Skip to content

Commit 345cc92

Browse files
committed
fix dataloader unit-test defect and nng perf test defect
1 parent 3fa5319 commit 345cc92

File tree

4 files changed

+11
-5
lines changed

4 files changed

+11
-5
lines changed

Diff for: codecov.yml

+3-2
Original file line numberDiff line numberDiff line change
@@ -11,5 +11,6 @@ coverage:
1111
# The unittests of the torchrpc module are tested by different runners and cannot be included
1212
# in the test_unittest's coverage report. To keep CI happy, we don't count torchrpc related coverage.
1313
ignore:
14-
- /mnt/cache/wangguoteng/DI-engine/ding/framework/message_queue/torch_rpc.py
15-
- /mnt/cache/wangguoteng/DI-engine/ding/framework/message_queue/perfs/*
14+
- ./ding/framework/message_queue/torch_rpc.py
15+
- ./ding/framework/message_queue/tests/test_torch_rpc.py
16+
- ./ding/framework/message_queue/perfs/*

Diff for: ding/framework/message_queue/perfs/perf_nng.py

+2
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,8 @@ def recv_loop():
127127
continue
128128
elif topic == "f":
129129
finish_tag.append(1)
130+
send_t("f")
131+
mq.stop()
130132
return
131133
else:
132134
raise RuntimeError("Unkown topic")

Diff for: ding/utils/data/dataloader.py

+5-2
Original file line numberDiff line numberDiff line change
@@ -116,12 +116,12 @@ def __init__(
116116
p, c = self.mp_context.Pipe()
117117

118118
# Async process (Main worker): Process data if num_workers <= 1; Assign job to other workers if num_workers > 1.
119-
self.async_process = self.mp_context.Process(target=self._async_loop, args=(p, c))
119+
self.async_process = self.mp_context.Process(target=self._async_loop, args=(p, c), name="async_process")
120120
self.async_process.daemon = True
121121
self.async_process.start()
122122

123123
# Get data thread: Get data from ``data_source`` and send it to ``async_process``.`
124-
self.get_data_thread = threading.Thread(target=self._get_data, args=(p, c))
124+
self.get_data_thread = threading.Thread(target=self._get_data, args=(p, c), name="get_data_thread")
125125
self.get_data_thread.daemon = True
126126
self.get_data_thread.start()
127127

@@ -350,6 +350,9 @@ def close(self) -> None:
350350
self.end_flag = True
351351
self.async_process.terminate()
352352
self.async_process.join()
353+
if self.use_cuda:
354+
self.cuda_thread.join()
355+
self.get_data_thread.join()
353356
if self.num_workers > 1:
354357
for w in self.worker:
355358
w.terminate()

Diff for: ding/utils/data/tests/test_dataloader.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -101,4 +101,4 @@ def entry(self, batch_size, num_workers, chunk_size, use_cuda):
101101
assert total_data_time <= 7 * 0.008
102102
dataloader.__del__()
103103
time.sleep(0.5)
104-
assert len(threading.enumerate()) <= 2, threading.enumerate()
104+
assert len(threading.enumerate()) <= 3, threading.enumerate()

0 commit comments

Comments
 (0)