Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions deepdrivewe/workflows/ddwe.py
Original file line number Diff line number Diff line change
Expand Up @@ -314,21 +314,27 @@ def process_simulation_result(self, result: Result) -> None:
# If we have all the simulation results, submit the inference task
# using the previous iteration's model
if len(self.sim_output) == len(self.ensemble.next_sims):
# We need to wait for the streaming train task to finish
if not self.use_stale_model:
# We need to wait for the first streaming train task to finish
if self.use_stale_model and self.train_output is None:
# Wait for the streaming train task to finish
self.logger.info('Waiting for streaming train task to finish')
self.logger.info(
'Waiting for first streaming train task to finish',
)
while self.train_output is None:
time.sleep(10)

elif self.use_stale_model:
self.logger.info('Waiting for streaming train task to finish')
# We need to wait for the next streaming train task to finish
# to get a fresh model
elif not self.use_stale_model:
self.logger.info(
'Waiting for next streaming train task to finish',
)
while self.train_iteration < self.ensemble.iteration:
time.sleep(10)
# This should hold (see train_stream_processor)
assert self.train_output is not None

# If it's okay to use the stale model, submit the inference task
# Submit the inference task using either a stale or fresh model
self.submit_task('inference', self.sim_output, self.train_output)

@agent()
Expand Down