[wenet/bin/train.py] add barrier before destory process group (#2590)

Co-authored-by: tianyukun <tianyukun@cambricon.com>
wenet-e2e · Aug 6, 2024 · bc58f83 · bc58f83
1 parent ec3d80f
commit bc58f83
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/wenet/bin/train.py b/wenet/bin/train.py
@@ -178,6 +178,8 @@ def main():
             final_model_path) else None
         os.symlink('{}.pt'.format(final_epoch), final_model_path)
         writer.close()
+    dist.barrier(
+    )  # NOTE(yktian): Ensure all ranks end Train before destroy process group.
     dist.destroy_process_group()