Run on_validation_end only on main process in DDP (#1125)

Co-authored-by: xingzhaolee <xingzhaolee@users.noreply.github.com>
Lightning-AI · Mar 12, 2020 · b4d4e48 · b4d4e48
1 parent b29613b
commit b4d4e48
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/pytorch_lightning/callbacks/model_checkpoint.py b/pytorch_lightning/callbacks/model_checkpoint.py
@@ -175,6 +175,10 @@ def format_checkpoint_name(self, epoch, metrics, ver=None):
         return filepath
 
     def on_validation_end(self, trainer, pl_module):
+        # only run on main process
+        if trainer.proc_rank != 0:
+            return
+
         metrics = trainer.callback_metrics
         epoch = trainer.current_epoch
         self.epochs_since_last_check += 1