add distributed property in accelerate_state

examples/by_feature/fsdp_with_peak_mem_tracking.py

@@ -280,7 +280,7 @@ def collate_fn(examples):
                 predictions, references = accelerator.gather(
                     (predictions, batch["labels"])
                 )  # If we are in a multiprocess environment, the last batch has duplicates
-                if accelerator.num_processes > 1:
+                if accelerator.use_distributed:
                     if step == len(eval_dataloader) - 1:
                         predictions = predictions[: len(eval_dataloader.dataset) - samples_seen]
                         references = references[: len(eval_dataloader.dataset) - samples_seen]

examples/by_feature/multi_process_metrics.py

@@ -178,7 +178,7 @@ def training_function(config, args):
             predictions, references = accelerator.gather((predictions, batch["labels"]))
             # New Code #
             # First we check if it's a distributed system
-            if accelerator.num_processes > 1:
+            if accelerator.use_distributed:
                 # Then see if we're on the last batch of our eval dataloader
                 if step == len(eval_dataloader) - 1:
                     # Last batch needs to be truncated on distributed systems as it contains additional samples

examples/complete_cv_example.py

@@ -242,7 +242,7 @@ def training_function(config, args):
                 outputs = model(inputs)
             predictions = outputs.argmax(dim=-1)
             predictions, references = accelerator.gather((predictions, batch["label"]))
-            if accelerator.num_processes > 1:
+            if accelerator.use_distributed:
                 if step == len(eval_dataloader) - 1:
                     predictions = predictions[: len(eval_dataloader) - samples_seen]
                     references = references[: len(eval_dataloader) - samples_seen]

examples/complete_nlp_example.py

@@ -222,7 +222,7 @@ def collate_fn(examples):
             predictions, references = accelerator.gather(
                 (predictions, batch["labels"])
             )  # If we are in a multiprocess environment, the last batch has duplicates
-            if accelerator.num_processes > 1:
+            if accelerator.use_distributed:
                 if step == len(eval_dataloader) - 1:
                     predictions = predictions[: len(eval_dataloader.dataset) - samples_seen]
                     references = references[: len(eval_dataloader.dataset) - samples_seen]

src/accelerate/accelerator.py

@@ -273,6 +273,10 @@ def __init__(
         if self.rng_types is None:
             self.rng_types = ["torch"] if is_torch_version("<=", "1.5.1") else ["generator"]
 
+    @property
+    def use_distributed(self):
+        return self.distributed_type != DistributedType.NO
+
     @property
     def distributed_type(self):
         return self.state.distributed_type
@@ -361,7 +365,7 @@ def no_sync(self, model):
                 PyTorch Module that was prepared with `Accelerator.prepare`
         """
         context = contextlib.nullcontext
-        if self.num_processes > 1:
+        if self.use_distributed:
             context = getattr(model, "no_sync", context)
 
         with context():