diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py
index 8c224665cc39..616d9ff5c2dc 100755
--- a/src/transformers/trainer.py
+++ b/src/transformers/trainer.py
@@ -2517,6 +2517,7 @@ def _inner_training_loop(
                     context = (
                         functools.partial(self.accelerator.no_sync, model=model)
                         if i != len(batch_samples) - 1
+                        and self.accelerator.distributed_type != DistributedType.DEEPSPEED
                         else contextlib.nullcontext
                     )
                     with context():