Enables reload of dataloaders on every n epochs from every epoch

CHANGELOG.md

@@ -112,6 +112,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
     * Added parallel plugins for DP, DDP, DDPSpawn, DDP2 and Horovod ([#5714](https://github.com/PyTorchLightning/pytorch-lightning/pull/5714))
 
 
+- Changed `Trainer` arg and functionality from `reload_dataloaders_every_epoch` to `reload_dataloaders_every_n_epochs` ([#5043](https://github.com/PyTorchLightning/pytorch-lightning/pull/5043))
+
+
 ### Deprecated
 
 - Function `stat_scores_multiple_classes` is deprecated in favor of `stat_scores` ([#4839](https://github.com/PyTorchLightning/pytorch-lightning/pull/4839))
@@ -126,6 +129,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - Deprecated `LightningDataParallel` in favor of new wrapper module `LightningParallelModule` ([#5670](https://github.com/PyTorchLightning/pytorch-lightning/pull/5670))
 
 
+- Deprecated `reload_dataloaders_every_epoch` argument of `Trainer` ([#5043](https://github.com/PyTorchLightning/pytorch-lightning/pull/5043))
+
+
 ### Removed
 
 - Removed deprecated checkpoint argument `filepath` ([#5321](https://github.com/PyTorchLightning/pytorch-lightning/pull/5321))

docs/source/common/trainer.rst

@@ -1263,8 +1263,8 @@ Note:
       Lightning will set it to 20 in these environments if the user does not provide a value.
     - This argument is ignored if a custom callback is passed to :paramref:`~Trainer.callbacks`.
 
-reload_dataloaders_every_epoch
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+reload_dataloaders_every_n_epochs
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 .. raw:: html
 
@@ -1274,19 +1274,20 @@ reload_dataloaders_every_epoch
 
 |
 
-Set to True to reload dataloaders every epoch.
+Set to a postive integer to reload dataloaders every n epochs.
 
 .. code-block:: python
 
-    # if False (default)
+    # if 0 (default)
     train_loader = model.train_dataloader()
     for epoch in epochs:
         for batch in train_loader:
             ...
 
-    # if True
+    # if a positive integer
     for epoch in epochs:
-        train_loader = model.train_dataloader()
+        if not epoch % reload_dataloaders_every_n_epochs:
+            train_loader = model.train_dataloader()
         for batch in train_loader:
 
 replace_sampler_ddp

pytorch_lightning/core/hooks.py

@@ -369,8 +369,9 @@ def train_dataloader(self) -> DataLoader:
         Return:
             Single PyTorch :class:`~torch.utils.data.DataLoader`.
 
-        The dataloader you return will not be called every epoch unless you set
-        :paramref:`~pytorch_lightning.trainer.Trainer.reload_dataloaders_every_epoch` to ``True``.
+        The dataloader you return will not be reloaded unless you set
+        :paramref:`~pytorch_lightning.trainer.Trainer.reload_dataloaders_every_n_epochs` to
+        a positive integer.
 
         For data processing use the following pattern:
 
@@ -414,8 +415,9 @@ def test_dataloader(self) -> Union[DataLoader, List[DataLoader]]:
         r"""
         Implement one or multiple PyTorch DataLoaders for testing.
 
-        The dataloader you return will not be called every epoch unless you set
-        :paramref:`~pytorch_lightning.trainer.Trainer.reload_dataloaders_every_epoch` to ``True``.
+        The dataloader you return will not be reloaded unless you set
+        :paramref:`~pytorch_lightning.trainer.Trainer.reload_dataloaders_every_n_epochs` to
+        a postive integer.
 
         For data processing use the following pattern:
 
@@ -474,8 +476,9 @@ def val_dataloader(self) -> Union[DataLoader, List[DataLoader]]:
         r"""
         Implement one or multiple PyTorch DataLoaders for validation.
 
-        The dataloader you return will not be called every epoch unless you set
-        :paramref:`~pytorch_lightning.trainer.Trainer.reload_dataloaders_every_epoch` to ``True``.
+        The dataloader you return will not be reloaded unless you set
+        :paramref:`~pytorch_lightning.trainer.Trainer.reload_dataloaders_every_n_epochs` to
+        a positive integer.
 
         It's recommended that all data downloads and preparation happen in :meth:`prepare_data`.
 

pytorch_lightning/trainer/connectors/data_connector.py

@@ -17,21 +17,42 @@
 from torch.utils.data import DataLoader
 
 from pytorch_lightning.core.datamodule import LightningDataModule
+from pytorch_lightning.utilities import rank_zero_warn
 from pytorch_lightning.utilities.exceptions import MisconfigurationException
-from pytorch_lightning.utilities.model_helpers import is_overridden
+from pytorch_lightning.utilities.model_utils import is_overridden
 
 
 class DataConnector(object):
 
     def __init__(self, trainer):
         self.trainer = trainer
 
-    def on_trainer_init(self, check_val_every_n_epoch, reload_dataloaders_every_epoch, prepare_data_per_node):
+    def on_trainer_init(
+        self,
+        check_val_every_n_epoch,
+        reload_dataloaders_every_n_epochs,
+        reload_dataloaders_every_epoch,
+        prepare_data_per_node
+    ):
         self.trainer.datamodule = None
         self.trainer.prepare_data_per_node = prepare_data_per_node
 
         self.trainer.check_val_every_n_epoch = check_val_every_n_epoch
-        self.trainer.reload_dataloaders_every_epoch = reload_dataloaders_every_epoch
+
+        if reload_dataloaders_every_epoch:
+            reload_dataloaders_every_n_epochs = int(reload_dataloaders_every_epoch)
+            rank_zero_warn(
+                "`reload_dataloaders_every_epoch` is deprecated in v1.2 and will be removed in v1.4."
+                " Please use `reload_dataloaders_every_n_epochs` in Trainer.", DeprecationWarning
+            )
+
+        if not isinstance(reload_dataloaders_every_n_epochs, int) or (reload_dataloaders_every_n_epochs < 0):
+            raise MisconfigurationException(
+                "`reload_dataloaders_every_n_epochs` should be an int >= 0,"
+                f" got {reload_dataloaders_every_n_epochs}."
+            )
+
+        self.trainer.reload_dataloaders_every_n_epochs = reload_dataloaders_every_n_epochs
         self.trainer._is_data_prepared = False
 
     def get_profiled_train_dataloader(self, train_dataloader):

pytorch_lightning/trainer/evaluation_loop.py

@@ -58,8 +58,8 @@ def get_evaluation_dataloaders(self, max_batches):
         else:
             # val
             in_sanity_check = self.trainer.running_sanity_check
-            should_reload_every_epoch = self.trainer.reload_dataloaders_every_epoch
-            if (self.trainer.val_dataloaders is None or should_reload_every_epoch) and not in_sanity_check:
+
+            if (self.trainer.val_dataloaders is None or self.trainer.should_reload_dl_epoch) and not in_sanity_check:
                 self.trainer.reset_val_dataloader(model)
 
             dataloaders = self.trainer.val_dataloaders

pytorch_lightning/trainer/properties.py

@@ -55,6 +55,8 @@ class TrainerProperties(ABC):
     model: LightningModule
     data_parallel_device_ids: Optional[List[int]]
     _progress_bar_callback: ProgressBarBase
+    current_epoch: int
+    reload_dataloaders_every_n_epochs: int
     limit_val_batches: int
     _default_root_dir: str
     _weights_save_path: str
@@ -195,6 +197,15 @@ def progress_bar_dict(self) -> dict:
         ref_model = cast(LightningModule, ref_model)
         return dict(**ref_model.get_progress_bar_dict(), **self.logger_connector.progress_bar_metrics)
 
+    @property
+    def should_reload_dl_epoch(self) -> bool:
+        """ Check if dataloader should be reloaded in the current epoch. """
+        reload_dl_every_n_epochs = self.reload_dataloaders_every_n_epochs
+        return (
+            reload_dl_every_n_epochs
+            and (not self.current_epoch % reload_dl_every_n_epochs)
+        )
+
     @property
     def disable_validation(self) -> bool:
         """ Check if validation is disabled during training. """

pytorch_lightning/trainer/trainer.py

@@ -122,6 +122,7 @@ def __init__(
         profiler: Optional[Union[BaseProfiler, bool, str]] = None,
         benchmark: bool = False,
         deterministic: bool = False,
+        reload_dataloaders_every_n_epochs: int = 0,
         reload_dataloaders_every_epoch: bool = False,
         auto_lr_find: Union[bool, str] = False,
         replace_sampler_ddp: bool = True,
@@ -246,8 +247,15 @@ def __init__(
             num_sanity_val_steps: Sanity check runs n validation batches before starting the training routine.
                 Set it to `-1` to run all batches in all validation dataloaders. Default: 2
 
+            reload_dataloaders_every_n_epochs: Set to a non negative integer to reload dataloaders every n epochs.
+                Default: 0
+
             reload_dataloaders_every_epoch: Set to True to reload dataloaders every epoch.
 
+            .. warning::
+               ``reload_dataloaders_every_epoch`` has been deprecated in v1.2 and will be removed in v1.4.
+               Please use ``reload_dataloaders_every_n_epochs``.
+
             replace_sampler_ddp: Explicitly enables or disables sampler replacement. If not specified this
                 will toggled automatically when DDP is used. By default it will add ``shuffle=True`` for
                 train sampler and ``shuffle=False`` for val/test sampler. If you want to customize it,
@@ -344,7 +352,10 @@ def __init__(
 
         # init data flags
         self.data_connector.on_trainer_init(
-            check_val_every_n_epoch, reload_dataloaders_every_epoch, prepare_data_per_node
+            check_val_every_n_epoch,
+            reload_dataloaders_every_n_epochs,
+            reload_dataloaders_every_epoch,
+            prepare_data_per_node
         )
 
         # init training tricks

pytorch_lightning/trainer/training_loop.py

@@ -249,7 +249,7 @@ def on_train_epoch_start(self, epoch):
         model = self.trainer.get_model()
 
         # reset train dataloader
-        if self.trainer.reload_dataloaders_every_epoch:
+        if self.trainer.should_reload_dl_epoch:
             self.trainer.reset_train_dataloader(model)
 
         # todo: specify the possible exception
@@ -283,10 +283,10 @@ def on_train_batch_end(self, epoch_output, epoch_end_outputs, batch, batch_idx,
         self.trainer.logger_connector.on_train_batch_end()
 
     def reset_train_val_dataloaders(self, model):
-        if not self.trainer.reload_dataloaders_every_epoch:
+        if not self.trainer.reload_dataloaders_every_n_epochs:
             self.trainer.reset_train_dataloader(model)
 
-        if self.trainer.val_dataloaders is None and not self.trainer.reload_dataloaders_every_epoch:
+        if self.trainer.val_dataloaders is None and not self.trainer.reload_dataloaders_every_n_epochs:
             self.trainer.reset_val_dataloader(model)
 
     def track_epoch_end_reduce_metrics(self, epoch_output, epoch_end_outputs):

tests/core/test_datamodules.py

@@ -426,9 +426,11 @@ def transfer_batch_to_device(self, data, device):
     assert batch_gpu.samples.device == batch_gpu.targets.device == expected
 
 
-def test_dm_reload_dataloaders_every_epoch(tmpdir):
-    """Test datamodule, where trainer argument
-    reload_dataloaders_every_epoch is set to True/False"""
+def test_dm_reload_dataloaders_every_n_epochs(tmpdir):
+    """
+    Test datamodule, where trainer argument
+    reload_dataloaders_every_n_epochs is set to a non negative integer
+    """
     class CustomBoringDataModule(BoringDataModule):
         def __init__(self):
             super().__init__()
@@ -451,9 +453,10 @@ def train_dataloader(self):
 
     trainer = Trainer(
         default_root_dir=tmpdir,
-        max_epochs=2,
-        limit_train_batches=0.01,
-        reload_dataloaders_every_epoch=True,
+        max_epochs=3,
+        limit_train_batches=2,
+        reload_dataloaders_every_n_epochs=2,
     )
     results = trainer.fit(model, dm)
+    assert trainer.state == TrainerState.FINISHED, f"Training failed with {trainer.state}"
     assert results

tests/deprecated_api/test_remove_1-4.py

@@ -12,7 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Test deprecated functionality which will be removed in vX.Y.Z"""
+import os
 import sys
+from unittest import mock
 
 import pytest
 import torch
@@ -198,3 +200,26 @@ def test_v1_4_0_deprecated_lightning_data_parallel():
         dp_model = LightningDataParallel(model, device_ids=[0])
     assert isinstance(dp_model, torch.nn.DataParallel)
     assert isinstance(dp_model.module, LightningParallelModule)
+
+
+@mock.patch.dict(os.environ, {"PL_DEV_DEBUG": "1"})
+def test_reload_dataloaders_every_epoch_remove_in_v1_4_0(tmpdir):
+
+    model = BoringModel()
+
+    with pytest.deprecated_call(match='will be removed in v1.4'):
+        trainer = Trainer(
+            default_root_dir=tmpdir,
+            limit_train_batches=0.3,
+            limit_val_batches=0.3,
+            reload_dataloaders_every_epoch=True,
+            max_epochs=3,
+        )
+    trainer.fit(model)
+    trainer.test()
+
+    # verify the sequence
+    calls = trainer.dev_debugger.dataloader_sequence_calls
+    expected_sequence = ['val_dataloader'] + ['train_dataloader', 'val_dataloader'] * 3 + ['test_dataloader']
+    for call, expected in zip(calls, expected_sequence):
+        assert call['name'] == expected

tests/trainer/test_dataloaders.py

@@ -1046,7 +1046,7 @@ def test_dataloaders_load_only_once_val_interval(tmpdir):
         limit_train_batches=10,
         limit_val_batches=10,
         val_check_interval=0.3,
-        reload_dataloaders_every_epoch=True,
+        reload_dataloaders_every_n_epochs=True,
         max_epochs=3,
     )
     trainer.fit(model)
@@ -1110,44 +1110,46 @@ def test_dataloaders_load_only_once_no_sanity_check(tmpdir):
         assert call['name'] == expected
 
 
-@mock.patch.dict(os.environ, {"PL_DEV_DEBUG": "1"})
-def test_dataloaders_load_every_epoch(tmpdir):
+@pytest.mark.parametrize("n", [1, 2])
+def test_dataloaders_load_every_n_epochs(tmpdir, n):
 
-    model = EvalModelTemplate()
+    model = BoringModel()
 
-    # logger file to get meta
     trainer = Trainer(
         default_root_dir=tmpdir,
         limit_train_batches=0.3,
         limit_val_batches=0.3,
-        reload_dataloaders_every_epoch=True,
+        reload_dataloaders_every_n_epochs=n,
         max_epochs=3,
     )
     trainer.fit(model)
     assert trainer.state == TrainerState.FINISHED, f"Training failed with {trainer.state}"
 
     trainer.test()
 
-    assert len(trainer.dev_debugger.val_dataloader_calls) == 4
-    assert len(trainer.dev_debugger.train_dataloader_calls) == 3
-    assert len(trainer.dev_debugger.test_dataloader_calls) == 1
-
     # verify the sequence
     calls = trainer.dev_debugger.dataloader_sequence_calls
-    expected_sequence = [
-        'val_dataloader',
-        'train_dataloader',
-        'val_dataloader',
-        'train_dataloader',
-        'val_dataloader',
-        'train_dataloader',
-        'val_dataloader',
-        'test_dataloader'
-    ]
+    expected_sequence = ['val_dataloader']
+    if n == 1:
+        expected_sequence += ['train_dataloader', 'val_dataloader'] * 3
+    elif n == 2:
+        expected_sequence +=['train_dataloader', 'val_dataloader'] * 2
+    expected_sequence += ['test_dataloader']
+
     for call, expected in zip(calls, expected_sequence):
         assert call['name'] == expected
 
 
+@pytest.mark.parametrize("n", ['test', -1])
+def test_dataloaders_load_every_n_epochs_exception(tmpdir, n):
+
+    with pytest.raises(MisconfigurationException, match='should be an int >'):
+        Trainer(
+            default_root_dir=tmpdir,
+            reload_dataloaders_every_n_epochs=n,
+        )
+
+
 @mock.patch.dict(os.environ, {"PL_DEV_DEBUG": "1"})
 def test_dataloaders_load_every_epoch_no_sanity_check(tmpdir):
 
@@ -1159,7 +1161,7 @@ def test_dataloaders_load_every_epoch_no_sanity_check(tmpdir):
         limit_train_batches=0.3,
         limit_val_batches=0.3,
         num_sanity_val_steps=0,
-        reload_dataloaders_every_epoch=True,
+        reload_dataloaders_every_n_epochs=True,
         max_epochs=3,
     )
     trainer.fit(model)