Merge branch 'master' into feat_artifacts

.github/workflows/ci_test-conda.yml

@@ -44,12 +44,30 @@ jobs:
     - name: Tests
       run: |
         # NOTE: run coverage on tests does not propagare faler status for Win, https://github.com/nedbat/coveragepy/issues/1003
-        python -m pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml
+        python -m pytest pytorch_lightning tests --cov=pytorch_lightning -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-torch${{ matrix.pytorch-version }}.xml
       shell: bash -l {0}
 
-    - name: Upload pytest test results
+    - name: Upload pytest results
       uses: actions/upload-artifact@v2
       with:
         name: pytest-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}
         path: junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml
       if: failure()
+
+    - name: Statistics
+      if: success()
+      run: |
+        coverage report
+        coverage xml
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v1
+      if: always()
+      # see: https://github.com/actions/toolkit/issues/399
+      continue-on-error: true
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        file: coverage.xml
+        flags: cpu,pytest,torch${{ matrix.pytorch-version }}
+        name: CPU-coverage
+        fail_ci_if_error: false

.github/workflows/ci_test-full.yml

@@ -138,13 +138,13 @@ jobs:
     - name: Tests
       run: |
         # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003
-        coverage run --source pytorch_lightning -m pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml
+        python -m pytest pytorch_lightning tests --cov=pytorch_lightning -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}.xml
 
     - name: Examples
       run: |
         python -m pytest pl_examples -v --durations=10
 
-    - name: Upload pytest test results
+    - name: Upload pytest results
       uses: actions/upload-artifact@v2
       with:
         name: pytest-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}
@@ -165,6 +165,6 @@ jobs:
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         file: coverage.xml
-        flags: cpu,pytest
+        flags: cpu,pytest,python${{ matrix.python-version }}
         name: CPU-coverage
         fail_ci_if_error: false

CHANGELOG.md

@@ -21,9 +21,15 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - Added `TrainerState.{FITTING,VALIDATING,TESTING,PREDICTING,TUNING}` ([#4945](https://github.com/PyTorchLightning/pytorch-lightning/pull/4945))
 
 
+- Added `Trainer.validate()` method to perform one evaluation epoch over the validation set ([#4948](https://github.com/PyTorchLightning/pytorch-lightning/pull/4948))
+
+
 - Added `LightningEnvironment` for Lightning-specific DDP ([#5915](https://github.com/PyTorchLightning/pytorch-lightning/pull/5915))
 
 
+- Added `auto_insert_metric_name` parameter to `ModelCheckpoint` ([#6277](https://github.com/PyTorchLightning/pytorch-lightning/pull/6277))
+
+
 - Added arg to `self.log` that enables users to give custom names when dealing with multiple dataloaders ([#6274](https://github.com/PyTorchLightning/pytorch-lightning/pull/6274))
 
 
@@ -181,6 +187,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - Fixed error message for AMP + CPU incompatibility ([#6107](https://github.com/PyTorchLightning/pytorch-lightning/pull/6107))
 
 
+- Disabled batch transfer in DP mode ([#6093](https://github.com/PyTorchLightning/pytorch-lightning/pull/6093))
+
+
 ## [1.2.0] - 2021-02-18
 
 ### Added

docs/source/common/hyperparameters.rst

@@ -53,10 +53,10 @@ a module (i.e.: if your project has a model that trains on Imagenet and another
 
         @staticmethod
         def add_model_specific_args(parent_parser):
-            parser = ArgumentParser(parents=[parent_parser], add_help=False)
+            parser = parent_parser.add_argument_group("LitModel")
             parser.add_argument('--encoder_layers', type=int, default=12)
             parser.add_argument('--data_path', type=str, default='/some/path')
-            return parser
+            return parent_parser
 
 Now in your main trainer file, add the ``Trainer`` args, the program args, and add the model args
 
@@ -226,9 +226,9 @@ polluting the ``main.py`` file, the ``LightningModule`` lets you define argument
 
         @staticmethod
         def add_model_specific_args(parent_parser):
-            parser = ArgumentParser(parents=[parent_parser], add_help=False)
+            parser = parent_parser.add_argument_group("LitMNIST")
             parser.add_argument('--layer_1_dim', type=int, default=128)
-            return parser
+            return parent_parser
 
 .. testcode::
 
@@ -240,9 +240,9 @@ polluting the ``main.py`` file, the ``LightningModule`` lets you define argument
 
         @staticmethod
         def add_model_specific_args(parent_parser):
-            parser = ArgumentParser(parents=[parent_parser], add_help=False)
+            parser = parent_parser.add_argument_group("GoodGAN")
             parser.add_argument('--encoder_layers', type=int, default=12)
-            return parser
+            return parent_parser
 
 
 Now we can allow each model to inject the arguments it needs in the ``main.py``

docs/source/common/trainer.rst

@@ -128,10 +128,7 @@ So you can run it like so:
 
     if __name__ == '__main__':
         parser = ArgumentParser()
-        parser = Trainer.add_argparse_args(
-            # group the Trainer arguments together
-            parser.add_argument_group(title="pl.Trainer args")
-        )
+        parser = Trainer.add_argparse_args()
         args = parser.parse_args()
 
         main(args)
@@ -151,14 +148,27 @@ So you can run it like so:
 
 ------------
 
+Validation
+----------
+You can perform an evaluation epoch over the validation set, outside of the training loop,
+using :meth:`pytorch_lightning.trainer.trainer.Trainer.validate`. This might be
+useful if you want to collect new metrics from a model right at its initialization
+or after it has already been trained.
+
+.. code-block:: python
+
+    trainer.validate(val_dataloaders=val_dataloaders)
+
+------------
+
 Testing
 -------
 Once you're done training, feel free to run the test set!
 (Only right before publishing your paper or pushing to production)
 
 .. code-block:: python
 
-    trainer.test(test_dataloaders=test_dataloader)
+    trainer.test(test_dataloaders=test_dataloaders)
 
 ------------
 

notebooks/04-transformers-text-classification.ipynb

@@ -368,12 +368,13 @@
     "\n",
     "    @staticmethod\n",
     "    def add_model_specific_args(parent_parser):\n",
+    "        parser = parent_parser.add_argument_group(\"GLUETransformer\")",
     "        parser = ArgumentParser(parents=[parent_parser], add_help=False)\n",
     "        parser.add_argument(\"--learning_rate\", default=2e-5, type=float)\n",
     "        parser.add_argument(\"--adam_epsilon\", default=1e-8, type=float)\n",
     "        parser.add_argument(\"--warmup_steps\", default=0, type=int)\n",
     "        parser.add_argument(\"--weight_decay\", default=0.0, type=float)\n",
-    "        return parser"
+    "        return parent_parser"
    ]
   },
   {

pl_examples/basic_examples/autoencoder.py

@@ -22,9 +22,10 @@
 import pytorch_lightning as pl
 from pl_examples import _DATASETS_PATH, _TORCHVISION_AVAILABLE, _TORCHVISION_MNIST_AVAILABLE, cli_lightning_logo
 
-if _TORCHVISION_AVAILABLE and _TORCHVISION_MNIST_AVAILABLE:
+if _TORCHVISION_AVAILABLE:
     from torchvision import transforms
-    from torchvision.datasets.mnist import MNIST
+if _TORCHVISION_MNIST_AVAILABLE:
+    from torchvision.datasets import MNIST
 else:
     from tests.helpers.datasets import MNIST
 

pl_examples/basic_examples/backbone_image_classifier.py

@@ -21,9 +21,10 @@
 import pytorch_lightning as pl
 from pl_examples import _DATASETS_PATH, _TORCHVISION_AVAILABLE, _TORCHVISION_MNIST_AVAILABLE, cli_lightning_logo
 
-if _TORCHVISION_AVAILABLE and _TORCHVISION_MNIST_AVAILABLE:
+if _TORCHVISION_AVAILABLE:
     from torchvision import transforms
-    from torchvision.datasets.mnist import MNIST
+if _TORCHVISION_MNIST_AVAILABLE:
+    from torchvision.datasets import MNIST
 else:
     from tests.helpers.datasets import MNIST
 
@@ -92,9 +93,9 @@ def configure_optimizers(self):
 
     @staticmethod
     def add_model_specific_args(parent_parser):
-        parser = ArgumentParser(parents=[parent_parser], add_help=False)
+        parser = parent_parser.add_argument_group("LitClassifier")
         parser.add_argument('--learning_rate', type=float, default=0.0001)
-        return parser
+        return parent_parser
 
 
 def cli_main():

pl_examples/basic_examples/dali_image_classifier.py

@@ -31,9 +31,10 @@
     cli_lightning_logo,
 )
 
-if _TORCHVISION_AVAILABLE and _TORCHVISION_MNIST_AVAILABLE:
+if _TORCHVISION_AVAILABLE:
     from torchvision import transforms
-    from torchvision.datasets.mnist import MNIST
+if _TORCHVISION_MNIST_AVAILABLE:
+    from torchvision.datasets import MNIST
 else:
     from tests.helpers.datasets import MNIST
 
@@ -174,10 +175,10 @@ def configure_optimizers(self):
 
     @staticmethod
     def add_model_specific_args(parent_parser):
-        parser = ArgumentParser(parents=[parent_parser], add_help=False)
+        parser = parent_parser.add_argument_group("LitClassifier")
         parser.add_argument('--hidden_dim', type=int, default=128)
         parser.add_argument('--learning_rate', type=float, default=0.0001)
-        return parser
+        return parent_parser
 
 
 def cli_main():

pl_examples/basic_examples/mnist_datamodule.py

@@ -20,8 +20,9 @@
 from pl_examples import _DATASETS_PATH, _TORCHVISION_AVAILABLE, _TORCHVISION_MNIST_AVAILABLE
 from pytorch_lightning import LightningDataModule
 
-if _TORCHVISION_AVAILABLE and _TORCHVISION_MNIST_AVAILABLE:
+if _TORCHVISION_AVAILABLE:
     from torchvision import transforms as transform_lib
+if _TORCHVISION_MNIST_AVAILABLE:
     from torchvision.datasets import MNIST
 else:
     from tests.helpers.datasets import MNIST
@@ -54,6 +55,8 @@ def __init__(
             val_split: how many of the training images to use for the validation split
             num_workers: how many workers to use for loading data
             normalize: If true applies image normalize
+            seed: starting seed for RNG.
+            batch_size: desired batch size.
         """
         super().__init__(*args, **kwargs)
         if num_workers and platform.system() == "Windows":

pl_examples/basic_examples/simple_image_classifier.py

@@ -68,10 +68,10 @@ def configure_optimizers(self):
 
     @staticmethod
     def add_model_specific_args(parent_parser):
-        parser = ArgumentParser(parents=[parent_parser], add_help=False)
+        parser = parent_parser.add_argument_group("LitClassifier")
         parser.add_argument('--hidden_dim', type=int, default=128)
         parser.add_argument('--learning_rate', type=float, default=0.0001)
-        return parser
+        return parent_parser
 
 
 def cli_main():
@@ -105,8 +105,6 @@ def cli_main():
     # ------------
     # testing
     # ------------
-    # todo: without passing model it fails for missing best weights
-    # MisconfigurationException, 'ckpt_path is "best", but ModelCheckpoint is not configured to save the best model.'
     result = trainer.test(model, datamodule=dm)
     pprint(result)
 

pl_examples/domain_templates/computer_vision_fine_tuning.py

@@ -147,14 +147,14 @@ def val_dataloader(self):
 
     @staticmethod
     def add_model_specific_args(parent_parser):
-        parser = argparse.ArgumentParser(parents=[parent_parser])
+        parser = parent_parser.add_argument_group("CatDogImageDataModule")
         parser.add_argument(
             "--num-workers", default=0, type=int, metavar="W", help="number of CPU workers", dest="num_workers"
         )
         parser.add_argument(
             "--batch-size", default=8, type=int, metavar="W", help="number of sample in a batch", dest="batch_size"
         )
-        return parser
+        return parent_parser
 
 
 #  --- Pytorch-lightning module ---
@@ -268,7 +268,7 @@ def configure_optimizers(self):
 
     @staticmethod
     def add_model_specific_args(parent_parser):
-        parser = argparse.ArgumentParser(parents=[parent_parser])
+        parser = parent_parser.add_argument_group("TransferLearningModel")
         parser.add_argument(
             "--backbone",
             default="resnet50",
@@ -303,7 +303,7 @@ def add_model_specific_args(parent_parser):
         parser.add_argument(
             "--milestones", default=[2, 4], type=list, metavar="M", help="List of two epochs milestones"
         )
-        return parser
+        return parent_parser
 
 
 def main(args: argparse.Namespace) -> None:

pl_examples/domain_templates/generative_adversarial_net.py

@@ -32,9 +32,10 @@
 from pytorch_lightning.core import LightningDataModule, LightningModule
 from pytorch_lightning.trainer import Trainer
 
-if _TORCHVISION_AVAILABLE and _TORCHVISION_MNIST_AVAILABLE:
+if _TORCHVISION_AVAILABLE:
     import torchvision
-    import torchvision.transforms as transforms
+    from torchvision import transforms
+if _TORCHVISION_MNIST_AVAILABLE:
     from torchvision.datasets import MNIST
 else:
     from tests.helpers.datasets import MNIST
@@ -134,14 +135,18 @@ def __init__(
         self.example_input_array = torch.zeros(2, self.hparams.latent_dim)
 
     @staticmethod
-    def add_argparse_args(parent_parser: ArgumentParser):
-        parser = ArgumentParser(parents=[parent_parser], add_help=False)
+    def add_argparse_args(parent_parser: ArgumentParser, *, use_argument_group=True):
+        if use_argument_group:
+            parser = parent_parser.add_argument_group("pl.GAN")
+            parser_out = parent_parser
+        else:
+            parser = ArgumentParser(parents=[parent_parser], add_help=False)
+            parser_out = parser
         parser.add_argument("--lr", type=float, default=0.0002, help="adam: learning rate")
         parser.add_argument("--b1", type=float, default=0.5, help="adam: decay of first order momentum of gradient")
         parser.add_argument("--b2", type=float, default=0.999, help="adam: decay of second order momentum of gradient")
         parser.add_argument("--latent_dim", type=int, default=100, help="dimensionality of the latent space")
-
-        return parser
+        return parser_out
 
     def forward(self, z):
         return self.generator(z)

pl_examples/domain_templates/imagenet.py

@@ -198,7 +198,7 @@ def substitute_val_keys(out):
 
     @staticmethod
     def add_model_specific_args(parent_parser):  # pragma: no-cover
-        parser = ArgumentParser(parents=[parent_parser])
+        parser = parent_parser.add_argument_group("ImageNetLightningModel")
         parser.add_argument(
             '-a',
             '--arch',
@@ -233,7 +233,7 @@ def add_model_specific_args(parent_parser):  # pragma: no-cover
             dest='weight_decay'
         )
         parser.add_argument('--pretrained', dest='pretrained', action='store_true', help='use pre-trained model')
-        return parser
+        return parent_parser
 
 
 def main(args: Namespace) -> None:

pl_examples/domain_templates/reinforce_learn_Qnet.py

@@ -390,7 +390,7 @@ def get_device(self, batch) -> str:
 
     @staticmethod
     def add_model_specific_args(parent_parser):  # pragma: no-cover
-        parser = argparse.ArgumentParser(parents=[parent_parser])
+        parser = parent_parser.add_argument_group("DQNLightning")
         parser.add_argument("--batch_size", type=int, default=16, help="size of the batches")
         parser.add_argument("--lr", type=float, default=1e-2, help="learning rate")
         parser.add_argument("--env", type=str, default="CartPole-v0", help="gym environment tag")
@@ -407,7 +407,7 @@ def add_model_specific_args(parent_parser):  # pragma: no-cover
         parser.add_argument("--eps_start", type=float, default=1.0, help="starting value of epsilon")
         parser.add_argument("--eps_end", type=float, default=0.01, help="final value of epsilon")
         parser.add_argument("--episode_length", type=int, default=200, help="max length of an episode")
-        return parser
+        return parent_parser
 
 
 def main(args) -> None: