Weekly patch release v1.6.5

.actions/pull_legacy_checkpoints.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+# Run this script from the project root.
+URL="https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip"
+mkdir -p legacy
+# wget is simpler but does not work on Windows
+python -c "from urllib.request import urlretrieve; urlretrieve('$URL', 'legacy/checkpoints.zip')"
+ls -l legacy/
+unzip -o legacy/checkpoints.zip -d legacy/
+ls -l legacy/checkpoints/

.azure-pipelines/gpu-tests.yml

@@ -69,10 +69,7 @@ jobs:
         python requirements/check-avail-extras.py
       displayName: 'Env details'
 
-    - bash: |
-        wget https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip -P legacy/
-        unzip -o legacy/checkpoints.zip -d legacy/
-        ls -l legacy/checkpoints/
+    - bash: bash .actions/pull_legacy_checkpoints.sh
       displayName: 'Get legacy checkpoints'
 
     - bash: |

.github/workflows/ci_test-conda.yml

@@ -31,19 +31,27 @@ jobs:
     timeout-minutes: 30
     steps:
     - name: Workaround for https://github.com/actions/checkout/issues/760
-      run: git config --global --add safe.directory /__w/pytorch-lightning/pytorch-lightning
+      run: git config --global --add safe.directory /__w/lightning/lightning
 
     - uses: actions/checkout@v2
 
-    - name: Update dependencies
+    - name: Update base dependencies
+      run: |
+        conda info
+        conda list
+        pip install -r requirements/test.txt
+
+    - name: DocTests
+      run: |
+        coverage run --source pytorch_lightning -m pytest pytorch_lightning
+
+    - name: Update all dependencies
       env:
         HOROVOD_BUILD_ARCH_FLAGS: "-mfma"
         HOROVOD_WITHOUT_MXNET: 1
         HOROVOD_WITHOUT_TENSORFLOW: 1
       run: |
         set -e
-        conda info
-        conda list
         # adjust versions according installed Torch version
         python ./requirements/adjust-versions.py requirements/extra.txt
         python ./requirements/adjust-versions.py requirements/examples.txt
@@ -55,17 +63,12 @@ jobs:
         # sanity check
         python requirements/check-avail-extras.py
 
-    - name: Pull checkpoints from S3
-      working-directory: ./legacy
-      run: |
-        # enter legacy and update checkpoints from S3
-        curl https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip --output checkpoints.zip
-        unzip -o checkpoints.zip
-        ls -l checkpoints/
+    - name: Pull legacy checkpoints
+      run: bash .actions/pull_legacy_checkpoints.sh
 
-    - name: Tests
+    - name: UnitTests
       run: |
-        coverage run --source pytorch_lightning -m pytest --timeout 150 pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-torch${{ matrix.pytorch-version }}.xml
+        coverage run --source pytorch_lightning -m pytest --timeout 150 tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-torch${{ matrix.pytorch-version }}.xml
 
     - name: Upload pytest results
       uses: actions/upload-artifact@v2

.github/workflows/ci_test-full.yml

@@ -76,27 +76,21 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-pip-td${{ env.TIME_PERIOD }}-py${{ matrix.python-version }}-${{ matrix.release }}-${{ matrix.requires }}-
 
-    - name: Pull checkpoints from S3
-      working-directory: ./legacy
-      run: |
-        # wget is simpler but does not work on Windows
-        python -c "from urllib.request import urlretrieve ; urlretrieve('https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip', 'checkpoints.zip')"
-        ls -l .
-        unzip -o checkpoints.zip
-        ls -l checkpoints/
+    - name: Pull legacy checkpoints
+      run: bash .actions/pull_legacy_checkpoints.sh
 
     - name: Install dependencies
       run: |
         flag=$(python -c "print('--pre' if '${{matrix.release}}' == 'pre' else '')" 2>&1)
         url=$(python -c "print('test/cpu/torch_test.html' if '${{matrix.release}}' == 'pre' else 'cpu/torch_stable.html')" 2>&1)
         pip install -r requirements.txt --upgrade $flag --find-links "https://download.pytorch.org/whl/${url}"
-        # adjust versions according installed Torch version
-        python ./requirements/adjust-versions.py requirements/examples.txt
-        pip install -r requirements/examples.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade
         pip install -r requirements/test.txt --upgrade
         pip list
       shell: bash
 
+    - name: DocTests
+      run: coverage run --source pytorch_lightning -m pytest pytorch_lightning
+
     - name: Install extra dependencies
       run: |
         # adjust versions according installed Torch version
@@ -132,13 +126,16 @@ jobs:
       run: |
         python requirements/check-avail-extras.py
 
-    - name: Tests
+    - name: UnitTests
       run: |
         # NOTE: do not include coverage report here, see: https://github.com/nedbat/coveragepy/issues/1003
         coverage run --source pytorch_lightning -m pytest pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-py${{ matrix.python-version }}-${{ matrix.requires }}-${{ matrix.release }}.xml
 
     - name: Examples
       run: |
+        # adjust versions according installed Torch version
+        python ./requirements/adjust-versions.py requirements/examples.txt
+        pip install -r requirements/examples.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html --upgrade
         python -m pytest pl_examples -v --durations=10
 
     - name: Upload pytest results

CHANGELOG.md

@@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 
+## [1.6.5] - 2022-07-05
+
+### Fixed
+
+- Fixed `estimated_stepping_batches` requiring distributed comms in `configure_optimizers` for the `DeepSpeedStrategy` ([#13350](https://github.com/PyTorchLightning/pytorch-lightning/pull/13350))
+- Fixed bug with Python version check that prevented use with development versions of Python ([#13420](https://github.com/PyTorchLightning/pytorch-lightning/pull/13420))
+- The loops now call `.set_epoch()` also on batch samplers if the dataloader has one wrapped in a distributed sampler ([#13396](https://github.com/PyTorchLightning/pytorch-lightning/pull/13396))
+
+
 ## [1.6.4] - 2022-06-01
 
 ### Added

dockers/nvidia/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG PYTORCH_VERSION=21.11
+ARG PYTORCH_VERSION=22.04
 
 # https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes
 FROM nvcr.io/nvidia/pytorch:${PYTORCH_VERSION}-py3
@@ -37,20 +37,19 @@ RUN \
         cd .. ; \
     fi && \
 # save the examples
-    mv pytorch-lightning/_notebooks notebooks && \
+    mv pytorch-lightning/_notebooks/.notebooks/ notebooks && \
     mv pytorch-lightning/pl_examples . && \
 
 # Installations \
     pip install -q fire && \
-    python ./pytorch-lightning/.actions/assistant.py requirements_prune_pkgs horovod --req_files ./pytorch-lightning/requirements/extra.txt  && \
-    pip install "Pillow>=8.2, !=8.3.0" "cryptography>=3.4" "py>=1.10" --no-cache-dir --upgrade-strategy only-if-needed && \
-    pip install -r ./pytorch-lightning/requirements/extra.txt --no-cache-dir --upgrade-strategy only-if-needed && \
-    pip install -r ./pytorch-lightning/requirements/examples.txt --no-cache-dir --upgrade-strategy only-if-needed && \
-    pip install ./pytorch-lightning --no-cache-dir && \
+    pip install "Pillow>=8.2, !=8.3.0" "cryptography>=3.4" "py>=1.10" --no-cache-dir && \
+    pip install ./pytorch-lightning["extra","loggers","strategies"] --no-cache-dir && \
+    pip install -r ./pytorch-lightning/requirements/examples.txt --no-cache-dir && \
     rm -rf pytorch-lightning && \
-    pip install jupyterlab[all] -U && \
     pip list
 
+RUN pip install jupyterlab[all] -U
+
 RUN pip install lightning-grid -U && \
     pip install "py>=1.10" "protobuf>=3.15.6" --upgrade-strategy only-if-needed
 

dockers/release/Dockerfile

@@ -36,7 +36,7 @@ RUN \
         mv pytorch-lightning-*/ pytorch-lightning ; \
         rm *.zip ; \
     fi && \
-    pip install ./pytorch-lightning["extra"] --no-cache-dir && \
+    pip install ./pytorch-lightning["extra","loggers","strategies"] --no-cache-dir && \
     rm -rf pytorch-lightning
 
 RUN python --version && \

dockers/tpu-tests/Dockerfile

@@ -22,10 +22,7 @@ LABEL maintainer="PyTorchLightning <https://github.com/PyTorchLightning>"
 COPY ./ ./pytorch-lightning/
 
 # Pull the legacy checkpoints
-RUN cd pytorch-lightning && \
-    wget https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip -P legacy/ && \
-    unzip -o legacy/checkpoints.zip -d legacy/ && \
-    ls -l legacy/checkpoints/
+RUN cd pytorch-lightning && bash .actions/pull_legacy_checkpoints.sh
 
 RUN \
     pip install -q fire && \

legacy/README.md

@@ -7,8 +7,7 @@ At this moment we focus on ability running old checkpoints, so the flow here is
 If you want to pull all saved version-checkpoints for local testing/development, call
 
 ```bash
-wget https://pl-public-data.s3.amazonaws.com/legacy/checkpoints.zip
-unzip -o checkpoints.zip
+bash .actions/pull_legacy_checkpoints.sh
 ```
 
 To back populate collection with past version you can use following bash:

pl_examples/basic_examples/mnist_datamodule.py

@@ -36,7 +36,7 @@ class _MNIST(Dataset):
     """Carbon copy of ``tests.helpers.datasets.MNIST``.
 
     We cannot import the tests as they are not distributed with the package.
-    See https://github.com/PyTorchLightning/pytorch-lightning/pull/7614#discussion_r671183652 for more context.
+    See https://github.com/Lightning-AI/lightning/pull/7614#discussion_r671183652 for more context.
     """
 
     RESOURCES = (

pl_examples/domain_templates/reinforce_learn_ppo.py

@@ -125,7 +125,7 @@ def get_log_prob(self, pi: Normal, actions: torch.Tensor):
 
 
 class ExperienceSourceDataset(IterableDataset):
-    """Implementation from PyTorch Lightning Bolts: https://github.com/PyTorchLightning/lightning-
+    """Implementation from PyTorch Lightning Bolts: https://github.com/Lightning-AI/lightning-
     bolts/blob/master/pl_bolts/datamodules/experience_source.py.
 
     Basic experience source dataset. Takes a generate_batch function that returns an iterator. The logic for the

pytorch_lightning/callbacks/progress/rich_progress.py

@@ -12,13 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import math
+import operator
 from dataclasses import dataclass
 from datetime import timedelta
 from typing import Any, Dict, Optional, Union
 
 import pytorch_lightning as pl
 from pytorch_lightning.callbacks.progress.base import ProgressBarBase
-from pytorch_lightning.utilities.imports import _RICH_AVAILABLE
+from pytorch_lightning.utilities.imports import _compare_version, _package_available
+
+_RICH_AVAILABLE: bool = _package_available("rich") and _compare_version("rich", operator.ge, "10.2.2")
 
 Task, Style = None, None
 if _RICH_AVAILABLE:

pytorch_lightning/callbacks/rich_model_summary.py

@@ -14,7 +14,7 @@
 from typing import List, Tuple
 
 from pytorch_lightning.callbacks import ModelSummary
-from pytorch_lightning.utilities.imports import _RICH_AVAILABLE
+from pytorch_lightning.callbacks.progress.rich_progress import _RICH_AVAILABLE
 from pytorch_lightning.utilities.model_summary import get_human_readable_count
 
 if _RICH_AVAILABLE:

pytorch_lightning/core/lightning.py

@@ -46,7 +46,7 @@
 from pytorch_lightning.utilities.cloud_io import get_filesystem
 from pytorch_lightning.utilities.distributed import distributed_available, sync_ddp
 from pytorch_lightning.utilities.exceptions import MisconfigurationException
-from pytorch_lightning.utilities.imports import _TORCH_GREATER_EQUAL_1_12
+from pytorch_lightning.utilities.imports import _TORCH_GREATER_EQUAL_1_11, _TORCH_GREATER_EQUAL_1_12
 from pytorch_lightning.utilities.memory import get_model_size_mb
 from pytorch_lightning.utilities.model_summary import ModelSummary, summarize
 from pytorch_lightning.utilities.parsing import collect_init_args
@@ -2064,7 +2064,10 @@ def _register_sharded_tensor_state_dict_hooks_if_available(self) -> None:
             rank_zero_debug("Could not register sharded tensor state dict hooks")
             return
 
-        from torch.distributed._sharded_tensor import pre_load_state_dict_hook, state_dict_hook
+        if _TORCH_GREATER_EQUAL_1_11:
+            from torch.distributed._shard.sharded_tensor import pre_load_state_dict_hook, state_dict_hook
+        else:
+            from torch.distributed._sharded_tensor import pre_load_state_dict_hook, state_dict_hook
 
         self._register_state_dict_hook(state_dict_hook)