Weekly Patch Release v1.5.10

.azure-pipelines/gpu-benchmark.yml

@@ -28,15 +28,15 @@ jobs:
     cancelTimeoutInMinutes: "2"
     pool: gridai-spot-pool
     container:
-      # base ML image: mcr.microsoft.com/azureml/openmpi3.1.2-cuda10.2-cudnn8-ubuntu18.04
-      image: "pytorchlightning/pytorch_lightning:base-cuda-py3.8-torch1.8"
+      # should match the one in '.azure-pipelines/gpu-benchmark.yml'
+      image: "pytorchlightning/pytorch_lightning:base-cuda-py3.7-torch1.8"
       options: "--runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=all --shm-size=32g"
     workspace:
       clean: all
 
     steps:
       - bash: |
-          python -m pytest benchmarks -v --durations=0
+          python -m pytest tests/benchmarks -v --durations=0
         displayName: 'Testing: benchmarks'
         env:
           PL_RUNNING_BENCHMARKS: 1

.azure-pipelines/gpu-tests.yml

@@ -51,7 +51,7 @@ jobs:
     - bash: |
         python -c "fname = 'requirements/extra.txt' ; lines = [line for line in open(fname).readlines() if 'horovod' not in line] ; open(fname, 'w').writelines(lines)"
         pip install fairscale==0.4.0
-        pip install deepspeed==0.5.4
+        pip install deepspeed==0.5.7
         pip install . --requirement requirements/devel.txt
         pip list
       displayName: 'Install dependencies'
@@ -68,14 +68,14 @@ jobs:
       displayName: 'Get legacy checkpoints'
 
     - bash: |
-        python -m coverage run --source pytorch_lightning -m pytest pytorch_lightning tests -v --junitxml=$(Build.StagingDirectory)/test-results.xml --durations=50
+        python -m coverage run --source pytorch_lightning -m pytest pytorch_lightning tests --ignore tests/benchmarks -v --junitxml=$(Build.StagingDirectory)/test-results.xml --durations=50
       displayName: 'Testing: standard'
 
     - bash: |
-        bash tests/special_tests.sh
+        bash tests/standalone_tests.sh
       env:
         PL_USE_MOCKED_MNIST: "1"
-      displayName: 'Testing: special'
+      displayName: 'Testing: standalone'
 
     - bash: |
         python -m coverage report
@@ -113,5 +113,5 @@ jobs:
       displayName: 'Testing: examples'
 
     - bash: |
-        python -m pytest benchmarks -v --maxfail=2 --durations=0
+        python -m pytest tests/benchmarks -v --maxfail=2 --durations=0
       displayName: 'Testing: benchmarks'

.circleci/config.yml

@@ -5,6 +5,19 @@ orbs:
   go: circleci/go@1.3.0
   codecov: codecov/codecov@1.1.0
 
+trigger:
+  tags:
+    include:
+      - '*'
+  branches:
+    include:
+      - "master"
+      - "release/*"
+      - "refs/tags/*"
+pr:
+  - "master"
+  - "release/*"
+
 # Workflow Steps:
 #  1. Checkout
 #  2. Install GO

.github/ISSUE_TEMPLATE/bug_report.md

@@ -16,11 +16,11 @@ assignees: ''
 Please reproduce using the BoringModel!
 
 You can use the following Colab link:
-https://colab.research.google.com/drive/1HvWVVTK8j2Nj52qU4Q4YCyzOm0_aLQF3?usp=sharing
+https://colab.research.google.com/github/PytorchLightning/pytorch-lightning/blob/master/pl_examples/bug_report/bug_report_model.ipynb
 IMPORTANT: has to be public.
 
 or this simple template:
-https://github.com/PyTorchLightning/pytorch-lightning/blob/master/pl_examples/bug_report_model.py
+https://github.com/PyTorchLightning/pytorch-lightning/blob/master/pl_examples/bug_report/bug_report_model.py
 
 If you could not reproduce using the BoringModel and still think there's a bug, please post here
 but remember, bugs with code are fixed faster!
@@ -46,9 +46,9 @@ python collect_env_details.py
 You can also fill out the list below manually.
 -->
 
-- PyTorch Lightning Version (e.g., 1.3.0):
-- PyTorch Version (e.g., 1.8)
-- Python version:
+- PyTorch Lightning Version (e.g., 1.5.0):
+- PyTorch Version (e.g., 1.10):
+- Python version (e.g., 3.9):
 - OS (e.g., Linux):
 - CUDA/cuDNN version:
 - GPU models and configuration:

.github/set-min-requirements.py

@@ -2,8 +2,8 @@
     "requirements.txt",
     "requirements/extra.txt",
     "requirements/loggers.txt",
-    "requirements/test.txt",
     "requirements/examples.txt",
+    # "requirements/test.txt",  # Don't use old testing packages
 )
 
 

.github/workflows/ci_dockers.yml

@@ -1,4 +1,4 @@
-name: CI build Docker
+name: Docker
 # https://www.docker.com/blog/first-docker-github-action-is-here
 # https://github.com/docker/build-push-action
 # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
@@ -23,9 +23,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # should be the config used in '.github/workflows/release-docker.yml', but we just keep one to check.
-        python_version: ["3.9"]
-        pytorch_version: ["1.9"]
+        # the config used in '.azure-pipelines/gpu-tests.yml' since the Dockerfile uses the cuda image
+        python_version: ["3.7"]
+        pytorch_version: ["1.8"]
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -92,8 +92,8 @@ jobs:
       fail-fast: false
       matrix:
         # the config used in '.github/workflows/ci_test-conda.yml'
-        python_version: ["3.7"]
-        pytorch_version: ["1.6", "1.7", "1.8", "1.9", "1.10"]
+        python_version: ["3.8"]
+        pytorch_version: ["1.7", "1.8", "1.9", "1.10"]
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -119,7 +119,7 @@ jobs:
       fail-fast: false
       matrix:
         # the config used in 'dockers/ipu-ci-runner/Dockerfile'
-        python_version: ["3.8"]  # latest
+        python_version: ["3.9"]  # latest
         # TODO: upgrade - PopTorch 2.2 uses torch 1.9, see:
         # https://docs.graphcore.ai/projects/poptorch-user-guide/en/latest/installation.html#version-compatibility
         pytorch_version: ["1.7"]

.github/workflows/ci_pkg-install.yml

@@ -1,4 +1,4 @@
-name: Install pkg
+name: Package
 
 # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
 on:  # Trigger the workflow on push or pull request, but only for the master branch
@@ -9,7 +9,7 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
 
 jobs:
 
-  pkg-install:
+  install:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -26,12 +26,11 @@ jobs:
 
       - name: Prepare env
         run: |
-          pip install check-manifest "twine==3.2" setuptools wheel
+          pip install "twine==3.2" setuptools wheel
 
       - name: Create package
         run: |
-          check-manifest
-          # python setup.py check --metadata --strict
+          python setup.py check --metadata --strict
           python setup.py sdist bdist_wheel
 
       - name: Check package

.github/workflows/ci_schema.yml

@@ -1,11 +1,11 @@
-name: CI action schema
+name: Schema
 on: # Trigger the workflow on push or pull request, but only for the master branch
   push: {}
   pull_request:
     branches: [master, "release/*"]
 
 jobs:
-  validate-schema:
+  check:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout

.github/workflows/ci_test-base.yml

@@ -1,6 +1,6 @@
 # this jobs runs `pytest` over the source directory. It does not install any extra dependencies.
 # this is useful to catch errors where an import has been added which is not part of the basic dependencies.
-name: CI basic testing
+name: Test
 
 # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
 on:  # Trigger the workflow on push or pull request, but only for the master branch
@@ -10,8 +10,7 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
     branches: [master, "release/*"]
 
 jobs:
-  doctest:
-
+  source:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -20,7 +19,7 @@ jobs:
         # this will install stable torch
         python-version: [3.9]
 
-    # Timeout: https://stackoverflow.com/a/59076067/4521646
+    # lower timeout as this should run very quickly
     timeout-minutes: 20
     steps:
     - uses: actions/checkout@v2
@@ -60,7 +59,6 @@ jobs:
 
     - name: Test Package [only]
       run: |
-        # NOTE: run coverage on tests does not propagate failure status for Win, https://github.com/nedbat/coveragepy/issues/1003
         coverage run --source pytorch_lightning -m pytest pytorch_lightning -v --junitxml=junit/test-results-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.requires }}.xml
 
     - name: Upload pytest test results

.github/workflows/ci_test-conda.yml

@@ -1,4 +1,4 @@
-name: PyTorch & Conda
+name: Test
 
 # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
 on:  # Trigger the workflow on push or pull request, but only for the master branch
@@ -14,11 +14,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7"]
-        pytorch-version: ["1.6", "1.7", "1.8", "1.9", "1.10"]
+        python-version: ["3.8"]  # previous to last Python version as that one is already used in test-full
+        pytorch-version: ["1.7", "1.8", "1.9", "1.10"]  # nightly: add when there's a release candidate
 
-    # Timeout: https://stackoverflow.com/a/59076067/4521646
-    timeout-minutes: 35
+    timeout-minutes: 30
     steps:
     - uses: actions/checkout@v2
 
@@ -30,7 +29,8 @@ jobs:
         python ./requirements/adjust_versions.py requirements/extra.txt
         python ./requirements/adjust_versions.py requirements/examples.txt
         pip install --requirement requirements/devel.txt --find-links https://download.pytorch.org/whl/nightly/torch_nightly.html
-        pip install pytest-random-order
+        # set a per-test timeout of 2.5 minutes to fail sooner. this aids with hanging tests
+        pip install pytest-timeout
         pip list
 
     - name: Pull checkpoints from S3
@@ -43,8 +43,7 @@ jobs:
 
     - name: Tests
       run: |
-        # NOTE: run coverage on tests does not propagate failure status for Win, https://github.com/nedbat/coveragepy/issues/1003
-        coverage run --source pytorch_lightning -m pytest --random-order-seed=1 pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-torch${{ matrix.pytorch-version }}.xml
+        coverage run --source pytorch_lightning -m pytest --timeout 150 pytorch_lightning tests -v --durations=50 --junitxml=junit/test-results-${{ runner.os }}-torch${{ matrix.pytorch-version }}.xml
       shell: bash -l {0}
 
     - name: Upload pytest results

.github/workflows/ci_test-full.yml

@@ -1,4 +1,4 @@
-name: CI complete testing
+name: Test
 
 # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows
 on:  # Trigger the workflow on push or pull request, but only for the master branch
@@ -10,28 +10,28 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
 
 jobs:
 
-  pytest:
+  cpu:
 
     runs-on: ${{ matrix.os }}
     if: github.event.pull_request.draft == false
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-18.04, windows-2019, macOS-10.15]
-        python-version: [3.6, 3.8, 3.9]
-        requires: ['minimal', 'latest']
-        release: ['stable']
-        exclude:
-          - python-version: 3.9
-            requires: 'minimal'
+        python-version: ["3.7", "3.9"]  # minimum, maximum
+        requires: ["oldest", "latest"]
+        release: ["stable"]
         include:
-          - os: ubuntu-20.04
-            python-version: 3.9
-            requires: 'latest'
-            release: 'pre'
+          # test 3.6 only on oldest until EOL: https://github.com/PyTorchLightning/pytorch-lightning/issues/9981
+          - {os: ubuntu-18.04, python-version: "3.6", requires: "oldest", release: "stable"}
+          - {os: windows-2019, python-version: "3.6", requires: "oldest", release: "stable"}
+          - {os: macOS-10.15, python-version: "3.6", requires: "oldest", release: "stable"}
+          # nightly: add when there's a release candidate
+          #- {os: ubuntu-20.04, python-version: "3.10", requires: "latest", release: "pre"}
+        exclude:
+          # Skip if torch<1.8 and py3.9 on Linux: https://github.com/pytorch/pytorch/issues/50014
+          - {os: ubuntu-18.04, python-version: "3.9", requires: "oldest", release: "stable"}
 
-    # Timeout: https://stackoverflow.com/a/59076067/4521646
-    # TODO: the macOS is taking too long, probably caching did not work...
     timeout-minutes: 40
 
     steps:
@@ -64,7 +64,7 @@ jobs:
         python .github/prune-packages.py requirements/extra.txt "horovod"
 
     - name: Set min. dependencies
-      if: matrix.requires == 'minimal'
+      if: matrix.requires == 'oldest'
       run: |
         python .github/set-min-requirements.py
 

.github/workflows/code-checks.yml

@@ -1,4 +1,4 @@
-name: "Check code"
+name: Test
 
 on:  # Trigger the workflow on push or pull request, but only for the master branch
   push:
@@ -7,8 +7,7 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
     branches: [master, "release/*"]
 
 jobs:
-  python-typing-mypy:
-    name: Python typing Mypy
+  mypy:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@master

.github/workflows/docs-checks.yml

@@ -1,4 +1,4 @@
-name: "Docs check"
+name: Test
 # https://github.com/marketplace/actions/sphinx-build
 
 on:  # Trigger the workflow on push or pull request, but only for the master branch
@@ -8,7 +8,7 @@ on:  # Trigger the workflow on push or pull request, but only for the master bra
     branches: [master, "release/*"]
 
 jobs:
-  test-docs:
+  doctest:
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v2

.github/workflows/events-nightly.yml

@@ -1,4 +1,4 @@
-name: Nightly events
+name: Nightly
 
 # https://jasonet.co/posts/scheduled-actions/
 # https://git.luolix.topmunity/t/distinct-job-for-each-schedule/17811/2
@@ -122,8 +122,8 @@ jobs:
       fail-fast: false
       matrix:
         # the config used in '.github/workflows/ci_test-conda.yml'
-        python_version: ["3.7"]
-        pytorch_version: ["1.6", "1.7", "1.8", "1.9", "1.10"]
+        python_version: ["3.8"]
+        pytorch_version: ["1.7", "1.8", "1.9", "1.10"]
 
     steps:
       - name: Checkout
@@ -163,7 +163,7 @@ jobs:
       matrix:
         # the config used in 'dockers/ipu-ci-runner/Dockerfile'
         include:
-          - python_version: "3.8"
+          - python_version: "3.9"
             pytorch_version: "1.7"
 
     steps:

.github/workflows/events-recurrent.yml

@@ -1,4 +1,4 @@
-name: Recurrent events
+name: Recurrent
 
 # https://jasonet.co/posts/scheduled-actions/
 # https://git.luolix.topmunity/t/distinct-job-for-each-schedule/17811/2