Add CI on pre-commits and tests

.github/workflows/publish.yml

@@ -0,0 +1,135 @@
+name: Publish
+
+on:
+  release:
+    types:
+      - created
+
+env:
+  CACHE_NUMBER: 0  # increase to reset cache manually
+  CONDA_ENV_FILE: './env.yaml'
+  CONDA_ENV_NAME: 'nn-template'
+
+jobs:
+  build:
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.9']
+        include:
+          - os: ubuntu-20.04
+            label: linux-64
+            prefix: /usr/share/miniconda3/envs/
+
+    name: ${{ matrix.label }}-py${{ matrix.python-version }}
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Parametrize conda env name
+        run: echo "PY_CONDA_ENV_NAME=${{ env.CONDA_ENV_NAME }}-${{ matrix.python-version }}" >> $GITHUB_ENV
+      - name: echo conda env name
+        run: echo ${{ env.PY_CONDA_ENV_NAME }}
+
+      - name: Parametrize conda prefix
+        run: echo "PY_PREFIX=${{ matrix.prefix }}${{ env.PY_CONDA_ENV_NAME }}" >> $GITHUB_ENV
+      - name: echo conda prefix
+        run: echo ${{ env.PY_PREFIX }}
+
+      # extract the first two digits from the release note
+      - name: Set release notes tag
+        run: |
+          export RELEASE_TAG_VERSION=${{ github.event.release.tag_name }}
+          echo "RELEASE_TAG_VERSION=${RELEASE_TAG_VERSION%.*}">> $GITHUB_ENV
+
+      - name: Echo release notes tag
+        run: |
+          echo "${RELEASE_TAG_VERSION}"
+
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      # Remove the python version pin from the env.yml which could be inconsistent
+      - name: Remove explicit python version from the environment
+        shell: bash -l {0}
+        run: |
+          sed -Ei '/^\s*-?\s*python\s*([#=].*)?$/d' ${{ env.CONDA_ENV_FILE }}
+          cat  ${{ env.CONDA_ENV_FILE }}
+
+      - name: Setup Mambaforge
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+            miniforge-variant: Mambaforge
+            miniforge-version: latest
+            activate-environment: ${{ env.PY_CONDA_ENV_NAME }}
+            python-version: ${{ matrix.python-version }}
+            use-mamba: true
+
+      - uses: actions/cache@v2
+        name: Conda cache
+        with:
+          path: ${{ env.PY_PREFIX }}
+          key: ${{ matrix.label }}-conda-${{ matrix.python-version }}-${{ env.CACHE_NUMBER }}-${{ env.PY_CONDA_ENV_NAME }}-${{ hashFiles(env.CONDA_ENV_FILE) }}-${{hashFiles('./setup.cfg') }}
+        id: conda_cache
+
+      - uses: actions/cache@v2
+        name: Pip cache
+        with:
+          path: ~/.cache/pip
+          key: ${{ matrix.label }}-pip-${{ matrix.python-version }}-${{ env.CACHE_NUMBER }}-${{ env.PY_CONDA_ENV_NAME }}-${{ hashFiles(env.CONDA_ENV_FILE) }}-${{hashFiles('./setup.cfg') }}
+
+      - uses: actions/cache@v2
+        name: Pre-commit cache
+        with:
+          path: ~/.cache/pre-commit
+          key: ${{ matrix.label }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}-${{ matrix.python-version }}-${{ env.CACHE_NUMBER }}-${{ env.PY_CONDA_ENV_NAME }}-${{ hashFiles(env.CONDA_ENV_FILE) }}-${{hashFiles('./setup.cfg') }}
+
+      # Ensure the hack for the python version worked
+      - name: Ensure we have the right Python
+        shell: bash -l {0}
+        run: |
+          echo "Installed Python: $(python --version)"
+          echo "Expected: ${{ matrix.python-version }}"
+          python --version | grep "Python ${{ matrix.python-version }}"
+
+      - name: Update conda environment
+        run: mamba env update -n ${{ env.PY_CONDA_ENV_NAME }} -f ${{ env.CONDA_ENV_FILE }}
+        if: steps.conda_cache.outputs.cache-hit != 'true'
+
+      # Update pip env whether or not there was a conda cache hit
+      - name: Update pip environment
+        shell: bash -l {0}
+        run: pip install -e ".[dev]"
+        if: steps.conda_cache.outputs.cache-hit == 'true'
+
+      - run: pip3 list
+        shell: bash -l {0}
+      - run: mamba info
+      - run: mamba list
+
+       # Ensure the hack for the python version worked
+      - name: Ensure we have the right Python
+        shell: bash -l {0}
+        run: |
+          echo "Installed Python: $(python --version)"
+          echo "Expected: ${{ matrix.python-version }}"
+          python --version | grep "Python ${{ matrix.python-version }}"
+
+      - name: Run pre-commits
+        shell: bash -l {0}
+        run: |
+          pre-commit install
+          pre-commit run -v --all-files --show-diff-on-failure
+
+      - name: Test with pytest
+        shell: bash -l {0}
+        run: |
+          pytest -v
+
+      - name: Build docs website
+        shell: bash -l {0}
+        run: |
+          git config user.name ci-bot
+          git config user.email ci-bot@ci.com
+          mike deploy --push --rebase --update-aliases ${RELEASE_TAG_VERSION} latest

.github/workflows/test_suite.yml

@@ -0,0 +1,142 @@
+name: Test Suite
+
+on:
+  push:
+    branches:
+      - main
+      - develop
+
+  pull_request:
+    types:
+      - opened
+      - edited
+      - reopened
+      - synchronize
+      - ready_for_review
+      - review_requested
+      - auto_merge_enabled
+
+env:
+  CACHE_NUMBER: 1  # increase to reset cache manually
+  CONDA_ENV_FILE: './env.yaml'
+  CONDA_ENV_NAME: 'nn-template'
+
+jobs:
+  build:
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.8', '3.9']
+        include:
+          - os: ubuntu-20.04
+            label: linux-64
+            prefix: /usr/share/miniconda3/envs/
+
+#           - os: macos-latest
+#             label: osx-64
+#             prefix: /Users/runner/miniconda3/envs/$CONDA_ENV_NAME
+
+#           - os: windows-latest
+#             label: win-64
+#             prefix: C:\Miniconda3\envs\$CONDA_ENV_NAME
+
+    name: ${{ matrix.label }}-py${{ matrix.python-version }}
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Parametrize conda env name
+        run: echo "PY_CONDA_ENV_NAME=${{ env.CONDA_ENV_NAME }}-${{ matrix.python-version }}" >> $GITHUB_ENV
+      - name: echo conda env name
+        run: echo ${{ env.PY_CONDA_ENV_NAME }}
+
+      - name: Parametrize conda prefix
+        run: echo "PY_PREFIX=${{ matrix.prefix }}${{ env.PY_CONDA_ENV_NAME }}" >> $GITHUB_ENV
+      - name: echo conda prefix
+        run: echo ${{ env.PY_PREFIX }}
+
+      - uses: actions/checkout@v2
+
+      # Remove the python version pin from the env.yml which could be inconsistent
+      - name: Remove explicit python version from the environment
+        shell: bash -l {0}
+        run: |
+          sed -Ei '/^\s*-?\s*python\s*([#=].*)?$/d' ${{ env.CONDA_ENV_FILE }}
+          cat  ${{ env.CONDA_ENV_FILE }}
+
+      - name: Setup Mambaforge
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+            miniforge-variant: Mambaforge
+            miniforge-version: latest
+            activate-environment: ${{ env.PY_CONDA_ENV_NAME }}
+            python-version: ${{ matrix.python-version }}
+            use-mamba: true
+
+      - uses: actions/cache@v2
+        name: Conda cache
+        with:
+          path: ${{ env.PY_PREFIX }}
+          key: ${{ matrix.label }}-conda-${{ matrix.python-version }}-${{ env.CACHE_NUMBER }}-${{ env.PY_CONDA_ENV_NAME }}-${{ hashFiles(env.CONDA_ENV_FILE) }}-${{hashFiles('./setup.cfg') }}
+        id: conda_cache
+
+      - uses: actions/cache@v2
+        name: Pip cache
+        with:
+          path: ~/.cache/pip
+          key: ${{ matrix.label }}-pip-${{ matrix.python-version }}-${{ env.CACHE_NUMBER }}-${{ env.PY_CONDA_ENV_NAME }}-${{ hashFiles(env.CONDA_ENV_FILE) }}-${{hashFiles('./setup.cfg') }}
+
+      - uses: actions/cache@v2
+        name: Pre-commit cache
+        with:
+          path: ~/.cache/pre-commit
+          key: ${{ matrix.label }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}-${{ matrix.python-version }}-${{ env.CACHE_NUMBER }}-${{ env.PY_CONDA_ENV_NAME }}-${{ hashFiles(env.CONDA_ENV_FILE) }}-${{hashFiles('./setup.cfg') }}
+
+      # Ensure the hack for the python version worked
+      - name: Ensure we have the right Python
+        shell: bash -l {0}
+        run: |
+          echo "Installed Python: $(python --version)"
+          echo "Expected: ${{ matrix.python-version }}"
+          python --version | grep "Python ${{ matrix.python-version }}"
+
+      - name: Update conda environment
+        run: mamba env update -n ${{ env.PY_CONDA_ENV_NAME }} -f ${{ env.CONDA_ENV_FILE }}
+        if: steps.conda_cache.outputs.cache-hit != 'true'
+
+      # Update pip env whether or not there was a conda cache hit
+      - name: Update pip environment
+        shell: bash -l {0}
+        run: pip install -e ".[dev]"
+        if: steps.conda_cache.outputs.cache-hit == 'true'
+
+      - run: pip3 list
+        shell: bash -l {0}
+      - run: mamba info
+      - run: mamba list
+
+       # Ensure the hack for the python version worked
+      - name: Ensure we have the right Python
+        shell: bash -l {0}
+        run: |
+          echo "Installed Python: $(python --version)"
+          echo "Expected: ${{ matrix.python-version }}"
+          python --version | grep "Python ${{ matrix.python-version }}"
+
+      - name: Run pre-commits
+        shell: bash -l {0}
+        run: |
+          pre-commit install
+          pre-commit run -v --all-files --show-diff-on-failure
+
+      # https://stackoverflow.com/questions/70520120/attributeerror-module-setuptools-distutils-has-no-attribute-version
+      # https://github.com/pytorch/pytorch/pull/69904
+      - name: Downgrade setuptools due to a but in PyTorch 1.10.1
+        shell: bash -l {0}
+        run: |
+          pip install setuptools==59.5.0
+
+      - name: Test with pytest
+        shell: bash -l {0}
+        run: |
+          pytest -v

conf/nn/default.yaml

@@ -44,4 +44,4 @@ module:
     T_mult: 2
     eta_min: 0 # min value for the lr
     last_epoch: -1
-    verbose: True
+    verbose: False

env.yaml

@@ -4,7 +4,7 @@ channels:
   - pytorch
 dependencies:
   - python=3.9
-  - pytorch>=1.10,<2.0
+  - pytorch=1.10.*
   - torchvision=0.11.*
   - pip
   - pip:

pyproject.toml

@@ -39,5 +39,5 @@ write_to = "src/nn_template/_version.py"
 write_to_template = '__version__ = "{version}"'
 
 [build-system]
-requires = ["setuptools>=42", "wheel", "setuptools_scm[toml]>=6.3.1"]
+requires = ["setuptools==59.5", "wheel", "setuptools_scm[toml]>=6.3.1"]
 build-backend = "setuptools.build_meta"

setup.cfg

@@ -14,7 +14,7 @@ package_dir=
     =src
 packages=find:
 install_requires =
-    nn-template-core>=0.0.1,<1.0.0
+    nn-template-core==0.0.3
 
     # Add project specific dependencies
     # Stuff easy to break with updates

src/nn_template/run.py

@@ -4,6 +4,8 @@
 # See https://github.com/PyTorchLightning/pytorch-lightning/issues/1503
 #
 # Force the execution of __init__.py if this file is executed directly.
+import nn_template  # isort:skip # noqa
+
 import logging
 from operator import xor
 from typing import List, Optional, Tuple
@@ -19,9 +21,6 @@
 from nn_core.model_logging import NNLogger
 from nn_core.resume import resolve_ckpt, resolve_run_path, resolve_run_version
 
-import nn_template  # isort:skip # noqa
-
-
 pylogger = logging.getLogger(__name__)
 
 

tests/conftest.py

@@ -47,6 +47,9 @@ def cfg(tmp_path_factory: TempPathFactory) -> DictConfig:
 def cfg_simple_train(cfg: DictConfig) -> DictConfig:
     cfg = OmegaConf.create(cfg)
 
+    # Disable gpus
+    cfg.train.trainer.gpus = 0
+
     # Disable logger
     cfg.train.logging.logger.mode = "disabled"