build: migrate project toolchain to python 3.11 and ubuntu 22.04

This commit upgrades all of the project configuration files, GitHub Action workflows, and Dockerfiles to use Python 3.11 and Ubuntu 22.04. The Dockerfiles use the deadsnakes PPA to install a recent version of Python 3.11, since the Ubuntu package repository only provides the first release candidate for Python 3.11.0, which is missing bug fixes. References to Python 3.9 have been replaced with Python 3.11 in the project documentation. This commit also upgrades the pinned version of Tensorflow to 2.16.1 and PyTorch to 2.2.2, which both have installable wheel support for Python 3.11. This change required updating some of the Dioptra plugins to work with the default eager execution model and the current API for Tensorflow. The detectron2 and pytorch-lightning packages have been dropped due to lack of support and compatibility with these versions of PyTorch and Python. This commit also adds async_timeout as an explicit dependency for the worker and restapi images. This fixes the problem introduced in redis/redis-py#2602 where async_timeout was removed as a dependency for Python 3.11, which had a ripple effect that resulted in unexpected breakage. Adding the dependency back fixes the issue. Co-authored-by: Keith Manville <kmanville@mitre.org>
usnistgov · May 2, 2024 · c8f2c01 · c8f2c01
1 parent d39c986
commit c8f2c01
Show file tree

Hide file tree

Showing 90 changed files with 11,424 additions and 12,574 deletions.
diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml
@@ -35,7 +35,7 @@ env:
 
 jobs:
   build-docker-dioptra-apps:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         dioptra-app:
@@ -111,7 +111,7 @@ jobs:
           path: /tmp/${{ env.PROJECT_PREFIX }}_${{ matrix.dioptra-app }}_dev.tar.gz
 
   container-test:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     if: ${{ false }}  # disable for now
     needs:
       - build-docker-dioptra-apps
@@ -123,10 +123,10 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: setup python 3.9
+      - name: setup python 3.11
         uses: actions/setup-python@v5.1.0
         with:
-          python-version: "3.9"
+          python-version: "3.11"
 
       - name: upgrade pip
         run: |
@@ -170,7 +170,7 @@ jobs:
           tox -e containers-${{ matrix.dioptra-app }}
 
   integration-test:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     if: ${{ false }}  # disable for now
     needs:
       - build-docker-dioptra-apps
@@ -183,10 +183,10 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: setup python 3.9
+      - name: setup python 3.11
         uses: actions/setup-python@v5.1.0
         with:
-          python-version: "3.9"
+          python-version: "3.11"
 
       - name: upgrade pip
         run: |

diff --git a/.github/workflows/pip-compile.yml b/.github/workflows/pip-compile.yml
@@ -25,13 +25,22 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: ["ubuntu-20.04", "macos-13", "windows-latest"]
-        python-version: ["3.9"]
-        architecture: ["x86_64"]
+        os: ["ubuntu-22.04", "macos-13", "macos-latest", "windows-latest"]
+        python-version: ["3.11"]
+        architecture: ["x86_64", "aarch64"]
         requirements:
           - "requirements-dev"
           - "requirements-dev-pytorch"
           - "requirements-dev-tensorflow"
+        exclude:
+          - os: "macos-latest"
+            architecture: "x86_64"
+          - os: "ubuntu-22.04"
+            architecture: "aarch64"
+          - os: "macos-13"
+            architecture: "aarch64"
+          - os: "windows-latest"
+            architecture: "aarch64"
 
     steps:
       - uses: actions/checkout@v4
@@ -77,35 +86,20 @@ jobs:
       - name: install dependencies
         run: python3 -m pip install --upgrade tox
 
-      - name: run tox (Linux, Python 3.9)
-        if: ${{ matrix.os == 'ubuntu-20.04' && matrix.python-version == '3.9' }}
-        run: |
-          python3 -m tox run -e py39-linux-${{ matrix.architecture }}-${{ matrix.requirements }}
-
-      - name: run tox (Linux, Python 3.10)
-        if: ${{ matrix.os == 'ubuntu-20.04' && matrix.python-version == '3.10' }}
-        run: |
-          python3 -m tox run -e py310-linux-${{ matrix.architecture }}-${{ matrix.requirements }}
-
-      - name: run tox (MacOS, Python 3.9)
-        if: ${{ matrix.os == 'macos-13' && matrix.python-version == '3.9' }}
-        run: |
-          python3 -m tox run -e py39-macos-${{ matrix.architecture }}-${{ matrix.requirements }}
-
-      - name: run tox (MacOS, Python 3.10)
-        if: ${{ matrix.os == 'macos-13' && matrix.python-version == '3.10' }}
+      - name: run tox (Linux, Python 3.11)
+        if: ${{ matrix.os == 'ubuntu-22.04' && matrix.python-version == '3.11' }}
         run: |
-          python3 -m tox run -e py310-macos-${{ matrix.architecture }}-${{ matrix.requirements }}
+          python3 -m tox run -e py311-linux-${{ matrix.architecture }}-${{ matrix.requirements }}
 
-      - name: run tox (Windows, Python 3.9)
-        if: ${{ matrix.os == 'windows-latest' && matrix.python-version == '3.9' }}
+      - name: run tox (MacOS, Python 3.11)
+        if: ${{ (matrix.os == 'macos-13' || matrix.os == 'macos-latest') && matrix.python-version == '3.11' }}
         run: |
-          python3 -m tox run -e py39-win-${{ matrix.architecture }}-${{ matrix.requirements }}
+          python3 -m tox run -e py311-macos-${{ matrix.architecture }}-${{ matrix.requirements }}
 
-      - name: run tox (Windows, Python 3.10)
-        if: ${{ matrix.os == 'windows-latest' && matrix.python-version == '3.10' }}
+      - name: run tox (Windows, Python 3.11)
+        if: ${{ matrix.os == 'windows-latest' && matrix.python-version == '3.11' }}
         run: |
-          python3 -m tox run -e py310-win-${{ matrix.architecture }}-${{ matrix.requirements }}
+          python3 -m tox run -e py311-win-${{ matrix.architecture }}-${{ matrix.requirements }}
 
       - name: archive the requirements file
         uses: actions/upload-artifact@v4
@@ -117,8 +111,8 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: ["ubuntu-20.04"]
-        python-version: ["3.9"]
+        os: ["ubuntu-22.04"]
+        python-version: ["3.11"]
         architecture: ["x86_64"]
         requirements:
           - "mlflow-tracking-requirements"
@@ -158,7 +152,7 @@ jobs:
 
       - name: run tox
         run: |
-          python3 -m tox run -e py39-linux-${{ matrix.architecture }}-${{ matrix.requirements }}
+          python3 -m tox run -e py311-linux-${{ matrix.architecture }}-${{ matrix.requirements }}
 
       - name: archive the requirements file
         uses: actions/upload-artifact@v4
@@ -167,17 +161,17 @@ jobs:
           path: venvs
 
   pip-compile-aarch64:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         tox-testenv:
-          - "py39-linux-aarch64-requirements-dev"
-          - "py39-linux-aarch64-requirements-dev-pytorch"
-          - "py39-linux-aarch64-requirements-dev-tensorflow"
-          - "py39-linux-aarch64-mlflow-tracking-requirements"
-          - "py39-linux-aarch64-restapi-requirements"
-          - "py39-linux-aarch64-pytorch-cpu-requirements"
-          - "py39-linux-aarch64-tensorflow2-cpu-requirements"
+          - "py311-linux-aarch64-requirements-dev"
+          - "py311-linux-aarch64-requirements-dev-pytorch"
+          - "py311-linux-aarch64-requirements-dev-tensorflow"
+          - "py311-linux-aarch64-mlflow-tracking-requirements"
+          - "py311-linux-aarch64-restapi-requirements"
+          - "py311-linux-aarch64-pytorch-cpu-requirements"
+          - "py311-linux-aarch64-tensorflow2-cpu-requirements"
 
     steps:
       - uses: actions/checkout@v4
@@ -190,7 +184,7 @@ jobs:
 
       - name: setup dockerfile and output folder
         run: |
-          echo 'FROM python:3.9-slim-bullseye' > Dockerfile.aarch64
+          echo 'FROM python:3.11-slim-bullseye' > Dockerfile.aarch64
           echo 'COPY . /app' >> Dockerfile.aarch64
           echo 'WORKDIR /app' >> Dockerfile.aarch64
           echo 'RUN python -m pip install --upgrade pip tox && \' >> Dockerfile.aarch64
@@ -218,7 +212,7 @@ jobs:
           path: /image/app/venvs
 
   consolidate-requirements-files:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     needs:
       - pip-compile
       - pip-compile-containers

diff --git a/.github/workflows/sphinx-docs.yml b/.github/workflows/sphinx-docs.yml
@@ -25,14 +25,14 @@ on:
 
 jobs:
   docs:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
 
-      - name: setup python 3.9
+      - name: setup python 3.11
         uses: actions/setup-python@v5.1.0
         with:
-          python-version: "3.9"
+          python-version: "3.11"
 
       - name: upgrade pip
         run: |

diff --git a/.github/workflows/tox-tests.yml b/.github/workflows/tox-tests.yml
@@ -25,10 +25,10 @@ on:
 
 jobs:
   linting-and-style-checks:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: ["3.9"]
+        python-version: ["3.11"]
         tox-testenv:
           - "black"
           - "flake8"
@@ -71,28 +71,17 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: ["ubuntu-20.04", "windows-latest"]
-        python-version: ["3.9", "3.10"]
+        os: ["ubuntu-22.04", "windows-latest"]
+        python-version: ["3.11"]
         tox-testenv:
-          - "clean,py39-pytest-cov,report"
-          - "clean,py310-pytest-cov,report"
-          - "py39-cookiecutter"
-          - "py310-cookiecutter"
-        exclude:
-          - python-version: "3.9"
-            tox-testenv: "clean,py310-pytest-cov,report"
-          - python-version: "3.9"
-            tox-testenv: "py310-cookiecutter"
-          - python-version: "3.10"
-            tox-testenv: "clean,py39-pytest-cov,report"
-          - python-version: "3.10"
-            tox-testenv: "py39-cookiecutter"
+          - "clean,py311-pytest-cov,report"
+          - "py311-cookiecutter"
 
     steps:
       - uses: actions/checkout@v4
 
       - name: install English words dictionary
-        if: ${{ matrix.os == 'ubuntu-20.04' }}
+        if: ${{ matrix.os == 'ubuntu-22.04' }}
         run: sudo apt install -y wamerican
 
       - name: setup python ${{ matrix.python-version }}

diff --git a/README.md b/README.md
@@ -27,26 +27,26 @@ For instructions on how to build and run a fresh instance of Dioptra, see [cooki
 
 ### Setting up the Python virtual environment
 
-Developers must use Python 3.9 and create a virtual environment using one of the requirements.txt files in the `requirements/` directory in order to make contributions to this project.
-Ensure that you have Python 3.9 installed and that it is available in your PATH, and then identify the requirements file that you want to use:
+Developers must use Python 3.11 and create a virtual environment using one of the requirements.txt files in the `requirements/` directory in order to make contributions to this project.
+Ensure that you have Python 3.11 installed and that it is available in your PATH, and then identify the requirements file that you want to use:
 
 | Filename | OS | Architecture | Tensorflow | PyTorch |
 | :--- | :---: | :---: | :--- | :--- |
-| linux-amd64-py3.9-requirements-dev.txt | Linux | x86-64 | ❌ | ❌ |
-| linux-amd64-py3.9-requirements-dev-tensorflow.txt | Linux | x86-64 | ✅ | ❌ |
-| linux-amd64-py3.9-requirements-dev-pytorch.txt | Linux | x86-64 | ❌ | ✅ |
-| linux-arm64-py3.9-requirements-dev.txt | Linux | arm64 | ❌ | ❌ |
-| linux-arm64-py3.9-requirements-dev-tensorflow.txt | Linux | arm64 | ❌ | ❌ |
-| linux-arm64-py3.9-requirements-dev-pytorch.txt | Linux | arm64 | ❌ | ✅ |
-| macos-amd64-py3.9-requirements-dev.txt | MacOS | x86-64 | ❌ | ❌ |
-| macos-amd64-py3.9-requirements-dev-tensorflow.txt | MacOS | x86-64 | ✅ | ❌ |
-| macos-amd64-py3.9-requirements-dev-pytorch.txt | MacOS | x86-64 | ❌ | ✅ |
-| macos-arm64-py3.9-requirements-dev.txt | MacOS | arm64 | ❌ | ❌ |
-| macos-arm64-py3.9-requirements-dev-tensorflow.txt | MacOS | arm64 | ✅ | ❌ |
-| macos-arm64-py3.9-requirements-dev-pytorch.txt | MacOS | arm64 | ❌ | ✅ |
-| win-amd64-py3.9-requirements-dev.txt | Windows | x86-64 | ❌ | ❌ |
-| win-amd64-py3.9-requirements-dev-tensorflow.txt | Windows | x86-64 | ✅ | ❌ |
-| win-amd64-py3.9-requirements-dev-pytorch.txt | Windows | x86-64 | ❌ | ✅ |
+| linux-amd64-py3.11-requirements-dev.txt | Linux | x86-64 | ❌ | ❌ |
+| linux-amd64-py3.11-requirements-dev-tensorflow.txt | Linux | x86-64 | ✅ | ❌ |
+| linux-amd64-py3.11-requirements-dev-pytorch.txt | Linux | x86-64 | ❌ | ✅ |
+| linux-arm64-py3.11-requirements-dev.txt | Linux | arm64 | ❌ | ❌ |
+| linux-arm64-py3.11-requirements-dev-tensorflow.txt | Linux | arm64 | ✅ | ❌ |
+| linux-arm64-py3.11-requirements-dev-pytorch.txt | Linux | arm64 | ❌ | ✅ |
+| macos-amd64-py3.11-requirements-dev.txt | MacOS | x86-64 | ❌ | ❌ |
+| macos-amd64-py3.11-requirements-dev-tensorflow.txt | MacOS | x86-64 | ✅ | ❌ |
+| macos-amd64-py3.11-requirements-dev-pytorch.txt | MacOS | x86-64 | ❌ | ✅ |
+| macos-arm64-py3.11-requirements-dev.txt | MacOS | arm64 | ❌ | ❌ |
+| macos-arm64-py3.11-requirements-dev-tensorflow.txt | MacOS | arm64 | ✅ | ❌ |
+| macos-arm64-py3.11-requirements-dev-pytorch.txt | MacOS | arm64 | ❌ | ✅ |
+| win-amd64-py3.11-requirements-dev.txt | Windows | x86-64 | ❌ | ❌ |
+| win-amd64-py3.11-requirements-dev-tensorflow.txt | Windows | x86-64 | ✅ | ❌ |
+| win-amd64-py3.11-requirements-dev-pytorch.txt | Windows | x86-64 | ❌ | ✅ |
 
 Next, use the `venv` module to create a new virtual environment:
 
@@ -77,15 +77,15 @@ Finally, use `pip-sync` to install the dependencies in your chosen requirements
 On MacOS/Linux:
 
 ```sh
-# Replace "linux-amd64-py3.9-requirements-dev.txt" with your chosen file
-pip-sync requirements/linux-amd64-py3.9-requirements-dev.txt
+# Replace "linux-amd64-py3.11-requirements-dev.txt" with your chosen file
+pip-sync requirements/linux-amd64-py3.11-requirements-dev.txt
 ```
 
 On Windows:
 
 ```powershell
-# Replace "win-amd64-py3.9-requirements-dev.txt" with your chosen file
-pip-sync requirements\win-amd64-py3.9-requirements-dev.txt
+# Replace "win-amd64-py3.11-requirements-dev.txt" with your chosen file
+pip-sync requirements\win-amd64-py3.11-requirements-dev.txt
 ```
 
 If the requirements file you used is updated, or if you want to switch to another requirements file (you need access to the Tensorflow library, for example), just run `pip-sync` again using the appropriate filename.

diff --git a/container-vars.mk b/container-vars.mk
@@ -24,7 +24,7 @@
 CONTAINER_MLFLOW_TRACKING_INCLUDE_FILES =\
     docker/configs/aws-config\
     docker/configs/build.pip.conf\
-    docker/requirements/linux-$(DETECTED_ARCH)-py3.9-mlflow-tracking-requirements.txt\
+    docker/requirements/linux-$(DETECTED_ARCH)-py3.11-mlflow-tracking-requirements.txt\
     docker/shellscripts/entrypoint-mlflow-tracking.m4\
     docker/shellscripts/fix-permissions.m4\
     docker/shellscripts/parse-uri.m4\
@@ -43,7 +43,7 @@ CONTAINER_NGINX_INCLUDE_FILES =\
 CONTAINER_PYTORCH_CPU_INCLUDE_FILES =\
     docker/configs/aws-config\
     docker/configs/build.pip.conf\
-    docker/requirements/linux-$(DETECTED_ARCH)-py3.9-pytorch-cpu-requirements.txt\
+    docker/requirements/linux-$(DETECTED_ARCH)-py3.11-pytorch-cpu-requirements.txt\
     docker/shellscripts/entrypoint-worker.m4\
     docker/shellscripts/fix-permissions.m4\
     docker/shellscripts/parse-uri.m4\
@@ -56,7 +56,7 @@ CONTAINER_PYTORCH_CPU_INCLUDE_FILES =\
 CONTAINER_PYTORCH_GPU_INCLUDE_FILES =\
     docker/configs/aws-config\
     docker/configs/build.pip.conf\
-    docker/requirements/linux-amd64-py3.9-pytorch-gpu-requirements.txt\
+    docker/requirements/linux-amd64-py3.11-pytorch-gpu-requirements.txt\
     docker/shellscripts/entrypoint-worker.m4\
     docker/shellscripts/fix-permissions.m4\
     docker/shellscripts/parse-uri.m4\
@@ -70,7 +70,7 @@ CONTAINER_RESTAPI_INCLUDE_FILES =\
     docker/configs/aws-config\
     docker/configs/build.pip.conf\
     docker/configs/gunicorn.restapi.conf.py\
-    docker/requirements/linux-$(DETECTED_ARCH)-py3.9-restapi-requirements.txt\
+    docker/requirements/linux-$(DETECTED_ARCH)-py3.11-restapi-requirements.txt\
     docker/shellscripts/entrypoint-restapi.m4\
     docker/shellscripts/fix-permissions.m4\
     docker/shellscripts/parse-uri.m4\
@@ -84,7 +84,7 @@ CONTAINER_RESTAPI_INCLUDE_FILES =\
 CONTAINER_TENSORFLOW2_CPU_INCLUDE_FILES =\
     docker/configs/aws-config\
     docker/configs/build.pip.conf\
-    docker/requirements/linux-$(DETECTED_ARCH)-py3.9-tensorflow2-cpu-requirements.txt\
+    docker/requirements/linux-$(DETECTED_ARCH)-py3.11-tensorflow2-cpu-requirements.txt\
     docker/shellscripts/entrypoint-worker.m4\
     docker/shellscripts/fix-permissions.m4\
     docker/shellscripts/parse-uri.m4\
@@ -97,7 +97,7 @@ CONTAINER_TENSORFLOW2_CPU_INCLUDE_FILES =\
 CONTAINER_TENSORFLOW2_GPU_INCLUDE_FILES =\
     docker/configs/aws-config\
     docker/configs/build.pip.conf\
-    docker/requirements/linux-amd64-py3.9-tensorflow2-gpu-requirements.txt\
+    docker/requirements/linux-amd64-py3.11-tensorflow2-gpu-requirements.txt\
     docker/shellscripts/entrypoint-worker.m4\
     docker/shellscripts/fix-permissions.m4\
     docker/shellscripts/parse-uri.m4\

diff --git a/cookiecutter-templates/cookiecutter-dioptra-deployment/README.md b/cookiecutter-templates/cookiecutter-dioptra-deployment/README.md
@@ -5,7 +5,7 @@ A Cookiecutter template that generates the scripts, configuration files, and Doc
 ## Prerequisites
 
 -   [Bash v5 or higher](https://tiswww.case.edu/php/chet/bash/bashtop.html)
--   [Python 3.9 or higher](https://www.python.org/)
+-   [Python 3.11 or higher](https://www.python.org/)
 -   [Cruft 2.15.0 or higher](https://cruft.github.io/cruft/)
 -   [Docker Engine 20.10.13 or higher](https://docs.docker.com/engine/install/)
 -   [Docker Compose](https://docs.docker.com/compose/install/)

diff --git a/...lates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/README.md b/...lates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/README.md
@@ -267,7 +267,7 @@ If you run `./init-deployment.sh --help`, you will print the script's help messa
                                    image
             --branch: The Dioptra GitHub branch to use when syncing the built-in task plugins
                       and the frontend files (default: 'main')
-            --python: Command for invoking the Python interpreter. Must be Python 3.9 or
+            --python: Command for invoking the Python interpreter. Must be Python 3.11 or
                       greater, and the jinja2 package must be installed.
                       (default: 'python')
             --worker-ssl-service: Image to use when bootstrapping the SSL named volumes for

diff --git a/...plates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/init-deployment.sh b/...plates/cookiecutter-dioptra-deployment/{{cookiecutter.__project_slug}}/init-deployment.sh
@@ -84,7 +84,7 @@ print_help() {
 		                               image
 		        --branch: The Dioptra GitHub branch to use when syncing the built-in task plugins
 		                  and the frontend files (default: '${DEFAULT_ARG_BRANCH}')
-		        --python: Command for invoking the Python interpreter. Must be Python 3.9 or
+		        --python: Command for invoking the Python interpreter. Must be Python 3.11 or
 		                  greater, and the jinja2 package must be installed.
 		                  (default: '${DEFAULT_ARG_PYTHON}')
 		        --worker-ssl-service: Image to use when bootstrapping the SSL named volumes for