Merge pull request #1986 from ElderMedic/prov_data_input_output

Update and Check loadListing for input dir provenance
common-workflow-language · Mar 27, 2024 · c7963b3 · c7963b3
2 parents 4d8ae62 + 96d1e20
commit c7963b3
Show file tree

Hide file tree

Showing 133 changed files with 1,563 additions and 2,464 deletions.
diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs
@@ -6,3 +6,6 @@
 
 # upgrade to black 23.1, longer lines (100)
 7fe8c0739b0515d00daabc7db87bc5fad926d345
+
+# upgrade to black 24
+80c9ec01b4602fc51ac1f53744f98df8baba3c31
diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml
@@ -28,26 +28,19 @@ jobs:
 
   tox:
     name: Tox
-    runs-on: ubuntu-20.04  # 22.04 doesn't support Python 3.6
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         py-ver-major: [3]
-        py-ver-minor: [6, 7, 8, 9, 10, 11]
+        py-ver-minor: [8, 9, 10, 11, 12]
         step: [lint, unit, bandit, mypy]
-        exclude:
-          - py-ver-major: 3
-            py-ver-minor: 6
-            step: mypy
-          - py-ver-major: 3
-            py-ver-minor: 6
-            step: lint
 
     env:
       py-semver: ${{ format('{0}.{1}', matrix.py-ver-major, matrix.py-ver-minor) }}
       TOXENV: ${{ format('py{0}{1}-{2}', matrix.py-ver-major, matrix.py-ver-minor, matrix.step) }}
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -62,9 +55,10 @@ jobs:
         run: sudo usermod -c 'CI Runner' "$(whoami)"
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ env.py-semver }}
+          allow-prereleases: true
           cache: pip
           cache-dependency-path: |
             requirements.txt
@@ -77,20 +71,21 @@ jobs:
 
       - name: MyPy cache
         if: ${{ matrix.step == 'mypy' }}
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: .mypy_cache/${{ env.py-semver }}
           key: mypy-${{ env.py-semver }}
 
       - name: Test with tox
-        run: tox
+        run: APPTAINER_TMPDIR=${RUNNER_TEMP} tox
 
       - name: Upload coverage to Codecov
         if: ${{ matrix.step == 'unit' }}
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           fail_ci_if_error: true
-          token: ${{ secrets.CODECOV_TOKEN }}
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
   tox-style:
     name: Linters
@@ -101,16 +96,16 @@ jobs:
         step: [lintreadme, shellcheck, pydocstyle]
 
     env:
-      py-semver: "3.11"
-      TOXENV: ${{ format('py311-{0}', matrix.step) }}
+      py-semver: "3.12"
+      TOXENV: ${{ format('py312-{0}', matrix.step) }}
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ env.py-semver }}
           cache: pip
@@ -131,9 +126,9 @@ jobs:
     name: No leftovers
     runs-on: ubuntu-22.04
     env:
-      py-semver: "3.11"
+      py-semver: "3.12"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -146,7 +141,7 @@ jobs:
         run: sudo usermod -c 'CI Runner' "$(whoami)"
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ env.py-semver }}
           cache: pip
@@ -162,7 +157,7 @@ jobs:
           chmod a-w .
 
       - name: run tests
-        run: make test
+        run: APPTAINER_TMPDIR=${RUNNER_TEMP} make test
 
 
   conformance_tests:
@@ -180,7 +175,7 @@ jobs:
             extras: "--fast-parser"
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Singularity
         if: ${{ matrix.container == 'singularity' }}
@@ -190,7 +185,7 @@ jobs:
 
       - name: Singularity cache
         if: ${{ matrix.container == 'singularity' }}
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: sifcache
           key: singularity
@@ -200,39 +195,46 @@ jobs:
         run: sudo rm -f /usr/bin/docker ; sudo apt-get install -y podman
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.11
+          python-version: 3.12
           cache: pip
 
       - name: "Test CWL ${{ matrix.cwl-version }} conformance"
         env:
           VERSION: ${{ matrix.cwl-version }}
           CONTAINER: ${{ matrix.container }}
-          GIT_TARGET: main
-          CWLTOOL_OPTIONS: ${{ matrix.extras }}
+          GIT_TARGET: ${{ matrix.cwl-version == 'v1.2' && '1.2.1_proposed' || 'main' }}
+          CWLTOOL_OPTIONS: ${{ matrix.cwl-version == 'v1.2' && '--relax-path-checks' || '' }} ${{ matrix.extras }}
         run: ./conformance-test.sh
+      - name: Archive test results
+        uses: actions/upload-artifact@v4
+        with:
+          name: cwl-${{ matrix.cwl-version }}-${{ matrix.container }}${{ matrix.extras }}-conformance-results
+          path: |
+            **/cwltool_conf*.xml
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           fail_ci_if_error: true
-          token: ${{ secrets.CODECOV_TOKEN }}
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
   release_test:
     name: cwltool release test
     runs-on: ubuntu-22.04
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up Singularity
         run: |
           wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-jammy_amd64.deb
           sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.11
+          python-version: 3.12
           cache: pip
           cache-dependency-path: |
             requirements.txt
@@ -254,23 +256,27 @@ jobs:
   build_test_container:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: record cwltool version
+        run: pip install -U setuptools wheel && pip install setuptools_scm[toml] && python setup.py --version
       - name: build & test cwltool_module container
         run: ./build-cwltool-docker.sh
 
   macos:
     name: Test on macos-latest
     runs-on: macos-latest
     env:
-      TOXENV: py311-unit
+      TOXENV: py312-unit
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.11
+          python-version: 3.12
           cache: pip
           cache-dependency-path: |
             requirements.txt
@@ -281,7 +287,7 @@ jobs:
           pip install "tox<4" "tox-gh-actions<3"
       # # docker for mac install is not currently stable
       # - name: 'SETUP MacOS: load Homebrew cache'
-      #   uses: actions/cache@v3
+      #   uses: actions/cache@v4
       #   if: runner.os == 'macOS'
       #   with:
       #      path: |
@@ -293,7 +299,8 @@ jobs:
       - name: Test with tox
         run: tox
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           fail_ci_if_error: true
-          token: ${{ secrets.CODECOV_TOKEN }}
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -23,13 +23,13 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: python
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3
diff --git a/.github/workflows/quay-publish.yml b/.github/workflows/quay-publish.yml
@@ -8,31 +8,39 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
       - name: Get image tags
         id: image_tags
         run: |
-          echo -n ::set-output "name=IMAGE_TAGS::${GITHUB_REF#refs/*/}"
+          echo -n "IMAGE_TAGS=${GITHUB_REF#refs/*/}" >> "$GITHUB_OUTPUT"
+      - name: record cwltool version
+        run: |
+          pip install "setuptools>=61"
+          pip install setuptools_scm[toml] wheel
+          python setup.py --version
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Cache Docker layers
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: /tmp/.buildx-cache
           key: ${{ runner.os }}-multi-buildx-${{ github.sha }}
           restore-keys: |
             ${{ runner.os }}-multi-buildx
       - name: Login to Quay.io
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ${{ secrets.REGISTRY_SERVER }}
           username: ${{ secrets.REGISTRY_USERNAME }}
           password: ${{ secrets.REGISTRY_PASSWORD }}
       - name: Build and publish cwltool_module image to Quay
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
+          context: .
           file: cwltool.Dockerfile
           tags: quay.io/commonwl/cwltool_module:${{ steps.image_tags.outputs.IMAGE_TAGS }},quay.io/commonwl/cwltool_module:latest
           target: module
@@ -41,8 +49,9 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
       - name: Build and publish cwltool image to Quay
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
+          context: .
           file: cwltool.Dockerfile
           tags: quay.io/commonwl/cwltool:${{ steps.image_tags.outputs.IMAGE_TAGS }},quay.io/commonwl/cwltool:latest
           platforms: linux/amd64,linux/arm64

diff --git a/.gitignore b/.gitignore
@@ -53,6 +53,8 @@ value
 
 .python-version
 
+cwltool/_version.py
+
 # Folder created when using make
 cwltool_deps
 docs/_build/

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -1,6 +1,6 @@
 Style guide:
 - PEP-8 (as implemented by the `black` code formatting tool)
-- Python 3.6+ compatible code
+- Python 3.8+ compatible code
 - PEP-484 type hints
 
 The development is done using `git`, we encourage you to get familiar with it.

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -2,7 +2,7 @@ include README.rst CODE_OF_CONDUCT.md CONTRIBUTING.md
 include MANIFEST.in
 include LICENSE.txt
 include *requirements.txt mypy.ini tox.ini
-include gittaggers.py Makefile cwltool.py
+include Makefile cwltool.py
 recursive-include mypy-stubs *.pyi *.py
 include tests/*
 include tests/cwl-conformance/cwltool-conftest.py