Refactor package checks to be test and remove duplication (apache#44318)

Package checks are already "tests" not only checks since we introduced compatibility complete tests. This PR updates names of files and also removes duplication of wheel/sdist package build and verification and adding conditional steps.
romsharon98 · Nov 24, 2024 · 18d1b97 · 18d1b97
1 parent 33afa8f
commit 18d1b97
Show file tree

Hide file tree

Showing 6 changed files with 46 additions and 95 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -109,7 +109,8 @@ jobs:
       postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
       postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }}
       prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }}
-      providers-compatibility-checks: ${{ steps.selective-checks.outputs.providers-compatibility-checks }}
+      # yamllint disable rule:line-length
+      providers-compatibility-tests-matrix: ${{ steps.selective-checks.outputs.providers-compatibility-tests-matrix }}
       providers-test-types-list-as-string: >-
         ${{ steps.selective-checks.outputs.providers-test-types-list-as-string }}
       pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }}
@@ -323,8 +324,8 @@ jobs:
       docs-build: ${{ needs.build-info.outputs.docs-build }}
 
   providers:
-    name: "Provider checks"
-    uses: ./.github/workflows/check-providers.yml
+    name: "Provider packages tests"
+    uses: ./.github/workflows/test-provider-packages.yml
     needs: [build-info, wait-for-ci-images]
     permissions:
       contents: read
@@ -340,7 +341,8 @@ jobs:
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       selected-providers-list-as-string: ${{ needs.build-info.outputs.selected-providers-list-as-string }}
-      providers-compatibility-checks: ${{ needs.build-info.outputs.providers-compatibility-checks }}
+      # yamllint disable rule:line-length
+      providers-compatibility-tests-matrix: ${{ needs.build-info.outputs.providers-compatibility-tests-matrix }}
       skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }}
       python-versions: ${{ needs.build-info.outputs.python-versions }}
       providers-test-types-list-as-string: ${{ needs.build-info.outputs.providers-test-types-list-as-string }}

diff --git a/.github/workflows/check-providers.yml → .github/workflows/test-provider-packages.yml b/.github/workflows/check-providers.yml → .github/workflows/test-provider-packages.yml
@@ -44,9 +44,9 @@ on:  # yamllint disable-line rule:truthy
         description: "List of affected providers as string"
         required: false
         type: string
-      providers-compatibility-checks:
+      providers-compatibility-tests-matrix:
         description: >
-          JSON-formatted array of providers compatibility checks in the form of array of dicts
+          JSON-formatted array of providers compatibility tests in the form of array of dicts
           (airflow-version, python-versions, remove-providers, run-tests)
         required: true
         type: string
@@ -63,10 +63,14 @@ on:  # yamllint disable-line rule:truthy
         required: true
         type: string
 jobs:
-  prepare-install-verify-provider-packages-wheel:
+  prepare-install-verify-provider-packages:
     timeout-minutes: 80
-    name: "Provider packages wheel build and verify"
+    name: "Providers ${{ matrix.package-format }} tests"
     runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
+    strategy:
+      fail-fast: false
+      matrix:
+        package-format: ["wheel", "sdist"]
     env:
       GITHUB_REPOSITORY: ${{ github.repository }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -94,120 +98,65 @@ jobs:
         run: >
           breeze release-management prepare-provider-documentation --include-not-ready-providers
           --non-interactive
-      - name: "Prepare provider packages: wheel"
+        if: matrix.package-format == 'wheel'
+      - name: "Prepare provider packages: ${{ matrix.package-format }}"
         run: >
           breeze release-management prepare-provider-packages --include-not-ready-providers
-          --version-suffix-for-pypi dev0 --package-format wheel
-      - name: "Prepare airflow package: wheel"
-        run: breeze release-management prepare-airflow-package --version-suffix-for-pypi dev0
-      - name: "Verify wheel packages with twine"
+          --version-suffix-for-pypi dev0 --package-format ${{ matrix.package-format }}
+      - name: "Prepare airflow package: ${{ matrix.package-format }}"
+        run: >
+          breeze release-management prepare-airflow-package --version-suffix-for-pypi dev0
+          --package-format ${{ matrix.package-format }}
+      - name: "Verify ${{ matrix.package-format }} packages with twine"
         run: |
           uv tool uninstall twine || true
-          uv tool install twine && twine check dist/*.whl
+          uv tool install twine && twine check dist/*
       - name: "Test providers issue generation automatically"
         run: >
           breeze release-management generate-issue-content-providers
           --only-available-in-dist --disable-progress
+        if: matrix.package-format == 'wheel'
       - name: Remove Python 3.9-incompatible provider packages
         run: |
           echo "Removing Python 3.9-incompatible provider: cloudant"
-          rm -vf dist/apache_airflow_providers_cloudant*
+          rm -vf dist/*cloudant*
       - name: "Generate source constraints from CI image"
         shell: bash
         run: >
           breeze release-management generate-constraints
           --airflow-constraints-mode constraints-source-providers --answer yes
-      - name: "Install and verify all provider packages and airflow via wheel files"
+      - name: "Install and verify wheel provider packages"
         run: >
           breeze release-management verify-provider-packages
           --use-packages-from-dist
-          --package-format wheel
-          --use-airflow-version wheel
+          --package-format ${{ matrix.package-format }}
+          --use-airflow-version ${{ matrix.package-format }}
           --airflow-constraints-reference default
           --providers-constraints-location
           /files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
         env:
           AIRFLOW_SKIP_CONSTRAINTS: "${{ inputs.upgrade-to-newer-dependencies }}"
-      - name: "Prepare airflow package: wheel without suffix and skipping the tag check"
-        run: >
-          breeze release-management prepare-provider-packages --skip-tag-check --package-format wheel
-
-  prepare-install-provider-packages-sdist:
-    timeout-minutes: 80
-    name: "Provider packages sdist build and install"
-    runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
-    env:
-      GITHUB_REPOSITORY: ${{ github.repository }}
-      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      GITHUB_USERNAME: ${{ github.actor }}
-      IMAGE_TAG: "${{ inputs.image-tag }}"
-      INCLUDE_NOT_READY_PROVIDERS: "true"
-      PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
-      VERBOSE: "true"
-    steps:
-      - name: "Cleanup repo"
-        shell: bash
-        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
-      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-        uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-      - name: "Cleanup docker"
-        run: ./scripts/ci/cleanup_docker.sh
-      - name: >
-          Prepare breeze & CI image: ${{ inputs.default-python-version }}:${{ inputs.image-tag }}
-        uses: ./.github/actions/prepare_breeze_and_image
-      - name: "Cleanup dist files"
-        run: rm -fv ./dist/*
-      - name: "Prepare provider packages: sdist"
-        run: >
-          breeze release-management prepare-provider-packages --include-not-ready-providers
-          --version-suffix-for-pypi dev0 --package-format sdist
-          ${{ inputs.selected-providers-list-as-string }}
-      - name: "Prepare airflow package: sdist"
-        run: >
-          breeze release-management prepare-airflow-package
-          --version-suffix-for-pypi dev0 --package-format sdist
-      - name: "Verify sdist packages with twine"
-        run: |
-          uv tool uninstall twine || true
-          uv tool install twine && twine check dist/*.tar.gz
-      - name: "Generate source constraints from CI image"
-        shell: bash
-        run: >
-          breeze release-management generate-constraints
-          --airflow-constraints-mode constraints-source-providers --answer yes
-      - name: "Install all provider packages and airflow via sdist files"
-        run: >
-          breeze release-management install-provider-packages
-          --use-packages-from-dist
-          --package-format sdist
-          --use-airflow-version sdist
-          --airflow-constraints-reference default
-          --providers-constraints-location
-          /files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
-          --run-in-parallel
-        if: inputs.selected-providers-list-as-string == ''
-      - name: "Install affected provider packages and airflow via sdist files"
+        if: matrix.package-format == 'wheel'
+      - name: "Install all sdist provider packages and airflow"
         run: >
           breeze release-management install-provider-packages
           --use-packages-from-dist
-          --package-format sdist
-          --use-airflow-version sdist
+          --package-format ${{ matrix.package-format }}
+          --use-airflow-version ${{ matrix.package-format }}
           --airflow-constraints-reference default
           --providers-constraints-location
           /files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
           --run-in-parallel
-        if: inputs.selected-providers-list-as-string != ''
+        if: matrix.package-format == 'sdist'
 
-  providers-compatibility-checks:
+  providers-compatibility-tests-matrix:
     timeout-minutes: 80
-    name: Compat ${{ matrix.airflow-version }}:P${{ matrix.python-version }} provider check
+    name: Compat ${{ matrix.airflow-version }}:P${{ matrix.python-version }} providers test
     runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
     strategy:
       fail-fast: false
       matrix:
-        include: ${{fromJSON(inputs.providers-compatibility-checks)}}
+        include: ${{fromJSON(inputs.providers-compatibility-tests-matrix)}}
     env:
       GITHUB_REPOSITORY: ${{ github.repository }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md
@@ -225,7 +225,7 @@ Github Actions to pass the list of parameters to a command to execute
 | postgres-exclude                               | Which versions of Postgres to exclude for tests as JSON array                                          | []                                      |      |
 | postgres-versions                              | Which versions of Postgres to use for tests as JSON array                                              | \['12'\]                                |      |
 | prod-image-build                               | Whether PROD image build is needed                                                                     | true                                    |      |
-| providers-compatibility-checks                 | List of dicts: (python_version, airflow_version, removed_providers) for compatibility checks           | \[{}\]                                  |      |
+| providers-compatibility-tests-matrix           | Matrix of providers compatibility tests: (python_version, airflow_version, removed_providers)          | \[{}\]                                  |      |
 | providers-test-types-list-as-string            | Which test types should be run for unit tests for providers                                            | Providers Providers\[-google\]          | *    |
 | pyproject-toml-changed                         | When pyproject.toml changed in the PR.                                                                 | false                                   |      |
 | python-versions                                | List of python versions to use for that build                                                          | \['3.9'\]                               |      |

diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -609,7 +609,7 @@ def get_airflow_extras():
 CHICKEN_EGG_PROVIDERS = " ".join(["standard amazon common.sql"])
 
 
-BASE_PROVIDERS_COMPATIBILITY_CHECKS: list[dict[str, str | list[str]]] = [
+PROVIDERS_COMPATIBILITY_TESTS_MATRIX: list[dict[str, str | list[str]]] = [
     {
         "python-version": "3.9",
         "airflow-version": "2.8.4",

diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -31,7 +31,6 @@
 from airflow_breeze.global_constants import (
     ALL_PYTHON_MAJOR_MINOR_VERSIONS,
     APACHE_AIRFLOW_GITHUB_REPOSITORY,
-    BASE_PROVIDERS_COMPATIBILITY_CHECKS,
     CHICKEN_EGG_PROVIDERS,
     COMMITTERS,
     CURRENT_KUBERNETES_VERSIONS,
@@ -45,6 +44,7 @@
     DISABLE_TESTABLE_INTEGRATIONS_FROM_CI,
     HELM_VERSION,
     KIND_VERSION,
+    PROVIDERS_COMPATIBILITY_TESTS_MATRIX,
     RUNS_ON_PUBLIC_RUNNER,
     RUNS_ON_SELF_HOSTED_ASF_RUNNER,
     RUNS_ON_SELF_HOSTED_RUNNER,
@@ -1376,12 +1376,12 @@ def chicken_egg_providers(self) -> str:
         return CHICKEN_EGG_PROVIDERS
 
     @cached_property
-    def providers_compatibility_checks(self) -> str:
-        """Provider compatibility input checks for the current run. Filter out python versions not built"""
+    def providers_compatibility_tests_matrix(self) -> str:
+        """Provider compatibility input matrix for the current run. Filter out python versions not built"""
         return json.dumps(
             [
                 check
-                for check in BASE_PROVIDERS_COMPATIBILITY_CHECKS
+                for check in PROVIDERS_COMPATIBILITY_TESTS_MATRIX
                 if check["python-version"] in self.python_versions
             ]
         )

diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py
@@ -25,9 +25,9 @@
 from rich.console import Console
 
 from airflow_breeze.global_constants import (
-    BASE_PROVIDERS_COMPATIBILITY_CHECKS,
     COMMITTERS,
     DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
+    PROVIDERS_COMPATIBILITY_TESTS_MATRIX,
     GithubEvents,
 )
 from airflow_breeze.utils.packages import get_available_packages
@@ -2291,10 +2291,10 @@ def test_has_migrations(files: tuple[str, ...], has_migrations: bool):
         pytest.param(
             (),
             {
-                "providers-compatibility-checks": json.dumps(
+                "providers-compatibility-tests-matrix": json.dumps(
                     [
                         check
-                        for check in BASE_PROVIDERS_COMPATIBILITY_CHECKS
+                        for check in PROVIDERS_COMPATIBILITY_TESTS_MATRIX
                         if check["python-version"] == DEFAULT_PYTHON_MAJOR_MINOR_VERSION
                     ]
                 ),
@@ -2303,7 +2303,7 @@ def test_has_migrations(files: tuple[str, ...], has_migrations: bool):
         ),
         pytest.param(
             ("all versions",),
-            {"providers-compatibility-checks": json.dumps(BASE_PROVIDERS_COMPATIBILITY_CHECKS)},
+            {"providers-compatibility-tests-matrix": json.dumps(PROVIDERS_COMPATIBILITY_TESTS_MATRIX)},
             id="full tests",
         ),
     ],