Refactor package checks to be test and remove duplication

Package checks are already "tests" not only checks since we
introduced compatibility complete tests. This PR updates
names of files and also removes duplication of wheel/sdist
package build and verification and adding conditional steps.

.github/workflows/ci.yml

@@ -109,7 +109,8 @@ jobs:
       postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
       postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }}
       prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }}
-      providers-compatibility-checks: ${{ steps.selective-checks.outputs.providers-compatibility-checks }}
+      # yamllint disable rule:line-length
+      providers-compatibility-tests-matrix: ${{ steps.selective-checks.outputs.providers-compatibility-tests-matrix }}
       providers-test-types-list-as-string: >-
         ${{ steps.selective-checks.outputs.providers-test-types-list-as-string }}
       pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }}
@@ -323,8 +324,8 @@ jobs:
       docs-build: ${{ needs.build-info.outputs.docs-build }}
 
   providers:
-    name: "Provider checks"
-    uses: ./.github/workflows/check-providers.yml
+    name: "Provider packages build and compat tests"
+    uses: ./.github/workflows/test-provider-packages.yml
     needs: [build-info, wait-for-ci-images]
     permissions:
       contents: read
@@ -340,7 +341,8 @@ jobs:
       default-python-version: ${{ needs.build-info.outputs.default-python-version }}
       upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
       selected-providers-list-as-string: ${{ needs.build-info.outputs.selected-providers-list-as-string }}
-      providers-compatibility-checks: ${{ needs.build-info.outputs.providers-compatibility-checks }}
+      # yamllint disable rule:line-length
+      providers-compatibility-tests-matrix: ${{ needs.build-info.outputs.providers-compatibility-tests-matrix }}
       skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }}
       python-versions: ${{ needs.build-info.outputs.python-versions }}
       providers-test-types-list-as-string: ${{ needs.build-info.outputs.providers-test-types-list-as-string }}

.github/workflows/check-providers.yml → .github/workflows/test-provider-packages.yml

@@ -44,9 +44,9 @@ on:  # yamllint disable-line rule:truthy
         description: "List of affected providers as string"
         required: false
         type: string
-      providers-compatibility-checks:
+      providers-compatibility-tests-matrix:
         description: >
-          JSON-formatted array of providers compatibility checks in the form of array of dicts
+          JSON-formatted array of providers compatibility tests in the form of array of dicts
           (airflow-version, python-versions, remove-providers, run-tests)
         required: true
         type: string
@@ -63,10 +63,14 @@ on:  # yamllint disable-line rule:truthy
         required: true
         type: string
 jobs:
-  prepare-install-verify-provider-packages-wheel:
+  prepare-install-verify-provider-packages:
     timeout-minutes: 80
-    name: "Provider packages wheel build and verify"
+    name: "Providers ${{ matrix.package-format }} tests"
     runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
+    strategy:
+      fail-fast: false
+      matrix:
+        package-format: ["wheel", "sdist"]
     env:
       GITHUB_REPOSITORY: ${{ github.repository }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -94,120 +98,54 @@ jobs:
         run: >
           breeze release-management prepare-provider-documentation --include-not-ready-providers
           --non-interactive
-      - name: "Prepare provider packages: wheel"
+        if: matrix.package-format == 'wheel'
+      - name: "Prepare provider packages: ${{ matrix.package-format }}"
         run: >
           breeze release-management prepare-provider-packages --include-not-ready-providers
-          --version-suffix-for-pypi dev0 --package-format wheel
-      - name: "Prepare airflow package: wheel"
-        run: breeze release-management prepare-airflow-package --version-suffix-for-pypi dev0
-      - name: "Verify wheel packages with twine"
+          --version-suffix-for-pypi dev0 --package-format ${{ matrix.package-format }}
+      - name: "Prepare airflow package: ${{ matrix.package-format }}"
+        run: >
+          breeze release-management prepare-airflow-package --version-suffix-for-pypi dev0
+          --package-format ${{ matrix.package-format }}
+      - name: "Verify ${{ matrix.package-format }} packages with twine"
         run: |
           uv tool uninstall twine || true
-          uv tool install twine && twine check dist/*.whl
+          uv tool install twine && twine check dist/*
       - name: "Test providers issue generation automatically"
         run: >
           breeze release-management generate-issue-content-providers
           --only-available-in-dist --disable-progress
+        if: matrix.package-format == 'wheel'
       - name: Remove Python 3.9-incompatible provider packages
         run: |
           echo "Removing Python 3.9-incompatible provider: cloudant"
-          rm -vf dist/apache_airflow_providers_cloudant*
+          rm -vf dist/*cloudant*
       - name: "Generate source constraints from CI image"
         shell: bash
         run: >
           breeze release-management generate-constraints
           --airflow-constraints-mode constraints-source-providers --answer yes
-      - name: "Install and verify all provider packages and airflow via wheel files"
+      - name: "Install and verify provider packages and airflow via ${{ matrix.package-format }} files"
         run: >
           breeze release-management verify-provider-packages
           --use-packages-from-dist
-          --package-format wheel
-          --use-airflow-version wheel
+          --package-format ${{ matrix.package-format }}
+          --use-airflow-version ${{ matrix.package-format }}
           --airflow-constraints-reference default
           --providers-constraints-location
           /files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
+          --run-in-parallel
         env:
           AIRFLOW_SKIP_CONSTRAINTS: "${{ inputs.upgrade-to-newer-dependencies }}"
-      - name: "Prepare airflow package: wheel without suffix and skipping the tag check"
-        run: >
-          breeze release-management prepare-provider-packages --skip-tag-check --package-format wheel
-
-  prepare-install-provider-packages-sdist:
-    timeout-minutes: 80
-    name: "Provider packages sdist build and install"
-    runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
-    env:
-      GITHUB_REPOSITORY: ${{ github.repository }}
-      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      GITHUB_USERNAME: ${{ github.actor }}
-      IMAGE_TAG: "${{ inputs.image-tag }}"
-      INCLUDE_NOT_READY_PROVIDERS: "true"
-      PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
-      VERBOSE: "true"
-    steps:
-      - name: "Cleanup repo"
-        shell: bash
-        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
-      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-        uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-      - name: "Cleanup docker"
-        run: ./scripts/ci/cleanup_docker.sh
-      - name: >
-          Prepare breeze & CI image: ${{ inputs.default-python-version }}:${{ inputs.image-tag }}
-        uses: ./.github/actions/prepare_breeze_and_image
-      - name: "Cleanup dist files"
-        run: rm -fv ./dist/*
-      - name: "Prepare provider packages: sdist"
-        run: >
-          breeze release-management prepare-provider-packages --include-not-ready-providers
-          --version-suffix-for-pypi dev0 --package-format sdist
-          ${{ inputs.selected-providers-list-as-string }}
-      - name: "Prepare airflow package: sdist"
-        run: >
-          breeze release-management prepare-airflow-package
-          --version-suffix-for-pypi dev0 --package-format sdist
-      - name: "Verify sdist packages with twine"
-        run: |
-          uv tool uninstall twine || true
-          uv tool install twine && twine check dist/*.tar.gz
-      - name: "Generate source constraints from CI image"
-        shell: bash
-        run: >
-          breeze release-management generate-constraints
-          --airflow-constraints-mode constraints-source-providers --answer yes
-      - name: "Install all provider packages and airflow via sdist files"
-        run: >
-          breeze release-management install-provider-packages
-          --use-packages-from-dist
-          --package-format sdist
-          --use-airflow-version sdist
-          --airflow-constraints-reference default
-          --providers-constraints-location
-          /files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
-          --run-in-parallel
-        if: inputs.selected-providers-list-as-string == ''
-      - name: "Install affected provider packages and airflow via sdist files"
-        run: >
-          breeze release-management install-provider-packages
-          --use-packages-from-dist
-          --package-format sdist
-          --use-airflow-version sdist
-          --airflow-constraints-reference default
-          --providers-constraints-location
-          /files/constraints-${{env.PYTHON_MAJOR_MINOR_VERSION}}/constraints-source-providers-${{env.PYTHON_MAJOR_MINOR_VERSION}}.txt
-          --run-in-parallel
-        if: inputs.selected-providers-list-as-string != ''
 
-  providers-compatibility-checks:
+  providers-compatibility-tests-matrix:
     timeout-minutes: 80
-    name: Compat ${{ matrix.airflow-version }}:P${{ matrix.python-version }} provider check
+    name: Compat ${{ matrix.airflow-version }}:P${{ matrix.python-version }} providers test
     runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
     strategy:
       fail-fast: false
       matrix:
-        include: ${{fromJSON(inputs.providers-compatibility-checks)}}
+        include: ${{fromJSON(inputs.providers-compatibility-tests-matrix)}}
     env:
       GITHUB_REPOSITORY: ${{ github.repository }}
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

dev/breeze/doc/ci/04_selective_checks.md

@@ -225,7 +225,7 @@ Github Actions to pass the list of parameters to a command to execute
 | postgres-exclude                               | Which versions of Postgres to exclude for tests as JSON array                                          | []                                      |      |
 | postgres-versions                              | Which versions of Postgres to use for tests as JSON array                                              | \['12'\]                                |      |
 | prod-image-build                               | Whether PROD image build is needed                                                                     | true                                    |      |
-| providers-compatibility-checks                 | List of dicts: (python_version, airflow_version, removed_providers) for compatibility checks           | \[{}\]                                  |      |
+| providers-compatibility-tests-matrix           | Matrix of providers compatibility tests: (python_version, airflow_version, removed_providers)          | \[{}\]                                  |      |
 | providers-test-types-list-as-string            | Which test types should be run for unit tests for providers                                            | Providers Providers\[-google\]          | *    |
 | pyproject-toml-changed                         | When pyproject.toml changed in the PR.                                                                 | false                                   |      |
 | python-versions                                | List of python versions to use for that build                                                          | \['3.9'\]                               |      |

dev/breeze/src/airflow_breeze/global_constants.py

@@ -609,7 +609,7 @@ def get_airflow_extras():
 CHICKEN_EGG_PROVIDERS = " ".join(["standard amazon common.sql"])
 
 
-BASE_PROVIDERS_COMPATIBILITY_CHECKS: list[dict[str, str | list[str]]] = [
+PROVIDERS_COMPATIBILITY_TESTS_MATRIX: list[dict[str, str | list[str]]] = [
     {
         "python-version": "3.9",
         "airflow-version": "2.8.4",

dev/breeze/src/airflow_breeze/utils/selective_checks.py

@@ -31,7 +31,6 @@
 from airflow_breeze.global_constants import (
     ALL_PYTHON_MAJOR_MINOR_VERSIONS,
     APACHE_AIRFLOW_GITHUB_REPOSITORY,
-    BASE_PROVIDERS_COMPATIBILITY_CHECKS,
     CHICKEN_EGG_PROVIDERS,
     COMMITTERS,
     CURRENT_KUBERNETES_VERSIONS,
@@ -45,6 +44,7 @@
     DISABLE_TESTABLE_INTEGRATIONS_FROM_CI,
     HELM_VERSION,
     KIND_VERSION,
+    PROVIDERS_COMPATIBILITY_TESTS_MATRIX,
     RUNS_ON_PUBLIC_RUNNER,
     RUNS_ON_SELF_HOSTED_ASF_RUNNER,
     RUNS_ON_SELF_HOSTED_RUNNER,
@@ -1376,12 +1376,12 @@ def chicken_egg_providers(self) -> str:
         return CHICKEN_EGG_PROVIDERS
 
     @cached_property
-    def providers_compatibility_checks(self) -> str:
-        """Provider compatibility input checks for the current run. Filter out python versions not built"""
+    def providers_compatibility_tests_matrix(self) -> str:
+        """Provider compatibility input matrix for the current run. Filter out python versions not built"""
         return json.dumps(
             [
                 check
-                for check in BASE_PROVIDERS_COMPATIBILITY_CHECKS
+                for check in PROVIDERS_COMPATIBILITY_TESTS_MATRIX
                 if check["python-version"] in self.python_versions
             ]
         )

dev/breeze/tests/test_selective_checks.py

@@ -25,9 +25,9 @@
 from rich.console import Console
 
 from airflow_breeze.global_constants import (
-    BASE_PROVIDERS_COMPATIBILITY_CHECKS,
     COMMITTERS,
     DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
+    PROVIDERS_COMPATIBILITY_TESTS_MATRIX,
     GithubEvents,
 )
 from airflow_breeze.utils.packages import get_available_packages
@@ -2291,10 +2291,10 @@ def test_has_migrations(files: tuple[str, ...], has_migrations: bool):
         pytest.param(
             (),
             {
-                "providers-compatibility-checks": json.dumps(
+                "providers-compatibility-tests-matrix": json.dumps(
                     [
                         check
-                        for check in BASE_PROVIDERS_COMPATIBILITY_CHECKS
+                        for check in PROVIDERS_COMPATIBILITY_TESTS_MATRIX
                         if check["python-version"] == DEFAULT_PYTHON_MAJOR_MINOR_VERSION
                     ]
                 ),
@@ -2303,7 +2303,7 @@ def test_has_migrations(files: tuple[str, ...], has_migrations: bool):
         ),
         pytest.param(
             ("all versions",),
-            {"providers-compatibility-checks": json.dumps(BASE_PROVIDERS_COMPATIBILITY_CHECKS)},
+            {"providers-compatibility-tests-matrix": json.dumps(PROVIDERS_COMPATIBILITY_TESTS_MATRIX)},
             id="full tests",
         ),
     ],