From 343abea4f78db257db1c830669181359652426e5 Mon Sep 17 00:00:00 2001 From: Pankaj Koti Date: Wed, 24 Jan 2024 14:32:16 +0530 Subject: [PATCH] Bump min Airflow version to >=2.7 and min Python version to >=3.8 --- .github/workflows/ci-python-sdk.yaml | 87 +--------------------------- python-sdk/noxfile.py | 31 +++------- python-sdk/pyproject.toml | 6 +- 3 files changed, 11 insertions(+), 113 deletions(-) diff --git a/.github/workflows/ci-python-sdk.yaml b/.github/workflows/ci-python-sdk.yaml index 1a7e70af3..76ce141e6 100644 --- a/.github/workflows/ci-python-sdk.yaml +++ b/.github/workflows/ci-python-sdk.yaml @@ -538,96 +538,14 @@ jobs: AZURE_WASB_CONN_STRING: ${{ secrets.AZURE_WASB_CONN_STRING }} AZURE_WASB_ACCESS_KEY: ${{ secrets.AZURE_WASB_ACCESS_KEY }} - Run-example-dag-tests-Airflow-2-2-5: - if: >- - github.event_name == 'push' || - ( - github.event_name == 'pull_request' && - github.event.pull_request.head.repo.fork == false - ) || - ( - github.event_name == 'pull_request_target' && - contains(github.event.pull_request.labels.*.name, 'safe to test') - )|| - ( - github.event_name == 'release' - ) - runs-on: ubuntu-latest - services: - postgres: - # Docker Hub image - image: dimberman/pagila-test - env: - POSTGRES_PASSWORD: postgres - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - sftp: - image: ghcr.io/astronomer/astro-sdk/sftp_docker - ports: - - 2222:22 - ftp: - image: ghcr.io/astronomer/astro-sdk/ftp_docker - ports: - - 21:21 - - 30000-30009:30000-30009 - env: - FTP_USER_NAME: ${{ secrets.SFTP_USERNAME }} - FTP_USER_PASS: ${{ secrets.SFTP_PASSWORD }} - FTP_USER_HOME: /home/foo - PUBLICHOST: "localhost" - steps: - - uses: actions/checkout@v3 - if: github.event_name != 'pull_request_target' - - - name: Checkout pull/${{ github.event.number }} - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha }} - if: github.event_name == 'pull_request_target' - - - uses: actions/setup-python@v3 - with: - python-version: '3.8' - architecture: 'x64' - - uses: actions/cache@v3 - with: - path: | - ~/.cache/pip - .nox - key: ${{ runner.os }}-2.2.5-${{ hashFiles('python-sdk/pyproject.toml') }}-${{ hashFiles('python-sdk/src/astro/__init__.py') }} - - run: cat ../.github/ci-test-connections.yaml > test-connections.yaml - - run: python -c 'import os; print(os.getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON", "").strip())' > ${{ env.GOOGLE_APPLICATION_CREDENTIALS }} - - run: sqlite3 /tmp/sqlite_default.db "VACUUM;" - - run: pip3 install nox - - run: nox -s "test-3.8(airflow='2.2.5')" -- "tests_integration/test_example_dags.py" "tests_integration/integration_test_dag.py" -k "not redshift" - env: - GOOGLE_APPLICATION_CREDENTIALS_JSON: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_JSON }} - GOOGLE_APPLICATION_CREDENTIALS: /tmp/google_credentials.json - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - REDSHIFT_NATIVE_LOAD_IAM_ROLE_ARN: ${{ secrets.REDSHIFT_NATIVE_LOAD_IAM_ROLE_ARN }} - REDSHIFT_USERNAME: ${{ secrets.REDSHIFT_USERNAME }} - REDSHIFT_PASSWORD: ${{ secrets.REDSHIFT_PASSWORD }} - SNOWFLAKE_ACCOUNT_NAME: ${{ secrets.SNOWFLAKE_UNAME }} - SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} - AIRFLOW__ASTRO_SDK__DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} - AZURE_WASB_CONN_STRING: ${{ secrets.AZURE_WASB_CONN_STRING }} - AZURE_WASB_ACCESS_KEY: ${{ secrets.AZURE_WASB_ACCESS_KEY }} Generate-Constraints: if: (github.event_name == 'release' || github.event_name == 'push') strategy: fail-fast: false matrix: - python: [ '3.7', '3.8', '3.9', '3.10' ] - airflow: [ '2.2.5', '2.3', '2.4', '2.5', '2.6', '2.7', '2.8'] + python: [ '3.8', '3.9', '3.10' ] + airflow: [ '2.7', '2.8'] runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -724,7 +642,6 @@ jobs: name: Build and publish Python 🐍 distributions 📦 to PyPI needs: - Run-Unit-tests-Airflow-2-8 - - Run-example-dag-tests-Airflow-2-2-5 - Run-Integration-tests-Airflow-2-8 - Run-load-file-Integration-Airflow-2-8 - Run-example-dag-Integration-Airflow-2-8 diff --git a/python-sdk/noxfile.py b/python-sdk/noxfile.py index c93c3874a..d49473d41 100644 --- a/python-sdk/noxfile.py +++ b/python-sdk/noxfile.py @@ -19,34 +19,17 @@ def dev(session: nox.Session) -> None: session.install("-e", ".[all,tests]") -@nox.session(python=["3.7", "3.8", "3.9", "3.10", "3.11"]) -@nox.parametrize("airflow", ["2.2.5", "2.4", "2.5", "2.6", "2.7", "2.8"]) +@nox.session(python=["3.8", "3.9", "3.10", "3.11"]) +@nox.parametrize("airflow", ["2.7", "2.8"]) def test(session: nox.Session, airflow) -> None: """Run both unit and integration tests.""" env = { "AIRFLOW_HOME": f"~/airflow-{airflow}-python-{session.python}", + "AIRFLOW__CORE__ALLOWED_DESERIALIZATION_CLASSES": "airflow\\.* astro\\.*", } - if airflow == "2.2.5": - env[ - "AIRFLOW__CORE__XCOM_BACKEND" - ] = "astro.custom_backend.astro_custom_backend.AstroCustomXcomBackend" - env["AIRFLOW__ASTRO_SDK__STORE_DATA_LOCAL_DEV"] = "True" - - # If you need a pinned version of a provider to be present in a nox session then - # update the constraints file used below with that version of provider - # For example as part of MSSQL support we need apache-airflow-providers-microsoft-mssql>=3.2 and this - # has been updated in the below constraint file. - session.install(f"apache-airflow=={airflow}", "-c", "tests/modified_constraint_file.txt") - session.install("-e", ".[all,tests]", "-c", "tests/modified_constraint_file.txt") - session.install("apache-airflow-providers-common-sql==1.2.0") - # install smart-open 6.3.0 since it has FTP implementation - session.install("smart-open>=6.3.0") - else: - env["AIRFLOW__CORE__ALLOWED_DESERIALIZATION_CLASSES"] = "airflow\\.* astro\\.*" - - session.install(f"apache-airflow~={airflow}") - session.install("-e", ".[all,tests]") + session.install(f"apache-airflow~={airflow}") + session.install("-e", ".[all,tests]") # Log all the installed dependencies session.log("Installed Dependencies:") @@ -150,8 +133,8 @@ def build_docs(session: nox.Session) -> None: session.run("make", "html") -@nox.session(python=["3.7", "3.8", "3.9", "3.10", "3.11"]) -@nox.parametrize("airflow", ["2.2.5", "2.3", "2.4", "2.5", "2.6", "2.7", "2.8"]) +@nox.session(python=["3.8", "3.9", "3.10", "3.11"]) +@nox.parametrize("airflow", ["2.7", "2.8"]) def generate_constraints(session: nox.Session, airflow) -> None: """Generate constraints file""" session.install("wheel") diff --git a/python-sdk/pyproject.toml b/python-sdk/pyproject.toml index cd3890c15..b1d78627d 100644 --- a/python-sdk/pyproject.toml +++ b/python-sdk/pyproject.toml @@ -15,16 +15,15 @@ authors = [ readme = "README.md" license = { file = "LICENSE" } -requires-python = ">=3.7" +requires-python = ">=3.8" dependencies = [ - "apache-airflow>=2.0", + "apache-airflow>=2.7", "attrs>=20.3.0", "pandas", "pyarrow", "python-frontmatter", "smart-open", "SQLAlchemy>=1.3.18", - "cached_property>=1.5.0;python_version<='3.7'", "Flask-Session<0.6.0" # This release breaking our tests, let's pin it as a temporary workaround ] @@ -36,7 +35,6 @@ classifiers = [ "Framework :: Apache Airflow", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10",