From f181cd8bd5eefd8d6d6445b8b219d5d7c554e212 Mon Sep 17 00:00:00 2001 From: Pankaj Date: Tue, 11 Jul 2023 20:24:54 +0530 Subject: [PATCH] Run test for Python-3.11 Run unit tests for Python3.11 exclude hive extra from it because of issue https://github.com/cloudera/python-sasl/issues/30 --- .circleci/config.yml | 35 +++++++++++++++++++ .circleci/scripts/pre_commit_readme_extra.py | 2 +- ..._commit_setup_cfg_python_3_11_all_extra.py | 35 +++++++++++++++++++ .pre-commit-config.yaml | 6 ++++ .../providers/microsoft/azure/hooks/wasb.py | 6 ++-- setup.cfg | 26 ++++++++++++++ tests/core/triggers/test_external_task.py | 9 +++-- 7 files changed, 109 insertions(+), 10 deletions(-) create mode 100755 .circleci/scripts/pre_commit_setup_cfg_python_3_11_all_extra.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 557d7535f..6f29b593c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -42,6 +42,10 @@ workflows: <<: *main_and_release_branches - build-docs: <<: *all_branches_and_version_tag + - test-python3-11: + <<: *all_branches_and_version_tag + requires: + - markdown-link-check - test: name: test-python<< matrix.python_version >> matrix: @@ -164,6 +168,37 @@ jobs: - ~/.pyenv/versions/ key: docs-{{ .Branch }}-{{ checksum "setup.cfg" }}-{{ checksum ".readthedocs.yaml" }} + test-python3-11: + description: "Test Python-3.11" + executor: + name: docker-executor + python_version: "3.11" + parallelism: 4 + steps: + - checkout + - restore_cache: + keys: + - deps-{{ .Branch }}-{{ checksum "setup.cfg" }}-{{ checksum "/home/circleci/.pyenv/version" }} + - deps-main-{{ checksum "setup.cfg" }}-{{ checksum "/home/circleci/.pyenv/version" }} + - run: + name: Install Dependencies + command: pip install -U -e .[test_python_3_11,tests] + - run: + name: Run tests + command: | + set -e + TEST_FILES=$(circleci tests \ + glob "tests/**/test_*.py" | sed '/tests\/apache\/hive/d' \ + | circleci tests split --split-by=timings \ + ) + pytest --junit-xml=test-report/report.xml $TEST_FILES + - store_test_results: + path: test-report + - save_cache: + paths: + - ~/.cache/pip + - ~/.pyenv/versions/ + key: deps-{{ .Branch }}-{{ checksum "setup.cfg" }}-{{ checksum "/home/circleci/.pyenv/version" }} test: parameters: diff --git a/.circleci/scripts/pre_commit_readme_extra.py b/.circleci/scripts/pre_commit_readme_extra.py index 94618fed9..f62307970 100755 --- a/.circleci/scripts/pre_commit_readme_extra.py +++ b/.circleci/scripts/pre_commit_readme_extra.py @@ -10,7 +10,7 @@ config.read(repo_dir / "setup.cfg") all_extra = [] -extra_to_exclude = {"tests", "mypy", "docs"} +extra_to_exclude = {"tests", "mypy", "docs", "test_python_3_11"} all_extras = set(config["options.extras_require"].keys()) - extra_to_exclude readme_path = repo_dir / "README.rst" diff --git a/.circleci/scripts/pre_commit_setup_cfg_python_3_11_all_extra.py b/.circleci/scripts/pre_commit_setup_cfg_python_3_11_all_extra.py new file mode 100755 index 000000000..2b5432f17 --- /dev/null +++ b/.circleci/scripts/pre_commit_setup_cfg_python_3_11_all_extra.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +""" +Pre-commit hook to sync an all Python3.11 extra in setup.cfg. +It will contain all the dependencies apart from tests and mypy. +""" +import configparser +from pathlib import Path + +repo_dir = Path(__file__).parent.parent.parent + +config = configparser.ConfigParser(strict=False) +config.read(repo_dir / "setup.cfg") + +all_extra = [] +extra_to_exclude = {"tests", "mypy", "docs", "all", "test_python_3_11", "apache.hive"} +for k in config["options.extras_require"].keys(): + if k in extra_to_exclude: + continue + reqs = config["options.extras_require"][k].split() + all_extra.extend(reqs) + +expected_all_extra = set(all_extra) +found_all_extra = set(config["options.extras_require"].get("test_python_3_11", "").split()) + +if not found_all_extra: + raise SystemExit("Missing 'all' extra in setup.cfg") + +""" +Use XOR operator ^ to find the missing dependencies instead of set A - set B +set A - set B will only show difference of set A from set B, but we want see overall diff +""" +diff_extras = expected_all_extra ^ found_all_extra +if diff_extras: + diff_extras_str = "\n \t" + "\n \t".join(sorted(diff_extras)) + raise SystemExit(f"'all' extra in setup.cfg is missing some dependencies:\n {diff_extras_str}") diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 77683e277..10feed956 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -106,6 +106,12 @@ repos: files: ^setup.cfg$ pass_filenames: false entry: .circleci/scripts/pre_commit_setup_cfg_all_extra.py + - id: sync-all-python-3-11-extras-setup.cfg + name: Sync python-3-11 extra in setup.cfg + language: python + files: ^setup.cfg$ + pass_filenames: false + entry: .circleci/scripts/pre_commit_setup_cfg_python_3_11_all_extra.py - id: sync-version-setup.cfg-docs-conf name: Sync "version" of the package in setup.cfg and docs/conf.py language: python diff --git a/astronomer/providers/microsoft/azure/hooks/wasb.py b/astronomer/providers/microsoft/azure/hooks/wasb.py index 4cee7c147..722d28148 100644 --- a/astronomer/providers/microsoft/azure/hooks/wasb.py +++ b/astronomer/providers/microsoft/azure/hooks/wasb.py @@ -55,9 +55,7 @@ def get_conn(self) -> BlobServiceClient: app_id = conn.login app_secret = conn.password token_credential = ClientSecretCredential(tenant, app_id, app_secret) - return BlobServiceClient( - account_url=conn.host, credential=token_credential, **extra # type:ignore[arg-type] - ) + return BlobServiceClient(account_url=conn.host, credential=token_credential, **extra) sas_token = extra.pop("sas_token", extra.pop("extra__wasb__sas_token", None)) if sas_token: @@ -133,7 +131,7 @@ async def get_blobs_list_async( blob_list = [] blobs = container.walk_blobs(name_starts_with=prefix, include=include, delimiter=delimiter, **kwargs) async for blob in blobs: - blob_list.append(blob.name) + blob_list.append(blob) return blob_list async def check_for_prefix_async(self, container_name: str, prefix: str, **kwargs: Any) -> bool: diff --git a/setup.cfg b/setup.cfg index 5d9f8f19b..3a7d6314e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,6 +22,7 @@ classifiers = Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 project_urls = Source Code=https://github.com/astronomer/astronomer-providers/ Homepage=https://astronomer.io/ @@ -138,6 +139,31 @@ all = paramiko snowflake-sqlalchemy>=1.4.4 # Temporary solution for https://github.com/astronomer/astronomer-providers/issues/958, we should pin apache-airflow-providers-snowflake version after it pins this package to great than or equal to 1.4.4. +# Hive provider does not work for python-3-11 +# See GihHub issue https://github.com/cloudera/python-sasl/issues/30 +# So creating this target for testing +test_python_3_11 = + aiobotocore>=2.1.1 + apache-airflow-providers-amazon>=3.0.0 + apache-airflow-providers-apache-livy + apache-airflow-providers-cncf-kubernetes>=4 + apache-airflow-providers-databricks>=2.2.0 + apache-airflow-providers-google>=8.1.0 + apache-airflow-providers-http + apache-airflow-providers-snowflake + apache-airflow-providers-sftp + apache-airflow-providers-microsoft-azure + asyncssh>=2.12.0 + databricks-sql-connector>=2.0.4;python_version>='3.10' + apache-airflow-providers-dbt-cloud>=2.1.0 + gcloud-aio-bigquery + gcloud-aio-storage + kubernetes_asyncio + openlineage-airflow>=0.12.0 + paramiko + snowflake-sqlalchemy>=1.4.4 # Temporary solution for https://github.com/astronomer/astronomer-providers/issues/958, we should pin apache-airflow-providers-snowflake version after it pins this package to great than or equal to 1.4.4. + + [options.packages.find] include = astronomer.* diff --git a/tests/core/triggers/test_external_task.py b/tests/core/triggers/test_external_task.py index f94b80283..a1a5cdb9d 100644 --- a/tests/core/triggers/test_external_task.py +++ b/tests/core/triggers/test_external_task.py @@ -2,10 +2,9 @@ from unittest import mock from unittest.mock import AsyncMock -import asynctest import pytest -from airflow import AirflowException -from airflow.operators.dummy import DummyOperator +from airflow.exceptions import AirflowException +from airflow.operators.empty import EmptyOperator from airflow.triggers.base import TriggerEvent from airflow.utils.state import DagRunState, TaskInstanceState @@ -35,7 +34,7 @@ async def test_task_state_trigger(self, session, dag): session.add(dag_run) session.commit() - external_task = DummyOperator(task_id=self.TASK_ID, dag=dag) + external_task = EmptyOperator(task_id=self.TASK_ID, dag=dag) instance = get_task_instance(external_task) session.add(instance) session.commit() @@ -222,7 +221,7 @@ async def test_deployment_task_exception(self, mock_run): assert TriggerEvent({"state": "error", "message": "Test exception"}) == actual @pytest.mark.asyncio - @asynctest.patch("astronomer.providers.http.hooks.http.HttpHookAsync.run") + @mock.patch("astronomer.providers.http.hooks.http.HttpHookAsync.run") async def test_deployment_complete(self, mock_run): """Assert ExternalDeploymentTaskTrigger runs and complete the run in success state""" mock.AsyncMock(HttpHookAsync)