Bring back hive provider for Python3.11 (#1317)

* Bring back hive provider for Python3.11 * Revert #1237 * Bump hive provider verion * Update config
astronomer · Aug 29, 2023 · 923cb8c · 923cb8c
1 parent fe12ff2
commit 923cb8c
Show file tree

Hide file tree

Showing 4 changed files with 5 additions and 107 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -45,27 +45,22 @@ workflows:
           matrix:
             parameters:
               python_version: ["3.11"]
-              airflow_version: ["2.6.3"]
+              airflow_version: ["2.6.3", "2.7.0"]
           <<: *main_and_release_branches
       - build-docs:
           <<: *all_branches_and_version_tag
-      - test-python3-11:
-          <<: *all_branches_and_version_tag
-          requires:
-            - markdown-link-check
       - test:
           name: test-python<< matrix.python_version >>
           matrix:
             parameters:
-              python_version: ["3.8", "3.9", "3.10"]
+              python_version: ["3.8", "3.9", "3.10", "3.11"]
           <<: *all_branches_and_version_tag
           requires:
             - markdown-link-check
       - build-and-verify:
           <<: *all_branches_and_version_tag
           requires:
             - test
-            - test-python3-11
             - build-docs
             - mypy
       - hold:
@@ -178,44 +173,6 @@ jobs:
           # yamllint disable-line rule:line-length
           key: docs-{{ .Branch }}-{{ checksum "setup.cfg" }}-{{ checksum ".readthedocs.yaml" }}-{{ .Environment.CACHE_VERSION }}
 
-  # This runs unit tests for Python3.11 only and excludes the Apache Hive test since
-  # Currently, the Airflow Hive provider is excluded from Python3.11 in Airflow
-  # Check the issue https://github.com/cloudera/python-sasl/issues/30 for detail
-  # PR to bring back Airflow Apache Hive provider https://github.com/apache/airflow/pull/32607
-  test-python3-11:
-    description: "Test Python-3.11"
-    executor:
-      name: docker-executor
-      python_version: "3.11"
-    parallelism: 4
-    steps:
-      - checkout
-      - restore_cache:
-          keys:
-            # yamllint disable-line rule:line-length
-            - deps-{{ .Branch }}-{{ checksum "setup.cfg" }}-{{ checksum "/home/circleci/.pyenv/version" }}-{{ .Environment.CACHE_VERSION }}
-      - run:
-          name: Install Dependencies
-          command: pip install -U -e .[test_python_3_11,tests]
-      - run:
-          name: Run tests
-          command: |
-            set -e
-            TEST_FILES=$(circleci tests \
-                glob "tests/**/test_*.py" | \
-                sed '/tests\/apache\/hive/d' | \
-                circleci tests split --split-by=timings \
-            )
-            pytest --junit-xml=test-report/report.xml $TEST_FILES
-      - store_test_results:
-          path: test-report
-      - save_cache:
-          paths:
-            - ~/.cache/pip
-            - ~/.pyenv/versions/
-          # yamllint disable-line rule:line-length
-          key: deps-{{ .Branch }}-{{ checksum "setup.cfg" }}-{{ checksum "/home/circleci/.pyenv/version" }}-{{ .Environment.CACHE_VERSION }}
-
   test:
     parameters:
       python_version:
@@ -312,7 +269,7 @@ jobs:
             sudo apt-get install libsasl2-dev
       - run:
           name: Install Dependencies
-          command: pip install -U -e .[test_python_3_11,tests] apache-airflow==<<parameters.airflow_version>>
+          command: pip install -U -e .[all,tests] apache-airflow==<<parameters.airflow_version>>
       - run:
           name: Generate Constraints
           command: pip freeze

diff --git a/.circleci/scripts/pre_commit_setup_cfg_python_3_11_all_extra.py b/.circleci/scripts/pre_commit_setup_cfg_python_3_11_all_extra.py
diff --git a/README.rst b/README.rst
@@ -113,9 +113,6 @@ Extras
 
 .. EXTRA_DOC_END
 
-.. note::
-    astronomer-providers[apache.hive] is not supported for Python3.11
-
 Example Usage
 -------------
 
@@ -188,7 +185,6 @@ Limitations
 
 - In Airflow sensors have a param ``mode`` which can be ``poke`` and ``reschedule``.
   In async sensors, this param has no usage since tasks gets deferred to Triggerer.
-- astronomer-providers[apache.hive] is not supported for Python3.11
 
 License
 -------

diff --git a/setup.cfg b/setup.cfg
@@ -48,7 +48,7 @@ amazon =
     apache-airflow-providers-amazon>=3.0.0
     aiobotocore>=2.1.1
 apache.hive =
-    apache-airflow-providers-apache-hive
+    apache-airflow-providers-apache-hive>=6.1.5
     impyla
 apache.livy =
     apache-airflow-providers-apache-livy
@@ -119,7 +119,7 @@ mypy =
 all =
     aiobotocore>=2.1.1
     apache-airflow-providers-amazon>=3.0.0
-    apache-airflow-providers-apache-hive
+    apache-airflow-providers-apache-hive>=6.1.5
     apache-airflow-providers-apache-livy
     apache-airflow-providers-cncf-kubernetes>=4
     apache-airflow-providers-databricks>=2.2.0
@@ -139,27 +139,6 @@ all =
     paramiko
     snowflake-sqlalchemy>=1.4.4  # Temporary solution for https://github.com/astronomer/astronomer-providers/issues/958, we should pin apache-airflow-providers-snowflake version after it pins this package to great than or equal to 1.4.4.
 
-test_python_3_11 =
-    aiobotocore>=2.1.1
-    apache-airflow-providers-amazon>=3.0.0
-    apache-airflow-providers-apache-livy
-    apache-airflow-providers-cncf-kubernetes>=4
-    apache-airflow-providers-databricks>=2.2.0
-    apache-airflow-providers-google>=8.1.0
-    apache-airflow-providers-http
-    apache-airflow-providers-snowflake
-    apache-airflow-providers-sftp
-    apache-airflow-providers-microsoft-azure
-    asyncssh>=2.12.0
-    databricks-sql-connector>=2.0.4;python_version>='3.10'
-    apache-airflow-providers-dbt-cloud>=2.1.0
-    gcloud-aio-bigquery
-    gcloud-aio-storage
-    kubernetes_asyncio
-    openlineage-airflow>=0.12.0
-    paramiko
-    snowflake-sqlalchemy>=1.4.4  # Temporary solution for https://github.com/astronomer/astronomer-providers/issues/958, we should pin apache-airflow-providers-snowflake version after it pins this package to great than or equal to 1.4.4.
-
 [options.packages.find]
 include =
     astronomer.*