From 4c338a8b8a941bd13926d1af99a6f23773dec7e0 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 23 May 2023 11:11:01 +0200 Subject: [PATCH] Bring back min-airflow-version for preinstalled providers (#31469) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the last wave of providers #31416 we bumped min-airlfow-version to 2.4 and added mechanism to verify min-airflow version is ok while importing, but it turned out that there are cases where installing just old version of airflow (with no constraints) brings the latest version of those providers and causes new installation of airflow to fail. This is far too common to ignore or require to use constraints, unfortunately  We do not have min-airlfow-version in the preinstalled providers for one reason only. For some tools that are NOT conforming to standards (such as bazel), having min-airflow-version for those providers causes circular dependency (even though technically dependencies in PyPI can - and often are - circular, because dependencies in Python are not DAG and can contain cycles. This was in response to this issue in 2021 #17795. However, this is really bazel issue, and on top of it - it's recognized as such and being fixed very recently in https://github.com/bazelbuild/rules_python/issues/1188 because there are other packages that have similar problems (pytorch and triton being popular couple). Also Bazel is not that popular in the Python world.  Therefore, rather than trying to workaround the problem of bazel, we encourage them to merge and release the fix https://github.com/bazelbuild/rules_python/pull/1166#issuecomment-1558690218 and call it out in our installation instructions, that bazel installation might lead to problems like that. If bazel does not fix it, this will only be a problem for Future installations of airflow in a few months most likely. It will not impact current bazel users installing old versions of Airflow (actually they might start having problems now if we do not fix it and yank the 5 providers released yesterday) GitOrigin-RevId: 547e352578fac92f072b269dc257d21cdc279d97 --- CONTRIBUTING.rst | 10 ++++++++++ IMAGES.rst | 5 +++++ LOCAL_VIRTUALENV.rst | 10 ++++++++++ README.md | 5 +++++ airflow/providers/common/sql/provider.yaml | 1 + airflow/providers/ftp/provider.yaml | 3 ++- airflow/providers/http/provider.yaml | 1 + airflow/providers/imap/provider.yaml | 3 ++- airflow/providers/sqlite/provider.yaml | 1 + .../PROVIDER_INDEX_TEMPLATE.rst.jinja2 | 14 ------------- .../prepare_provider_packages.py | 7 +++---- .../installing-from-pypi.rst | 20 +++++++++++++------ .../upgrading-from-1-10/upgrade-check.rst | 6 ++++++ .../installation/installing-from-pypi.rst | 19 ++++++++++++------ docs/apache-airflow/start.rst | 5 +++++ generated/provider_dependencies.json | 13 +++++++++--- 16 files changed, 88 insertions(+), 35 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index f88a12979a4..d8bb9d2b952 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -593,6 +593,11 @@ Airflow dependencies ``pip`` - especially when it comes to constraint vs. requirements management. Installing via ``Poetry`` or ``pip-tools`` is not currently supported. + There are known issues with ``bazel`` that might lead to circular dependencies when using it to install + Airflow. Please switch to ``pip`` if you encounter such problems. ``Bazel`` community works on fixing + the problem in `this PR `_ so it might be that + newer versions of ``bazel`` will handle it. + If you wish to install airflow using those tools you should use the constraint files and convert them to appropriate format and workflow that your tool requires. @@ -796,6 +801,11 @@ Pinned constraint files ``pip`` - especially when it comes to constraint vs. requirements management. Installing via ``Poetry`` or ``pip-tools`` is not currently supported. + There are known issues with ``bazel`` that might lead to circular dependencies when using it to install + Airflow. Please switch to ``pip`` if you encounter such problems. ``Bazel`` community works on fixing + the problem in `this PR `_ so it might be that + newer versions of ``bazel`` will handle it. + If you wish to install airflow using those tools you should use the constraint files and convert them to appropriate format and workflow that your tool requires. diff --git a/IMAGES.rst b/IMAGES.rst index d28db992784..9034077ce30 100644 --- a/IMAGES.rst +++ b/IMAGES.rst @@ -147,6 +147,11 @@ This will build the image using command similar to: ``pip`` - especially when it comes to constraint vs. requirements management. Installing via ``Poetry`` or ``pip-tools`` is not currently supported. + There are known issues with ``bazel`` that might lead to circular dependencies when using it to install + Airflow. Please switch to ``pip`` if you encounter such problems. ``Bazel`` community works on fixing + the problem in `this PR `_ so it might be that + newer versions of ``bazel`` will handle it. + If you wish to install airflow using those tools you should use the constraint files and convert them to appropriate format and workflow that your tool requires. diff --git a/LOCAL_VIRTUALENV.rst b/LOCAL_VIRTUALENV.rst index 00205215b19..8fc030e1f2a 100644 --- a/LOCAL_VIRTUALENV.rst +++ b/LOCAL_VIRTUALENV.rst @@ -70,6 +70,11 @@ Extra Packages ``pip`` - especially when it comes to constraint vs. requirements management. Installing via ``Poetry`` or ``pip-tools`` is not currently supported. + There are known issues with ``bazel`` that might lead to circular dependencies when using it to install + Airflow. Please switch to ``pip`` if you encounter such problems. ``Bazel`` community works on fixing + the problem in `this PR `_ so it might be that + n ewer versions of ``bazel`` will handle it. + If you wish to install airflow using those tools you should use the constraint files and convert them to appropriate format and workflow that your tool requires. @@ -136,6 +141,11 @@ To create and initialize the local virtualenv: ``pip`` - especially when it comes to constraint vs. requirements management. Installing via ``Poetry`` or ``pip-tools`` is not currently supported. + There are known issues with ``bazel`` that might lead to circular dependencies when using it to install + Airflow. Please switch to ``pip`` if you encounter such problems. ``Bazel`` community works on fixing + the problem in `this PR `_ so it might be that + newer versions of ``bazel`` will handle it. + If you wish to install airflow using those tools you should use the constraint files and convert them to appropriate format and workflow that your tool requires. diff --git a/README.md b/README.md index 95265f16f5e..cf24d6efe96 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,11 @@ While it is possible to install Airflow with tools like [Poetry](https://python- `pip` - especially when it comes to constraint vs. requirements management. Installing via `Poetry` or `pip-tools` is not currently supported. +There are known issues with ``bazel`` that might lead to circular dependencies when using it to install +Airflow. Please switch to ``pip`` if you encounter such problems. ``Bazel`` community works on fixing +the problem in `this PR `_ so it might be that +newer versions of ``bazel`` will handle it. + If you wish to install Airflow using those tools, you should use the constraint files and convert them to the appropriate format and workflow that your tool requires. diff --git a/airflow/providers/common/sql/provider.yaml b/airflow/providers/common/sql/provider.yaml index b5d7d1931c2..e43b6087b60 100644 --- a/airflow/providers/common/sql/provider.yaml +++ b/airflow/providers/common/sql/provider.yaml @@ -35,6 +35,7 @@ versions: - 1.0.0 dependencies: + - apache-airflow>=2.4.0 - sqlparse>=0.4.2 additional-extras: diff --git a/airflow/providers/ftp/provider.yaml b/airflow/providers/ftp/provider.yaml index 0fc7b6ff477..986b26923cf 100644 --- a/airflow/providers/ftp/provider.yaml +++ b/airflow/providers/ftp/provider.yaml @@ -38,7 +38,8 @@ versions: - 1.0.1 - 1.0.0 -dependencies: [] +dependencies: + - apache-airflow>=2.4.0 integrations: - integration-name: File Transfer Protocol (FTP) diff --git a/airflow/providers/http/provider.yaml b/airflow/providers/http/provider.yaml index c31638bebf5..feb010f1c3a 100644 --- a/airflow/providers/http/provider.yaml +++ b/airflow/providers/http/provider.yaml @@ -42,6 +42,7 @@ versions: - 1.0.0 dependencies: + - apache-airflow>=2.4.0 # The 2.26.0 release of requests got rid of the chardet LGPL mandatory dependency, allowing us to # release it as a requirement for airflow - requests>=2.26.0 diff --git a/airflow/providers/imap/provider.yaml b/airflow/providers/imap/provider.yaml index 7449a83a9bd..20b26922b0a 100644 --- a/airflow/providers/imap/provider.yaml +++ b/airflow/providers/imap/provider.yaml @@ -38,7 +38,8 @@ versions: - 1.0.1 - 1.0.0 -dependencies: [] +dependencies: + - apache-airflow>=2.4.0 integrations: - integration-name: Internet Message Access Protocol (IMAP) diff --git a/airflow/providers/sqlite/provider.yaml b/airflow/providers/sqlite/provider.yaml index 5c6cea93dfa..fa821f77b43 100644 --- a/airflow/providers/sqlite/provider.yaml +++ b/airflow/providers/sqlite/provider.yaml @@ -42,6 +42,7 @@ versions: - 1.0.0 dependencies: + - apache-airflow>=2.4.0 - apache-airflow-providers-common-sql>=1.3.1 integrations: diff --git a/dev/provider_packages/PROVIDER_INDEX_TEMPLATE.rst.jinja2 b/dev/provider_packages/PROVIDER_INDEX_TEMPLATE.rst.jinja2 index 7da1baccd97..51175274ad0 100644 --- a/dev/provider_packages/PROVIDER_INDEX_TEMPLATE.rst.jinja2 +++ b/dev/provider_packages/PROVIDER_INDEX_TEMPLATE.rst.jinja2 @@ -56,20 +56,6 @@ Requirements The minimum Apache Airflow version supported by this provider package is ``{{ MIN_AIRFLOW_VERSION }}``. -{%- if PREINSTALLED_PROVIDER %} - -This provider package is preinstalled by default when Apache Airflow is installed. You do not need to -install it separately. You can upgrade and downgrade it independently of Apache Airflow package though. - -.. note:: - - The minimum Apache Airflow version for this package is {{ MIN_AIRFLOW_VERSION }} and it will fail - import at runtime if the version of Airflow is lower even if there is no requirement specified in - the dependencies - this is because the provider is preinstalled and specifying minimum Apache - Airflow version would create a dependency cycle, which confuses dependency tools. - -{%- endif %} - {%- if PIP_REQUIREMENTS %} {{ PIP_REQUIREMENTS_TABLE_RST | safe}} diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py index f78f53a122d..bf7da691c43 100755 --- a/dev/provider_packages/prepare_provider_packages.py +++ b/dev/provider_packages/prepare_provider_packages.py @@ -1348,10 +1348,9 @@ def update_min_airflow_version(provider_package_id: str, version_suffix: str): context=jinja_context, target_path=provider_details.source_provider_package_path, ) - if not jinja_context["PREINSTALLED_PROVIDER"]: - replace_min_airflow_version_in_provider_yaml( - context=jinja_context, target_path=provider_details.source_provider_package_path - ) + replace_min_airflow_version_in_provider_yaml( + context=jinja_context, target_path=provider_details.source_provider_package_path + ) def update_setup_files( diff --git a/docs/apache-airflow-providers/installing-from-pypi.rst b/docs/apache-airflow-providers/installing-from-pypi.rst index 227398d6111..7d360591558 100644 --- a/docs/apache-airflow-providers/installing-from-pypi.rst +++ b/docs/apache-airflow-providers/installing-from-pypi.rst @@ -26,12 +26,20 @@ Installation tools Only ``pip`` installation is currently officially supported. -While there are some successes with using other tools like `poetry `_ or -`pip-tools `_, they do not share the same workflow as -``pip`` - especially when it comes to constraint vs. requirements management. -Installing via ``Poetry`` or ``pip-tools`` is not currently supported. If you wish to install airflow -using those tools you should use the constraints and convert them to appropriate -format and workflow that your tool requires. +.. note:: + + While there are some successes with using other tools like `poetry `_ or + `pip-tools `_, they do not share the same workflow as + ``pip`` - especially when it comes to constraint vs. requirements management. + Installing via ``Poetry`` or ``pip-tools`` is not currently supported. If you wish to install airflow + using those tools you should use the constraints and convert them to appropriate + format and workflow that your tool requires. + + There are known issues with ``bazel`` that might lead to circular dependencies when using it to install + Airflow. Please switch to ``pip`` if you encounter such problems. ``Bazel`` community works on fixing + the problem in `this PR `_ so it might be that + newer versions of ``bazel`` will handle it. + Typical command to install airflow from PyPI looks like below (you need to use the right Airflow version and Python version): diff --git a/docs/apache-airflow/howto/upgrading-from-1-10/upgrade-check.rst b/docs/apache-airflow/howto/upgrading-from-1-10/upgrade-check.rst index 8049d2ddd3f..3f33770fac3 100644 --- a/docs/apache-airflow/howto/upgrading-from-1-10/upgrade-check.rst +++ b/docs/apache-airflow/howto/upgrading-from-1-10/upgrade-check.rst @@ -43,6 +43,12 @@ time of writing: 1.10.15) and then to download this package and run the script a ``pip`` - especially when it comes to constraint vs. requirements management. Installing via ``Poetry`` or ``pip-tools`` is not currently supported. + There are known issues with ``bazel`` that might lead to circular dependencies when using it to install + Airflow. Please switch to ``pip`` if you encounter such problems. ``Bazel`` community works on fixing + the problem in `this PR `_ so it might be that + newer versions of ``bazel`` will handle it. + + If you wish to install airflow using those tools you should use the constraint files and convert them to appropriate format and workflow that your tool requires. diff --git a/docs/apache-airflow/installation/installing-from-pypi.rst b/docs/apache-airflow/installation/installing-from-pypi.rst index 97cf3ba41c6..0ae5991f10e 100644 --- a/docs/apache-airflow/installation/installing-from-pypi.rst +++ b/docs/apache-airflow/installation/installing-from-pypi.rst @@ -26,12 +26,19 @@ Installation tools Only ``pip`` installation is currently officially supported. -While there are some successes with using other tools like `poetry `_ or -`pip-tools `_, they do not share the same workflow as -``pip`` - especially when it comes to constraint vs. requirements management. -Installing via ``Poetry`` or ``pip-tools`` is not currently supported. If you wish to install airflow -using those tools you should use the constraints and convert them to appropriate -format and workflow that your tool requires. +.. note:: + + While there are some successes with using other tools like `poetry `_ or + `pip-tools `_, they do not share the same workflow as + ``pip`` - especially when it comes to constraint vs. requirements management. + Installing via ``Poetry`` or ``pip-tools`` is not currently supported. If you wish to install airflow + using those tools you should use the constraints and convert them to appropriate + format and workflow that your tool requires. + + There are known issues with ``bazel`` that might lead to circular dependencies when using it to install + Airflow. Please switch to ``pip`` if you encounter such problems. ``Bazel`` community works on fixing + the problem in `this PR `_ so it might be that + newer versions of ``bazel`` will handle it. Typical command to install airflow from PyPI looks like below: diff --git a/docs/apache-airflow/start.rst b/docs/apache-airflow/start.rst index 6f198dfded5..2ea8c6d4b9c 100644 --- a/docs/apache-airflow/start.rst +++ b/docs/apache-airflow/start.rst @@ -34,6 +34,11 @@ This quick start guide will help you bootstrap an Airflow standalone instance on ``pip`` - especially when it comes to constraint vs. requirements management. Installing via ``Poetry`` or ``pip-tools`` is not currently supported. + There are known issues with ``bazel`` that might lead to circular dependencies when using it to install + Airflow. Please switch to ``pip`` if you encounter such problems. ``Bazel`` community works on fixing + the problem in `this PR `_ so it might be that + newer versions of ``bazel`` will handle it. + If you wish to install Airflow using those tools you should use the constraint files and convert them to appropriate format and workflow that your tool requires. diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index ae286f97e33..b42a3f60aa6 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -255,6 +255,7 @@ }, "common.sql": { "deps": [ + "apache-airflow>=2.4.0", "sqlparse>=0.4.2" ], "cross-providers-deps": [], @@ -356,7 +357,9 @@ "excluded-python-versions": [] }, "ftp": { - "deps": [], + "deps": [ + "apache-airflow>=2.4.0" + ], "cross-providers-deps": [], "excluded-python-versions": [] }, @@ -467,6 +470,7 @@ "http": { "deps": [ "aiohttp", + "apache-airflow>=2.4.0", "asgiref", "requests>=2.26.0", "requests_toolbelt" @@ -475,7 +479,9 @@ "excluded-python-versions": [] }, "imap": { - "deps": [], + "deps": [ + "apache-airflow>=2.4.0" + ], "cross-providers-deps": [], "excluded-python-versions": [] }, @@ -795,7 +801,8 @@ }, "sqlite": { "deps": [ - "apache-airflow-providers-common-sql>=1.3.1" + "apache-airflow-providers-common-sql>=1.3.1", + "apache-airflow>=2.4.0" ], "cross-providers-deps": [ "common.sql"