Migrate all packages to use PEP 517

.pre-commit-config.yaml

@@ -264,12 +264,12 @@ repos:
         pass_filenames: true
         require_serial: true
       - id: setup-order
-        name: Check order of dependencies in setup.cfg and setup.py
+        name: Check order of dependencies in pyproject.toml, setup.cfg, and setup.py
         language: python
-        files: ^setup\.cfg$|^setup\.py$
+        files: ^setup\.cfg$|^setup\.py$|^pyproject\.toml$
         pass_filenames: false
         entry: ./scripts/ci/pre_commit/pre_commit_check_order_setup.py
-        additional_dependencies: ['rich']
+        additional_dependencies: ['rich', 'toml']
       - id: setup-extra-packages
         name: Checks setup extra packages
         description: Checks if all the libraries in setup.py are listed in extra-packages-ref.rst file

Dockerfile

@@ -45,6 +45,7 @@ ARG AIRFLOW_GID="50000"
 ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
 
 ARG AIRFLOW_PIP_VERSION=21.1.1
+ARG AIRFLOW_PYPA_BUILD_VERSION=0.4.0
 
 # By default PIP has progress bar but you can disable it.
 ARG PIP_PROGRESS_BAR="on"
@@ -182,6 +183,7 @@ ENV INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT} \
     DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH} \
     PATH=${PATH}:/root/.local/bin \
     AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
+    AIRFLOW_PYPA_BUILD_VERSION=${AIRFLOW_PYPA_BUILD_VERSION} \
     PIP_PROGRESS_BAR=${PIP_PROGRESS_BAR} \
     # Install Airflow with "--user" flag, so that we can copy the whole .local folder to the final image
     # from the build image and always in non-editable mode
@@ -320,7 +322,8 @@ ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} \
     # Make sure noninteractive debian install is used and language variables set
     DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \
     LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 \
-    AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION}
+    AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
+    AIRFLOW_PYPA_BUILD_VERSION=${AIRFLOW_PYPA_BUILD_VERSION}
 
 # Install curl and gnupg2 - needed for many other installation steps
 RUN apt-get update \

Dockerfile.ci

@@ -219,6 +219,7 @@ ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true"
 ARG INSTALL_PROVIDERS_FROM_SOURCES="true"
 ARG INSTALL_FROM_PYPI="true"
 ARG AIRFLOW_PIP_VERSION=21.1.1
+ARG AIRFLOW_PYPA_BUILD_VERSION=0.4.0
 # Setup PIP
 # By default PIP install run without cache to make image smaller
 ARG PIP_NO_CACHE_DIR="true"
@@ -244,6 +245,7 @@ ENV AIRFLOW_REPO=${AIRFLOW_REPO}\
     INSTALL_PROVIDERS_FROM_SOURCES=${INSTALL_PROVIDERS_FROM_SOURCES} \
     INSTALL_FROM_PYPI=${INSTALL_FROM_PYPI} \
     AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
+    AIRFLOW_PYPA_BUILD_VERSION=${AIRFLOW_PYPA_BUILD_VERSION} \
 # In the CI image we always:
 # * install MySQL
 # * install airflow from current sources, not from PyPI package

MANIFEST.in

@@ -24,6 +24,7 @@ graft licenses
 graft airflow/www
 graft airflow/www/static
 graft airflow/www/templates
+graft scripts/pep517
 graft scripts/systemd
 graft scripts/upstart
 graft airflow/config_templates

dev/provider_packages/MANIFEST_TEMPLATE.in.jinja2

@@ -38,4 +38,6 @@ include NOTICE
 include LICENSE
 include CHANGELOG.txt
 include README.md
+include pyproject.toml
+graft pep517
 global-exclude __pycache__  *.pyc

dev/provider_packages/SETUP_TEMPLATE.py.jinja2

@@ -59,7 +59,6 @@ def do_setup():
         zip_safe=False,
         include_package_data=True,
         install_requires={{ INSTALL_REQUIREMENTS }},
-        setup_requires={{ SETUP_REQUIREMENTS }},
         extras_require={{ EXTRAS_REQUIREMENTS }},
         classifiers=[
             'Development Status :: 5 - Production/Stable',

dev/provider_packages/prepare_provider_packages.py

@@ -370,14 +370,6 @@ def get_install_requirements(provider_package_id: str) -> List[str]:
     return install_requires
 
 
-def get_setup_requirements() -> List[str]:
-    """
-    Returns setup requirements (common for all package for now).
-    :return: setup requirements
-    """
-    return ['setuptools', 'wheel']
-
-
 def get_package_extras(provider_package_id: str) -> Dict[str, List[str]]:
     """
     Finds extras for the package specified.
@@ -1538,7 +1530,6 @@ def get_provider_jinja_context(
         "INSTALL_REQUIREMENTS": get_install_requirements(
             provider_package_id=provider_details.provider_package_id
         ),
-        "SETUP_REQUIREMENTS": get_setup_requirements(),
         "EXTRAS_REQUIREMENTS": get_package_extras(provider_package_id=provider_details.provider_package_id),
         "CROSS_PROVIDERS_DEPENDENCIES_TABLE": cross_providers_dependencies_table,
         "CROSS_PROVIDERS_DEPENDENCIES_TABLE_RST": cross_providers_dependencies_table_rst,
@@ -1997,44 +1988,33 @@ def build_provider_packages(
 
     See `list-providers-packages` subcommand for the possible PACKAGE_ID values
     """
-
-    import tempfile
-
-    # we cannot use context managers because if the directory gets deleted (which bdist_wheel does),
-    # the context manager will throw an exception when trying to delete it again
-    tmp_build_dir = tempfile.TemporaryDirectory().name
-    tmp_dist_dir = tempfile.TemporaryDirectory().name
-    try:
-        provider_package_id = package_id
-        with with_group(f"Prepare provider package for '{provider_package_id}'"):
-            current_tag = get_current_tag(provider_package_id, version_suffix, git_update, verbose)
-            if tag_exists_for_version(provider_package_id, current_tag, verbose):
-                print(f"[yellow]The tag {current_tag} exists. Skipping the package.[/]")
-                return False
-            print(f"Changing directory to ${TARGET_PROVIDER_PACKAGES_PATH}")
-            os.chdir(TARGET_PROVIDER_PACKAGES_PATH)
-            cleanup_remnants(verbose)
-            provider_package = package_id
-            verify_setup_py_prepared(provider_package)
-
-            print(f"Building provider package: {provider_package} in format {package_format}")
-            command = ["python3", "setup.py", "build", "--build-temp", tmp_build_dir]
-            if version_suffix is not None:
-                command.extend(['egg_info', '--tag-build', version_suffix])
-            if package_format in ['sdist', 'both']:
-                command.append("sdist")
-            if package_format in ['wheel', 'both']:
-                command.extend(["bdist_wheel", "--bdist-dir", tmp_dist_dir])
-            print(f"Executing command: '{' '.join(command)}'")
-            try:
-                subprocess.check_call(command, stdout=subprocess.DEVNULL)
-            except subprocess.CalledProcessError as ex:
-                print(ex.output.decode())
-                raise Exception("The command returned an error %s", command)
-            print(f"[green]Prepared provider package {provider_package} in format {package_format}[/]")
-    finally:
-        shutil.rmtree(tmp_build_dir, ignore_errors=True)
-        shutil.rmtree(tmp_dist_dir, ignore_errors=True)
+    provider_package_id = package_id
+    with with_group(f"Prepare provider package for '{provider_package_id}'"):
+        current_tag = get_current_tag(provider_package_id, version_suffix, git_update, verbose)
+        if tag_exists_for_version(provider_package_id, current_tag, verbose):
+            print(f"[yellow]The tag {current_tag} exists. Skipping the package.[/]")
+            return False
+        print(f"Changing directory to ${TARGET_PROVIDER_PACKAGES_PATH}")
+        os.chdir(TARGET_PROVIDER_PACKAGES_PATH)
+        cleanup_remnants(verbose)
+        provider_package = package_id
+        verify_setup_py_prepared(provider_package)
+
+        print(f"Building provider package: {provider_package} in format {package_format}")
+        command = ["python", "-m", "build"]
+        if version_suffix is not None:
+            command.extend(["--config-setting", f"airflow-tag-build={version_suffix}"])
+        if package_format in ['sdist', 'both']:
+            command.append("--sdist")
+        if package_format in ['wheel', 'both']:
+            command.append("--wheel")
+        print(f"Executing command: '{' '.join(command)}'")
+        try:
+            subprocess.check_call(command, stdout=subprocess.DEVNULL)
+        except subprocess.CalledProcessError as ex:
+            print(ex.output.decode())
+            raise Exception("The command returned an error %s", command)
+        print(f"[green]Prepared provider package {provider_package} in format {package_format}[/]")
 
 
 def verify_provider_classes_for_single_provider(imported_classes: List[str], provider_package_id: str):

provider_packages/pep517/backend.py

@@ -0,0 +1,72 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import subprocess
+import sys
+
+from setuptools import build_meta
+
+
+def _run_setup(*args):
+    subprocess.check_call([sys.executable, "setup.py", *args])
+
+
+def _tag_build_if_needed(config_settings):
+    """Tag build before building a dist."""
+    if not config_settings:
+        return
+    tag_build = config_settings.get("airflow-tag-build")
+    if not tag_build:
+        return
+    return _run_setup("egg_info", "--tag-build", tag_build)
+
+
+def build_sdist(sdist_directory, config_settings=None):
+    _tag_build_if_needed(config_settings)
+    return build_meta.build_sdist(
+        sdist_directory=sdist_directory,
+        config_settings=config_settings,
+    )
+
+
+def build_wheel(wheel_directory, config_settings=None, metadata_directory=None):
+    _tag_build_if_needed(config_settings)
+    return build_meta.build_wheel(
+        wheel_directory=wheel_directory,
+        config_settings=config_settings,
+        metadata_directory=metadata_directory,
+    )
+
+
+def prepare_metadata_for_build_wheel(metadata_directory, config_settings=None):
+    _tag_build_if_needed(config_settings)
+    return build_meta.prepare_metadata_for_build_wheel(
+        metadata_directory=metadata_directory,
+        config_settings=config_settings,
+    )
+
+
+get_requires_for_build_sdist = build_meta.get_requires_for_build_sdist
+get_requires_for_build_wheel = build_meta.get_requires_for_build_wheel
+
+
+__all__ = [
+    "build_sdist",
+    "build_wheel",
+    "get_requires_for_build_sdist",
+    "get_requires_for_build_wheel",
+    "prepare_metadata_for_build_wheel",
+]

provider_packages/pyproject.toml

@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+[build-system]
+requires = ['setuptools', 'wheel']
+build-backend = 'backend'
+backend-path = ['pep517']
+
+[tool.black]
+line-length = 110
+target-version = ['py36', 'py37', 'py38']
+skip-string-normalization = true

pyproject.toml

@@ -14,6 +14,11 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+[build-system]
+requires = ['gitpython', 'setuptools', 'wheel']
+build-backend = 'backend'
+backend-path = ['scripts/pep517']
+
 [tool.black]
 line-length = 110
 target-version = ['py36', 'py37', 'py38']

scripts/ci/libraries/_initialization.sh

@@ -423,6 +423,9 @@ function initialization::initialize_image_build_variables() {
     AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION:="21.1"}
     export AIRFLOW_PIP_VERSION
 
+    AIRFLOW_PYPA_BUILD_VERSION=${AIRFLOW_PYPA_BUILD_VERSION:="0.4.0"}
+    export AIRFLOW_PYPA_BUILD_VERSION
+
     # We also pin version of wheel used to get consistent builds
     WHEEL_VERSION=${WHEEL_VERSION:="0.36.2"}
     export WHEEL_VERSION

scripts/ci/pre_commit/pre_commit_check_order_setup.py

@@ -17,14 +17,14 @@
 # specific language governing permissions and limitations
 # under the License.
 """
-Test for an order of dependencies in setup.py
+Test for an order of dependencies in setup.py, setup.cfg, and pyproject.toml
 """
 import os
 import re
 import sys
-from os.path import abspath, dirname
 from typing import List
 
+import toml
 from rich import print
 
 errors = []
@@ -107,24 +107,38 @@ def check_variable_order(var_name: str) -> None:
         _check_list_sorted(var, f"Order of dependencies in: {var_name}")
 
 
-def check_install_and_setup_requires() -> None:
+def check_install_requires() -> None:
     """
-    Test for an order of dependencies in function do_setup section
-    install_requires and setup_requires in setup.cfg
+    Test for an order of dependencies in section install_requires in setup.cfg
     """
 
     from setuptools.config import read_configuration
 
-    path = abspath(os.path.join(dirname(__file__), os.pardir, os.pardir, os.pardir, 'setup.cfg'))
+    path = os.path.join(SOURCE_DIR_PATH, 'setup.cfg')
     config = read_configuration(path)
 
     pattern_dependent_version = re.compile('[~|><=;].*')
 
-    for key in ('install_requires', 'setup_requires'):
-        print(f"[blue]Checking setup.cfg group {key}[/]")
-        deps = config['options'][key]
-        dists = [pattern_dependent_version.sub('', p) for p in deps]
-        _check_list_sorted(dists, f"Order of dependencies in do_setup section: {key}")
+    key = 'install_requires'
+    print(f"[blue]Checking setup.cfg group {key}[/]")
+    deps = config['options'][key]
+    dists = [pattern_dependent_version.sub('', p) for p in deps]
+    _check_list_sorted(dists, f"Order of dependencies in setup.cfg section: {key}")
+
+
+def check_build_system_requires() -> None:
+    """
+    Test for an order of dependencies in [build-system] requires in pyproject.toml
+    """
+    with open(os.path.join(SOURCE_DIR_PATH, "pyproject.toml")) as f:
+        pyproject = toml.load(f)
+    key = "[build-system] requires"
+
+    pattern_dependent_version = re.compile('[~|><=;].*')
+    print(f"[blue]Checking pyproject.toml key {key} [/]")
+    deps = pyproject["build-system"]["requires"]
+    dists = [pattern_dependent_version.sub('', p) for p in deps]
+    _check_list_sorted(dists, f"Order of dependencies in pyproject.toml section: {key}")
 
 
 if __name__ == '__main__':
@@ -139,7 +153,8 @@ def check_install_and_setup_requires() -> None:
     check_variable_order("ADDITIONAL_EXTRAS_REQUIREMENTS")
     check_variable_order("EXTRAS_DEPRECATED_ALIASES")
     check_variable_order("PREINSTALLED_PROVIDERS")
-    check_install_and_setup_requires()
+    check_install_requires()
+    check_build_system_requires()
 
     print()
     print()