From 460b196d4af73c9d0c05257c75bb51855cd40841 Mon Sep 17 00:00:00 2001 From: Andrey Anshin Date: Fri, 1 Sep 2023 18:09:43 +0400 Subject: [PATCH 1/4] Add name and version of project into 'pyproject.toml' --- .pre-commit-config.yaml | 11 +++++++++++ STATIC_CODE_CHECKS.rst | 4 ++++ dev/breeze/src/airflow_breeze/pre_commit_ids.py | 2 ++ images/breeze/output-commands-hash.txt | 2 +- images/breeze/output_static-checks.svg | 4 ++-- pyproject.toml | 2 ++ scripts/ci/pre_commit/pre_commit_update_versions.py | 1 + 7 files changed, 23 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b1988fcd1061f..4b89f432c680f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -292,6 +292,11 @@ repos: - --ignore-words=docs/spelling_wordlist.txt - --skip=airflow/providers/*/*.rst,airflow/www/*.log,docs/*/commits.rst,docs/apache-airflow/tutorial/pipeline_example.csv,*.min.js,*.lock,INTHEWILD.md - --exclude-file=.codespellignorelines + - repo: https://github.com/abravalheri/validate-pyproject + rev: v0.14 + hooks: + - id: validate-pyproject + name: Validate pyproject.toml - repo: local # Note that this is the 2nd "local" repo group in the .pre-commit-config.yaml file. This is because # we try to minimise the number of passes that must happen in order to apply some of the changes @@ -424,6 +429,12 @@ repos: language: python files: ^docs pass_filenames: false + - id: update-project-version + name: Update project version to the latest version + entry: ./scripts/ci/pre_commit/pre_commit_update_versions.py + language: python + files: ^pyproject.toml + pass_filenames: false - id: check-pydevd-left-in-code language: pygrep name: Check for pydevd debug statements accidentally left diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst index a6fbb5b93bfcc..5dd6eee4dc485 100644 --- a/STATIC_CODE_CHECKS.rst +++ b/STATIC_CODE_CHECKS.rst @@ -349,6 +349,8 @@ require Breeze Docker image to be built locally. +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | update-migration-references | Update migration ref doc | * | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ +| update-project-version | Update project version to the latest version | | ++-----------------------------------------------------------+--------------------------------------------------------------+---------+ | update-providers-dependencies | Update cross-dependencies for providers packages | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | update-spelling-wordlist-to-be-sorted | Sort alphabetically and uniquify spelling_wordlist.txt | | @@ -359,6 +361,8 @@ require Breeze Docker image to be built locally. +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | update-version | Update version to the latest version in the documentation | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ +| validate-pyproject | Validate pyproject.toml | | ++-----------------------------------------------------------+--------------------------------------------------------------+---------+ | yamllint | Check YAML files with yamllint | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py b/dev/breeze/src/airflow_breeze/pre_commit_ids.py index a9f797e84ce0f..49f64b68ce0f2 100644 --- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py +++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py @@ -120,10 +120,12 @@ "update-installed-providers-to-be-sorted", "update-local-yml-file", "update-migration-references", + "update-project-version", "update-providers-dependencies", "update-spelling-wordlist-to-be-sorted", "update-supported-versions", "update-vendored-in-k8s-json-schema", "update-version", + "validate-pyproject", "yamllint", ] diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 92905b64e49ae..0d9686fa0462d 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -64,7 +64,7 @@ setup:version:be116d90a21c2afe01087f7609774e1e setup:fd391bab5277ad3aca65987a84144d51 shell:1462cde6f7e11a73cb42d4eec93c598e start-airflow:d4815dea2cfc0af7038697c1d9a13996 -static-checks:d643498c1537236f20c475c0036f1a30 +static-checks:603a7e3c6f1eee2489db045ca2ea826c testing:docker-compose-tests:a4dfe7dadbe3e95fdf2b8d2107f7e208 testing:helm-tests:0669be17b744ba057adbf38681bd8e68 testing:integration-tests:e745af9dd595adaa6f17ef02fbaae3b5 diff --git a/images/breeze/output_static-checks.svg b/images/breeze/output_static-checks.svg index aff980e81db24..9c821c0c86b46 100644 --- a/images/breeze/output_static-checks.svg +++ b/images/breeze/output_static-checks.svg @@ -329,10 +329,10 @@ │update-breeze-readme-config-hash | update-common-sql-api-stubs |                 │ │update-er-diagram | update-extras | update-in-the-wild-to-be-sorted |            │ │update-inlined-dockerfile-scripts | update-installed-providers-to-be-sorted |    │ -│update-local-yml-file | update-migration-references |                            │ +│update-local-yml-file | update-migration-references | update-project-version |   │ │update-providers-dependencies | update-spelling-wordlist-to-be-sorted |          │ │update-supported-versions | update-vendored-in-k8s-json-schema | update-version |│ -│yamllint)                                                                        │ +│validate-pyproject | yamllint)                                                   │ │--show-diff-on-failure-sShow diff for files modified by the checks.│ │--initialize-environmentInitialize environment before running checks.│ │--max-initialization-attemptsMaximum number of attempts to initialize environment before giving up.│ diff --git a/pyproject.toml b/pyproject.toml index 1732edc78fb1e..c27a9a1d61e32 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,8 @@ requires = ['setuptools==67.2.0'] build-backend = "setuptools.build_meta" [project] +name = "apache-airflow" +version = "2.8.0.dev0" # sentinel comment for project version requires-python = ">=3.8" [tool.ruff] diff --git a/scripts/ci/pre_commit/pre_commit_update_versions.py b/scripts/ci/pre_commit/pre_commit_update_versions.py index 19cba2c6488a0..95918a1fc2d66 100755 --- a/scripts/ci/pre_commit/pre_commit_update_versions.py +++ b/scripts/ci/pre_commit/pre_commit_update_versions.py @@ -46,6 +46,7 @@ def update_version(pattern: re.Pattern, v: str, file_path: Path): r"(apache\/airflow:)[^-]*(\-)": "docs/docker-stack/entrypoint.rst", r"(`apache/airflow:(?:slim-)?)[0-9].*?((?:-pythonX.Y)?`)": "docs/docker-stack/README.md", r"(\(Assuming Airflow version `).*(`\))": "docs/docker-stack/README.md", + r"""(^version\s?=\s?[\"']).*([\"']\s?#\s?sentinel.*$)""": "pyproject.toml", } From 724179edd2f83fe988df5fcb5b15c43dc8a23067 Mon Sep 17 00:00:00 2001 From: Andrey Anshin Date: Sat, 2 Sep 2023 18:29:18 +0400 Subject: [PATCH 2/4] Remove [project] section from `pyproject.toml` --- .pre-commit-config.yaml | 6 ------ STATIC_CODE_CHECKS.rst | 2 -- dev/breeze/src/airflow_breeze/pre_commit_ids.py | 1 - images/breeze/output-commands-hash.txt | 2 +- images/breeze/output_static-checks.svg | 2 +- pyproject.toml | 5 ----- 6 files changed, 2 insertions(+), 16 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4b89f432c680f..ba5d0a4456077 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -429,12 +429,6 @@ repos: language: python files: ^docs pass_filenames: false - - id: update-project-version - name: Update project version to the latest version - entry: ./scripts/ci/pre_commit/pre_commit_update_versions.py - language: python - files: ^pyproject.toml - pass_filenames: false - id: check-pydevd-left-in-code language: pygrep name: Check for pydevd debug statements accidentally left diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst index 5dd6eee4dc485..be446735cbd13 100644 --- a/STATIC_CODE_CHECKS.rst +++ b/STATIC_CODE_CHECKS.rst @@ -349,8 +349,6 @@ require Breeze Docker image to be built locally. +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | update-migration-references | Update migration ref doc | * | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ -| update-project-version | Update project version to the latest version | | -+-----------------------------------------------------------+--------------------------------------------------------------+---------+ | update-providers-dependencies | Update cross-dependencies for providers packages | | +-----------------------------------------------------------+--------------------------------------------------------------+---------+ | update-spelling-wordlist-to-be-sorted | Sort alphabetically and uniquify spelling_wordlist.txt | | diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py b/dev/breeze/src/airflow_breeze/pre_commit_ids.py index 49f64b68ce0f2..a083ee15ea5be 100644 --- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py +++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py @@ -120,7 +120,6 @@ "update-installed-providers-to-be-sorted", "update-local-yml-file", "update-migration-references", - "update-project-version", "update-providers-dependencies", "update-spelling-wordlist-to-be-sorted", "update-supported-versions", diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 0d9686fa0462d..666a12b13d92a 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -64,7 +64,7 @@ setup:version:be116d90a21c2afe01087f7609774e1e setup:fd391bab5277ad3aca65987a84144d51 shell:1462cde6f7e11a73cb42d4eec93c598e start-airflow:d4815dea2cfc0af7038697c1d9a13996 -static-checks:603a7e3c6f1eee2489db045ca2ea826c +static-checks:19926b8fcea5784b28d4a0d99865363c testing:docker-compose-tests:a4dfe7dadbe3e95fdf2b8d2107f7e208 testing:helm-tests:0669be17b744ba057adbf38681bd8e68 testing:integration-tests:e745af9dd595adaa6f17ef02fbaae3b5 diff --git a/images/breeze/output_static-checks.svg b/images/breeze/output_static-checks.svg index 9c821c0c86b46..f694d823b7722 100644 --- a/images/breeze/output_static-checks.svg +++ b/images/breeze/output_static-checks.svg @@ -329,7 +329,7 @@ │update-breeze-readme-config-hash | update-common-sql-api-stubs |                 │ │update-er-diagram | update-extras | update-in-the-wild-to-be-sorted |            │ │update-inlined-dockerfile-scripts | update-installed-providers-to-be-sorted |    │ -│update-local-yml-file | update-migration-references | update-project-version |   │ +│update-local-yml-file | update-migration-references |                            │ │update-providers-dependencies | update-spelling-wordlist-to-be-sorted |          │ │update-supported-versions | update-vendored-in-k8s-json-schema | update-version |│ │validate-pyproject | yamllint)                                                   │ diff --git a/pyproject.toml b/pyproject.toml index c27a9a1d61e32..604f377a76faa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,11 +28,6 @@ target-version = ['py37', 'py38', 'py39', 'py310'] requires = ['setuptools==67.2.0'] build-backend = "setuptools.build_meta" -[project] -name = "apache-airflow" -version = "2.8.0.dev0" # sentinel comment for project version -requires-python = ">=3.8" - [tool.ruff] typing-modules = ["airflow.typing_compat"] line-length = 110 From d4ec44d0dad0b16965c31b6f52dce953717d276a Mon Sep 17 00:00:00 2001 From: Andrey Anshin Date: Sat, 2 Sep 2023 18:33:55 +0400 Subject: [PATCH 3/4] Remove regex for update pyproject [project] version --- scripts/ci/pre_commit/pre_commit_update_versions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/ci/pre_commit/pre_commit_update_versions.py b/scripts/ci/pre_commit/pre_commit_update_versions.py index 95918a1fc2d66..19cba2c6488a0 100755 --- a/scripts/ci/pre_commit/pre_commit_update_versions.py +++ b/scripts/ci/pre_commit/pre_commit_update_versions.py @@ -46,7 +46,6 @@ def update_version(pattern: re.Pattern, v: str, file_path: Path): r"(apache\/airflow:)[^-]*(\-)": "docs/docker-stack/entrypoint.rst", r"(`apache/airflow:(?:slim-)?)[0-9].*?((?:-pythonX.Y)?`)": "docs/docker-stack/README.md", r"(\(Assuming Airflow version `).*(`\))": "docs/docker-stack/README.md", - r"""(^version\s?=\s?[\"']).*([\"']\s?#\s?sentinel.*$)""": "pyproject.toml", } From c91497e349e61bd24fdc1bfefda7d7ef516b6354 Mon Sep 17 00:00:00 2001 From: Andrey Anshin Date: Sat, 2 Sep 2023 21:00:33 +0400 Subject: [PATCH 4/4] Fight agains static checks --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 604f377a76faa..38d80503b8f56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ # under the License. [tool.black] line-length = 110 -target-version = ['py37', 'py38', 'py39', 'py310'] +target-version = ['py38', 'py39', 'py310', 'py311'] # Editable installs are currently broken using setuptools 64.0.0 and above. The problem is tracked in # https://github.com/pypa/setuptools/issues/3548. We're also discussing how we could potentially fix @@ -29,6 +29,7 @@ requires = ['setuptools==67.2.0'] build-backend = "setuptools.build_meta" [tool.ruff] +target-version = "py38" typing-modules = ["airflow.typing_compat"] line-length = 110 extend-exclude = [