From da454983fe542c17c5d2f0b57d91b725e77be48c Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sat, 6 Jan 2024 17:34:31 +0100 Subject: [PATCH] Change detection mechanism for Breeze self-upgrade Breeze auto-detects if it should upgrade itself - based on finding Airflow directory it is in and calculating the hash of the pyproject.toml it uses. Finding the airflow sources to act on was using setup.cfg from Airflow and checking the package name inside, but since we are about to remove setup.cfg, and move all project configuration to pyproject.toml (see #36537), this mechanism will stop working. This PR changes it by just checking if `airflow` subdir is present, and contains `__init__.py` with "airflow" inside. That should be "good enough" and fast, and also it should be backwards compatible in case new Breeze is used in older airflow sources. --- .../src/airflow_breeze/utils/path_utils.py | 23 +++++++++++-------- .../tests/test_find_airflow_directory.py | 2 +- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py b/dev/breeze/src/airflow_breeze/utils/path_utils.py index 23f8f09ddd0d8..f6e75a9fc5684 100644 --- a/dev/breeze/src/airflow_breeze/utils/path_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py @@ -34,16 +34,21 @@ from airflow_breeze.utils.reinstall import reinstall_breeze, warn_dependencies_changed, warn_non_editable from airflow_breeze.utils.shared_options import get_verbose, set_forced_answer -AIRFLOW_CFG_FILE = "setup.cfg" +PYPROJECT_TOML_FILE = "pyproject.toml" def search_upwards_for_airflow_sources_root(start_from: Path) -> Path | None: root = Path(start_from.root) d = start_from while d != root: - attempt = d / AIRFLOW_CFG_FILE - if attempt.exists() and "name = apache-airflow\n" in attempt.read_text(): - return attempt.parent + airflow_candidate = d / "airflow" + airflow_candidate_init_py = airflow_candidate / "__init__.py" + if ( + airflow_candidate.is_dir() + and airflow_candidate_init_py.is_file() + and "airflow" in airflow_candidate_init_py.read_text().lower() + ): + return airflow_candidate.parent d = d.parent return None @@ -97,10 +102,10 @@ def get_package_setup_metadata_hash() -> str: return "NOT FOUND" -def get_sources_setup_metadata_hash(sources: Path) -> str: +def get_pyproject_toml_hash(sources: Path) -> str: try: the_hash = hashlib.new("blake2b") - the_hash.update((sources / "dev" / "breeze" / "pyproject.toml").read_bytes()) + the_hash.update((sources / "dev" / "breeze" / PYPROJECT_TOML_FILE).read_bytes()) return the_hash.hexdigest() except FileNotFoundError as e: return f"Missing file {e.filename}" @@ -108,7 +113,7 @@ def get_sources_setup_metadata_hash(sources: Path) -> str: def get_installation_sources_config_metadata_hash() -> str: """ - Retrieves hash of setup.py and setup.cfg files from the source of installation of Breeze. + Retrieves hash of pyproject.toml from the source of installation of Breeze. This is used in order to determine if we need to upgrade Breeze, because some setup files changed. Blake2b algorithm will not be flagged by security checkers @@ -118,14 +123,14 @@ def get_installation_sources_config_metadata_hash() -> str: installation_sources = get_installation_airflow_sources() if installation_sources is None: return "NOT FOUND" - return get_sources_setup_metadata_hash(installation_sources) + return get_pyproject_toml_hash(installation_sources) def get_used_sources_setup_metadata_hash() -> str: """ Retrieves hash of setup files from the currently used sources. """ - return get_sources_setup_metadata_hash(get_used_airflow_sources()) + return get_pyproject_toml_hash(get_used_airflow_sources()) def set_forced_answer_for_upgrade_check(): diff --git a/dev/breeze/tests/test_find_airflow_directory.py b/dev/breeze/tests/test_find_airflow_directory.py index 3182cb065eb75..9902e9d939a02 100644 --- a/dev/breeze/tests/test_find_airflow_directory.py +++ b/dev/breeze/tests/test_find_airflow_directory.py @@ -42,7 +42,7 @@ def test_find_airflow_root_upwards_from_file(capsys): assert output == "" -@mock.patch("airflow_breeze.utils.path_utils.AIRFLOW_CFG_FILE", "bad_name.cfg") +@mock.patch("airflow_breeze.utils.path_utils.PYPROJECT_TOML_FILE", "bad_name.toml") @mock.patch("airflow_breeze.utils.path_utils.Path.cwd") def test_find_airflow_root_from_installation_dir(mock_cwd, capsys): mock_cwd.return_value = ROOT_PATH