From ca222e244bac22802b53955eb4d86e5c78f21970 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Thu, 31 Mar 2022 22:54:23 +0200 Subject: [PATCH] Fix file permissions in all build commands (#22656) Moves group permission cleanup to build command in new Breeze2. Seems that we should fix group permissions in all build commands. That will shave extra few minutes on most builds where they modify sources. Also the output from the build will be a bit cleaner by removing come of the console outputs. GitOrigin-RevId: 89a4c4d94d21e4fd9e50e792fc5c0025695538de --- dev/breeze/src/airflow_breeze/ci/build_image.py | 10 ++++------ dev/breeze/src/airflow_breeze/prod/build_prod_image.py | 6 ++---- dev/breeze/src/airflow_breeze/shell/enter_shell.py | 2 -- dev/breeze/src/airflow_breeze/utils/run_utils.py | 1 + 4 files changed, 7 insertions(+), 12 deletions(-) diff --git a/dev/breeze/src/airflow_breeze/ci/build_image.py b/dev/breeze/src/airflow_breeze/ci/build_image.py index ad0975901b..af955e1dea 100644 --- a/dev/breeze/src/airflow_breeze/ci/build_image.py +++ b/dev/breeze/src/airflow_breeze/ci/build_image.py @@ -19,9 +19,8 @@ from airflow_breeze.cache import check_cache_and_write_if_not_cached, touch_cache_file, write_to_cache_file from airflow_breeze.ci.build_params import BuildParams -from airflow_breeze.console import console from airflow_breeze.utils.path_utils import AIRFLOW_SOURCE, BUILD_CACHE_DIR -from airflow_breeze.utils.run_utils import filter_out_none, run_command +from airflow_breeze.utils.run_utils import filter_out_none, fix_group_permissions, run_command PARAMS_CI_IMAGE = [ "python_base_image", @@ -83,6 +82,7 @@ def construct_docker_command(ci_image: BuildParams) -> List[str]: def build_image(verbose, **kwargs): + fix_group_permissions() parameters_passed = filter_out_none(**kwargs) ci_image_params = get_image_build_params(parameters_passed) ci_image_cache_dir = Path(BUILD_CACHE_DIR, ci_image_params.airflow_branch) @@ -91,17 +91,15 @@ def build_image(verbose, **kwargs): f"built_{ci_image_params.python_version}", root_dir=ci_image_cache_dir, ) - output = run_command( + run_command( ["docker", "rmi", "--no-prune", "--force", ci_image_params.airflow_ci_image_name], verbose=verbose, cwd=AIRFLOW_SOURCE, text=True, suppress_raise_exception=True, ) - console.print(f"[blue]{output}") cmd = construct_docker_command(ci_image_params) - output = run_command(cmd, verbose=verbose, cwd=AIRFLOW_SOURCE, text=True) - console.print(f"[blue]{output}") + run_command(cmd, verbose=verbose, cwd=AIRFLOW_SOURCE, text=True) def get_image_build_params(parameters_passed: Dict[str, str]): diff --git a/dev/breeze/src/airflow_breeze/prod/build_prod_image.py b/dev/breeze/src/airflow_breeze/prod/build_prod_image.py index b74303bff2..1573b547c4 100644 --- a/dev/breeze/src/airflow_breeze/prod/build_prod_image.py +++ b/dev/breeze/src/airflow_breeze/prod/build_prod_image.py @@ -171,16 +171,14 @@ def build_production_image(verbose, **kwargs): login_to_docker_registry(prod_params) cmd = construct_docker_command(prod_params) - output = run_command( + run_command( ["docker", "rmi", "--no-prune", "--force", prod_params.airflow_prod_image_name], verbose=verbose, cwd=AIRFLOW_SOURCE, text=True, suppress_raise_exception=True, ) - console.print(f"[blue]{output}") - output = run_command(cmd, verbose=verbose, cwd=AIRFLOW_SOURCE, text=True) - console.print(f"[blue]{output}") + run_command(cmd, verbose=verbose, cwd=AIRFLOW_SOURCE, text=True) if prod_params.prepare_buildx_cache: run_command(['docker', 'push', prod_params.airflow_prod_image_name], verbose=True, text=True) diff --git a/dev/breeze/src/airflow_breeze/shell/enter_shell.py b/dev/breeze/src/airflow_breeze/shell/enter_shell.py index a4d8a61f14..783ca6348d 100644 --- a/dev/breeze/src/airflow_breeze/shell/enter_shell.py +++ b/dev/breeze/src/airflow_breeze/shell/enter_shell.py @@ -50,7 +50,6 @@ from airflow_breeze.utils.path_utils import BUILD_CACHE_DIR from airflow_breeze.utils.run_utils import ( filter_out_none, - fix_group_permissions, get_latest_sha, instruct_build_image, instruct_for_setup, @@ -181,7 +180,6 @@ def build_image_if_needed_steps(verbose: bool, shell_params: ShellBuilder): def build_image_checks(verbose: bool, shell_params: ShellBuilder): - fix_group_permissions() build_ci_image_check_cache = Path( BUILD_CACHE_DIR, shell_params.airflow_branch, f".built_{shell_params.python_version}" ) diff --git a/dev/breeze/src/airflow_breeze/utils/run_utils.py b/dev/breeze/src/airflow_breeze/utils/run_utils.py index 7b4f785ffe..f48d16e540 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/run_utils.py @@ -194,6 +194,7 @@ def change_directory_permission(directory_to_fix: Path): @working_directory(AIRFLOW_SOURCE) def fix_group_permissions(): + console.print("[blue]Fixing group permissions[/]") files_to_fix_result = run_command(['git', 'ls-files', './'], capture_output=True, text=True) if files_to_fix_result.returncode == 0: files_to_fix = files_to_fix_result.stdout.strip().split('\n')