Skip to content

Commit

Permalink
Fix file permissions in all build commands (#22656)
Browse files Browse the repository at this point in the history
Moves group permission cleanup to build command in new Breeze2.

Seems that we should fix group permissions in all build commands.
That will shave extra few minutes on most builds where they
modify sources. Also the output from the build will be a bit
cleaner by removing come of the console outputs.

GitOrigin-RevId: 89a4c4d94d21e4fd9e50e792fc5c0025695538de
  • Loading branch information
potiuk authored and Cloud Composer Team committed Sep 12, 2024
1 parent ecf93d4 commit ca222e2
Show file tree
Hide file tree
Showing 4 changed files with 7 additions and 12 deletions.
10 changes: 4 additions & 6 deletions dev/breeze/src/airflow_breeze/ci/build_image.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,8 @@

from airflow_breeze.cache import check_cache_and_write_if_not_cached, touch_cache_file, write_to_cache_file
from airflow_breeze.ci.build_params import BuildParams
from airflow_breeze.console import console
from airflow_breeze.utils.path_utils import AIRFLOW_SOURCE, BUILD_CACHE_DIR
from airflow_breeze.utils.run_utils import filter_out_none, run_command
from airflow_breeze.utils.run_utils import filter_out_none, fix_group_permissions, run_command

PARAMS_CI_IMAGE = [
"python_base_image",
Expand Down Expand Up @@ -83,6 +82,7 @@ def construct_docker_command(ci_image: BuildParams) -> List[str]:


def build_image(verbose, **kwargs):
fix_group_permissions()
parameters_passed = filter_out_none(**kwargs)
ci_image_params = get_image_build_params(parameters_passed)
ci_image_cache_dir = Path(BUILD_CACHE_DIR, ci_image_params.airflow_branch)
Expand All @@ -91,17 +91,15 @@ def build_image(verbose, **kwargs):
f"built_{ci_image_params.python_version}",
root_dir=ci_image_cache_dir,
)
output = run_command(
run_command(
["docker", "rmi", "--no-prune", "--force", ci_image_params.airflow_ci_image_name],
verbose=verbose,
cwd=AIRFLOW_SOURCE,
text=True,
suppress_raise_exception=True,
)
console.print(f"[blue]{output}")
cmd = construct_docker_command(ci_image_params)
output = run_command(cmd, verbose=verbose, cwd=AIRFLOW_SOURCE, text=True)
console.print(f"[blue]{output}")
run_command(cmd, verbose=verbose, cwd=AIRFLOW_SOURCE, text=True)


def get_image_build_params(parameters_passed: Dict[str, str]):
Expand Down
6 changes: 2 additions & 4 deletions dev/breeze/src/airflow_breeze/prod/build_prod_image.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,16 +171,14 @@ def build_production_image(verbose, **kwargs):
login_to_docker_registry(prod_params)

cmd = construct_docker_command(prod_params)
output = run_command(
run_command(
["docker", "rmi", "--no-prune", "--force", prod_params.airflow_prod_image_name],
verbose=verbose,
cwd=AIRFLOW_SOURCE,
text=True,
suppress_raise_exception=True,
)
console.print(f"[blue]{output}")
output = run_command(cmd, verbose=verbose, cwd=AIRFLOW_SOURCE, text=True)
console.print(f"[blue]{output}")
run_command(cmd, verbose=verbose, cwd=AIRFLOW_SOURCE, text=True)
if prod_params.prepare_buildx_cache:
run_command(['docker', 'push', prod_params.airflow_prod_image_name], verbose=True, text=True)

Expand Down
2 changes: 0 additions & 2 deletions dev/breeze/src/airflow_breeze/shell/enter_shell.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@
from airflow_breeze.utils.path_utils import BUILD_CACHE_DIR
from airflow_breeze.utils.run_utils import (
filter_out_none,
fix_group_permissions,
get_latest_sha,
instruct_build_image,
instruct_for_setup,
Expand Down Expand Up @@ -181,7 +180,6 @@ def build_image_if_needed_steps(verbose: bool, shell_params: ShellBuilder):


def build_image_checks(verbose: bool, shell_params: ShellBuilder):
fix_group_permissions()
build_ci_image_check_cache = Path(
BUILD_CACHE_DIR, shell_params.airflow_branch, f".built_{shell_params.python_version}"
)
Expand Down
1 change: 1 addition & 0 deletions dev/breeze/src/airflow_breeze/utils/run_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,7 @@ def change_directory_permission(directory_to_fix: Path):

@working_directory(AIRFLOW_SOURCE)
def fix_group_permissions():
console.print("[blue]Fixing group permissions[/]")
files_to_fix_result = run_command(['git', 'ls-files', './'], capture_output=True, text=True)
if files_to_fix_result.returncode == 0:
files_to_fix = files_to_fix_result.stdout.strip().split('\n')
Expand Down

0 comments on commit ca222e2

Please sign in to comment.