Skip to content

Commit

Permalink
Fix constraints generation when there are no differences
Browse files Browse the repository at this point in the history
The new constraints generation implemented in #36158 had a bug that
constraint files have not been generated when there was no changes
(wrong file has been deleted). This resulted in PROD files generation
failing when there were no new releases to one of 670 dependencies.

This PR fixes it and improves the diagnostics of comstraints
generation:

* more information printed about generated files
* using shorten method wnen running commands (will add ellipsis if
  longer)
* failing when there are not constraints to upload - indicating
  a bug in the processs of constraints generation rather than when
  downloaded constraints are missing.
  • Loading branch information
potiuk committed Dec 11, 2023
1 parent fcd993b commit 4c3a0b4
Show file tree
Hide file tree
Showing 5 changed files with 26 additions and 4 deletions.
1 change: 1 addition & 0 deletions .github/actions/build-ci-images/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,3 +45,4 @@ runs:
name: source-constraints
path: ./files/constraints-*/constraints-*.txt
retention-days: 7
if-no-files-found: error
1 change: 1 addition & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -536,6 +536,7 @@ jobs:
name: constraints
path: ./files/constraints-*/constraints-*.txt
retention-days: 7
if-no-files-found: error


static-checks:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -613,6 +613,7 @@ def run_generate_constraints(
command="/opt/airflow/scripts/in_container/run_generate_constraints.py",
output=output,
)

return (
result.returncode,
f"Constraints {shell_params.airflow_constraints_mode}:{shell_params.python}",
Expand All @@ -624,6 +625,15 @@ def run_generate_constraints(
)


def list_generated_constraints(output: Output | None):
get_console(output=output).print("\n[info]List of generated files in './files' folder:[/]\n")
found_files = Path("./files").rglob("*")
for file in sorted(found_files):
if file.is_file():
get_console(output=output).print(file.as_posix())
get_console(output=output).print()


def run_generate_constraints_in_parallel(
shell_params_list: list[ShellParams],
python_version_list: list[str],
Expand Down Expand Up @@ -764,6 +774,7 @@ def generate_constraints(
if return_code != 0:
get_console().print(f"[error]There was an error when generating constraints: {info}[/]")
sys.exit(return_code)
list_generated_constraints(output=None)


SDIST_FILENAME_PREFIX = "apache-airflow-providers-"
Expand Down
3 changes: 2 additions & 1 deletion scripts/in_container/in_container_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import shlex
import subprocess
import textwrap
from contextlib import contextmanager

import rich_click as click
Expand All @@ -30,7 +31,7 @@
@contextmanager
def ci_group(group_name: str, github_actions: bool):
if github_actions:
console.print(f"::group::{group_name[:200]}[/]", markup=False)
console.print(f"::group::{textwrap.shorten(group_name, width=200)}", markup=False)
console.print(group_name, markup=False)
try:
yield
Expand Down
14 changes: 11 additions & 3 deletions scripts/in_container/run_generate_constraints.py
Original file line number Diff line number Diff line change
Expand Up @@ -154,16 +154,19 @@ def freeze_packages_to_file(config_params: ConfigParams, file: TextIO) -> None:
check=True,
capture_output=True,
)
count_lines = 0
for line in sorted(result.stdout.split("\n")):
if line.startswith(("apache_airflow", "apache-airflow==", "/opt/airflow", "#", "-e")):
continue
if "@" in line:
continue
if line.strip() == "":
continue
count_lines += 1
file.write(line)
file.write("\n")
console.print(f"[green]Constraints generated to file: {file.name}")
file.flush()
console.print(f"[green]Constraints generated to file: {file.name}. Wrote {count_lines} lines")


def download_latest_constraint_file(config_params: ConfigParams):
Expand Down Expand Up @@ -201,8 +204,8 @@ def diff_constraints(config_params: ConfigParams) -> None:
)
if result.returncode == 0:
console.print("[green]No changes in constraints files. exiting")
config_params.current_constraints_file.unlink(missing_ok=True)
sys.exit(0)
config_params.constraints_diff_file.unlink(missing_ok=True)
return
result = run_command(
[
"diff",
Expand Down Expand Up @@ -444,6 +447,11 @@ def generate_constraints(
else:
console.print(f"[red]Unknown constraints mode: {airflow_constraints_mode}")
sys.exit(1)
console.print("[green]Generated constraints:")
files = config_params.constraints_dir.rglob("*.txt")
for file in files:
console.print(file.as_posix())
console.print()


if __name__ == "__main__":
Expand Down

0 comments on commit 4c3a0b4

Please sign in to comment.