Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix constraints generation when there are no differences #36168

Merged
merged 1 commit into from
Dec 11, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/actions/build-ci-images/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,3 +45,4 @@ runs:
name: source-constraints
path: ./files/constraints-*/constraints-*.txt
retention-days: 7
if-no-files-found: error
1 change: 1 addition & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -536,6 +536,7 @@ jobs:
name: constraints
path: ./files/constraints-*/constraints-*.txt
retention-days: 7
if-no-files-found: error


static-checks:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -613,6 +613,7 @@ def run_generate_constraints(
command="/opt/airflow/scripts/in_container/run_generate_constraints.py",
output=output,
)

return (
result.returncode,
f"Constraints {shell_params.airflow_constraints_mode}:{shell_params.python}",
Expand All @@ -624,6 +625,15 @@ def run_generate_constraints(
)


def list_generated_constraints(output: Output | None):
get_console(output=output).print("\n[info]List of generated files in './files' folder:[/]\n")
found_files = Path("./files").rglob("*")
for file in sorted(found_files):
if file.is_file():
get_console(output=output).print(file.as_posix())
get_console(output=output).print()


def run_generate_constraints_in_parallel(
shell_params_list: list[ShellParams],
python_version_list: list[str],
Expand Down Expand Up @@ -764,6 +774,7 @@ def generate_constraints(
if return_code != 0:
get_console().print(f"[error]There was an error when generating constraints: {info}[/]")
sys.exit(return_code)
list_generated_constraints(output=None)


SDIST_FILENAME_PREFIX = "apache-airflow-providers-"
Expand Down
3 changes: 2 additions & 1 deletion scripts/in_container/in_container_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

import shlex
import subprocess
import textwrap
from contextlib import contextmanager

import rich_click as click
Expand All @@ -30,7 +31,7 @@
@contextmanager
def ci_group(group_name: str, github_actions: bool):
if github_actions:
console.print(f"::group::{group_name[:200]}[/]", markup=False)
console.print(f"::group::{textwrap.shorten(group_name, width=200)}", markup=False)
console.print(group_name, markup=False)
try:
yield
Expand Down
14 changes: 11 additions & 3 deletions scripts/in_container/run_generate_constraints.py
Original file line number Diff line number Diff line change
Expand Up @@ -154,16 +154,19 @@ def freeze_packages_to_file(config_params: ConfigParams, file: TextIO) -> None:
check=True,
capture_output=True,
)
count_lines = 0
for line in sorted(result.stdout.split("\n")):
if line.startswith(("apache_airflow", "apache-airflow==", "/opt/airflow", "#", "-e")):
continue
if "@" in line:
continue
if line.strip() == "":
continue
count_lines += 1
file.write(line)
file.write("\n")
console.print(f"[green]Constraints generated to file: {file.name}")
file.flush()
console.print(f"[green]Constraints generated to file: {file.name}. Wrote {count_lines} lines")


def download_latest_constraint_file(config_params: ConfigParams):
Expand Down Expand Up @@ -201,8 +204,8 @@ def diff_constraints(config_params: ConfigParams) -> None:
)
if result.returncode == 0:
console.print("[green]No changes in constraints files. exiting")
config_params.current_constraints_file.unlink(missing_ok=True)
sys.exit(0)
config_params.constraints_diff_file.unlink(missing_ok=True)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was the error. Wrong file was removed :(

return
result = run_command(
[
"diff",
Expand Down Expand Up @@ -444,6 +447,11 @@ def generate_constraints(
else:
console.print(f"[red]Unknown constraints mode: {airflow_constraints_mode}")
sys.exit(1)
console.print("[green]Generated constraints:")
files = config_params.constraints_dir.rglob("*.txt")
for file in files:
console.print(file.as_posix())
console.print()


if __name__ == "__main__":
Expand Down