From 6778ecf960693b834aadca37239ce6cfa268dc55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Thu, 2 May 2024 21:42:36 +0200 Subject: [PATCH 1/5] Use sum type for `WorkflowRunType` --- src/ci/github-actions/calculate-job-matrix.py | 40 ++++++++++++------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/src/ci/github-actions/calculate-job-matrix.py b/src/ci/github-actions/calculate-job-matrix.py index 68565f489c939..1801904d1e79f 100755 --- a/src/ci/github-actions/calculate-job-matrix.py +++ b/src/ci/github-actions/calculate-job-matrix.py @@ -8,10 +8,10 @@ and filters them based on the event that happened on CI. """ import dataclasses -import enum import json import logging import os +import typing from pathlib import Path from typing import List, Dict, Any, Optional @@ -44,10 +44,22 @@ def add_base_env(jobs: List[Job], environment: Dict[str, str]) -> List[Job]: return jobs -class WorkflowRunType(enum.Enum): - PR = enum.auto() - Try = enum.auto() - Auto = enum.auto() +@dataclasses.dataclass +class PRRunType: + pass + + +@dataclasses.dataclass +class TryRunType: + custom_jobs: List[str] + + +@dataclasses.dataclass +class AutoRunType: + pass + + +WorkflowRunType = typing.Union[PRRunType, TryRunType, AutoRunType] @dataclasses.dataclass @@ -59,7 +71,7 @@ class GitHubCtx: def find_run_type(ctx: GitHubCtx) -> Optional[WorkflowRunType]: if ctx.event_name == "pull_request": - return WorkflowRunType.PR + return PRRunType() elif ctx.event_name == "push": old_bors_try_build = ( ctx.ref in ("refs/heads/try", "refs/heads/try-perf") and @@ -72,20 +84,20 @@ def find_run_type(ctx: GitHubCtx) -> Optional[WorkflowRunType]: try_build = old_bors_try_build or new_bors_try_build if try_build: - return WorkflowRunType.Try + return TryRunType() if ctx.ref == "refs/heads/auto" and ctx.repository == "rust-lang-ci/rust": - return WorkflowRunType.Auto + return AutoRunType() return None def calculate_jobs(run_type: WorkflowRunType, job_data: Dict[str, Any]) -> List[Job]: - if run_type == WorkflowRunType.PR: + if isinstance(run_type, PRRunType): return add_base_env(name_jobs(job_data["pr"], "PR"), job_data["envs"]["pr"]) - elif run_type == WorkflowRunType.Try: + elif isinstance(run_type, TryRunType): return add_base_env(name_jobs(job_data["try"], "try"), job_data["envs"]["try"]) - elif run_type == WorkflowRunType.Auto: + elif isinstance(run_type, AutoRunType): return add_base_env(name_jobs(job_data["auto"], "auto"), job_data["envs"]["auto"]) return [] @@ -107,11 +119,11 @@ def get_github_ctx() -> GitHubCtx: def format_run_type(run_type: WorkflowRunType) -> str: - if run_type == WorkflowRunType.PR: + if isinstance(run_type, PRRunType): return "pr" - elif run_type == WorkflowRunType.Auto: + elif isinstance(run_type, AutoRunType): return "auto" - elif run_type == WorkflowRunType.Try: + elif isinstance(run_type, TryRunType): return "try" else: raise AssertionError() From 74dbe8ae9ef42615875ae36c0e524a48ce0fb16f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Thu, 2 May 2024 21:44:47 +0200 Subject: [PATCH 2/5] Parse try build CI job name from commit message --- .github/workflows/ci.yml | 2 + src/ci/github-actions/calculate-job-matrix.py | 51 +++++++++++++++++-- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a2769a2c2749d..1c5330ec25dd6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,8 @@ jobs: - name: Checkout the source code uses: actions/checkout@v4 - name: Calculate the CI job matrix + env: + COMMIT_MESSAGE: ${{ github.event.head_commit.message }} run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT id: jobs job: diff --git a/src/ci/github-actions/calculate-job-matrix.py b/src/ci/github-actions/calculate-job-matrix.py index 1801904d1e79f..4aba5059d419e 100755 --- a/src/ci/github-actions/calculate-job-matrix.py +++ b/src/ci/github-actions/calculate-job-matrix.py @@ -11,6 +11,7 @@ import json import logging import os +import re import typing from pathlib import Path from typing import List, Dict, Any, Optional @@ -67,6 +68,23 @@ class GitHubCtx: event_name: str ref: str repository: str + commit_message: Optional[str] + + +def get_custom_jobs(ctx: GitHubCtx) -> List[str]: + """ + Tries to parse names of specific CI jobs that should be executed in the form of + ci-job: + from the commit message of the passed GitHub context. + """ + if ctx.commit_message is None: + return [] + + regex = re.compile(r"ci-job: (.*)") + jobs = [] + for match in regex.finditer(ctx.commit_message): + jobs.append(match.group(1)) + return jobs def find_run_type(ctx: GitHubCtx) -> Optional[WorkflowRunType]: @@ -84,7 +102,8 @@ def find_run_type(ctx: GitHubCtx) -> Optional[WorkflowRunType]: try_build = old_bors_try_build or new_bors_try_build if try_build: - return TryRunType() + jobs = get_custom_jobs(ctx) + return TryRunType(custom_jobs=jobs) if ctx.ref == "refs/heads/auto" and ctx.repository == "rust-lang-ci/rust": return AutoRunType() @@ -96,8 +115,24 @@ def calculate_jobs(run_type: WorkflowRunType, job_data: Dict[str, Any]) -> List[ if isinstance(run_type, PRRunType): return add_base_env(name_jobs(job_data["pr"], "PR"), job_data["envs"]["pr"]) elif isinstance(run_type, TryRunType): - return add_base_env(name_jobs(job_data["try"], "try"), job_data["envs"]["try"]) - elif isinstance(run_type, AutoRunType): + jobs = job_data["try"] + custom_jobs = run_type.custom_jobs + if custom_jobs: + if len(custom_jobs) > 10: + raise Exception( + f"It is only possible to schedule up to 10 custom jobs," + f"received {len(custom_jobs)} jobs" + ) + + jobs = [] + for custom_job in custom_jobs: + job = [j for j in job_data["auto"] if j["image"] == custom_job] + if not job: + raise Exception(f"Custom job `{custom_job}` not found in auto jobs") + jobs.append(job[0]) + + return add_base_env(name_jobs(jobs, "try"), job_data["envs"]["try"]) + elif run_type is AutoRunType: return add_base_env(name_jobs(job_data["auto"], "auto"), job_data["envs"]["auto"]) return [] @@ -111,10 +146,16 @@ def skip_jobs(jobs: List[Dict[str, Any]], channel: str) -> List[Job]: def get_github_ctx() -> GitHubCtx: + event_name = os.environ["GITHUB_EVENT_NAME"] + + commit_message = None + if event_name == "push": + commit_message = os.environ["COMMIT_MESSAGE"] return GitHubCtx( - event_name=os.environ["GITHUB_EVENT_NAME"], + event_name=event_name, ref=os.environ["GITHUB_REF"], - repository=os.environ["GITHUB_REPOSITORY"] + repository=os.environ["GITHUB_REPOSITORY"], + commit_message=commit_message ) From e2e280610dba7f37685cd656faf3446d968bfbe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Thu, 2 May 2024 22:41:11 +0200 Subject: [PATCH 3/5] Make the regex more robust --- src/ci/github-actions/calculate-job-matrix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/github-actions/calculate-job-matrix.py b/src/ci/github-actions/calculate-job-matrix.py index 4aba5059d419e..7c42c4ec5c6b9 100755 --- a/src/ci/github-actions/calculate-job-matrix.py +++ b/src/ci/github-actions/calculate-job-matrix.py @@ -80,7 +80,7 @@ def get_custom_jobs(ctx: GitHubCtx) -> List[str]: if ctx.commit_message is None: return [] - regex = re.compile(r"ci-job: (.*)") + regex = re.compile(r"^ci-job: (.*)", re.MULTILINE) jobs = [] for match in regex.finditer(ctx.commit_message): jobs.append(match.group(1)) From bf8bcc4c2d3b0b9c44bc7286b9377a9674b67186 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Fri, 3 May 2024 10:06:13 +0200 Subject: [PATCH 4/5] Address review comments --- src/ci/github-actions/calculate-job-matrix.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/ci/github-actions/calculate-job-matrix.py b/src/ci/github-actions/calculate-job-matrix.py index 7c42c4ec5c6b9..115f2e0ad7835 100755 --- a/src/ci/github-actions/calculate-job-matrix.py +++ b/src/ci/github-actions/calculate-job-matrix.py @@ -74,13 +74,13 @@ class GitHubCtx: def get_custom_jobs(ctx: GitHubCtx) -> List[str]: """ Tries to parse names of specific CI jobs that should be executed in the form of - ci-job: + try-job: from the commit message of the passed GitHub context. """ if ctx.commit_message is None: return [] - regex = re.compile(r"^ci-job: (.*)", re.MULTILINE) + regex = re.compile(r"^try-job: (.*)", re.MULTILINE) jobs = [] for match in regex.finditer(ctx.commit_message): jobs.append(match.group(1)) @@ -120,16 +120,20 @@ def calculate_jobs(run_type: WorkflowRunType, job_data: Dict[str, Any]) -> List[ if custom_jobs: if len(custom_jobs) > 10: raise Exception( - f"It is only possible to schedule up to 10 custom jobs," + f"It is only possible to schedule up to 10 custom jobs, " f"received {len(custom_jobs)} jobs" ) jobs = [] + unknown_jobs = [] for custom_job in custom_jobs: job = [j for j in job_data["auto"] if j["image"] == custom_job] if not job: - raise Exception(f"Custom job `{custom_job}` not found in auto jobs") + unknown_jobs.append(custom_job) + continue jobs.append(job[0]) + if unknown_jobs: + raise Exception(f"Custom job(s) `{unknown_jobs}` not found in auto jobs") return add_base_env(name_jobs(jobs, "try"), job_data["envs"]["try"]) elif run_type is AutoRunType: From b3e9060178780b51c21b50c30f04f2b3d339164a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Sun, 5 May 2024 08:05:10 +0200 Subject: [PATCH 5/5] CI: fix auto builds and make sure that we always have at least a single CI job --- .github/workflows/ci.yml | 8 -------- src/ci/github-actions/calculate-job-matrix.py | 6 +++++- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c5330ec25dd6..5364c1e9f466c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,14 +77,6 @@ jobs: matrix: # Check the `calculate_matrix` job to see how is the matrix defined. include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }} - # GitHub Actions fails the workflow if an empty list of jobs is provided to - # the workflow, so we need to skip this job if nothing was produced by - # the Python script. - # - # Unfortunately checking whether a list is empty is not possible in a nice - # way due to GitHub Actions expressions limits. - # This hack is taken from https://github.com/ferrocene/ferrocene/blob/d43edc6b7697cf1719ec1c17c54904ab94825763/.github/workflows/release.yml#L75-L82 - if: fromJSON(needs.calculate_matrix.outputs.jobs)[0] != null steps: - if: contains(matrix.os, 'windows') uses: msys2/setup-msys2@v2.22.0 diff --git a/src/ci/github-actions/calculate-job-matrix.py b/src/ci/github-actions/calculate-job-matrix.py index 115f2e0ad7835..4f9bc39a628ce 100755 --- a/src/ci/github-actions/calculate-job-matrix.py +++ b/src/ci/github-actions/calculate-job-matrix.py @@ -136,7 +136,7 @@ def calculate_jobs(run_type: WorkflowRunType, job_data: Dict[str, Any]) -> List[ raise Exception(f"Custom job(s) `{unknown_jobs}` not found in auto jobs") return add_base_env(name_jobs(jobs, "try"), job_data["envs"]["try"]) - elif run_type is AutoRunType: + elif isinstance(run_type, AutoRunType): return add_base_env(name_jobs(job_data["auto"], "auto"), job_data["envs"]["auto"]) return [] @@ -192,6 +192,10 @@ def format_run_type(run_type: WorkflowRunType) -> str: if run_type is not None: jobs = calculate_jobs(run_type, data) jobs = skip_jobs(jobs, channel) + + if not jobs: + raise Exception("Scheduled job list is empty, this is an error") + run_type = format_run_type(run_type) logging.info(f"Output:\n{yaml.dump(dict(jobs=jobs, run_type=run_type), indent=4)}")