From d006b378f1ebe5fcdd1e8662ef30edb789588e91 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 13 Mar 2024 19:37:06 +0100 Subject: [PATCH] Changesets --- .buildkite/pipeline.py | 106 +++++++++++++++++++++++----------------- auditbeat/buildkite.yml | 8 +-- filebeat/buildkite.yml | 8 +-- 3 files changed, 68 insertions(+), 54 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 786dbd5ba83..68016e3a0c6 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -2,6 +2,8 @@ import yaml import os from dataclasses import dataclass, field +import subprocess +import fnmatch from jinja2 import Template from pathlib import Path @@ -24,7 +26,6 @@ def create_entity(self): msg = tm.render(pipeline=self) return msg - @dataclass(unsafe_hash=True) class Group: """Buildkite Group object""" @@ -92,8 +93,9 @@ def create_entity(self): # Conditions: -def is_pr(value) -> bool: - return os.getenv('BUILDKITE_PULL_REQUEST') == value +def is_pr() -> bool: + pr = os.getenv('BUILDKITE_PULL_REQUEST') + return pr and os.getenv('BUILDKITE_PULL_REQUEST') != "false" def step_comment(step: Step) -> bool: comment = os.getenv('GITHUB_PR_TRIGGER_COMMENT') @@ -106,17 +108,48 @@ def step_comment(step: Step) -> bool: return True # i.e: /test filebeat unitTest return comment_prefix + " " + step.name in comment + else: + return True + +def group_comment(group: Group) -> bool: + comment = os.getenv('GITHUB_PR_TRIGGER_COMMENT') + if comment: + # the comment should be a subset of the values in .buildkite/pull-requests.json + # TODO: change /test + comment_prefix = "buildkite test" + if group.category == "mandatory": + # i.e: /test filebeat + return comment_prefix + " " + group.project in comment + else: + # i.e: test filebeat extended + return comment_prefix + " " + group.project + " " + group.category in comment + +changed_files = None + +def get_pr_changeset(): + if not changed_files: + base_branch = os.getenv('BUILDKITE_PULL_REQUEST_BASE_BRANCH') + diff_command = ["git", "diff", "--name-only", "{}...HEAD".format(base_branch)] + result = subprocess.run(diff_command, stdout=subprocess.PIPE) + changed_files = result.stdout.decode().splitlines() + print("Changed files: {}".format(changed_files)) + + return changed_files + +def filter_files_by_glob(files, patterns: list[str]): + for pattern in patterns: + ## TODO: Support glob extended patterns: ^ and etc. + ## Now it supports only linux glob patterns + if fnmatch.filter(files, pattern): + return True + return False -def is_in_changeset() -> bool: - # TODO - return True - -def is_step_enabled(step: Step, conditions) -> bool: - # TODO: - # If PR then - # If Changeset +def is_in_pr_changeset(project_changeset_filters: list[str]) -> bool: + changeset = get_pr_changeset() + return filter_files_by_glob(changeset, project_changeset_filters) - if is_pr("false"): +def is_step_enabled(step: Step) -> bool: + if not is_pr(): return True if step_comment(step): @@ -130,31 +163,15 @@ def is_step_enabled(step: Step, conditions) -> bool: return True return False + +def is_group_enabled(group: Group, changeset_filters: list[str]) -> bool: + if not is_pr(): + return True - -def is_group_enabled(group: Group, conditions) -> bool: - # TODO: - # If PR then - # If GitHub label matches project name + category (I'm not sure we wanna use this approach since GH comments support it) - # If Changeset - - pull_request = os.getenv('BUILDKITE_PULL_REQUEST') - if pull_request and pull_request == "false": - return True - - comment = os.getenv('GITHUB_PR_TRIGGER_COMMENT') - if comment: - # the comment should be a subset of the values in .buildkite/pull-requests.json - # TODO: change /test - comment_prefix = "buildkite test" - if group.category == "mandatory": - # i.e: /test filebeat - return comment_prefix + " " + group.project in comment - else: - # i.e: test filebeat extended - return comment_prefix + " " + group.project + " " + group.category in comment - - return group.category.startswith("mandatory") + if is_pr() and is_in_pr_changeset(changeset_filters) and group.category.startswith("mandatory"): + return True + + return group_comment(group) def fetch_stage(name: str, stage, project: str, category: str) -> Step: """Create a step given the yaml object.""" @@ -171,7 +188,7 @@ def fetch_stage(name: str, stage, project: str, category: str) -> Step: provider="gcp") -def fetch_group(stages, project: str, category: str, conditions) -> Group: +def fetch_group(stages, project: str, category: str) -> Group: """Create a group given the yaml object.""" steps = [] @@ -183,14 +200,13 @@ def fetch_group(stages, project: str, category: str, conditions) -> Group: project=project, stage=stages[stage]) - if is_step_enabled(step, conditions): + if is_step_enabled(step): steps.append(step) return Group( project=project, category=category, - steps=steps - ) + steps=steps) # TODO: validate unique stages! @@ -214,18 +230,16 @@ def main() -> None: group = fetch_group(stages=project_obj["stages"]["mandatory"], project=project, - category="mandatory", - conditions=conditions) + category="mandatory") - if is_group_enabled(group, conditions): + if is_group_enabled(group, project_obj["when"]["changeset"]): groups.append(group) group = fetch_group(stages=project_obj["stages"]["extended"], project=project, - category="extended", - conditions=conditions) + category="extended") - if is_group_enabled(group, conditions): + if is_group_enabled(group, project_obj["when"]["changeset"]): extended_groups.append(group) # TODO: improve this merging lists diff --git a/auditbeat/buildkite.yml b/auditbeat/buildkite.yml index c85c176947e..709f50209f1 100644 --- a/auditbeat/buildkite.yml +++ b/auditbeat/buildkite.yml @@ -1,8 +1,8 @@ when: - changeset: ## when PR contains any of those entries in the changeset - - "^auditbeat/.*" - - "@ci" ## special token regarding the changeset for the ci - - "@oss" ## special token regarding the changeset for the oss + changeset: ## when PR contains any of those entries in the changeset + - "auditbeat/**" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss stages: # mandatory stage - it runs always for: # - branches/tags diff --git a/filebeat/buildkite.yml b/filebeat/buildkite.yml index faffac3313e..9e8a2c61356 100644 --- a/filebeat/buildkite.yml +++ b/filebeat/buildkite.yml @@ -1,8 +1,8 @@ when: - changeset: ## when PR contains any of those entries in the changeset - - "^filebeat/.*" - - "@ci" ## special token regarding the changeset for the ci - - "@oss" ## special token regarding the changeset for the oss + changeset: ## when PR contains any of those entries in the changeset + - "filebeat/**" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss stages: # default stage - it runs always for: # - branches/tags