From 05b112c956ede66ab9f58e683e7f7c5a6d0af531 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 1 Mar 2024 13:53:37 +0100 Subject: [PATCH 1/9] buildkite: python generator --- .buildkite/buildkite.yml | 44 +++++++++ .buildkite/pipeline.py | 200 +++++++++++++++++++++++++++++++++++++++ auditbeat/buildkite.yml | 50 ++++++++++ filebeat/buildkite.yml | 49 ++++++++++ 4 files changed, 343 insertions(+) create mode 100644 .buildkite/buildkite.yml create mode 100755 .buildkite/pipeline.py create mode 100644 auditbeat/buildkite.yml create mode 100644 filebeat/buildkite.yml diff --git a/.buildkite/buildkite.yml b/.buildkite/buildkite.yml new file mode 100644 index 000000000000..4707707e07cf --- /dev/null +++ b/.buildkite/buildkite.yml @@ -0,0 +1,44 @@ +projects: + - "auditbeat" + - "deploy/kubernetes" + - "filebeat" + - "heartbeat" + - "libbeat" + - "metricbeat" + - "packetbeat" + - "winlogbeat" + - "x-pack/auditbeat" + - "x-pack/dockerlogbeat" + - "x-pack/filebeat" + - "x-pack/functionbeat" + - "x-pack/heartbeat" + - "x-pack/libbeat" + - "x-pack/metricbeat" + - "x-pack/osquerybeat" + - "x-pack/packetbeat" + - "x-pack/winlogbeat" + +## Changeset macros that are defined here and used in each specific 3.0 pipeline. +changeset: + ci: + - "^Jenkinsfile" + - "^\\.ci/scripts/.*" + oss: + - "^go.mod" + - "^pytest.ini" + - "^dev-tools/.*" + - "^libbeat/.*" + - "^testing/.*" + xpack: + - "^go.mod" + - "^pytest.ini" + - "^dev-tools/.*" + - "^libbeat/.*" + - "^testing/.*" + - "^x-pack/libbeat/.*" + +disabled: + when: + labels: ## Skip the GitHub Pull Request builds if any of the given GitHub labels match with the assigned labels in the PR. + - skip-ci + draft: true ## Skip the GitHub Pull Request builds with Draft PRs. diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py new file mode 100755 index 000000000000..af0e0f048739 --- /dev/null +++ b/.buildkite/pipeline.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +import yaml +import os +from dataclasses import dataclass, field + +from jinja2 import Template +from pathlib import Path + + +@dataclass() +class Pipeline: + """Buildkite Pipeline object""" + groups: list[str] + + def create_entity(self): + data = """ +steps: +{% for group in pipeline.groups -%} +{{ group.create_entity() }} +{% endfor -%} +""" + + tm = Template(data) + msg = tm.render(pipeline=self) + return msg + + +@dataclass(unsafe_hash=True) +class Group: + """Buildkite Group object""" + + project: str + category: str + steps: list[str] + + def __lt__(self, other): + return self.project < other.project + + def create_entity(self): + data = """ + - group: "{{ group.project }} {{ group.category }}" + key: "{{ group.project }}-{{ group.category }}" + steps: + {% for step in group.steps|sort -%} + {{ step.create_entity() }} + {% endfor -%} +""" + + tm = Template(data) + msg = tm.render(group=self) + return msg + + +@dataclass(unsafe_hash=True) +class Step: + """Buildkite Step object""" + + command: str + name: str + runner: str + project: str + provider: str + label: str = field(init=False) + comment: str = field(init=False) + + def __post_init__(self): + self.comment = "/test " + self.project + " " + self.name + self.label = self.name + + def __lt__(self, other): + return self.name < other.name + + def create_entity(self): + data = """ + - label: "{{ stage.project }} {{ stage.name }}" + command: + - {{ stage.command }} + notify: + - github_commit_status: + context: "{{ stage.project }}: {{ stage.name }}" + agents: + provider: {{ stage.provider }} + image:{{ stage.runner }} +""" + + tm = Template(data) + msg = tm.render(stage=self) + return msg + + +def is_step_enabled(step: Step, conditions) -> bool: + # TODO: + # If branch + # If PR then + # If GitHub label matches project name + # If GitHub comment + # If Changeset + return True + + +def is_group_enabled(group: Group, conditions) -> bool: + # TODO: + # If branch + # If PR then + # If GitHub label matches project name + category + # If GitHub comment + # If Changeset + return True + + +def fetch_stage(name: str, stage, project: str) -> Step: + """Create a step given the yaml object.""" + + # TODO: need to accomodate the provider type. + # maybe in the buildkite.yml or some dynamic analysis based on the + # name of the runners. + return Step( + command=stage["command"], + name=name, + runner=stage["platform"], + project=project, + provider="gcp") + +# TODO: validate unique stages! + +def main() -> None: + + groups = [] + extended_groups = [] + with open(".buildkite/buildkite.yml", "r", encoding="utf8") as file: + doc = yaml.load(file, yaml.FullLoader) + + for project in doc["projects"]: + project_file = os.path.join(project, "buildkite.yml") + if not os.path.isfile(project_file): + continue + # TODO: data structure when things run. + conditions = None + with open(project_file, "r", encoding="utf8") as file: + steps = [] + project_obj = yaml.load(file, yaml.FullLoader) + + # Given the mandatory list first + mandatory = project_obj["stages"]["mandatory"] + for stage in mandatory: + step = fetch_stage( + name=stage, + project=project, + stage=mandatory[stage]) + + if is_step_enabled(step, conditions): + steps.append(step) + + group = Group( + project=project, + category="mandatory", + steps=steps + ) + + if is_group_enabled(group, conditions): + extended_groups.append(group) + + # Given the extended list if needed + # TODO: Validate if included + extended_steps = [] + + extended = project_obj["stages"]["extended"] + for stage in extended: + step = fetch_stage( + name=stage, + project=project, + stage=extended[stage]) + + if is_step_enabled(step, conditions): + extended_steps.append(step) + + group = Group( + project=project, + category="extended", + steps=extended_steps + ) + + if is_group_enabled(group, conditions): + extended_groups.append(group) + + # TODO: improve this merging lists + all_groups = [] + for group in sorted(groups): + all_groups.append(group) + for group in sorted(extended_groups): + all_groups.append(group) + + # Produce now the pipeline + print(Pipeline(all_groups).create_entity()) + + +if __name__ == "__main__": + + # pylint: disable=E1120 + main() diff --git a/auditbeat/buildkite.yml b/auditbeat/buildkite.yml new file mode 100644 index 000000000000..c85c176947e2 --- /dev/null +++ b/auditbeat/buildkite.yml @@ -0,0 +1,50 @@ +when: + changeset: ## when PR contains any of those entries in the changeset + - "^auditbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss +stages: + # mandatory stage - it runs always for: + # - branches/tags + # - on PRs + # - GitHub comment /test auditbeat + # - GitHub label auditbeat + mandatory: + # NOTE: stage name should be unique! + unitTest: + command: "mage build unitTest" + platform: "family/core-ubuntu-2204" + crosscompile: + command: "make -C auditbeat crosscompile" + platform: "family/core-ubuntu-2204" + rhel-9: + command: "mage build unitTest" + platform: "family/core-rhel-9" + unitTest-windows-2022: + command: "mage build unitTest" + platform: "windows-2022" + unitTest-windows-2016: + command: "mage build unitTest" + platform: "family/core-windows-2016" + # optional stage - it runs on: + # - branches/tags + # - on PRs if: + # - GitHub comment /test auditbeat . i.e: /test auditbeat integTest + # - GitHub label . i.e: integTest or unitTest-arm or unitTest-macos ... + extended: + # NOTE: stage name should be unique! + integTest: + command: "mage build integTest" + platform: "core-ubuntu-2004-aarch64" + integTest-arm: + command: "mage build integTest" + platform: "core-ubuntu-2004-aarch64" + unitTest-arm: + command: "mage build unitTest" + platform: "core-ubuntu-2004-aarch64" + unitTest-macos: + command: "mage build unitTest" + platform: "generic-13-ventura-x64" + unitTest-windows-2019: + command: "mage build unitTest" + platform: "family/core-windows-2019" diff --git a/filebeat/buildkite.yml b/filebeat/buildkite.yml new file mode 100644 index 000000000000..faffac3313e2 --- /dev/null +++ b/filebeat/buildkite.yml @@ -0,0 +1,49 @@ +when: + changeset: ## when PR contains any of those entries in the changeset + - "^filebeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss +stages: + # default stage - it runs always for: + # - branches/tags + # - on PRs + # - GitHub comment /test filebeat + # - GitHub label filebeat + mandatory: + # NOTE: stage name should be unique! + unitTest: + command: "mage build unitTest" + platform: "family/core-ubuntu-2204" + crosscompile: + command: "make -C filebeat crosscompile" + platform: "family/core-ubuntu-2204" + goIntegTest: + command: "mage goIntegTest" + platform: "family/core-ubuntu-2204" + pythonIntegTest: + command: "mage pythonIntegTest" + platform: "family/core-ubuntu-2204" + rhel-9: + command: "mage build unitTest" + platform: "family/core-rhel-9" + unitTest-windows-2022: + command: "mage build unitTest" + platform: "windows-2022" + unitTest-windows-2016: + command: "mage build unitTest" + platform: "family/core-windows-2016" + # optional stage - it runs on: + # - branches/tags + # - on PRs if: + # - GitHub comment /test filebeat . i.e: /test filebeat integTest + # - GitHub label . i.e: integTest or unitTest-arm or unitTest-macos ... + extended: + unitTest-arm: + command: "mage build unitTest" + platform: "core-ubuntu-2004-aarch64" + unitTest-macos: + command: "mage build unitTest" + platform: "generic-13-ventura-x64" + unitTest-windows-2019: + command: "mage build unitTest" + platform: "family/core-windows-2019" From 0d08a6a5ca6460b6466558a6fd6c83003b9699bc Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 4 Mar 2024 09:49:49 +0100 Subject: [PATCH 2/9] for testing purposes --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 34321b61161b..2fd83fb493bf 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -2,4 +2,4 @@ steps: - label: "Example test" - command: echo "Hello!" + command: "python3 .buildkite/pipeline.py | buildkite-agent pipeline upload" From 9548f719dd41883e63a13af7c7de45a25bf9f0d6 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 4 Mar 2024 10:00:38 +0100 Subject: [PATCH 3/9] use intermediate script to run the pipeline generation with the required tools --- .buildkite/pipeline.yml | 4 ++-- .buildkite/scripts/generate_pipeline.sh | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100755 .buildkite/scripts/generate_pipeline.sh diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 2fd83fb493bf..dc6b27176331 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,5 +1,5 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json steps: - - label: "Example test" - command: "python3 .buildkite/pipeline.py | buildkite-agent pipeline upload" + - label: "Test my dynamic pipeline" + command: ".buildkite/scripts/generate_pipeline.sh" diff --git a/.buildkite/scripts/generate_pipeline.sh b/.buildkite/scripts/generate_pipeline.sh new file mode 100755 index 000000000000..4e1d11ba43a1 --- /dev/null +++ b/.buildkite/scripts/generate_pipeline.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Install dependencies +pip3 install --quiet jinja2 +pip3 install --quiet PyYAML + +# Run the python generator - likely this should be called in the +# catalog-info.yaml +python3 .buildkite/pipeline.py | buildkite-agent pipeline upload From 821ef1357efec69d6c9c5be806730d2f8b824f96 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 4 Mar 2024 10:06:25 +0100 Subject: [PATCH 4/9] debug traces --- .buildkite/scripts/generate_pipeline.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.buildkite/scripts/generate_pipeline.sh b/.buildkite/scripts/generate_pipeline.sh index 4e1d11ba43a1..26901680cb1c 100755 --- a/.buildkite/scripts/generate_pipeline.sh +++ b/.buildkite/scripts/generate_pipeline.sh @@ -1,10 +1,14 @@ #!/usr/bin/env bash set -euo pipefail -# Install dependencies +echo "--- Install dependencies" pip3 install --quiet jinja2 pip3 install --quiet PyYAML +echo "--- Run pipeline generator in dry-run mode" +python3 .buildkite/pipeline.py || true + # Run the python generator - likely this should be called in the # catalog-info.yaml +echo "--- Upload pipeline" python3 .buildkite/pipeline.py | buildkite-agent pipeline upload From 05879eec63f21984beef944150b3459c03e013b0 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 4 Mar 2024 10:15:30 +0100 Subject: [PATCH 5/9] fix typo --- .buildkite/pipeline.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index af0e0f048739..af402863397b 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -74,13 +74,13 @@ def create_entity(self): data = """ - label: "{{ stage.project }} {{ stage.name }}" command: - - {{ stage.command }} + - "{{ stage.command }}" notify: - github_commit_status: context: "{{ stage.project }}: {{ stage.name }}" agents: - provider: {{ stage.provider }} - image:{{ stage.runner }} + provider: "{{ stage.provider }}" + image: "{{ stage.runner }}" """ tm = Template(data) From 6b368115d2556dd74b85da652789fbe284e0a505 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 4 Mar 2024 16:19:51 +0100 Subject: [PATCH 6/9] support for PRs, labels and GH commands --- .buildkite/pipeline.py | 48 +++++++++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index af402863397b..93449a1f2c46 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -90,22 +90,54 @@ def create_entity(self): def is_step_enabled(step: Step, conditions) -> bool: # TODO: - # If branch # If PR then - # If GitHub label matches project name - # If GitHub comment # If Changeset - return True + + pull_request = os.getenv('BUILDKITE_PULL_REQUEST') + if pull_request and pull_request == "false": + return True + + comment = os.getenv('GITHUB_PR_TRIGGER_COMMENT') + if comment: + # the comment should be a subset of the values in .buildkite/pull-requests.json + # TODO: change /test + comment_prefix = "buildkite test" + # i.e: test filebeat unitTest + return comment_prefix + " " + step.project + " " + step.name in comment + + labels_env = os.getenv('GITHUB_PR_LABELS') + if labels_env: + labels = labels_env.split() + # i.e: filebeat-unitTest + if step.project + '-' + step.name in labels: + return True + + return False def is_group_enabled(group: Group, conditions) -> bool: # TODO: - # If branch # If PR then - # If GitHub label matches project name + category - # If GitHub comment + # If GitHub label matches project name + category (I'm not sure we wanna use this approach since GH comments support it) # If Changeset - return True + + pull_request = os.getenv('BUILDKITE_PULL_REQUEST') + if pull_request and pull_request == "false": + return True + + comment = os.getenv('GITHUB_PR_TRIGGER_COMMENT') + if comment: + # the comment should be a subset of the values in .buildkite/pull-requests.json + # TODO: change /test + comment_prefix = "buildkite test" + if group.category.startswith("mandatory"): + # i.e: test filebeat + return comment_prefix + " " + group.project in comment + else: + # i.e: test filebeat extended + return comment_prefix + " " + group.project + " " + group.category in comment + + return group.category.startswith("mandatory") def fetch_stage(name: str, stage, project: str) -> Step: From 935f49bd4f678d3e2621831eedc0fda15fdabb48 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 4 Mar 2024 16:36:53 +0100 Subject: [PATCH 7/9] support category in the steps so github comments for mandatory stages work --- .buildkite/pipeline.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 93449a1f2c46..4a5e8da0cfa8 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -60,6 +60,7 @@ class Step: runner: str project: str provider: str + category: str label: str = field(init=False) comment: str = field(init=False) @@ -101,9 +102,12 @@ def is_step_enabled(step: Step, conditions) -> bool: if comment: # the comment should be a subset of the values in .buildkite/pull-requests.json # TODO: change /test - comment_prefix = "buildkite test" - # i.e: test filebeat unitTest - return comment_prefix + " " + step.project + " " + step.name in comment + comment_prefix = "buildkite test " + step.project + # i.e: /test filebeat should run all the mandatory stages + if step.category == "mandatory" and comment_prefix == comment: + return True + # i.e: /test filebeat unitTest + return comment_prefix + " " + step.name in comment labels_env = os.getenv('GITHUB_PR_LABELS') if labels_env: @@ -130,8 +134,8 @@ def is_group_enabled(group: Group, conditions) -> bool: # the comment should be a subset of the values in .buildkite/pull-requests.json # TODO: change /test comment_prefix = "buildkite test" - if group.category.startswith("mandatory"): - # i.e: test filebeat + if group.category == "mandatory": + # i.e: /test filebeat return comment_prefix + " " + group.project in comment else: # i.e: test filebeat extended @@ -140,13 +144,14 @@ def is_group_enabled(group: Group, conditions) -> bool: return group.category.startswith("mandatory") -def fetch_stage(name: str, stage, project: str) -> Step: +def fetch_stage(name: str, stage, project: str, category: str) -> Step: """Create a step given the yaml object.""" # TODO: need to accomodate the provider type. # maybe in the buildkite.yml or some dynamic analysis based on the # name of the runners. return Step( + category=category, command=stage["command"], name=name, runner=stage["platform"], @@ -176,6 +181,7 @@ def main() -> None: mandatory = project_obj["stages"]["mandatory"] for stage in mandatory: step = fetch_stage( + category="mandatory", name=stage, project=project, stage=mandatory[stage]) @@ -199,6 +205,7 @@ def main() -> None: extended = project_obj["stages"]["extended"] for stage in extended: step = fetch_stage( + category="extended", name=stage, project=project, stage=extended[stage]) From 7559617ae685690a13c3fabea46ed51a67af5bf8 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 4 Mar 2024 16:40:52 +0100 Subject: [PATCH 8/9] avoid empty groups --- .buildkite/pipeline.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 4a5e8da0cfa8..b46c44a10308 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -38,12 +38,14 @@ def __lt__(self, other): def create_entity(self): data = """ +{% if group.steps|length > 0 %} - group: "{{ group.project }} {{ group.category }}" key: "{{ group.project }}-{{ group.category }}" steps: {% for step in group.steps|sort -%} {{ step.create_entity() }} {% endfor -%} +{% endif -%} """ tm = Template(data) From fad7d2a36bf38b7a8d0eba18d4cee60984d32a95 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 4 Mar 2024 17:40:00 +0100 Subject: [PATCH 9/9] refactored --- .buildkite/pipeline.py | 70 +++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 38 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index b46c44a10308..9b6c0dd888e7 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -160,6 +160,29 @@ def fetch_stage(name: str, stage, project: str, category: str) -> Step: project=project, provider="gcp") + +def fetch_group(stages, project: str, category: str, conditions) -> Group: + """Create a group given the yaml object.""" + + steps = [] + + for stage in stages: + step = fetch_stage( + category=category, + name=stage, + project=project, + stage=stages[stage]) + + if is_step_enabled(step, conditions): + steps.append(step) + + return Group( + project=project, + category=category, + steps=steps + ) + + # TODO: validate unique stages! def main() -> None: @@ -179,47 +202,18 @@ def main() -> None: steps = [] project_obj = yaml.load(file, yaml.FullLoader) - # Given the mandatory list first - mandatory = project_obj["stages"]["mandatory"] - for stage in mandatory: - step = fetch_stage( - category="mandatory", - name=stage, - project=project, - stage=mandatory[stage]) - - if is_step_enabled(step, conditions): - steps.append(step) - - group = Group( - project=project, - category="mandatory", - steps=steps - ) + group = fetch_group(stages=project_obj["stages"]["mandatory"], + project=project, + category="mandatory", + conditions=conditions) if is_group_enabled(group, conditions): - extended_groups.append(group) + groups.append(group) - # Given the extended list if needed - # TODO: Validate if included - extended_steps = [] - - extended = project_obj["stages"]["extended"] - for stage in extended: - step = fetch_stage( - category="extended", - name=stage, - project=project, - stage=extended[stage]) - - if is_step_enabled(step, conditions): - extended_steps.append(step) - - group = Group( - project=project, - category="extended", - steps=extended_steps - ) + group = fetch_group(stages=project_obj["stages"]["extended"], + project=project, + category="extended", + conditions=conditions) if is_group_enabled(group, conditions): extended_groups.append(group)