From 55e3666502de5dfefc0ecc2229b4b76537a2f228 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 1 Mar 2024 13:53:37 +0100 Subject: [PATCH 01/43] buildkite: python generator --- .buildkite/buildkite.yml | 44 +++++++++ .buildkite/pipeline.py | 200 +++++++++++++++++++++++++++++++++++++++ auditbeat/buildkite.yml | 50 ++++++++++ filebeat/buildkite.yml | 49 ++++++++++ 4 files changed, 343 insertions(+) create mode 100644 .buildkite/buildkite.yml create mode 100755 .buildkite/pipeline.py create mode 100644 auditbeat/buildkite.yml create mode 100644 filebeat/buildkite.yml diff --git a/.buildkite/buildkite.yml b/.buildkite/buildkite.yml new file mode 100644 index 00000000000..4707707e07c --- /dev/null +++ b/.buildkite/buildkite.yml @@ -0,0 +1,44 @@ +projects: + - "auditbeat" + - "deploy/kubernetes" + - "filebeat" + - "heartbeat" + - "libbeat" + - "metricbeat" + - "packetbeat" + - "winlogbeat" + - "x-pack/auditbeat" + - "x-pack/dockerlogbeat" + - "x-pack/filebeat" + - "x-pack/functionbeat" + - "x-pack/heartbeat" + - "x-pack/libbeat" + - "x-pack/metricbeat" + - "x-pack/osquerybeat" + - "x-pack/packetbeat" + - "x-pack/winlogbeat" + +## Changeset macros that are defined here and used in each specific 3.0 pipeline. +changeset: + ci: + - "^Jenkinsfile" + - "^\\.ci/scripts/.*" + oss: + - "^go.mod" + - "^pytest.ini" + - "^dev-tools/.*" + - "^libbeat/.*" + - "^testing/.*" + xpack: + - "^go.mod" + - "^pytest.ini" + - "^dev-tools/.*" + - "^libbeat/.*" + - "^testing/.*" + - "^x-pack/libbeat/.*" + +disabled: + when: + labels: ## Skip the GitHub Pull Request builds if any of the given GitHub labels match with the assigned labels in the PR. + - skip-ci + draft: true ## Skip the GitHub Pull Request builds with Draft PRs. diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py new file mode 100755 index 00000000000..af0e0f04873 --- /dev/null +++ b/.buildkite/pipeline.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +import yaml +import os +from dataclasses import dataclass, field + +from jinja2 import Template +from pathlib import Path + + +@dataclass() +class Pipeline: + """Buildkite Pipeline object""" + groups: list[str] + + def create_entity(self): + data = """ +steps: +{% for group in pipeline.groups -%} +{{ group.create_entity() }} +{% endfor -%} +""" + + tm = Template(data) + msg = tm.render(pipeline=self) + return msg + + +@dataclass(unsafe_hash=True) +class Group: + """Buildkite Group object""" + + project: str + category: str + steps: list[str] + + def __lt__(self, other): + return self.project < other.project + + def create_entity(self): + data = """ + - group: "{{ group.project }} {{ group.category }}" + key: "{{ group.project }}-{{ group.category }}" + steps: + {% for step in group.steps|sort -%} + {{ step.create_entity() }} + {% endfor -%} +""" + + tm = Template(data) + msg = tm.render(group=self) + return msg + + +@dataclass(unsafe_hash=True) +class Step: + """Buildkite Step object""" + + command: str + name: str + runner: str + project: str + provider: str + label: str = field(init=False) + comment: str = field(init=False) + + def __post_init__(self): + self.comment = "/test " + self.project + " " + self.name + self.label = self.name + + def __lt__(self, other): + return self.name < other.name + + def create_entity(self): + data = """ + - label: "{{ stage.project }} {{ stage.name }}" + command: + - {{ stage.command }} + notify: + - github_commit_status: + context: "{{ stage.project }}: {{ stage.name }}" + agents: + provider: {{ stage.provider }} + image:{{ stage.runner }} +""" + + tm = Template(data) + msg = tm.render(stage=self) + return msg + + +def is_step_enabled(step: Step, conditions) -> bool: + # TODO: + # If branch + # If PR then + # If GitHub label matches project name + # If GitHub comment + # If Changeset + return True + + +def is_group_enabled(group: Group, conditions) -> bool: + # TODO: + # If branch + # If PR then + # If GitHub label matches project name + category + # If GitHub comment + # If Changeset + return True + + +def fetch_stage(name: str, stage, project: str) -> Step: + """Create a step given the yaml object.""" + + # TODO: need to accomodate the provider type. + # maybe in the buildkite.yml or some dynamic analysis based on the + # name of the runners. + return Step( + command=stage["command"], + name=name, + runner=stage["platform"], + project=project, + provider="gcp") + +# TODO: validate unique stages! + +def main() -> None: + + groups = [] + extended_groups = [] + with open(".buildkite/buildkite.yml", "r", encoding="utf8") as file: + doc = yaml.load(file, yaml.FullLoader) + + for project in doc["projects"]: + project_file = os.path.join(project, "buildkite.yml") + if not os.path.isfile(project_file): + continue + # TODO: data structure when things run. + conditions = None + with open(project_file, "r", encoding="utf8") as file: + steps = [] + project_obj = yaml.load(file, yaml.FullLoader) + + # Given the mandatory list first + mandatory = project_obj["stages"]["mandatory"] + for stage in mandatory: + step = fetch_stage( + name=stage, + project=project, + stage=mandatory[stage]) + + if is_step_enabled(step, conditions): + steps.append(step) + + group = Group( + project=project, + category="mandatory", + steps=steps + ) + + if is_group_enabled(group, conditions): + extended_groups.append(group) + + # Given the extended list if needed + # TODO: Validate if included + extended_steps = [] + + extended = project_obj["stages"]["extended"] + for stage in extended: + step = fetch_stage( + name=stage, + project=project, + stage=extended[stage]) + + if is_step_enabled(step, conditions): + extended_steps.append(step) + + group = Group( + project=project, + category="extended", + steps=extended_steps + ) + + if is_group_enabled(group, conditions): + extended_groups.append(group) + + # TODO: improve this merging lists + all_groups = [] + for group in sorted(groups): + all_groups.append(group) + for group in sorted(extended_groups): + all_groups.append(group) + + # Produce now the pipeline + print(Pipeline(all_groups).create_entity()) + + +if __name__ == "__main__": + + # pylint: disable=E1120 + main() diff --git a/auditbeat/buildkite.yml b/auditbeat/buildkite.yml new file mode 100644 index 00000000000..c85c176947e --- /dev/null +++ b/auditbeat/buildkite.yml @@ -0,0 +1,50 @@ +when: + changeset: ## when PR contains any of those entries in the changeset + - "^auditbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss +stages: + # mandatory stage - it runs always for: + # - branches/tags + # - on PRs + # - GitHub comment /test auditbeat + # - GitHub label auditbeat + mandatory: + # NOTE: stage name should be unique! + unitTest: + command: "mage build unitTest" + platform: "family/core-ubuntu-2204" + crosscompile: + command: "make -C auditbeat crosscompile" + platform: "family/core-ubuntu-2204" + rhel-9: + command: "mage build unitTest" + platform: "family/core-rhel-9" + unitTest-windows-2022: + command: "mage build unitTest" + platform: "windows-2022" + unitTest-windows-2016: + command: "mage build unitTest" + platform: "family/core-windows-2016" + # optional stage - it runs on: + # - branches/tags + # - on PRs if: + # - GitHub comment /test auditbeat . i.e: /test auditbeat integTest + # - GitHub label . i.e: integTest or unitTest-arm or unitTest-macos ... + extended: + # NOTE: stage name should be unique! + integTest: + command: "mage build integTest" + platform: "core-ubuntu-2004-aarch64" + integTest-arm: + command: "mage build integTest" + platform: "core-ubuntu-2004-aarch64" + unitTest-arm: + command: "mage build unitTest" + platform: "core-ubuntu-2004-aarch64" + unitTest-macos: + command: "mage build unitTest" + platform: "generic-13-ventura-x64" + unitTest-windows-2019: + command: "mage build unitTest" + platform: "family/core-windows-2019" diff --git a/filebeat/buildkite.yml b/filebeat/buildkite.yml new file mode 100644 index 00000000000..faffac3313e --- /dev/null +++ b/filebeat/buildkite.yml @@ -0,0 +1,49 @@ +when: + changeset: ## when PR contains any of those entries in the changeset + - "^filebeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss +stages: + # default stage - it runs always for: + # - branches/tags + # - on PRs + # - GitHub comment /test filebeat + # - GitHub label filebeat + mandatory: + # NOTE: stage name should be unique! + unitTest: + command: "mage build unitTest" + platform: "family/core-ubuntu-2204" + crosscompile: + command: "make -C filebeat crosscompile" + platform: "family/core-ubuntu-2204" + goIntegTest: + command: "mage goIntegTest" + platform: "family/core-ubuntu-2204" + pythonIntegTest: + command: "mage pythonIntegTest" + platform: "family/core-ubuntu-2204" + rhel-9: + command: "mage build unitTest" + platform: "family/core-rhel-9" + unitTest-windows-2022: + command: "mage build unitTest" + platform: "windows-2022" + unitTest-windows-2016: + command: "mage build unitTest" + platform: "family/core-windows-2016" + # optional stage - it runs on: + # - branches/tags + # - on PRs if: + # - GitHub comment /test filebeat . i.e: /test filebeat integTest + # - GitHub label . i.e: integTest or unitTest-arm or unitTest-macos ... + extended: + unitTest-arm: + command: "mage build unitTest" + platform: "core-ubuntu-2004-aarch64" + unitTest-macos: + command: "mage build unitTest" + platform: "generic-13-ventura-x64" + unitTest-windows-2019: + command: "mage build unitTest" + platform: "family/core-windows-2019" From 172c2bbcf9113ee886519a5fb5d8c0f4b7002bc7 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 4 Mar 2024 09:49:49 +0100 Subject: [PATCH 02/43] for testing purposes --- .buildkite/pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 34321b61161..2fd83fb493b 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -2,4 +2,4 @@ steps: - label: "Example test" - command: echo "Hello!" + command: "python3 .buildkite/pipeline.py | buildkite-agent pipeline upload" From 59de509a28ecae58240a530e34f1acd54f531d0e Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 4 Mar 2024 10:00:38 +0100 Subject: [PATCH 03/43] use intermediate script to run the pipeline generation with the required tools --- .buildkite/pipeline.yml | 4 ++-- .buildkite/scripts/generate_pipeline.sh | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100755 .buildkite/scripts/generate_pipeline.sh diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 2fd83fb493b..dc6b2717633 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,5 +1,5 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json steps: - - label: "Example test" - command: "python3 .buildkite/pipeline.py | buildkite-agent pipeline upload" + - label: "Test my dynamic pipeline" + command: ".buildkite/scripts/generate_pipeline.sh" diff --git a/.buildkite/scripts/generate_pipeline.sh b/.buildkite/scripts/generate_pipeline.sh new file mode 100755 index 00000000000..4e1d11ba43a --- /dev/null +++ b/.buildkite/scripts/generate_pipeline.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Install dependencies +pip3 install --quiet jinja2 +pip3 install --quiet PyYAML + +# Run the python generator - likely this should be called in the +# catalog-info.yaml +python3 .buildkite/pipeline.py | buildkite-agent pipeline upload From 306b2ff3ebf17e5a001ae6a1bc4f67d843eb0475 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 4 Mar 2024 10:06:25 +0100 Subject: [PATCH 04/43] debug traces --- .buildkite/scripts/generate_pipeline.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.buildkite/scripts/generate_pipeline.sh b/.buildkite/scripts/generate_pipeline.sh index 4e1d11ba43a..26901680cb1 100755 --- a/.buildkite/scripts/generate_pipeline.sh +++ b/.buildkite/scripts/generate_pipeline.sh @@ -1,10 +1,14 @@ #!/usr/bin/env bash set -euo pipefail -# Install dependencies +echo "--- Install dependencies" pip3 install --quiet jinja2 pip3 install --quiet PyYAML +echo "--- Run pipeline generator in dry-run mode" +python3 .buildkite/pipeline.py || true + # Run the python generator - likely this should be called in the # catalog-info.yaml +echo "--- Upload pipeline" python3 .buildkite/pipeline.py | buildkite-agent pipeline upload From 05fa8c487268f3988ec9922eafe349878ab0b457 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 4 Mar 2024 10:15:30 +0100 Subject: [PATCH 05/43] fix typo --- .buildkite/pipeline.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index af0e0f04873..af402863397 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -74,13 +74,13 @@ def create_entity(self): data = """ - label: "{{ stage.project }} {{ stage.name }}" command: - - {{ stage.command }} + - "{{ stage.command }}" notify: - github_commit_status: context: "{{ stage.project }}: {{ stage.name }}" agents: - provider: {{ stage.provider }} - image:{{ stage.runner }} + provider: "{{ stage.provider }}" + image: "{{ stage.runner }}" """ tm = Template(data) From 340f6d1ff5fe58ba916f05206c24bfab15834a2d Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 4 Mar 2024 16:19:51 +0100 Subject: [PATCH 06/43] support for PRs, labels and GH commands --- .buildkite/pipeline.py | 48 +++++++++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index af402863397..93449a1f2c4 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -90,22 +90,54 @@ def create_entity(self): def is_step_enabled(step: Step, conditions) -> bool: # TODO: - # If branch # If PR then - # If GitHub label matches project name - # If GitHub comment # If Changeset - return True + + pull_request = os.getenv('BUILDKITE_PULL_REQUEST') + if pull_request and pull_request == "false": + return True + + comment = os.getenv('GITHUB_PR_TRIGGER_COMMENT') + if comment: + # the comment should be a subset of the values in .buildkite/pull-requests.json + # TODO: change /test + comment_prefix = "buildkite test" + # i.e: test filebeat unitTest + return comment_prefix + " " + step.project + " " + step.name in comment + + labels_env = os.getenv('GITHUB_PR_LABELS') + if labels_env: + labels = labels_env.split() + # i.e: filebeat-unitTest + if step.project + '-' + step.name in labels: + return True + + return False def is_group_enabled(group: Group, conditions) -> bool: # TODO: - # If branch # If PR then - # If GitHub label matches project name + category - # If GitHub comment + # If GitHub label matches project name + category (I'm not sure we wanna use this approach since GH comments support it) # If Changeset - return True + + pull_request = os.getenv('BUILDKITE_PULL_REQUEST') + if pull_request and pull_request == "false": + return True + + comment = os.getenv('GITHUB_PR_TRIGGER_COMMENT') + if comment: + # the comment should be a subset of the values in .buildkite/pull-requests.json + # TODO: change /test + comment_prefix = "buildkite test" + if group.category.startswith("mandatory"): + # i.e: test filebeat + return comment_prefix + " " + group.project in comment + else: + # i.e: test filebeat extended + return comment_prefix + " " + group.project + " " + group.category in comment + + return group.category.startswith("mandatory") def fetch_stage(name: str, stage, project: str) -> Step: From 4b20d19d120d4ec3a76bc28fdee2c819769c330a Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 4 Mar 2024 16:36:53 +0100 Subject: [PATCH 07/43] support category in the steps so github comments for mandatory stages work --- .buildkite/pipeline.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 93449a1f2c4..4a5e8da0cfa 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -60,6 +60,7 @@ class Step: runner: str project: str provider: str + category: str label: str = field(init=False) comment: str = field(init=False) @@ -101,9 +102,12 @@ def is_step_enabled(step: Step, conditions) -> bool: if comment: # the comment should be a subset of the values in .buildkite/pull-requests.json # TODO: change /test - comment_prefix = "buildkite test" - # i.e: test filebeat unitTest - return comment_prefix + " " + step.project + " " + step.name in comment + comment_prefix = "buildkite test " + step.project + # i.e: /test filebeat should run all the mandatory stages + if step.category == "mandatory" and comment_prefix == comment: + return True + # i.e: /test filebeat unitTest + return comment_prefix + " " + step.name in comment labels_env = os.getenv('GITHUB_PR_LABELS') if labels_env: @@ -130,8 +134,8 @@ def is_group_enabled(group: Group, conditions) -> bool: # the comment should be a subset of the values in .buildkite/pull-requests.json # TODO: change /test comment_prefix = "buildkite test" - if group.category.startswith("mandatory"): - # i.e: test filebeat + if group.category == "mandatory": + # i.e: /test filebeat return comment_prefix + " " + group.project in comment else: # i.e: test filebeat extended @@ -140,13 +144,14 @@ def is_group_enabled(group: Group, conditions) -> bool: return group.category.startswith("mandatory") -def fetch_stage(name: str, stage, project: str) -> Step: +def fetch_stage(name: str, stage, project: str, category: str) -> Step: """Create a step given the yaml object.""" # TODO: need to accomodate the provider type. # maybe in the buildkite.yml or some dynamic analysis based on the # name of the runners. return Step( + category=category, command=stage["command"], name=name, runner=stage["platform"], @@ -176,6 +181,7 @@ def main() -> None: mandatory = project_obj["stages"]["mandatory"] for stage in mandatory: step = fetch_stage( + category="mandatory", name=stage, project=project, stage=mandatory[stage]) @@ -199,6 +205,7 @@ def main() -> None: extended = project_obj["stages"]["extended"] for stage in extended: step = fetch_stage( + category="extended", name=stage, project=project, stage=extended[stage]) From 596890d1d1fe36eaf8d238fa68bd097bdaa3f1cb Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 4 Mar 2024 16:40:52 +0100 Subject: [PATCH 08/43] avoid empty groups --- .buildkite/pipeline.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 4a5e8da0cfa..b46c44a1030 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -38,12 +38,14 @@ def __lt__(self, other): def create_entity(self): data = """ +{% if group.steps|length > 0 %} - group: "{{ group.project }} {{ group.category }}" key: "{{ group.project }}-{{ group.category }}" steps: {% for step in group.steps|sort -%} {{ step.create_entity() }} {% endfor -%} +{% endif -%} """ tm = Template(data) From df55062c782c702092f8f520cd7e17e1de015272 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 4 Mar 2024 17:40:00 +0100 Subject: [PATCH 09/43] refactored --- .buildkite/pipeline.py | 70 +++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 38 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index b46c44a1030..9b6c0dd888e 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -160,6 +160,29 @@ def fetch_stage(name: str, stage, project: str, category: str) -> Step: project=project, provider="gcp") + +def fetch_group(stages, project: str, category: str, conditions) -> Group: + """Create a group given the yaml object.""" + + steps = [] + + for stage in stages: + step = fetch_stage( + category=category, + name=stage, + project=project, + stage=stages[stage]) + + if is_step_enabled(step, conditions): + steps.append(step) + + return Group( + project=project, + category=category, + steps=steps + ) + + # TODO: validate unique stages! def main() -> None: @@ -179,47 +202,18 @@ def main() -> None: steps = [] project_obj = yaml.load(file, yaml.FullLoader) - # Given the mandatory list first - mandatory = project_obj["stages"]["mandatory"] - for stage in mandatory: - step = fetch_stage( - category="mandatory", - name=stage, - project=project, - stage=mandatory[stage]) - - if is_step_enabled(step, conditions): - steps.append(step) - - group = Group( - project=project, - category="mandatory", - steps=steps - ) + group = fetch_group(stages=project_obj["stages"]["mandatory"], + project=project, + category="mandatory", + conditions=conditions) if is_group_enabled(group, conditions): - extended_groups.append(group) + groups.append(group) - # Given the extended list if needed - # TODO: Validate if included - extended_steps = [] - - extended = project_obj["stages"]["extended"] - for stage in extended: - step = fetch_stage( - category="extended", - name=stage, - project=project, - stage=extended[stage]) - - if is_step_enabled(step, conditions): - extended_steps.append(step) - - group = Group( - project=project, - category="extended", - steps=extended_steps - ) + group = fetch_group(stages=project_obj["stages"]["extended"], + project=project, + category="extended", + conditions=conditions) if is_group_enabled(group, conditions): extended_groups.append(group) From 0f4df5d870492267b1194979f55abd68ff0227b4 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 13 Mar 2024 15:15:32 +0100 Subject: [PATCH 10/43] changeset --- .buildkite/pipeline.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 9b6c0dd888e..786dbd5ba83 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -90,16 +90,12 @@ def create_entity(self): msg = tm.render(stage=self) return msg +# Conditions: -def is_step_enabled(step: Step, conditions) -> bool: - # TODO: - # If PR then - # If Changeset - - pull_request = os.getenv('BUILDKITE_PULL_REQUEST') - if pull_request and pull_request == "false": - return True +def is_pr(value) -> bool: + return os.getenv('BUILDKITE_PULL_REQUEST') == value +def step_comment(step: Step) -> bool: comment = os.getenv('GITHUB_PR_TRIGGER_COMMENT') if comment: # the comment should be a subset of the values in .buildkite/pull-requests.json @@ -111,6 +107,21 @@ def is_step_enabled(step: Step, conditions) -> bool: # i.e: /test filebeat unitTest return comment_prefix + " " + step.name in comment +def is_in_changeset() -> bool: + # TODO + return True + +def is_step_enabled(step: Step, conditions) -> bool: + # TODO: + # If PR then + # If Changeset + + if is_pr("false"): + return True + + if step_comment(step): + return True + labels_env = os.getenv('GITHUB_PR_LABELS') if labels_env: labels = labels_env.split() @@ -145,7 +156,6 @@ def is_group_enabled(group: Group, conditions) -> bool: return group.category.startswith("mandatory") - def fetch_stage(name: str, stage, project: str, category: str) -> Step: """Create a step given the yaml object.""" From d006b378f1ebe5fcdd1e8662ef30edb789588e91 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 13 Mar 2024 19:37:06 +0100 Subject: [PATCH 11/43] Changesets --- .buildkite/pipeline.py | 106 +++++++++++++++++++++++----------------- auditbeat/buildkite.yml | 8 +-- filebeat/buildkite.yml | 8 +-- 3 files changed, 68 insertions(+), 54 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 786dbd5ba83..68016e3a0c6 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -2,6 +2,8 @@ import yaml import os from dataclasses import dataclass, field +import subprocess +import fnmatch from jinja2 import Template from pathlib import Path @@ -24,7 +26,6 @@ def create_entity(self): msg = tm.render(pipeline=self) return msg - @dataclass(unsafe_hash=True) class Group: """Buildkite Group object""" @@ -92,8 +93,9 @@ def create_entity(self): # Conditions: -def is_pr(value) -> bool: - return os.getenv('BUILDKITE_PULL_REQUEST') == value +def is_pr() -> bool: + pr = os.getenv('BUILDKITE_PULL_REQUEST') + return pr and os.getenv('BUILDKITE_PULL_REQUEST') != "false" def step_comment(step: Step) -> bool: comment = os.getenv('GITHUB_PR_TRIGGER_COMMENT') @@ -106,17 +108,48 @@ def step_comment(step: Step) -> bool: return True # i.e: /test filebeat unitTest return comment_prefix + " " + step.name in comment + else: + return True + +def group_comment(group: Group) -> bool: + comment = os.getenv('GITHUB_PR_TRIGGER_COMMENT') + if comment: + # the comment should be a subset of the values in .buildkite/pull-requests.json + # TODO: change /test + comment_prefix = "buildkite test" + if group.category == "mandatory": + # i.e: /test filebeat + return comment_prefix + " " + group.project in comment + else: + # i.e: test filebeat extended + return comment_prefix + " " + group.project + " " + group.category in comment + +changed_files = None + +def get_pr_changeset(): + if not changed_files: + base_branch = os.getenv('BUILDKITE_PULL_REQUEST_BASE_BRANCH') + diff_command = ["git", "diff", "--name-only", "{}...HEAD".format(base_branch)] + result = subprocess.run(diff_command, stdout=subprocess.PIPE) + changed_files = result.stdout.decode().splitlines() + print("Changed files: {}".format(changed_files)) + + return changed_files + +def filter_files_by_glob(files, patterns: list[str]): + for pattern in patterns: + ## TODO: Support glob extended patterns: ^ and etc. + ## Now it supports only linux glob patterns + if fnmatch.filter(files, pattern): + return True + return False -def is_in_changeset() -> bool: - # TODO - return True - -def is_step_enabled(step: Step, conditions) -> bool: - # TODO: - # If PR then - # If Changeset +def is_in_pr_changeset(project_changeset_filters: list[str]) -> bool: + changeset = get_pr_changeset() + return filter_files_by_glob(changeset, project_changeset_filters) - if is_pr("false"): +def is_step_enabled(step: Step) -> bool: + if not is_pr(): return True if step_comment(step): @@ -130,31 +163,15 @@ def is_step_enabled(step: Step, conditions) -> bool: return True return False + +def is_group_enabled(group: Group, changeset_filters: list[str]) -> bool: + if not is_pr(): + return True - -def is_group_enabled(group: Group, conditions) -> bool: - # TODO: - # If PR then - # If GitHub label matches project name + category (I'm not sure we wanna use this approach since GH comments support it) - # If Changeset - - pull_request = os.getenv('BUILDKITE_PULL_REQUEST') - if pull_request and pull_request == "false": - return True - - comment = os.getenv('GITHUB_PR_TRIGGER_COMMENT') - if comment: - # the comment should be a subset of the values in .buildkite/pull-requests.json - # TODO: change /test - comment_prefix = "buildkite test" - if group.category == "mandatory": - # i.e: /test filebeat - return comment_prefix + " " + group.project in comment - else: - # i.e: test filebeat extended - return comment_prefix + " " + group.project + " " + group.category in comment - - return group.category.startswith("mandatory") + if is_pr() and is_in_pr_changeset(changeset_filters) and group.category.startswith("mandatory"): + return True + + return group_comment(group) def fetch_stage(name: str, stage, project: str, category: str) -> Step: """Create a step given the yaml object.""" @@ -171,7 +188,7 @@ def fetch_stage(name: str, stage, project: str, category: str) -> Step: provider="gcp") -def fetch_group(stages, project: str, category: str, conditions) -> Group: +def fetch_group(stages, project: str, category: str) -> Group: """Create a group given the yaml object.""" steps = [] @@ -183,14 +200,13 @@ def fetch_group(stages, project: str, category: str, conditions) -> Group: project=project, stage=stages[stage]) - if is_step_enabled(step, conditions): + if is_step_enabled(step): steps.append(step) return Group( project=project, category=category, - steps=steps - ) + steps=steps) # TODO: validate unique stages! @@ -214,18 +230,16 @@ def main() -> None: group = fetch_group(stages=project_obj["stages"]["mandatory"], project=project, - category="mandatory", - conditions=conditions) + category="mandatory") - if is_group_enabled(group, conditions): + if is_group_enabled(group, project_obj["when"]["changeset"]): groups.append(group) group = fetch_group(stages=project_obj["stages"]["extended"], project=project, - category="extended", - conditions=conditions) + category="extended") - if is_group_enabled(group, conditions): + if is_group_enabled(group, project_obj["when"]["changeset"]): extended_groups.append(group) # TODO: improve this merging lists diff --git a/auditbeat/buildkite.yml b/auditbeat/buildkite.yml index c85c176947e..709f50209f1 100644 --- a/auditbeat/buildkite.yml +++ b/auditbeat/buildkite.yml @@ -1,8 +1,8 @@ when: - changeset: ## when PR contains any of those entries in the changeset - - "^auditbeat/.*" - - "@ci" ## special token regarding the changeset for the ci - - "@oss" ## special token regarding the changeset for the oss + changeset: ## when PR contains any of those entries in the changeset + - "auditbeat/**" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss stages: # mandatory stage - it runs always for: # - branches/tags diff --git a/filebeat/buildkite.yml b/filebeat/buildkite.yml index faffac3313e..9e8a2c61356 100644 --- a/filebeat/buildkite.yml +++ b/filebeat/buildkite.yml @@ -1,8 +1,8 @@ when: - changeset: ## when PR contains any of those entries in the changeset - - "^filebeat/.*" - - "@ci" ## special token regarding the changeset for the ci - - "@oss" ## special token regarding the changeset for the oss + changeset: ## when PR contains any of those entries in the changeset + - "filebeat/**" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss stages: # default stage - it runs always for: # - branches/tags From e51daa02052c4437cf65e90016d1325e2df39c1b Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Thu, 14 Mar 2024 23:15:48 +0100 Subject: [PATCH 12/43] fix global variable --- .buildkite/pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 68016e3a0c6..76e24def861 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -127,13 +127,13 @@ def group_comment(group: Group) -> bool: changed_files = None def get_pr_changeset(): + global changed_files if not changed_files: base_branch = os.getenv('BUILDKITE_PULL_REQUEST_BASE_BRANCH') diff_command = ["git", "diff", "--name-only", "{}...HEAD".format(base_branch)] result = subprocess.run(diff_command, stdout=subprocess.PIPE) changed_files = result.stdout.decode().splitlines() - print("Changed files: {}".format(changed_files)) - + print("Changed files: {}".format(changed_files)) return changed_files def filter_files_by_glob(files, patterns: list[str]): From b92ffd764ebbfa403b7db70949ae4ddcd9978ef1 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Thu, 14 Mar 2024 23:57:00 +0100 Subject: [PATCH 13/43] Use beats images --- .buildkite/auditbeat/scripts/unit-tests.sh | 3 +-- auditbeat/buildkite.yml | 20 ++++++++--------- filebeat/buildkite.yml | 26 +++++++++++----------- 3 files changed, 24 insertions(+), 25 deletions(-) diff --git a/.buildkite/auditbeat/scripts/unit-tests.sh b/.buildkite/auditbeat/scripts/unit-tests.sh index c1f5685c77f..a87b2426b56 100755 --- a/.buildkite/auditbeat/scripts/unit-tests.sh +++ b/.buildkite/auditbeat/scripts/unit-tests.sh @@ -7,6 +7,5 @@ source .buildkite/env-scripts/linux-env.sh echo "--- Running Unit Tests" sudo chmod -R go-w auditbeat/ -cd auditbeat umask 0022 -mage build unitTest +mage -d auditbeat build unitTest diff --git a/auditbeat/buildkite.yml b/auditbeat/buildkite.yml index 709f50209f1..2e9e60fa29f 100644 --- a/auditbeat/buildkite.yml +++ b/auditbeat/buildkite.yml @@ -12,20 +12,20 @@ stages: mandatory: # NOTE: stage name should be unique! unitTest: - command: "mage build unitTest" - platform: "family/core-ubuntu-2204" + command: ".buildkite/auditbeat/scripts/unit-tests.sh" + platform: "family/platform-ingest-beats-ubuntu-2204" crosscompile: - command: "make -C auditbeat crosscompile" - platform: "family/core-ubuntu-2204" + command: ".buildkite/auditbeat/scripts/crosscompile.sh" + platform: "family/platform-ingest-beats-ubuntu-2204" rhel-9: - command: "mage build unitTest" - platform: "family/core-rhel-9" + command: ".buildkite/auditbeat/scripts/unit-tests.sh" + platform: "family/platform-ingest-beats-rhel-9" unitTest-windows-2022: - command: "mage build unitTest" - platform: "windows-2022" + command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1" + platform: "family/platform-ingest-beats-windows-2022" unitTest-windows-2016: - command: "mage build unitTest" - platform: "family/core-windows-2016" + command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1" + platform: "family/platform-ingest-beats-windows-2016" # optional stage - it runs on: # - branches/tags # - on PRs if: diff --git a/filebeat/buildkite.yml b/filebeat/buildkite.yml index 9e8a2c61356..4bd0c82ae2a 100644 --- a/filebeat/buildkite.yml +++ b/filebeat/buildkite.yml @@ -12,26 +12,26 @@ stages: mandatory: # NOTE: stage name should be unique! unitTest: - command: "mage build unitTest" - platform: "family/core-ubuntu-2204" + command: "mage -d filebeat build unitTest" + platform: "family/platform-ingest-beats-ubuntu-2204" crosscompile: command: "make -C filebeat crosscompile" - platform: "family/core-ubuntu-2204" + platform: "family/platform-ingest-beats-ubuntu-2204" goIntegTest: - command: "mage goIntegTest" - platform: "family/core-ubuntu-2204" + command: "mage -d filebeat goIntegTest" + platform: "family/platform-ingest-beats-ubuntu-2204" pythonIntegTest: - command: "mage pythonIntegTest" - platform: "family/core-ubuntu-2204" + command: "mage -d filebeat pythonIntegTest" + platform: "family/platform-ingest-beats-ubuntu-2204" rhel-9: - command: "mage build unitTest" - platform: "family/core-rhel-9" + command: "mage -d filebeat build unitTest" + platform: "family/platform-ingest-beats-rhel-9" unitTest-windows-2022: - command: "mage build unitTest" - platform: "windows-2022" + command: "mage -d filebeat build unitTest" + platform: "family/platform-ingest-beats-windows-2022" unitTest-windows-2016: - command: "mage build unitTest" - platform: "family/core-windows-2016" + command: "mage -d filebeat build unitTest" + platform: "family/platform-ingest-beats-windows-2016" # optional stage - it runs on: # - branches/tags # - on PRs if: From 3449137b6e360720aca7c1b0b5998f8f1a78f523 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Fri, 15 Mar 2024 00:18:45 +0100 Subject: [PATCH 14/43] added tool versions and enabled pre-command for beats pipeline --- .buildkite/hooks/pre-command | 2 +- .tool-versions | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 .tool-versions diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 282fb5a0085..215f4d94a8d 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -23,7 +23,7 @@ retry() { return 0 } -if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then source .buildkite/env-scripts/env.sh source .buildkite/env-scripts/util.sh source .buildkite/env-scripts/win-env.sh diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 00000000000..8fecea33ef6 --- /dev/null +++ b/.tool-versions @@ -0,0 +1 @@ +mage 1.15.0 \ No newline at end of file From ab9eecbd88a977c1feb81aa9ce2eed5792d63a43 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Fri, 15 Mar 2024 00:35:38 +0100 Subject: [PATCH 15/43] Use beets VM images for filebeat --- .buildkite/filebeat/scripts/crosscompile.sh | 8 ++++++++ filebeat/buildkite.yml | 22 ++++++++++----------- 2 files changed, 19 insertions(+), 11 deletions(-) create mode 100755 .buildkite/filebeat/scripts/crosscompile.sh diff --git a/.buildkite/filebeat/scripts/crosscompile.sh b/.buildkite/filebeat/scripts/crosscompile.sh new file mode 100755 index 00000000000..d7182458a97 --- /dev/null +++ b/.buildkite/filebeat/scripts/crosscompile.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/linux-env.sh + +echo "--- Executing Crosscompile" +make -C filebeat crosscompile diff --git a/filebeat/buildkite.yml b/filebeat/buildkite.yml index 4bd0c82ae2a..db4d4113152 100644 --- a/filebeat/buildkite.yml +++ b/filebeat/buildkite.yml @@ -12,25 +12,25 @@ stages: mandatory: # NOTE: stage name should be unique! unitTest: - command: "mage -d filebeat build unitTest" + command: ".buildkite/filebeat/scripts/unit-tests.sh" platform: "family/platform-ingest-beats-ubuntu-2204" crosscompile: - command: "make -C filebeat crosscompile" + command: ".buildkite/filebeat/scripts/crosscompile.sh" platform: "family/platform-ingest-beats-ubuntu-2204" goIntegTest: - command: "mage -d filebeat goIntegTest" + command: ".buildkite/filebeat/scripts/integration-gotests.sh" platform: "family/platform-ingest-beats-ubuntu-2204" pythonIntegTest: - command: "mage -d filebeat pythonIntegTest" + command: ".buildkite/filebeat/scripts/integration-pytests.sh" platform: "family/platform-ingest-beats-ubuntu-2204" rhel-9: - command: "mage -d filebeat build unitTest" + command: ".buildkite/filebeat/scripts/unit-tests.sh" platform: "family/platform-ingest-beats-rhel-9" unitTest-windows-2022: - command: "mage -d filebeat build unitTest" + command: ".buildkite/filebeat/scripts/unit-tests-win.ps1" platform: "family/platform-ingest-beats-windows-2022" unitTest-windows-2016: - command: "mage -d filebeat build unitTest" + command: ".buildkite/filebeat/scripts/unit-tests-win.ps1" platform: "family/platform-ingest-beats-windows-2016" # optional stage - it runs on: # - branches/tags @@ -39,11 +39,11 @@ stages: # - GitHub label . i.e: integTest or unitTest-arm or unitTest-macos ... extended: unitTest-arm: - command: "mage build unitTest" + command: ".buildkite/filebeat/scripts/unit-tests.sh" platform: "core-ubuntu-2004-aarch64" unitTest-macos: - command: "mage build unitTest" + command: ".buildkite/filebeat/scripts/unit-tests.sh" platform: "generic-13-ventura-x64" unitTest-windows-2019: - command: "mage build unitTest" - platform: "family/core-windows-2019" + command: ".buildkite/filebeat/scripts/unit-tests-win.ps1" + platform: "family/platform-ingest-beats-windows-2019" From b5cdc4c978d054dc5bcdf9b1ddcf3b50b720eea5 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Fri, 15 Mar 2024 01:33:42 +0100 Subject: [PATCH 16/43] AWS And Orka support --- .buildkite/auditbeat/scripts/integ-test.sh | 11 +++ .buildkite/pipeline.py | 92 ++++++++++++++++++---- .buildkite/pipeline.yml | 2 +- auditbeat/buildkite.yml | 20 +++-- filebeat/buildkite.yml | 4 +- 5 files changed, 103 insertions(+), 26 deletions(-) create mode 100644 .buildkite/auditbeat/scripts/integ-test.sh diff --git a/.buildkite/auditbeat/scripts/integ-test.sh b/.buildkite/auditbeat/scripts/integ-test.sh new file mode 100644 index 00000000000..919bb6420a3 --- /dev/null +++ b/.buildkite/auditbeat/scripts/integ-test.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/linux-env.sh + +echo "--- Executing Integration Tests" +sudo chmod -R go-w auditbeat/ + +umask 0022 +mage -d auditbeat build integTest diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 76e24def861..9311b0e9b23 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -53,16 +53,74 @@ def create_entity(self): msg = tm.render(group=self) return msg +@dataclass(unsafe_hash=True) +class Agent: + """Buildkite Agent object""" + + image: str + + def create_entity(self): + raise NotImplementedError("Not implemented yet") + +@dataclass(unsafe_hash=True) +class AWSAgent(Agent): + """AWS Agent object""" + + image: str + + def create_entity(self): + data = """ + agents: + provider: "aws" + imagePrefix: "{{ agent.image }}" + instanceType: "t4g.large" +""" + + tm = Template(data) + msg = tm.render(agent=self) + return msg + +@dataclass(unsafe_hash=True) +class GCPAgent(Agent): + """GCP Agent object""" + + image: str + + def create_entity(self): + data = """ + agents: + provider: "gcp" + image: "{{ agent.image }}" +""" + + tm = Template(data) + msg = tm.render(agent=self) + return msg +@dataclass(unsafe_hash=True) +class OrkaAgent(Agent): + """Orka Agent object""" + + image: str + + def create_entity(self): + data = """ + agents: + provider: "orka" + imagePrefix: "{{ agent.image }}" +""" + + tm = Template(data) + msg = tm.render(agent=self) + return msg @dataclass(unsafe_hash=True) class Step: """Buildkite Step object""" command: str + agent: Agent name: str - runner: str project: str - provider: str category: str label: str = field(init=False) comment: str = field(init=False) @@ -76,21 +134,20 @@ def __lt__(self, other): def create_entity(self): data = """ - - label: "{{ stage.project }} {{ stage.name }}" + - label: "{{ step.project }} {{ step.name }}" command: - - "{{ stage.command }}" + - "{{ step.command }}" notify: - github_commit_status: - context: "{{ stage.project }}: {{ stage.name }}" - agents: - provider: "{{ stage.provider }}" - image: "{{ stage.runner }}" + context: "{{ step.project }}: {{ step.name }}" + {{ step.agent.create_entity() }} """ tm = Template(data) - msg = tm.render(stage=self) + msg = tm.render(step=self) return msg + # Conditions: def is_pr() -> bool: @@ -176,17 +233,20 @@ def is_group_enabled(group: Group, changeset_filters: list[str]) -> bool: def fetch_stage(name: str, stage, project: str, category: str) -> Step: """Create a step given the yaml object.""" - # TODO: need to accomodate the provider type. - # maybe in the buildkite.yml or some dynamic analysis based on the - # name of the runners. + agent: Agent = None + if (not "provider" in stage) or stage["provider"] == "gcp": + agent = GCPAgent(image=stage["platform"]) + elif stage["provider"] == "aws": + agent = AWSAgent(image=stage["platform"]) + elif stage["provider"] == "orka": + agent = OrkaAgent(image=stage["platform"]) + return Step( category=category, command=stage["command"], name=name, - runner=stage["platform"], - project=project, - provider="gcp") - + agent=agent, + project=project) def fetch_group(stages, project: str, category: str) -> Group: """Create a group given the yaml object.""" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index dc6b2717633..d968ea3a304 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,5 +1,5 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json steps: - - label: "Test my dynamic pipeline" + - label: "Generate dynamic pipeline" command: ".buildkite/scripts/generate_pipeline.sh" diff --git a/auditbeat/buildkite.yml b/auditbeat/buildkite.yml index 2e9e60fa29f..220b4401fe6 100644 --- a/auditbeat/buildkite.yml +++ b/auditbeat/buildkite.yml @@ -34,17 +34,21 @@ stages: extended: # NOTE: stage name should be unique! integTest: - command: "mage build integTest" - platform: "core-ubuntu-2004-aarch64" + command: ".buildkite/auditbeat/scripts/integ-test.sh" + platform: "platform-ingest-beats-ubuntu-2204-aarch64" + provider: "aws" integTest-arm: - command: "mage build integTest" - platform: "core-ubuntu-2004-aarch64" + command: ".buildkite/auditbeat/scripts/integ-test.sh" + platform: "platform-ingest-beats-ubuntu-2204-aarch64" + provider: "aws" unitTest-arm: - command: "mage build unitTest" - platform: "core-ubuntu-2004-aarch64" + command: ".buildkite/auditbeat/scripts/unit-tests.sh" + platform: "platform-ingest-beats-ubuntu-2204-aarch64" + provider: "aws" unitTest-macos: - command: "mage build unitTest" + command: ".buildkite/auditbeat/scripts/unit-tests.sh" platform: "generic-13-ventura-x64" + provider: "orka" unitTest-windows-2019: - command: "mage build unitTest" + command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1" platform: "family/core-windows-2019" diff --git a/filebeat/buildkite.yml b/filebeat/buildkite.yml index db4d4113152..ca50e7dc378 100644 --- a/filebeat/buildkite.yml +++ b/filebeat/buildkite.yml @@ -40,10 +40,12 @@ stages: extended: unitTest-arm: command: ".buildkite/filebeat/scripts/unit-tests.sh" - platform: "core-ubuntu-2004-aarch64" + platform: "platform-ingest-beats-ubuntu-2204-aarch64" + provider: "aws" unitTest-macos: command: ".buildkite/filebeat/scripts/unit-tests.sh" platform: "generic-13-ventura-x64" + provider: "orka" unitTest-windows-2019: command: ".buildkite/filebeat/scripts/unit-tests-win.ps1" platform: "family/platform-ingest-beats-windows-2019" From 2835ba18ffacdf80fcf8f15799fe86b50143a9a2 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Fri, 15 Mar 2024 10:08:33 +0100 Subject: [PATCH 17/43] Update auditbeat/buildkite.yml --- auditbeat/buildkite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auditbeat/buildkite.yml b/auditbeat/buildkite.yml index 220b4401fe6..f074cf4ce24 100644 --- a/auditbeat/buildkite.yml +++ b/auditbeat/buildkite.yml @@ -17,7 +17,7 @@ stages: crosscompile: command: ".buildkite/auditbeat/scripts/crosscompile.sh" platform: "family/platform-ingest-beats-ubuntu-2204" - rhel-9: + unitTest-rhel-9: command: ".buildkite/auditbeat/scripts/unit-tests.sh" platform: "family/platform-ingest-beats-rhel-9" unitTest-windows-2022: From 9c42231bd145e94f2a07df17b2f0e9393cb79086 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Mon, 18 Mar 2024 13:15:28 +0100 Subject: [PATCH 18/43] Use native yaml marshalling --- .buildkite/pipeline.py | 233 +++++++++++------------- .buildkite/scripts/generate_pipeline.sh | 8 +- 2 files changed, 114 insertions(+), 127 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 9311b0e9b23..836a2306a71 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -1,117 +1,63 @@ #!/usr/bin/env python3 +from typing import Any import yaml import os from dataclasses import dataclass, field import subprocess import fnmatch -from jinja2 import Template -from pathlib import Path - - -@dataclass() -class Pipeline: - """Buildkite Pipeline object""" - groups: list[str] - - def create_entity(self): - data = """ -steps: -{% for group in pipeline.groups -%} -{{ group.create_entity() }} -{% endfor -%} -""" - - tm = Template(data) - msg = tm.render(pipeline=self) - return msg - -@dataclass(unsafe_hash=True) -class Group: - """Buildkite Group object""" - - project: str - category: str - steps: list[str] - - def __lt__(self, other): - return self.project < other.project - - def create_entity(self): - data = """ -{% if group.steps|length > 0 %} - - group: "{{ group.project }} {{ group.category }}" - key: "{{ group.project }}-{{ group.category }}" - steps: - {% for step in group.steps|sort -%} - {{ step.create_entity() }} - {% endfor -%} -{% endif -%} -""" - - tm = Template(data) - msg = tm.render(group=self) - return msg @dataclass(unsafe_hash=True) class Agent: """Buildkite Agent object""" - image: str def create_entity(self): raise NotImplementedError("Not implemented yet") - + + @dataclass(unsafe_hash=True) class AWSAgent(Agent): """AWS Agent object""" - - image: str + image: str - def create_entity(self): - data = """ - agents: - provider: "aws" - imagePrefix: "{{ agent.image }}" - instanceType: "t4g.large" -""" + def create_entity(self) -> dict[str, dict[str, str]]: + return { + "agents": { + "provider": "aws", + "imagePrefix": self.image, + "instanceType": "t4g.large", + } + } - tm = Template(data) - msg = tm.render(agent=self) - return msg @dataclass(unsafe_hash=True) class GCPAgent(Agent): """GCP Agent object""" - - image: str + image: str + + def create_entity(self) -> dict[str, dict[str, str]]: + return { + "agents": { + "provider": "gcp", + "image": self.image, + } + } + - def create_entity(self): - data = """ - agents: - provider: "gcp" - image: "{{ agent.image }}" -""" - - tm = Template(data) - msg = tm.render(agent=self) - return msg @dataclass(unsafe_hash=True) class OrkaAgent(Agent): """Orka Agent object""" - - image: str + image: str - def create_entity(self): - data = """ - agents: - provider: "orka" - imagePrefix: "{{ agent.image }}" -""" + def create_entity(self) -> dict[str, dict[str, str]]: + return { + "agents": { + "provider": "orka", + "imagePrefix": self.image, + } + } - tm = Template(data) - msg = tm.render(agent=self) - return msg @dataclass(unsafe_hash=True) class Step: @@ -132,32 +78,61 @@ def __post_init__(self): def __lt__(self, other): return self.name < other.name - def create_entity(self): - data = """ - - label: "{{ step.project }} {{ step.name }}" - command: - - "{{ step.command }}" - notify: - - github_commit_status: - context: "{{ step.project }}: {{ step.name }}" - {{ step.agent.create_entity() }} -""" + def create_entity(self) -> dict[str, Any]: + data = { + "label": f"{self.project} {self.name}", + "command": [self.command], + "notify": [{ + "github_commit_status": { + "context": f"{self.project}: {self.name}", + } + }], + } + data.update(self.agent.create_entity()) + return data - tm = Template(data) - msg = tm.render(step=self) - return msg +@dataclass(unsafe_hash=True) +class Group: + """Buildkite Group object""" + + project: str + category: str + steps: list[Step] + + def __lt__(self, other): + return self.project < other.project + + def create_entity(self) -> dict[str, Any] | None: + if len(self.steps) == 0: + return + + data = { + "group": f"{self.project} {self.category}", + "key": f"{self.project}-{self.category}", + "steps": [step.create_entity() for step in sorted(self.steps)], + } + + return data + + +@dataclass() +class Pipeline: + """Buildkite Pipeline object""" + groups: list[Group] + + def create_entity(self): + data = {"steps": [group.create_entity() for group in self.groups]} + return data -# Conditions: def is_pr() -> bool: - pr = os.getenv('BUILDKITE_PULL_REQUEST') - return pr and os.getenv('BUILDKITE_PULL_REQUEST') != "false" + return os.getenv('BUILDKITE_PULL_REQUEST') != "false" + def step_comment(step: Step) -> bool: comment = os.getenv('GITHUB_PR_TRIGGER_COMMENT') if comment: - # the comment should be a subset of the values in .buildkite/pull-requests.json # TODO: change /test comment_prefix = "buildkite test " + step.project # i.e: /test filebeat should run all the mandatory stages @@ -167,11 +142,13 @@ def step_comment(step: Step) -> bool: return comment_prefix + " " + step.name in comment else: return True - + + def group_comment(group: Group) -> bool: comment = os.getenv('GITHUB_PR_TRIGGER_COMMENT') if comment: - # the comment should be a subset of the values in .buildkite/pull-requests.json + # the comment should be a subset of the values + # in .buildkite/pull-requests.json # TODO: change /test comment_prefix = "buildkite test" if group.category == "mandatory": @@ -179,36 +156,46 @@ def group_comment(group: Group) -> bool: return comment_prefix + " " + group.project in comment else: # i.e: test filebeat extended - return comment_prefix + " " + group.project + " " + group.category in comment + return ( + comment_prefix + " " + group.project + " " + group.category + in comment + ) + changed_files = None + def get_pr_changeset(): global changed_files if not changed_files: base_branch = os.getenv('BUILDKITE_PULL_REQUEST_BASE_BRANCH') - diff_command = ["git", "diff", "--name-only", "{}...HEAD".format(base_branch)] + diff_command = [ + "git", "diff", "--name-only", "{}...HEAD".format(base_branch) + ] result = subprocess.run(diff_command, stdout=subprocess.PIPE) changed_files = result.stdout.decode().splitlines() - print("Changed files: {}".format(changed_files)) + print("Changed files: {}".format(changed_files)) return changed_files + def filter_files_by_glob(files, patterns: list[str]): for pattern in patterns: - ## TODO: Support glob extended patterns: ^ and etc. - ## Now it supports only linux glob patterns + # TODO: Support glob extended patterns: ^ and etc. + # Now it supports only linux glob patterns if fnmatch.filter(files, pattern): return True return False + def is_in_pr_changeset(project_changeset_filters: list[str]) -> bool: changeset = get_pr_changeset() return filter_files_by_glob(changeset, project_changeset_filters) -def is_step_enabled(step: Step) -> bool: + +def is_step_enabled(step: Step) -> bool: if not is_pr(): return True - + if step_comment(step): return True @@ -220,27 +207,30 @@ def is_step_enabled(step: Step) -> bool: return True return False - + + def is_group_enabled(group: Group, changeset_filters: list[str]) -> bool: if not is_pr(): - return True + return True + + if is_pr() and is_in_pr_changeset(changeset_filters) and \ + group.category.startswith("mandatory"): + return True - if is_pr() and is_in_pr_changeset(changeset_filters) and group.category.startswith("mandatory"): - return True - return group_comment(group) + def fetch_stage(name: str, stage, project: str, category: str) -> Step: """Create a step given the yaml object.""" agent: Agent = None - if (not "provider" in stage) or stage["provider"] == "gcp": + if ("provider" not in stage) or stage["provider"] == "gcp": agent = GCPAgent(image=stage["platform"]) elif stage["provider"] == "aws": agent = AWSAgent(image=stage["platform"]) elif stage["provider"] == "orka": agent = OrkaAgent(image=stage["platform"]) - + return Step( category=category, command=stage["command"], @@ -248,6 +238,7 @@ def fetch_stage(name: str, stage, project: str, category: str) -> Step: agent=agent, project=project) + def fetch_group(stages, project: str, category: str) -> Group: """Create a group given the yaml object.""" @@ -270,7 +261,6 @@ def fetch_group(stages, project: str, category: str) -> Group: # TODO: validate unique stages! - def main() -> None: groups = [] @@ -282,10 +272,7 @@ def main() -> None: project_file = os.path.join(project, "buildkite.yml") if not os.path.isfile(project_file): continue - # TODO: data structure when things run. - conditions = None with open(project_file, "r", encoding="utf8") as file: - steps = [] project_obj = yaml.load(file, yaml.FullLoader) group = fetch_group(stages=project_obj["stages"]["mandatory"], @@ -299,7 +286,7 @@ def main() -> None: project=project, category="extended") - if is_group_enabled(group, project_obj["when"]["changeset"]): + if is_group_enabled(group, project_obj["when"]["changeset"]): extended_groups.append(group) # TODO: improve this merging lists @@ -310,7 +297,7 @@ def main() -> None: all_groups.append(group) # Produce now the pipeline - print(Pipeline(all_groups).create_entity()) + print(yaml.dump(Pipeline(all_groups).create_entity())) if __name__ == "__main__": diff --git a/.buildkite/scripts/generate_pipeline.sh b/.buildkite/scripts/generate_pipeline.sh index 26901680cb1..3f14a913e37 100755 --- a/.buildkite/scripts/generate_pipeline.sh +++ b/.buildkite/scripts/generate_pipeline.sh @@ -2,13 +2,13 @@ set -euo pipefail echo "--- Install dependencies" -pip3 install --quiet jinja2 pip3 install --quiet PyYAML echo "--- Run pipeline generator in dry-run mode" -python3 .buildkite/pipeline.py || true +python3 .buildkite/pipeline.py | yq . # Run the python generator - likely this should be called in the # catalog-info.yaml -echo "--- Upload pipeline" -python3 .buildkite/pipeline.py | buildkite-agent pipeline upload +# echo "--- Upload pipeline" +## Remove when is ready +# python3 .buildkite/pipeline.py | buildkite-agent pipeline upload From 99dffd7a4456f6a09ffe828882223f7df1aa9c19 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Mon, 18 Mar 2024 16:34:40 +0100 Subject: [PATCH 19/43] fix image --- .buildkite/pipeline.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index d968ea3a304..c4bbcbb3c7c 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -3,3 +3,6 @@ steps: - label: "Generate dynamic pipeline" command: ".buildkite/scripts/generate_pipeline.sh" + agents: + provider: "gcp" + image: "family/platform-ingest-beats-ubuntu-2204" From 9dc603a6775e52f904e7a658eb7de9a26792232a Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Tue, 19 Mar 2024 13:26:13 +0100 Subject: [PATCH 20/43] comments pipeline generation: /test ... --- .buildkite/pipeline.py | 168 +++++++++++++++++++++++++++++++---------- 1 file changed, 127 insertions(+), 41 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 836a2306a71..6f389ac58b0 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -130,20 +130,6 @@ def is_pr() -> bool: return os.getenv('BUILDKITE_PULL_REQUEST') != "false" -def step_comment(step: Step) -> bool: - comment = os.getenv('GITHUB_PR_TRIGGER_COMMENT') - if comment: - # TODO: change /test - comment_prefix = "buildkite test " + step.project - # i.e: /test filebeat should run all the mandatory stages - if step.category == "mandatory" and comment_prefix == comment: - return True - # i.e: /test filebeat unitTest - return comment_prefix + " " + step.name in comment - else: - return True - - def group_comment(group: Group) -> bool: comment = os.getenv('GITHUB_PR_TRIGGER_COMMENT') if comment: @@ -181,7 +167,7 @@ def get_pr_changeset(): def filter_files_by_glob(files, patterns: list[str]): for pattern in patterns: # TODO: Support glob extended patterns: ^ and etc. - # Now it supports only linux glob patterns + # Now it supports only linux glob syntax if fnmatch.filter(files, pattern): return True return False @@ -192,23 +178,6 @@ def is_in_pr_changeset(project_changeset_filters: list[str]) -> bool: return filter_files_by_glob(changeset, project_changeset_filters) -def is_step_enabled(step: Step) -> bool: - if not is_pr(): - return True - - if step_comment(step): - return True - - labels_env = os.getenv('GITHUB_PR_LABELS') - if labels_env: - labels = labels_env.split() - # i.e: filebeat-unitTest - if step.project + '-' + step.name in labels: - return True - - return False - - def is_group_enabled(group: Group, changeset_filters: list[str]) -> bool: if not is_pr(): return True @@ -245,14 +214,11 @@ def fetch_group(stages, project: str, category: str) -> Group: steps = [] for stage in stages: - step = fetch_stage( + steps.append(fetch_stage( category=category, name=stage, project=project, - stage=stages[stage]) - - if is_step_enabled(step): - steps.append(step) + stage=stages[stage])) return Group( project=project, @@ -260,9 +226,7 @@ def fetch_group(stages, project: str, category: str) -> Group: steps=steps) -# TODO: validate unique stages! -def main() -> None: - +def fetch_pr_pipeline() -> list[Group]: groups = [] extended_groups = [] with open(".buildkite/buildkite.yml", "r", encoding="utf8") as file: @@ -286,7 +250,7 @@ def main() -> None: project=project, category="extended") - if is_group_enabled(group, project_obj["when"]["changeset"]): + if is_group_enabled(group, project_obj["when"]["changeset"]): extended_groups.append(group) # TODO: improve this merging lists @@ -296,6 +260,128 @@ def main() -> None: for group in sorted(extended_groups): all_groups.append(group) + return all_groups + + +class PRComment: + command: str + group: str + project: str + step: str + + def __init__(self, comment: str): + words = comment.split() + self.command = words.pop(0) if words else "" + self.project = words.pop(0) if words else "" + self.group = words.pop(0) if words else "" + self.step = words.pop(0) if words else "" + + +# A comment like "/test filebeat extended" +# Returns a group of steps corresponding to the comment +def fetch_pr_comment_group_pipeline(comment: PRComment) -> list[Group]: + groups = [] + with open(".buildkite/buildkite.yml", "r", encoding="utf8") as file: + doc = yaml.load(file, yaml.FullLoader) + if comment.project in doc["projects"]: + project_file = os.path.join(comment.project, "buildkite.yml") + + if not os.path.isfile(project_file): + raise FileNotFoundError("buildkite.yml not found in: " + + "{}".format(comment.project)) + with open(project_file, "r", encoding="utf8") as file: + project_obj = yaml.load(file, yaml.FullLoader) + + if not project_obj["stages"][comment.group]: + raise ValueError("Group not found in {} buildkike.yml: {}" + .format(comment.project, comment.group)) + + group = fetch_group( + stages=project_obj["stages"][comment.group], + project=comment.project, + category="mandatory" + ) + groups.append(group) + + return groups + + +# A comment like "/test filebeat extended unitTest-macos" +def fetch_pr_comment_step_pipeline(comment: PRComment) -> list[Group]: + groups = [] + with open(".buildkite/buildkite.yml", "r", encoding="utf8") as file: + doc = yaml.load(file, yaml.FullLoader) + if comment.project in doc["projects"]: + project_file = os.path.join(comment.project, "buildkite.yml") + + if not os.path.isfile(project_file): + raise FileNotFoundError("buildkite.yml not found in: " + + "{}".format(comment.project)) + with open(project_file, "r", encoding="utf8") as file: + project_obj = yaml.load(file, yaml.FullLoader) + + if not project_obj["stages"][comment.group]: + raise ValueError("Group not found in {} buildkike.yml: {}" + .format(comment.project, comment.group)) + + group = fetch_group( + stages=project_obj["stages"][comment.group], + project=comment.project, + category="mandatory" + ) + + filtered_steps = list(filter( + lambda step: step.name == comment.step, + group.steps + )) + + if not filtered_steps: + raise ValueError("Step {} not found in {} buildkike.yml" + .format(comment.step, comment.project)) + group.steps = filtered_steps + groups.append(group) + + return groups + + +def pr_comment_pipeline(pr_comment: PRComment) -> list[Group]: + + if pr_comment.command == "/test": + + # A comment like "/test" for a PR + # We rerun the PR pipeline + if not pr_comment.group: + return fetch_pr_pipeline() + + # A comment like "/test filebeat" + # We don't know what group to run hence raise an error + if pr_comment.project and not pr_comment.group: + raise ValueError( + "Specify group or/and step for {}".format(pr_comment.project) + ) + + # A comment like "/test filebeat extended" + # We rerun the filebeat extended pipeline for the PR + if pr_comment.group and not pr_comment.step: + return fetch_pr_comment_group_pipeline(pr_comment) + + # A comment like "/test filebeat extended unitTest-macos" + if pr_comment.step: + return fetch_pr_comment_step_pipeline(pr_comment) + + +# TODO: validate unique stages! +def main() -> None: + all_groups = [] + if is_pr() and not os.getenv('GITHUB_PR_TRIGGER_COMMENT'): + all_groups = fetch_pr_pipeline() + + if is_pr() and os.getenv('GITHUB_PR_TRIGGER_COMMENT'): + print("GITHUB_PR_TRIGGER_COMMENT: {}".format( + os.getenv('GITHUB_PR_TRIGGER_COMMENT'))) + comment = PRComment(os.getenv('GITHUB_PR_TRIGGER_COMMENT')) + all_groups = pr_comment_pipeline(comment) + # Produce now the pipeline print(yaml.dump(Pipeline(all_groups).create_entity())) From a74a5b684b6fd8a8f94f64d9ce04e87b641723dc Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 19 Mar 2024 15:54:30 +0200 Subject: [PATCH 21/43] Cleanup - Replace yaml with Ruamel - Respect order from the blueprint of the pipeline - Remove dataclasses Signed-off-by: Alexandros Sapranidis --- .buildkite/pipeline.py | 308 +++++++++++++----------- .buildkite/scripts/generate_pipeline.sh | 10 +- 2 files changed, 176 insertions(+), 142 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 6f389ac58b0..91827247c2c 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -1,77 +1,78 @@ #!/usr/bin/env python3 from typing import Any -import yaml +from ruamel.yaml import YAML import os -from dataclasses import dataclass, field import subprocess import fnmatch +import sys -@dataclass(unsafe_hash=True) class Agent: """Buildkite Agent object""" - image: str + + def __init__(self, image: str, provider: str): + self.image: str = image + self.provider: str = provider def create_entity(self): raise NotImplementedError("Not implemented yet") -@dataclass(unsafe_hash=True) class AWSAgent(Agent): """AWS Agent object""" - image: str - def create_entity(self) -> dict[str, dict[str, str]]: + def __init__(self, image: str, instance_type: str = None): + super().__init__(image, "aws") + if instance_type is None: + self.instance_type: str = "t4g.large" + else: + self.instance_type = instance_type + + def create_entity(self) -> dict[str, str]: return { - "agents": { - "provider": "aws", - "imagePrefix": self.image, - "instanceType": "t4g.large", - } + "provider": self.provider, + "imagePrefix": self.image, + "instanceType": self.instance_type, } -@dataclass(unsafe_hash=True) class GCPAgent(Agent): """GCP Agent object""" - image: str - def create_entity(self) -> dict[str, dict[str, str]]: + def __init__(self, image: str): + super().__init__(image, "gcp") + + def create_entity(self) -> dict[str, str]: return { - "agents": { - "provider": "gcp", - "image": self.image, - } + "provider": self.provider, + "image": self.image, } -@dataclass(unsafe_hash=True) class OrkaAgent(Agent): """Orka Agent object""" - image: str - def create_entity(self) -> dict[str, dict[str, str]]: + def __init__(self, image: str): + super().__init__(image, "orka") + + def create_entity(self) -> dict[str, str]: return { - "agents": { - "provider": "orka", - "imagePrefix": self.image, - } + "provider": self.provider, + "imagePrefix": self.image, } -@dataclass(unsafe_hash=True) class Step: """Buildkite Step object""" - command: str - agent: Agent - name: str - project: str - category: str - label: str = field(init=False) - comment: str = field(init=False) - - def __post_init__(self): + def __init__( + self, name: str, command: str, project: str, category: str, agent: Agent + ): + self.command: str = command + self.agent: Agent = agent + self.name: str = name + self.project: str = project + self.category: str = category self.comment = "/test " + self.project + " " + self.name self.label = self.name @@ -82,44 +83,65 @@ def create_entity(self) -> dict[str, Any]: data = { "label": f"{self.project} {self.name}", "command": [self.command], - "notify": [{ - "github_commit_status": { - "context": f"{self.project}: {self.name}", + "notify": [ + { + "github_commit_status": { + "context": f"{self.project}: {self.name}", + } } - }], + ], + "agents": self.agent.create_entity(), } - data.update(self.agent.create_entity()) return data -@dataclass(unsafe_hash=True) class Group: """Buildkite Group object""" - project: str - category: str - steps: list[Step] + def __init__(self, project: str, category: str, steps: list[Step]): + self.project: str = project + self.category: str = category + self.steps: list[Step] = steps def __lt__(self, other): return self.project < other.project - def create_entity(self) -> dict[str, Any] | None: + def create_entity(self) -> dict[str, Any]: if len(self.steps) == 0: - return + return {} data = { "group": f"{self.project} {self.category}", "key": f"{self.project}-{self.category}", - "steps": [step.create_entity() for step in sorted(self.steps)], + "steps": [step.create_entity() for step in self.steps], } return data -@dataclass() -class Pipeline: +class GitHelper: + def __init__(self): + self.files: list[str] = [] + + def get_pr_changeset(self) -> list[str]: + base_branch = os.getenv("BUILDKITE_PULL_REQUEST_BASE_BRANCH", "main") + diff_command = ["git", "diff", "--name-only", "{}...HEAD".format(base_branch)] + result = subprocess.run(diff_command, stdout=subprocess.PIPE) + if result.returncode == 0: + self.files = result.stdout.decode().splitlines() + else: + print(f"Detecting changed files failed, exiting [{result.returncode}]") + exit(result.returncode) + return self.files + + +class BuildkitePipeline: """Buildkite Pipeline object""" - groups: list[Group] + + def __init__(self, groups: list[Group] = None): + if groups is None: + groups = [] + self.groups: list[Group] = groups def create_entity(self): data = {"steps": [group.create_entity() for group in self.groups]} @@ -127,11 +149,11 @@ def create_entity(self): def is_pr() -> bool: - return os.getenv('BUILDKITE_PULL_REQUEST') != "false" + return os.getenv("BUILDKITE_PULL_REQUEST") != "false" def group_comment(group: Group) -> bool: - comment = os.getenv('GITHUB_PR_TRIGGER_COMMENT') + comment = os.getenv("GITHUB_PR_TRIGGER_COMMENT") if comment: # the comment should be a subset of the values # in .buildkite/pull-requests.json @@ -143,27 +165,10 @@ def group_comment(group: Group) -> bool: else: # i.e: test filebeat extended return ( - comment_prefix + " " + group.project + " " + group.category - in comment + comment_prefix + " " + group.project + " " + group.category in comment ) -changed_files = None - - -def get_pr_changeset(): - global changed_files - if not changed_files: - base_branch = os.getenv('BUILDKITE_PULL_REQUEST_BASE_BRANCH') - diff_command = [ - "git", "diff", "--name-only", "{}...HEAD".format(base_branch) - ] - result = subprocess.run(diff_command, stdout=subprocess.PIPE) - changed_files = result.stdout.decode().splitlines() - print("Changed files: {}".format(changed_files)) - return changed_files - - def filter_files_by_glob(files, patterns: list[str]): for pattern in patterns: # TODO: Support glob extended patterns: ^ and etc. @@ -173,17 +178,23 @@ def filter_files_by_glob(files, patterns: list[str]): return False -def is_in_pr_changeset(project_changeset_filters: list[str]) -> bool: - changeset = get_pr_changeset() +def is_in_pr_changeset( + project_changeset_filters: list[str], changeset: list[str] +) -> bool: return filter_files_by_glob(changeset, project_changeset_filters) -def is_group_enabled(group: Group, changeset_filters: list[str]) -> bool: +def is_group_enabled( + group: Group, changeset_filters: list[str], changeset: list[str] +) -> bool: if not is_pr(): return True - if is_pr() and is_in_pr_changeset(changeset_filters) and \ - group.category.startswith("mandatory"): + if ( + is_pr() + and is_in_pr_changeset(changeset_filters, changeset) + and group.category.startswith("mandatory") + ): return True return group_comment(group) @@ -196,16 +207,19 @@ def fetch_stage(name: str, stage, project: str, category: str) -> Step: if ("provider" not in stage) or stage["provider"] == "gcp": agent = GCPAgent(image=stage["platform"]) elif stage["provider"] == "aws": - agent = AWSAgent(image=stage["platform"]) + agent = AWSAgent( + image=stage["platform"], + ) elif stage["provider"] == "orka": agent = OrkaAgent(image=stage["platform"]) return Step( - category=category, - command=stage["command"], - name=name, - agent=agent, - project=project) + category=category, + command=stage["command"], + name=name, + agent=agent, + project=project, + ) def fetch_group(stages, project: str, category: str) -> Group: @@ -214,50 +228,51 @@ def fetch_group(stages, project: str, category: str) -> Group: steps = [] for stage in stages: - steps.append(fetch_stage( - category=category, - name=stage, - project=project, - stage=stages[stage])) + steps.append( + fetch_stage( + category=category, name=stage, project=project, stage=stages[stage] + ) + ) - return Group( - project=project, - category=category, - steps=steps) + return Group(project=project, category=category, steps=steps) def fetch_pr_pipeline() -> list[Group]: - groups = [] - extended_groups = [] - with open(".buildkite/buildkite.yml", "r", encoding="utf8") as file: - doc = yaml.load(file, yaml.FullLoader) + groups: list[Group] = [] + git_helper = GitHelper() + changeset = git_helper.get_pr_changeset() + yaml = YAML(typ="safe") + with open(".buildkite/buildkite.yml", "r", encoding="utf8") as file: + doc = yaml.load(file) for project in doc["projects"]: project_file = os.path.join(project, "buildkite.yml") if not os.path.isfile(project_file): continue - with open(project_file, "r", encoding="utf8") as file: - project_obj = yaml.load(file, yaml.FullLoader) + with open(project_file, "r", encoding="utf8") as project_fp: + project_obj = yaml.load(project_fp) - group = fetch_group(stages=project_obj["stages"]["mandatory"], - project=project, - category="mandatory") + group = fetch_group( + stages=project_obj["stages"]["mandatory"], + project=project, + category="mandatory", + ) - if is_group_enabled(group, project_obj["when"]["changeset"]): + if is_group_enabled(group, project_obj["when"]["changeset"], changeset): groups.append(group) - group = fetch_group(stages=project_obj["stages"]["extended"], - project=project, - category="extended") + group = fetch_group( + stages=project_obj["stages"]["extended"], + project=project, + category="extended", + ) - if is_group_enabled(group, project_obj["when"]["changeset"]): - extended_groups.append(group) + if is_group_enabled(group, project_obj["when"]["changeset"], changeset): + groups.append(group) # TODO: improve this merging lists all_groups = [] - for group in sorted(groups): - all_groups.append(group) - for group in sorted(extended_groups): + for group in groups: all_groups.append(group) return all_groups @@ -281,25 +296,30 @@ def __init__(self, comment: str): # Returns a group of steps corresponding to the comment def fetch_pr_comment_group_pipeline(comment: PRComment) -> list[Group]: groups = [] + yaml = YAML(typ="safe") with open(".buildkite/buildkite.yml", "r", encoding="utf8") as file: - doc = yaml.load(file, yaml.FullLoader) + doc = yaml.load(file) if comment.project in doc["projects"]: project_file = os.path.join(comment.project, "buildkite.yml") if not os.path.isfile(project_file): - raise FileNotFoundError("buildkite.yml not found in: " + - "{}".format(comment.project)) - with open(project_file, "r", encoding="utf8") as file: - project_obj = yaml.load(file, yaml.FullLoader) + raise FileNotFoundError( + "buildkite.yml not found in: " + "{}".format(comment.project) + ) + with open(project_file, "r", encoding="utf8") as project_fp: + project_obj = yaml.load(project_fp) if not project_obj["stages"][comment.group]: - raise ValueError("Group not found in {} buildkike.yml: {}" - .format(comment.project, comment.group)) + raise ValueError( + "Group not found in {} buildkike.yml: {}".format( + comment.project, comment.group + ) + ) group = fetch_group( stages=project_obj["stages"][comment.group], project=comment.project, - category="mandatory" + category="mandatory", ) groups.append(group) @@ -309,35 +329,42 @@ def fetch_pr_comment_group_pipeline(comment: PRComment) -> list[Group]: # A comment like "/test filebeat extended unitTest-macos" def fetch_pr_comment_step_pipeline(comment: PRComment) -> list[Group]: groups = [] + yaml = YAML(typ="safe") with open(".buildkite/buildkite.yml", "r", encoding="utf8") as file: - doc = yaml.load(file, yaml.FullLoader) + doc = yaml.load(file) if comment.project in doc["projects"]: project_file = os.path.join(comment.project, "buildkite.yml") if not os.path.isfile(project_file): - raise FileNotFoundError("buildkite.yml not found in: " + - "{}".format(comment.project)) - with open(project_file, "r", encoding="utf8") as file: - project_obj = yaml.load(file, yaml.FullLoader) + raise FileNotFoundError( + "buildkite.yml not found in: " + "{}".format(comment.project) + ) + with open(project_file, "r", encoding="utf8") as project_fp: + project_obj = yaml.load(project_fp) if not project_obj["stages"][comment.group]: - raise ValueError("Group not found in {} buildkike.yml: {}" - .format(comment.project, comment.group)) + raise ValueError( + "Group not found in {} buildkike.yml: {}".format( + comment.project, comment.group + ) + ) group = fetch_group( stages=project_obj["stages"][comment.group], project=comment.project, - category="mandatory" + category="mandatory", ) - filtered_steps = list(filter( - lambda step: step.name == comment.step, - group.steps - )) + filtered_steps = list( + filter(lambda step: step.name == comment.step, group.steps) + ) if not filtered_steps: - raise ValueError("Step {} not found in {} buildkike.yml" - .format(comment.step, comment.project)) + raise ValueError( + "Step {} not found in {} buildkike.yml".format( + comment.step, comment.project + ) + ) group.steps = filtered_steps groups.append(group) @@ -373,17 +400,24 @@ def pr_comment_pipeline(pr_comment: PRComment) -> list[Group]: # TODO: validate unique stages! def main() -> None: all_groups = [] - if is_pr() and not os.getenv('GITHUB_PR_TRIGGER_COMMENT'): + if is_pr() and not os.getenv("GITHUB_PR_TRIGGER_COMMENT"): all_groups = fetch_pr_pipeline() - if is_pr() and os.getenv('GITHUB_PR_TRIGGER_COMMENT'): - print("GITHUB_PR_TRIGGER_COMMENT: {}".format( - os.getenv('GITHUB_PR_TRIGGER_COMMENT'))) - comment = PRComment(os.getenv('GITHUB_PR_TRIGGER_COMMENT')) + if is_pr() and os.getenv("GITHUB_PR_TRIGGER_COMMENT"): + print( + "GITHUB_PR_TRIGGER_COMMENT: {}".format( + os.getenv("GITHUB_PR_TRIGGER_COMMENT") + ) + ) + comment = PRComment(os.getenv("GITHUB_PR_TRIGGER_COMMENT")) all_groups = pr_comment_pipeline(comment) # Produce now the pipeline - print(yaml.dump(Pipeline(all_groups).create_entity())) + print( + "# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json" + ) + yaml = YAML(typ="safe") + yaml.dump(BuildkitePipeline(all_groups).create_entity(), sys.stdout) if __name__ == "__main__": diff --git a/.buildkite/scripts/generate_pipeline.sh b/.buildkite/scripts/generate_pipeline.sh index 3f14a913e37..2a1cc3b1235 100755 --- a/.buildkite/scripts/generate_pipeline.sh +++ b/.buildkite/scripts/generate_pipeline.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash set -euo pipefail -echo "--- Install dependencies" -pip3 install --quiet PyYAML +echo "~~~ Install dependencies" +pip3 install --quiet "ruamel.yaml<0.18.0" -echo "--- Run pipeline generator in dry-run mode" +echo "~~~ Run pipeline generator in dry-run mode" python3 .buildkite/pipeline.py | yq . # Run the python generator - likely this should be called in the # catalog-info.yaml -# echo "--- Upload pipeline" +echo "~~~ Upload pipeline" ## Remove when is ready -# python3 .buildkite/pipeline.py | buildkite-agent pipeline upload +python3 .buildkite/pipeline.py | buildkite-agent pipeline upload From 921ac8a4f30ae26e7e7171aabd46ba708e2ffb66 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 19 Mar 2024 16:08:34 +0200 Subject: [PATCH 22/43] Fix typo Signed-off-by: Alexandros Sapranidis --- .buildkite/pipeline.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 91827247c2c..67d36b607ce 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -311,7 +311,7 @@ def fetch_pr_comment_group_pipeline(comment: PRComment) -> list[Group]: if not project_obj["stages"][comment.group]: raise ValueError( - "Group not found in {} buildkike.yml: {}".format( + "Group not found in {} buildkite.yml: {}".format( comment.project, comment.group ) ) @@ -344,7 +344,7 @@ def fetch_pr_comment_step_pipeline(comment: PRComment) -> list[Group]: if not project_obj["stages"][comment.group]: raise ValueError( - "Group not found in {} buildkike.yml: {}".format( + "Group not found in {} buildkite.yml: {}".format( comment.project, comment.group ) ) @@ -361,7 +361,7 @@ def fetch_pr_comment_step_pipeline(comment: PRComment) -> list[Group]: if not filtered_steps: raise ValueError( - "Step {} not found in {} buildkike.yml".format( + "Step {} not found in {} buildkite.yml".format( comment.step, comment.project ) ) From b15fc29f01c51e7274d7bef87a6be11c2041a0fe Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 19 Mar 2024 16:12:48 +0200 Subject: [PATCH 23/43] Use mage commands inside the steps Signed-off-by: Alexandros Sapranidis --- auditbeat/buildkite.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/auditbeat/buildkite.yml b/auditbeat/buildkite.yml index f074cf4ce24..937373ad8bd 100644 --- a/auditbeat/buildkite.yml +++ b/auditbeat/buildkite.yml @@ -12,10 +12,10 @@ stages: mandatory: # NOTE: stage name should be unique! unitTest: - command: ".buildkite/auditbeat/scripts/unit-tests.sh" + command: "mage -d auditbeat build unitTest" platform: "family/platform-ingest-beats-ubuntu-2204" crosscompile: - command: ".buildkite/auditbeat/scripts/crosscompile.sh" + command: "make -C auditbeat crosscompile" platform: "family/platform-ingest-beats-ubuntu-2204" unitTest-rhel-9: command: ".buildkite/auditbeat/scripts/unit-tests.sh" From a52814227c0947919aaf6635f2966197cba9e26e Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 19 Mar 2024 16:34:14 +0200 Subject: [PATCH 24/43] Create artifacts_path Signed-off-by: Alexandros Sapranidis --- .buildkite/pipeline.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 67d36b607ce..3f909494920 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -91,6 +91,9 @@ def create_entity(self) -> dict[str, Any]: } ], "agents": self.agent.create_entity(), + "artifact_paths": [ + f"{self.project}/build/*.xml" f"{self.project}/build/*.json" + ], } return data From 8b0b92ada2e943e07b8aa8673318b7f67a30e864 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 19 Mar 2024 16:39:46 +0200 Subject: [PATCH 25/43] Remove more wrapper scripts Signed-off-by: Alexandros Sapranidis --- auditbeat/buildkite.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/auditbeat/buildkite.yml b/auditbeat/buildkite.yml index 937373ad8bd..8c6d0ed76f6 100644 --- a/auditbeat/buildkite.yml +++ b/auditbeat/buildkite.yml @@ -12,19 +12,19 @@ stages: mandatory: # NOTE: stage name should be unique! unitTest: - command: "mage -d auditbeat build unitTest" + command: "mage auditbeat build unitTest" platform: "family/platform-ingest-beats-ubuntu-2204" crosscompile: command: "make -C auditbeat crosscompile" platform: "family/platform-ingest-beats-ubuntu-2204" unitTest-rhel-9: - command: ".buildkite/auditbeat/scripts/unit-tests.sh" + command: "mage auditbeat build unitTest" platform: "family/platform-ingest-beats-rhel-9" unitTest-windows-2022: - command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1" + command: "mage build unitTest" platform: "family/platform-ingest-beats-windows-2022" unitTest-windows-2016: - command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1" + command: "mage build unitTest" platform: "family/platform-ingest-beats-windows-2016" # optional stage - it runs on: # - branches/tags From 79d7a486df2971987a3329e8a818b172dcd00333 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 19 Mar 2024 16:42:03 +0200 Subject: [PATCH 26/43] fixup! Remove more wrapper scripts --- auditbeat/buildkite.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/auditbeat/buildkite.yml b/auditbeat/buildkite.yml index 8c6d0ed76f6..32019bcd38c 100644 --- a/auditbeat/buildkite.yml +++ b/auditbeat/buildkite.yml @@ -12,13 +12,13 @@ stages: mandatory: # NOTE: stage name should be unique! unitTest: - command: "mage auditbeat build unitTest" + command: "mage -d auditbeat build unitTest" platform: "family/platform-ingest-beats-ubuntu-2204" crosscompile: command: "make -C auditbeat crosscompile" platform: "family/platform-ingest-beats-ubuntu-2204" unitTest-rhel-9: - command: "mage auditbeat build unitTest" + command: "mage -d auditbeat build unitTest" platform: "family/platform-ingest-beats-rhel-9" unitTest-windows-2022: command: "mage build unitTest" From 049404a9d40a1cb165c207140e1857844167eae3 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 19 Mar 2024 17:12:14 +0200 Subject: [PATCH 27/43] Cleanup and removed doublicate code Signed-off-by: Alexandros Sapranidis --- .buildkite/pipeline.py | 188 ++++++++++++++++++++--------------------- 1 file changed, 91 insertions(+), 97 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 3f909494920..a45174f82b9 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -240,38 +240,33 @@ def fetch_group(stages, project: str, category: str) -> Group: return Group(project=project, category=category, steps=steps) -def fetch_pr_pipeline() -> list[Group]: - groups: list[Group] = [] - +def fetch_pr_pipeline(yaml: YAML) -> list[Group]: git_helper = GitHelper() changeset = git_helper.get_pr_changeset() - yaml = YAML(typ="safe") - with open(".buildkite/buildkite.yml", "r", encoding="utf8") as file: - doc = yaml.load(file) - for project in doc["projects"]: - project_file = os.path.join(project, "buildkite.yml") - if not os.path.isfile(project_file): - continue - with open(project_file, "r", encoding="utf8") as project_fp: - project_obj = yaml.load(project_fp) - - group = fetch_group( - stages=project_obj["stages"]["mandatory"], - project=project, - category="mandatory", - ) + groups: list[Group] = [] + doc = pipeline_loader(yaml) + for project in doc["projects"]: + project_file = os.path.join(project, "buildkite.yml") + if not os.path.isfile(project_file): + continue + project_obj = project_loader(yaml, project_file) + group = fetch_group( + stages=project_obj["stages"]["mandatory"], + project=project, + category="mandatory", + ) - if is_group_enabled(group, project_obj["when"]["changeset"], changeset): - groups.append(group) + if is_group_enabled(group, project_obj["when"]["changeset"], changeset): + groups.append(group) - group = fetch_group( - stages=project_obj["stages"]["extended"], - project=project, - category="extended", - ) + group = fetch_group( + stages=project_obj["stages"]["extended"], + project=project, + category="extended", + ) - if is_group_enabled(group, project_obj["when"]["changeset"], changeset): - groups.append(group) + if is_group_enabled(group, project_obj["when"]["changeset"], changeset): + groups.append(group) # TODO: improve this merging lists all_groups = [] @@ -297,91 +292,80 @@ def __init__(self, comment: str): # A comment like "/test filebeat extended" # Returns a group of steps corresponding to the comment -def fetch_pr_comment_group_pipeline(comment: PRComment) -> list[Group]: +def fetch_pr_comment_group_pipeline(comment: PRComment, yaml: YAML) -> list[Group]: groups = [] - yaml = YAML(typ="safe") - with open(".buildkite/buildkite.yml", "r", encoding="utf8") as file: - doc = yaml.load(file) - if comment.project in doc["projects"]: - project_file = os.path.join(comment.project, "buildkite.yml") - - if not os.path.isfile(project_file): - raise FileNotFoundError( - "buildkite.yml not found in: " + "{}".format(comment.project) - ) - with open(project_file, "r", encoding="utf8") as project_fp: - project_obj = yaml.load(project_fp) - - if not project_obj["stages"][comment.group]: - raise ValueError( - "Group not found in {} buildkite.yml: {}".format( - comment.project, comment.group - ) - ) - - group = fetch_group( - stages=project_obj["stages"][comment.group], - project=comment.project, - category="mandatory", + doc = pipeline_loader(yaml) + if comment.project in doc["projects"]: + project_file = os.path.join(comment.project, "buildkite.yml") + if not os.path.isfile(project_file): + raise FileNotFoundError( + "buildkite.yml not found in: " + "{}".format(comment.project) + ) + project_obj = project_loader(yaml, project_file) + if not project_obj["stages"][comment.group]: + raise ValueError( + "Group not found in {} buildkite.yml: {}".format( + comment.project, comment.group ) - groups.append(group) + ) + + group = fetch_group( + stages=project_obj["stages"][comment.group], + project=comment.project, + category="mandatory", + ) + groups.append(group) return groups # A comment like "/test filebeat extended unitTest-macos" -def fetch_pr_comment_step_pipeline(comment: PRComment) -> list[Group]: +def fetch_pr_comment_step_pipeline(comment: PRComment, yaml: YAML) -> list[Group]: groups = [] - yaml = YAML(typ="safe") - with open(".buildkite/buildkite.yml", "r", encoding="utf8") as file: - doc = yaml.load(file) - if comment.project in doc["projects"]: - project_file = os.path.join(comment.project, "buildkite.yml") - - if not os.path.isfile(project_file): - raise FileNotFoundError( - "buildkite.yml not found in: " + "{}".format(comment.project) - ) - with open(project_file, "r", encoding="utf8") as project_fp: - project_obj = yaml.load(project_fp) - - if not project_obj["stages"][comment.group]: - raise ValueError( - "Group not found in {} buildkite.yml: {}".format( - comment.project, comment.group - ) - ) - - group = fetch_group( - stages=project_obj["stages"][comment.group], - project=comment.project, - category="mandatory", + doc = pipeline_loader(yaml) + if comment.project in doc["projects"]: + project_file = os.path.join(comment.project, "buildkite.yml") + if not os.path.isfile(project_file): + raise FileNotFoundError( + "buildkite.yml not found in: " + "{}".format(comment.project) + ) + project_obj = project_loader(yaml, project_file) + if not project_obj["stages"][comment.group]: + raise ValueError( + "Group not found in {} buildkite.yml: {}".format( + comment.project, comment.group ) + ) + group = fetch_group( + stages=project_obj["stages"][comment.group], + project=comment.project, + category="mandatory", + ) - filtered_steps = list( - filter(lambda step: step.name == comment.step, group.steps) - ) + filtered_steps = list( + filter(lambda step: step.name == comment.step, group.steps) + ) - if not filtered_steps: - raise ValueError( - "Step {} not found in {} buildkite.yml".format( - comment.step, comment.project - ) - ) - group.steps = filtered_steps - groups.append(group) + if not filtered_steps: + raise ValueError( + "Step {} not found in {} buildkite.yml".format( + comment.step, comment.project + ) + ) + group.steps = filtered_steps + groups.append(group) - return groups + return groups -def pr_comment_pipeline(pr_comment: PRComment) -> list[Group]: +def pr_comment_pipeline(pr_comment: PRComment, yaml: YAML) -> list[Group]: if pr_comment.command == "/test": # A comment like "/test" for a PR # We rerun the PR pipeline if not pr_comment.group: - return fetch_pr_pipeline() + return fetch_pr_pipeline(yaml) # A comment like "/test filebeat" # We don't know what group to run hence raise an error @@ -393,15 +377,16 @@ def pr_comment_pipeline(pr_comment: PRComment) -> list[Group]: # A comment like "/test filebeat extended" # We rerun the filebeat extended pipeline for the PR if pr_comment.group and not pr_comment.step: - return fetch_pr_comment_group_pipeline(pr_comment) + return fetch_pr_comment_group_pipeline(pr_comment, yaml) # A comment like "/test filebeat extended unitTest-macos" if pr_comment.step: - return fetch_pr_comment_step_pipeline(pr_comment) + return fetch_pr_comment_step_pipeline(pr_comment, yaml) # TODO: validate unique stages! def main() -> None: + yaml = YAML(typ="safe") all_groups = [] if is_pr() and not os.getenv("GITHUB_PR_TRIGGER_COMMENT"): all_groups = fetch_pr_pipeline() @@ -413,16 +398,25 @@ def main() -> None: ) ) comment = PRComment(os.getenv("GITHUB_PR_TRIGGER_COMMENT")) - all_groups = pr_comment_pipeline(comment) + all_groups = pr_comment_pipeline(comment, yaml) - # Produce now the pipeline + # Produce the dynamic pipeline print( "# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json" ) - yaml = YAML(typ="safe") yaml.dump(BuildkitePipeline(all_groups).create_entity(), sys.stdout) +def pipeline_loader(yaml: YAML = YAML(typ="safe")): + with open(".buildkite/buildkite.yml", "r", encoding="utf8") as file: + return yaml.load(file) + + +def project_loader(yaml: YAML = YAML(typ="safe"), project_file: str = ""): + with open(project_file, "r", encoding="utf8") as project_fp: + return yaml.load(project_fp) + + if __name__ == "__main__": # pylint: disable=E1120 From 239f60afd4885d5b18d423b410e46d42e4994978 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 19 Mar 2024 17:17:00 +0200 Subject: [PATCH 28/43] Add missing parameter Signed-off-by: Alexandros Sapranidis --- .buildkite/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index a45174f82b9..624171f403d 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -389,7 +389,7 @@ def main() -> None: yaml = YAML(typ="safe") all_groups = [] if is_pr() and not os.getenv("GITHUB_PR_TRIGGER_COMMENT"): - all_groups = fetch_pr_pipeline() + all_groups = fetch_pr_pipeline(yaml) if is_pr() and os.getenv("GITHUB_PR_TRIGGER_COMMENT"): print( From d49d8c0674d288d60cb2aa0f11496143261c1f71 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 19 Mar 2024 17:21:28 +0200 Subject: [PATCH 29/43] fixup! Create artifacts_path --- .buildkite/pipeline.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 624171f403d..2ff21dc2ed1 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -92,7 +92,8 @@ def create_entity(self) -> dict[str, Any]: ], "agents": self.agent.create_entity(), "artifact_paths": [ - f"{self.project}/build/*.xml" f"{self.project}/build/*.json" + f"{self.project}/build/*.xml", + f"{self.project}/build/*.json" ], } return data From c9e50c4abd34c4d7c025aa18470e33c71806e3ec Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 19 Mar 2024 17:29:15 +0200 Subject: [PATCH 30/43] Remove print this is going to break yaml parsing, because we pipe this scripts output into buildkite-agent Signed-off-by: Alexandros Sapranidis --- .buildkite/pipeline.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 2ff21dc2ed1..5f932a8eafe 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -393,11 +393,6 @@ def main() -> None: all_groups = fetch_pr_pipeline(yaml) if is_pr() and os.getenv("GITHUB_PR_TRIGGER_COMMENT"): - print( - "GITHUB_PR_TRIGGER_COMMENT: {}".format( - os.getenv("GITHUB_PR_TRIGGER_COMMENT") - ) - ) comment = PRComment(os.getenv("GITHUB_PR_TRIGGER_COMMENT")) all_groups = pr_comment_pipeline(comment, yaml) From 6c52899b2bbd1b49d7d6155fa93e980899007ef2 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 19 Mar 2024 17:32:59 +0200 Subject: [PATCH 31/43] Set magefile version as env var Signed-off-by: Alexandros Sapranidis --- .buildkite/pipeline.yml | 2 ++ .tool-versions | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) delete mode 100644 .tool-versions diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index c4bbcbb3c7c..8f124fea4ae 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,4 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +env: + ASDF_MAGE_VERSION: '1.15.0' steps: - label: "Generate dynamic pipeline" diff --git a/.tool-versions b/.tool-versions deleted file mode 100644 index 8fecea33ef6..00000000000 --- a/.tool-versions +++ /dev/null @@ -1 +0,0 @@ -mage 1.15.0 \ No newline at end of file From 9ef55b216a4ad5104f466fcd59e9d71260778964 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 19 Mar 2024 17:36:42 +0200 Subject: [PATCH 32/43] Remove the usage of wrapper scripts Signed-off-by: Alexandros Sapranidis --- auditbeat/buildkite.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/auditbeat/buildkite.yml b/auditbeat/buildkite.yml index 32019bcd38c..d3dea9c7a56 100644 --- a/auditbeat/buildkite.yml +++ b/auditbeat/buildkite.yml @@ -34,21 +34,21 @@ stages: extended: # NOTE: stage name should be unique! integTest: - command: ".buildkite/auditbeat/scripts/integ-test.sh" + command: "mage -d auditbeat build integTest" platform: "platform-ingest-beats-ubuntu-2204-aarch64" provider: "aws" integTest-arm: - command: ".buildkite/auditbeat/scripts/integ-test.sh" + command: "mage -d auditbeat build integTest" platform: "platform-ingest-beats-ubuntu-2204-aarch64" provider: "aws" unitTest-arm: - command: ".buildkite/auditbeat/scripts/unit-tests.sh" + command: "mage -d auditbeat build unitTest" platform: "platform-ingest-beats-ubuntu-2204-aarch64" provider: "aws" unitTest-macos: - command: ".buildkite/auditbeat/scripts/unit-tests.sh" + command: "mage -d auditbeat build unitTest" platform: "generic-13-ventura-x64" provider: "orka" unitTest-windows-2019: - command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1" + command: "mage build unitTest" platform: "family/core-windows-2019" From bec043b86048e2f096c23abd20cb4a4dbfe6babc Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 19 Mar 2024 17:40:49 +0200 Subject: [PATCH 33/43] Remove more wrapper scripts Signed-off-by: Alexandros Sapranidis --- .buildkite/auditbeat/scripts/integ-test.sh | 11 ----------- filebeat/buildkite.yml | 14 +++++++------- 2 files changed, 7 insertions(+), 18 deletions(-) delete mode 100644 .buildkite/auditbeat/scripts/integ-test.sh diff --git a/.buildkite/auditbeat/scripts/integ-test.sh b/.buildkite/auditbeat/scripts/integ-test.sh deleted file mode 100644 index 919bb6420a3..00000000000 --- a/.buildkite/auditbeat/scripts/integ-test.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -source .buildkite/env-scripts/linux-env.sh - -echo "--- Executing Integration Tests" -sudo chmod -R go-w auditbeat/ - -umask 0022 -mage -d auditbeat build integTest diff --git a/filebeat/buildkite.yml b/filebeat/buildkite.yml index ca50e7dc378..f17cb2b394c 100644 --- a/filebeat/buildkite.yml +++ b/filebeat/buildkite.yml @@ -12,25 +12,25 @@ stages: mandatory: # NOTE: stage name should be unique! unitTest: - command: ".buildkite/filebeat/scripts/unit-tests.sh" + command: "mage -d filebeat unitTest" platform: "family/platform-ingest-beats-ubuntu-2204" crosscompile: - command: ".buildkite/filebeat/scripts/crosscompile.sh" + command: "make -C filebeat crosscompile" platform: "family/platform-ingest-beats-ubuntu-2204" goIntegTest: - command: ".buildkite/filebeat/scripts/integration-gotests.sh" + command: "mage -d filebeat goIntegTest" platform: "family/platform-ingest-beats-ubuntu-2204" pythonIntegTest: - command: ".buildkite/filebeat/scripts/integration-pytests.sh" + command: "mage -d filebeat pythonIntegTest" platform: "family/platform-ingest-beats-ubuntu-2204" rhel-9: - command: ".buildkite/filebeat/scripts/unit-tests.sh" + command: "mage -d filebeat unitTest" platform: "family/platform-ingest-beats-rhel-9" unitTest-windows-2022: - command: ".buildkite/filebeat/scripts/unit-tests-win.ps1" + command: "mage build unitTest" platform: "family/platform-ingest-beats-windows-2022" unitTest-windows-2016: - command: ".buildkite/filebeat/scripts/unit-tests-win.ps1" + command: "mage build unitTest" platform: "family/platform-ingest-beats-windows-2016" # optional stage - it runs on: # - branches/tags From 2cb2431efe2ef76eb264bf19f7fff3e33e7605a5 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 19 Mar 2024 17:47:10 +0200 Subject: [PATCH 34/43] Remove filebeat crosscompile wrapper Signed-off-by: Alexandros Sapranidis --- .buildkite/filebeat/scripts/crosscompile.sh | 8 -------- 1 file changed, 8 deletions(-) delete mode 100755 .buildkite/filebeat/scripts/crosscompile.sh diff --git a/.buildkite/filebeat/scripts/crosscompile.sh b/.buildkite/filebeat/scripts/crosscompile.sh deleted file mode 100755 index d7182458a97..00000000000 --- a/.buildkite/filebeat/scripts/crosscompile.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -source .buildkite/env-scripts/linux-env.sh - -echo "--- Executing Crosscompile" -make -C filebeat crosscompile From 5c20f456b5d918e4a05d1e86e58539033c5eba79 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 19 Mar 2024 17:50:34 +0200 Subject: [PATCH 35/43] Rework the logical if when loading the projects Signed-off-by: Alexandros Sapranidis --- .buildkite/pipeline.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 5f932a8eafe..8988cde09cd 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -389,12 +389,13 @@ def pr_comment_pipeline(pr_comment: PRComment, yaml: YAML) -> list[Group]: def main() -> None: yaml = YAML(typ="safe") all_groups = [] - if is_pr() and not os.getenv("GITHUB_PR_TRIGGER_COMMENT"): - all_groups = fetch_pr_pipeline(yaml) - - if is_pr() and os.getenv("GITHUB_PR_TRIGGER_COMMENT"): - comment = PRComment(os.getenv("GITHUB_PR_TRIGGER_COMMENT")) - all_groups = pr_comment_pipeline(comment, yaml) + if is_pr(): + if os.getenv("GITHUB_PR_TRIGGER_COMMENT"): + comment = PRComment(os.getenv("GITHUB_PR_TRIGGER_COMMENT")) + all_groups = pr_comment_pipeline(comment, yaml) + else: + all_groups = fetch_pr_pipeline(yaml) + # TODO what to load when not in PR # Produce the dynamic pipeline print( From 996d322b2439670db554709c046c14eb0b92055c Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 19 Mar 2024 17:52:22 +0200 Subject: [PATCH 36/43] Remove comments Signed-off-by: Alexandros Sapranidis --- .buildkite/scripts/generate_pipeline.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/.buildkite/scripts/generate_pipeline.sh b/.buildkite/scripts/generate_pipeline.sh index 2a1cc3b1235..fd495579f81 100755 --- a/.buildkite/scripts/generate_pipeline.sh +++ b/.buildkite/scripts/generate_pipeline.sh @@ -7,8 +7,5 @@ pip3 install --quiet "ruamel.yaml<0.18.0" echo "~~~ Run pipeline generator in dry-run mode" python3 .buildkite/pipeline.py | yq . -# Run the python generator - likely this should be called in the -# catalog-info.yaml echo "~~~ Upload pipeline" -## Remove when is ready python3 .buildkite/pipeline.py | buildkite-agent pipeline upload From 1dcce414d3526029475190789d6901b7d77f5225 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 19 Mar 2024 17:58:16 +0200 Subject: [PATCH 37/43] Show pipeline in expanded mode The pipeline that this script is running is only creating a dynamic pipeline and thats why its ok to have it by default show expanded output Signed-off-by: Alexandros Sapranidis --- .buildkite/scripts/generate_pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/generate_pipeline.sh b/.buildkite/scripts/generate_pipeline.sh index fd495579f81..342dc8bf7ac 100755 --- a/.buildkite/scripts/generate_pipeline.sh +++ b/.buildkite/scripts/generate_pipeline.sh @@ -4,7 +4,7 @@ set -euo pipefail echo "~~~ Install dependencies" pip3 install --quiet "ruamel.yaml<0.18.0" -echo "~~~ Run pipeline generator in dry-run mode" +echo "+++ Run pipeline generator in dry-run mode" python3 .buildkite/pipeline.py | yq . echo "~~~ Upload pipeline" From 0ce5a62b7711c3d020d9cb96fabc53e43b0d46da Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 19 Mar 2024 18:28:05 +0200 Subject: [PATCH 38/43] Fix windows mage build targets Signed-off-by: Alexandros Sapranidis --- auditbeat/buildkite.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/auditbeat/buildkite.yml b/auditbeat/buildkite.yml index d3dea9c7a56..3619d20c98f 100644 --- a/auditbeat/buildkite.yml +++ b/auditbeat/buildkite.yml @@ -21,10 +21,10 @@ stages: command: "mage -d auditbeat build unitTest" platform: "family/platform-ingest-beats-rhel-9" unitTest-windows-2022: - command: "mage build unitTest" + command: "mage -d auditbeat build unitTest" platform: "family/platform-ingest-beats-windows-2022" unitTest-windows-2016: - command: "mage build unitTest" + command: "mage -d auditbeat build unitTest" platform: "family/platform-ingest-beats-windows-2016" # optional stage - it runs on: # - branches/tags @@ -50,5 +50,5 @@ stages: platform: "generic-13-ventura-x64" provider: "orka" unitTest-windows-2019: - command: "mage build unitTest" + command: "mage -d auditbeat build unitTest" platform: "family/core-windows-2019" From c839a6793cf0236630666586da109a6d7fb44bd9 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Tue, 19 Mar 2024 18:32:26 +0200 Subject: [PATCH 39/43] Make github context start with capital letter Signed-off-by: Alexandros Sapranidis --- .buildkite/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 8988cde09cd..f2585dd9704 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -86,7 +86,7 @@ def create_entity(self) -> dict[str, Any]: "notify": [ { "github_commit_status": { - "context": f"{self.project}: {self.name}", + "context": f"{self.project.title()}: {self.name}", } } ], From 3a1cc1b26207c2035bad5312d72d5596735c5f91 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Wed, 20 Mar 2024 11:55:55 +0200 Subject: [PATCH 40/43] Add crosscompile GOX_FLAGS Signed-off-by: Alexandros Sapranidis --- .buildkite/pipeline.py | 2 +- auditbeat/buildkite.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 4ab7c6652e9..27626483e67 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -103,7 +103,7 @@ def create_entity(self) -> dict[str, Any]: ], } if len(self.envs) > 0: - data["env"] = [f"{k}:{v}" for k, v in self.envs] + data["env"] = self.envs return data diff --git a/auditbeat/buildkite.yml b/auditbeat/buildkite.yml index 3619d20c98f..d26828d9766 100644 --- a/auditbeat/buildkite.yml +++ b/auditbeat/buildkite.yml @@ -17,6 +17,8 @@ stages: crosscompile: command: "make -C auditbeat crosscompile" platform: "family/platform-ingest-beats-ubuntu-2204" + env: + GOX_FLAGS: "-arch amd64" unitTest-rhel-9: command: "mage -d auditbeat build unitTest" platform: "family/platform-ingest-beats-rhel-9" From 48a785172236022455f12a54c45fe643e1457c35 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Wed, 20 Mar 2024 13:29:38 +0200 Subject: [PATCH 41/43] Fix mage command for Filebeat windows steps Signed-off-by: Alexandros Sapranidis --- filebeat/buildkite.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/filebeat/buildkite.yml b/filebeat/buildkite.yml index 97cb6e97322..5cdd2d2603e 100644 --- a/filebeat/buildkite.yml +++ b/filebeat/buildkite.yml @@ -29,10 +29,10 @@ stages: command: "mage -d filebeat unitTest" platform: "family/platform-ingest-beats-rhel-9" unitTest-windows-2022: - command: "mage build unitTest" + command: "mage -d filebeat build unitTest" platform: "family/platform-ingest-beats-windows-2022" unitTest-windows-2016: - command: "mage build unitTest" + command: "mage -d filebeat build unitTest" platform: "family/platform-ingest-beats-windows-2016" # optional stage - it runs on: # - branches/tags From a090777a499c53bbf9fb9061f31558f07e5b40cb Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Wed, 20 Mar 2024 14:09:29 +0200 Subject: [PATCH 42/43] Make steps run from inside the proper project folder Signed-off-by: Alexandros Sapranidis --- .buildkite/pipeline.py | 17 +++++++++++++---- auditbeat/buildkite.yml | 20 ++++++++++---------- filebeat/buildkite.yml | 14 +++++++------- 3 files changed, 30 insertions(+), 21 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 27626483e67..1ee368aff7f 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -74,7 +74,7 @@ def __init__( definition: dict[str, Any], ): self.command = definition.get("command", "") - self.envs = definition.get("env", []) + self.env = definition.get("env", {}) self.agent: Agent = agent self.name: str = name self.project: str = project @@ -85,10 +85,19 @@ def __init__( def __lt__(self, other): return self.name < other.name + def step_command(self) -> list[str]: + commands = [ + # pushd works for windows as well + # https://en.wikipedia.org/wiki/Pushd_and_popd#:~:text=In%20Windows%20PowerShell%2C%20pushd%20is,the%20pushd%20and%20popd%20commands. + f"pushd {self.project}", + self.command, + ] + return commands + def create_entity(self) -> dict[str, Any]: data = { "label": f"{self.project} {self.name}", - "command": [self.command], + "command": self.step_command(), "notify": [ { "github_commit_status": { @@ -102,8 +111,8 @@ def create_entity(self) -> dict[str, Any]: f"{self.project}/build/*.json", ], } - if len(self.envs) > 0: - data["env"] = self.envs + if self.env: + data["env"] = self.env return data diff --git a/auditbeat/buildkite.yml b/auditbeat/buildkite.yml index d26828d9766..705207cb07e 100644 --- a/auditbeat/buildkite.yml +++ b/auditbeat/buildkite.yml @@ -12,21 +12,21 @@ stages: mandatory: # NOTE: stage name should be unique! unitTest: - command: "mage -d auditbeat build unitTest" + command: "build unitTest" platform: "family/platform-ingest-beats-ubuntu-2204" crosscompile: - command: "make -C auditbeat crosscompile" + command: "make crosscompile" platform: "family/platform-ingest-beats-ubuntu-2204" env: GOX_FLAGS: "-arch amd64" unitTest-rhel-9: - command: "mage -d auditbeat build unitTest" + command: "mage build unitTest" platform: "family/platform-ingest-beats-rhel-9" unitTest-windows-2022: - command: "mage -d auditbeat build unitTest" + command: "mage build unitTest" platform: "family/platform-ingest-beats-windows-2022" unitTest-windows-2016: - command: "mage -d auditbeat build unitTest" + command: "mage build unitTest" platform: "family/platform-ingest-beats-windows-2016" # optional stage - it runs on: # - branches/tags @@ -36,21 +36,21 @@ stages: extended: # NOTE: stage name should be unique! integTest: - command: "mage -d auditbeat build integTest" + command: "mage build integTest" platform: "platform-ingest-beats-ubuntu-2204-aarch64" provider: "aws" integTest-arm: - command: "mage -d auditbeat build integTest" + command: "mage build integTest" platform: "platform-ingest-beats-ubuntu-2204-aarch64" provider: "aws" unitTest-arm: - command: "mage -d auditbeat build unitTest" + command: "mage build unitTest" platform: "platform-ingest-beats-ubuntu-2204-aarch64" provider: "aws" unitTest-macos: - command: "mage -d auditbeat build unitTest" + command: "mage build unitTest" platform: "generic-13-ventura-x64" provider: "orka" unitTest-windows-2019: - command: "mage -d auditbeat build unitTest" + command: "mage build unitTest" platform: "family/core-windows-2019" diff --git a/filebeat/buildkite.yml b/filebeat/buildkite.yml index 5cdd2d2603e..372fbcf8caf 100644 --- a/filebeat/buildkite.yml +++ b/filebeat/buildkite.yml @@ -12,27 +12,27 @@ stages: mandatory: # NOTE: stage name should be unique! unitTest: - command: "mage -d filebeat unitTest" + command: "mage unitTest" platform: "family/platform-ingest-beats-ubuntu-2204" crosscompile: - command: "make -C filebeat crosscompile" + command: "make crosscompile" platform: "family/platform-ingest-beats-ubuntu-2204" env: GOX_FLAGS: "-arch amd64" goIntegTest: - command: "mage -d filebeat goIntegTest" + command: "mage goIntegTest" platform: "family/platform-ingest-beats-ubuntu-2204" pythonIntegTest: - command: "mage -d filebeat pythonIntegTest" + command: "mage pythonIntegTest" platform: "family/platform-ingest-beats-ubuntu-2204" rhel-9: - command: "mage -d filebeat unitTest" + command: "mage unitTest" platform: "family/platform-ingest-beats-rhel-9" unitTest-windows-2022: - command: "mage -d filebeat build unitTest" + command: "mage build unitTest" platform: "family/platform-ingest-beats-windows-2022" unitTest-windows-2016: - command: "mage -d filebeat build unitTest" + command: "mage build unitTest" platform: "family/platform-ingest-beats-windows-2016" # optional stage - it runs on: # - branches/tags From ca2c8fc9564d6a3d66468d34fe97922e46025d16 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Wed, 20 Mar 2024 14:16:25 +0200 Subject: [PATCH 43/43] Use more compatible cd and restore missed mage Signed-off-by: Alexandros Sapranidis --- .buildkite/pipeline.py | 4 +--- auditbeat/buildkite.yml | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 1ee368aff7f..95530b89628 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -87,9 +87,7 @@ def __lt__(self, other): def step_command(self) -> list[str]: commands = [ - # pushd works for windows as well - # https://en.wikipedia.org/wiki/Pushd_and_popd#:~:text=In%20Windows%20PowerShell%2C%20pushd%20is,the%20pushd%20and%20popd%20commands. - f"pushd {self.project}", + f"cd {self.project}", self.command, ] return commands diff --git a/auditbeat/buildkite.yml b/auditbeat/buildkite.yml index 705207cb07e..2abf9d68407 100644 --- a/auditbeat/buildkite.yml +++ b/auditbeat/buildkite.yml @@ -12,7 +12,7 @@ stages: mandatory: # NOTE: stage name should be unique! unitTest: - command: "build unitTest" + command: "mage build unitTest" platform: "family/platform-ingest-beats-ubuntu-2204" crosscompile: command: "make crosscompile"