From b5cdc4c978d054dc5bcdf9b1ddcf3b50b720eea5 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Fri, 15 Mar 2024 01:33:42 +0100 Subject: [PATCH] AWS And Orka support --- .buildkite/auditbeat/scripts/integ-test.sh | 11 +++ .buildkite/pipeline.py | 92 ++++++++++++++++++---- .buildkite/pipeline.yml | 2 +- auditbeat/buildkite.yml | 20 +++-- filebeat/buildkite.yml | 4 +- 5 files changed, 103 insertions(+), 26 deletions(-) create mode 100644 .buildkite/auditbeat/scripts/integ-test.sh diff --git a/.buildkite/auditbeat/scripts/integ-test.sh b/.buildkite/auditbeat/scripts/integ-test.sh new file mode 100644 index 00000000000..919bb6420a3 --- /dev/null +++ b/.buildkite/auditbeat/scripts/integ-test.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/linux-env.sh + +echo "--- Executing Integration Tests" +sudo chmod -R go-w auditbeat/ + +umask 0022 +mage -d auditbeat build integTest diff --git a/.buildkite/pipeline.py b/.buildkite/pipeline.py index 76e24def861..9311b0e9b23 100755 --- a/.buildkite/pipeline.py +++ b/.buildkite/pipeline.py @@ -53,16 +53,74 @@ def create_entity(self): msg = tm.render(group=self) return msg +@dataclass(unsafe_hash=True) +class Agent: + """Buildkite Agent object""" + + image: str + + def create_entity(self): + raise NotImplementedError("Not implemented yet") + +@dataclass(unsafe_hash=True) +class AWSAgent(Agent): + """AWS Agent object""" + + image: str + + def create_entity(self): + data = """ + agents: + provider: "aws" + imagePrefix: "{{ agent.image }}" + instanceType: "t4g.large" +""" + + tm = Template(data) + msg = tm.render(agent=self) + return msg + +@dataclass(unsafe_hash=True) +class GCPAgent(Agent): + """GCP Agent object""" + + image: str + + def create_entity(self): + data = """ + agents: + provider: "gcp" + image: "{{ agent.image }}" +""" + + tm = Template(data) + msg = tm.render(agent=self) + return msg +@dataclass(unsafe_hash=True) +class OrkaAgent(Agent): + """Orka Agent object""" + + image: str + + def create_entity(self): + data = """ + agents: + provider: "orka" + imagePrefix: "{{ agent.image }}" +""" + + tm = Template(data) + msg = tm.render(agent=self) + return msg @dataclass(unsafe_hash=True) class Step: """Buildkite Step object""" command: str + agent: Agent name: str - runner: str project: str - provider: str category: str label: str = field(init=False) comment: str = field(init=False) @@ -76,21 +134,20 @@ def __lt__(self, other): def create_entity(self): data = """ - - label: "{{ stage.project }} {{ stage.name }}" + - label: "{{ step.project }} {{ step.name }}" command: - - "{{ stage.command }}" + - "{{ step.command }}" notify: - github_commit_status: - context: "{{ stage.project }}: {{ stage.name }}" - agents: - provider: "{{ stage.provider }}" - image: "{{ stage.runner }}" + context: "{{ step.project }}: {{ step.name }}" + {{ step.agent.create_entity() }} """ tm = Template(data) - msg = tm.render(stage=self) + msg = tm.render(step=self) return msg + # Conditions: def is_pr() -> bool: @@ -176,17 +233,20 @@ def is_group_enabled(group: Group, changeset_filters: list[str]) -> bool: def fetch_stage(name: str, stage, project: str, category: str) -> Step: """Create a step given the yaml object.""" - # TODO: need to accomodate the provider type. - # maybe in the buildkite.yml or some dynamic analysis based on the - # name of the runners. + agent: Agent = None + if (not "provider" in stage) or stage["provider"] == "gcp": + agent = GCPAgent(image=stage["platform"]) + elif stage["provider"] == "aws": + agent = AWSAgent(image=stage["platform"]) + elif stage["provider"] == "orka": + agent = OrkaAgent(image=stage["platform"]) + return Step( category=category, command=stage["command"], name=name, - runner=stage["platform"], - project=project, - provider="gcp") - + agent=agent, + project=project) def fetch_group(stages, project: str, category: str) -> Group: """Create a group given the yaml object.""" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index dc6b2717633..d968ea3a304 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,5 +1,5 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json steps: - - label: "Test my dynamic pipeline" + - label: "Generate dynamic pipeline" command: ".buildkite/scripts/generate_pipeline.sh" diff --git a/auditbeat/buildkite.yml b/auditbeat/buildkite.yml index 2e9e60fa29f..220b4401fe6 100644 --- a/auditbeat/buildkite.yml +++ b/auditbeat/buildkite.yml @@ -34,17 +34,21 @@ stages: extended: # NOTE: stage name should be unique! integTest: - command: "mage build integTest" - platform: "core-ubuntu-2004-aarch64" + command: ".buildkite/auditbeat/scripts/integ-test.sh" + platform: "platform-ingest-beats-ubuntu-2204-aarch64" + provider: "aws" integTest-arm: - command: "mage build integTest" - platform: "core-ubuntu-2004-aarch64" + command: ".buildkite/auditbeat/scripts/integ-test.sh" + platform: "platform-ingest-beats-ubuntu-2204-aarch64" + provider: "aws" unitTest-arm: - command: "mage build unitTest" - platform: "core-ubuntu-2004-aarch64" + command: ".buildkite/auditbeat/scripts/unit-tests.sh" + platform: "platform-ingest-beats-ubuntu-2204-aarch64" + provider: "aws" unitTest-macos: - command: "mage build unitTest" + command: ".buildkite/auditbeat/scripts/unit-tests.sh" platform: "generic-13-ventura-x64" + provider: "orka" unitTest-windows-2019: - command: "mage build unitTest" + command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1" platform: "family/core-windows-2019" diff --git a/filebeat/buildkite.yml b/filebeat/buildkite.yml index db4d4113152..ca50e7dc378 100644 --- a/filebeat/buildkite.yml +++ b/filebeat/buildkite.yml @@ -40,10 +40,12 @@ stages: extended: unitTest-arm: command: ".buildkite/filebeat/scripts/unit-tests.sh" - platform: "core-ubuntu-2004-aarch64" + platform: "platform-ingest-beats-ubuntu-2204-aarch64" + provider: "aws" unitTest-macos: command: ".buildkite/filebeat/scripts/unit-tests.sh" platform: "generic-13-ventura-x64" + provider: "orka" unitTest-windows-2019: command: ".buildkite/filebeat/scripts/unit-tests-win.ps1" platform: "family/platform-ingest-beats-windows-2019"