AWS And Orka support

.buildkite/auditbeat/scripts/integ-test.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/env-scripts/linux-env.sh
+
+echo "--- Executing Integration Tests"
+sudo chmod -R go-w auditbeat/
+
+umask 0022
+mage -d auditbeat build integTest

.buildkite/pipeline.py

@@ -53,16 +53,74 @@ def create_entity(self):
         msg = tm.render(group=self)
         return msg
 
+@dataclass(unsafe_hash=True)
+class Agent:
+    """Buildkite Agent object"""
+
+    image: str
+
+    def create_entity(self):
+        raise NotImplementedError("Not implemented yet")
+
+@dataclass(unsafe_hash=True)
+class AWSAgent(Agent):
+    """AWS Agent object"""
+
+    image: str 
+
+    def create_entity(self):
+        data = """
+        agents:
+          provider: "aws"
+          imagePrefix: "{{ agent.image }}"         
+          instanceType: "t4g.large"
+"""
+
+        tm = Template(data)
+        msg = tm.render(agent=self)
+        return msg
+
+@dataclass(unsafe_hash=True)
+class GCPAgent(Agent):
+    """GCP Agent object"""
+
+    image: str 
+
+    def create_entity(self):
+        data = """
+        agents:
+          provider: "gcp"
+          image: "{{ agent.image }}"                   
+"""
+
+        tm = Template(data)
+        msg = tm.render(agent=self)
+        return msg
+@dataclass(unsafe_hash=True)
+class OrkaAgent(Agent):
+    """Orka Agent object"""
+
+    image: str 
+
+    def create_entity(self):
+        data = """
+        agents:
+          provider: "orka"
+          imagePrefix: "{{ agent.image }}"                   
+"""
+
+        tm = Template(data)
+        msg = tm.render(agent=self)
+        return msg
 
 @dataclass(unsafe_hash=True)
 class Step:
     """Buildkite Step object"""
 
     command: str
+    agent: Agent
     name: str
-    runner: str
     project: str
-    provider: str
     category: str
     label: str = field(init=False)
     comment: str = field(init=False)
@@ -76,21 +134,20 @@ def __lt__(self, other):
 
     def create_entity(self):
         data = """
-      - label: "{{ stage.project }} {{ stage.name }}"
+      - label: "{{ step.project }} {{ step.name }}"
         command:
-          - "{{ stage.command }}"
+          - "{{ step.command }}"
         notify:
           - github_commit_status:
-              context: "{{ stage.project }}: {{ stage.name }}"
-        agents:
-          provider: "{{ stage.provider }}"
-          image: "{{ stage.runner }}"
+              context: "{{ step.project }}: {{ step.name }}"
+        {{ step.agent.create_entity() }}
 """
 
         tm = Template(data)
-        msg = tm.render(stage=self)
+        msg = tm.render(step=self)
         return msg
 
+
 # Conditions:
 
 def is_pr() -> bool:
@@ -176,17 +233,20 @@ def is_group_enabled(group: Group, changeset_filters: list[str]) -> bool:
 def fetch_stage(name: str, stage, project: str, category: str) -> Step:
     """Create a step given the yaml object."""
 
-    # TODO: need to accomodate the provider type.
-    # maybe in the buildkite.yml or some dynamic analysis based on the
-    # name of the runners.
+    agent: Agent = None
+    if (not "provider" in stage) or stage["provider"] == "gcp":
+        agent = GCPAgent(image=stage["platform"])
+    elif stage["provider"] == "aws":
+        agent = AWSAgent(image=stage["platform"])
+    elif stage["provider"] == "orka":
+        agent = OrkaAgent(image=stage["platform"])
+
     return Step(
             category=category,
             command=stage["command"],
             name=name,
-            runner=stage["platform"],
-            project=project,
-            provider="gcp")
-
+            agent=agent,
+            project=project)
 
 def fetch_group(stages, project: str, category: str) -> Group:
     """Create a group given the yaml object."""

.buildkite/pipeline.yml

@@ -1,5 +1,5 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
 steps:
-  - label: "Test my dynamic pipeline"
+  - label: "Generate dynamic pipeline"
     command: ".buildkite/scripts/generate_pipeline.sh"

auditbeat/buildkite.yml

@@ -34,17 +34,21 @@ stages:
     extended:
         # NOTE: stage name should be unique!
         integTest:
-            command: "mage build integTest"
-            platform: "core-ubuntu-2004-aarch64"
+            command: ".buildkite/auditbeat/scripts/integ-test.sh"
+            platform: "platform-ingest-beats-ubuntu-2204-aarch64"
+            provider: "aws"
         integTest-arm:
-            command: "mage build integTest"
-            platform: "core-ubuntu-2004-aarch64"
+            command: ".buildkite/auditbeat/scripts/integ-test.sh"
+            platform: "platform-ingest-beats-ubuntu-2204-aarch64"
+            provider: "aws"
         unitTest-arm:
-            command: "mage build unitTest"
-            platform: "core-ubuntu-2004-aarch64"
+            command: ".buildkite/auditbeat/scripts/unit-tests.sh"
+            platform: "platform-ingest-beats-ubuntu-2204-aarch64"
+            provider: "aws"
         unitTest-macos:
-            command: "mage build unitTest"
+            command: ".buildkite/auditbeat/scripts/unit-tests.sh"
             platform: "generic-13-ventura-x64"
+            provider: "orka"
         unitTest-windows-2019:
-            command: "mage build unitTest"
+            command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1"
             platform: "family/core-windows-2019"

filebeat/buildkite.yml

@@ -40,10 +40,12 @@ stages:
     extended:
         unitTest-arm:
             command: ".buildkite/filebeat/scripts/unit-tests.sh"
-            platform: "core-ubuntu-2004-aarch64"
+            platform: "platform-ingest-beats-ubuntu-2204-aarch64"
+            provider: "aws"
         unitTest-macos:
             command: ".buildkite/filebeat/scripts/unit-tests.sh"
             platform: "generic-13-ventura-x64"
+            provider: "orka"
         unitTest-windows-2019:
             command: ".buildkite/filebeat/scripts/unit-tests-win.ps1"
             platform: "family/platform-ingest-beats-windows-2019"