Use native yaml marshalling

.buildkite/pipeline.py

@@ -1,117 +1,63 @@
 #!/usr/bin/env python3
+from typing import Any
 import yaml
 import os
 from dataclasses import dataclass, field
 import subprocess
 import fnmatch
 
-from jinja2 import Template
-from pathlib import Path
-
-
-@dataclass()
-class Pipeline:
-    """Buildkite Pipeline object"""
-    groups: list[str]
-
-    def create_entity(self):
-        data = """
-steps:
-{% for group in pipeline.groups -%}
-{{ group.create_entity() }}
-{% endfor -%}
-"""
-
-        tm = Template(data)
-        msg = tm.render(pipeline=self)
-        return msg
-
-@dataclass(unsafe_hash=True)
-class Group:
-    """Buildkite Group object"""
-
-    project: str
-    category: str
-    steps: list[str]
-
-    def __lt__(self, other):
-        return self.project < other.project
-
-    def create_entity(self):
-        data = """
-{% if group.steps|length > 0 %}
-  - group: "{{ group.project }} {{ group.category }}"
-    key: "{{ group.project }}-{{ group.category }}"
-    steps:
-      {% for step in group.steps|sort -%}
-        {{ step.create_entity() }}
-      {% endfor -%}
-{% endif -%}
-"""
-
-        tm = Template(data)
-        msg = tm.render(group=self)
-        return msg
 
 @dataclass(unsafe_hash=True)
 class Agent:
     """Buildkite Agent object"""
-
     image: str
 
     def create_entity(self):
         raise NotImplementedError("Not implemented yet")
-
+
+
 @dataclass(unsafe_hash=True)
 class AWSAgent(Agent):
     """AWS Agent object"""
-
-    image: str 
+    image: str
 
-    def create_entity(self):
-        data = """
-        agents:
-          provider: "aws"
-          imagePrefix: "{{ agent.image }}"         
-          instanceType: "t4g.large"
-"""
+    def create_entity(self) -> dict[str, dict[str, str]]:
+        return {
+            "agents": {
+                "provider": "aws",
+                "imagePrefix": self.image,
+                "instanceType": "t4g.large",
+            }
+        }
 
-        tm = Template(data)
-        msg = tm.render(agent=self)
-        return msg
 
 @dataclass(unsafe_hash=True)
 class GCPAgent(Agent):
     """GCP Agent object"""
-
-    image: str 
+    image: str
+
+    def create_entity(self) -> dict[str, dict[str, str]]:
+        return {
+            "agents": {
+                "provider": "gcp",
+                "image": self.image,
+            }
+        }
+
 
-    def create_entity(self):
-        data = """
-        agents:
-          provider: "gcp"
-          image: "{{ agent.image }}"                   
-"""
-
-        tm = Template(data)
-        msg = tm.render(agent=self)
-        return msg
 @dataclass(unsafe_hash=True)
 class OrkaAgent(Agent):
     """Orka Agent object"""
-
-    image: str 
+    image: str
 
-    def create_entity(self):
-        data = """
-        agents:
-          provider: "orka"
-          imagePrefix: "{{ agent.image }}"                   
-"""
+    def create_entity(self) -> dict[str, dict[str, str]]:
+        return {
+            "agents": {
+                "provider": "orka",
+                "imagePrefix": self.image,
+            }
+        }
 
-        tm = Template(data)
-        msg = tm.render(agent=self)
-        return msg
 
 @dataclass(unsafe_hash=True)
 class Step:
@@ -132,32 +78,61 @@ def __post_init__(self):
     def __lt__(self, other):
         return self.name < other.name
 
-    def create_entity(self):
-        data = """
-      - label: "{{ step.project }} {{ step.name }}"
-        command:
-          - "{{ step.command }}"
-        notify:
-          - github_commit_status:
-              context: "{{ step.project }}: {{ step.name }}"
-        {{ step.agent.create_entity() }}
-"""
+    def create_entity(self) -> dict[str, Any]:
+        data = {
+            "label": f"{self.project} {self.name}",
+            "command": [self.command],
+            "notify": [{
+                "github_commit_status": {
+                    "context": f"{self.project}: {self.name}",
+                }
+            }],
+        }
+        data.update(self.agent.create_entity())
+        return data
 
-        tm = Template(data)
-        msg = tm.render(step=self)
-        return msg
 
+@dataclass(unsafe_hash=True)
+class Group:
+    """Buildkite Group object"""
+
+    project: str
+    category: str
+    steps: list[Step]
+
+    def __lt__(self, other):
+        return self.project < other.project
+
+    def create_entity(self) -> dict[str, Any] | None:
+        if len(self.steps) == 0:
+            return
+
+        data = {
+            "group": f"{self.project} {self.category}",
+            "key": f"{self.project}-{self.category}",
+            "steps": [step.create_entity() for step in sorted(self.steps)],
+        }
+
+        return data
+
+
+@dataclass()
+class Pipeline:
+    """Buildkite Pipeline object"""
+    groups: list[Group]
+
+    def create_entity(self):
+        data = {"steps": [group.create_entity() for group in self.groups]}
+        return data
 
-# Conditions:
 
 def is_pr() -> bool:
-    pr = os.getenv('BUILDKITE_PULL_REQUEST')
-    return pr and os.getenv('BUILDKITE_PULL_REQUEST') != "false"
+    return os.getenv('BUILDKITE_PULL_REQUEST') != "false"
+
 
 def step_comment(step: Step) -> bool:
     comment = os.getenv('GITHUB_PR_TRIGGER_COMMENT')
     if comment:
-        # the comment should be a subset of the values in .buildkite/pull-requests.json
         # TODO: change /test
         comment_prefix = "buildkite test " + step.project
         # i.e: /test filebeat should run all the mandatory stages
@@ -167,48 +142,60 @@ def step_comment(step: Step) -> bool:
         return comment_prefix + " " + step.name in comment
     else:
         return True
-
+
+
 def group_comment(group: Group) -> bool:
     comment = os.getenv('GITHUB_PR_TRIGGER_COMMENT')
     if comment:
-        # the comment should be a subset of the values in .buildkite/pull-requests.json
+        # the comment should be a subset of the values
+        # in .buildkite/pull-requests.json
         # TODO: change /test
         comment_prefix = "buildkite test"
         if group.category == "mandatory":
             # i.e: /test filebeat
             return comment_prefix + " " + group.project in comment
         else:
             # i.e: test filebeat extended
-            return comment_prefix + " " + group.project + " " + group.category in comment    
+            return (
+                comment_prefix + " " + group.project + " " + group.category
+                in comment
+            )
+
 
 changed_files = None
 
+
 def get_pr_changeset():
     global changed_files
     if not changed_files:
         base_branch = os.getenv('BUILDKITE_PULL_REQUEST_BASE_BRANCH')
-        diff_command = ["git", "diff", "--name-only", "{}...HEAD".format(base_branch)]
+        diff_command = [
+            "git", "diff", "--name-only", "{}...HEAD".format(base_branch)
+        ]
         result = subprocess.run(diff_command, stdout=subprocess.PIPE)
         changed_files = result.stdout.decode().splitlines()
-        print("Changed files: {}".format(changed_files))        
+        print("Changed files: {}".format(changed_files))
     return changed_files
 
+
 def filter_files_by_glob(files, patterns: list[str]):
     for pattern in patterns:
-        ## TODO: Support glob extended patterns: ^ and etc.
-        ## Now it supports only linux glob patterns
+        # TODO: Support glob extended patterns: ^ and etc.
+        # Now it supports only linux glob patterns
         if fnmatch.filter(files, pattern):
             return True
     return False
 
+
 def is_in_pr_changeset(project_changeset_filters: list[str]) -> bool:
     changeset = get_pr_changeset()
     return filter_files_by_glob(changeset, project_changeset_filters)
 
-def is_step_enabled(step: Step) -> bool:    
+
+def is_step_enabled(step: Step) -> bool:
     if not is_pr():
         return True
-    
+
     if step_comment(step):
         return True
 
@@ -220,34 +207,38 @@ def is_step_enabled(step: Step) -> bool:
             return True
 
     return False
-
+
+
 def is_group_enabled(group: Group, changeset_filters: list[str]) -> bool:
     if not is_pr():
-        return True    
+        return True
+
+    if is_pr() and is_in_pr_changeset(changeset_filters) and \
+            group.category.startswith("mandatory"):
+        return True
 
-    if is_pr() and is_in_pr_changeset(changeset_filters) and group.category.startswith("mandatory"):
-        return True        
-
     return group_comment(group)
 
+
 def fetch_stage(name: str, stage, project: str, category: str) -> Step:
     """Create a step given the yaml object."""
 
     agent: Agent = None
-    if (not "provider" in stage) or stage["provider"] == "gcp":
+    if ("provider" not in stage) or stage["provider"] == "gcp":
         agent = GCPAgent(image=stage["platform"])
     elif stage["provider"] == "aws":
         agent = AWSAgent(image=stage["platform"])
     elif stage["provider"] == "orka":
         agent = OrkaAgent(image=stage["platform"])
-    
+
     return Step(
             category=category,
             command=stage["command"],
             name=name,
             agent=agent,
             project=project)
 
+
 def fetch_group(stages, project: str, category: str) -> Group:
     """Create a group given the yaml object."""
 
@@ -270,7 +261,6 @@ def fetch_group(stages, project: str, category: str) -> Group:
 
 
 # TODO: validate unique stages!
-
 def main() -> None:
 
     groups = []
@@ -282,10 +272,7 @@ def main() -> None:
             project_file = os.path.join(project, "buildkite.yml")
             if not os.path.isfile(project_file):
                 continue
-            # TODO: data structure when things run.
-            conditions = None
             with open(project_file, "r", encoding="utf8") as file:
-                steps = []
                 project_obj = yaml.load(file, yaml.FullLoader)
 
                 group = fetch_group(stages=project_obj["stages"]["mandatory"],
@@ -299,7 +286,7 @@ def main() -> None:
                                     project=project,
                                     category="extended")
 
-                if is_group_enabled(group, project_obj["when"]["changeset"]):
+                if is_group_enabled(group, project_obj["when"]["changeset"]):                    
                     extended_groups.append(group)
 
     # TODO: improve this merging lists
@@ -310,7 +297,7 @@ def main() -> None:
         all_groups.append(group)
 
     # Produce now the pipeline
-    print(Pipeline(all_groups).create_entity())
+    print(yaml.dump(Pipeline(all_groups).create_entity()))
 
 
 if __name__ == "__main__":

.buildkite/scripts/generate_pipeline.sh

@@ -2,13 +2,13 @@
 set -euo pipefail
 
 echo "--- Install dependencies"
-pip3 install --quiet jinja2
 pip3 install --quiet PyYAML
 
 echo "--- Run pipeline generator in dry-run mode"
-python3 .buildkite/pipeline.py || true
+python3 .buildkite/pipeline.py | yq .
 
 # Run the python generator - likely this should be called in the
 # catalog-info.yaml
-echo "--- Upload pipeline"
-python3 .buildkite/pipeline.py | buildkite-agent pipeline upload
+# echo "--- Upload pipeline"
+## Remove when is ready
+# python3 .buildkite/pipeline.py | buildkite-agent pipeline upload