Make job timeout configurable (#84)

* Make job timeout configurable via cluster type extraVars * Move job timeout to cluster type spec * Update unit tests * Fix jobTimeout type annotation --------- Co-authored-by: sd109 <scott@stackhpc.com> Co-authored-by: John Garbutt <john.garbutt@stackhpc.com>
azimuth-cloud · Jan 16, 2024 · 5885a8a · 5885a8a
1 parent 4045f01
commit 5885a8a
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 2 deletions.
diff --git a/azimuth_caas_operator/models/v1alpha1/cluster_type.py b/azimuth_caas_operator/models/v1alpha1/cluster_type.py
@@ -26,6 +26,9 @@ class ClusterTypeSpec(schema.BaseModel):
     gitVersion: str
     # Playbook is contained in the above git repo
     playbook: str
+    # The timeout (in seconds) to apply to the kubernetes job resource
+    # which creates, updates and deletes the cluster instances
+    jobTimeout: int = pydantic.Field(default=1200)
     # Option to add cloud specific details, like the image
     extraVars: schema.Dict[str, schema.Any] = pydantic.Field(default_factory=dict)
     # Option to define cluster-type specific details, like inventory

diff --git a/azimuth_caas_operator/tests/models/test_crds.py b/azimuth_caas_operator/tests/models/test_crds.py
@@ -58,6 +58,9 @@ def test_cluster_type_crd_json(self):
                   "playbook": {
                     "type": "string"
                   },
+                  "jobTimeout": {
+                    "type": "integer"
+                  },
                   "extraVars": {
                     "additionalProperties": {
                       "x-kubernetes-preserve-unknown-fields": true
@@ -264,6 +267,9 @@ def test_cluster_crd_json(self):
                       "playbook": {
                         "type": "string"
                       },
+                      "jobTimeout": {
+                        "type": "integer"
+                      },
                       "extraVars": {
                         "additionalProperties": {
                           "x-kubernetes-preserve-unknown-fields": true

diff --git a/azimuth_caas_operator/utils/ansible_runner.py b/azimuth_caas_operator/utils/ansible_runner.py
@@ -294,8 +294,8 @@ def get_job(
           defaultMode: 256
           optional: true
   backoffLimit: {1 if remove else 0}
-  # timeout after 20 mins
-  activeDeadlineSeconds: 1200"""  # noqa
+  # Set timeout so that jobs don't get stuck in configuring state if something goes wrong
+  activeDeadlineSeconds: {cluster_type_spec.jobTimeout}"""  # noqa
     return yaml.safe_load(job_yaml)