[minor] Output verify and sanity test log to test record (#218)

ibm-mas · Nov 11, 2024 · 62ef5ee · 62ef5ee
1 parent 6df02d1
commit 62ef5ee
Show file tree

Hide file tree

Showing 5 changed files with 232 additions and 90 deletions.
diff --git a/cluster-applications/200-cluster-promotion/templates/02-cluster-verify_Job.yaml b/cluster-applications/200-cluster-promotion/templates/02-cluster-verify_Job.yaml
@@ -35,7 +35,7 @@ roleRef:
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: cluster-verify-{{ .Values.cluster_id }}-v1-{{ omit .Values "junitreporter" | toYaml | adler32sum }}
+  name: cluster-verify-{{ .Values.cluster_id }}-v2-{{ omit .Values "junitreporter" | toYaml | adler32sum }}
   namespace: mas-syncres
   annotations:
     argocd.argoproj.io/sync-wave: "202"
@@ -114,12 +114,20 @@ spec:
                       FOUND_CATALOG_VERION=$(oc get configmap -n $MAS_APP_NAMESPACE $MAS_APP_CM -o jsonpath='{.data.mas_catalog_version}')
                       if [[ $FOUND_CATALOG_VERION == $MAS_CATALOG_VERSION ]]; then
                           echo "Found ConfigMap $MAS_APP_CM with correct MAS_CATALOG_VERSION: $MAS_CATALOG_VERSION"
-                          SUCCESS=true
+
+                          TEST_PASSED=$(oc get configmap -n $MAS_APP_NAMESPACE $MAS_APP_CM -o jsonpath='{.data.test_passed}')
+                          echo "Found ConfigMap $MAS_APP_CM with TEST_PASSED value of: $TEST_PASSED"
+                          if [[ $TEST_PASSED ]]; then
+                            SUCCESS=true
+                          else
+                            SUCCESS=false
+                          fi
                       else
                           echo "Found ConfigMap $MAS_APP_CM with incorrect MAS_CATALOG_VERSION of $FOUND_CATALOG_VERION, expecting $MAS_CATALOG_VERSION"
                           SUCCESS=false
                       fi
 
+
                       if [[ $SUCCESS == "true" ]]; then
                           break
                       fi

diff --git a/...ce-applications/510-550-ibm-mas-suite-app-config/templates/04-postsync-manage-sanity.yaml b/...ce-applications/510-550-ibm-mas-suite-app-config/templates/04-postsync-manage-sanity.yaml
@@ -86,6 +86,14 @@ rules:
       - manageapp
       - manageworkspaces
       - manageapps
+  - verbs:
+      - get
+      - list
+      - patch
+    apiGroups:
+      - ""
+    resources:
+      - configmaps
 ---
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@@ -930,14 +938,36 @@ data:
         response = session.delete(asset_url, headers=headers, params=querystring)
         assert_that(response.status_code).is_equal_to(204)
 
+
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: {{ $record_cm_name }}
+  namespace: {{ $ns }}
+  annotations:
+    argocd.argoproj.io/sync-wave: "604"
+    argocd.argoproj.io/hook: PostSync
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+  labels:
+    type: mas-app-sanity-record
+{{- if .Values.custom_labels }}
+{{ .Values.custom_labels | toYaml | indent 4 }}
+{{- end }}
+immutable: false
+data:
+  mas_app: "manage"
+  mas_catalog_version: "{{ .Values.mas_catalog_version }}"
+  test_passed: "unknown"
+
 ---
 apiVersion: batch/v1
 kind: Job
 metadata:
   name: {{ $job_name }}
   namespace: {{ $ns }}
   annotations:
-    argocd.argoproj.io/sync-wave: "604"
+    argocd.argoproj.io/sync-wave: "605"
     argocd.argoproj.io/hook: PostSync
     argocd.argoproj.io/hook-delete-policy: HookSucceeded,BeforeHookCreation
 {{- if .Values.custom_labels }}
@@ -974,6 +1004,8 @@ spec:
               value: "{{ .Values.mas_app_namespace }}"
             - name: MAS_NAMESPACE
               value: "mas-{{ .Values.instance_id }}-core"
+            - name: TEST_RECORD_CM
+              value: "{{ $record_cm_name }}"
           volumeMounts:
             - name: tests
               mountPath: /tmp/tests
@@ -984,7 +1016,29 @@ spec:
               python -m venv .venv
               source .venv/bin/activate
               pip install -r /tmp/tests/requirements.txt
-              pytest -o cache_dir=/tmp/__pycache__ /tmp/tests/tests.py 
+
+              set -o pipefail
+              echo "Running tests..."
+              pytest -v --junit-xml=junitxml_test_output.xml -o cache_dir=/tmp/__pycache__ /tmp/tests/tests.py 2>&1 | tee test_log.txt
+              if [[ $? -ne 0 ]]; then
+                  TEST_PASSED=false
+              else
+                  TEST_PASSED=true
+              fi
+              echo "Test Result Passed: $TEST_PASSED"
+              set +o pipefail
+
+              set -e
+              echo "Updating $TEST_RECORD_CM configmap with test result"
+              oc set data cm $TEST_RECORD_CM test_passed=$TEST_PASSED
+              oc set data cm $TEST_RECORD_CM --from-file=junitxml_test_output.xml
+              oc set data cm $TEST_RECORD_CM --from-file=test_log.txt
+              
+              if [[ $TEST_PASSED == "false" ]]; then
+                echo "Test Result failed, exit 1"
+                exit 1
+              fi
+
       restartPolicy: Never
       serviceAccountName: {{ $sa_name }}
       volumes:
@@ -1000,24 +1054,4 @@ spec:
             optional: false
   backoffLimit: 4
 
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: {{ $record_cm_name }}
-  namespace: {{ $ns }}
-  annotations:
-    argocd.argoproj.io/sync-wave: "605"
-    argocd.argoproj.io/hook: PostSync
-    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
-  labels:
-    type: mas-app-verification-record
-{{- if .Values.custom_labels }}
-{{ .Values.custom_labels | toYaml | indent 4 }}
-{{- end }}
-immutable: false
-data:
-  mas_app: "manage"
-  mas_catalog_version: "{{ .Values.mas_catalog_version }}"
-
 {{- end }}
diff --git a/...ce-applications/510-550-ibm-mas-suite-app-config/templates/04-postsync-manage-verify.yaml b/...ce-applications/510-550-ibm-mas-suite-app-config/templates/04-postsync-manage-verify.yaml
@@ -83,6 +83,14 @@ rules:
       - manageapp
       - manageworkspaces
       - manageapps
+  - verbs:
+      - get
+      - list
+      - patch
+    apiGroups:
+      - ""
+    resources:
+      - configmaps
 ---
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
@@ -207,6 +215,27 @@ data:
       if spec_serverbundles is not None:
         assert status_serverbundles == spec_serverbundles, f"Expected serverbundles set in the spec: {spec_serverbundles} to be equal to the languages in the status: {status_serverbundles}"
 
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: {{ $record_cm_name }}
+  namespace: {{ $ns }}
+  annotations:
+    argocd.argoproj.io/sync-wave: "604"
+    argocd.argoproj.io/hook: PostSync
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+  labels:
+    type: mas-app-verification-record
+{{- if .Values.custom_labels }}
+{{ .Values.custom_labels | toYaml | indent 4 }}
+{{- end }}
+immutable: false
+data:
+  mas_app: "manage"
+  mas_catalog_version: "{{ .Values.mas_catalog_version }}"
+  test_passed: "unknown"
+
 
 ---
 apiVersion: batch/v1
@@ -215,7 +244,7 @@ metadata:
   name: {{ $job_name }}
   namespace: {{ $ns }}
   annotations:
-    argocd.argoproj.io/sync-wave: "604"
+    argocd.argoproj.io/sync-wave: "605"
     argocd.argoproj.io/hook: PostSync
     argocd.argoproj.io/hook-delete-policy: HookSucceeded,BeforeHookCreation
 {{- if .Values.custom_labels }}
@@ -248,15 +277,40 @@ spec:
               value: "{{ .Values.instance_id }}"
             - name: MAS_WORKSPACE_ID
               value: "{{ .Values.mas_workspace_id }}"
+            - name: TEST_RECORD_CM
+              value: "{{ $record_cm_name }}"
           volumeMounts:
             - name: tests
               mountPath: /tmp/tests
           command:
             - /bin/sh
             - -c
             - |
+              python -m venv .venv
+              source .venv/bin/activate
               pip install -r /tmp/tests/requirements.txt
-              pytest -o cache_dir=/tmp/__pycache__ /tmp/tests/tests.py 
+
+              set -o pipefail
+              echo "Running tests..."
+              pytest -v --junit-xml=junitxml_test_output.xml -o cache_dir=/tmp/__pycache__ /tmp/tests/tests.py 2>&1 | tee test_log.txt
+              if [[ $? -ne 0 ]]; then
+                  TEST_PASSED=false
+              else
+                  TEST_PASSED=true
+              fi
+              echo "Test Result Passed: $TEST_PASSED"
+              set +o pipefail
+
+              set -e
+              echo "Updating $TEST_RECORD_CM configmap with test result"
+              oc set data cm $TEST_RECORD_CM test_passed=$TEST_PASSED
+              oc set data cm $TEST_RECORD_CM --from-file=junitxml_test_output.xml
+              oc set data cm $TEST_RECORD_CM --from-file=test_log.txt
+
+              if [[ $TEST_PASSED == "false" ]]; then
+                echo "Test Result failed, exit 1"
+                exit 1
+              fi
       restartPolicy: Never
       serviceAccountName: {{ $sa_name }}
       volumes:
@@ -272,24 +326,5 @@ spec:
             optional: false
   backoffLimit: 4
 
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: {{ $record_cm_name }}
-  namespace: {{ $ns }}
-  annotations:
-    argocd.argoproj.io/sync-wave: "605"
-    argocd.argoproj.io/hook: PostSync
-    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
-  labels:
-    type: mas-app-verification-record
-{{- if .Values.custom_labels }}
-{{ .Values.custom_labels | toYaml | indent 4 }}
-{{- end }}
-immutable: false
-data:
-  mas_app: "manage"
-  mas_catalog_version: "{{ .Values.mas_catalog_version }}"
 
 {{- end }}
diff --git a/instance-applications/510-550-ibm-mas-suite-app-config/templates/04-postsync-mvi-sanity.yaml b/instance-applications/510-550-ibm-mas-suite-app-config/templates/04-postsync-mvi-sanity.yaml
@@ -85,6 +85,14 @@ rules:
       - "route.openshift.io"
     resources:
       - routes
+  - verbs:
+      - get
+      - list
+      - patch
+    apiGroups:
+      - ""
+    resources:
+      - configmaps
 
 ---
 kind: RoleBinding
@@ -451,6 +459,27 @@ data:
         print("Placeholder for sanity tests to be added")
 
 
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: {{ $record_cm_name }}
+  namespace: {{ $ns }}
+  annotations:
+    argocd.argoproj.io/sync-wave: "604"
+    argocd.argoproj.io/hook: PostSync
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+  labels:
+      type: mas-app-sanity-record
+{{- if .Values.custom_labels }}
+{{ .Values.custom_labels | toYaml | indent 4 }}
+{{- end }}
+immutable: false
+data:
+  mas_app: "visualinspection"
+  mas_catalog_version: "{{ .Values.mas_catalog_version }}"
+  test_passed: "unknown"
+
 
 ---
 apiVersion: batch/v1
@@ -459,7 +488,7 @@ metadata:
   name: {{ $job_name }}
   namespace: {{ $ns }}
   annotations:
-    argocd.argoproj.io/sync-wave: "604"
+    argocd.argoproj.io/sync-wave: "605"
     argocd.argoproj.io/hook: PostSync
     argocd.argoproj.io/hook-delete-policy: HookSucceeded,BeforeHookCreation
 {{- if .Values.custom_labels }}
@@ -492,16 +521,40 @@ spec:
               value: "{{ .Values.instance_id }}"
             - name: MAS_WORKSPACE_ID
               value: "{{ .Values.mas_workspace_id }}"
+            - name: TEST_RECORD_CM
+              value: "{{ $record_cm_name }}"
           volumeMounts:
             - name: tests
               mountPath: /tmp/tests
           command:
             - /bin/sh
             - -c
             - |
+              python -m venv .venv
+              source .venv/bin/activate
               pip install -r /tmp/tests/requirements.txt
-              pytest -o cache_dir=/tmp/__pycache__ /tmp/tests/tests.py
 
+              set -o pipefail
+              echo "Running tests..."
+              pytest -v --junit-xml=junitxml_test_output.xml -o cache_dir=/tmp/__pycache__ /tmp/tests/tests.py 2>&1 | tee test_log.txt
+              if [[ $? -ne 0 ]]; then
+                  TEST_PASSED=false
+              else
+                  TEST_PASSED=true
+              fi
+              echo "Test Result Passed: $TEST_PASSED"
+              set +o pipefail
+
+              set -e
+              echo "Updating $TEST_RECORD_CM configmap with test result"
+              oc set data cm $TEST_RECORD_CM test_passed=$TEST_PASSED
+              oc set data cm $TEST_RECORD_CM --from-file=junitxml_test_output.xml
+              oc set data cm $TEST_RECORD_CM --from-file=test_log.txt
+
+              if [[ $TEST_PASSED == "false" ]]; then
+                echo "Test Result failed, exit 1"
+                exit 1
+              fi
       restartPolicy: Never
       serviceAccountName: {{ $sa_name }}
       volumes:
@@ -517,24 +570,4 @@ spec:
             optional: false
   backoffLimit: 4
 
----
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: {{ $record_cm_name }}
-  namespace: {{ $ns }}
-  annotations:
-    argocd.argoproj.io/sync-wave: "605"
-    argocd.argoproj.io/hook: PostSync
-    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
-  labels:
-      type: mas-app-verification-record
-{{- if .Values.custom_labels }}
-{{ .Values.custom_labels | toYaml | indent 4 }}
-{{- end }}
-immutable: false
-data:
-  mas_app: "visualinspection"
-  mas_catalog_version: "{{ .Values.mas_catalog_version }}"
-
 {{- end }}