WIP: Rework kuttl test to be organize like in nova-operator

Makefile

@@ -364,3 +364,43 @@ scale-down-placement-controller-csv:
 	if [ "$(has_webhooks)" != "null" ]; then \
 	    oc patch csv -n $(OPERATOR_NAMESPACE) placement-operator.v0.0.1 --type json -p='[{"op": "remove", "path": "/spec/webhookdefinitions"}]'; \
 	fi
+
+KUTTL_SUITE ?= default
+KUTTL_NAMESPACE ?= placement-kuttl-tests
+KUTTL_SUITE_DIR ?= tests/kuttl/test-suites/$(KUTTL_SUITE)
+
+.PHONY: kuttl-test-prep
+kuttl-test-prep:
+	oc apply -k $(KUTTL_SUITE_DIR)/deps/ --timeout=120s
+	oc wait -n $(KUTTL_NAMESPACE) openstackcontrolplane openstack --for condition=Ready --timeout=300s
+
+.PHONY: kuttl-test-run
+kuttl-test-run: export NAMESPACE = $(KUTTL_NAMESPACE)
+kuttl-test-run:
+	oc kuttl test --v 1 --start-kind=false --config $(KUTTL_SUITE_DIR)/config.yaml
+
+.PHONY: kuttl-test
+kuttl-test: kuttl-test-prep kuttl-test-run
+
+.PHONY: kuttl-test-cleanup
+kuttl-test-cleanup:
+	# only cleanup if the $(KUTTL_NAMESPACE) exists
+	$(eval namespace_exists=$(shell oc get namespace $(KUTTL_NAMESPACE) --ignore-not-found -o name))
+	# We need to order the deletion. Simply deleting the namespace will
+	# result in errors in mariadb- and keystone-operator and then
+	# finalizer removal get stuck blocking the namespace deletion.
+	if [ "${namespace_exists}" != "" ]; then \
+		if [ "$(KUTTL_SUITE)" == "autoscaling" ]; then \
+			oc delete --wait=true --all=true -n $(KUTTL_NAMESPACE) --timeout=120s Autoscaling; \
+		fi; \
+		if [ "$(KUTTL_SUITE)" == "ceilometer" ]; then \
+			oc delete --wait=true --all=true -n $(KUTTL_NAMESPACE) --timeout=120s Ceilometer; \
+		fi; \
+		if [ "$(KUTTL_SUITE)" == "default" ]; then \
+			oc delete --wait=true --all=true -n $(KUTTL_NAMESPACE) --timeout=120s Telemetry; \
+		fi; \
+		oc delete --wait=true --all=true -n $(KUTTL_NAMESPACE) --timeout=120s OpenStackControlPlane; \
+		oc delete --wait=true namespace $(KUTTL_NAMESPACE); \
+	else \
+		echo "Namespce already cleaned up. Nothing to do"; \
+	fi

renovate.json

@@ -1,6 +1,6 @@
 {
   "extends": [
-    "github>openstack-k8s-operators/renovate-config"
+    "github>openstack-k8s-operators/renovate-config:default.json5"
   ],
   "constraints": {
     "go": "1.19"

tests/kuttl/common/assert_sample_deployment.yaml

@@ -5,9 +5,6 @@ metadata:
   - PlacementAPI
   name: placement
 spec:
-  customServiceConfig: |
-    [DEFAULT]
-    debug = true
   databaseInstance: openstack
   databaseUser: placement
   debug:
@@ -18,7 +15,7 @@ spec:
     service: PlacementPassword
   preserveJobs: false
   replicas: 1
-  secret: osp-secret
+  secret: placement-secret
   serviceUser: placement
 status:
   databaseHostname: openstack
@@ -188,12 +185,12 @@ spec:
           valueFrom:
             secretKeyRef:
               key: PlacementDatabasePassword
-              name: osp-secret
+              name: placement-secret
         - name: PlacementPassword
           valueFrom:
             secretKeyRef:
               key: PlacementPassword
-              name: osp-secret
+              name: placement-secret
         - name: DatabaseHost
           value: openstack
         - name: DatabaseName

tests/kuttl/test-suites/default/config.yaml

@@ -0,0 +1,14 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestSuite
+reportFormat: JSON
+reportName: kuttl-test-placement
+namespace: placement-kuttl-tests
+# we could set this lower, but the initial image pull can take a while
+timeout: 300
+parallel: 1
+skipDelete: true
+testDirs:
+  - tests/kuttl/test-suites/default/
+suppress:
+  - events
+artifactsDir: tests/kuttl/test-suites/default/output

tests/kuttl/test-suites/default/deps/OpenStackControlPlane.yaml

@@ -0,0 +1,16 @@
+apiVersion: core.openstack.org/v1beta1
+kind: OpenStackControlPlane
+metadata:
+  name: openstack
+spec:
+  storageClass: "crc-csi-hostpath-provisioner"
+  ironic:
+    enabled: false
+    template:
+      ironicConductors: []
+  manila:
+    enabled: false
+    template:
+      manilaShares: {}
+  horizon:
+    enabled: false

tests/kuttl/test-suites/default/deps/infra.yaml

@@ -0,0 +1,32 @@
+apiVersion: core.openstack.org/v1beta1
+kind: OpenStackControlPlane
+metadata:
+  name: openstack
+spec:
+  mariadb:
+    enabled: false
+  galera:
+    enabled: true
+    templates:
+      openstack:
+        storageRequest: 500M
+      openstack-cell1:
+        storageRequest: 500M
+  rabbitmq:
+    templates:
+      rabbitmq:
+        replicas: 1
+      rabbitmq-cell1:
+        replicas: 1
+  memcached:
+    templates:
+      memcached:
+        replicas: 1
+  ovn:
+    enabled: false
+    template:
+      ovnController:
+        external-ids:
+          ovn-encap-type: geneve
+  ovs:
+    enabled: false

tests/kuttl/test-suites/default/deps/keystone.yaml

@@ -0,0 +1,9 @@
+apiVersion: core.openstack.org/v1beta1
+kind: OpenStackControlPlane
+metadata:
+  name: openstack
+spec:
+  keystone:
+    template:
+      databaseInstance: openstack
+      secret: placement-secret

tests/kuttl/test-suites/default/deps/kustomization.yaml

@@ -0,0 +1,42 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: placement-kuttl-tests
+
+secretGenerator:
+- literals:
+  - AdminPassword=password
+  - DbRootPassword=password
+  - DatabasePassword=password
+  - KeystoneDatabasePassword=password
+  - PlacementPassword=password
+  - PlacementDatabasePassword=password
+  - GlancePassword=password
+  - GlanceDatabasePassword=password
+  - NeutronPassword=password
+  - NeutronDatabasePassword=password
+  - NovaPassword=password
+  - NovaAPIDatabasePassword=password
+  - NovaCell0DatabasePassword=password
+  - NovaCell1DatabasePassword=password
+  - MetadataSecret=42
+  name: placement-secret
+generatorOptions:
+  disableNameSuffixHash: true
+  labels:
+    type: placement-secret
+
+resources:
+- namespace.yaml
+- OpenStackControlPlane.yaml
+
+patches:
+- patch: |-
+    apiVersion: core.openstack.org/v1beta1
+    kind: OpenStackControlPlane
+    metadata:
+      name: openstack
+    spec:
+      secret: placement-secret
+- path: infra.yaml
+- path: keystone.yaml
+- path: placement.yaml

tests/kuttl/test-suites/default/deps/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: placement-kuttl-tests

tests/kuttl/test-suites/default/deps/placement.yaml

@@ -0,0 +1,9 @@
+apiVersion: core.openstack.org/v1beta1
+kind: OpenStackControlPlane
+metadata:
+  name: openstack
+spec:
+  placement:
+    template:
+      databaseInstance: openstack
+      secret: placement-secret

tests/kuttl/test-suites/default/output/kuttl-default-results.json

@@ -0,0 +1,36 @@
+{
+   "name": "",
+   "tests": 3,
+   "failures": 0,
+   "time": "214.322",
+   "testsuite": [
+     {
+       "tests": 3,
+       "failures": 0,
+       "timestamp": "2023-08-04T11:39:21.251975663+02:00",
+       "time": "202.907",
+       "name": "test/kuttl/test-suites/default/",
+       "testcase": [
+         {
+           "classname": "default",
+           "name": "deps",
+           "timestamp": "2023-08-04T11:39:21.252056484+02:00",
+           "time": "0.000"
+         },
+         {
+           "classname": "default",
+           "name": "scale-tests",
+           "timestamp": "2023-08-04T11:39:21.257387947+02:00",
+           "time": "202.901",
+           "assertions": 69
+         },
+         {
+           "classname": "default",
+           "name": "output",
+           "timestamp": "2023-08-04T11:42:44.158903202+02:00",
+           "time": "0.000"
+         }
+       ]
+     }
+   ]
+ }

tests/kuttl/test-suites/default/output/kuttl-test-placement.json

@@ -0,0 +1,40 @@
+{
+   "name": "",
+   "tests": 3,
+   "failures": 1,
+   "time": "334.084",
+   "testsuite": [
+     {
+       "tests": 3,
+       "failures": 1,
+       "timestamp": "2024-01-05T16:01:14.739927108+01:00",
+       "time": "322.664",
+       "name": "tests/kuttl/test-suites/default/",
+       "testcase": [
+         {
+           "classname": "default",
+           "name": "deps",
+           "timestamp": "2024-01-05T16:01:14.740035321+01:00",
+           "time": "0.000"
+         },
+         {
+           "classname": "default",
+           "name": "scale-tests",
+           "timestamp": "2024-01-05T16:01:14.743540703+01:00",
+           "time": "322.661",
+           "assertions": 18,
+           "failure": {
+             "text": "resource Deployment:placement-kuttl-tests/placement: .spec.replicas: value mismatch, expected: 2 != actual: 1",
+             "message": "failed in step 3-scale-placementapi"
+           }
+         },
+         {
+           "classname": "default",
+           "name": "output",
+           "timestamp": "2024-01-05T16:06:37.404168159+01:00",
+           "time": "0.000"
+         }
+       ]
+     }
+   ]
+ }

tests/kuttl/test-suites/default/scale-tests/00-cleanup-placement.yaml

@@ -0,0 +1,7 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+delete:
+- apiVersion: placement.openstack.org/v1beta1
+  kind: PlacementAPI
+  name: placement-kuttl
+  namespace: placement-kuttl-tests