make compute kit job deploy openstack and run minimal tempest

.zuul.yaml

@@ -11,10 +11,10 @@
       This is the base job for all nova-operator tests that will deploy
       openstack. the base job does not deploy openstack but it does use
       the nova-operator-content-provider job to provide a registry with
-      built operators and installs them on a crc envionment. child jobs
-      should take this deploy crc and use the installed operators to
+      built operators and installs them on a crc environment. Child jobs
+      should take this deployed crc and use the installed operators to
       deploy openstack and then test it. This job will fetch the job
-      output directory so child jobs can just add addtional logs to that
+      output directory so child jobs can just add additional logs to that
       in there post-playbooks.
     parent: cifmw-base-crc-openstack
     abstract: true
@@ -31,6 +31,7 @@
       crc_parameters: "--memory 18000 --disk-size 120 --cpus 6"
       network_isolation: true
       crc_attach_default_interface: true
+      cifmw_edpm_prepare_skip_crc_storage_creation: true
 
 - job:
     name: nova-operator-compute-kit
@@ -45,11 +46,11 @@
     requires:
       - "nova-operator-content"
     run:
-      - ci/nova-operator-compute-kit/playbooks/run.yaml
+      - ci/nova-operator-compute-kit/playbooks/deploy-openstack.yaml
+      - ci/nova-operator-compute-kit/playbooks/tempest.yaml
 
 - project:
     name: openstack-k8s-operators/nova-operator
     github-check:
       jobs:
         - nova-operator-content-provider
-        - nova-operator-compute-kit

ci/nova-operator-base/playbooks/collect-logs.yaml

@@ -7,7 +7,7 @@
       ansible.builtin.file:
         path: "{{ ansible_user_dir }}/zuul-output/logs"
         state: directory
-        mode: 0755
+        mode: "0755"
 
 - hosts: controller
   name: Collect logs on the controller
@@ -17,17 +17,11 @@
       ansible.builtin.file:
         path: "{{ ansible_user_dir }}/zuul-output/logs/controller"
         state: directory
-        mode: 0755
+        mode: "0755"
 
-    - name: Collect pod logs
+    - name: Collect general openshift cr info
       ansible.builtin.shell: |
         source ~/.bashrc
-        pushd {{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/install_yamls/out/openstack
-        mkdir {{ ansible_user_dir }}/zuul-output/logs/openstack
-        cp namespace.yaml {{ ansible_user_dir }}/zuul-output/logs/openstack
-        cp -R openstack/cr {{ ansible_user_dir }}/zuul-output/logs/openstack
-        cp -R dataplane/cr {{ ansible_user_dir }}/zuul-output/logs/openstack
-        popd
         cp /etc/resolv.conf .
         cp /etc/hosts .
         ip a > network.txt
@@ -41,14 +35,27 @@
         oc get -o yaml openstackdataplane > openstackdataplane.yaml
         oc get -o yaml openstackdataplanerole > openstackdataplanerole.yaml
         oc get -o yaml openstackdataplanenode > openstackdataplanenode.yaml
-        all_pods=$(oc get pods -o name)
-        mkdir pod
-        for pod in $all_pods; do
-          echo $pod
-          oc logs $pod > ${pod}-logs.txt
-          oc get -o yaml $pod > ${pod}.yaml
-          oc describe $pod > ${pod}-describe.txt
-        done
+        pod_dir="{{ ansible_user_dir }}/zuul-output/logs/controller/pods"
+        mkdir ${pod_dir}
+        pushd ${pod_dir}
+          all_pods=$(oc get pods | awk '{print $1}' | awk -F '.' '{print $1}')
+          for pod in $all_pods; do
+            echo $pod
+            oc logs pod/${pod} > ${pod}-logs.txt
+            oc get -o yaml pod/${pod} > ${pod}.yaml
+            oc describe pod/${pod} > ${pod}-describe.txt
+          done
+        popd
+        crd_logs="{{ ansible_user_dir }}/zuul-output/logs/controller/crd"
+        mkdir ${crd_logs}
+        pushd ${crd_logs}
+          all_crds=$(oc get crd | grep openstack | awk '{print $1}' | awk -F '.' '{print $1}')
+          for cr in $all_crds; do
+            echo $crd
+            oc get -o yaml $cr > ${cr}.yaml
+            oc describe $cr > ${cr}-describe.txt
+          done
+        popd
       args:
         chdir: "{{ ansible_user_dir }}/zuul-output/logs/controller"
       changed_when: true

ci/nova-operator-base/playbooks/pre.yaml

@@ -19,10 +19,6 @@
         oc patch operatorhubs/cluster --type merge --patch '{"spec":{"sources":[{"disabled": true,"name": "redhat-marketplace"}]}}'
       changed_when: true
 
-    - name: Create Openstack Namesapce
-      ansible.builtin.shell: |
-        oc apply -f "{{ nova_operator_basedir }}/ci/nova-operator-base/namespace.yaml"
-
     - name: Use OpenStack Namespace
       ansible.builtin.command: oc project openstack
       changed_when: false
@@ -35,3 +31,10 @@
       changed_when: false
       retries: 30
       delay: 30
+
+    # Fixme: {{ zuul_ssh_key_dest }} is not defined in the zuul job so we need to
+    # figure out how to get the ssh key from the zuul job or generate a new one.
+    # - name: register zuul build ssh key  as a secret
+    #   ansible.builtin.command: |
+    #     oc create secret generic zuul-ssh-key  --type 'kubernetes.io/ssh-auth' \
+    #     --from-file=ssh-privatekey=~/.ssh/{{ zuul_ssh_key_dest }} --from-file=ssh-publickey=~/.ssh/{{ zuul_ssh_key_dest }}.pub

ci/nova-operator-compute-kit/playbooks/deploy-openstack.yaml

@@ -0,0 +1,24 @@
+---
+# Note make download_tools and make openstack commands run
+# in https://github.com/rdo-infra/review.rdoproject.org-config/blob/master/playbooks/crc/openstack.yaml
+# We can override the openstack operator image by defining openstack_img in the zuul job
+
+- hosts: controller
+  name: Deploy Openstack Operators
+  vars:
+    nova_operator_basedir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/nova-operator"
+  tasks:
+    - name: Deploy OpenStack
+      ansible.builtin.command: "oc apply -k {{ nova_operator_basedir }}/ci/nova-operator-compute-kit/topology/ci"
+      changed_when: true
+
+    # note(sean-k-mooney): the openstack operator does not properly init its ready status
+    # so we need to wait for the nova and openstack cr to be ready before we can proceed.
+    # This can be removed later once the operator is fixed.
+    - name: Make sure Nova CR is ready
+      ansible.builtin.command: oc wait Nova nova --for condition=Ready --timeout=120s
+      changed_when: false
+
+    - name: Make sure Openstack CR is ready
+      ansible.builtin.command: oc wait OpenstackControlPlane openstack --for condition=Ready --timeout=120s
+      changed_when: false

ci/nova-operator-compute-kit/playbooks/tempest.yaml

@@ -0,0 +1,36 @@
+---
+- name: Tempest
+  hosts: controller
+  vars:
+    nova_operator_basedir: "{{ ansible_user_dir }}/src/github.com/openstack-k8s-operators/nova-operator"
+  tasks:
+    - name: Use OpenStack Namespace
+      ansible.builtin.command: oc project openstack
+      changed_when: false
+
+    - name: Deploy OpenStack
+      ansible.builtin.command: "oc apply -k {{ nova_operator_basedir }}/ci/nova-operator-compute-kit/topology/tempest"
+      changed_when: true
+
+    - name: Wait for tempest-worker pod to start
+      ansible.builtin.command: oc wait pod tempest-worker --for condition=Ready --timeout=120s
+      changed_when: false
+
+    - name: Create local tempest log dir
+      ansible.builtin.file:
+        path: "{{ ansible_user_dir }}/zuul-output/logs/controller/tempest"
+        state: directory
+        mode: "0755"
+
+    - name: Run tempest tests
+      ansible.builtin.command: "oc rsh pod/tempest-worker /var/lib/tempest/run_tempest.sh"
+      register: tempest_output
+      changed_when: true
+
+    - name: Save tempest output
+      ansible.builtin.copy:
+        dest: "{{ ansible_user_dir }}/zuul-output/logs/controller/tempest/tempest-output.txt"
+        content: "{{ tempest_output.stdout }}"
+        mode: "0644"
+
+    # TODO(sean-k-mooney): copy the html test report to the zuul output dir.

ci/nova-operator-compute-kit/topology/ci/kustomization.yaml

@@ -1,14 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: openstack
-
-bases:
- - ../network-isolation
-
-# patches:
-#   - path: keystone.yaml
-#   - path: glance.yaml
-#   - path: placement.yaml
-#   - path: neutron.yaml
-#   - path: nova.yaml
-#   - path: infra.yaml
+patches:
+- path: storage.yaml
+resources:
+- ../base

ci/nova-operator-compute-kit/topology/ci/storage.yaml

@@ -3,6 +3,7 @@ kind: OpenStackControlPlane
 metadata:
   name: openstack
 spec:
+   storageClass: "crc-csi-hostpath-provisioner"
    glance:
     template:
-      storage-class:
+      storageClass: "crc-csi-hostpath-provisioner"

ci/nova-operator-compute-kit/topology/sno/storage.yaml

@@ -3,7 +3,12 @@ kind: OpenStackControlPlane
 metadata:
   name: openstack
 spec:
-   storageClass: "lvms-vg1"
-   glance:
+  storageClass: "lvms-vg1"
+  glance:
     template:
-      storageClass: "lvms-vg1"
+      glanceAPIInternal:
+        replicas: 1
+      glanceAPIExternal:
+        replicas: 1
+      storageClass: "nfs"
+      storageRequest: 15G

ci/nova-operator-compute-kit/topology/tempest/kustomization.yaml

@@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: openstack
+# TODO: add generation of tempest test specification and run_tempest.sh script.
+# currently we are just running the minimal set of tempest tests using the
+# run_tempest.sh script from the tempest container. To be able to control the
+# concurrency of the tests we need to generate a replacement script which can
+# be injected into the container via a configmap. The is also needed to enable
+# us to do a two phase run where we run the api tests in parallel and then run the
+# scenario tests serially. as is done in the integrated-compute job.
+# https://github.com/openstack/tempest/blob/master/tox.ini#L201-L214
+resources:
+- tempest.yaml

ci/nova-operator-compute-kit/topology/tempest/tempest.yaml

@@ -0,0 +1,38 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: tempest-worker
+spec:
+  volumes:
+    - name: tempest-workdir
+      emptyDir: {}
+    - name: cloud-passwd
+      secret:
+         secretName: openstack-config-secret
+    - name: clouds-config
+      configMap:
+          name: openstack-config
+    - name: pre-install
+      emptyDir: {}
+
+  containers:
+    - name: tempest-container
+      image: quay.io/podified-antelope-centos9/openstack-tempest:current-podified
+      command: ["/usr/bin/dumb-init", "sleep", "infinity"]
+      volumeMounts:
+        - mountPath: "/home"
+          name: tempest-workdir
+        - mountPath: "/etc/openstack"
+          name: pre-install
+        - mountPath: "/etc/openstack/clouds.yaml"
+          name: clouds-config
+          subPath: clouds.yaml
+        - mountPath: "/etc/openstack/secure.yaml"
+          name: cloud-passwd
+          subPath: secure.yaml
+      env:
+        - name: OS_CLOUD
+          valueFrom:
+            configMapKeyRef:
+              name: openstack-config
+              key: OS_CLOUD