Kubernetes testing

* Set up a Kubernetes cluster (using Kind) with an image registry and an ingress controller
* Build container images using wildfly:image goal and push them to the cluster image registry
* Add an ingress to access the application (until wildfly/wildfly-charts#52 is resolved)

* Add tests for helloworld & todo-backend

Signed-off-by: Jeff Mesnil <jmesnil@redhat.com>

.github/workflows/testing.yml

@@ -0,0 +1,142 @@
+name: Kubernetes Testing
+
+on:
+  - push
+  - pull_request
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+          cache: 'maven'
+
+      - name: Install Helm
+        uses: azure/setup-helm@v3
+        with:
+          version: v3.12.1
+
+      - name: Create k8s Kind Cluster
+        uses: helm/kind-action@v1.5.0
+        with:
+          node_image: "kindest/node:v1.27.3"
+          config: ./test-setup/kind-config.yaml
+
+      - name: Setup Image registry & ingress controller
+        run: |-
+          ./test-setup/setup-image-registry.sh
+          ./test-setup/setup-nginx-ingress-controller.sh
+           sudo echo "127.0.0.1 wildfly.local" | sudo tee -a /etc/hosts
+
+      - name: Run helloworld test
+        run: |-
+          cd helloworld
+          mvn -B -Popenshift package wildfly:image
+          docker tag helloworld localhost:5001/helloworld
+          docker push localhost:5001/helloworld
+          
+          cat << EOF > override.yaml
+          image:
+            name: localhost:5001/helloworld
+          build:
+            enabled: false
+          deploy:
+            route:
+              enabled: false
+          EOF
+
+          helm install helloworld \
+            --repo https://docs.wildfly.org/wildfly-charts wildfly \
+            --wait --timeout=90s \
+            -f charts/helm.yaml -f override.yaml
+
+          cat << EOF | kubectl create -f -
+          apiVersion: networking.k8s.io/v1
+          kind: Ingress
+          metadata:
+            name: helloworld-ingress
+          spec:
+            rules:
+              - host: wildfly.local
+                http:
+                 paths:
+                   - path: /
+                     pathType: Prefix
+                     backend:
+                       service:
+                        name: helloworld
+                        port:
+                          number: 8080
+          EOF
+          sleep 5
+
+          SERVER_HOST=http://wildfly.local/ mvn -B -P integration-testing verify
+
+          helm delete helloworld
+          kubectl delete ingress helloworld-ingress
+
+          echo "  * :white_check_mark: Helloworld" >> $GITHUB_STEP_SUMMARY
+
+      - name: Run todo-backend test
+        run: |-
+          cd todo-backend
+          mvn -B -Popenshift package wildfly:image
+          docker tag todo-backend localhost:5001/todo-backend
+          docker push localhost:5001/todo-backend
+          
+          cat << EOF > override.yaml
+          wildfly:
+            image:
+              name: localhost:5001/todo-backend
+            build:
+              enabled: false
+            deploy:
+              route:
+                enabled: false
+          EOF
+
+          helm dependency update todo-backend-chart/
+          echo install todo-backend helm chart
+          helm install todo-backend todo-backend-chart \
+            -f override.yaml
+
+          kubectl wait \
+            --for=condition=ready pod \
+            --selector=app.kubernetes.io/name=todo-backend \
+            --timeout=120s  
+
+          cat << EOF | kubectl create -f -
+          apiVersion: networking.k8s.io/v1
+          kind: Ingress
+          metadata:
+            name: todo-backend-ingress
+          spec:
+            rules:
+              - host: wildfly.local
+                http:
+                 paths:
+                   - path: /
+                     pathType: Prefix
+                     backend:
+                       service:
+                        name: todo-backend
+                        port:
+                          number: 8080
+          EOF
+          sleep 5
+
+          SERVER_HOST=http://wildfly.local/ mvn -B -P arq-remote verify
+
+          helm delete todo-backend
+          kubectl delete ingress todo-backend-ingress
+
+          echo "  * :white_check_mark: TODO Backend" >> $GITHUB_STEP_SUMMARY

test-setup/kind-config.yaml

@@ -0,0 +1,21 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry]
+    config_path = "/etc/containerd/certs.d"
+nodes:
+- role: control-plane
+  kubeadmConfigPatches:
+  - |
+    kind: InitConfiguration
+    nodeRegistration:
+      kubeletExtraArgs:
+        node-labels: "ingress-ready=true"
+  extraPortMappings:
+  - containerPort: 80
+    hostPort: 80
+    protocol: TCP
+  - containerPort: 443
+    hostPort: 443
+    protocol: TCP  

test-setup/setup-image-registry.sh

@@ -0,0 +1,49 @@
+#!/bin/sh
+set -o errexit
+
+echo 1. Create registry container unless it already exists
+reg_name='kind-registry'
+reg_port='5001'
+if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then
+  docker run \
+    -d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" \
+    registry:2
+fi
+
+echo 2. Add the registry config to the nodes
+
+# This is necessary because localhost resolves to loopback addresses that are
+# network-namespace local.
+# In other words: localhost in the container is not localhost on the host.
+#
+# We want a consistent name that works from both ends, so we tell containerd to
+# alias localhost:${reg_port} to the registry container when pulling images
+kind get nodes --name chart-testing
+REGISTRY_DIR="/etc/containerd/certs.d/localhost:${reg_port}"
+for node in $(kind get nodes --name chart-testing); do
+  docker exec "${node}" mkdir -p "${REGISTRY_DIR}"
+  cat <<EOF | docker exec -i "${node}" cp /dev/stdin "${REGISTRY_DIR}/hosts.toml"
+[host."http://${reg_name}:5000"]
+EOF
+done
+
+echo 3. Connect the registry to the cluster network if not already connected
+# This allows kind to bootstrap the network but ensures they're on the same network
+if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name}")" = 'null' ]; then
+  docker network connect "kind" "${reg_name}"
+fi
+
+echo 4. Document the local registry
+# https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: local-registry-hosting
+  namespace: kube-public
+data:
+  localRegistryHosting.v1: |
+    host: "localhost:${reg_port}"
+    help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
+EOF
+

test-setup/setup-nginx-ingress-controller.sh

@@ -0,0 +1,9 @@
+kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml
+
+echo Wait for the ingress nginx controller to be ready...
+sleep 20
+
+kubectl wait --namespace ingress-nginx \
+  --for=condition=ready pod \
+  --selector=app.kubernetes.io/component=controller \
+  --timeout=90s  

test-setup/setup.sh

@@ -0,0 +1,6 @@
+kind create cluster \
+  --name chart-testing \
+  --config=./kind-config.yaml
+
+./setup-image-registry.sh
+./setup-nginx-ingress-controller.sh

todo-backend/pom.xml

@@ -317,6 +317,9 @@
                                     <artifactId>wildfly-datasources-galleon-pack</artifactId>
                                     <version>${version.wildfly-datasources-galleon-pack}</version>
                                 </feature-pack>
+                                <feature-pack>
+                                    <location>org.wildfly.cloud:wildfly-cloud-galleon-pack:${version.cloud.fp}</location>
+                                </feature-pack>
                             </feature-packs>
                             <layers>
                                 <layer>cloud-server</layer>

todo-backend/todo-backend-chart/values.yaml

@@ -31,7 +31,6 @@ postgresql:
         enabled: false
         securityContext:
             runAsUser: "auto"
-
 wildfly:
     build:
       uri: https://github.com/wildfly/quickstart.git
@@ -53,41 +52,17 @@ wildfly:
           value: todo-backend-postgresql
         - name: POSTGRESQL_SERVICE_PORT
           value: "5432"
-        # Env vars to configure CORS filter
-        - name: FILTERS
-          value: acao, acam, acah, acac, acma
-        - name: acao_FILTER_RESPONSE_HEADER_NAME
-          value: Access-Control-Allow-Origin
-        - name: acao_FILTER_RESPONSE_HEADER_VALUE
-          value: "*"
-        - name: acao_FILTER_REF_NAME
-          value: Access-Control-Allow-Origin
-        - name: acam_FILTER_RESPONSE_HEADER_NAME
-          value: Access-Control-Allow-Methods
-        - name: acam_FILTER_RESPONSE_HEADER_VALUE
-          value: GET, POST, OPTION, PUT, DELETE, PATCH
-        - name: acam_FILTER_REF_NAME
-          value: Access-Control-Allow-Methods
-        - name: acah_FILTER_RESPONSE_HEADER_NAME
-          value: Access-Control-Allow-Headers
-        - name: acah_FILTER_RESPONSE_HEADER_VALUE
-          value: accept, authorization, content-type, x-requested-with
-        - name: acah_FILTER_REF_NAME
-          value: Access-Control-Allow-Headers
-        - name: acac_FILTER_RESPONSE_HEADER_NAME
-          value: Access-Control-Allow-Credentials
-        - name: acac_FILTER_RESPONSE_HEADER_VALUE
-          value: "true"
-        - name: acac_FILTER_REF_NAME
-          value: Access-Control-Allow-Credentials
-        - name: acma_FILTER_RESPONSE_HEADER_NAME
-          value: Access-Control-Max-Age
-        - name: acma_FILTER_RESPONSE_HEADER_VALUE
-          value: "1"
-        - name: acma_FILTER_REF_NAME
-          value: Access-Control-Max-Age
-        # Env to avoid OOME
-        - name: GC_MAX_METASPACE_SIZE
-          value: "256"
-        - name: GC_METASPACE_SIZE
-          value: "96"
+      startupProbe:
+        httpGet:
+          path: /health/ready
+          port: 9990
+        initialDelaySeconds: 20
+        failureThreshold: 5
+        periodSeconds: 10
+      livenessProbe:
+        httpGet:
+          path: /health/ready
+          port: 9990
+        initialDelaySeconds: 20
+        failureThreshold: 5
+        periodSeconds: 10