test(chart): Chart template render and assert output

Makefile

@@ -364,30 +364,34 @@ test_video: video hub chrome firefox edge
 	docker run -v $$(pwd):$$(pwd) -w $$(pwd) $(FFMPEG_BASED_NAME)/ffmpeg:$(FFMPEG_BASED_TAG) -v error -i ./tests/videos/edge_video.mp4 -f null - 2>error.log
 
 chart_setup_env:
-	./tests/K8s/chart_setup_env.sh
+	./tests/charts/make/chart_setup_env.sh
 
 chart_test: chart_lint \
+ chart_test_template \
  chart_install_chrome \
  chart_install_firefox \
  chart_install_edge
 
+chart_test_template:
+	./tests/charts/bootstrap.sh
+
 chart_cluster_setup:
-	VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/K8s/chart_cluster_setup.sh
+	VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/charts/make/chart_cluster_setup.sh
 
 chart_lint:
-	./tests/K8s/chart_lint.sh
+	./tests/charts/make/chart_lint.sh
 
 chart_install_chrome:
-	VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/K8s/chart_install.sh NodeChrome
+	VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/charts/make/chart_install.sh NodeChrome
 
 chart_install_firefox:
-	VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/K8s/chart_install.sh NodeFirefox
+	VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/charts/make/chart_install.sh NodeFirefox
 
 chart_install_edge:
-	VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/K8s/chart_install.sh NodeEdge
+	VERSION=$(TAG_VERSION) NAMESPACE=$(NAMESPACE) ./tests/charts/make/chart_install.sh NodeEdge
 
 chart_cluster_cleanup:
-	./tests/K8s/chart_cluster_cleanup.sh
+	./tests/charts/make/chart_cluster_cleanup.sh
 
 .PHONY: \
 	all \

charts/selenium-grid/README.md

@@ -36,13 +36,13 @@ helm install selenium-grid --set ingress.hostname=selenium-grid.k8s.local docker
 Selenium Grid has the ability to autoscaling browser nodes up/down based on the pending requests in the 
 session queue.
 
-To do this [KEDA](https://keda.sh/docs/2.12/scalers/selenium-grid-scaler/) is used. When enabling
+To do this [KEDA](https://keda.sh/docs/latest/scalers/selenium-grid-scaler/) is used. When enabling
 autoscaling using `autoscaling.enabling` KEDA is installed automatically. To instead use an existing
 installation of KEDA you can enable autoscaling with `autoscaling.enableWithExistingKEDA` instead.
 
 KEDA can scale either with
-[deployments](https://keda.sh/docs/2.12/concepts/scaling-deployments/#scaling-of-deployments-and-statefulsets)
-or [jobs](https://keda.sh/docs/2.12/concepts/scaling-jobs/) and the charts support both types. This
+[deployments](https://keda.sh/docs/latest/concepts/scaling-deployments/#scaling-of-deployments-and-statefulsets)
+or [jobs](https://keda.sh/docs/latest/concepts/scaling-jobs/) and the charts support both types. This
 chart support both modes.  It is controlled with `autoscaling.scalingType` that can be set to either
 job (default) or deployment.
 

charts/selenium-grid/TESTING.md

@@ -4,28 +4,46 @@ All related testing to this helm chart will be documented in this file.
 
 ## Test Traceability Matrix
 
-| Features               | TC Description                                                       | Coverage |
-|------------------------|----------------------------------------------------------------------|----------|
-| Basic Auth             | Basic Auth is disabled                                               | ✓  |
-|                        | Basic Auth is enabled                                                | ✗  |
-| Auto scaling           | Auto scaling with `enableWithExistingKEDA` is `true`                 | ✓  |
-|                        | Auto scaling with `scalingType` is `job`                             | ✓  |
-|                        | Auto scaling with `scalingType` is `deployment`                      | ✗  |
-|                        | Auto scaling with `autoscaling.scaledOptions.minReplicaCount` is `0` | ✓  |
-| Ingress                | Ingress is enabled without `hostname`                                | ✓  |
-|                        | Ingress is enabled with `hostname` is set                            | ✗  |
-|                        | Hub `sub-path` is set with Ingress `ImplementationSpecific` paths    | ✓  |
-| Distributed components | `isolateComponents` is enabled                                       | ✓  |
-|                        | `isolateComponents` is disabled                                      | ✗  |
-| Browser Nodes          | Node `nameOverride` is set                                           | ✓  |
-|                        | Sanity tests in node                                                 | ✓  |
-|                        | Video recorder is enabled in node                                    | ✗  |
-|                        | Node `extraEnvironmentVariables` is set value                        | ✓  |
-| General                | Set new image registry via `global.seleniumGrid.imageRegistry`       | ✓  |
-| Tracing                | Enable tracing via `SE_ENABLE_TRACING`                               | ✓  |
-|                        | Disable tracing via `SE_ENABLE_TRACING`                              | ✗  |
-
-## Build & test Docker images with Helm charts
+| Features               | TC Description                                                       | Coverage | Test via |
+|------------------------|----------------------------------------------------------------------|----------|----------|
+| Basic Auth             | Basic Auth is disabled                                               | ✓  | Cluster  |
+|                        | Basic Auth is enabled                                                | ✗  |          |
+| Auto scaling           | Auto scaling with `enableWithExistingKEDA` is `true`                 | ✓  | Cluster  |
+|                        | Auto scaling with `scalingType` is `job`                             | ✓  | Cluster  |
+|                        | Auto scaling with `scalingType` is `deployment`                      | ✗  |          |
+|                        | Auto scaling with `autoscaling.scaledOptions.minReplicaCount` is `0` | ✓  | Cluster  |
+| Ingress                | Ingress is enabled without `hostname`                                | ✓  | Cluster  |
+|                        | Ingress is enabled with `hostname` is set                            | ✗  |          |
+|                        | Hub `sub-path` is set with Ingress `ImplementationSpecific` paths    | ✓  | Cluster  |
+| Distributed components | `isolateComponents` is enabled                                       | ✓  | Cluster  |
+|                        | `isolateComponents` is disabled                                      | ✗  |          |
+| Browser Nodes          | Node `nameOverride` is set                                           | ✓  | Cluster  |
+|                        | Sanity tests in node                                                 | ✓  | Cluster  |
+|                        | Video recorder is enabled in node                                    | ✗  |          |
+|                        | Node `extraEnvironmentVariables` is set value                        | ✓  | Cluster  |
+| General                | Set new image registry via `global.seleniumGrid.imageRegistry`       | ✓  | Cluster  |
+|                        | Components are able to set `.affinity`                               | ✓  | Template |
+| Tracing                | Enable tracing via `SE_ENABLE_TRACING`                               | ✓  | Cluster  |
+|                        | Disable tracing via `SE_ENABLE_TRACING`                              | ✗  |          |
+
+## Test Chart Template
+- By using `helm template` command, the chart template is tested without installing it to Kubernetes cluster.
+- Templates are rendered and the output as a YAML manifest file. The manifest file is then asserted with [pyyaml](https://pyyaml.org/wiki/PyYAMLDocumentation).
+- Set of values are used to render the templates located in [tests/charts/templates/render](../../tests/charts/templates/render).
+
+```bash
+# Back to root directory
+cd ../..
+
+# Build chart dependencies and lint
+make chart_lint
+
+# Test chart template
+make chart_test_template
+```
+- Build chart dependencies and lint requires [Chart Testing `ct`](https://github.com/helm/chart-testing). There is a config file [ct.yaml](../../tests/charts/config/ct.yaml) to configure the chart testing.
+
+## Build & test Docker images with deploy to Kubernetes cluster
 Noted: These `make` commands are composed and tested on Linux x86_64.
 Run entire commands to build and test Docker images with Helm charts in local environment.
 
@@ -48,3 +66,5 @@ make chart_test
 # Cleanup Kubernetes cluster
 make chart_cluster_cleanup
 ```
+- Setup Kubernetes environment requires [Kind](https://kind.sigs.k8s.io/docs/user/quick-start/) and [Kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/).
+- Set of values are used to deploy the chart to Kubernetes cluster located in [tests/charts/ci](../../tests/charts/ci).

tests/charts/bootstrap.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+mkdir -p tests/tests
+cd tests || true
+
+if [ "${CI:-false}" = "false" ]; then
+  pip3 install virtualenv | grep -v 'Requirement already satisfied'
+  virtualenv docker-selenium-tests
+  source docker-selenium-tests/bin/activate
+fi
+
+python -m pip install pyyaml==6.0.1 \
+                      | grep -v 'Requirement already satisfied'
+
+cd ..
+helm template dummy --values tests/charts/templates/render/dummy.yaml \
+  charts/selenium-grid > ./tests/tests/output_deployment.yaml
+
+python tests/charts/templates/test.py "./tests/tests/output_deployment.yaml"
+ret_code=$?
+
+if [ "${CI:-false}" = "false" ]; then
+  deactivate
+fi
+
+exit $ret_code

charts/selenium-grid/ci/NodeChrome-values.yaml → tests/charts/ci/NodeChrome-values.yaml

@@ -1,4 +1,4 @@
-# This is used in Helm chart testing. This disables the basic auth on selenium grid
+# This is used in Helm chart testing
 # Configuration for chrome nodes
 chromeNode:
   nameOverride: my-chrome-name

charts/selenium-grid/ci/NodeEdge-values.yaml → tests/charts/ci/NodeEdge-values.yaml

@@ -1,4 +1,4 @@
-# This is used in Helm chart testing. This disables the basic auth on selenium grid
+# This is used in Helm chart testing
 # Configuration for chrome nodes
 chromeNode:
   enabled: false

charts/selenium-grid/ci/NodeFirefox-values.yaml → tests/charts/ci/NodeFirefox-values.yaml

@@ -1,4 +1,4 @@
-# This is used in Helm chart testing. This disables the basic auth on selenium grid
+# This is used in Helm chart testing
 # Configuration for chrome nodes
 chromeNode:
   enabled: false

tests/K8s/chart_cluster_setup.sh → tests/charts/make/chart_cluster_setup.sh

@@ -8,7 +8,7 @@ KEDA_NAMESPACE=${KEDA_NAMESPACE:-"keda"}
 INGRESS_NAMESPACE=${INGRESS_NAMESPACE:-"ingress-nginx"}
 SUB_PATH=${SUB_PATH:-"/selenium"}
 CHART_PATH=${CHART_PATH:-"charts/selenium-grid"}
-TEST_VALUES_PATH=${TEST_VALUES_PATH:-"charts/selenium-grid/ci"}
+TEST_VALUES_PATH=${TEST_VALUES_PATH:-"tests/charts/ci"}
 SELENIUM_GRID_HOST=${SELENIUM_GRID_HOST:-"localhost"}
 SELENIUM_GRID_PORT=${SELENIUM_GRID_PORT:-"80"}
 WAIT_TIMEOUT=${WAIT_TIMEOUT:-"90s"}
@@ -17,7 +17,7 @@ SKIP_CLEANUP=${SKIP_CLEANUP:-"false"} # For debugging purposes, retain the clust
 # Function to clean up for retry step on workflow
 cleanup() {
   if [ "${SKIP_CLEANUP}" = "false" ]; then
-    ./tests/K8s/chart_cluster_cleanup.sh
+    ./tests/charts/make/chart_cluster_cleanup.sh
   fi
 }
 
@@ -33,7 +33,7 @@ on_failure() {
 trap 'on_failure' ERR
 
 echo "Create Kind cluster"
-kind create cluster --wait ${WAIT_TIMEOUT} --name ${CLUSTER_NAME} --config tests/K8s/kind-cluster-config.yaml
+kind create cluster --wait ${WAIT_TIMEOUT} --name ${CLUSTER_NAME} --config tests/charts/config/kind-cluster.yaml
 
 echo "Install KEDA core on kind kubernetes cluster"
 kubectl apply --server-side -f https://github.com/kedacore/keda/releases/download/v2.12.1/keda-2.12.1-core.yaml

tests/K8s/chart_install.sh → tests/charts/make/chart_install.sh

@@ -8,7 +8,7 @@ KEDA_NAMESPACE=${KEDA_NAMESPACE:-"keda"}
 INGRESS_NAMESPACE=${INGRESS_NAMESPACE:-"ingress-nginx"}
 SUB_PATH=${SUB_PATH:-"/selenium"}
 CHART_PATH=${CHART_PATH:-"charts/selenium-grid"}
-TEST_VALUES_PATH=${TEST_VALUES_PATH:-"charts/selenium-grid/ci"}
+TEST_VALUES_PATH=${TEST_VALUES_PATH:-"tests/charts/ci"}
 SELENIUM_GRID_HOST=${SELENIUM_GRID_HOST:-"localhost"}
 SELENIUM_GRID_PORT=${SELENIUM_GRID_PORT:-"80"}
 MATRIX_BROWSER=${1:-"NodeChrome"}

tests/K8s/chart_lint.sh → tests/charts/make/chart_lint.sh

@@ -22,7 +22,8 @@ python -m pip install yamale==4.0.4 \
                       | grep -v 'Requirement already satisfied'
 
 cd ..
-ct lint --all --config tests/K8s/chart-testing.yaml
+rm -rf ./charts/**/Chart.lock
+ct lint --all --config tests/charts/config/ct.yaml
 
 if [ "${CI:-false}" = "false" ]; then
   deactivate

tests/charts/templates/render/dummy.yaml

@@ -0,0 +1,24 @@
+# This is dummy values file for chart template testing
+global:
+  seleniumGrid:
+    affinity: &affinity
+      podAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+                - key: app
+                  operator: In
+                  values:
+                    - selenium
+            topologyKey: "kubernetes.io/hostname"
+
+isolateComponents: true
+
+chromeNode:
+  affinity: *affinity
+
+firefoxNode:
+    affinity: *affinity
+
+edgeNode:
+    affinity: *affinity

tests/charts/templates/test.py

@@ -0,0 +1,42 @@
+import yaml
+import unittest
+import sys
+import logging
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+logger = logging.getLogger(__name__)
+
+def load_template(yaml_file):
+    try:
+        with open(yaml_file, 'r') as file:
+            documents = yaml.safe_load_all(file)
+            list_of_documents = [doc for doc in documents]
+            return list_of_documents
+    except yaml.YAMLError as error:
+        print("Error in configuration file: ", error)
+
+class ChartTemplateTests(unittest.TestCase):
+    def test_set_affinity(self):
+        resources_name = ['selenium-chrome-node', 'selenium-distributor', 'selenium-edge-node', 'selenium-firefox-node',
+                'selenium-event-bus', 'selenium-router', 'selenium-session-map', 'selenium-session-queue']
+        count = 0
+        for doc in LIST_OF_DOCUMENTS:
+            if doc['metadata']['name'] in resources_name and doc['kind'] == 'Deployment':
+                self.assertTrue(doc['spec']['template']['spec']['affinity']['podAffinity']['requiredDuringSchedulingIgnoredDuringExecution'][0]['labelSelector']['matchExpressions'] is not None)
+                count += 1
+        self.assertEqual(count, len(resources_name), "Not all resources have affinity set")
+
+if __name__ == '__main__':
+    failed = False
+    try:
+        FILE_NAME = sys.argv[1]
+        LIST_OF_DOCUMENTS = load_template(FILE_NAME)
+        suite = unittest.TestLoader().loadTestsFromTestCase(ChartTemplateTests)
+        test_runner = unittest.TextTestRunner(verbosity=3)
+        failed = not test_runner.run(suite).wasSuccessful()
+    except Exception as e:
+        logger.fatal(e)
+        failed = True
+
+    if failed:
+        exit(1)