Autoscaling selenium grid on kubernetes

Autoscale selenium browser nodes running in kubernetes based on the request pending in session queue using KEDA. Toggle autoscaling on/off using 'autoscalingEnabled' option in helm charts.
SeleniumHQ · Nov 1, 2022 · 7e4f215 · 7e4f215
1 parent b1d15ef
commit 7e4f215
Show file tree

Hide file tree

Showing 6 changed files with 134 additions and 33 deletions.
diff --git a/charts/selenium-grid/README.md b/charts/selenium-grid/README.md
@@ -29,6 +29,11 @@ helm install selenium-grid docker-selenium/selenium-grid --version <version>
 helm install selenium-grid --set ingress.hostname=selenium-grid.k8s.local docker-selenium/chart/selenium-grid/.
 ```
 
+## Enable Selenium Grid Autoscaling
+Selenium Grid has the ability to autoscale browser nodes up/down based on the requests pending in session queue. You can enable it setting 'autoscalingEnabled' to `true`. You need to install KEDA by following the [instructions](https://keda.sh/docs/2.8/deploy/#helm) in order for autoscaling to work. 
+
+The hpa.url value is configured to work for grid installed in `default` namespace. If you are installing the grid in some other namespace make sure to update the value of hpa.url accordingly. 
+
 ## Updating Selenium-Grid release
 
 Once you have a new chart version, you can update your selenium-grid running:
@@ -66,6 +71,7 @@ This table contains the configuration parameters of the chart and their default
 | `ingress.annotations`                   | `{}`                               | Custom annotations for ingress resource                                                                                    |
 | `ingress.hostname`                      | `selenium-grid.local`              | Default host for the ingress resource                                                                                      |
 | `ingress.tls`                           | `[]`                               | TLS backend configuration for ingress resource                                                                             |
+| `ingress.path`                          | `/`                                | Default path for ingress resource                                                                             |
 | `busConfigMap.annotations`              | `{}`                               | Custom annotations for configmap                                                                                           |
 | `chromeNode.enabled`                    | `true`                             | Enable chrome nodes                                                                                                        |
 | `chromeNode.deploymentEnabled`          | `true`                             | Enable creation of Deployment for chrome nodes                                                                             |
@@ -95,6 +101,11 @@ This table contains the configuration parameters of the chart and their default
 | `chromeNode.lifecycle`                  | `{}`                               | hooks to make pod correctly shutdown or started                                                                            |
 | `chromeNode.extraVolumeMounts`          | `[]`                               | Extra mounts of declared ExtraVolumes into pod                                                                             |
 | `chromeNode.extraVolumes`               | `[]`                               | Extra Volumes declarations to be used in the pod (can be any supported volume type: ConfigMap, Secret, PVC, NFS, etc.)     |
+| `chromeNode.autoscalingEnabled`         | `false`                            | Enable/Disable autoscaling of browser nodes |
+| `chromeNode.hpa.url`                    | `http://selenium-hub.default:4444/graphql` | Graphql Url of the hub or the router |
+| `chromeNode.hpa.browserName`            | `chrome`                           | BrowserName from the capability |
+| `chromeNode.hpa.browserVersion`         | ``                                 | BrowserVersion from the capability |
+| `chromeNode.maxReplicaCount`            | `8`                                | Max number of replicas that this browsernode can auto scale up to |
 | `firefoxNode.enabled`                   | `true`                             | Enable firefox nodes                                                                                                       |
 | `firefoxNode.deploymentEnabled`         | `true`                             | Enable creation of Deployment for firefox nodes                                                                            |
 | `firefoxNode.replicas`                  | `1`                                | Number of firefox nodes                                                                                                    |
@@ -123,6 +134,11 @@ This table contains the configuration parameters of the chart and their default
 | `firefoxNode.lifecycle`                 | `{}`                               | hooks to make pod correctly shutdown or started                                                                            |
 | `firefoxNode.extraVolumeMounts`         | `[]`                               | Extra mounts of declared ExtraVolumes into pod                                                                             |
 | `firefoxNode.extraVolumes`              | `[]`                               | Extra Volumes declarations to be used in the pod (can be any supported volume type: ConfigMap, Secret, PVC, NFS, etc.)     |
+| `firefoxNode.autoscalingEnabled`        | `false`                            | Enable/Disable autoscaling of browser nodes |
+| `firefoxNode.hpa.url`                   | `http://selenium-hub.default:4444/graphql` | Graphql Url of the hub or the router |
+| `firefoxNode.hpa.browserName`           | `firefox`                          | BrowserName from the capability |
+| `firefoxNode.hpa.browserVersion`        | ``                                 | BrowserVersion from the capability |
+| `firefoxNode.maxReplicaCount`           | `8`                                | Max number of replicas that this browsernode can auto scale up to |
 | `edgeNode.enabled`                      | `true`                             | Enable edge nodes                                                                                                          |
 | `edgeNode.deploymentEnabled`            | `true`                             | Enable creation of Deployment for edge nodes                                                                               |
 | `edgeNode.replicas`                     | `1`                                | Number of edge nodes                                                                                                       |
@@ -151,6 +167,11 @@ This table contains the configuration parameters of the chart and their default
 | `edgeNode.lifecycle`                    | `{}`                               | hooks to make pod correctly shutdown or started                                                                            |
 | `edgeNode.extraVolumeMounts`            | `[]`                               | Extra mounts of declared ExtraVolumes into pod                                                                             |
 | `edgeNode.extraVolumes`                 | `[]`                               | Extra Volumes declarations to be used in the pod (can be any supported volume type: ConfigMap, Secret, PVC, NFS, etc.)     |
+| `edgeNode.autoscalingEnabled`          | `false`                            | Enable/Disable autoscaling of browser nodes |
+| `edgeNode.hpa.url`                     | `http://selenium-hub.default:4444/graphql` | Graphql Url of the hub or the router |
+| `edgeNode.hpa.browserName`             | `edge`                           | BrowserName from the capability |
+| `edgeNode.hpa.browserVersion`          | ``                               | BrowserVersion from the capability |
+| `edgeNode.maxReplicaCount`             | `8`                                | Max number of replicas that this browsernode can auto scale up to |
 | `customLabels`                          | `{}`                               | Custom labels for k8s resources                                                                                            |
 
 

diff --git a/charts/selenium-grid/templates/chrome-node-hpa.yaml b/charts/selenium-grid/templates/chrome-node-hpa.yaml
@@ -0,0 +1,18 @@
+{{- if and .Values.chromeNode.enabled .Values.chromeNode.autoscalingEnabled }}
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: selenium-grid-chrome-scaledobject
+  namespace: {{ .Release.Namespace }}
+  labels:
+    deploymentName: {{ template "seleniumGrid.chromeNode.fullname" . }}
+spec:
+  maxReplicaCount: {{ .Values.chromeNode.maxReplicaCount }}
+  scaleTargetRef:
+    name: {{ template "seleniumGrid.chromeNode.fullname" . }}
+  triggers:
+    - type: selenium-grid
+    {{- with .Values.chromeNode.hpa }}
+      metadata: {{- toYaml . | nindent 8 }}
+    {{- end }}
+{{- end }}
diff --git a/charts/selenium-grid/templates/edge-node-hpa.yaml b/charts/selenium-grid/templates/edge-node-hpa.yaml
@@ -0,0 +1,18 @@
+{{- if and .Values.edgeNode.enabled .Values.edgeNode.autoscalingEnabled }}
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: selenium-grid-edge-scaledobject
+  namespace: {{ .Release.Namespace }}
+  labels:
+    deploymentName: {{ template "seleniumGrid.edgeNode.fullname" . }}
+spec:
+  maxReplicaCount: {{ .Values.edgeNode.maxReplicaCount }}
+  scaleTargetRef:
+    name: {{ template "seleniumGrid.edgeNode.fullname" . }}
+  triggers:
+    - type: selenium-grid
+    {{- with .Values.edgeNode.hpa }}
+      metadata: {{- toYaml . | nindent 8 }}
+    {{- end }}
+{{- end }}
diff --git a/charts/selenium-grid/templates/firefox-node-hpa.yaml b/charts/selenium-grid/templates/firefox-node-hpa.yaml
@@ -0,0 +1,18 @@
+{{- if and .Values.firefoxNode.enabled .Values.firefoxNode.autoscalingEnabled }}
+apiVersion: keda.sh/v1alpha1
+kind: ScaledObject
+metadata:
+  name: selenium-grid-firefox-scaledobject
+  namespace: {{ .Release.Namespace }}
+  labels:
+    deploymentName: {{ template "seleniumGrid.firefoxNode.fullname" . }}
+spec:
+  maxReplicaCount: {{ .Values.firefoxNode.maxReplicaCount }}
+  scaleTargetRef:
+    name: {{ template "seleniumGrid.firefoxNode.fullname" . }}
+  triggers:
+    - type: selenium-grid
+    {{- with .Values.firefoxNode.hpa }}
+      metadata: {{- toYaml . | nindent 8 }}
+    {{- end }}
+{{- end }}
diff --git a/charts/selenium-grid/templates/ingress.yaml b/charts/selenium-grid/templates/ingress.yaml
@@ -45,7 +45,11 @@ spec:
     - http:
       {{- end }}
         paths:
+            {{- if $.Values.ingress.path }}
+          - path: {{ .Values.ingress.path }}
+            {{- else }}
           - path: /
+            {{- end }}  
             pathType: Prefix
             backend:
               service:

diff --git a/charts/selenium-grid/values.yaml b/charts/selenium-grid/values.yaml
@@ -22,6 +22,7 @@ ingress:
   hostname: selenium-grid.local
   # TLS backend configuration for ingress resource
   tls: []
+  path: /
 
 # ConfigMap that contains SE_EVENT_BUS_HOST, SE_EVENT_BUS_PUBLISH_PORT and SE_EVENT_BUS_SUBSCRIBE_PORT variables
 busConfigMap:
@@ -363,7 +364,7 @@ chromeNode:
     # Custom annotations for service
     annotations: {}
   # Size limit for DSH volume mounted in container (if not set, default is "1Gi")
-  dshmVolumeSizeLimit: 1Gi
+  dshmVolumeSizeLimit: 2Gi
   # Priority class name for chrome-node pods
   priorityClassName: ""
 
@@ -375,17 +376,17 @@ chromeNode:
     # failureThreshold: 120
     # periodSeconds: 5
   # Time to wait for pod termination
-  terminationGracePeriodSeconds: 30
+  terminationGracePeriodSeconds: 3600
   # Allow pod correctly shutdown
-  lifecycle: {}
-    # preStop:
-    #   exec:
-    #     command:
-    #       - bash
-    #       - -c
-    #       - |
-    #         curl -X POST 127.0.0.1:5555/se/grid/node/drain --header 'X-REGISTRATION-SECRET;' && \
-    #         while curl 127.0.0.1:5555/status; do sleep 1; done
+  lifecycle: 
+    preStop:
+      exec:
+        command:
+          - bash
+          - -c
+          - |
+            curl -X POST 127.0.0.1:5555/se/grid/node/drain --header 'X-REGISTRATION-SECRET;' && \
+            while curl 127.0.0.1:5555/status; do sleep 1; done;
 
   extraVolumeMounts: []
   # - name: my-extra-volume
@@ -398,6 +399,15 @@ chromeNode:
   #   persistentVolumeClaim:
   #     claimName: my-pv-claim
 
+  # Keda scaled object configuration
+  autoscalingEnabled: false
+  maxReplicaCount: 8
+  hpa:
+    url: http://selenium-hub.default:4444/graphql # Repalce your http graphql url here
+    browserName: chrome
+    # browserVersion: '91.0' # Optional. Only required when supporting multiple versions of browser in your Selenium Grid.
+    unsafeSsl : 'true' # Optional
+
 # Configuration for firefox nodes
 firefoxNode:
   # Enable firefox nodes
@@ -475,7 +485,7 @@ firefoxNode:
     # Custom annotations for service
     annotations: {}
   # Size limit for DSH volume mounted in container (if not set, default is "1Gi")
-  dshmVolumeSizeLimit: 1Gi
+  dshmVolumeSizeLimit: 2Gi
   # Priority class name for firefox-node pods
   priorityClassName: ""
 
@@ -487,17 +497,17 @@ firefoxNode:
     # failureThreshold: 120
     # periodSeconds: 5
   # Time to wait for pod termination
-  terminationGracePeriodSeconds: 30
+  terminationGracePeriodSeconds: 3600
   # Allow pod correctly shutdown
-  lifecycle: {}
-    # preStop:
-    #   exec:
-    #     command:
-    #       - bash
-    #       - -c
-    #       - |
-    #         curl -X POST 127.0.0.1:5555/se/grid/node/drain --header 'X-REGISTRATION-SECRET;' && \
-    #         while curl 127.0.0.1:5555/status; do sleep 1; done
+  lifecycle: 
+    preStop:
+      exec:
+        command:
+          - bash
+          - -c
+          - |
+            curl -X POST 127.0.0.1:5555/se/grid/node/drain --header 'X-REGISTRATION-SECRET;' && \
+            while curl 127.0.0.1:5555/status; do sleep 1; done;
 
   extraVolumeMounts: []
   # - name: my-extra-volume
@@ -509,6 +519,12 @@ firefoxNode:
   # - name: my-extra-volume-from-pvc
   #   persistentVolumeClaim:
   #     claimName: my-pv-claim
+  # Keda scaled object configuration
+  autoscalingEnabled: false
+  maxReplicaCount: 8
+  hpa:
+    url: http://selenium-hub.default:4444/graphql # Repalce your http graphql url here
+    browserName: firefox
 
 # Configuration for edge nodes
 edgeNode:
@@ -587,7 +603,7 @@ edgeNode:
     annotations:
       hello: world
   # Size limit for DSH volume mounted in container (if not set, default is "1Gi")
-  dshmVolumeSizeLimit: 1Gi
+  dshmVolumeSizeLimit: 2Gi
   # Priority class name for edge-node pods
   priorityClassName: ""
 
@@ -599,17 +615,17 @@ edgeNode:
     # failureThreshold: 120
     # periodSeconds: 5
   # Time to wait for pod termination
-  terminationGracePeriodSeconds: 30
+  terminationGracePeriodSeconds: 3600
   # Allow pod correctly shutdown
-  lifecycle: {}
-    # preStop:
-    #   exec:
-    #     command:
-    #       - bash
-    #       - -c
-    #       - |
-    #         curl -X POST 127.0.0.1:5555/se/grid/node/drain --header 'X-REGISTRATION-SECRET;' && \
-    #         while curl 127.0.0.1:5555/status; do sleep 1; done
+  lifecycle: 
+    preStop:
+      exec:
+        command:
+          - bash
+          - -c
+          - |
+            curl -X POST 127.0.0.1:5555/se/grid/node/drain --header 'X-REGISTRATION-SECRET;' && \
+            while curl 127.0.0.1:5555/status; do sleep 1; done;
 
   extraVolumeMounts: []
   # - name: my-extra-volume
@@ -621,6 +637,12 @@ edgeNode:
   # - name: my-extra-volume-from-pvc
   #   persistentVolumeClaim:
   #     claimName: my-pv-claim
+  # Keda scaled object configuration
+  autoscalingEnabled: false
+  maxReplicaCount: 8
+  hpa:
+    url: http://selenium-hub.default:4444/graphql # Repalce your http graphql url here
+    browserName: MicrosoftEdge
 
 # Custom labels for k8s resources
 customLabels: {}