Support gateway configurations without http enabled. Add tests for ht…

…tps and NodePort service with both https and http exposed.
trinodb · Dec 11, 2024 · f0a4380 · f0a4380
1 parent 6d86367
commit f0a4380
Show file tree

Hide file tree

Showing 10 changed files with 209 additions and 15 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,3 @@
 .idea/*
-tests/trino/cert.key
-tests/trino/cert.crt
+*/*/cert.key
+*/*/cert.crt
diff --git a/charts/gateway/README.md b/charts/gateway/README.md
@@ -55,7 +55,7 @@ A Helm chart for Trino Gateway
   Startup command for Trino Gateway process. Add additional Java options and other modifications as desired.
 * `service` - object, default: `{"spec":{"ports":[{"name":"request","protocol":"TCP"}],"type":"ClusterIP"}}`  
 
-  service for accessing the gateway. The `port` and `targetPort` of the first element of the ports list will automatically be set to the value of `config.serverConfig."http-server.http.port"`. Additional ports (for JMX or a Java Agent for example) can be configured by adding additional elements to the ports list. The selector is also automatically configured. All other values are passed through as is.
+  service for accessing the gateway. The `port` and `targetPort` of the first element of the ports list will automatically be set to the value of `config.serverConfig."http-server.http[s].port"`. If both https and http ports are defined, the https port is used. In this case, an additional service for the http port must be configured manually. Additional ports (for JMX or a Java Agent for example) can be configured by adding additional elements to the ports list. The selector is also automatically configured. All other values are passed through as is.
 * `ingress.enabled` - bool, default: `false`
 * `ingress.className` - string, default: `""`
 * `ingress.annotations` - object, default: `{}`

diff --git a/charts/gateway/templates/deployment.yaml b/charts/gateway/templates/deployment.yaml
@@ -46,16 +46,22 @@ spec:
               protocol: TCP
           livenessProbe:
             httpGet:
-                path: /trino-gateway
-                port: {{ index .Values "config" "serverConfig" "http-server.http.port" }}
+              path: /trino-gateway
+              port: {{ coalesce (index .Values "config" "serverConfig" "http-server.https.port") (index .Values "config" "serverConfig" "http-server.http.port") }}
+              {{- if index .Values "config" "serverConfig" "http-server.https.port" }}
+              scheme: HTTPS
+              {{- end }}
             initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
             failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
             timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
           readinessProbe:
             httpGet:
-                path: /trino-gateway
-                port: {{ index .Values "config" "serverConfig" "http-server.http.port" }}
+              path: /trino-gateway
+              port: {{ coalesce (index .Values "config" "serverConfig" "http-server.https.port") (index .Values "config" "serverConfig" "http-server.http.port") }}
+              {{- if index .Values "config" "serverConfig" "http-server.https.port" }}
+              scheme: HTTPS
+              {{- end }}
             initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
             failureThreshold: {{ .Values.readinessProbe.failureThreshold }}

diff --git a/charts/gateway/templates/service.yaml b/charts/gateway/templates/service.yaml
@@ -4,11 +4,15 @@ metadata:
   name: {{ include "trino-gateway.fullname" . }}
   labels:
     {{- include "trino-gateway.labels" . | nindent 4 }}
-{{- $httpPort := index .Values "config" "serverConfig" "http-server.http.port" }}
-{{- $portDefault := dict "port" $httpPort "targetPort" $httpPort }}
+{{- $gatewayPort := coalesce (index .Values "config" "serverConfig" "http-server.https.port")
+                    (index .Values "config" "serverConfig" "http-server.http.port") }}
+{{- if empty $gatewayPort }}
+{{- fail "Error: No port defined in serverConfig!"}}
+{{- end -}}
+{{- $portDefault := dict "port" $gatewayPort "targetPort" $gatewayPort }}
 {{- $portValues := .Values.service.spec.ports | default list | first | default $portDefault}}
-{{- $_0 := set $portValues "port" $httpPort}}
-{{- $_1 := set $portValues "targetPort" $httpPort}}
+{{- $_0 := set $portValues "port" $gatewayPort}}
+{{- $_1 := set $portValues "targetPort" $gatewayPort}}
 {{- $ports := list $portValues }}
 {{- $additionalPorts := .Values.service.spec.ports | default list | rest }}
 {{- $allPorts := concat $ports $additionalPorts}}

diff --git a/charts/gateway/templates/tests/test-connection.yaml b/charts/gateway/templates/tests/test-connection.yaml
@@ -12,6 +12,7 @@ spec:
   initContainers:
     - name: extract-persistence-sql
       image: "trinodb/trino-gateway"
+      imagePullPolicy: IfNotPresent
       command:
         - "/bin/sh"
         - "-c"
@@ -24,6 +25,7 @@ spec:
           mountPath: /etc/persistence
     - name: initialize-db
       image: bitnami/postgresql:17.1.0
+      imagePullPolicy: IfNotPresent
       command:
         - "/bin/sh"
         - "-c"
@@ -38,13 +40,18 @@ spec:
           mountPath: /etc/persistence
   containers:
     - name: wget
-      image: busybox
+      image: alpine
+      imagePullPolicy: IfNotPresent
       # Get the list of backends, which should return an empty list, "[]". For this test to pass
       # the gateway must successfully connect to an initialized backend database
       command:
         - "sh"
         - "-c"
-        - '[ "$(wget {{ include "trino-gateway.fullname" . }}:{{ .Values.service.port }}/entity/GATEWAY_BACKEND -O -)" = "[]" ]'
+        {{- if index .Values "config" "serverConfig" "http-server.https.port" }}
+        - '[ "$(wget --no-check-certificate https://{{ include "trino-gateway.fullname" . }}:{{ index .Values "config" "serverConfig" "http-server.https.port"}}/entity/GATEWAY_BACKEND -O -)" = "[]" ]'
+        {{- else }}
+        - '[ "$(wget http://{{ include "trino-gateway.fullname" . }}:{{ index .Values "config" "serverConfig" "http-server.http.port"}}/entity/GATEWAY_BACKEND -O -)" = "[]" ]'
+        {{- end }}
   volumes:
     - name: persistence-sql
       emptyDir:

diff --git a/charts/gateway/templates/tests/test-nodeport.yaml b/charts/gateway/templates/tests/test-nodeport.yaml
@@ -0,0 +1,31 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{ include "trino-gateway.fullname" . }}-test-connection"
+  labels:
+    {{- include "trino-gateway.labels" . | nindent 4 }}
+    app.kubernetes.io/component: test
+    test: service
+  annotations:
+    "helm.sh/hook": test
+spec:
+  containers:
+    - name: wget
+      image: alpine
+      imagePullPolicy: IfNotPresent
+      # Get the list of backends, which should return an empty list, "[]". For this test to pass
+      # the gateway must successfully connect to an initialized backend database
+      command:
+        - "sh"
+        - "-c"
+        - |
+          {{- if eq .Values.service.spec.type "NodePort" }}
+          [ "$(wget --no-check-certificate https://${NODE_IP}:{{ index .Values.service.spec.ports 0 "nodePort"}}/entity/GATEWAY_BACKEND -O -)" = "[]" ] &&
+          [ "$(wget http://${NODE_IP}:{{ index .Values.service.spec.ports 1 "nodePort"}}/entity/GATEWAY_BACKEND -O -)" = "[]" ]
+          {{- else }}
+          echo non NodePort service type
+          {{- end }}
+      envFrom:
+        - secretRef:
+            name: node-ip
+  restartPolicy: Never
diff --git a/charts/gateway/values.yaml b/charts/gateway/values.yaml
@@ -59,7 +59,9 @@ command:
   - "/etc/gateway/config.yaml"
 
 # -- service for accessing the gateway. The `port` and `targetPort` of the first element of the ports list will
-# automatically be set to the value of `config.serverConfig."http-server.http.port"`. Additional ports (for JMX
+# automatically be set to the value of `config.serverConfig."http-server.http[s].port"`. If both https and http
+# ports are defined, the https port is used. In this case, an additional service for the http port must be
+# configured manually. Additional ports (for JMX
 # or a Java Agent for example) can be configured by adding additional elements to the ports list. The selector is
 # also automatically configured. All other values are passed through as is.
 service:

diff --git a/tests/gateway/test-https.yaml b/tests/gateway/test-https.yaml
@@ -0,0 +1,57 @@
+replicaCount: 1
+
+image:
+  # -- Repository location of the Trino Gateway image, typically `organization/imagename`
+  repository: "trinodb/trino-gateway"
+  pullPolicy: IfNotPresent
+
+command:
+  - "/bin/sh"
+  - "-c"
+  - |
+    cat /etc/certificates/tls.crt /etc/certificates/tls.key > /etc/scratch/tls.pem && \
+    java -XX:MinRAMPercentage=80.0 -XX:MaxRAMPercentage=80.0 -jar /usr/lib/trino/gateway-ha-jar-with-dependencies.jar /etc/gateway/config.yaml
+
+config:
+  serverConfig:
+    node.environment: test
+    http-server.http.enabled: false
+    http-server.https.enabled: true
+    http-server.https.port: 8443
+    http-server.https.keystore.path: /etc/scratch/tls.pem
+  dataStore:
+    # The connection details for the backend database for Trino Gateway and Trino query history
+    jdbcUrl: jdbc:postgresql://gateway-backend-db-postgresql.postgres-gateway.svc.cluster.local:5432/gateway
+    user: gateway
+    password: pass0000
+    driver: org.postgresql.Driver
+  clusterStatsConfiguration:
+    monitorType: INFO_API
+  modules:
+    - io.trino.gateway.ha.module.HaGatewayProviderModule
+    - io.trino.gateway.ha.module.ClusterStateListenerModule
+    - io.trino.gateway.ha.module.ClusterStatsMonitorModule
+  managedApps:
+    - io.trino.gateway.ha.clustermonitor.ActiveClusterMonitor
+
+volumes:
+  - name: certificates
+    secret:
+      secretName: certificates
+  - name: scratch
+    emptyDir:
+      sizeLimit: 10Mi
+
+volumeMounts:
+  - name: certificates
+    mountPath: /etc/certificates
+  - name: scratch
+    mountPath: /etc/scratch
+
+resources:
+  limits:
+    cpu: 500m
+    memory: 256Mi
+  requests:
+    cpu: 250m
+    memory: 256Mi
diff --git a/tests/gateway/test-nodeport.yaml b/tests/gateway/test-nodeport.yaml
@@ -0,0 +1,70 @@
+replicaCount: 1
+
+image:
+  # -- Repository location of the Trino Gateway image, typically `organization/imagename`
+  repository: "trinodb/trino-gateway"
+  pullPolicy: IfNotPresent
+
+command:
+  - "/bin/sh"
+  - "-c"
+  - |
+    cat /etc/certificates/tls.crt /etc/certificates/tls.key > /etc/scratch/tls.pem && \
+    java -XX:MinRAMPercentage=80.0 -XX:MaxRAMPercentage=80.0 -jar /usr/lib/trino/gateway-ha-jar-with-dependencies.jar /etc/gateway/config.yaml
+
+config:
+  serverConfig:
+    node.environment: test
+    http-server.http.enabled: true
+    http-server.https.enabled: true
+    http-server.https.port: 8443
+    http-server.https.keystore.path: /etc/scratch/tls.pem
+  dataStore:
+    # The connection details for the backend database for Trino Gateway and Trino query history
+    jdbcUrl: jdbc:postgresql://gateway-backend-db-postgresql.postgres-gateway.svc.cluster.local:5432/gateway
+    user: gateway
+    password: pass0000
+    driver: org.postgresql.Driver
+  clusterStatsConfiguration:
+    monitorType: INFO_API
+  modules:
+    - io.trino.gateway.ha.module.HaGatewayProviderModule
+    - io.trino.gateway.ha.module.ClusterStateListenerModule
+    - io.trino.gateway.ha.module.ClusterStatsMonitorModule
+  managedApps:
+    - io.trino.gateway.ha.clustermonitor.ActiveClusterMonitor
+
+resources:
+  limits:
+    cpu: 500m
+    memory: 256Mi
+  requests:
+    cpu: 250m
+    memory: 256Mi
+
+service:
+  spec:
+    type: NodePort
+    ports:
+      - protocol: TCP
+        name: request
+        nodePort: 30443
+      - protocol: TCP
+        name: gateway-http
+        nodePort: 30080
+        port: 8080
+        targetPort: 8080
+
+volumes:
+  - name: certificates
+    secret:
+      secretName: certificates
+  - name: scratch
+    emptyDir:
+      sizeLimit: 10Mi
+
+volumeMounts:
+  - name: certificates
+    mountPath: /etc/certificates
+  - name: scratch
+    mountPath: /etc/scratch
diff --git a/tests/gateway/test.sh b/tests/gateway/test.sh
@@ -5,13 +5,19 @@ set -euo pipefail
 declare -A testCases=(
     [complete_values]="--values test-values.yaml"
     [env_from]="--values test-values-with-env.yaml"
+    [nodeport]="--values test-nodeport.yaml"
+    [https]="--values test-https.yaml"
 )
 
 declare -A testCaseCharts=(
     [complete_values]="../../charts/gateway"
+    [nodeport]="../../charts/gateway"
+    [https]="../../charts/gateway"
     [env_from]="../../charts/gateway"
 )
 
+TEST_NAMES=(complete_values nodeport https env_from)
+
 function join_by {
     local d=${1-} f=${2-}
     if shift 2; then
@@ -24,13 +30,24 @@ NAMESPACE=trino-gateway-$(LC_ALL=C tr -dc 'a-z0-9' </dev/urandom | head -c 6 ||
 DB_NAMESPACE=postgres-gateway
 kubectl create namespace "${NAMESPACE}" --dry-run=client --output yaml | kubectl apply --filename -
 kubectl create namespace "${DB_NAMESPACE}" --dry-run=client --output yaml | kubectl apply --filename -
+
+NODE_IP=$(kubectl get nodes -o json  -o jsonpath='{.items[0].status.addresses[0].address}')
+kubectl -n "${NAMESPACE}" create secret generic node-ip --from-literal=NODE_IP="${NODE_IP}"
+
+echo 1>&2 "Generating a self-signed TLS certificate"
+openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 \
+    -subj "/O=Trino Software Foundation" \
+    -addext "subjectAltName=DNS:localhost,DNS:*.$NAMESPACE,DNS:*.$NAMESPACE.svc,DNS:*.$NAMESPACE.svc.cluster.local,IP:127.0.0.1,IP:${NODE_IP}" \
+    -keyout cert.key -out cert.crt
+kubectl -n "$NAMESPACE" create secret tls certificates --cert=cert.crt --key=cert.key --dry-run=client --output yaml | kubectl apply --filename -
+
+
 HELM_EXTRA_SET_ARGS=
 CT_ARGS=(
     --skip-clean-up
     --helm-extra-args="--timeout 2m"
 )
 CLEANUP_NAMESPACE=true
-TEST_NAMES=(complete_values env_from)
 
 usage() {
     cat <<EOF 1>&2