Helm Chart: Add external temporal option (#14597)

* conflict env configmap and chart lock * reverting lock * add eof lines and documentation on values yaml * conflict json file * rollback json * solve conflict * correct minio with new version Co-authored-by: Guy Feldman <gfeldman@86labs.com>
airbytehq · Jul 12, 2022 · fc2a590 · fc2a590
1 parent ef49424
commit fc2a590
Show file tree

Hide file tree

Showing 10 changed files with 52 additions and 59 deletions.
diff --git a/charts/airbyte/Chart.lock b/charts/airbyte/Chart.lock
@@ -1,12 +1,12 @@
 dependencies:
 - name: common
   repository: https://charts.bitnami.com/bitnami
-  version: 1.15.1
+  version: 1.16.0
 - name: postgresql
   repository: https://charts.bitnami.com/bitnami
   version: 10.16.2
 - name: minio
   repository: https://charts.bitnami.com/bitnami
-  version: 11.6.3
-digest: sha256:616b7b5af8ca1ac0bb11374a5589f7c00cad416faf5ac59a3484585be768fee3
-generated: "2022-06-02T18:25:41.835008+02:00"
+  version: 11.7.9
+digest: sha256:34bb2b1da02d7f529520b62d3a2fdb30849579f0d9d68979d8abc5978da55076
+generated: "2022-07-11T17:42:53.084477-03:00"
diff --git a/charts/airbyte/templates/_helpers.tpl b/charts/airbyte/templates/_helpers.tpl
@@ -154,6 +154,24 @@ Add environment variables to configure database values
 {{- printf "jdbc:postgresql://%s:%s/%s" $host $port $dbName -}}
 {{- end -}}
 
+{{/*
+Add variable for temporal host
+*/}}
+{{- define "airbyte.temporal.host" -}}
+{{- ternary (printf "%s-%s" (include "common.names.fullname" .) "temporal") .Values.externalTemporal.host .Values.temporal.enabled -}}
+{{- end -}}
+
+{{/*
+Add variable for temporal port
+*/}}
+{{- define "airbyte.temporal.port" -}}
+{{- ternary  .Values.temporal.service.port .Values.externalTemporal.port .Values.temporal.enabled -}}
+{{- end -}}
+
+{{- define "airbyte.temporal.url" -}}
+{{- (printf "%s:%d" (include "airbyte.temporal.host" .) (include "airbyte.temporal.port" . | default 7233 | int)) -}}
+{{- end -}}
+
 {{/*
 Create a default fully qualified minio name.
 We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).

diff --git a/charts/airbyte/templates/env-configmap.yaml b/charts/airbyte/templates/env-configmap.yaml
@@ -46,7 +46,8 @@ data:
   S3_PATH_STYLE_ACCESS: {{ include "airbyte.s3PathStyleAccess" . | quote }}
   STATE_STORAGE_MINIO_BUCKET_NAME: airbyte-state-storage
   STATE_STORAGE_MINIO_ENDPOINT: {{ include "airbyte.minio.endpoint" . | quote }}
-  TEMPORAL_HOST: {{ include "common.names.fullname" . }}-temporal:{{ .Values.temporal.service.port }}
+  SUBMITTER_NUM_THREADS: "10"
+  TEMPORAL_HOST: {{ include "airbyte.temporal.url" . }}
   TEMPORAL_WORKER_PORTS: 9001,9002,9003,9004,9005,9006,9007,9008,9009,9010,9011,9012,9013,9014,9015,9016,9017,9018,9019,9020,9021,9022,9023,9024,9025,9026,9027,9028,9029,9030,9031,9032,9033,9034,9035,9036,9037,9038,9039,9040
   TRACKING_STRATEGY: segment
   WEBAPP_URL: http://{{ include "common.names.fullname" . }}-webapp:{{ .Values.webapp.service.port }}

diff --git a/charts/airbyte/templates/secret.yaml b/charts/airbyte/templates/secret.yaml
@@ -7,11 +7,7 @@ metadata:
     helm.sh/hook-weight: "-1"
 type: Opaque
 stringData:
-  AWS_ACCESS_KEY_ID: {{ .Values.logs.accessKey.password | quote }}
-  AWS_SECRET_ACCESS_KEY: {{ .Values.logs.secretKey.password | quote }}
   {{ if and (not .Values.postgresql.enabled) (eq .Values.externalDatabase.existingSecret "") -}}
   DATABASE_PASSWORD: {{ .Values.externalDatabase.password | quote }}
   {{ end -}}
   DATABASE_USER: {{ include "airbyte.database.user" . }}
-  STATE_STORAGE_MINIO_ACCESS_KEY: {{ .Values.minio.accessKey.password | quote }}
-  STATE_STORAGE_MINIO_SECRET_ACCESS_KEY: {{ .Values.minio.secretKey.password | quote }}
diff --git a/charts/airbyte/templates/server/deployment.yaml b/charts/airbyte/templates/server/deployment.yaml
@@ -121,32 +121,16 @@ spec:
             configMapKeyRef:
               name: {{ include "common.names.fullname" . }}-env
               key: S3_LOG_BUCKET_REGION
-        {{- if and .Values.logs.accessKey.existingSecret .Values.logs.accessKey.existingSecretKey }}
         - name: AWS_ACCESS_KEY_ID
           valueFrom:
             secretKeyRef:
               name: {{ .Values.logs.accessKey.existingSecret }}
               key: {{ .Values.logs.accessKey.existingSecretKey }}
-        {{- else }}
-        - name: AWS_ACCESS_KEY_ID
-          valueFrom:
-            secretKeyRef:
-              name: {{ include "common.names.fullname" . }}-secrets
-              key: AWS_ACCESS_KEY_ID
-        {{- end }}
-        {{- if and .Values.logs.secretKey.existingSecret .Values.logs.secretKey.existingSecretKey }}
         - name: AWS_SECRET_ACCESS_KEY
           valueFrom:
             secretKeyRef:
               name: {{ .Values.logs.secretKey.existingSecret }}
               key: {{ .Values.logs.secretKey.existingSecretKey }}
-        {{- else }}
-        - name: AWS_SECRET_ACCESS_KEY
-          valueFrom:
-            secretKeyRef:
-              name: {{ include "common.names.fullname" . }}-secrets
-              key: AWS_SECRET_ACCESS_KEY
-        {{- end }}
         - name: S3_MINIO_ENDPOINT
           valueFrom:
             configMapKeyRef:

diff --git a/charts/airbyte/templates/temporal/configmap.yaml b/charts/airbyte/templates/temporal/configmap.yaml
@@ -1,3 +1,4 @@
+{{- if .Values.temporal.enabled -}}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -46,3 +47,4 @@ data:
     limit.blobSize.warn:
       - value: 10485760 # 10MB
         constraints: {}
+{{- end -}}
diff --git a/charts/airbyte/templates/temporal/deployment.yaml b/charts/airbyte/templates/temporal/deployment.yaml
@@ -1,3 +1,4 @@
+{{- if .Values.temporal.enabled }}
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -114,3 +115,4 @@ spec:
       {{- if .Values.temporal.extraVolumes }}
 {{ toYaml .Values.temporal.extraVolumes | nindent 6 }}
       {{- end }}
+{{- end }}
diff --git a/charts/airbyte/templates/temporal/service.yaml b/charts/airbyte/templates/temporal/service.yaml
@@ -1,3 +1,4 @@
+{{- if .Values.temporal.enabled }}
 ---
 apiVersion: v1
 kind: Service
@@ -11,3 +12,4 @@ spec:
     targetPort: 7233
   selector:
     airbyte: temporal
+{{- end }}
diff --git a/charts/airbyte/templates/worker/deployment.yaml b/charts/airbyte/templates/worker/deployment.yaml
@@ -173,32 +173,16 @@ spec:
             configMapKeyRef:
               name: {{ include "common.names.fullname" . }}-env
               key: S3_LOG_BUCKET_REGION
-        {{- if and .Values.logs.accessKey.existingSecret .Values.logs.accessKey.existingSecretKey }}
         - name: AWS_ACCESS_KEY_ID
           valueFrom:
             secretKeyRef:
               name: {{ .Values.logs.accessKey.existingSecret }}
               key: {{ .Values.logs.accessKey.existingSecretKey }}
-        {{- else }}
-        - name: AWS_ACCESS_KEY_ID
-          valueFrom:
-            secretKeyRef:
-              name: {{ include "common.names.fullname" . }}-secrets
-              key: AWS_ACCESS_KEY_ID
-        {{- end }}
-        {{- if and .Values.logs.secretKey.existingSecret .Values.logs.secretKey.existingSecretKey }}
         - name: AWS_SECRET_ACCESS_KEY
           valueFrom:
             secretKeyRef:
               name: {{ .Values.logs.secretKey.existingSecret }}
               key: {{ .Values.logs.secretKey.existingSecretKey }}
-        {{- else }}
-        - name: AWS_SECRET_ACCESS_KEY
-          valueFrom:
-            secretKeyRef:
-              name: {{ include "common.names.fullname" . }}-secrets
-              key: AWS_SECRET_ACCESS_KEY
-        {{- end }}
         - name: S3_MINIO_ENDPOINT
           valueFrom:
             configMapKeyRef:
@@ -227,13 +211,13 @@ spec:
         - name: STATE_STORAGE_MINIO_ACCESS_KEY
           valueFrom:
             secretKeyRef:
-              name: {{ include "common.names.fullname" . }}-secrets
-              key: STATE_STORAGE_MINIO_ACCESS_KEY
+              name: {{ .Values.logs.accessKey.existingSecret }}
+              key: {{ .Values.logs.accessKey.existingSecretKey }}
         - name: STATE_STORAGE_MINIO_SECRET_ACCESS_KEY
           valueFrom:
             secretKeyRef:
-              name: {{ include "common.names.fullname" . }}-secrets
-              key: STATE_STORAGE_MINIO_SECRET_ACCESS_KEY
+              name: {{ .Values.logs.secretKey.existingSecret }}
+              key: {{ .Values.logs.secretKey.existingSecretKey }}
         - name: STATE_STORAGE_MINIO_ENDPOINT
           valueFrom:
             configMapKeyRef:

diff --git a/charts/airbyte/values.yaml b/charts/airbyte/values.yaml
@@ -612,6 +612,7 @@ bootloader:
 ## TODO: Move to consuming temporal from a dedicated helm chart
 
 temporal:
+  enabled: true
   ## @param temporal.replicaCount The number of temporal replicas to deploy
   replicaCount: 1
 
@@ -751,6 +752,14 @@ temporal:
     ##    memory: 1Gi
     limits: {}
 
+## External Temporal configuration
+## All of these values are only used when temporal.enabled is set to false
+## @param externalTemporal.host temporal API host
+## @param externalDatabase.port temporal API port
+externalTemporal:
+  host: ""
+  port: ""
+
 ## @section Airbyte Database parameters
 
 ## PostgreSQL chart configuration
@@ -798,20 +807,16 @@ externalDatabase:
 
 ## @section Logs parameters
 logs:
-  ## @param logs.accessKey.password Logs Access Key
   ## @param logs.accessKey.existingSecret
   ## @param logs.accessKey.existingSecretKey
   accessKey:
-    password: minio
-    existingSecret: ""
-    existingSecretKey: ""
-  ## @param logs.secretKey.password Logs Secret Key
+    existingSecret: "airbyte-minio"
+    existingSecretKey: "root-user"
   ## @param logs.secretKey.existingSecret
   ## @param logs.secretKey.existingSecretKey
   secretKey:
-    password: minio123
-    existingSecret: ""
-    existingSecretKey: ""
+    existingSecret: "airbyte-minio"
+    existingSecretKey: "root-password"
 
   ## @param logs.minio.enabled Switch to enable or disable the Minio helm chart
   minio:
@@ -846,13 +851,12 @@ logs:
     credentialsJson: ""
 
 ## @section Minio chart overwrites
-## @param minio.accessKey.password Minio Access Key
-## @param minio.secretKey.password Minio Secret Key
+## @param minio.rootUser Minio User
+## @param minio.rootPassword Minio Password
 minio:
-  accessKey:
-    password: minio
-  secretKey:
-    password: minio123
+  rootUser: airbyte-user
+  rootPassword: airbyte-password-123
+
 
 jobs:
   ## Jobs resource requests and limits