feat: add data-migrate helm chart

.prettierignore

@@ -0,0 +1 @@
+helm/*/templates/*.yaml

helm/cas-ciip-portal-data-migrate/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

helm/cas-ciip-portal-data-migrate/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v2
+name: cas-ciip-portal-data-migrate
+description: A Helm chart for Kubernetes
+version: 0.1.0
+appVersion: 0.1.0

helm/cas-ciip-portal-data-migrate/templates/_helpers.tpl

@@ -0,0 +1,62 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "cas-ciip-portal-data-migrate.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "cas-ciip-portal-data-migrate.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "cas-ciip-portal-data-migrate.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "cas-ciip-portal-data-migrate.labels" -}}
+helm.sh/chart: {{ include "cas-ciip-portal-data-migrate.chart" . }}
+{{ include "cas-ciip-portal-data-migrate.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "cas-ciip-portal-data-migrate.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "cas-ciip-portal-data-migrate.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "cas-ciip-portal-data-migrate.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "cas-ciip-portal-data-migrate.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

helm/cas-ciip-portal-data-migrate/templates/job.yaml

@@ -0,0 +1,156 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ template "cas-ciip-portal-data-migrate.fullname" . }}-{{ randAlphaNum 5 | lower }}
+  labels: {{ include "cas-ciip-portal-data-migrate.labels" . | nindent 4 }}
+  # annotations:
+  #   "helm.sh/hook": "pre-upgrade,pre-install"
+  #   "helm.sh/hook-delete-policy": "before-hook-creation"
+  #   "helm.sh/hook-weight": "-5"
+spec:
+  backoffLimit: 0
+  activeDeadlineSeconds: 3600
+  template:
+    metadata:
+      labels: {{ include "cas-ciip-portal-data-migrate.labels" . | nindent 8 }}
+        release: cas-ciip-portal
+    spec:
+      activeDeadlineSeconds: 3600
+      restartPolicy: Never
+      containers:
+        - env:
+            - name: PORTAL_USER
+              value: postgres
+            - name: PORTAL_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  key: password-superuser
+                  name: cas-ciip-portal-patroni
+            - name: PORTAL_DATABASE
+              valueFrom:
+                secretKeyRef:
+                  key: database-name
+                  name: cas-ciip-portal
+            - name: PORTAL_APP_USER
+              valueFrom:
+                secretKeyRef:
+                  key: database-app-user
+                  name: cas-ciip-portal
+            - name: PORTAL_READONLY_USER
+              valueFrom:
+                secretKeyRef:
+                  key: database-readonly-user
+                  name: cas-ciip-portal
+            - name: PORTAL_HOST
+              value: cas-ciip-portal-patroni
+            - name: PORTAL_PORT
+              value: "5432"
+            - name: GOOGLE_APPLICATION_CREDENTIALS
+              value: "/home/postgres/credentials/credentials.json"
+            - name: GOOGLE_APPLICATION_CREDENTIALS_JSON
+              valueFrom:
+                secretKeyRef:
+                  key: credentials.json
+                  name: {{ .Values.existingGcpSecret }}
+            - name: WALG_GS_PREFIX
+              value: "gs://wksv3k-prod-ciip-backups/wal"
+            - name: WALE_GS_PREFIX
+              value: "gs://wksv3k-prod-ciip-backups/wal"
+          name: {{ template "cas-ciip-portal-data-migrate.fullname" . }}-prod-test-restore
+          image: {{ .Values.image.psql.repository }}:{{ .Values.image.psql.tag }}
+          resources:
+            limits:
+              cpu: 500m
+              memory: 1Gi
+            requests:
+              cpu: 100m
+              memory: 500Mi
+          command:
+            - /usr/bin/env
+            - bash
+            - -c
+            - |
+              set -euo pipefail;
+              if ! whoami &> /dev/null; then
+                if [ -w /etc/passwd ]; then
+                  echo "postgres:x:$(id -u):0:postgres user:/home/postgres:/sbin/nologin" >> /etc/passwd
+                fi
+              fi
+              rm -rf prod-bak
+              rm -rf credentials
+              echo ADDING GCS CREDENTIALS...
+              mkdir credentials
+              echo $GOOGLE_APPLICATION_CREDENTIALS_JSON > credentials/credentials.json
+              echo DOING WAL-G FETCH NOW...
+              wal-g backup-fetch prod-bak LATEST
+              cd prod-bak
+              echo ADDING recovery.signal...
+              touch recovery.signal
+              echo FIXING PG_HBA...
+              echo 'local all             all                                trust' > pg_hba.conf
+              cd ..
+              chmod -R 0750 ./prod-bak
+              echo STARTING SERVER IN RECOVERY MODE...
+              pg_ctl -D prod-bak start -w -o "-c archive_command='' -c ssl='off' -c hba_file='/home/postgres/prod-bak/pg_hba.conf' -c ident_file='/home/postgres/prod-bak/pg_ident.conf' --restore_command='wal-g wal-fetch %f %p' --recovery_target_action=promote"
+              echo RESTART POSTGRES IN NON-RECOVERY MODE...
+              pg_ctl -D prod-bak stop
+              cd prod-bak
+              rm recovery.signal
+              rm -rf backup_label
+              rm -rf backup_label.old
+              cd ..
+              pg_ctl -D prod-bak start -w -o "-c log_statement='all' -c logging_collector=on -c archive_mode='off' -c archive_command='' -c ssl='off' -c hba_file='/home/postgres/prod-bak/pg_hba.conf' -c ident_file='/home/postgres/prod-bak/pg_ident.conf'"
+
+              echo "$PORTAL_HOST:$PORTAL_PORT:$PORTAL_DATABASE:$PORTAL_USER:$PORTAL_PASSWORD" >> .pgpass
+              chmod 0600 .pgpass
+              # we need to recreate users first as the ACL created in sqitch need to be dumped to DB
+              # no need to grant any privileges on schemas or set passwords here, this will be done in the app-users job
+              PGPASSWORD="$PORTAL_PASSWORD" PGHOST=$PORTAL_HOST PGUSER=$PORTAL_USER psql<<EOF
+                create user $PORTAL_APP_USER;
+                grant connect on database ciip to $PORTAL_APP_USER;
+                create user $PORTAL_READONLY_USER;
+                grant connect on database ciip to $PORTAL_READONLY_USER;
+                do
+                \$do\$
+                begin
+                  if not exists (
+                      select true
+                      from   pg_catalog.pg_roles
+                      where  rolname = 'ciip_administrator') then
+
+                      create role ciip_administrator;
+                  end if;
+                  if not exists (
+                      select true
+                      from   pg_catalog.pg_roles
+                      where  rolname = 'ciip_analyst') then
+
+                      create role ciip_analyst;
+                  end if;
+                  if not exists (
+                      select true
+                      from   pg_catalog.pg_roles
+                      where  rolname = 'ciip_industry_user') then
+
+                      create role ciip_industry_user;
+                  end if;
+                  if not exists (
+                      select true
+                      from   pg_catalog.pg_roles
+                      where  rolname = 'ciip_guest') then
+
+                      create role ciip_guest;
+                  end if;
+                end
+                \$do\$;
+              EOF
+              echo RESTORING TO TEST...
+              pg_dump --no-password --format=custom --schema=sqitch --schema=graphile_worker --schema=ggircs_portal --schema=ggircs_portal_private --schema=swrs ciip |
+              PGPASSFILE=".pgpass" pg_restore --clean --exit-on-error --no-password -d $PORTAL_DATABASE -h $PORTAL_HOST -p $PORTAL_PORT -U $PORTAL_USER
+          volumeMounts:
+            - mountPath: /home/postgres/
+              name: prod-restore
+      volumes:
+        - name: prod-restore
+          persistentVolumeClaim:
+            claimName: {{ template "cas-ciip-portal-data-migrate.fullname" . }}

helm/cas-ciip-portal-data-migrate/templates/pvc.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ template "cas-ciip-portal-data-migrate.fullname" . }}
+  labels: {{ include "cas-ciip-portal-data-migrate.labels" . | nindent 4 }}
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ .Values.storage.size }}
+  storageClassName: {{ .Values.storage.storageClass }}

helm/cas-ciip-portal-data-migrate/values.yaml

@@ -0,0 +1,10 @@
+storage:
+  size: 10Gi
+  storageClass: netapp-block-standard
+image:
+  psql:
+    repository: gcr.io/ggl-cas-storage/cas-postgres
+    pullPolicy: Always
+    tag: "0.2.0"
+
+existingGcpSecret: gcp-wksv3k-prod-read-only-service-account-key # pragma: allowlist secret