Merge pull request #2820 from uselagoon/custom_backup_restore_locatio…

…n_support

docs/using-lagoon-advanced/backups.md

@@ -25,3 +25,8 @@ Backups stored in Restic will be tracked within Lagoon, and can be recovered via
 
 Backups of development environments are attempted nightly and are strictly a best effort service.
 
+## Custom Backup and/or Restore Locations
+
+Lagoon supports custom backup and restore locations via the use of the "[Custom Backup Settings](https://github.com/uselagoon/lagoon/blob/main/docs/using-lagoon-advanced/environment-variables.md#custom-backup-settings)" and/or "[Custom Restore Settings](https://github.com/uselagoon/lagoon/blob/main/docs/using-lagoon-advanced/environment-variables.md#custom-restore-settings)" variables stored in the Lagoon API for each project.
+
+***Proceed with caution: Setting these variables will override backup/restore storage locations that may be configured at a cluster level. Any misconfiguration will cause backup/restore failures.***

docs/using-lagoon-advanced/environment-variables.md

@@ -167,3 +167,58 @@ On production environments, this value defaults to `E_ALL & ~E_DEPRECATED & ~E_S
 
 On development environments, this value defaults to `E_ALL & ~E_DEPRECATED & ~E_STRICT`.
 
+### Custom Backup Settings
+
+Lagoon can support custom backup locations and credentials for any project when all four of the following variables are set as `BUILD` type variables. Please note that any use of these variables means that all environment and db backups created and managed by Lagoon will be stored using these credentials, meaning that any interruption of these credentials' may lead to failed or inaccessible backups.
+
+#### `LAGOON_BAAS_CUSTOM_BACKUP_ENDPOINT`
+
+Specify the S3 compatible endpoint where any Lagoon initiated backups should be stored.
+
+An example custom setting would be: `http://10.144.1.224:9000`
+
+#### `LAGOON_BAAS_CUSTOM_BACKUP_BUCKET`
+
+Specify the bucket name where any Lagoon initiated backups should be stored.
+
+An example custom setting would be: `example-restore-bucket`
+
+#### `LAGOON_BAAS_CUSTOM_BACKUP_ACCESS_KEY`
+
+Specify the access key Lagoon should use to access the custom backup bucket.
+
+An example custom setting would be: `AAAAAAAAAAAA12345`
+
+#### `LAGOON_BAAS_CUSTOM_BACKUP_SECRET_KEY`
+
+Specify the secret key Lagoon should use to access the custom backup bucket.
+
+An example custom setting would be: `12345AAAAAAAAAAAA`
+
+### Custom Restore Location
+
+Lagoon can support custom restore locations and credentials for any project when all four of the following variables are set as `BUILD` type variables. Please note that any use of these variables means that all environment and db backups restored by Lagoon will be stored using these credentials, meaning that any interruption of these credentials' access may lead to failed or inaccessible restored files.
+
+#### `LAGOON_BAAS_CUSTOM_RESTORE_ENDPOINT`
+
+Specify the S3 compatible endpoint where any Lagoon initiated restores should be stored.
+
+An example custom setting would be: `http://10.144.1.224:9000`
+
+#### `LAGOON_BAAS_CUSTOM_RESTORE_BUCKET`
+
+Specify the bucket name where any Lagoon initiated restores should be stored.
+
+An example custom setting would be: `example-restore-bucket`
+
+#### `LAGOON_BAAS_CUSTOM_RESTORE_ACCESS_KEY`
+
+Specify the access key Lagoon should use to access the custom restore bucket.
+
+An example custom setting would be: `AAAAAAAAAAAA12345`
+
+#### `LAGOON_BAAS_CUSTOM_RESTORE_SECRET_KEY`
+
+Specify the secret key Lagoon should use to access the custom restore bucket.
+
+An example custom setting would be: `12345AAAAAAAAAAAA`

images/kubectl-build-deploy-dind/build-deploy-docker-compose.sh

@@ -1273,6 +1273,42 @@ fi
 # If k8up is supported by this cluster we create the schedule definition
 if [[ "${CAPABILITIES[@]}" =~ "backup.appuio.ch/v1alpha1/Schedule" ]]; then
 
+  # Parse out custom baas backup location variables
+  if [ ! -z "$LAGOON_PROJECT_VARIABLES" ]; then
+    BAAS_CUSTOM_BACKUP_ENDPOINT=($(echo $LAGOON_PROJECT_VARIABLES | jq -r '.[] | select(.name == "LAGOON_BAAS_CUSTOM_BACKUP_ENDPOINT") | "\(.value)"'))
+    BAAS_CUSTOM_BACKUP_BUCKET=($(echo $LAGOON_PROJECT_VARIABLES | jq -r '.[] | select(.name == "LAGOON_BAAS_CUSTOM_BACKUP_BUCKET") | "\(.value)"'))
+    BAAS_CUSTOM_BACKUP_ACCESS_KEY=($(echo $LAGOON_PROJECT_VARIABLES | jq -r '.[] | select(.name == "LAGOON_BAAS_CUSTOM_BACKUP_ACCESS_KEY") | "\(.value)"'))
+    BAAS_CUSTOM_BACKUP_SECRET_KEY=($(echo $LAGOON_PROJECT_VARIABLES | jq -r '.[] | select(.name == "LAGOON_BAAS_CUSTOM_BACKUP_SECRET_KEY") | "\(.value)"'))
+
+    if [ ! -z $BAAS_CUSTOM_BACKUP_ENDPOINT ] && [ ! -z $BAAS_CUSTOM_BACKUP_BUCKET ] && [ ! -z $BAAS_CUSTOM_BACKUP_ACCESS_KEY ] && [ ! -z $BAAS_CUSTOM_BACKUP_SECRET_KEY ]; then
+      CUSTOM_BAAS_BACKUP_ENABLED=1
+
+      HELM_CUSTOM_BAAS_BACKUP_ACCESS_KEY=${BAAS_CUSTOM_BACKUP_ACCESS_KEY}
+      HELM_CUSTOM_BAAS_BACKUP_SECRET_KEY=${BAAS_CUSTOM_BACKUP_SECRET_KEY}
+    else
+      set +x
+      kubectl --insecure-skip-tls-verify -n ${NAMESPACE} delete secret baas-custom-backup-credentials --ignore-not-found
+      set -x
+    fi
+  fi
+
+  # Parse out custom baas restore location variables
+  if [ ! -z "$LAGOON_PROJECT_VARIABLES" ]; then
+    BAAS_CUSTOM_RESTORE_ENDPOINT=($(echo $LAGOON_PROJECT_VARIABLES | jq -r '.[] | select(.name == "LAGOON_BAAS_CUSTOM_RESTORE_ENDPOINT") | "\(.value)"'))
+    BAAS_CUSTOM_RESTORE_BUCKET=($(echo $LAGOON_PROJECT_VARIABLES | jq -r '.[] | select(.name == "LAGOON_BAAS_CUSTOM_RESTORE_BUCKET") | "\(.value)"'))
+    BAAS_CUSTOM_RESTORE_ACCESS_KEY=($(echo $LAGOON_PROJECT_VARIABLES | jq -r '.[] | select(.name == "LAGOON_BAAS_CUSTOM_RESTORE_ACCESS_KEY") | "\(.value)"'))
+    BAAS_CUSTOM_RESTORE_SECRET_KEY=($(echo $LAGOON_PROJECT_VARIABLES | jq -r '.[] | select(.name == "LAGOON_BAAS_CUSTOM_RESTORE_SECRET_KEY") | "\(.value)"'))
+
+    if [ ! -z $BAAS_CUSTOM_RESTORE_ENDPOINT ] && [ ! -z $BAAS_CUSTOM_RESTORE_BUCKET ] && [ ! -z $BAAS_CUSTOM_RESTORE_ACCESS_KEY ] && [ ! -z $BAAS_CUSTOM_RESTORE_SECRET_KEY ]; then
+      HELM_CUSTOM_BAAS_RESTORE_ACCESS_KEY=${BAAS_CUSTOM_RESTORE_ACCESS_KEY}
+      HELM_CUSTOM_BAAS_RESTORE_SECRET_KEY=${BAAS_CUSTOM_RESTORE_SECRET_KEY}
+    else
+      set +x
+      kubectl --insecure-skip-tls-verify -n ${NAMESPACE} delete secret baas-custom-restore-credentials --ignore-not-found
+      set -x
+    fi
+  fi
+
   if ! kubectl --insecure-skip-tls-verify -n ${NAMESPACE} get secret baas-repo-pw &> /dev/null; then
     # Create baas-repo-pw secret based on the project secret
     set +x
@@ -1347,17 +1383,34 @@ if [[ "${CAPABILITIES[@]}" =~ "backup.appuio.ch/v1alpha1/Schedule" ]]; then
     PRUNE_SCHEDULE=$( /kubectl-build-deploy/scripts/convert-crontab.sh "${NAMESPACE}" "M H(3-6) * * 6")
   fi
 
+  # Set the S3 variables which should be passed to the helm chart
+  if [ ! -z $CUSTOM_BAAS_BACKUP_ENABLED ]; then
+    BAAS_BACKUP_ENDPOINT=${BAAS_CUSTOM_BACKUP_ENDPOINT}
+    BAAS_BACKUP_BUCKET=${BAAS_CUSTOM_BACKUP_BUCKET}
+    BAAS_BACKUP_SECRET_NAME='lagoon-baas-custom-backup-credentials'
+  else
+    BAAS_BACKUP_ENDPOINT=''
+    BAAS_BACKUP_BUCKET=${BAAS_BUCKET_NAME}
+    BAAS_BACKUP_SECRET_NAME=''
+  fi
+
   OPENSHIFT_TEMPLATE="/kubectl-build-deploy/openshift-templates/backup-schedule.yml"
   helm template k8up-lagoon-backup-schedule /kubectl-build-deploy/helmcharts/k8up-schedule \
     -f /kubectl-build-deploy/values.yaml \
     --set backup.schedule="${BACKUP_SCHEDULE}" \
     --set check.schedule="${CHECK_SCHEDULE}" \
-    --set prune.schedule="${PRUNE_SCHEDULE}" "${HELM_ARGUMENTS[@]}" \
-    --set baasBucketName="${BAAS_BUCKET_NAME}" > $YAML_FOLDER/k8up-lagoon-backup-schedule.yaml \
-    --set prune.retention.keepMonthly=$MONTHLY_BACKUP_RETENTION \
-    --set prune.retention.keepWeekly=$WEEKLY_BACKUP_RETENTION \
-    --set prune.retention.keepDaily=$DAILY_BACKUP_RETENTION \
-    --set prune.retention.keepHourly=$HOURLY_BACKUP_RETENTION
+    --set prune.schedule="${PRUNE_SCHEDULE}" \
+    --set prune.retention.keepMonthly=${MONTHLY_BACKUP_RETENTION} \
+    --set prune.retention.keepWeekly=${WEEKLY_BACKUP_RETENTION} \
+    --set prune.retention.keepDaily=${DAILY_BACKUP_RETENTION} \
+    --set prune.retention.keepHourly=${HOURLY_BACKUP_RETENTION} \
+    --set s3.endpoint="${BAAS_BACKUP_ENDPOINT}" \
+    --set s3.bucket="${BAAS_BACKUP_BUCKET}" \
+    --set s3.secretName="${BAAS_BACKUP_SECRET_NAME}" \
+    --set customRestoreLocation.accessKey="${BAAS_CUSTOM_RESTORE_ACCESS_KEY}" \
+    --set customRestoreLocation.secretKey="${BAAS_CUSTOM_RESTORE_SECRET_KEY}" \
+    --set customBackupLocation.accessKey="${BAAS_CUSTOM_BACKUP_ACCESS_KEY}" \
+    --set customBackupLocation.secretKey="${BAAS_CUSTOM_BACKUP_SECRET_KEY}" "${HELM_ARGUMENTS[@]}" > $YAML_FOLDER/k8up-lagoon-backup-schedule.yaml
 fi
 
 if [ "$(ls -A $YAML_FOLDER/)" ]; then

images/kubectl-build-deploy-dind/helmcharts/k8up-schedule/templates/custom-baas-backup-location-secret.yaml

@@ -0,0 +1,13 @@
+{{- if and (.Values.customBackupLocation.accessKey) (.Values.customBackupLocation.secretKey) }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: lagoon-baas-custom-backup-credentials
+  labels:
+    {{- include "k8up-schedule.labels" . | nindent 4 }}
+  annotations:
+    {{- include "k8up-schedule.annotations" . | nindent 4 }}
+stringData:
+  access-key: {{ .Values.customBackupLocation.accessKey }}
+  secret-key: {{ .Values.customBackupLocation.secretKey }}
+{{ end }}

images/kubectl-build-deploy-dind/helmcharts/k8up-schedule/templates/custom-baas-restore-location-secret.yaml

@@ -0,0 +1,13 @@
+{{- if and (.Values.customRestoreLocation.accessKey) (.Values.customRestoreLocation.secretKey) }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: lagoon-baas-custom-restore-credentials
+  labels:
+    {{- include "k8up-schedule.labels" . | nindent 4 }}
+  annotations:
+    {{- include "k8up-schedule.annotations" . | nindent 4 }}
+stringData:
+  access-key: {{ .Values.customRestoreLocation.accessKey }}
+  secret-key: {{ .Values.customRestoreLocation.secretKey }}
+{{ end }}

images/kubectl-build-deploy-dind/helmcharts/k8up-schedule/templates/schedule.yaml

@@ -12,7 +12,20 @@ spec:
       key: repo-pw
       name: baas-repo-pw
     s3:
-      bucket: '{{ .Values.baasBucketName }}'
+      {{- if .Values.s3.endpoint }}
+      endpoint: '{{ .Values.s3.endpoint }}'
+      {{- end }}
+      {{- if .Values.s3.bucket }}
+      bucket: '{{ .Values.s3.bucket }}'
+      {{- end }}
+      {{- if .Values.s3.secretName }}
+      accessKeyIDSecretRef:
+        name: '{{ .Values.s3.secretName }}'
+        key: access-key
+      secretAccessKeySecretRef:
+        name: '{{ .Values.s3.secretName }}'
+        key: secret-key
+      {{- end }}
   backup:
     schedule: '{{ .Values.backup.schedule }}'
   check:

images/kubectl-build-deploy-dind/helmcharts/k8up-schedule/values.yaml

@@ -16,4 +16,9 @@ prune:
     keepDaily: 7
     keepWeekly: 6
     keepMonthly: 1
-  schedule: '0 * * * 6'
+  schedule: '0 * * * 6'
+
+s3:
+  endpoint: ''
+  bucket: ''
+  secretName: ''

images/kubectl-build-deploy-dind/helmcharts/postgres-dbaas/templates/prebackuppod.yaml

@@ -8,7 +8,7 @@ metadata:
   annotations:
     {{- include "postgres-dbaas.annotations" . | nindent 4 }}
 spec:
-  backupCommand: /bin/sh -c "PGPASSWORD=$BACKUP_DB_PASSWORD pg_dump --host=$BACKUP_DB_HOST --port=$BACKUP_DB_PORT --dbname=$BACKUP_DB_NAME --format=t -w"
+  backupCommand: /bin/sh -c "PGPASSWORD=$BACKUP_DB_PASSWORD pg_dump --host=$BACKUP_DB_HOST --port=$BACKUP_DB_PORT --dbname=$BACKUP_DB_NAME --username=$BACKUP_DB_USERNAME --format=t -w"
   fileExtension: .{{ include "postgres-dbaas.fullname" . }}.tar
   pod:
     metadata:
@@ -44,7 +44,7 @@ spec:
             - name: BACKUP_DB_NAME
               valueFrom:
                 configMapKeyRef:
-                  key: {{ include "postgres-dbaas.fullnameUppercase" . }}_NAME
+                  key: {{ include "postgres-dbaas.fullnameUppercase" . }}_DATABASE
                   name: lagoon-env
           image: imagecache.amazeeio.cloud/uselagoon/php-8.0-cli
           imagePullPolicy: Always

node-packages/commons/src/tasks.ts

@@ -950,7 +950,7 @@ export const createRemoveTask = async function(removeData: any) {
 }
 
 // creates the restore job configuration for use in the misc task
-const restoreConfig = (name, backupId, safeProjectName, baasBucketName) => {
+const restoreConfig = (name, backupId, safeProjectName, baasBucketName, backupS3Config, restoreS3Config) => {
   let config = {
     apiVersion: 'backup.appuio.ch/v1alpha1',
     kind: 'Restore',
@@ -960,10 +960,10 @@ const restoreConfig = (name, backupId, safeProjectName, baasBucketName) => {
     spec: {
       snapshot: backupId,
       restoreMethod: {
-        s3: {},
+        s3: restoreS3Config ? restoreS3Config : {},
       },
       backend: {
-        s3: {
+        s3: backupS3Config ? backupS3Config : {
           bucket: baasBucketName ? baasBucketName : `baas-${safeProjectName}`
         },
         repoPasswordSecretRef: {
@@ -1070,8 +1070,94 @@ export const createMiscTask = async function(taskData: any) {
           if (baasBucketName) {
             baasBucketName = baasBucketName.value
           }
+
+          // Handle custom backup configurations
+          let lagoonBaasCustomBackupEndpoint = result.project.envVariables.find(obj => {
+            return obj.name === "LAGOON_BAAS_CUSTOM_BACKUP_ENDPOINT"
+          })
+          if (lagoonBaasCustomBackupEndpoint) {
+            lagoonBaasCustomBackupEndpoint = lagoonBaasCustomBackupEndpoint.value
+          }
+          let lagoonBaasCustomBackupBucket = result.project.envVariables.find(obj => {
+            return obj.name === "LAGOON_BAAS_CUSTOM_BACKUP_BUCKET"
+          })
+          if (lagoonBaasCustomBackupBucket) {
+            lagoonBaasCustomBackupBucket = lagoonBaasCustomBackupBucket.value
+          }
+          let lagoonBaasCustomBackupAccessKey = result.project.envVariables.find(obj => {
+            return obj.name === "LAGOON_BAAS_CUSTOM_BACKUP_ACCESS_KEY"
+          })
+          if (lagoonBaasCustomBackupAccessKey) {
+            lagoonBaasCustomBackupAccessKey = lagoonBaasCustomBackupAccessKey.value
+          }
+          let lagoonBaasCustomBackupSecretKey = result.project.envVariables.find(obj => {
+            return obj.name === "LAGOON_BAAS_CUSTOM_BACKUP_SECRET_KEY"
+          })
+          if (lagoonBaasCustomBackupSecretKey) {
+            lagoonBaasCustomBackupSecretKey = lagoonBaasCustomBackupSecretKey.value
+          }
+
+          let backupS3Config = {}
+          if (lagoonBaasCustomBackupEndpoint && lagoonBaasCustomBackupBucket && lagoonBaasCustomBackupAccessKey && lagoonBaasCustomBackupSecretKey) {
+            backupS3Config = {
+              endpoint: lagoonBaasCustomBackupEndpoint,
+              bucket: lagoonBaasCustomBackupBucket,
+              accessKeyIDSecretRef: {
+                name: "lagoon-baas-custom-backup-credentials",
+                key: "access-key"
+              },
+              secretAccessKeySecretRef: {
+                name: "lagoon-baas-custom-backup-credentials",
+                key: "secret-key"
+              }
+            }
+          }
+
+          // Handle custom restore configurations
+          let lagoonBaasCustomRestoreEndpoint = result.project.envVariables.find(obj => {
+            return obj.name === "LAGOON_BAAS_CUSTOM_RESTORE_ENDPOINT"
+          })
+          if (lagoonBaasCustomRestoreEndpoint) {
+            lagoonBaasCustomRestoreEndpoint = lagoonBaasCustomRestoreEndpoint.value
+          }
+          let lagoonBaasCustomRestoreBucket = result.project.envVariables.find(obj => {
+            return obj.name === "LAGOON_BAAS_CUSTOM_RESTORE_BUCKET"
+          })
+          if (lagoonBaasCustomRestoreBucket) {
+            lagoonBaasCustomRestoreBucket = lagoonBaasCustomRestoreBucket.value
+          }
+          let lagoonBaasCustomRestoreAccessKey = result.project.envVariables.find(obj => {
+            return obj.name === "LAGOON_BAAS_CUSTOM_RESTORE_ACCESS_KEY"
+          })
+          if (lagoonBaasCustomRestoreAccessKey) {
+            lagoonBaasCustomRestoreAccessKey = lagoonBaasCustomRestoreAccessKey.value
+          }
+          let lagoonBaasCustomRestoreSecretKey = result.project.envVariables.find(obj => {
+            return obj.name === "LAGOON_BAAS_CUSTOM_RESTORE_SECRET_KEY"
+          })
+          if (lagoonBaasCustomRestoreSecretKey) {
+            lagoonBaasCustomRestoreSecretKey = lagoonBaasCustomRestoreSecretKey.value
+          }
+
+          let restoreS3Config = {}
+          if (lagoonBaasCustomRestoreEndpoint && lagoonBaasCustomRestoreBucket && lagoonBaasCustomRestoreAccessKey && lagoonBaasCustomRestoreSecretKey) {
+            restoreS3Config = {
+              endpoint: lagoonBaasCustomRestoreEndpoint,
+              bucket: lagoonBaasCustomRestoreBucket,
+              accessKeyIDSecretRef: {
+                name: "lagoon-baas-custom-restore-credentials",
+                key: "access-key"
+              },
+              secretAccessKeySecretRef: {
+                name: "lagoon-baas-custom-restore-credentials",
+                key: "secret-key"
+              }
+            }
+          }
+
           // generate the restore CRD
-          const restoreConf = restoreConfig(restoreName, taskData.data.backup.backupId, makeSafe(taskData.data.project.name), baasBucketName)
+          const restoreConf = restoreConfig(restoreName, taskData.data.backup.backupId, makeSafe(taskData.data.project.name), baasBucketName, backupS3Config, restoreS3Config)
+          //logger.info(restoreConf)
           // base64 encode it
           const restoreBytes = new Buffer(JSON.stringify(restoreConf).replace(/\\n/g, "\n")).toString('base64')
           miscTaskData.misc.miscResource = restoreBytes