add github workflows for DR test

DFE-Digital · Aug 21, 2024 · 35e7520 · 35e7520
1 parent 1854eb9
commit 35e7520
Show file tree

Hide file tree

Showing 3 changed files with 174 additions and 48 deletions.
diff --git a/.github/workflows/database-backup.yml b/.github/workflows/database-backup.yml
@@ -4,72 +4,69 @@ concurrency: build_and_deploy_main
 on:
   workflow_dispatch:
     inputs:
+      environment:
+        description: Environment to backup
+        required: true
+        default: staging
+        type: choice
+        options:
+        - 'staging'
+        - 'production'
       overwriteThisMorningsBackup:
         required: true
         type: boolean
         default: false
+      PTRdatabase:
+        description: Are you backing up a PTR database copy
+        required: false
+        type: boolean
+        default: false
   schedule: # 03:00 UTC
     - cron: '0 3 * * *'
 
+env:
+  SERVICE_NAME: itt-mentor-services
+  SERVICE_SHORT: ittms
+  TF_VARS_PATH: terraform/application/config
+
 jobs:
   backup:
-    name: Backup AKS Database (production)
+    name: Backup AKS Database
     if: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.overwriteThisMorningsBackup == 'true') }}
     runs-on: ubuntu-latest
     environment:
-        name: production
+      name: ${{ inputs.environment || 'production' }}
+    env:
+      DEPLOY_ENV: ${{ inputs.environment || 'production'  }}
 
     steps:
     - uses: actions/checkout@v4
       name: Checkout
 
-    - uses: azure/login@v2
-      with:
-        creds: ${{ secrets.AZURE_CREDENTIALS }}
-
-    - name: Setup postgres client
-      uses: DFE-Digital/github-actions/install-postgres-client@master
-      with:
-        version: 14
-
-    - name: Install kubectl
-      uses: azure/setup-kubectl@v4
-      with:
-        version: "v1.26.1" # default is latest stable
-
-    - uses: DFE-Digital/github-actions/set-kubelogin-environment@master
-      with:
-        azure-credentials: ${{ secrets.AZURE_CREDENTIALS }}
-
-    - name: K8 setup
-      shell: bash
-      run: |
-        make ci production get-cluster-credentials
-        make bin/konduit.sh
-
-    - name: Set environment variable
-      run: echo "BACKUP_FILE_NAME=ittm_prod_$(date +"%F")" >> $GITHUB_ENV
-
-    - name: Backup Prod DB
+    - name: Set environment variables
       run: |
-        bin/konduit.sh -t 7200 itt-mentor-services-production -- pg_dump -E utf8 --clean --if-exists --no-owner --verbose --no-password -f ${BACKUP_FILE_NAME}.sql
-        tar -cvzf ${BACKUP_FILE_NAME}.tar.gz ${BACKUP_FILE_NAME}.sql
+        source global_config/${{ env.DEPLOY_ENV }}.sh
+        tf_vars_file=${{ env.TF_VARS_PATH }}/${{ env.DEPLOY_ENV }}.tfvars.json
+        echo "CLUSTER=$(jq -r '.cluster' ${tf_vars_file})" >> $GITHUB_ENV
+        echo "RESOURCE_GROUP_NAME=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-rg" >> $GITHUB_ENV
+        echo "STORAGE_ACCOUNT_NAME=${AZURE_RESOURCE_PREFIX}${SERVICE_SHORT}dbbkp${CONFIG_SHORT}sa" >> $GITHUB_ENV
+        TODAY=$(date +"%F")
+        echo "BACKUP_FILE=${SERVICE_SHORT}_${CONFIG_SHORT}_${TODAY}.sql" >> $GITHUB_ENV
+        echo "DB=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-pg" >> $GITHUB_ENV
 
-    - name: Set up environment variables
-      shell: bash
+    - name: Set PTR variables
+      if: ${{ github.event.inputs.PTRdatabase == 'true' }}
       run: |
-        echo "STORAGE_ACCOUNT_RG=s189p01-ittms-pd-rg" >> $GITHUB_ENV
-        echo "STORAGE_ACCOUNT_NAME=s189p01ittmsdbbkppdsa" >> $GITHUB_ENV
+        echo "PTR_DB=${{ env.DB }}-PTR" >> $GITHUB_ENV
 
-    - name: Set Connection String
-      run: |
-        STORAGE_CONN_STR=$(az storage account show-connection-string -g  $STORAGE_ACCOUNT_RG -n $STORAGE_ACCOUNT_NAME --query 'connectionString')
-        echo "::add-mask::$STORAGE_CONN_STR"
-        echo "AZURE_STORAGE_CONNECTION_STRING=$STORAGE_CONN_STR" >> $GITHUB_ENV
-
-    - name: Upload Backup to Azure Storage
-      run: |
-        az config set extension.use_dynamic_install=yes_without_prompt
-        az config set core.only_show_errors=true
-        az storage azcopy blob upload --container database-backup \
-        --source ${BACKUP_FILE_NAME}.tar.gz
+    - name: Backup ${{ env.DEPLOY_ENV }} postgres
+      if: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.overwriteThisMorningsBackup == 'true' ) }}
+      uses: DFE-Digital/github-actions/backup-postgres@master
+      with:
+        storage-account: ${{ env.STORAGE_ACCOUNT_NAME }}
+        resource-group: ${{ env.RESOURCE_GROUP_NAME }}
+        app-name: ${{ env.SERVICE_NAME }}-${{ env.DEPLOY_ENV }}
+        cluster: ${{ env.CLUSTER }}
+        azure-credentials: ${{ secrets.AZURE_CREDENTIALS }}
+        backup-file: ${{ env.BACKUP_FILE }}
+        ptr-db-server-name: ${{ env.PTR_DB }}
diff --git a/.github/workflows/postgres-ptr.yml b/.github/workflows/postgres-ptr.yml
@@ -0,0 +1,60 @@
+name: Restore Database from point in time
+concurrency: build_and_deploy_main
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: Environment to restore
+        required: true
+        default: staging
+        type: choice
+        options:
+        - 'staging'
+#        - 'production'
+      confirm-production:
+        description: Set to true to restore nightly backup if production
+        required: true
+        default: 'false'
+        type: choice
+        options:
+        - 'false'
+        - 'true'
+      restore-time:
+        description: Restore point in time. e.g. 2024-07-24T06:00:00
+        type: string
+        required: true
+
+env:
+  SERVICE_SHORT: ittms
+  TF_VARS_PATH: terraform/application/config
+
+jobs:
+  ptr-restore:
+    name: PTR Restore AKS Database
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+
+    steps:
+    - uses: actions/checkout@v4
+      name: Checkout
+
+    - name: Set environment variables
+      run: |
+        source global_config/${{ inputs.environment }}.sh
+        tf_vars_file=${{ env.TF_VARS_PATH }}/${{ inputs.environment }}.tfvars.json
+        echo "CLUSTER=$(jq -r '.cluster' ${tf_vars_file})" >> $GITHUB_ENV
+        echo "RESOURCE_GROUP_NAME=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-rg" >> $GITHUB_ENV
+        echo "DB=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-pg" >> $GITHUB_ENV
+
+      - name: Restore ${{ inputs.environment }} postgres
+        if: ${{ inputs.environment != 'production' || (inputs.environment == 'production' && github.event.inputs.confirm-production == 'true' )  }}
+
+        uses: DFE-Digital/github-actions/ptr-postgres@master
+        with:
+          resource-group: ${{ env.RESOURCE_GROUP_NAME }}
+          source-server: ${{ env.DB }}
+          new-server: ${{ env.DB }}-ptr
+          restore-time: ${{ inputs.restore-time }}
+          cluster: ${{ env.CLUSTER }}
+          azure-credentials: ${{ secrets.AZURE_CREDENTIALS}}
diff --git a/.github/workflows/postgres-restore.yml b/.github/workflows/postgres-restore.yml
@@ -0,0 +1,69 @@
+name: Restore Database from Azure Storage
+concurrency: build_and_deploy_main
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: Environment to restore
+        required: true
+        default: staging
+        type: choice
+        options:
+        - 'staging'
+#        - 'production'
+      confirm-production:
+        description: Set to true to restore nightly backup if production
+        required: true
+        default: 'false'
+        type: choice
+        options:
+        - 'false'
+        - 'true'
+      backup-file:
+        description: Name of the backup file in Azure storage. e.g. ittms_prod_2024-08-09.tar.gz. The default value is today's backup.
+        type: string
+        required: false
+
+env:
+  SERVICE_NAME: itt-mentor-services
+  SERVICE_SHORT: ittms
+  TF_VARS_PATH: terraform/application/config
+
+jobs:
+  restore:
+    name: Restore AKS Database
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+
+    steps:
+    - uses: actions/checkout@v4
+      name: Checkout
+
+    - name: Set environment variables
+      run: |
+        source global_config/${{ inputs.environment }}.sh
+        tf_vars_file=${{ env.TF_VARS_PATH }}/${{ inputs.environment }}.tfvars.json
+        echo "CLUSTER=$(jq -r '.cluster' ${tf_vars_file})" >> $GITHUB_ENV
+        echo "RESOURCE_GROUP_NAME=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-rg" >> $GITHUB_ENV
+        echo "STORAGE_ACCOUNT_NAME=${AZURE_RESOURCE_PREFIX}${SERVICE_SHORT}dbbkp${CONFIG_SHORT}sa" >> $GITHUB_ENV
+        TODAY=$(date +"%F")
+        echo "BACKUP_FILE=${SERVICE_SHORT}_${CONFIG_SHORT}_${TODAY}.sql" >> $GITHUB_ENV
+        if [ "${{ inputs.backup-file }}" != "" ]; then
+          BACKUP_FILE=${{ inputs.backup-file }}
+        else
+          BACKUP_FILE=${SERVICE_SHORT}_${CONFIG_SHORT}_${TODAY}.sql.gz
+        fi
+        echo "BACKUP_FILE=$BACKUP_FILE" >> $GITHUB_ENV
+
+      - name: Restore ${{ inputs.environment }} postgres
+        if: ${{ inputs.environment != 'production' || (inputs.environment == 'production' && github.event.inputs.confirm-production == 'true' )  }}
+
+        uses: DFE-Digital/github-actions/restore-postgres-backup@master
+        with:
+          storage-account: ${{ env.STORAGE_ACCOUNT_NAME }}
+          resource-group: ${{ env.RESOURCE_GROUP_NAME }}
+          app-name: ${{ env.SERVICE_NAME }}-${{ inputs.environment }}
+          cluster: ${{ env.CLUSTER }}
+          azure-credentials: ${{ secrets.AZURE_CREDENTIALS }}
+          backup-file: ${{ env.BACKUP_FILE }}