Update blueprints (#1004)

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
kanisterio · May 20, 2021 · 6c11103 · 6c11103
1 parent fec0016
commit 6c11103
Show file tree

Hide file tree

Showing 10 changed files with 190 additions and 141 deletions.
diff --git a/examples/stable/couchbase/blueprint-v2/couchbase-blueprint.yaml b/examples/stable/couchbase/blueprint-v2/couchbase-blueprint.yaml
@@ -7,9 +7,10 @@ actions:
     kind: CustomResource
     outputArtifacts:
       cbBackup:
-        keyValue:
-          backupPath: "{{ .Phases.TakeBackup.Output.backupPath }}"
-          backupID: "{{ .Phases.TakeBackup.Output.backupID }}"
+        # Capture the kopia snapshot information for subsequent actions
+        # The information includes the kopia snapshot ID which is essential for restore and delete to succeed
+        # `kopiaOutput` is the name provided to kando using `--output-name` flag
+        kopiaSnapshot: "{{ .Phases.TakeBackup.Output.kopiaOutput }}"
     phases:
     - func: KubeTask
       name: TakeBackup
@@ -20,7 +21,7 @@ actions:
           namespace: "{{ .Object.metadata.namespace }}"
       args:
         namespace: "{{ .Object.metadata.namespace }}"
-        image: ghcr.io/kanisterio/couchbase-tools:0.53.0
+        image: ghcr.io/kanisterio/couchbase-tools:0.57.0
         command:
           - bash
           - -o
@@ -38,13 +39,14 @@ actions:
               --cluster couchbase://{{ .Object.metadata.name }}.{{ .Object.metadata.namespace }}.svc    \
               --username {{ .Phases.TakeBackup.Secrets.authSecret.Data.username | toString }} \
               --password {{ .Phases.TakeBackup.Secrets.authSecret.Data.password | toString }}
-            backup_path='/couchbase-backups/{{ toDate "2006-01-02T15:04:05.999999999Z07:00" .Time  | date "2006-01-02T15-04-05" }}/backup.tgz'
-            tar zcvf - -C /backups/ test | kando location push --profile '{{ toJson .Profile }}' --path ${backup_path} -
-            kando output backupPath ${backup_path}
+            backup_file_path='backup.tgz'
+            tar zcvf - -C /backups/ test | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" -
 
   restore:
     kind: CustomResource
     inputArtifactNames:
+    # The kopia snapshot info created in backup phase can be used here
+    # Use the `--kopia-snapshot` flag in kando to pass in `cbBackup.KopiaSnapshot`
     - cbBackup
     phases:
     - func: KubeTask
@@ -67,7 +69,8 @@ actions:
           - |
             rm -rf /backups/test
             mkdir -p /backups/test
-            kando location pull --profile '{{ toJson .Profile }}' --path '{{ .ArtifactsIn.cbBackup.KeyValue.backupPath }}' --backupID '{{ .ArtifactsIn.cbBackup.KeyValue.backupID }}' - | tar zxvf - -C /backups/
+            backup_file_path='backup.tgz'
+            kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot '{{ .ArtifactsIn.cbBackup.KopiaSnapshot }}' - | tar zxvf - -C /backups/
             cbbackupmgr restore                                             \
               --archive /backups/test                                       \
               --repo couchbase                                              \
@@ -78,6 +81,10 @@ actions:
 
   delete:
     type: Namespace
+    inputArtifactNames:
+    # The kopia snapshot info created in backup phase can be used here
+    # Use the `--kopia-snapshot` flag in kando to pass in `cbBackup.KopiaSnapshot`
+    - cbBackup
     phases:
     - func: KubeTask
       name: deleteBackup
@@ -92,6 +99,5 @@ actions:
           - errexit
           - -c
           - |
-            backup_path='{{ .ArtifactsIn.cbBackup.KeyValue.backupPath }}'
-            backup_id='{{ .ArtifactsIn.cbBackup.KeyValue.backupID }}'
-            kando location delete --profile '{{ toJson .Profile }}' --path ${backup_path} --backupID ${backup_id}
+            backup_file_path='backup.tgz'
+            kando location delete --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot '{{ .ArtifactsIn.cbBackup.KopiaSnapshot }}'
diff --git a/examples/stable/elasticsearch/blueprint-v2/elasticsearch-blueprint.yaml b/examples/stable/elasticsearch/blueprint-v2/elasticsearch-blueprint.yaml
@@ -7,15 +7,16 @@ actions:
     type: StatefulSet
     outputArtifacts:
       esBackup:
-        keyValue:
-          backupPath: "{{ .Phases.backupToStore.Output.backupPath }}"
-          backupID: "{{ .Phases.backupToStore.Output.backupID }}"
+        # Capture the kopia snapshot information for subsequent actions
+        # The information includes the kopia snapshot ID which is essential for restore and delete to succeed
+        # `kopiaOutput` is the name provided to kando using `--output-name` flag
+        kopiaSnapshot: "{{ .Phases.backupToStore.Output.kopiaOutput }}"
     phases:
     - func: KubeTask
       name: backupToStore
       args:
         namespace: "{{ .StatefulSet.Namespace }}"
-        image: "ghcr.io/kanisterio/es-sidecar:0.53.0"
+        image: "ghcr.io/kanisterio/es-sidecar:0.57.0"
         command:
         - bash
         - -o
@@ -25,21 +26,22 @@ actions:
         - -c
         - |
           host_name="{{ .Object.spec.serviceName }}.{{ .StatefulSet.Namespace }}.svc.cluster.local"
-          BACKUP_PATH=es_backups/{{ .StatefulSet.Namespace }}/{{ .StatefulSet.Name }}/{{ toDate "2006-01-02T15:04:05.999999999Z07:00" .Time | date "2006-01-02T15:04:05Z07:00" }}/backup.gz
+          backup_file_path='backup.gz'
           elasticdump --bulk=true --input=http://${host_name}:9200 --output=/backup
           gzip /backup
-          kando location push --profile '{{ toJson .Profile }}'  /backup.gz --path $BACKUP_PATH
-          kando output backupPath $BACKUP_PATH
+          kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" /backup.gz
   restore:
     type: StatefulSet
     inputArtifactNames:
+    # The kopia snapshot info created in backup phase can be used here
+    # Use the `--kopia-snapshot` flag in kando to pass in `esBackup.KopiaSnapshot`
     - esBackup
     phases:
     - func: KubeTask
       name: restoreFromStore
       args:
         namespace: "{{ .StatefulSet.Namespace }}"
-        image: "ghcr.io/kanisterio/es-sidecar:0.53.0"
+        image: "ghcr.io/kanisterio/es-sidecar:0.57.0"
         command:
         - bash
         - -o
@@ -49,12 +51,14 @@ actions:
         - -c
         - |
           host_name="{{ .Object.spec.serviceName }}.{{ .StatefulSet.Namespace }}.svc.cluster.local"
-          backup_path='{{ .ArtifactsIn.esBackup.KeyValue.backupPath }}'
-          backup_id='{{ .ArtifactsIn.esBackup.KeyValue.backupID }}'
-          kando location pull --profile '{{ toJson .Profile }}' --path ${backup_path} --backupID ${backup_id} - | gunzip | elasticdump --bulk=true --input=$ --output=http://${host_name}:9200
+          backup_file_path='backup.gz'
+          kopia_snap='{{ .ArtifactsIn.esBackup.KopiaSnapshot }}'
+          kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - | gunzip | elasticdump --bulk=true --input=$ --output=http://${host_name}:9200
   delete:
     type: Namespace
     inputArtifactNames:
+    # The kopia snapshot info created in backup phase can be used here
+    # Use the `--kopia-snapshot` flag in kando to pass in `esBackup.KopiaSnapshot`
     - esBackup
     phases:
     - func: KubeTask
@@ -70,6 +74,6 @@ actions:
         - pipefail
         - -c
         - |
-          backup_path='{{ .ArtifactsIn.esBackup.KeyValue.backupPath }}'
-          backup_id='{{ .ArtifactsIn.esBackup.KeyValue.backupID }}'
-          kando location delete --profile '{{ toJson .Profile }}' --path ${backup_path} --backupID ${backup_id}
+          backup_file_path='backup.gz'
+          kopia_snap='{{ .ArtifactsIn.esBackup.KopiaSnapshot }}'
+          kando location delete --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}"
diff --git a/examples/stable/foundationdb/blueprint-v2/foundationdb-blueprint.yaml b/examples/stable/foundationdb/blueprint-v2/foundationdb-blueprint.yaml
@@ -7,9 +7,10 @@ actions:
     kind: CustomResource
     outputArtifacts:
       fdbBackup:
-        keyValue:
-          backupPath: "{{ .Phases.TakeBackup.Output.backupPath }}"
-          backupID: "{{ .Phases.TakeBackup.Output.backupID }}"
+        # Capture the kopia snapshot information for subsequent actions
+        # The information includes the kopia snapshot ID which is essential for restore and delete to succeed
+        # `kopiaOutput` is the name provided to kando using `--output-name` flag
+        kopiaSnapshot: "{{ .Phases.TakeBackup.Output.kopiaOutput }}"
     phases:
     - func: KubeExec
       name: TakeBackup
@@ -33,13 +34,14 @@ actions:
             chmod -R 777 /data/fdbbackup
             fdbbackup start  -d file:///data/fdbbackup -w
             pkill -9 backup_agent
-            backup_path='/foundationdb-backups/{{ toDate "2006-01-02T15:04:05.999999999Z07:00" .Time  | date "2006-01-02T15-04-05" }}/backup.tgz'
-            tar zcvf - -C /data/ fdbbackup | kando location push --profile '{{ toJson .Profile }}' --path ${backup_path} -
+            backup_file_path='backup.tgz'
+            tar zcvf - -C /data/ fdbbackup | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" -
             rm -rf /data/fdbbackup
-            kando output backupPath ${backup_path}
   restore:
     kind: CustomResource
     inputArtifactNames:
+    # The kopia snapshot info created in backup phase can be used here
+    # Use the `--kopia-snapshot` flag in kando to pass in `fdbBackup.KopiaSnapshot`
     - fdbBackup
     phases:
     - func: KubeExec
@@ -57,9 +59,9 @@ actions:
           - -c
           - |
             mkdir -p /data/restored
-            backup_path='{{ .ArtifactsIn.fdbBackup.KeyValue.backupPath }}'
-            backup_id='{{ .ArtifactsIn.fdbBackup.KeyValue.backupID }}'
-            kando location pull --profile '{{ toJson .Profile }}' --path ${backup_path} --backupID ${backup_id} - | tar zxvf - -C /data/restored
+            backup_file_path='backup.tgz'
+            kopia_snap='{{ .ArtifactsIn.fdbBackup.KopiaSnapshot }}'
+            kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - | tar zxvf - -C /data/restored
             /usr/bin/backup_agent -C /var/dynamic-conf/fdb.cluster  &
             fdbcli --exec "writemode on; clearrange '' \xFF"
             backupname=$(ls /data/restored/fdbbackup/)
@@ -68,13 +70,15 @@ actions:
   delete:
     type: Namespace
     inputArtifactNames:
+    # The kopia snapshot info created in backup phase can be used here
+    # Use the `--kopia-snapshot` flag in kando to pass in `fdbBackup.KopiaSnapshot`
     - fdbBackup
     phases:
     - func: KubeTask
       name: deleteBackup
       args:
         namespace: "{{ .Namespace.Name }}"
-        image: ghcr.io/kanisterio/kanister-tools:0.53.0
+        image: ghcr.io/kanisterio/kanister-tools:0.57.0
         command:
           - bash
           - -o
@@ -83,6 +87,6 @@ actions:
           - errexit
           - -c
           - |
-            backup_path='{{ .ArtifactsIn.fdbBackup.KeyValue.backupPath }}'
-            backup_id='{{ .ArtifactsIn.fdbBackup.KeyValue.backupID }}'
-            kando location delete --profile '{{ toJson .Profile }}' --path ${backup_path} --backupID ${backup_id}
+            backup_file_path='backup.tgz'
+            kopia_snap='{{ .ArtifactsIn.fdbBackup.KopiaSnapshot }}'
+            kando location delete --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}"
diff --git a/examples/stable/maria/blueprint-v2/maria-blueprint.yaml b/examples/stable/maria/blueprint-v2/maria-blueprint.yaml
@@ -7,9 +7,10 @@ actions:
     type: StatefulSet
     outputArtifacts:
       mariaBackup:
-        keyValue:
-          backupPath: "{{ .Phases.dumpToStore.Output.backupPath }}"
-          backupID: "{{ .Phases.dumpToStore.Output.backupID }}"
+        # Capture the kopia snapshot information for subsequent actions
+        # The information includes the kopia snapshot ID which is essential for restore and delete to succeed
+        # `kopiaOutput` is the name provided to kando using `--output-name` flag
+        kopiaSnapshot: "{{ .Phases.dumpToStore.Output.kopiaOutput }}"
     phases:
     - func: KubeTask
       name: dumpToStore
@@ -19,7 +20,7 @@ actions:
           name: '{{ .StatefulSet.Name }}'
           namespace: '{{ .StatefulSet.Namespace }}'
       args:
-        image: ghcr.io/kanisterio/mysql-sidecar:0.53.0
+        image: ghcr.io/kanisterio/mysql-sidecar:0.57.0
         namespace: "{{ .StatefulSet.Namespace }}"
         command:
         - bash
@@ -29,19 +30,21 @@ actions:
         - pipefail
         - -c
         - |
-          backup_path="/maria-backups/{{ .StatefulSet.Namespace }}/{{ .StatefulSet.Name }}/{{ toDate "2006-01-02T15:04:05.999999999Z07:00" .Time  | date "2006-01-02T15-04-05" }}/dump.sql.gz"
+          backup_file_path="dump.sql.gz"
           root_password="{{ index .Phases.dumpToStore.Secrets.mariaSecret.Data "mariadb-root-password" | toString }}"
           exclude_db="('information_schema', 'mysql', 'performance_schema')"
           dump_databases=""
           for db in `mysql -u root  -h {{ .StatefulSet.Name }} --password=${root_password} -ANe "use information_schema; select schema_name from schemata where schema_name not in ${exclude_db}"`
           do
                   dump_databases="${dump_databases} ${db}"
           done
-          mysqldump -u root --password=${root_password} -h {{ .StatefulSet.Name }} --column-statistics=0 --single-transaction --databases ${dump_databases} | gzip - | kando location push --profile '{{ toJson .Profile }}' --path ${backup_path} -
-          kando output backupPath ${backup_path}
+          dump_cmd="mysqldump -u root --password=${root_password} -h {{ .StatefulSet.Name }} --column-statistics=0 --single-transaction --databases ${dump_databases}"
+          ${dump_cmd} | gzip - | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" -
   restore:
     type: StatefulSet
     inputArtifactNames:
+    # The kopia snapshot info created in backup phase can be used here
+    # Use the `--kopia-snapshot` flag in kando to pass in `mariaBackup.KopiaSnapshot`
     - mariaBackup
     phases:
     - func: KubeTask
@@ -52,7 +55,7 @@ actions:
           name: '{{ .StatefulSet.Name }}'
           namespace: '{{ .StatefulSet.Namespace }}'
       args:
-        image: ghcr.io/kanisterio/mysql-sidecar:0.53.0
+        image: ghcr.io/kanisterio/mysql-sidecar:0.57.0
         namespace: "{{ .StatefulSet.Namespace }}"
         command:
         - bash
@@ -62,19 +65,21 @@ actions:
         - pipefail
         - -c
         - |
-          backup_path="{{ .ArtifactsIn.mariaBackup.KeyValue.backupPath }}"
-          backup_id="{{ .ArtifactsIn.mariaBackup.KeyValue.backupID }}"
+          backup_file_path="dump.sql.gz"
+          kopia_snap='{{ .ArtifactsIn.mariaBackup.KopiaSnapshot }}'
           root_password="{{ index .Phases.restoreFromStore.Secrets.mariaSecret.Data "mariadb-root-password" | toString }}"
-          kando location pull --profile '{{ toJson .Profile }}' --path ${backup_path} --backupID ${backup_id} - | gunzip | mysql -u root --password=${root_password} -h {{ .StatefulSet.Name }}
+          kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - | gunzip | mysql -u root --password=${root_password} -h {{ .StatefulSet.Name }}
   delete:
     type: Namespace
     inputArtifactNames:
+    # The kopia snapshot info created in backup phase can be used here
+    # Use the `--kopia-snapshot` flag in kando to pass in `mariaBackup.KopiaSnapshot`
     - mariaBackup
     phases:
     - func: KubeTask
       name: deleteFromStore
       args:
-        image: ghcr.io/kanisterio/mysql-sidecar:0.53.0
+        image: ghcr.io/kanisterio/mysql-sidecar:0.57.0
         namespace: "{{ .Namespace.Name }}"
         command:
         - bash
@@ -84,6 +89,6 @@ actions:
         - pipefail
         - -c
         - |
-          backup_path="{{ .ArtifactsIn.mariaBackup.KeyValue.backupPath }}"
-          backup_id="{{ .ArtifactsIn.mariaBackup.KeyValue.backupID }}"
-          kando location delete --profile '{{ toJson .Profile }}' --path ${backup_path} --backupID ${backup_id}
+          backup_file_path="dump.sql.gz"
+          kopia_snap='{{ .ArtifactsIn.mariaBackup.KopiaSnapshot }}'
+          kando location delete --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}"
diff --git a/examples/stable/mongodb-deploymentconfig/blueprint-v2/mongo-dep-config-blueprint.yaml b/examples/stable/mongodb-deploymentconfig/blueprint-v2/mongo-dep-config-blueprint.yaml
@@ -7,9 +7,10 @@ actions:
     type: DeploymentConfig
     outputArtifacts:
       mongoBackup:
-        keyValue:
-          backupPath: "{{ .Phases.mongoDump.Output.backupPath }}"
-          backupID: "{{ .Phases.mongoDump.Output.backupID }}"
+        # Capture the kopia snapshot information for subsequent actions
+        # The information includes the kopia snapshot ID which is essential for restore and delete to succeed
+        # `kopiaOutput` is the name provided to kando using `--output-name` flag
+        kopiaSnapshot: "{{ .Phases.mongoDump.Output.kopiaOutput }}"
     phases:
     - func: KubeTask
       name: mongoDump
@@ -20,7 +21,7 @@ actions:
           namespace: "{{ .DeploymentConfig.Namespace }}"
       args:
         namespace: "{{ .DeploymentConfig.Namespace }}"
-        image: ghcr.io/kanisterio/mongodb:0.51.0
+        image: ghcr.io/kanisterio/mongodb:0.57.0
         command:
           - bash
           - -o
@@ -33,12 +34,13 @@ actions:
             dbPassword='{{ index .Phases.mongoDump.Secrets.mongosecret.Data "database-admin-password" | toString }}'
             dump_cmd="mongodump --gzip --archive --host ${host} -u admin -p ${dbPassword}"
             echo $dump_cmd
-            backup_path='/mongodb-replicaset-backups/{{ .DeploymentConfig.Name }}/{{ toDate "2006-01-02T15:04:05.999999999Z07:00" .Time  | date "2006-01-02T15-04-05" }}/rs_backup.gz'
-            ${dump_cmd} | kando location push --profile '{{ toJson .Profile }}' --path ${backup_path} -
-            kando output backupPath ${backup_path}
+            backup_file_path='rs_backup.gz'
+            ${dump_cmd} | kando location push --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --output-name "kopiaOutput" -
   restore:
     type: DeploymentConfig
     inputArtifactNames:
+    # The kopia snapshot info created in backup phase can be used here
+    # Use the `--kopia-snapshot` flag in kando to pass in `mongoBackup.KopiaSnapshot`
       - mongoBackup
     phases:
     - func: KubeTask
@@ -50,7 +52,7 @@ actions:
           namespace: "{{ .DeploymentConfig.Namespace }}"
       args:
         namespace: "{{ .DeploymentConfig.Namespace }}"
-        image: ghcr.io/kanisterio/mongodb:0.51.0
+        image: ghcr.io/kanisterio/mongodb:0.57.0
         command:
           - bash
           - -o
@@ -62,19 +64,21 @@ actions:
             host="{{ .DeploymentConfig.Name }}.{{ .DeploymentConfig.Namespace }}.svc.cluster.local"
             dbPassword='{{ index .Phases.pullFromStore.Secrets.mongosecret.Data "database-admin-password" | toString }}'
             restore_cmd="mongorestore --gzip --archive --drop --host ${host} -u admin -p ${dbPassword}"
-            backup_path='{{ .ArtifactsIn.mongoBackup.KeyValue.backupPath }}'
-            backup_id='{{ .ArtifactsIn.mongoBackup.KeyValue.backupID }}'
-            kando location pull --profile '{{ toJson .Profile }}' --path ${backup_path} --backupID ${backup_id} - | ${restore_cmd}
+            backup_file_path='rs_backup.gz'
+            kopia_snap='{{ .ArtifactsIn.mongoBackup.KopiaSnapshot }}'
+            kando location pull --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}" - | ${restore_cmd}
   delete:
     type: Namespace
     inputArtifactNames:
+    # The kopia snapshot info created in backup phase can be used here
+    # Use the `--kopia-snapshot` flag in kando to pass in `mongoBackup.KopiaSnapshot`
       - mongoBackup
     phases:
     - func: KubeTask
       name: deleteFromStore
       args:
         namespace: "{{ .Namespace.Name }}"
-        image: ghcr.io/kanisterio/mongodb:0.51.0
+        image: ghcr.io/kanisterio/mongodb:0.57.0
         command:
           - bash
           - -o
@@ -83,5 +87,6 @@ actions:
           - pipefail
           - -c
           - |
-            backup_path="{{ .ArtifactsIn.mongoBackup.KeyValue.path }}"
-            kando location delete --profile '{{ toJson .Profile }}' --path ${backup_path} --backupID ${backup_id}
+            backup_file_path='rs_backup.gz'
+            kopia_snap='{{ .ArtifactsIn.mongoBackup.KopiaSnapshot }}'
+            kando location delete --profile '{{ toJson .Profile }}' --path "${backup_file_path}" --kopia-snapshot "${kopia_snap}"