Refactor Backup/Restore/CopyData (#5232)

1. Clean up the functions
2. Generate `backupTag` inside the `BackupData` function
3. Use `backupTag` as the identifier during restore.

docs/functions.rst

@@ -269,6 +269,8 @@ This function backs up data from a container into an S3 compatible object store.
    sidecar container. This sidecar is necessary to run the
    tools that capture path on a volume and store it on the object store.
 
+Arguments:
+
 .. csv-table::
    :header: "Argument", "Required", "Type", "Description"
    :align: left
@@ -279,22 +281,37 @@ This function backs up data from a container into an S3 compatible object store.
    `container`, Yes, `string`, container in which to execute
    `includePath`, Yes, `string`, path of the data to be backed up
    `backupArtifactPrefix`, Yes, `string`, path to store the backup on the object store
-   `backupIdentifier`, Yes, `string`, unique string to identify the backup
+
+Outputs:
+
+.. csv-table::
+   :header: "Output", "Type", "Description"
+   :align: left
+   :widths: 5,5,15
+
+   `backupTag`,`string`, unique tag added to the backup
 
 Example:
 
 .. code-block:: yaml
   :linenos:
 
-  - func: BackupData
-    name: BackupToObjectStore
-    args:
-      namespace: "{{ .Deployment.Namespace }}"
-      pod: "{{ index .Deployment.Pods 0 }}"
-      container: kanister-tools
-      includePath: /mnt/data
-      backupArtifactPrefix: s3-bucket/path/artifactPrefix
-      backupIdentifier: "{{ .Time }}"
+  actions:
+    backup:
+      type: Deployment
+      outputArtifacts:
+        backupInfo:
+          keyValue:
+            backupIdentifier: "{{ .Phases.BackupToObjectStore.Output.backupTag }}"
+      phases:
+        - func: BackupData
+          name: BackupToObjectStore
+          args:
+            namespace: "{{ .Deployment.Namespace }}"
+            pod: "{{ index .Deployment.Pods 0 }}"
+            container: kanister-tools
+            includePath: /mnt/data
+            backupArtifactPrefix: s3-bucket/path/artifactPrefix
 
 .. _restoredata:
 
@@ -322,7 +339,7 @@ and restores data to the specified path.
    `namespace`, Yes, `string`, namespace in which to execute
    `image`, Yes, `string`, image to be used for running restore
    `backupArtifactPrefix`, Yes, `string`, path to the backup on the object store
-   `backupIdentifier`, Yes, `string`, unique string to identify the backup
+   `backupTag`, Yes, `string`, unique tag added during the backup
    `restorePath`, No, `string`, path where data is restored
    `pod`, No, `string`, pod to which the volumes are attached
    `volumes`, No, `map[string]string`, Mapping of `pvcName` to `mountPath` under which the volume will be available
@@ -336,6 +353,11 @@ and restores data to the specified path.
 
 Example:
 
+Consider a scenario where you wish to restore the data backed up by the
+:ref:`backupdata` function. We will first scale down the application,
+restore the data and then scale it back up.
+For this phase, we will use the `backupInfo` Artifact provided by backup function.
+
 .. code-block:: yaml
   :linenos:
 
@@ -344,7 +366,7 @@ Example:
     args:
       namespace: "{{ .Deployment.Namespace }}"
       name: "{{ .Deployment.Name }}"
-      kind: deployment
+      kind: Deployment
       replicas: 0
   - func: RestoreData
     name: RestoreFromObjectStore
@@ -353,7 +375,14 @@ Example:
       pod: "{{ index .Deployment.Pods 0 }}"
       image: kanisterio/kanister-tools:0.18.0
       backupArtifactPrefix: s3-bucket/path/artifactPrefix
-      backupIdentifier: "{{ .Time }}"
+      backupTag: "{{ .ArtifactsIn.backupInfo.KeyValue.backupIdentifier }}"
+  - func: ScaleWorkload
+    name: StartupApplication
+    args:
+      namespace: "{{ .Deployment.Namespace }}"
+      name: "{{ .Deployment.Name }}"
+      kind: Deployment
+      replicas: 1
 
 CopyVolumeData
 --------------
@@ -388,7 +417,7 @@ Outputs:
    :widths: 5,5,15
 
    `backupArtifactLocation`,`string`, location in objectstore where data was copied
-   `backupID`,`string`,  unique string to identify this data copy
+   `backupTag`,`string`,  unique string to identify this data copy
 
 Example:
 

pkg/function/args.go

@@ -26,7 +26,7 @@ func OptArg(args map[string]interface{}, argName string, result interface{}, def
 	return mapstructure.Decode(defaultValue, result)
 }
 
-// ArgPresent checks if the argument exists
+// ArgExists checks if the argument exists
 func ArgExists(args map[string]interface{}, argName string) bool {
 	_, ok := args[argName]
 	return ok

pkg/function/backup_data.go

@@ -5,6 +5,7 @@ import (
 	"regexp"
 
 	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/util/rand"
 
 	kanister "github.com/kanisterio/kanister/pkg"
 	crv1alpha1 "github.com/kanisterio/kanister/pkg/apis/cr/v1alpha1"
@@ -25,10 +26,12 @@ const (
 	BackupDataIncludePathArg = "includePath"
 	// BackupDataBackupArtifactPrefixArg provides the path to store artifacts on the object store
 	BackupDataBackupArtifactPrefixArg = "backupArtifactPrefix"
-	// BackupDataBackupIdentifierArg provides a unique ID added to the artifacts
-	BackupDataBackupIdentifierArg = "backupIdentifier"
 	// BackupDataEncryptionKeyArg provides the encryption key to be used for backups
 	BackupDataEncryptionKeyArg = "encryptionKey"
+	// BackupDataOutputBackupID is the key used for returning backup ID output
+	BackupDataOutputBackupID = "backupID"
+	// BackupDataOutputBackupTag is the key used for returning backupTag output
+	BackupDataOutputBackupTag = "backupTag"
 )
 
 func init() {
@@ -70,7 +73,7 @@ func getSnapshotIDFromLog(output string) string {
 }
 
 func (*backupDataFunc) Exec(ctx context.Context, tp param.TemplateParams, args map[string]interface{}) (map[string]interface{}, error) {
-	var namespace, pod, container, includePath, backupArtifactPrefix, backupIdentifier, encryptionKey string
+	var namespace, pod, container, includePath, backupArtifactPrefix, encryptionKey string
 	var err error
 	if err = Arg(args, BackupDataNamespaceArg, &namespace); err != nil {
 		return nil, err
@@ -87,9 +90,6 @@ func (*backupDataFunc) Exec(ctx context.Context, tp param.TemplateParams, args m
 	if err = Arg(args, BackupDataBackupArtifactPrefixArg, &backupArtifactPrefix); err != nil {
 		return nil, err
 	}
-	if err = Arg(args, BackupDataBackupIdentifierArg, &backupIdentifier); err != nil {
-		return nil, err
-	}
 	if err = OptArg(args, BackupDataEncryptionKeyArg, &encryptionKey, restic.GeneratePassword()); err != nil {
 		return nil, err
 	}
@@ -107,21 +107,27 @@ func (*backupDataFunc) Exec(ctx context.Context, tp param.TemplateParams, args m
 	}
 
 	// Create backup and dump it on the object store
-	cmd := restic.BackupCommand(tp.Profile, backupArtifactPrefix, backupIdentifier, includePath, encryptionKey)
+	backupTag := rand.String(10)
+	cmd := restic.BackupCommandByTag(tp.Profile, backupArtifactPrefix, backupTag, includePath, encryptionKey)
 	stdout, stderr, err := kube.Exec(cli, namespace, pod, container, cmd)
 	format.Log(pod, container, stdout)
 	format.Log(pod, container, stderr)
 	if err != nil {
 		return nil, errors.Wrapf(err, "Failed to create and upload backup")
 	}
 	// Get the snapshot ID from log
-	if snapID := getSnapshotIDFromLog(stdout); snapID != "" {
-		return map[string]interface{}{"snapshotID": snapID}, nil
+	backupID := getSnapshotIDFromLog(stdout)
+	if backupID == "" {
+		return nil, errors.New("Failed to parse the backup ID from logs")
+	}
+	output := map[string]interface{}{
+		BackupDataOutputBackupID:  backupID,
+		BackupDataOutputBackupTag: backupTag,
 	}
-	return nil, nil
+	return output, nil
 }
 
 func (*backupDataFunc) RequiredArgs() []string {
 	return []string{BackupDataNamespaceArg, BackupDataPodArg, BackupDataContainerArg,
-		BackupDataIncludePathArg, BackupDataBackupArtifactPrefixArg, BackupDataBackupIdentifierArg}
+		BackupDataIncludePathArg, BackupDataBackupArtifactPrefixArg}
 }

pkg/function/copy_volume_data.go

@@ -27,6 +27,7 @@ const (
 	CopyVolumeDataOutputBackupRoot             = "backupRoot"
 	CopyVolumeDataOutputBackupArtifactLocation = "backupArtifactLocation"
 	CopyVolumeDataEncryptionKeyArg             = "encryptionKey"
+	CopyVolumeDataOutputBackupTag              = "backupTag"
 )
 
 func init() {
@@ -71,18 +72,24 @@ func copyVolumeData(ctx context.Context, cli kubernetes.Interface, tp param.Temp
 	}
 
 	// Copy data to object store
-	backupIdentifier := rand.String(10)
-	cmd := restic.BackupCommand(tp.Profile, targetPath, backupIdentifier, mountPoint, encryptionKey)
+	backupTag := rand.String(10)
+	cmd := restic.BackupCommandByTag(tp.Profile, targetPath, backupTag, mountPoint, encryptionKey)
 	stdout, stderr, err := kube.Exec(cli, namespace, pod.Name, pod.Spec.Containers[0].Name, cmd)
 	format.Log(pod.Name, pod.Spec.Containers[0].Name, stdout)
 	format.Log(pod.Name, pod.Spec.Containers[0].Name, stderr)
 	if err != nil {
 		return nil, errors.Wrapf(err, "Failed to create and upload backup")
 	}
+	// Get the snapshot ID from log
+	backupID := getSnapshotIDFromLog(stdout)
+	if backupID == "" {
+		return nil, errors.New("Failed to parse the backup ID from logs")
+	}
 	return map[string]interface{}{
-			CopyVolumeDataOutputBackupID:               backupIdentifier,
+			CopyVolumeDataOutputBackupID:               backupID,
 			CopyVolumeDataOutputBackupRoot:             mountPoint,
 			CopyVolumeDataOutputBackupArtifactLocation: targetPath,
+			CopyVolumeDataOutputBackupTag:              backupTag,
 		},
 		nil
 }

pkg/function/data_test.go

@@ -64,12 +64,10 @@ func (s *DataSuite) TearDownSuite(c *C) {
 	}
 }
 
-const actionName = "backupAndRestore"
-
 func newRestoreDataBlueprint(pvc string) *crv1alpha1.Blueprint {
 	return &crv1alpha1.Blueprint{
 		Actions: map[string]*crv1alpha1.BlueprintAction{
-			actionName: &crv1alpha1.BlueprintAction{
+			"restore": &crv1alpha1.BlueprintAction{
 				Kind: param.StatefulSetKind,
 				SecretNames: []string{
 					"backupKey",
@@ -83,7 +81,7 @@ func newRestoreDataBlueprint(pvc string) *crv1alpha1.Blueprint {
 							RestoreDataImageArg:                "kanisterio/kanister-tools:0.18.0",
 							RestoreDataBackupArtifactPrefixArg: "{{ .Profile.Location.Bucket }}/{{ .Profile.Location.Prefix }}",
 							RestoreDataRestorePathArg:          "/mnt/data",
-							RestoreDataBackupIdentifierArg:     "{{ .Time }}",
+							RestoreDataBackupTagArg:            fmt.Sprintf("{{ .Options.%s }}", BackupDataOutputBackupTag),
 							RestoreDataEncryptionKeyArg:        "{{ .Secrets.backupKey.Data.password | toString }}",
 							RestoreDataVolsArg: map[string]string{
 								pvc: "/mnt/data",
@@ -99,7 +97,7 @@ func newRestoreDataBlueprint(pvc string) *crv1alpha1.Blueprint {
 func newBackupDataBlueprint() *crv1alpha1.Blueprint {
 	return &crv1alpha1.Blueprint{
 		Actions: map[string]*crv1alpha1.BlueprintAction{
-			actionName: &crv1alpha1.BlueprintAction{
+			"backup": &crv1alpha1.BlueprintAction{
 				Kind: param.StatefulSetKind,
 				Phases: []crv1alpha1.BlueprintPhase{
 					crv1alpha1.BlueprintPhase{
@@ -111,7 +109,6 @@ func newBackupDataBlueprint() *crv1alpha1.Blueprint {
 							BackupDataContainerArg:            "{{ index .StatefulSet.Containers 0 0 }}",
 							BackupDataIncludePathArg:          "/etc",
 							BackupDataBackupArtifactPrefixArg: "{{ .Profile.Location.Bucket }}/{{ .Profile.Location.Prefix }}",
-							BackupDataBackupIdentifierArg:     "{{ .Time }}",
 							BackupDataEncryptionKeyArg:        "{{ .Secrets.backupKey.Data.password | toString }}",
 						},
 					},
@@ -173,20 +170,21 @@ func (s *DataSuite) TestBackupRestoreData(c *C) {
 	}
 	tp.Profile = testutil.ObjectStoreProfileOrSkip(c, objectstore.ProviderTypeS3, location)
 
-	for _, bp := range []crv1alpha1.Blueprint{
-		*newBackupDataBlueprint(),
-		*newRestoreDataBlueprint(pvc.GetName()),
-	} {
-		phases, err := kanister.GetPhases(bp, actionName, *tp)
-		c.Assert(err, IsNil)
-		for _, p := range phases {
-			out, err := p.Exec(context.Background(), bp, actionName, *tp)
-			if out != nil {
-				c.Assert(out["snapshotID"], NotNil)
-			}
-			c.Assert(err, IsNil)
-		}
+	// Test backup
+	bp := *newBackupDataBlueprint()
+	out := runAction(c, bp, "backup", tp)
+	c.Assert(out[BackupDataOutputBackupID].(string), Not(Equals), "")
+	c.Assert(out[BackupDataOutputBackupTag].(string), Not(Equals), "")
+
+	options := map[string]string{
+		BackupDataOutputBackupID:  out[BackupDataOutputBackupID].(string),
+		BackupDataOutputBackupTag: out[BackupDataOutputBackupTag].(string),
 	}
+	tp.Options = options
+
+	// Test restore
+	bp = *newRestoreDataBlueprint(pvc.GetName())
+	_ = runAction(c, bp, "restore", tp)
 }
 
 func newCopyDataTestBlueprint() crv1alpha1.Blueprint {
@@ -231,7 +229,7 @@ func newCopyDataTestBlueprint() crv1alpha1.Blueprint {
 							RestoreDataNamespaceArg:            "{{ .PVC.Namespace }}",
 							RestoreDataImageArg:                "kanisterio/kanister-tools:0.18.0",
 							RestoreDataBackupArtifactPrefixArg: fmt.Sprintf("{{ .Options.%s }}", CopyVolumeDataOutputBackupArtifactLocation),
-							RestoreDataBackupIdentifierArg:     fmt.Sprintf("{{ .Options.%s }}", CopyVolumeDataOutputBackupID),
+							RestoreDataBackupTagArg:            fmt.Sprintf("{{ .Options.%s }}", CopyVolumeDataOutputBackupTag),
 							RestoreDataVolsArg: map[string]string{
 								"{{ .PVC.Name }}": fmt.Sprintf("{{ .Options.%s }}", CopyVolumeDataOutputBackupRoot),
 							},
@@ -275,10 +273,12 @@ func (s *DataSuite) TestCopyData(c *C) {
 	c.Assert(out[CopyVolumeDataOutputBackupID].(string), Not(Equals), "")
 	c.Assert(out[CopyVolumeDataOutputBackupRoot].(string), Not(Equals), "")
 	c.Assert(out[CopyVolumeDataOutputBackupArtifactLocation].(string), Not(Equals), "")
+	c.Assert(out[CopyVolumeDataOutputBackupTag].(string), Not(Equals), "")
 	options := map[string]string{
 		CopyVolumeDataOutputBackupID:               out[CopyVolumeDataOutputBackupID].(string),
 		CopyVolumeDataOutputBackupRoot:             out[CopyVolumeDataOutputBackupRoot].(string),
 		CopyVolumeDataOutputBackupArtifactLocation: out[CopyVolumeDataOutputBackupArtifactLocation].(string),
+		CopyVolumeDataOutputBackupTag:              out[CopyVolumeDataOutputBackupTag].(string),
 	}
 
 	// Create a new PVC