Update location commands to produce kopia snapshot output (#1002)

* Update location push to output kopia snap json

* Update pull and delete cmds to accept kopia snap

* Address review comments

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

pkg/kando/location_delete.go

@@ -35,7 +35,7 @@ func newLocationDeleteCommand() *cobra.Command {
 			return runLocationDelete(c)
 		},
 	}
-	cmd.Flags().StringP(backupIDFlagName, "b", "", "Pass the backup ID from the location push command (optional)")
+	cmd.Flags().StringP(kopiaSnapshotFlagName, "k", "", "Pass the kopia snapshot information from the location push command (optional)")
 	return cmd
 }
 
@@ -46,16 +46,20 @@ func runLocationDelete(cmd *cobra.Command) error {
 	}
 	cmd.SilenceUsage = true
 	s := pathFlag(cmd)
-	id := backupIDFlag(cmd)
 	ctx := context.Background()
 	if p.Location.Type == crv1alpha1.LocationTypeKopia {
-		if id == "" {
-			return errors.New("Backup ID is required to delete data using kopia")
+		snapJSON := kopiaSnapshotFlag(cmd)
+		if snapJSON == "" {
+			return errors.New("kopia snapshot information is required to delete data using kopia")
+		}
+		kopiaSnap, err := kopia.UnmarshalKopiaSnapshot(snapJSON)
+		if err != nil {
+			return err
 		}
 		if err = connectToKopiaServer(ctx, p); err != nil {
 			return err
 		}
-		return kopiaLocationDelete(ctx, id, s)
+		return kopiaLocationDelete(ctx, kopiaSnap.ID, s)
 	}
 	return locationDelete(ctx, p, s)
 }

pkg/kando/location_pull.go

@@ -29,7 +29,7 @@ import (
 )
 
 const (
-	backupIDFlagName = "backupID"
+	kopiaSnapshotFlagName = "kopia-snaphot"
 )
 
 func newLocationPullCommand() *cobra.Command {
@@ -42,12 +42,12 @@ func newLocationPullCommand() *cobra.Command {
 			return runLocationPull(c, args)
 		},
 	}
-	cmd.Flags().StringP(backupIDFlagName, "b", "", "Pass the backup ID from the location push command (optional)")
+	cmd.Flags().StringP(kopiaSnapshotFlagName, "k", "", "Pass the kopia snapshot information from the location push command (optional)")
 	return cmd
 }
 
-func backupIDFlag(cmd *cobra.Command) string {
-	return cmd.Flag(backupIDFlagName).Value.String()
+func kopiaSnapshotFlag(cmd *cobra.Command) string {
+	return cmd.Flag(kopiaSnapshotFlagName).Value.String()
 }
 
 func runLocationPull(cmd *cobra.Command, args []string) error {
@@ -60,16 +60,20 @@ func runLocationPull(cmd *cobra.Command, args []string) error {
 		return err
 	}
 	s := pathFlag(cmd)
-	id := backupIDFlag(cmd)
 	ctx := context.Background()
 	if p.Location.Type == crv1alpha1.LocationTypeKopia {
-		if id == "" {
-			return errors.New("Backup ID is required to pull data using kopia")
+		snapJSON := kopiaSnapshotFlag(cmd)
+		if snapJSON == "" {
+			return errors.New("kopia snapshot information is required to pull data using kopia")
+		}
+		kopiaSnap, err := kopia.UnmarshalKopiaSnapshot(snapJSON)
+		if err != nil {
+			return err
 		}
 		if err = connectToKopiaServer(ctx, p); err != nil {
 			return err
 		}
-		return kopiaLocationPull(ctx, id, s, target)
+		return kopiaLocationPull(ctx, kopiaSnap.ID, s, target)
 	}
 	return locationPull(ctx, p, s, target)
 }
@@ -94,5 +98,14 @@ func kopiaLocationPull(ctx context.Context, backupID, path string, target io.Wri
 func connectToKopiaServer(ctx context.Context, kp *param.Profile) error {
 	contentCacheSize := kopia.GetDataStoreGeneralContentCacheSize(kp.Credential.KopiaServerSecret.ConnectOptions)
 	metadataCacheSize := kopia.GetDataStoreGeneralMetadataCacheSize(kp.Credential.KopiaServerSecret.ConnectOptions)
-	return kopia.ConnectToAPIServer(ctx, kp.Credential.KopiaServerSecret.Cert, kp.Credential.KopiaServerSecret.Password, kp.Credential.KopiaServerSecret.Hostname, kp.Location.Endpoint, kp.Credential.KopiaServerSecret.Username, contentCacheSize, metadataCacheSize)
+	return kopia.ConnectToAPIServer(
+		ctx,
+		kp.Credential.KopiaServerSecret.Cert,
+		kp.Credential.KopiaServerSecret.Password,
+		kp.Credential.KopiaServerSecret.Hostname,
+		kp.Location.Endpoint,
+		kp.Credential.KopiaServerSecret.Username,
+		contentCacheSize,
+		metadataCacheSize,
+	)
 }

pkg/kando/location_push.go

@@ -29,6 +29,11 @@ import (
 	"github.com/kanisterio/kanister/pkg/param"
 )
 
+const (
+	outputNameFlagName    = "output-name"
+	defaultKandoOutputKey = "kandoOutput"
+)
+
 func newLocationPushCommand() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "push <source>",
@@ -39,9 +44,14 @@ func newLocationPushCommand() *cobra.Command {
 			return runLocationPush(c, args)
 		},
 	}
+	cmd.Flags().StringP(outputNameFlagName, "o", defaultKandoOutputKey, "Specify a name to be used for the output produced by kando. Set to `kandoOutput` by default")
 	return cmd
 }
 
+func outputNameFlag(cmd *cobra.Command) string {
+	return cmd.Flag(outputNameFlagName).Value.String()
+}
+
 func runLocationPush(cmd *cobra.Command, args []string) error {
 	source, err := sourceReader(args[0])
 	if err != nil {
@@ -54,10 +64,11 @@ func runLocationPush(cmd *cobra.Command, args []string) error {
 	s := pathFlag(cmd)
 	ctx := context.Background()
 	if p.Location.Type == crv1alpha1.LocationTypeKopia {
+		outputName := outputNameFlag(cmd)
 		if err = connectToKopiaServer(ctx, p); err != nil {
 			return err
 		}
-		return kopiaLocationPush(ctx, s, source)
+		return kopiaLocationPush(ctx, s, outputName, source)
 	}
 	return locationPush(ctx, p, s, source)
 }
@@ -83,10 +94,14 @@ func locationPush(ctx context.Context, p *param.Profile, path string, source io.
 }
 
 // kopiaLocationPush pushes the data from the source using a kopia snapshot
-func kopiaLocationPush(ctx context.Context, path string, source io.Reader) error {
-	snapID, _, err := kopia.Write(ctx, path, source)
+func kopiaLocationPush(ctx context.Context, path, outputName string, source io.Reader) error {
+	snapInfo, err := kopia.Write(ctx, path, source)
 	if err != nil {
 		return errors.Wrap(err, "Failed to push data using kopia")
 	}
-	return output.PrintOutput(kopia.BackupIdentifierKey, snapID)
+	snapInfoJSON, err := kopia.MarshalKopiaSnapshot(snapInfo)
+	if err != nil {
+		return err
+	}
+	return output.PrintOutput(outputName, snapInfoJSON)
 }

pkg/kopia/stream.go

@@ -36,18 +36,28 @@ const (
 	pullRepoPurpose = "kando location pull"
 )
 
+// SnapshotInfo tracks kopia snapshot information produced by a kando command in a phase
+type SnapshotInfo struct {
+	// ID is the snapshot ID produced by kopia snapshot operation
+	ID string `json:"id"`
+	// LogicalSize is the sum of cached and hashed file size in bytes
+	LogicalSize int64 `json:"logicalSize"`
+	// LogicalSize is the uploaded size in bytes
+	PhysicalSize int64 `json:"physicalSize"`
+}
+
 // Write creates a kopia snapshot from the given reader
 // A virtual directory tree rooted at filepath.Dir(path) is created with
 // a kopia streaming file with filepath.Base(path) as name
-func Write(ctx context.Context, path string, source io.Reader) (string, string, error) {
+func Write(ctx context.Context, path string, source io.Reader) (*SnapshotInfo, error) {
 	password, ok := repo.GetPersistedPassword(ctx, defaultConfigFilePath)
 	if !ok || password == "" {
-		return "", "", errors.New("Failed to retrieve kopia client passphrase")
+		return nil, errors.New("Failed to retrieve kopia client passphrase")
 	}
 
 	rep, err := OpenRepository(ctx, defaultConfigFilePath, password, pushRepoPurpose)
 	if err != nil {
-		return "", "", errors.Wrap(err, "Failed to open kopia repository")
+		return nil, errors.Wrap(err, "Failed to open kopia repository")
 	}
 
 	// Populate the source info with source path
@@ -65,7 +75,20 @@ func Write(ctx context.Context, path string, source io.Reader) (string, string,
 	u := snapshotfs.NewUploader(rep)
 
 	// Create a kopia snapshot
-	return SnapshotSource(ctx, rep, u, sourceInfo, rootDir, "Kanister Database Backup")
+	snapID, _, err := SnapshotSource(ctx, rep, u, sourceInfo, rootDir, "Kanister Database Backup")
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO@pavan: Add kopia snapshot size information
+	zeroSize := int64(0)
+	snapshotInfo := &SnapshotInfo{
+		ID:           snapID,
+		LogicalSize:  zeroSize,
+		PhysicalSize: zeroSize,
+	}
+
+	return snapshotInfo, nil
 }
 
 // Read reads a kopia snapshot with the given ID and copies it to the given target

pkg/kopia/utils.go

@@ -191,3 +191,20 @@ func GetDataStoreGeneralMetadataCacheSize(opt map[string]int) int {
 	}
 	return defaultDataStoreGeneralMetadataCacheSizeMB
 }
+
+// MarshalKopiaSnapshot encodes kopia SnapshotInfo struct into a string
+func MarshalKopiaSnapshot(snapInfo *SnapshotInfo) (string, error) {
+	snap, err := json.Marshal(snapInfo)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to marshal kopia snapshot information")
+	}
+
+	return string(snap), nil
+}
+
+// UnmarshalKopiaSnapshot decodes a kopia snapshot JSON string into SnapshotInfo struct
+func UnmarshalKopiaSnapshot(snapInfoJSON string) (SnapshotInfo, error) {
+	snap := SnapshotInfo{}
+	err := json.Unmarshal([]byte(snapInfoJSON), &snap)
+	return snap, errors.Wrap(err, "failed to unmarshal kopia snapshot information")
+}