Add kopia repository command wrappers (#1700)

* Add utility to read file and directory from a pod

* Add copyright to new files

* Address review comments - fix error message and add comments

* Add Environment to PodOptions

* Add SecretMounts to PodOptions

* Add tests for new fields added to PodOptions

* Add function comment

* Revert file reader and SecretMounts implementation

* Remove stale test

* Add kopia repository storage args generation utilities

* Address review comments, add copyright to new files

* Address review comments - Update tests to use flags

* Use map[string][]byte instead of map[string]string

* Add utilities to generate EnvVars for secret

* Add kopia repository command wrappers

* Update kopia package error handling

* Add kopia repository connect or create utilities

* Remove location parameter from struct

* Use updated location map type

* Add missing copyright info

* Add test for ResolveS3Endpoint

* Set session token to env var as value

* Remove unused field

* Update test

pkg/kopia/command/common.go

@@ -15,6 +15,8 @@
 package command
 
 import (
+	"strconv"
+
 	"github.com/kanisterio/kanister/pkg/field"
 	"github.com/kanisterio/kanister/pkg/log"
 	"github.com/kanisterio/kanister/pkg/logsafe"
@@ -67,3 +69,16 @@ func addTags(tags []string, args logsafe.Cmd) logsafe.Cmd {
 func ExecKopiaArgs(configFilePath string) []string {
 	return commonArgs(&CommandArgs{ConfigFilePath: configFilePath}, false).StringSliceCMD()
 }
+
+const (
+	cacheDirectoryFlag      = "--cache-directory"
+	contentCacheSizeMBFlag  = "--content-cache-size-mb"
+	metadataCacheSizeMBFlag = "--metadata-cache-size-mb"
+)
+
+func kopiaCacheArgs(args logsafe.Cmd, cacheDirectory string, contentCacheMB, metadataCacheMB int) logsafe.Cmd {
+	args = args.AppendLoggableKV(cacheDirectoryFlag, cacheDirectory)
+	args = args.AppendLoggableKV(contentCacheSizeMBFlag, strconv.Itoa(contentCacheMB))
+	args = args.AppendLoggableKV(metadataCacheSizeMBFlag, strconv.Itoa(metadataCacheMB))
+	return args
+}

pkg/kopia/command/const.go

@@ -74,8 +74,34 @@ const (
 	userPasswordFlag          = "--user-password"
 
 	// Repository specific
-	repositorySubCommand = "repository"
+	repositorySubCommand      = "repository"
+	connectSubCommand         = "connect"
+	noCheckForUpdatesFlag     = "--no-check-for-updates"
+	overrideHostnameFlag      = "--override-hostname"
+	overrideUsernameFlag      = "--override-username"
+	pointInTimeConnectionFlag = "--point-in-time"
+	urlFlag                   = "--url"
+)
+
+// List of possible modifications to a policy, expressed as the kopia flag that will modify it
+const (
+	// Retention
+	keepLatest  = "--keep-latest"
+	keepHourly  = "--keep-hourly"
+	keepDaily   = "--keep-daily"
+	keepWeekly  = "--keep-weekly"
+	keepMonthly = "--keep-monthly"
+	keepAnnual  = "--keep-annual"
+
+	// Compression
+	compressionAlgorithm = "--compression"
 
+	// Compression Algorithms recognized by Kopia
+	s2DefaultComprAlgo = "s2-default"
+)
+
+// Constants for kopia defaults
+const (
 	// DefaultCacheDirectory is the directory where kopia content cache is created
 	DefaultCacheDirectory = "/tmp/kopia-cache"
 

pkg/kopia/command/policy_set_global.go

@@ -14,6 +14,8 @@
 
 package command
 
+import "strconv"
+
 type PolicySetGlobalCommandArgs struct {
 	*CommandArgs
 	Modifications policyChanges
@@ -29,3 +31,21 @@ func PolicySetGlobal(cmdArgs PolicySetGlobalCommandArgs) []string {
 
 	return stringSliceCommand(args)
 }
+
+func GetPolicyModifications() map[string]string {
+	const maxInt32 = 1<<31 - 1
+
+	pc := map[string]string{
+		// Retention changes
+		keepLatest:  strconv.Itoa(maxInt32),
+		keepHourly:  strconv.Itoa(0),
+		keepDaily:   strconv.Itoa(0),
+		keepWeekly:  strconv.Itoa(0),
+		keepMonthly: strconv.Itoa(0),
+		keepAnnual:  strconv.Itoa(0),
+
+		// Compression changes
+		compressionAlgorithm: s2DefaultComprAlgo,
+	}
+	return pc
+}

pkg/kopia/command/repository.go

@@ -14,6 +14,127 @@
 
 package command
 
+import (
+	"time"
+
+	"github.com/go-openapi/strfmt"
+	"github.com/pkg/errors"
+
+	"github.com/kanisterio/kanister/pkg/kopia/command/storage"
+)
+
+// RepositoryCommandArgs contains fields that are needed for
+// creating or connecting to a Kopia repository
+type RepositoryCommandArgs struct {
+	*CommandArgs
+	CacheDirectory  string
+	Hostname        string
+	ContentCacheMB  int
+	MetadataCacheMB int
+	Username        string
+	RepoPathPrefix  string
+	// PITFlag is only effective if set while repository connect
+	PITFlag  strfmt.DateTime
+	Location map[string][]byte
+}
+
+// RepositoryConnectCommand returns the kopia command for connecting to an existing repo
+func RepositoryConnectCommand(cmdArgs RepositoryCommandArgs) ([]string, error) {
+	args := commonArgs(cmdArgs.CommandArgs, false)
+	args = args.AppendLoggable(repositorySubCommand, connectSubCommand, noCheckForUpdatesFlag)
+
+	args = kopiaCacheArgs(args, cmdArgs.CacheDirectory, cmdArgs.ContentCacheMB, cmdArgs.MetadataCacheMB)
+
+	if cmdArgs.Hostname != "" {
+		args = args.AppendLoggableKV(overrideHostnameFlag, cmdArgs.Hostname)
+	}
+
+	if cmdArgs.Username != "" {
+		args = args.AppendLoggableKV(overrideUsernameFlag, cmdArgs.Username)
+	}
+
+	bsArgs, err := storage.KopiaStorageArgs(&storage.StorageCommandParams{
+		Location:       cmdArgs.Location,
+		RepoPathPrefix: cmdArgs.RepoPathPrefix,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "Failed to generate storage args")
+	}
+
+	if !time.Time(cmdArgs.PITFlag).IsZero() {
+		bsArgs = bsArgs.AppendLoggableKV(pointInTimeConnectionFlag, cmdArgs.PITFlag.String())
+	}
+
+	return stringSliceCommand(args.Combine(bsArgs)), nil
+}
+
+// RepositoryCreateCommand returns the kopia command for creation of a repo
+func RepositoryCreateCommand(cmdArgs RepositoryCommandArgs) ([]string, error) {
+	args := commonArgs(cmdArgs.CommandArgs, false)
+	args = args.AppendLoggable(repositorySubCommand, createSubCommand, noCheckForUpdatesFlag)
+
+	args = kopiaCacheArgs(args, cmdArgs.CacheDirectory, cmdArgs.ContentCacheMB, cmdArgs.MetadataCacheMB)
+
+	if cmdArgs.Hostname != "" {
+		args = args.AppendLoggableKV(overrideHostnameFlag, cmdArgs.Hostname)
+	}
+
+	if cmdArgs.Username != "" {
+		args = args.AppendLoggableKV(overrideUsernameFlag, cmdArgs.Username)
+	}
+
+	bsArgs, err := storage.KopiaStorageArgs(&storage.StorageCommandParams{
+		Location:       cmdArgs.Location,
+		RepoPathPrefix: cmdArgs.RepoPathPrefix,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "Failed to generate storage args")
+	}
+
+	return stringSliceCommand(args.Combine(bsArgs)), nil
+}
+
+// RepositoryServerCommandArgs contains fields required for connecting
+// to Kopia Repository API server
+type RepositoryServerCommandArgs struct {
+	UserPassword    string
+	ConfigFilePath  string
+	LogDirectory    string
+	CacheDirectory  string
+	Hostname        string
+	ServerURL       string
+	Fingerprint     string
+	Username        string
+	ContentCacheMB  int
+	MetadataCacheMB int
+}
+
+// RepositoryConnectServerCommand returns the kopia command for connecting to a remote
+// repository on Kopia Repository API server
+func RepositoryConnectServerCommand(cmdArgs RepositoryServerCommandArgs) []string {
+	args := commonArgs(&CommandArgs{
+		RepoPassword:   cmdArgs.UserPassword,
+		ConfigFilePath: cmdArgs.ConfigFilePath,
+		LogDirectory:   cmdArgs.LogDirectory,
+	}, false)
+	args = args.AppendLoggable(repositorySubCommand, connectSubCommand, serverSubCommand, noCheckForUpdatesFlag, noGrpcFlag)
+
+	args = kopiaCacheArgs(args, cmdArgs.CacheDirectory, cmdArgs.ContentCacheMB, cmdArgs.MetadataCacheMB)
+
+	if cmdArgs.Hostname != "" {
+		args = args.AppendLoggableKV(overrideHostnameFlag, cmdArgs.Hostname)
+	}
+
+	if cmdArgs.Username != "" {
+		args = args.AppendLoggableKV(overrideUsernameFlag, cmdArgs.Username)
+	}
+	args = args.AppendLoggableKV(urlFlag, cmdArgs.ServerURL)
+
+	args = args.AppendRedactedKV(serverCertFingerprint, cmdArgs.Fingerprint)
+
+	return stringSliceCommand(args)
+}
+
 type RepositoryStatusCommandArgs struct {
 	*CommandArgs
 }