[Kopia wrappers - B] Add command wrappers in the Kopia package (#1526)

* Add kopia wrappers * Add logsafe module * Modify utils package * Add errors package * Add apis/config package * Refactor kopia wrapper module * Remove downstream user reference * Address reviewer comments * Separate constant files in Kopia package * Add command logsafe * Add utility functions * Created sub-packages for repository and snapshot wrapper methods * Add license info in constant files * Move helper functions out of the kopia.go file * Rename kopia.go to commands.go * Rename OpenRepository, DeleteSnapshot and reportSnapshotStatus methods * Convert constants and functions from local to global * Fix constant in pkg/kopia/snapshot/stream.go * Fix const description in pkg/kopia/utils.go * Added some more utils functions * Update maintenance info command and add maintenance utils * Add Client keyword in constants and rename kankopia import alias in pkg/stream/stream.go * Rename kopiaArgs to commonArgs * Resolve cyclic import statements * Rename cmd package to command * Resolve cyclic commit * Refactor separate log_output_utils.go * Fix command utility import * Remove nested loop * Change log level to Debug * [Kopia wrappers - C] Separate out kopia commands from a single file (#1552) * Separate out kopia commands from a single file * Combine wrapper functions * Fix policyChanges in PolicySetGlobal command wrapper * Move command wrapper arguments * Add CommandArgs struct to pass args to command wrappers * Convert local arguments to exportable arguments * Change log level to Debug * Remove unused const * Fix for make gomod in CI * Move RegEx as a const * Rename pkg/kopia/command/kopia.go to pkg/kopia/command/common.go * Rename log_output_utils.go to parse_command_output.go and Move output parsing functions from helpers.go to parse_command_output.go * Group 'kopia blob' commands in single file * Group 'kopia maintenance' commands in single file * Group 'kopia server' commands in single file * Move parent command functions closer in pkg/kopia/command/server.go * Group 'kopia snapshot' commands in single file * Group 'kopia snapshot' commands in single file * rename commandArgs in all commands * Simplify nil check in if statements * Fix golang issues * Fix golint issues * Add recent changes to snapshot create stats parsed from the 'kopia snapshot create' command * Fix go.mod issue in golint
kanisterio · Sep 7, 2022 · 43fb264 · 43fb264
1 parent 26992a6
commit 43fb264
Show file tree

Hide file tree

Showing 15 changed files with 1,144 additions and 8 deletions.
diff --git a/go.mod b/go.mod
@@ -63,6 +63,8 @@ require (
 	sigs.k8s.io/controller-runtime v0.12.3
 )
 
+require github.com/dustin/go-humanize v1.0.0
+
 require (
 	cloud.google.com/go v0.102.1 // indirect
 	cloud.google.com/go/compute v1.7.0 // indirect
@@ -91,7 +93,6 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
-	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/edsrzf/mmap-go v1.1.0 // indirect
 	github.com/elazarl/goproxy v0.0.0-20190711103511-473e67f1d7d2 // indirect
 	github.com/emicklei/go-restful v2.9.5+incompatible // indirect

diff --git a/pkg/kopia/command/blob.go b/pkg/kopia/command/blob.go
@@ -0,0 +1,39 @@
+// Copyright 2022 The Kanister Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package command
+
+type BlobListCommandArgs struct {
+	*CommandArgs
+}
+
+// BlobList returns the kopia command for listing blobs in the repository with their sizes
+func BlobList(cmdArgs BlobListCommandArgs) []string {
+	args := commonArgs(cmdArgs.EncryptionKey, cmdArgs.ConfigFilePath, cmdArgs.LogDirectory, false)
+	args = args.AppendLoggable(blobSubCommand, listSubCommand)
+
+	return stringSliceCommand(args)
+}
+
+type BlobStatsCommandArgs struct {
+	*CommandArgs
+}
+
+// BlobStats returns the kopia command to get the blob stats
+func BlobStats(cmdArgs BlobStatsCommandArgs) []string {
+	args := commonArgs(cmdArgs.EncryptionKey, cmdArgs.ConfigFilePath, cmdArgs.LogDirectory, false)
+	args = args.AppendLoggable(blobSubCommand, statsSubCommand, rawFlag)
+
+	return stringSliceCommand(args)
+}
diff --git a/pkg/kopia/command/common.go b/pkg/kopia/command/common.go
@@ -0,0 +1,61 @@
+// Copyright 2022 The Kanister Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package command
+
+import (
+	"github.com/kanisterio/kanister/pkg/field"
+	"github.com/kanisterio/kanister/pkg/log"
+	"github.com/kanisterio/kanister/pkg/logsafe"
+)
+
+type CommandArgs struct {
+	EncryptionKey  string
+	ConfigFilePath string
+	LogDirectory   string
+}
+
+func bashCommand(args logsafe.Cmd) []string {
+	log.Debug().Print("Kopia Command", field.M{"Command": args.String()})
+	return []string{"bash", "-o", "errexit", "-c", args.PlainText()}
+}
+
+func stringSliceCommand(args logsafe.Cmd) []string {
+	log.Debug().Print("Kopia Command", field.M{"Command": args.String()})
+	return args.StringSliceCMD()
+}
+
+func commonArgs(password, configFilePath, logDirectory string, requireInfoLevel bool) logsafe.Cmd {
+	c := logsafe.NewLoggable(kopiaCommand)
+	if requireInfoLevel {
+		c = c.AppendLoggable(logLevelInfoFlag)
+	} else {
+		c = c.AppendLoggable(logLevelErrorFlag)
+	}
+	if configFilePath != "" {
+		c = c.AppendLoggableKV(configFileFlag, configFilePath)
+	}
+	if logDirectory != "" {
+		c = c.AppendLoggableKV(logDirectoryFlag, logDirectory)
+	}
+	if password != "" {
+		c = c.AppendRedactedKV(passwordFlag, password)
+	}
+	return c
+}
+
+// ExecKopiaArgs returns the basic Argv for executing kopia with the given config file path.
+func ExecKopiaArgs(configFilePath string) []string {
+	return commonArgs("", configFilePath, "", false).StringSliceCMD()
+}
diff --git a/pkg/kopia/command/const.go b/pkg/kopia/command/const.go
@@ -0,0 +1,74 @@
+// Copyright 2022 The Kanister Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package command
+
+const (
+	blobSubCommand        = "blob"
+	createSubCommand      = "create"
+	deleteSubCommand      = "delete"
+	expireSubCommand      = "expire"
+	gcSubCommand          = "gc"
+	infoSubCommand        = "info"
+	kopiaCommand          = "kopia"
+	listSubCommand        = "list"
+	maintenanceSubCommand = "maintenance"
+	manifestSubCommand    = "manifest"
+	policySubCommand      = "policy"
+	restoreSubCommand     = "restore"
+	runSubCommand         = "run"
+	setSubCommand         = "set"
+	snapshotSubCommand    = "snapshot"
+	statsSubCommand       = "stats"
+
+	allFlag                    = "--all"
+	configFileFlag             = "--config-file"
+	deleteFlag                 = "--delete"
+	deltaFlag                  = "--delta"
+	filterFlag                 = "--filter"
+	globalFlag                 = "--global"
+	jsonFlag                   = "--json"
+	logDirectoryFlag           = "--log-dir"
+	logLevelErrorFlag          = "--log-level=error"
+	logLevelInfoFlag           = "--log-level=info"
+	noGrpcFlag                 = "--no-grpc"
+	parallelFlag               = "--parallel"
+	passwordFlag               = "--password"
+	progressUpdateIntervalFlag = "--progress-update-interval"
+	rawFlag                    = "--raw"
+	showIdenticalFlag          = "--show-identical"
+	unsafeIgnoreSourceFlag     = "--unsafe-ignore-source"
+	ownerFlag                  = "--owner"
+	sparseFlag                 = "--sparse"
+
+	// Server specific
+	addSubCommand             = "add"
+	refreshSubCommand         = "refresh"
+	serverSubCommand          = "server"
+	startSubCommand           = "start"
+	statusSubCommand          = "status"
+	userSubCommand            = "user"
+	addressFlag               = "--address"
+	redirectToDevNull         = "> /dev/null 2>&1"
+	runInBackground           = "&"
+	serverControlPasswordFlag = "--server-control-password"
+	serverControlUsernameFlag = "--server-control-username"
+	serverPasswordFlag        = "--server-password"
+	serverUsernameFlag        = "--server-username"
+	serverCertFingerprint     = "--server-cert-fingerprint"
+	tlsCertFilePath           = "--tls-cert-file"
+	tlsGenerateCertFlag       = "--tls-generate-cert"
+	tlsKeyFilePath            = "--tls-key-file"
+	userPasswordFlag          = "--user-password"
+)
diff --git a/pkg/kopia/command/helpers.go b/pkg/kopia/command/helpers.go
@@ -0,0 +1,37 @@
+// Copyright 2022 The Kanister Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package command
+
+import (
+	"github.com/kanisterio/kanister/pkg/kopia"
+	"github.com/kanisterio/kanister/pkg/utils"
+)
+
+type policyChanges map[string]string
+
+// GetCacheSizeSettingsForSnapshot returns the feature setting cache size values to be used
+// for initializing repositories that will be performing general command workloads that benefit from
+// cacheing metadata only.
+func GetCacheSizeSettingsForSnapshot() (contentCacheMB, metadataCacheMB int) {
+	return utils.GetEnvAsIntOrDefault(kopia.DataStoreGeneralContentCacheSizeMBVarName, kopia.DefaultDataStoreGeneralContentCacheSizeMB),
+		utils.GetEnvAsIntOrDefault(kopia.DataStoreGeneralMetadataCacheSizeMBVarName, kopia.DefaultDataStoreGeneralMetadataCacheSizeMB)
+}
+
+// GetCacheSizeSettingsForRestore returns the feature setting cache size values to be used
+// for initializing repositories that will be performing restore workloads
+func GetCacheSizeSettingsForRestore() (contentCacheMB, metadataCacheMB int) {
+	return utils.GetEnvAsIntOrDefault(kopia.DataStoreRestoreContentCacheSizeMBVarName, kopia.DefaultDataStoreRestoreContentCacheSizeMB),
+		utils.GetEnvAsIntOrDefault(kopia.DataStoreRestoreMetadataCacheSizeMBVarName, kopia.DefaultDataStoreRestoreMetadataCacheSizeMB)
+}
diff --git a/pkg/kopia/command/maintenance.go b/pkg/kopia/command/maintenance.go
@@ -0,0 +1,56 @@
+// Copyright 2022 The Kanister Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package command
+
+type MaintenanceInfoCommandArgs struct {
+	*CommandArgs
+	GetJsonOutput bool
+}
+
+// MaintenanceInfo returns the kopia command to get maintenance info
+func MaintenanceInfo(cmdArgs MaintenanceInfoCommandArgs) []string {
+	args := commonArgs(cmdArgs.EncryptionKey, cmdArgs.ConfigFilePath, cmdArgs.LogDirectory, false)
+	args = args.AppendLoggable(maintenanceSubCommand, infoSubCommand)
+	if cmdArgs.GetJsonOutput {
+		args = args.AppendLoggable(jsonFlag)
+	}
+
+	return stringSliceCommand(args)
+}
+
+type MaintenanceSetOwnerCommandArgs struct {
+	*CommandArgs
+	CustomOwner string
+}
+
+// MaintenanceSetOwner returns the kopia command for setting custom maintenance owner
+func MaintenanceSetOwner(cmdArgs MaintenanceSetOwnerCommandArgs) []string {
+	args := commonArgs(cmdArgs.EncryptionKey, cmdArgs.ConfigFilePath, cmdArgs.LogDirectory, false)
+	args = args.AppendLoggable(maintenanceSubCommand, setSubCommand)
+	args = args.AppendLoggableKV(ownerFlag, cmdArgs.CustomOwner)
+	return stringSliceCommand(args)
+}
+
+type MaintenanceRunCommandArgs struct {
+	*CommandArgs
+}
+
+// MaintenanceRunCommand returns the kopia command to run manual maintenance
+func MaintenanceRunCommand(cmdArgs MaintenanceRunCommandArgs) []string {
+	args := commonArgs(cmdArgs.EncryptionKey, cmdArgs.ConfigFilePath, cmdArgs.LogDirectory, false)
+	args = args.AppendLoggable(maintenanceSubCommand, runSubCommand)
+
+	return stringSliceCommand(args)
+}