diff --git a/go.mod b/go.mod index 10c7a5dc44..155a76c05f 100644 --- a/go.mod +++ b/go.mod @@ -63,6 +63,8 @@ require ( sigs.k8s.io/controller-runtime v0.12.3 ) +require github.com/dustin/go-humanize v1.0.0 + require ( cloud.google.com/go v0.102.1 // indirect cloud.google.com/go/compute v1.7.0 // indirect @@ -91,7 +93,6 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/dimchansky/utfbom v1.1.1 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect github.com/edsrzf/mmap-go v1.1.0 // indirect github.com/elazarl/goproxy v0.0.0-20190711103511-473e67f1d7d2 // indirect github.com/emicklei/go-restful v2.9.5+incompatible // indirect diff --git a/pkg/kopia/command/blob.go b/pkg/kopia/command/blob.go new file mode 100644 index 0000000000..1f76868473 --- /dev/null +++ b/pkg/kopia/command/blob.go @@ -0,0 +1,39 @@ +// Copyright 2022 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +type BlobListCommandArgs struct { + *CommandArgs +} + +// BlobList returns the kopia command for listing blobs in the repository with their sizes +func BlobList(cmdArgs BlobListCommandArgs) []string { + args := commonArgs(cmdArgs.EncryptionKey, cmdArgs.ConfigFilePath, cmdArgs.LogDirectory, false) + args = args.AppendLoggable(blobSubCommand, listSubCommand) + + return stringSliceCommand(args) +} + +type BlobStatsCommandArgs struct { + *CommandArgs +} + +// BlobStats returns the kopia command to get the blob stats +func BlobStats(cmdArgs BlobStatsCommandArgs) []string { + args := commonArgs(cmdArgs.EncryptionKey, cmdArgs.ConfigFilePath, cmdArgs.LogDirectory, false) + args = args.AppendLoggable(blobSubCommand, statsSubCommand, rawFlag) + + return stringSliceCommand(args) +} diff --git a/pkg/kopia/command/common.go b/pkg/kopia/command/common.go new file mode 100644 index 0000000000..fc445efa87 --- /dev/null +++ b/pkg/kopia/command/common.go @@ -0,0 +1,61 @@ +// Copyright 2022 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +import ( + "github.com/kanisterio/kanister/pkg/field" + "github.com/kanisterio/kanister/pkg/log" + "github.com/kanisterio/kanister/pkg/logsafe" +) + +type CommandArgs struct { + EncryptionKey string + ConfigFilePath string + LogDirectory string +} + +func bashCommand(args logsafe.Cmd) []string { + log.Debug().Print("Kopia Command", field.M{"Command": args.String()}) + return []string{"bash", "-o", "errexit", "-c", args.PlainText()} +} + +func stringSliceCommand(args logsafe.Cmd) []string { + log.Debug().Print("Kopia Command", field.M{"Command": args.String()}) + return args.StringSliceCMD() +} + +func commonArgs(password, configFilePath, logDirectory string, requireInfoLevel bool) logsafe.Cmd { + c := logsafe.NewLoggable(kopiaCommand) + if requireInfoLevel { + c = c.AppendLoggable(logLevelInfoFlag) + } else { + c = c.AppendLoggable(logLevelErrorFlag) + } + if configFilePath != "" { + c = c.AppendLoggableKV(configFileFlag, configFilePath) + } + if logDirectory != "" { + c = c.AppendLoggableKV(logDirectoryFlag, logDirectory) + } + if password != "" { + c = c.AppendRedactedKV(passwordFlag, password) + } + return c +} + +// ExecKopiaArgs returns the basic Argv for executing kopia with the given config file path. +func ExecKopiaArgs(configFilePath string) []string { + return commonArgs("", configFilePath, "", false).StringSliceCMD() +} diff --git a/pkg/kopia/command/const.go b/pkg/kopia/command/const.go new file mode 100644 index 0000000000..6eb1b4eab3 --- /dev/null +++ b/pkg/kopia/command/const.go @@ -0,0 +1,74 @@ +// Copyright 2022 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +const ( + blobSubCommand = "blob" + createSubCommand = "create" + deleteSubCommand = "delete" + expireSubCommand = "expire" + gcSubCommand = "gc" + infoSubCommand = "info" + kopiaCommand = "kopia" + listSubCommand = "list" + maintenanceSubCommand = "maintenance" + manifestSubCommand = "manifest" + policySubCommand = "policy" + restoreSubCommand = "restore" + runSubCommand = "run" + setSubCommand = "set" + snapshotSubCommand = "snapshot" + statsSubCommand = "stats" + + allFlag = "--all" + configFileFlag = "--config-file" + deleteFlag = "--delete" + deltaFlag = "--delta" + filterFlag = "--filter" + globalFlag = "--global" + jsonFlag = "--json" + logDirectoryFlag = "--log-dir" + logLevelErrorFlag = "--log-level=error" + logLevelInfoFlag = "--log-level=info" + noGrpcFlag = "--no-grpc" + parallelFlag = "--parallel" + passwordFlag = "--password" + progressUpdateIntervalFlag = "--progress-update-interval" + rawFlag = "--raw" + showIdenticalFlag = "--show-identical" + unsafeIgnoreSourceFlag = "--unsafe-ignore-source" + ownerFlag = "--owner" + sparseFlag = "--sparse" + + // Server specific + addSubCommand = "add" + refreshSubCommand = "refresh" + serverSubCommand = "server" + startSubCommand = "start" + statusSubCommand = "status" + userSubCommand = "user" + addressFlag = "--address" + redirectToDevNull = "> /dev/null 2>&1" + runInBackground = "&" + serverControlPasswordFlag = "--server-control-password" + serverControlUsernameFlag = "--server-control-username" + serverPasswordFlag = "--server-password" + serverUsernameFlag = "--server-username" + serverCertFingerprint = "--server-cert-fingerprint" + tlsCertFilePath = "--tls-cert-file" + tlsGenerateCertFlag = "--tls-generate-cert" + tlsKeyFilePath = "--tls-key-file" + userPasswordFlag = "--user-password" +) diff --git a/pkg/kopia/command/helpers.go b/pkg/kopia/command/helpers.go new file mode 100644 index 0000000000..d55713c681 --- /dev/null +++ b/pkg/kopia/command/helpers.go @@ -0,0 +1,37 @@ +// Copyright 2022 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +import ( + "github.com/kanisterio/kanister/pkg/kopia" + "github.com/kanisterio/kanister/pkg/utils" +) + +type policyChanges map[string]string + +// GetCacheSizeSettingsForSnapshot returns the feature setting cache size values to be used +// for initializing repositories that will be performing general command workloads that benefit from +// cacheing metadata only. +func GetCacheSizeSettingsForSnapshot() (contentCacheMB, metadataCacheMB int) { + return utils.GetEnvAsIntOrDefault(kopia.DataStoreGeneralContentCacheSizeMBVarName, kopia.DefaultDataStoreGeneralContentCacheSizeMB), + utils.GetEnvAsIntOrDefault(kopia.DataStoreGeneralMetadataCacheSizeMBVarName, kopia.DefaultDataStoreGeneralMetadataCacheSizeMB) +} + +// GetCacheSizeSettingsForRestore returns the feature setting cache size values to be used +// for initializing repositories that will be performing restore workloads +func GetCacheSizeSettingsForRestore() (contentCacheMB, metadataCacheMB int) { + return utils.GetEnvAsIntOrDefault(kopia.DataStoreRestoreContentCacheSizeMBVarName, kopia.DefaultDataStoreRestoreContentCacheSizeMB), + utils.GetEnvAsIntOrDefault(kopia.DataStoreRestoreMetadataCacheSizeMBVarName, kopia.DefaultDataStoreRestoreMetadataCacheSizeMB) +} diff --git a/pkg/kopia/command/maintenance.go b/pkg/kopia/command/maintenance.go new file mode 100644 index 0000000000..c5bd4b8b99 --- /dev/null +++ b/pkg/kopia/command/maintenance.go @@ -0,0 +1,56 @@ +// Copyright 2022 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +type MaintenanceInfoCommandArgs struct { + *CommandArgs + GetJsonOutput bool +} + +// MaintenanceInfo returns the kopia command to get maintenance info +func MaintenanceInfo(cmdArgs MaintenanceInfoCommandArgs) []string { + args := commonArgs(cmdArgs.EncryptionKey, cmdArgs.ConfigFilePath, cmdArgs.LogDirectory, false) + args = args.AppendLoggable(maintenanceSubCommand, infoSubCommand) + if cmdArgs.GetJsonOutput { + args = args.AppendLoggable(jsonFlag) + } + + return stringSliceCommand(args) +} + +type MaintenanceSetOwnerCommandArgs struct { + *CommandArgs + CustomOwner string +} + +// MaintenanceSetOwner returns the kopia command for setting custom maintenance owner +func MaintenanceSetOwner(cmdArgs MaintenanceSetOwnerCommandArgs) []string { + args := commonArgs(cmdArgs.EncryptionKey, cmdArgs.ConfigFilePath, cmdArgs.LogDirectory, false) + args = args.AppendLoggable(maintenanceSubCommand, setSubCommand) + args = args.AppendLoggableKV(ownerFlag, cmdArgs.CustomOwner) + return stringSliceCommand(args) +} + +type MaintenanceRunCommandArgs struct { + *CommandArgs +} + +// MaintenanceRunCommand returns the kopia command to run manual maintenance +func MaintenanceRunCommand(cmdArgs MaintenanceRunCommandArgs) []string { + args := commonArgs(cmdArgs.EncryptionKey, cmdArgs.ConfigFilePath, cmdArgs.LogDirectory, false) + args = args.AppendLoggable(maintenanceSubCommand, runSubCommand) + + return stringSliceCommand(args) +} diff --git a/pkg/kopia/command/parse_command_output.go b/pkg/kopia/command/parse_command_output.go new file mode 100644 index 0000000000..73d86e92b1 --- /dev/null +++ b/pkg/kopia/command/parse_command_output.go @@ -0,0 +1,374 @@ +// Copyright 2022 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +import ( + "bufio" + "encoding/json" + "fmt" + "regexp" + "strconv" + "strings" + + "github.com/dustin/go-humanize" + "github.com/kopia/kopia/repo/manifest" + "github.com/kopia/kopia/snapshot" + "github.com/pkg/errors" + + "github.com/kanisterio/kanister/pkg/field" + "github.com/kanisterio/kanister/pkg/log" +) + +const ( + pathKey = "path" + typeKey = "type" + snapshotValue = "snapshot" + + snapshotCreateOutputRegEx = `(?P[|/\-\\\*]).+[^\d](?P\d+) hashed \((?P[^\)]+)\), (?P\d+) cached \((?P[^\)]+)\), uploaded (?P[^\)]+), (?:estimating...|estimated (?P[^\)]+) \((?P[^\)]+)\%\).+)` + extractSnapshotIDRegEx = `Created snapshot with root ([^\s]+) and ID ([^\s]+).*$` + repoTotalSizeFromBlobStatsRegEx = `Total: (\d+)$` + repoCountFromBlobStatsRegEx = `Count: (\d+)$` +) + +// SnapshotIDsFromSnapshot extracts root ID of a snapshot from the logs +func SnapshotIDsFromSnapshot(output string) (snapID, rootID string, err error) { + if output == "" { + return snapID, rootID, errors.New("Received empty output") + } + + logs := regexp.MustCompile("[\r\n]").Split(output, -1) + pattern := regexp.MustCompile(extractSnapshotIDRegEx) + for _, l := range logs { + // Log should contain "Created snapshot with root ABC and ID XYZ..." + match := pattern.FindAllStringSubmatch(l, 1) + if len(match) > 0 && len(match[0]) > 2 { + snapID = match[0][2] + rootID = match[0][1] + return + } + } + return snapID, rootID, errors.New("Failed to find Root ID from output") +} + +// LatestSnapshotInfoFromManifestList returns snapshot ID and backup path of the latest snapshot from `manifests list` output +func LatestSnapshotInfoFromManifestList(output string) (string, string, error) { + manifestList := []manifest.EntryMetadata{} + snapID := "" + backupPath := "" + + err := json.Unmarshal([]byte(output), &manifestList) + if err != nil { + return snapID, backupPath, errors.Wrap(err, "Failed to unmarshal manifest list") + } + for _, manifest := range manifestList { + for key, value := range manifest.Labels { + if key == pathKey { + backupPath = value + } + if key == typeKey && value == snapshotValue { + snapID = string(manifest.ID) + } + } + } + if snapID == "" { + return "", "", errors.New("Failed to get latest snapshot ID from manifest list") + } + if backupPath == "" { + return "", "", errors.New("Failed to get latest snapshot backup path from manifest list") + } + return snapID, backupPath, nil +} + +// SnapshotInfoFromSnapshotCreateOutput returns snapshot ID and root ID from snapshot create output +func SnapshotInfoFromSnapshotCreateOutput(output string) (string, string, error) { + snapID := "" + rootID := "" + scanner := bufio.NewScanner(strings.NewReader(output)) + for scanner.Scan() { + snapManifest := &snapshot.Manifest{} + err := json.Unmarshal([]byte(scanner.Text()), snapManifest) + if err != nil { + continue + } + if snapManifest == nil { + continue + } + snapID = string(snapManifest.ID) + if snapManifest.RootEntry != nil { + rootID = snapManifest.RootEntry.ObjectID.String() + } + } + if snapID == "" { + return "", "", errors.New(fmt.Sprintf("Failed to get snapshot ID from create snapshot output %s", output)) + } + if rootID == "" { + return "", "", errors.New(fmt.Sprintf("Failed to get root ID from create snapshot output %s", output)) + } + return snapID, rootID, nil +} + +// SnapSizeStatsFromSnapListAll returns a list of snapshot logical sizes assuming the input string +// is formatted as the output of a kopia snapshot list --all command. +func SnapSizeStatsFromSnapListAll(output string) (totalSizeB int64, numSnapshots int, err error) { + if output == "" { + return 0, 0, errors.New("Received empty output") + } + + snapList, err := parseSnapshotManifestList(output) + if err != nil { + return 0, 0, errors.Wrap(err, "Parsing snapshot list output as snapshot manifest list") + } + + totalSizeB = sumSnapshotSizes(snapList) + + return totalSizeB, len(snapList), nil +} + +func sumSnapshotSizes(snapList []*snapshot.Manifest) (sum int64) { + noSizeDataCount := 0 + for _, snapInfo := range snapList { + if snapInfo.RootEntry == nil || + snapInfo.RootEntry.DirSummary == nil { + noSizeDataCount++ + + continue + } + + sum += snapInfo.RootEntry.DirSummary.TotalFileSize + } + + if noSizeDataCount > 0 { + log.Error().Print("Found snapshot manifests without size data", field.M{"count": noSizeDataCount}) + } + + return sum +} + +func parseSnapshotManifestList(output string) ([]*snapshot.Manifest, error) { + snapInfoList := []*snapshot.Manifest{} + + if err := json.Unmarshal([]byte(output), &snapInfoList); err != nil { + return nil, errors.Wrap(err, "Failed to unmarshal snapshot manifest list") + } + + return snapInfoList, nil +} + +// SnapshotCreateInfo is a container for data that can be parsed from the output of +// `kopia snapshot create`. +type SnapshotCreateInfo struct { + SnapshotID string + RootID string + Stats *SnapshotCreateStats +} + +// ParseSnapshotCreateOutput parses the output of a snapshot create command into +// a new SnapshotCreateInfo struct and returns its pointer. The Stats field may be nil +// if the stats were unable to be parsed. The root ID and snapshot ID are fetched from +// structured stdout and stats are parsed from stderr output. +func ParseSnapshotCreateOutput(snapCreateStdoutOutput, snapCreateStderrOutput string) (*SnapshotCreateInfo, error) { + snapID, rootID, err := SnapshotInfoFromSnapshotCreateOutput(snapCreateStdoutOutput) + if err != nil { + return nil, err + } + + return &SnapshotCreateInfo{ + SnapshotID: snapID, + RootID: rootID, + Stats: SnapshotStatsFromSnapshotCreate(snapCreateStderrOutput, true), + }, nil +} + +// SnapshotCreateStats is a container for stats parsed from the output of a `kopia +// snapshot create` command. +type SnapshotCreateStats struct { + FilesHashed int64 + SizeHashedB int64 + FilesCached int64 + SizeCachedB int64 + SizeUploadedB int64 + SizeEstimatedB int64 + ProgressPercent int64 +} + +var kopiaProgressPattern = regexp.MustCompile(snapshotCreateOutputRegEx) //nolint:lll + +// SnapshotStatsFromSnapshotCreate parses the output of a kopia snapshot +// create execution for a log of the stats for that execution. +func SnapshotStatsFromSnapshotCreate(snapCreateStderrOutput string, matchOnlyFinished bool) (stats *SnapshotCreateStats) { + if snapCreateStderrOutput == "" { + return nil + } + logs := regexp.MustCompile("[\r\n]").Split(snapCreateStderrOutput, -1) + + // Match a pattern starting with "*" (signifying upload finished), and containing + // the repeated pattern "<\d+> (),", + // where is "hashed", "cached", and "uploaded". + // Example input: + // * 0 hashing, 1 hashed (2 B), 3 cached (40 KB), uploaded 6.7 GB, estimated 1092.3 MB (100.0%) 0s left + // Expected output: + // SnapshotCreateStats{ + // filesHashed: 1, + // sizeHashedB: 2, + // filesCached: 3, + // sizeCachedB: 40000, + // sizeUploadedB: 6700000000, + // sizeEstimatedB: 1092000000, + // progressPercent: 100, + // }, nil + + for _, l := range logs { + lineStats := parseKopiaProgressLine(l, matchOnlyFinished) + if lineStats != nil { + stats = lineStats + } + } + + return stats +} + +func parseKopiaProgressLine(line string, matchOnlyFinished bool) (stats *SnapshotCreateStats) { + match := kopiaProgressPattern.FindStringSubmatch(line) + if len(match) < 9 { + return nil + } + + groups := make(map[string]string) + for i, name := range kopiaProgressPattern.SubexpNames() { + if i != 0 && name != "" { + groups[name] = match[i] + } + } + + isFinalResult := groups["spinner"] == "*" + if matchOnlyFinished && !isFinalResult { + return nil + } + + numHashed, err := strconv.Atoi(groups["numHashed"]) + if err != nil { + log.WithError(err).Print("Skipping entry due to inability to parse number of hashed files", field.M{"numHashed": groups["numHashed"]}) + return nil + } + + numCached, err := strconv.Atoi(groups["numCached"]) + if err != nil { + log.WithError(err).Print("Skipping entry due to inability to parse number of cached files", field.M{"numCached": groups["numCached"]}) + return nil + } + + hashedSizeBytes, err := humanize.ParseBytes(groups["hashedSize"]) + if err != nil { + log.WithError(err).Print("Skipping entry due to inability to parse hashed size string", field.M{"hashedSize": groups["hashedSize"]}) + return nil + } + + cachedSizeBytes, err := humanize.ParseBytes(groups["cachedSize"]) + if err != nil { + log.WithError(err).Print("Skipping entry due to inability to parse cached size string", field.M{"cachedSize": groups["cachedSize"]}) + return nil + } + + uploadedSizeBytes, err := humanize.ParseBytes(groups["uploadedSize"]) + if err != nil { + log.WithError(err).Print("Skipping entry due to inability to parse uploaded size string", field.M{"uploadedSize": groups["uploadedSize"]}) + return nil + } + + var estimatedSizeBytes uint64 + var progressPercent float64 + stillEstimating := len(groups["estimatedSize"]) == 0 && len(groups["estimatedProgress"]) == 0 + + if !stillEstimating { // Estimation completed + estimatedSizeBytes, err = humanize.ParseBytes(groups["estimatedSize"]) + if err != nil { + log.WithError(err).Print("Skipping entry due to inability to parse estimated size string", field.M{"estimatedSize": groups["estimatedSize"]}) + return nil + } + + progressPercent, err = strconv.ParseFloat(groups["estimatedProgress"], 64) + if err != nil { + log.WithError(err).Print("Skipping entry due to inability to parse progress percent string", field.M{"estimatedProgress": groups["estimatedProgress"]}) + return nil + } + } else if isFinalResult { // It may happen that kopia will complete its job before estimation will be done + progressPercent = 100 + } + + return &SnapshotCreateStats{ + FilesHashed: int64(numHashed), + SizeHashedB: int64(hashedSizeBytes), + FilesCached: int64(numCached), + SizeCachedB: int64(cachedSizeBytes), + SizeUploadedB: int64(uploadedSizeBytes), + SizeEstimatedB: int64(estimatedSizeBytes), + ProgressPercent: int64(progressPercent), + } +} + +// RepoSizeStatsFromBlobStatsRaw takes a string as input, interprets it as a kopia blob stats +// output in an expected format (Contains the line "Total: "), and returns the integer +// size in bytes or an error if parsing is unsuccessful. +func RepoSizeStatsFromBlobStatsRaw(blobStats string) (phySizeTotal int64, blobCount int, err error) { + if blobStats == "" { + return phySizeTotal, blobCount, errors.New("received empty blob stats string") + } + + sizePattern := regexp.MustCompile(repoTotalSizeFromBlobStatsRegEx) + countPattern := regexp.MustCompile(repoCountFromBlobStatsRegEx) + + var countStr, sizeStr string + + for _, l := range strings.Split(blobStats, "\n") { + if countStr == "" { + countMatch := countPattern.FindStringSubmatch(l) + if len(countMatch) >= 2 { + countStr = countMatch[1] + } + } + + if sizeStr == "" { + sizeMatch := sizePattern.FindStringSubmatch(l) + if len(sizeMatch) >= 2 { + sizeStr = sizeMatch[1] + } + } + + if !(countStr == "" || sizeStr == "") { + // Both strings have been matched + break + } + } + + if countStr == "" { + return phySizeTotal, blobCount, errors.New("could not find count field in the blob stats") + } + + if sizeStr == "" { + return phySizeTotal, blobCount, errors.New("could not find size field in the blob stats") + } + + countVal, err := strconv.Atoi(countStr) + if err != nil { + return phySizeTotal, blobCount, errors.Wrap(err, fmt.Sprintf("unable to convert parsed count value %s", countStr)) + } + + sizeValBytes, err := strconv.Atoi(sizeStr) + if err != nil { + return phySizeTotal, blobCount, errors.Wrap(err, fmt.Sprintf("unable to convert parsed size value %s", countStr)) + } + + return int64(sizeValBytes), countVal, nil +} diff --git a/pkg/kopia/command/policy_set_global.go b/pkg/kopia/command/policy_set_global.go new file mode 100644 index 0000000000..880d432aa8 --- /dev/null +++ b/pkg/kopia/command/policy_set_global.go @@ -0,0 +1,31 @@ +// Copyright 2022 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +type PolicySetGlobalCommandArgs struct { + *CommandArgs + Modifications policyChanges +} + +// PolicySetGlobal returns the kopia command for modifying the global policy +func PolicySetGlobal(cmdArgs PolicySetGlobalCommandArgs) []string { + args := commonArgs(cmdArgs.EncryptionKey, cmdArgs.ConfigFilePath, cmdArgs.LogDirectory, false) + args = args.AppendLoggable(policySubCommand, setSubCommand, globalFlag) + for field, val := range cmdArgs.Modifications { + args = args.AppendLoggableKV(field, val) + } + + return stringSliceCommand(args) +} diff --git a/pkg/kopia/command/restore.go b/pkg/kopia/command/restore.go new file mode 100644 index 0000000000..5171e47162 --- /dev/null +++ b/pkg/kopia/command/restore.go @@ -0,0 +1,29 @@ +// Copyright 2022 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +type RestoreCommandArgs struct { + *CommandArgs + RootID string + TargetPath string +} + +// Restore returns the kopia command for restoring root of a snapshot with given root ID +func Restore(cmdArgs RestoreCommandArgs) []string { + args := commonArgs(cmdArgs.EncryptionKey, cmdArgs.ConfigFilePath, cmdArgs.LogDirectory, false) + args = args.AppendLoggable(restoreSubCommand, cmdArgs.RootID, cmdArgs.TargetPath) + + return stringSliceCommand(args) +} diff --git a/pkg/kopia/command/server.go b/pkg/kopia/command/server.go new file mode 100644 index 0000000000..c19b52f474 --- /dev/null +++ b/pkg/kopia/command/server.go @@ -0,0 +1,138 @@ +// Copyright 2022 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +type ServerStartCommandArgs struct { + *CommandArgs + ServerAddress string + TLSCertFile string + TLSKeyFile string + ServerUsername string + ServerPassword string + AutoGenerateCert bool + Background bool +} + +// ServerStart returns the kopia command for starting the Kopia API Server +func ServerStart(cmdArgs ServerStartCommandArgs) []string { + args := commonArgs("", cmdArgs.ConfigFilePath, cmdArgs.LogDirectory, false) + + if cmdArgs.AutoGenerateCert { + args = args.AppendLoggable(serverSubCommand, startSubCommand, tlsGenerateCertFlag) + } else { + args = args.AppendLoggable(serverSubCommand, startSubCommand) + } + args = args.AppendLoggableKV(addressFlag, cmdArgs.ServerAddress) + args = args.AppendLoggableKV(tlsCertFilePath, cmdArgs.TLSCertFile) + args = args.AppendLoggableKV(tlsKeyFilePath, cmdArgs.TLSKeyFile) + args = args.AppendLoggableKV(serverUsernameFlag, cmdArgs.ServerUsername) + args = args.AppendRedactedKV(serverPasswordFlag, cmdArgs.ServerPassword) + + args = args.AppendLoggableKV(serverControlUsernameFlag, cmdArgs.ServerUsername) + args = args.AppendRedactedKV(serverControlPasswordFlag, cmdArgs.ServerPassword) + + // TODO: Remove when GRPC support is added + args = args.AppendLoggable(noGrpcFlag) + + if cmdArgs.Background { + // To start the server and run in the background + args = args.AppendLoggable(redirectToDevNull, runInBackground) + } + + return bashCommand(args) +} + +type ServerRefreshCommandArgs struct { + *CommandArgs + ServerAddress string + ServerUsername string + ServerPassword string + Fingerprint string +} + +// ServerRefresh returns the kopia command for refreshing the Kopia API Server +// This helps allow new users to be able to connect to the Server instead of waiting for auto-refresh +func ServerRefresh(cmdArgs ServerRefreshCommandArgs) []string { + args := commonArgs(cmdArgs.EncryptionKey, cmdArgs.ConfigFilePath, cmdArgs.LogDirectory, false) + args = args.AppendLoggable(serverSubCommand, refreshSubCommand) + args = args.AppendRedactedKV(serverCertFingerprint, cmdArgs.Fingerprint) + args = args.AppendLoggableKV(addressFlag, cmdArgs.ServerAddress) + args = args.AppendLoggableKV(serverUsernameFlag, cmdArgs.ServerUsername) + args = args.AppendRedactedKV(serverPasswordFlag, cmdArgs.ServerPassword) + + return stringSliceCommand(args) +} + +type ServerStatusCommandArgs struct { + *CommandArgs + ServerAddress string + ServerUsername string + ServerPassword string + Fingerprint string +} + +// ServerStatus returns the kopia command for checking status of the Kopia API Server +func ServerStatus(cmdArgs ServerStatusCommandArgs) []string { + args := commonArgs("", cmdArgs.ConfigFilePath, cmdArgs.LogDirectory, false) + args = args.AppendLoggable(serverSubCommand, statusSubCommand) + args = args.AppendLoggableKV(addressFlag, cmdArgs.ServerAddress) + args = args.AppendRedactedKV(serverCertFingerprint, cmdArgs.Fingerprint) + args = args.AppendLoggableKV(serverUsernameFlag, cmdArgs.ServerUsername) + args = args.AppendRedactedKV(serverPasswordFlag, cmdArgs.ServerPassword) + + return stringSliceCommand(args) +} + +type ServerListUserCommmandArgs struct { + *CommandArgs +} + +// ServerListUser returns the kopia command to list users from the Kopia API Server +func ServerListUser(cmdArgs ServerListUserCommmandArgs) []string { + args := commonArgs(cmdArgs.EncryptionKey, cmdArgs.ConfigFilePath, cmdArgs.LogDirectory, false) + args = args.AppendLoggable(serverSubCommand, userSubCommand, listSubCommand, jsonFlag) + + return stringSliceCommand(args) +} + +type ServerSetUserCommandArgs struct { + *CommandArgs + NewUsername string + UserPassword string +} + +// ServerSetUser returns the kopia command setting password for existing user for the Kopia API Server +func ServerSetUser(cmdArgs ServerSetUserCommandArgs) []string { + args := commonArgs(cmdArgs.EncryptionKey, cmdArgs.ConfigFilePath, cmdArgs.LogDirectory, false) + args = args.AppendLoggable(serverSubCommand, userSubCommand, setSubCommand, cmdArgs.NewUsername) + args = args.AppendRedactedKV(userPasswordFlag, cmdArgs.UserPassword) + + return stringSliceCommand(args) +} + +type ServerAddUserCommandArgs struct { + *CommandArgs + NewUsername string + UserPassword string +} + +// ServerAddUser returns the kopia command adding a new user to the Kopia API Server +func ServerAddUser(cmdArgs ServerAddUserCommandArgs) []string { + args := commonArgs(cmdArgs.EncryptionKey, cmdArgs.ConfigFilePath, cmdArgs.LogDirectory, false) + args = args.AppendLoggable(serverSubCommand, userSubCommand, addSubCommand, cmdArgs.NewUsername) + args = args.AppendRedactedKV(userPasswordFlag, cmdArgs.UserPassword) + + return stringSliceCommand(args) +} diff --git a/pkg/kopia/command/snapshot.go b/pkg/kopia/command/snapshot.go new file mode 100644 index 0000000000..8d31c93599 --- /dev/null +++ b/pkg/kopia/command/snapshot.go @@ -0,0 +1,139 @@ +// Copyright 2022 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +import ( + "strconv" + + "github.com/kanisterio/kanister/pkg/kopia" + "github.com/kanisterio/kanister/pkg/utils" +) + +const ( + // kube.Exec might timeout after 4h if there is no output from the command + // Setting it to 1h instead of 1000000h so that kopia logs progress once every hour + longUpdateInterval = "1h" + + requireLogLevelInfo = true +) + +type SnapshotCreateCommandArgs struct { + *CommandArgs + PathToBackup string +} + +// SnapshotCreate returns the kopia command for creation of a snapshot +// TODO: Have better mechanism to apply global flags +func SnapshotCreate(cmdArgs SnapshotCreateCommandArgs) []string { + parallelismStr := strconv.Itoa(utils.GetEnvAsIntOrDefault(kopia.DataStoreParallelUploadVarName, kopia.DefaultDataStoreParallelUpload)) + args := commonArgs(cmdArgs.EncryptionKey, cmdArgs.ConfigFilePath, cmdArgs.LogDirectory, requireLogLevelInfo) + args = args.AppendLoggable(snapshotSubCommand, createSubCommand, cmdArgs.PathToBackup, jsonFlag) + args = args.AppendLoggableKV(parallelFlag, parallelismStr) + args = args.AppendLoggableKV(progressUpdateIntervalFlag, longUpdateInterval) + + return stringSliceCommand(args) +} + +type SnapshotRestoreCommandArgs struct { + *CommandArgs + SnapID string + TargetPath string + SparseRestore bool +} + +// SnapshotRestore returns kopia command restoring snapshots with given snap ID +func SnapshotRestore(cmdArgs SnapshotRestoreCommandArgs) []string { + args := commonArgs(cmdArgs.EncryptionKey, cmdArgs.ConfigFilePath, cmdArgs.LogDirectory, false) + args = args.AppendLoggable(snapshotSubCommand, restoreSubCommand, cmdArgs.SnapID, cmdArgs.TargetPath) + if cmdArgs.SparseRestore { + args = args.AppendLoggable(sparseFlag) + } + + return stringSliceCommand(args) +} + +type SnapshotDeleteCommandArgs struct { + *CommandArgs + SnapID string +} + +// SnapshotDelete returns the kopia command for deleting a snapshot with given snapshot ID +func SnapshotDelete(cmdArgs SnapshotDeleteCommandArgs) []string { + args := commonArgs(cmdArgs.EncryptionKey, cmdArgs.ConfigFilePath, cmdArgs.LogDirectory, false) + args = args.AppendLoggable(snapshotSubCommand, deleteSubCommand, cmdArgs.SnapID, unsafeIgnoreSourceFlag) + + return stringSliceCommand(args) +} + +type SnapshotExpireCommandArgs struct { + *CommandArgs + RootID string + MustDelete bool +} + +// SnapshotExpire returns the kopia command for removing snapshots with given root ID +func SnapshotExpire(cmdArgs SnapshotExpireCommandArgs) []string { + args := commonArgs(cmdArgs.EncryptionKey, cmdArgs.ConfigFilePath, cmdArgs.LogDirectory, false) + args = args.AppendLoggable(snapshotSubCommand, expireSubCommand, cmdArgs.RootID) + if cmdArgs.MustDelete { + args = args.AppendLoggable(deleteFlag) + } + + return stringSliceCommand(args) +} + +type SnapshotGCCommandArgs struct { + *CommandArgs +} + +// SnapshotGC returns the kopia command for issuing kopia snapshot gc +func SnapshotGC(cmdArgs SnapshotGCCommandArgs) []string { + args := commonArgs(cmdArgs.EncryptionKey, cmdArgs.ConfigFilePath, cmdArgs.LogDirectory, false) + args = args.AppendLoggable(snapshotSubCommand, gcSubCommand, deleteFlag) + + return stringSliceCommand(args) +} + +type SnapListAllCommandArgs struct { + *CommandArgs +} + +// SnapListAll returns the kopia command for listing all snapshots in the repository with their sizes +func SnapListAll(cmdArgs SnapListAllCommandArgs) []string { + args := commonArgs(cmdArgs.EncryptionKey, cmdArgs.ConfigFilePath, cmdArgs.LogDirectory, false) + args = args.AppendLoggable( + snapshotSubCommand, + listSubCommand, + allFlag, + deltaFlag, + showIdenticalFlag, + jsonFlag, + ) + + return stringSliceCommand(args) +} + +type SnapListAllWithSnapIDsCommandArgs struct { + *CommandArgs +} + +// SnapListAllWithSnapIDs returns the kopia command for listing all snapshots in the repository with snapshotIDs +func SnapListAllWithSnapIDs(cmdArgs SnapListAllWithSnapIDsCommandArgs) []string { + args := commonArgs(cmdArgs.EncryptionKey, cmdArgs.ConfigFilePath, cmdArgs.LogDirectory, false) + args = args.AppendLoggable(manifestSubCommand, listSubCommand, jsonFlag) + args = args.AppendLoggableKV(filterFlag, kopia.ManifestTypeSnapshotFilter) + + return stringSliceCommand(args) +} diff --git a/pkg/kopia/const.go b/pkg/kopia/const.go new file mode 100644 index 0000000000..1d7c3adb57 --- /dev/null +++ b/pkg/kopia/const.go @@ -0,0 +1,56 @@ +// Copyright 2022 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kopia + +const ( + // DefaultClientConfigFilePath is the file which contains kopia repo config + DefaultClientConfigFilePath = "/tmp/kopia-repository.config" + + // DefaultClientCacheDirectory is the directory where kopia content cache is created + DefaultClientCacheDirectory = "/tmp/kopia-cache" + + // DefaultDataStoreGeneralContentCacheSizeMB is the default content cache size for general command workloads + DefaultDataStoreGeneralContentCacheSizeMB = 0 + // DataStoreGeneralContentCacheSizeMBVarName is the name of the environment variable that controls + // kopia content cache size for general command workloads + DataStoreGeneralContentCacheSizeMBVarName = "DATA_STORE_GENERAL_CONTENT_CACHE_SIZE_MB" + + // DefaultDataStoreGeneralMetadataCacheSizeMB is the default metadata cache size for general command workloads + DefaultDataStoreGeneralMetadataCacheSizeMB = 500 + // DataStoreGeneralMetadataCacheSizeMBVarName is the name of the environment variable that controls + // kopia metadata cache size for general command workloads + DataStoreGeneralMetadataCacheSizeMBVarName = "DATA_STORE_GENERAL_METADATA_CACHE_SIZE_MB" + + // DefaultDataStoreRestoreContentCacheSizeMB is the default content cache size for restore workloads + DefaultDataStoreRestoreContentCacheSizeMB = 500 + // DataStoreRestoreContentCacheSizeMBVarName is the name of the environment variable that controls + // kopia content cache size for restore workloads + DataStoreRestoreContentCacheSizeMBVarName = "DATA_STORE_RESTORE_CONTENT_CACHE_SIZE_MB" + + // DefaultDataStoreRestoreMetadataCacheSizeMB is the default metadata cache size for restore workloads + DefaultDataStoreRestoreMetadataCacheSizeMB = 500 + // DataStoreRestoreMetadataCacheSizeMBVarName is the name of the environment variable that controls + // kopia metadata cache size for restore workloads + DataStoreRestoreMetadataCacheSizeMBVarName = "DATA_STORE_RESTORE_METADATA_CACHE_SIZE_MB" + + // DefaultDataStoreParallelUpload is the default value for data store parallelism + DefaultDataStoreParallelUpload = 8 + + // DataStoreParallelUploadVarName is the name of the environment variable that controls + // kopia parallelism during snapshot create commands + DataStoreParallelUploadVarName = "DATA_STORE_PARALLEL_UPLOAD" + + ManifestTypeSnapshotFilter = "type:snapshot" +) diff --git a/pkg/kopia/errors.go b/pkg/kopia/errors.go new file mode 100644 index 0000000000..ce3f499afc --- /dev/null +++ b/pkg/kopia/errors.go @@ -0,0 +1,26 @@ +// Copyright 2022 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kopia + +import ( + "errors" +) + +var ( + ErrInvalidPassword = errors.New("invalid repository password") + ErrOutOfMemory = errors.New("kanister-tools container ran out of memory") + ErrAccessDenied = errors.New("Access Denied") + ErrRepoNotFound = errors.New("repository not found") +) diff --git a/pkg/kopia/maintenance/get_maintenance_owner.go b/pkg/kopia/maintenance/get_maintenance_owner.go new file mode 100644 index 0000000000..9671b8502c --- /dev/null +++ b/pkg/kopia/maintenance/get_maintenance_owner.go @@ -0,0 +1,82 @@ +// Copyright 2022 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package maintenance + +import ( + "strings" + + "github.com/kopia/kopia/repo/manifest" + "github.com/pkg/errors" + "k8s.io/client-go/kubernetes" + + "github.com/kanisterio/kanister/pkg/format" + "github.com/kanisterio/kanister/pkg/kopia/command" + "github.com/kanisterio/kanister/pkg/kube" +) + +// KopiaUserProfile is a duplicate of struct for Kopia user profiles since Profile struct is in internal/user package and could not be imported +type KopiaUserProfile struct { + ManifestID manifest.ID `json:"-"` + + Username string `json:"username"` + PasswordHashVersion int `json:"passwordHashVersion"` + PasswordHash []byte `json:"passwordHash"` +} + +// GetMaintenanceOwnerForConnectedRepository executes maintenance info command, parses output +// and returns maintenance owner +func GetMaintenanceOwnerForConnectedRepository( + cli kubernetes.Interface, + namespace, + pod, + container, + encryptionKey, + configFilePath, + logDirectory string, +) (string, error) { + args := command.MaintenanceInfoCommandArgs{ + CommandArgs: &command.CommandArgs{ + EncryptionKey: encryptionKey, + ConfigFilePath: configFilePath, + LogDirectory: logDirectory, + }, + GetJsonOutput: false, + } + cmd := command.MaintenanceInfo(args) + stdout, stderr, err := kube.Exec(cli, namespace, pod, container, cmd, nil) + format.Log(pod, container, stdout) + format.Log(pod, container, stderr) + if err != nil { + return "", err + } + parsedOwner := parseOutput(stdout) + if parsedOwner == "" { + return "", errors.New("Failed parsing maintenance info output to get owner") + } + return parsedOwner, nil +} + +func parseOutput(output string) string { + lines := strings.Split(output, "\n") + for _, line := range lines { + if strings.Contains(line, "Owner") { + arr := strings.Split(line, ":") + if len(arr) == 2 { + return strings.TrimSpace(arr[1]) + } + } + } + return "" +} diff --git a/pkg/kopia/utils.go b/pkg/kopia/utils.go index da3fcee1d2..c9c9a769e7 100644 --- a/pkg/kopia/utils.go +++ b/pkg/kopia/utils.go @@ -35,12 +35,6 @@ import ( ) const ( - // DefaultClientConfigFilePath is the file which contains kopia repo config - DefaultClientConfigFilePath = "/tmp/kopia-repository.config" - - // DefaultClientCacheDirectory is the directory where kopia content cache is created - DefaultClientCacheDirectory = "/tmp/kopia-cache" - // defaultDataStoreGeneralContentCacheSizeMB is the default content cache size for general command workloads defaultDataStoreGeneralContentCacheSizeMB = 0 @@ -58,7 +52,6 @@ const ( // use when describing repo ObjectStorePathOption = "objectStorePath" - // Kopia profile option keys // DataStoreGeneralContentCacheSizeMBKey is the key to pass content cache size for general command workloads DataStoreGeneralContentCacheSizeMBKey = "dataStoreGeneralContentCacheSize" // DataStoreGeneralMetadataCacheSizeMBKey is the key to pass metadata cache size for general command workloads