Skip to content

Commit

Permalink
Remove cyclic dependency and allow to pass Parallelism as an argument (…
Browse files Browse the repository at this point in the history 
…#1859)

* Remove cyclic dependency from package kopia

Signed-off-by: Rajat Gupta <rajat.gupta@veeam.com>

* Update Tests

Signed-off-by: Rajat Gupta <rajat.gupta@veeam.com>

* Change Export Variables to Non Export type

Signed-off-by: Rajat Gupta <rajat.gupta@veeam.com>

Signed-off-by: Rajat Gupta <rajat.gupta@veeam.com>
  • Loading branch information
@r4rajat
r4rajat committed Jan 12, 2023
1 parent 1602d1b commit e65900e
Show file tree
Hide file tree
Showing 4 changed files with 34 additions and 42 deletions.
32 changes: 27 additions & 5 deletions pkg/kopia/command/helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,26 +15,48 @@
package command

import (
"github.com/kanisterio/kanister/pkg/kopia"
"github.com/kanisterio/kanister/pkg/logsafe"
"github.com/kanisterio/kanister/pkg/utils"
)

const (
// dataStoreGeneralContentCacheSizeMBVarName is the name of the environment variable that controls
// kopia content cache size for general command workloads
dataStoreGeneralContentCacheSizeMBVarName = "DATA_STORE_GENERAL_CONTENT_CACHE_SIZE_MB"
// defaultDataStoreGeneralMetadataCacheSizeMB is the default metadata cache size for general command workloads
defaultDataStoreGeneralMetadataCacheSizeMB = 500
// dataStoreGeneralMetadataCacheSizeMBVarName is the name of the environment variable that controls
// kopia metadata cache size for general command workloads
dataStoreGeneralMetadataCacheSizeMBVarName = "DATA_STORE_GENERAL_METADATA_CACHE_SIZE_MB"
// defaultDataStoreRestoreContentCacheSizeMB is the default content cache size for restore workloads
defaultDataStoreRestoreContentCacheSizeMB = 500
// defaultDataStoreGeneralContentCacheSizeMB is the default content cache size for general command workloads
defaultDataStoreGeneralContentCacheSizeMB = 0
// dataStoreRestoreContentCacheSizeMBVarName is the name of the environment variable that controls
// kopia content cache size for restore workloads
dataStoreRestoreContentCacheSizeMBVarName = "DATA_STORE_RESTORE_CONTENT_CACHE_SIZE_MB"
// defaultDataStoreRestoreMetadataCacheSizeMB is the default metadata cache size for restore workloads
defaultDataStoreRestoreMetadataCacheSizeMB = 500
// dataStoreRestoreMetadataCacheSizeMBVarName is the name of the environment variable that controls
// kopia metadata cache size for restore workloads
dataStoreRestoreMetadataCacheSizeMBVarName = "DATA_STORE_RESTORE_METADATA_CACHE_SIZE_MB"
)

type policyChanges map[string]string

// GetCacheSizeSettingsForSnapshot returns the feature setting cache size values to be used
// for initializing repositories that will be performing general command workloads that benefit from
// cacheing metadata only.
func GetCacheSizeSettingsForSnapshot() (contentCacheMB, metadataCacheMB int) {
return utils.GetEnvAsIntOrDefault(kopia.DataStoreGeneralContentCacheSizeMBVarName, kopia.DefaultDataStoreGeneralContentCacheSizeMB),
utils.GetEnvAsIntOrDefault(kopia.DataStoreGeneralMetadataCacheSizeMBVarName, kopia.DefaultDataStoreGeneralMetadataCacheSizeMB)
return utils.GetEnvAsIntOrDefault(dataStoreGeneralContentCacheSizeMBVarName, defaultDataStoreGeneralContentCacheSizeMB),
utils.GetEnvAsIntOrDefault(dataStoreGeneralMetadataCacheSizeMBVarName, defaultDataStoreGeneralMetadataCacheSizeMB)
}

// GetCacheSizeSettingsForRestore returns the feature setting cache size values to be used
// for initializing repositories that will be performing restore workloads
func GetCacheSizeSettingsForRestore() (contentCacheMB, metadataCacheMB int) {
return utils.GetEnvAsIntOrDefault(kopia.DataStoreRestoreContentCacheSizeMBVarName, kopia.DefaultDataStoreRestoreContentCacheSizeMB),
utils.GetEnvAsIntOrDefault(kopia.DataStoreRestoreMetadataCacheSizeMBVarName, kopia.DefaultDataStoreRestoreMetadataCacheSizeMB)
return utils.GetEnvAsIntOrDefault(dataStoreRestoreContentCacheSizeMBVarName, defaultDataStoreRestoreContentCacheSizeMB),
utils.GetEnvAsIntOrDefault(dataStoreRestoreMetadataCacheSizeMBVarName, defaultDataStoreRestoreMetadataCacheSizeMB)
}

type GeneralCommandArgs struct {
Expand Down
9 changes: 5 additions & 4 deletions pkg/kopia/command/snapshot.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,24 +18,25 @@ import (
"strconv"
"time"

"github.com/kanisterio/kanister/pkg/kopia"
"github.com/kanisterio/kanister/pkg/utils"
)

const (
requireLogLevelInfo = true
requireLogLevelInfo = true
manifestTypeSnapshotFilter = "type:snapshot"
)

type SnapshotCreateCommandArgs struct {
*CommandArgs
PathToBackup string
Tags []string
ProgressUpdateInterval time.Duration
Parallelism int
}

// SnapshotCreate returns the kopia command for creation of a snapshot
func SnapshotCreate(cmdArgs SnapshotCreateCommandArgs) []string {
parallelismStr := strconv.Itoa(utils.GetEnvAsIntOrDefault(kopia.DataStoreParallelUploadVarName, kopia.DefaultDataStoreParallelUpload))
parallelismStr := strconv.Itoa(cmdArgs.Parallelism)
args := commonArgs(cmdArgs.CommandArgs, requireLogLevelInfo)
args = args.AppendLoggable(snapshotSubCommand, createSubCommand, cmdArgs.PathToBackup, jsonFlag)
args = args.AppendLoggableKV(parallelFlag, parallelismStr)
Expand Down Expand Up @@ -145,7 +146,7 @@ type SnapListAllWithSnapIDsCommandArgs struct {
func SnapListAllWithSnapIDs(cmdArgs SnapListAllWithSnapIDsCommandArgs) []string {
args := commonArgs(cmdArgs.CommandArgs, false)
args = args.AppendLoggable(manifestSubCommand, listSubCommand, jsonFlag)
args = args.AppendLoggableKV(filterFlag, kopia.ManifestTypeSnapshotFilter)
args = args.AppendLoggableKV(filterFlag, manifestTypeSnapshotFilter)

return stringSliceCommand(args)
}
Expand Down
2 changes: 2 additions & 0 deletions pkg/kopia/command/snapshot_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ func (kSnapshot *KopiaSnapshotTestSuite) TestSnapshotCommands(c *C) {
CommandArgs: commandArgs,
PathToBackup: "path/to/backup",
ProgressUpdateInterval: 0,
Parallelism: 8,
}
return SnapshotCreate(args)
},
Expand All @@ -53,6 +54,7 @@ func (kSnapshot *KopiaSnapshotTestSuite) TestSnapshotCommands(c *C) {
CommandArgs: commandArgs,
PathToBackup: "path/to/backup",
ProgressUpdateInterval: 1*time.Minute + 35*time.Second,
Parallelism: 8,
}
return SnapshotCreate(args)
},
Expand Down
33 changes: 0 additions & 33 deletions pkg/kopia/const.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,37 +20,4 @@ const (

// DefaultClientCacheDirectory is the directory where kopia content cache is created
DefaultClientCacheDirectory = "/tmp/kopia-cache"

// DefaultDataStoreGeneralContentCacheSizeMB is the default content cache size for general command workloads
DefaultDataStoreGeneralContentCacheSizeMB = 0
// DataStoreGeneralContentCacheSizeMBVarName is the name of the environment variable that controls
// kopia content cache size for general command workloads
DataStoreGeneralContentCacheSizeMBVarName = "DATA_STORE_GENERAL_CONTENT_CACHE_SIZE_MB"

// DefaultDataStoreGeneralMetadataCacheSizeMB is the default metadata cache size for general command workloads
DefaultDataStoreGeneralMetadataCacheSizeMB = 500
// DataStoreGeneralMetadataCacheSizeMBVarName is the name of the environment variable that controls
// kopia metadata cache size for general command workloads
DataStoreGeneralMetadataCacheSizeMBVarName = "DATA_STORE_GENERAL_METADATA_CACHE_SIZE_MB"

// DefaultDataStoreRestoreContentCacheSizeMB is the default content cache size for restore workloads
DefaultDataStoreRestoreContentCacheSizeMB = 500
// DataStoreRestoreContentCacheSizeMBVarName is the name of the environment variable that controls
// kopia content cache size for restore workloads
DataStoreRestoreContentCacheSizeMBVarName = "DATA_STORE_RESTORE_CONTENT_CACHE_SIZE_MB"

// DefaultDataStoreRestoreMetadataCacheSizeMB is the default metadata cache size for restore workloads
DefaultDataStoreRestoreMetadataCacheSizeMB = 500
// DataStoreRestoreMetadataCacheSizeMBVarName is the name of the environment variable that controls
// kopia metadata cache size for restore workloads
DataStoreRestoreMetadataCacheSizeMBVarName = "DATA_STORE_RESTORE_METADATA_CACHE_SIZE_MB"

// DefaultDataStoreParallelUpload is the default value for data store parallelism
DefaultDataStoreParallelUpload = 8

// DataStoreParallelUploadVarName is the name of the environment variable that controls
// kopia parallelism during snapshot create commands
DataStoreParallelUploadVarName = "DATA_STORE_PARALLEL_UPLOAD"

ManifestTypeSnapshotFilter = "type:snapshot"
)

0 comments on commit e65900e

Please sign in to comment.