Skip to content

Commit

Permalink
Modify kopia utilities to use new kopia cache flags (#2393)
Browse files Browse the repository at this point in the history 
* add cache hard limit flags

Signed-off-by: Amruta Kale <amruta.kale@veeam.com>

* add cache structure

Signed-off-by: Amruta Kale <amruta.kale@veeam.com>

* remove test changes

Signed-off-by: Amruta Kale <amruta.kale@veeam.com>

* remove soft limit fields from the struct as we are not setting it

Signed-off-by: Amruta Kale <amruta.kale@veeam.com>

* move the unused function to child PR where the function is already used

Signed-off-by: Amruta Kale <amruta.kale@veeam.com>

* add copyright headers to cache.go

Signed-off-by: Amruta Kale <amruta.kale@veeam.com>

* modify repository create/connect command for new flags

Signed-off-by: Amruta Kale <amruta.kale@veeam.com>

* rename CacheCommandArgs to CacheArgs

Signed-off-by: Amruta Kale <amruta.kale@veeam.com>

* refactor tests according to changes in repository connect command

Signed-off-by: Amruta Kale <amruta.kale@veeam.com>

* fix data mover and repository controller tests according to changes in repository create and connect command

Signed-off-by: Amruta Kale <amruta.kale@veeam.com>

* add kopiaCacheArgs function

Signed-off-by: Amruta Kale <amruta.kale@veeam.com>

* changes in repository server connect SDK command

Signed-off-by: Amruta Kale <amruta.kale@veeam.com>

* changes to kopia client

Signed-off-by: Amruta Kale <amruta.kale@veeam.com>

* fix lint issues

Signed-off-by: Amruta Kale <amruta.kale@veeam.com>

* format comments

Signed-off-by: Amruta Kale <amruta.kale@veeam.com>

---------

Signed-off-by: Amruta Kale <amruta.kale@veeam.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
  • Loading branch information
@kale-amruta @mergify
kale-amruta and mergify[bot] authored Oct 27, 2023
1 parent 878a4c7 commit 3e93315
Show file tree
Hide file tree
Showing 11 changed files with 155 additions and 124 deletions.
12 changes: 7 additions & 5 deletions pkg/controllers/repositoryserver/repository.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,13 @@ func (h *RepoServerHandler) connectToKopiaRepository() error {
ConfigFilePath: repoConfiguration.ConfigFilePath,
LogDirectory: repoConfiguration.LogDirectory,
},
CacheDirectory: repoConfiguration.CacheDirectory,
Hostname: h.RepositoryServer.Spec.Repository.Hostname,
ContentCacheMB: *cacheSizeSettings.Content,
MetadataCacheMB: *cacheSizeSettings.Metadata,
Username: h.RepositoryServer.Spec.Repository.Username,
CacheDirectory: repoConfiguration.CacheDirectory,
Hostname: h.RepositoryServer.Spec.Repository.Hostname,
CacheArgs: command.CacheArgs{
ContentCacheLimitMB: *cacheSizeSettings.Content,
MetadataCacheLimitMB: *cacheSizeSettings.Metadata,
},
Username: h.RepositoryServer.Spec.Repository.Username,
// TODO(Amruta): Generate path for respository
RepoPathPrefix: h.RepositoryServer.Spec.Repository.RootPath,
Location: h.RepositoryServerSecrets.storage.Data,
Expand Down
16 changes: 9 additions & 7 deletions pkg/datamover/repository_server_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,13 +97,15 @@ func (rss *RepositoryServerSuite) setupKopiaRepositoryServer(c *C) {
ConfigFilePath: rss.kopiaConfigDir,
LogDirectory: rss.kopiaLogDir,
},
CacheDirectory: rss.kopiaCacheDir,
Hostname: rss.serverHost,
ContentCacheMB: contentCacheMB,
MetadataCacheMB: metadataCacheMB,
RepoPathPrefix: rss.repoPathPrefix,
Username: rss.repositoryUser,
Location: testutil.GetDefaultS3CompliantStorageLocation(),
CacheDirectory: rss.kopiaCacheDir,
Hostname: rss.serverHost,
CacheArgs: kopiacmd.CacheArgs{
ContentCacheLimitMB: contentCacheMB,
MetadataCacheLimitMB: metadataCacheMB,
},
RepoPathPrefix: rss.repoPathPrefix,
Username: rss.repositoryUser,
Location: testutil.GetDefaultS3CompliantStorageLocation(),
}
// First try to connect with Kopia Repository
c.Log("Connecting with Kopia Repository...")
Expand Down
22 changes: 12 additions & 10 deletions pkg/function/backup_data_using_kopia_server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,16 +198,18 @@ func backupDataUsingKopiaServer(
configFile, logDirectory := kankopia.CustomConfigFileAndLogDirectory(hostname)

cmd := kopiacmd.RepositoryConnectServerCommand(kopiacmd.RepositoryServerCommandArgs{
UserPassword: userPassphrase,
ConfigFilePath: configFile,
LogDirectory: logDirectory,
CacheDirectory: kopiacmd.DefaultCacheDirectory,
Hostname: hostname,
ServerURL: serverAddress,
Fingerprint: fingerprint,
Username: username,
ContentCacheMB: contentCacheMB,
MetadataCacheMB: metadataCacheMB,
UserPassword: userPassphrase,
ConfigFilePath: configFile,
LogDirectory: logDirectory,
CacheDirectory: kopiacmd.DefaultCacheDirectory,
Hostname: hostname,
ServerURL: serverAddress,
Fingerprint: fingerprint,
Username: username,
CacheArgs: kopiacmd.CacheArgs{
ContentCacheLimitMB: contentCacheMB,
MetadataCacheLimitMB: metadataCacheMB,
},
})

stdout, stderr, err := kube.Exec(cli, namespace, pod, container, cmd, nil)
Expand Down
22 changes: 12 additions & 10 deletions pkg/function/delete_data_using_kopia_server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,16 +190,18 @@ func deleteDataFromServerPodFunc(
configFile, logDirectory := kankopia.CustomConfigFileAndLogDirectory(hostname)
cmd := kopiacmd.RepositoryConnectServerCommand(
kopiacmd.RepositoryServerCommandArgs{
UserPassword: userPassphrase,
ConfigFilePath: configFile,
LogDirectory: logDirectory,
CacheDirectory: kopiacmd.DefaultCacheDirectory,
Hostname: hostname,
ServerURL: serverAddress,
Fingerprint: fingerprint,
Username: username,
ContentCacheMB: contentCacheMB,
MetadataCacheMB: metadataCacheMB,
UserPassword: userPassphrase,
ConfigFilePath: configFile,
LogDirectory: logDirectory,
CacheDirectory: kopiacmd.DefaultCacheDirectory,
Hostname: hostname,
ServerURL: serverAddress,
Fingerprint: fingerprint,
Username: username,
CacheArgs: kopiacmd.CacheArgs{
ContentCacheLimitMB: contentCacheMB,
MetadataCacheLimitMB: metadataCacheMB,
},
})

commandExecutor, err := pc.GetCommandExecutor()
Expand Down
22 changes: 12 additions & 10 deletions pkg/function/restore_data_using_kopia_server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -244,16 +244,18 @@ func restoreDataFromServerPodFunc(

cmd := kopiacmd.RepositoryConnectServerCommand(
kopiacmd.RepositoryServerCommandArgs{
UserPassword: userPassphrase,
ConfigFilePath: configFile,
LogDirectory: logDirectory,
CacheDirectory: kopiacmd.DefaultCacheDirectory,
Hostname: hostname,
ServerURL: serverAddress,
Fingerprint: fingerprint,
Username: username,
ContentCacheMB: contentCacheMB,
MetadataCacheMB: metadataCacheMB,
UserPassword: userPassphrase,
ConfigFilePath: configFile,
LogDirectory: logDirectory,
CacheDirectory: kopiacmd.DefaultCacheDirectory,
Hostname: hostname,
ServerURL: serverAddress,
Fingerprint: fingerprint,
Username: username,
CacheArgs: kopiacmd.CacheArgs{
ContentCacheLimitMB: contentCacheMB,
MetadataCacheLimitMB: metadataCacheMB,
},
})

commandExecutor, err := pc.GetCommandExecutor()
Expand Down
19 changes: 19 additions & 0 deletions pkg/kopia/command/cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,28 @@

package command

import (
"strconv"

"github.com/kanisterio/kanister/pkg/logsafe"
)

// CacheArgs has fields that can be used to set
// cache settings for different kopia repository operations
type CacheArgs struct {
ContentCacheLimitMB int
MetadataCacheLimitMB int
}

func (c CacheArgs) kopiaCacheArgs(args logsafe.Cmd, cacheDirectory string) logsafe.Cmd {
args = args.AppendLoggableKV(cacheDirectoryFlag, cacheDirectory)

// The hard limit flags for cache would be set using the env variables that
// are passed through Kanister helm settings.
// Soft limit flags would be set to the default values by kopia
// automatically in ``connectOptions.setup()`` function.
// Refer to https://github.com/kopia/kopia/blob/master/cli/command_repository_connect.go#L71
args = args.AppendLoggableKV(contentCacheSizeLimitMBFlag, strconv.Itoa(c.ContentCacheLimitMB))
args = args.AppendLoggableKV(metadataCacheSizeLimitMBFlag, strconv.Itoa(c.MetadataCacheLimitMB))
return args
}
9 changes: 0 additions & 9 deletions pkg/kopia/command/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,6 @@
package command

import (
"strconv"

"github.com/kanisterio/kanister/pkg/field"
"github.com/kanisterio/kanister/pkg/log"
"github.com/kanisterio/kanister/pkg/logsafe"
Expand Down Expand Up @@ -83,10 +81,3 @@ const (
contentCacheSizeLimitMBFlag = "--content-cache-size-limit-mb"
metadataCacheSizeLimitMBFlag = "--metadata-cache-size-limit-mb"
)

func kopiaCacheArgs(args logsafe.Cmd, cacheDirectory string, contentCacheMB, metadataCacheMB int) logsafe.Cmd {
args = args.AppendLoggableKV(cacheDirectoryFlag, cacheDirectory)
args = args.AppendLoggableKV(contentCacheSizeMBFlag, strconv.Itoa(contentCacheMB))
args = args.AppendLoggableKV(metadataCacheSizeMBFlag, strconv.Itoa(metadataCacheMB))
return args
}
40 changes: 19 additions & 21 deletions pkg/kopia/command/repository.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,12 @@ import (
// creating or connecting to a Kopia repository
type RepositoryCommandArgs struct {
*CommandArgs
CacheDirectory string
Hostname string
ContentCacheMB int
MetadataCacheMB int
Username string
RepoPathPrefix string
ReadOnly bool
CacheArgs
CacheDirectory string
Hostname string
Username string
RepoPathPrefix string
ReadOnly bool
// PITFlag is only effective if set while repository connect
PITFlag strfmt.DateTime
Location map[string][]byte
Expand All @@ -48,7 +47,7 @@ func RepositoryConnectCommand(cmdArgs RepositoryCommandArgs) ([]string, error) {
args = args.AppendLoggable(readOnlyFlag)
}

args = kopiaCacheArgs(args, cmdArgs.CacheDirectory, cmdArgs.ContentCacheMB, cmdArgs.MetadataCacheMB)
args = cmdArgs.kopiaCacheArgs(args, cmdArgs.CacheDirectory)

if cmdArgs.Hostname != "" {
args = args.AppendLoggableKV(overrideHostnameFlag, cmdArgs.Hostname)
Expand Down Expand Up @@ -78,7 +77,7 @@ func RepositoryCreateCommand(cmdArgs RepositoryCommandArgs) ([]string, error) {
args := commonArgs(cmdArgs.CommandArgs)
args = args.AppendLoggable(repositorySubCommand, createSubCommand, noCheckForUpdatesFlag)

args = kopiaCacheArgs(args, cmdArgs.CacheDirectory, cmdArgs.ContentCacheMB, cmdArgs.MetadataCacheMB)
args = cmdArgs.kopiaCacheArgs(args, cmdArgs.CacheDirectory)

if cmdArgs.Hostname != "" {
args = args.AppendLoggableKV(overrideHostnameFlag, cmdArgs.Hostname)
Expand All @@ -102,17 +101,16 @@ func RepositoryCreateCommand(cmdArgs RepositoryCommandArgs) ([]string, error) {
// RepositoryServerCommandArgs contains fields required for connecting
// to Kopia Repository API server
type RepositoryServerCommandArgs struct {
UserPassword string
ConfigFilePath string
LogDirectory string
CacheDirectory string
Hostname string
ServerURL string
Fingerprint string
Username string
ReadOnly bool
ContentCacheMB int
MetadataCacheMB int
UserPassword string
ConfigFilePath string
LogDirectory string
CacheDirectory string
Hostname string
ServerURL string
Fingerprint string
Username string
ReadOnly bool
CacheArgs
}

// RepositoryConnectServerCommand returns the kopia command for connecting to a remote
Expand All @@ -129,7 +127,7 @@ func RepositoryConnectServerCommand(cmdArgs RepositoryServerCommandArgs) []strin
args = args.AppendLoggable(readOnlyFlag)
}

args = kopiaCacheArgs(args, cmdArgs.CacheDirectory, cmdArgs.ContentCacheMB, cmdArgs.MetadataCacheMB)
args = cmdArgs.kopiaCacheArgs(args, cmdArgs.CacheDirectory)

if cmdArgs.Hostname != "" {
args = args.AppendLoggableKV(overrideHostnameFlag, cmdArgs.Hostname)
Expand Down
Loading

0 comments on commit 3e93315

Please sign in to comment.