Add Kopia repository create command (#2661)

* Add safecli dependency * add new flag implementations based on the safecli package for the Kopia CLI * apply go fmt Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Add common Kopia args and flags * Add Kopia storage core flags * Add kopia filesystem storage flags * cleanup storage tests * Add kopia GCS storage flags * add gcs flag tests * Add kopia azure storage flags * Add kopia s3 and s3 compliant storage flags * Use test.FlagSuite for storage tests * Fix typo * Add kopia CLI repository connect command * Fix Apply and test.Suit Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Remove variadic args for Common and Cache flags Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * pkg/kopia/cli/internal/flag is implemented in the safecli@v0.0.4 now Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Add pkg/kopia/cli package Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * go mod tidy Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Add Kopia storage helpers Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Implement Kopia storage Filesystem opts Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Add (c) headers Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Remove unused error Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Reorganize imports Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Add Kopia GCS storage opts Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Reorganize imports Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Add Kopia Azure storage opts Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Fix gcs test Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Add Kopia S3 and S3 compliant storage opts Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Add Kopia S3 and S3 compliant storage opts Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Cleanup tests Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Add Kopia repository create command Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Cleanup Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Reorganize tests Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Convert common flags from vars to funcs Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Add safecli dependency * add new flag implementations based on the safecli package for the Kopia CLI * apply go fmt Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Fix Apply and test.Suit Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * pkg/kopia/cli/internal/flag is implemented in the safecli@v0.0.4 now Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Add pkg/kopia/cli package Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * go mod tidy Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Update safecli to v0.0.5 Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Update safecli to v0.0.6 Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Fix tests Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Add Location.IsPointInTypeSupported Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Add tests for Location.IsPointInTypeSupported Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Fix s3 options Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Fix s3 options Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Fix options to return errors for empty args Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Fix options to return errors for empty args Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Fix options to return errors for empty args Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Support empty prefix Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Support empty prefix Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Support empty prefix Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Support empty prefix Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Support empty argument for hostname and username options Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Fix formatting Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * organize imports Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * organize imports Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Fix s3 tests Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> --------- Signed-off-by: pavel.larkin <pavel.larkin@veeam.com>
kanisterio · Mar 5, 2024 · e6c5cb5 · e6c5cb5
1 parent 84a1869
commit e6c5cb5
Show file tree

Hide file tree

Showing 7 changed files with 645 additions and 0 deletions.
diff --git a/pkg/kopia/cli/internal/kopia.go b/pkg/kopia/cli/internal/kopia.go
@@ -0,0 +1,29 @@
+// Copyright 2024 The Kanister Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package internal
+
+import (
+	"github.com/kanisterio/safecli"
+	"github.com/kanisterio/safecli/command"
+)
+
+const (
+	kopiaBinName = "kopia"
+)
+
+// NewKopiaCommand creates a new safecli.Builder for the kopia command.
+func NewKopiaCommand(args ...command.Applier) (*safecli.Builder, error) {
+	return command.New(kopiaBinName, args...)
+}
diff --git a/pkg/kopia/cli/internal/test/command_suite.go b/pkg/kopia/cli/internal/test/command_suite.go
@@ -0,0 +1,96 @@
+package test
+
+import (
+	"github.com/kanisterio/safecli"
+	"github.com/kanisterio/safecli/test"
+	"github.com/pkg/errors"
+	"gopkg.in/check.v1"
+)
+
+// CommandTest defines a single test for a command.
+type CommandTest struct {
+	// Name of the test. (required)
+	Name string
+
+	// Command to test. (required)
+	Command func() (*safecli.Builder, error)
+
+	// Expected CLI arguments. (optional)
+	ExpectedCLI []string
+
+	// Expected log output. (optional)
+	// if empty, it will be set to ExpectedCLI joined with space.
+	// if empty and ExpectedCLI is empty, it will be ignored.
+	ExpectedLog string
+
+	// Expected error. (optional)
+	// If nil, no error is expected and
+	// ExpectedCLI and ExpectedLog are checked.
+	ExpectedErr error
+}
+
+// CheckCommentString implements check.CommentInterface
+func (t *CommandTest) CheckCommentString() string {
+	return t.Name
+}
+
+// setDefaultExpectedLog sets the default value for ExpectedLog based on ExpectedCLI.
+func (t *CommandTest) setDefaultExpectedLog() {
+	if len(t.ExpectedLog) == 0 && len(t.ExpectedCLI) > 0 {
+		t.ExpectedLog = test.RedactCLI(t.ExpectedCLI)
+	}
+}
+
+// assertNoError makes sure there is no error.
+func (t *CommandTest) assertNoError(c *check.C, err error) {
+	c.Assert(err, check.IsNil, t)
+}
+
+// assertError checks the error against ExpectedErr.
+func (t *CommandTest) assertError(c *check.C, err error) {
+	actualErr := errors.Cause(err)
+	c.Assert(actualErr, check.Equals, t.ExpectedErr, t)
+}
+
+// assertCLI asserts the builder's CLI output against ExpectedCLI.
+func (t *CommandTest) assertCLI(c *check.C, b *safecli.Builder) {
+	if t.ExpectedCLI != nil {
+		c.Check(b.Build(), check.DeepEquals, t.ExpectedCLI, t)
+	}
+}
+
+// assertLog asserts the builder's log output against ExpectedLog.
+func (t *CommandTest) assertLog(c *check.C, b *safecli.Builder) {
+	if t.ExpectedCLI != nil {
+		t.setDefaultExpectedLog()
+		c.Check(b.String(), check.Equals, t.ExpectedLog, t)
+	}
+}
+
+func (t *CommandTest) Test(c *check.C) {
+	cmd, err := t.Command()
+	if t.ExpectedErr == nil {
+		t.assertNoError(c, err)
+	} else {
+		t.assertError(c, err)
+	}
+	t.assertCLI(c, cmd)
+	t.assertLog(c, cmd)
+}
+
+// CommandSuite defines a test suite for commands.
+type CommandSuite struct {
+	Commands []CommandTest
+}
+
+// TestCommands runs all tests in the suite.
+func (s *CommandSuite) TestCommands(c *check.C) {
+	for _, cmd := range s.Commands {
+		cmd.Test(c)
+	}
+}
+
+// NewCommandSuite creates a new CommandSuite.
+func NewCommandSuite(commands []CommandTest) *CommandSuite {
+	return &CommandSuite{Commands: commands}
+}
diff --git a/pkg/kopia/cli/repository/data_test.go b/pkg/kopia/cli/repository/data_test.go
@@ -0,0 +1,67 @@
+package repository
+
+import (
+	"time"
+
+	"github.com/kanisterio/kanister/pkg/kopia/cli/args"
+	"github.com/kanisterio/kanister/pkg/kopia/cli/internal"
+	rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver"
+)
+
+var (
+	common = args.Common{
+		RepoPassword:   "encr-key",
+		ConfigFilePath: "path/kopia.config",
+		LogDirectory:   "cache/log",
+	}
+
+	cache = args.Cache{
+		CacheDirectory:           "/tmp/cache.dir",
+		ContentCacheSizeLimitMB:  0,
+		MetadataCacheSizeLimitMB: 0,
+	}
+)
+
+var (
+	retentionMode   = "Locked"
+	retentionPeriod = 15 * time.Minute
+
+	locFS = internal.Location{
+		rs.TypeKey:   []byte("filestore"),
+		rs.PrefixKey: []byte("test-prefix"),
+	}
+
+	locAzure = internal.Location{
+		rs.TypeKey:   []byte("azure"),
+		rs.BucketKey: []byte("test-bucket"),
+		rs.PrefixKey: []byte("test-prefix"),
+	}
+
+	locGCS = internal.Location{
+		rs.TypeKey:   []byte("gcs"),
+		rs.BucketKey: []byte("test-bucket"),
+		rs.PrefixKey: []byte("test-prefix"),
+	}
+
+	locS3 = internal.Location{
+		rs.TypeKey:          []byte("s3"),
+		rs.EndpointKey:      []byte("test-endpoint"),
+		rs.RegionKey:        []byte("test-region"),
+		rs.BucketKey:        []byte("test-bucket"),
+		rs.PrefixKey:        []byte("test-prefix"),
+		rs.SkipSSLVerifyKey: []byte("false"),
+	}
+
+	locS3Compliant = internal.Location{
+		rs.TypeKey:          []byte("s3Compliant"),
+		rs.EndpointKey:      []byte("test-endpoint"),
+		rs.RegionKey:        []byte("test-region"),
+		rs.BucketKey:        []byte("test-bucket"),
+		rs.PrefixKey:        []byte("test-prefix"),
+		rs.SkipSSLVerifyKey: []byte("false"),
+	}
+
+	locFTP = internal.Location{
+		rs.TypeKey: []byte("ftp"),
+	}
+)
diff --git a/pkg/kopia/cli/repository/opts.go b/pkg/kopia/cli/repository/opts.go
@@ -0,0 +1,91 @@
+// Copyright 2024 The Kanister Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package repository
+
+import (
+	"time"
+
+	"github.com/kanisterio/safecli/command"
+	"github.com/pkg/errors"
+
+	"github.com/kanisterio/kanister/pkg/kopia/cli"
+	"github.com/kanisterio/kanister/pkg/kopia/cli/internal"
+	"github.com/kanisterio/kanister/pkg/kopia/cli/repository/storage/azure"
+	"github.com/kanisterio/kanister/pkg/kopia/cli/repository/storage/fs"
+	"github.com/kanisterio/kanister/pkg/kopia/cli/repository/storage/gcs"
+	"github.com/kanisterio/kanister/pkg/kopia/cli/repository/storage/s3"
+	"github.com/kanisterio/kanister/pkg/log"
+	rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver"
+)
+
+var (
+	cmdRepository = command.NewArgument("repository")
+
+	subcmdCreate = command.NewArgument("create")
+)
+
+// optHostname creates a new option for the hostname of the repository.
+// If the hostname is empty, the hostname option is not set.
+func optHostname(h string) command.Applier {
+	if h == "" {
+		return command.NewNoopArgument()
+	}
+	return command.NewOptionWithArgument("--override-hostname", h)
+}
+
+// optUsername creates a new option for the username of the repository.
+// If the username is empty, the username option is not set.
+func optUsername(u string) command.Applier {
+	if u == "" {
+		return command.NewNoopArgument()
+	}
+	return command.NewOptionWithArgument("--override-username", u)
+}
+
+// optBlobRetention creates new blob retention options with a given mode and period.
+// If mode is empty, the retention is disabled.
+func optBlobRetention(mode string, period time.Duration) command.Applier {
+	if mode == "" {
+		return command.NewNoopArgument()
+	}
+	return command.NewArguments(
+		command.NewOptionWithArgument("--retention-mode", mode),
+		command.NewOptionWithArgument("--retention-period", period.String()),
+	)
+}
+
+type storageBuilder func(internal.Location, string, log.Logger) command.Applier
+
+var storageBuilders = map[rs.LocType]storageBuilder{
+	rs.LocTypeFilestore:   fs.New,
+	rs.LocTypeAzure:       azure.New,
+	rs.LocTypeS3:          s3.New,
+	rs.LocTypes3Compliant: s3.New,
+	rs.LocTypeGCS:         gcs.New,
+}
+
+// optStorage creates a list of options for the specified storage location.
+func optStorage(l internal.Location, repoPathPrefix string, logger log.Logger) command.Applier {
+	sb := storageBuilders[l.Type()]
+	if sb == nil {
+		return errUnsupportedStorageType(l.Type())
+	}
+	return sb(l, repoPathPrefix, logger)
+}
+
+func errUnsupportedStorageType(t rs.LocType) command.Applier {
+	err := errors.Wrapf(cli.ErrUnsupportedStorage, "storage location: %v", t)
+	return command.NewErrorArgument(err)
+}
diff --git a/pkg/kopia/cli/repository/opts_test.go b/pkg/kopia/cli/repository/opts_test.go
@@ -0,0 +1,79 @@
+// Copyright 2024 The Kanister Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package repository
+
+import (
+	"testing"
+
+	"github.com/kanisterio/safecli/command"
+	"github.com/kanisterio/safecli/test"
+	"gopkg.in/check.v1"
+
+	"github.com/kanisterio/kanister/pkg/kopia/cli"
+)
+
+func TestRepositoryOptions(t *testing.T) { check.TestingT(t) }
+
+var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{
+	{
+		Name: "optHostname",
+		Argument: command.NewArguments(
+			optHostname("host"),
+			optHostname(""), // no output
+		),
+		ExpectedCLI: []string{"cmd", "--override-hostname=host"},
+	},
+	{
+		Name: "optUsername",
+		Argument: command.NewArguments(
+			optUsername("user"),
+			optUsername(""), // no output
+		),
+		ExpectedCLI: []string{"cmd", "--override-username=user"},
+	},
+	{
+		Name: "optBlobRetention",
+		Argument: command.NewArguments(
+			optBlobRetention(retentionMode, retentionPeriod),
+			optBlobRetention("", 0), // no output
+		),
+		ExpectedCLI: []string{"cmd", "--retention-mode=Locked", "--retention-period=15m0s"},
+	},
+	{
+		Name:        "optStorage FS",
+		Argument:    optStorage(locFS, "repoPathPrefix", nil),
+		ExpectedCLI: []string{"cmd", "filesystem", "--path=/mnt/data/test-prefix/repoPathPrefix/"},
+	},
+	{
+		Name:        "optStorage Azure",
+		Argument:    optStorage(locAzure, "repoPathPrefix", nil),
+		ExpectedCLI: []string{"cmd", "azure", "--container=test-bucket", "--prefix=test-prefix/repoPathPrefix/"},
+	},
+	{
+		Name:        "optStorage S3",
+		Argument:    optStorage(locS3, "repoPathPrefix", nil),
+		ExpectedCLI: []string{"cmd", "s3", "--region=test-region", "--bucket=test-bucket", "--endpoint=test-endpoint", "--prefix=test-prefix/repoPathPrefix/"},
+	},
+	{
+		Name:        "optStorage S3Compliant",
+		Argument:    optStorage(locS3Compliant, "repoPathPrefix", nil),
+		ExpectedCLI: []string{"cmd", "s3", "--region=test-region", "--bucket=test-bucket", "--endpoint=test-endpoint", "--prefix=test-prefix/repoPathPrefix/"},
+	},
+	{
+		Name:        "optStorage FTP Unsupported",
+		Argument:    optStorage(locFTP, "repoPathPrefix", nil),
+		ExpectedErr: cli.ErrUnsupportedStorage,
+	},
+}})