Add Kopia s3 and s3 compliant storage flags (#2659)

* Add safecli dependency * add new flag implementations based on the safecli package for the Kopia CLI * apply go fmt Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Add common Kopia args and flags * Add Kopia storage core flags * Add kopia filesystem storage flags * cleanup storage tests * Add kopia GCS storage flags * add gcs flag tests * Add kopia azure storage flags * Add kopia s3 and s3 compliant storage flags * Fix Apply and test.Suit Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Remove variadic args for Common and Cache flags Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * pkg/kopia/cli/internal/flag is implemented in the safecli@v0.0.4 now Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Add pkg/kopia/cli package Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * go mod tidy Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Add Kopia storage helpers Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Implement Kopia storage Filesystem opts Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Add (c) headers Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Remove unused error Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Reorganize imports Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Add Kopia GCS storage opts Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Reorganize imports Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Add Kopia Azure storage opts Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Fix gcs test Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Add Kopia S3 and S3 compliant storage opts Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Add Kopia S3 and S3 compliant storage opts Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Cleanup tests Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Convert common flags from vars to funcs Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Add safecli dependency * add new flag implementations based on the safecli package for the Kopia CLI * apply go fmt Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Fix Apply and test.Suit Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * pkg/kopia/cli/internal/flag is implemented in the safecli@v0.0.4 now Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Add pkg/kopia/cli package Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * go mod tidy Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Update safecli to v0.0.5 Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Update safecli to v0.0.6 Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Fix tests Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Add Location.IsPointInTypeSupported Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Add tests for Location.IsPointInTypeSupported Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Fix s3 options Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Fix s3 options Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Fix options to return errors for empty args Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Fix options to return errors for empty args Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Fix options to return errors for empty args Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Support empty prefix Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Support empty prefix Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Support empty prefix Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Support empty prefix Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Fix formatting Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * organize imports Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * organize imports Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> * Fix s3 tests Signed-off-by: pavel.larkin <pavel.larkin@veeam.com> --------- Signed-off-by: pavel.larkin <pavel.larkin@veeam.com>
kanisterio · Mar 5, 2024 · 84a1869 · 84a1869
1 parent dfca3a5
commit 84a1869
Show file tree

Hide file tree

Showing 6 changed files with 470 additions and 0 deletions.
diff --git a/pkg/kopia/cli/internal/log/log.go b/pkg/kopia/cli/internal/log/log.go
@@ -17,6 +17,7 @@ package storage
 import (
 	"context"
 	"io"
+	"regexp"
 
 	"github.com/kanisterio/kanister/pkg/field"
 	"github.com/kanisterio/kanister/pkg/log"
@@ -43,3 +44,31 @@ func (NopLogger) WithContext(ctx context.Context) log.Logger {
 func (NopLogger) WithError(err error) log.Logger {
 	return &NopLogger{}
 }
+
+// StringLogger is a logger that stores log messages in a slice of strings.
+type StringLogger []string
+
+func (l *StringLogger) Print(msg string, fields ...field.M) {
+	*l = append(*l, msg)
+}
+
+func (l *StringLogger) PrintTo(w io.Writer, msg string, fields ...field.M) {
+	*l = append(*l, msg)
+}
+
+func (l *StringLogger) WithContext(ctx context.Context) log.Logger {
+	return l
+}
+
+func (l *StringLogger) WithError(err error) log.Logger {
+	return l
+}
+
+func (l *StringLogger) MatchString(pattern string) bool {
+	for _, line := range *l {
+		if found, _ := regexp.MatchString(pattern, line); found {
+			return true
+		}
+	}
+	return false
+}
diff --git a/pkg/kopia/cli/internal/test/arg_suit.go b/pkg/kopia/cli/internal/test/arg_suit.go
@@ -0,0 +1,81 @@
+// Copyright 2024 The Kanister Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package test
+
+import (
+	"github.com/kanisterio/safecli/test"
+	"gopkg.in/check.v1"
+
+	intlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log"
+	"github.com/kanisterio/kanister/pkg/log"
+)
+
+// ArgumentTest extends test.ArgumentTest to include logger tests.
+type ArgumentTest struct {
+	test.ArgumentTest
+
+	Logger      log.Logger // Logger is the logger to use for the test. (optional)
+	LoggerRegex []string   // LoggerRegex is a list of regexs to match against the log output. (optional)
+}
+
+// Test runs the test with the given command and checks the log output.
+func (t *ArgumentTest) Test(c *check.C, cmd string) {
+	t.ArgumentTest.Test(c, cmd)
+	t.assertLog(c)
+}
+
+// assertLog checks the log output against the expected regexs.
+func (t *ArgumentTest) assertLog(c *check.C) {
+	if t.Logger == nil {
+		if len(t.LoggerRegex) > 0 {
+			c.Fatalf("t.Logger is nil but t.LoggerRegex is %#v", t.LoggerRegex)
+		}
+		return
+	}
+
+	log, ok := t.Logger.(*intlog.StringLogger)
+	if !ok {
+		c.Fatalf("t.Logger is not *intlog.StringLogger")
+	}
+	if t.isEmptyLogExpected() {
+		cmtLog := check.Commentf("FAIL: log should be empty but got %#v", log)
+		c.Assert(len([]string(*log)), check.Equals, 0, cmtLog)
+		return
+	}
+
+	// Check each regex.
+	for _, regex := range t.LoggerRegex {
+		cmtLog := check.Commentf("FAIL: %v\nlog %#v expected to match %#v", t.ArgumentTest.Name, log, regex)
+		c.Assert(log.MatchString(regex), check.Equals, true, cmtLog)
+	}
+}
+
+// isEmptyLogExpected returns true if the test expects an empty log.
+func (t *ArgumentTest) isEmptyLogExpected() bool {
+	return len(t.LoggerRegex) == 1 && t.LoggerRegex[0] == ""
+}
+
+// ArgumentSuite defines a suite of tests for a single ArgumentTest.
+type ArgumentSuite struct {
+	Cmd       string         // Cmd appends to the safecli.Builder before test if not empty.
+	Arguments []ArgumentTest // Tests to run.
+}
+
+// TestArguments runs all tests in the suite.
+func (s *ArgumentSuite) TestArguments(c *check.C) {
+	for _, arg := range s.Arguments {
+		arg.Test(c, s.Cmd)
+	}
+}
diff --git a/pkg/kopia/cli/repository/storage/s3/s3.go b/pkg/kopia/cli/repository/storage/s3/s3.go
@@ -0,0 +1,63 @@
+// Copyright 2024 The Kanister Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+	"strings"
+
+	"github.com/kanisterio/safecli/command"
+
+	"github.com/kanisterio/kanister/pkg/kopia/cli/internal"
+	intlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log"
+	"github.com/kanisterio/kanister/pkg/log"
+)
+
+// New creates a new subcommand for the S3 storage.
+func New(location internal.Location, repoPathPrefix string, logger log.Logger) command.Applier {
+	if logger == nil {
+		logger = intlog.NopLogger{}
+	}
+	endpoint := resolveS3Endpoint(location.Endpoint(), logger)
+	prefix := internal.GenerateFullRepoPath(location.Prefix(), repoPathPrefix)
+	return command.NewArguments(subcmdS3,
+		optRegion(location.Region()),
+		optBucket(location.BucketName()),
+		optEndpoint(endpoint),
+		optPrefix(prefix),
+		optDisableTLS(location.IsInsecureEndpoint()),
+		optDisableTLSVerify(location.HasSkipSSLVerify()),
+	)
+}
+
+// resolveS3Endpoint removes the trailing slash and
+// protocol from provided endpoint and
+// returns the absolute endpoint string.
+func resolveS3Endpoint(endpoint string, logger log.Logger) string {
+	if endpoint == "" {
+		return ""
+	}
+
+	if strings.HasSuffix(endpoint, "/") {
+		logger.Print("Removing trailing slashes from the endpoint")
+		endpoint = strings.TrimRight(endpoint, "/")
+	}
+
+	sp := strings.SplitN(endpoint, "://", 2)
+	if len(sp) > 1 {
+		logger.Print("Removing leading protocol from the endpoint")
+	}
+
+	return sp[len(sp)-1]
+}
diff --git a/pkg/kopia/cli/repository/storage/s3/s3_opts.go b/pkg/kopia/cli/repository/storage/s3/s3_opts.go
@@ -0,0 +1,68 @@
+// Copyright 2024 The Kanister Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+	"github.com/kanisterio/safecli/command"
+
+	"github.com/kanisterio/kanister/pkg/kopia/cli"
+)
+
+var (
+	subcmdS3 = command.NewArgument("s3")
+)
+
+// optBucket creates a new bucket option with a given name.
+// If the name is empty, it returns ErrInvalidBucketName.
+func optBucket(name string) command.Applier {
+	if name == "" {
+		return command.NewErrorArgument(cli.ErrInvalidBucketName)
+	}
+	return command.NewOptionWithArgument("--bucket", name)
+}
+
+// optEndpoint creates a new endpoint option with a given endpoint.
+// If the endpoint is empty, the endpoint option is not set.
+func optEndpoint(endpoint string) command.Applier {
+	if endpoint == "" {
+		return command.NewNoopArgument()
+	}
+	return command.NewOptionWithArgument("--endpoint", endpoint)
+}
+
+// optPrefix creates a new prefix option with a given prefix.
+// If the prefix is empty, the prefix option is not set.
+func optPrefix(prefix string) command.Applier {
+	return command.NewOptionWithArgument("--prefix", prefix)
+}
+
+// optRegion creates a new region option with a given region.
+// If the region is empty, the region option is not set.
+func optRegion(region string) command.Applier {
+	if region == "" {
+		return command.NewNoopArgument()
+	}
+	return command.NewOptionWithArgument("--region", region)
+}
+
+// optDisableTLS creates a new disable TLS option with a given value.
+func optDisableTLS(disable bool) command.Applier {
+	return command.NewOption("--disable-tls", disable)
+}
+
+// optDisableTLSVerify creates a new disable TLS verification option with a given value.
+func optDisableTLSVerify(disable bool) command.Applier {
+	return command.NewOption("--disable-tls-verification", disable)
+}
diff --git a/pkg/kopia/cli/repository/storage/s3/s3_opts_test.go b/pkg/kopia/cli/repository/storage/s3/s3_opts_test.go
@@ -0,0 +1,65 @@
+// Copyright 2024 The Kanister Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3
+
+import (
+	"testing"
+
+	"github.com/kanisterio/safecli/command"
+	"github.com/kanisterio/safecli/test"
+	"gopkg.in/check.v1"
+
+	"github.com/kanisterio/kanister/pkg/kopia/cli"
+)
+
+func TestS3Options(t *testing.T) { check.TestingT(t) }
+
+var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{
+	{
+		Name:        "optRegion",
+		Argument:    command.NewArguments(optRegion("region"), optRegion("")),
+		ExpectedCLI: []string{"cmd", "--region=region"},
+	},
+	{
+		Name:        "optBucket with bucketname should return option",
+		Argument:    optBucket("bucketname"),
+		ExpectedCLI: []string{"cmd", "--bucket=bucketname"},
+	},
+	{
+		Name:        "optBucket with empty bucketname should return error",
+		Argument:    optBucket(""),
+		ExpectedErr: cli.ErrInvalidBucketName,
+	},
+	{
+		Name:        "optEndpoint",
+		Argument:    command.NewArguments(optEndpoint("endpoint"), optEndpoint("")),
+		ExpectedCLI: []string{"cmd", "--endpoint=endpoint"},
+	},
+	{
+		Name:        "optPrefix",
+		Argument:    command.NewArguments(optPrefix("prefix"), optPrefix("")),
+		ExpectedCLI: []string{"cmd", "--prefix=prefix", "--prefix="},
+	},
+	{
+		Name:        "optDisableTLS",
+		Argument:    command.NewArguments(optDisableTLS(true), optDisableTLS(false)),
+		ExpectedCLI: []string{"cmd", "--disable-tls"},
+	},
+	{
+		Name:        "optDisableTLSVerify",
+		Argument:    command.NewArguments(optDisableTLSVerify(true), optDisableTLSVerify(false)),
+		ExpectedCLI: []string{"cmd", "--disable-tls-verification"},
+	},
+}})