From 84a18697366d1758ee1e9a6a67edb8434fe17d69 Mon Sep 17 00:00:00 2001 From: Pavel Larkin Date: Tue, 5 Mar 2024 13:49:32 -0800 Subject: [PATCH] Add Kopia s3 and s3 compliant storage flags (#2659) * Add safecli dependency * add new flag implementations based on the safecli package for the Kopia CLI * apply go fmt Signed-off-by: pavel.larkin * Add common Kopia args and flags * Add Kopia storage core flags * Add kopia filesystem storage flags * cleanup storage tests * Add kopia GCS storage flags * add gcs flag tests * Add kopia azure storage flags * Add kopia s3 and s3 compliant storage flags * Fix Apply and test.Suit Signed-off-by: pavel.larkin * Remove variadic args for Common and Cache flags Signed-off-by: pavel.larkin * pkg/kopia/cli/internal/flag is implemented in the safecli@v0.0.4 now Signed-off-by: pavel.larkin * Add pkg/kopia/cli package Signed-off-by: pavel.larkin * go mod tidy Signed-off-by: pavel.larkin * Add Kopia storage helpers Signed-off-by: pavel.larkin * Implement Kopia storage Filesystem opts Signed-off-by: pavel.larkin * Add (c) headers Signed-off-by: pavel.larkin * Remove unused error Signed-off-by: pavel.larkin * Reorganize imports Signed-off-by: pavel.larkin * Add Kopia GCS storage opts Signed-off-by: pavel.larkin * Reorganize imports Signed-off-by: pavel.larkin * Add Kopia Azure storage opts Signed-off-by: pavel.larkin * Fix gcs test Signed-off-by: pavel.larkin * Add Kopia S3 and S3 compliant storage opts Signed-off-by: pavel.larkin * Add Kopia S3 and S3 compliant storage opts Signed-off-by: pavel.larkin * Cleanup tests Signed-off-by: pavel.larkin * Convert common flags from vars to funcs Signed-off-by: pavel.larkin * Add safecli dependency * add new flag implementations based on the safecli package for the Kopia CLI * apply go fmt Signed-off-by: pavel.larkin * Fix Apply and test.Suit Signed-off-by: pavel.larkin * pkg/kopia/cli/internal/flag is implemented in the safecli@v0.0.4 now Signed-off-by: pavel.larkin * Add pkg/kopia/cli package Signed-off-by: pavel.larkin * go mod tidy Signed-off-by: pavel.larkin * Update safecli to v0.0.5 Signed-off-by: pavel.larkin * Update safecli to v0.0.6 Signed-off-by: pavel.larkin * Fix tests Signed-off-by: pavel.larkin * Add Location.IsPointInTypeSupported Signed-off-by: pavel.larkin * Add tests for Location.IsPointInTypeSupported Signed-off-by: pavel.larkin * Fix s3 options Signed-off-by: pavel.larkin * Fix s3 options Signed-off-by: pavel.larkin * Fix options to return errors for empty args Signed-off-by: pavel.larkin * Fix options to return errors for empty args Signed-off-by: pavel.larkin * Fix options to return errors for empty args Signed-off-by: pavel.larkin * Support empty prefix Signed-off-by: pavel.larkin * Support empty prefix Signed-off-by: pavel.larkin * Support empty prefix Signed-off-by: pavel.larkin * Support empty prefix Signed-off-by: pavel.larkin * Fix formatting Signed-off-by: pavel.larkin * organize imports Signed-off-by: pavel.larkin * organize imports Signed-off-by: pavel.larkin * Fix s3 tests Signed-off-by: pavel.larkin --------- Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/log/log.go | 29 ++++ pkg/kopia/cli/internal/test/arg_suit.go | 81 +++++++++ pkg/kopia/cli/repository/storage/s3/s3.go | 63 +++++++ .../cli/repository/storage/s3/s3_opts.go | 68 ++++++++ .../cli/repository/storage/s3/s3_opts_test.go | 65 +++++++ .../cli/repository/storage/s3/s3_test.go | 164 ++++++++++++++++++ 6 files changed, 470 insertions(+) create mode 100644 pkg/kopia/cli/internal/test/arg_suit.go create mode 100644 pkg/kopia/cli/repository/storage/s3/s3.go create mode 100644 pkg/kopia/cli/repository/storage/s3/s3_opts.go create mode 100644 pkg/kopia/cli/repository/storage/s3/s3_opts_test.go create mode 100644 pkg/kopia/cli/repository/storage/s3/s3_test.go diff --git a/pkg/kopia/cli/internal/log/log.go b/pkg/kopia/cli/internal/log/log.go index 4d2b29ffe3..e3d2ea2544 100644 --- a/pkg/kopia/cli/internal/log/log.go +++ b/pkg/kopia/cli/internal/log/log.go @@ -17,6 +17,7 @@ package storage import ( "context" "io" + "regexp" "github.com/kanisterio/kanister/pkg/field" "github.com/kanisterio/kanister/pkg/log" @@ -43,3 +44,31 @@ func (NopLogger) WithContext(ctx context.Context) log.Logger { func (NopLogger) WithError(err error) log.Logger { return &NopLogger{} } + +// StringLogger is a logger that stores log messages in a slice of strings. +type StringLogger []string + +func (l *StringLogger) Print(msg string, fields ...field.M) { + *l = append(*l, msg) +} + +func (l *StringLogger) PrintTo(w io.Writer, msg string, fields ...field.M) { + *l = append(*l, msg) +} + +func (l *StringLogger) WithContext(ctx context.Context) log.Logger { + return l +} + +func (l *StringLogger) WithError(err error) log.Logger { + return l +} + +func (l *StringLogger) MatchString(pattern string) bool { + for _, line := range *l { + if found, _ := regexp.MatchString(pattern, line); found { + return true + } + } + return false +} diff --git a/pkg/kopia/cli/internal/test/arg_suit.go b/pkg/kopia/cli/internal/test/arg_suit.go new file mode 100644 index 0000000000..21ccbf70e4 --- /dev/null +++ b/pkg/kopia/cli/internal/test/arg_suit.go @@ -0,0 +1,81 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "github.com/kanisterio/safecli/test" + "gopkg.in/check.v1" + + intlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" + "github.com/kanisterio/kanister/pkg/log" +) + +// ArgumentTest extends test.ArgumentTest to include logger tests. +type ArgumentTest struct { + test.ArgumentTest + + Logger log.Logger // Logger is the logger to use for the test. (optional) + LoggerRegex []string // LoggerRegex is a list of regexs to match against the log output. (optional) +} + +// Test runs the test with the given command and checks the log output. +func (t *ArgumentTest) Test(c *check.C, cmd string) { + t.ArgumentTest.Test(c, cmd) + t.assertLog(c) +} + +// assertLog checks the log output against the expected regexs. +func (t *ArgumentTest) assertLog(c *check.C) { + if t.Logger == nil { + if len(t.LoggerRegex) > 0 { + c.Fatalf("t.Logger is nil but t.LoggerRegex is %#v", t.LoggerRegex) + } + return + } + + log, ok := t.Logger.(*intlog.StringLogger) + if !ok { + c.Fatalf("t.Logger is not *intlog.StringLogger") + } + if t.isEmptyLogExpected() { + cmtLog := check.Commentf("FAIL: log should be empty but got %#v", log) + c.Assert(len([]string(*log)), check.Equals, 0, cmtLog) + return + } + + // Check each regex. + for _, regex := range t.LoggerRegex { + cmtLog := check.Commentf("FAIL: %v\nlog %#v expected to match %#v", t.ArgumentTest.Name, log, regex) + c.Assert(log.MatchString(regex), check.Equals, true, cmtLog) + } +} + +// isEmptyLogExpected returns true if the test expects an empty log. +func (t *ArgumentTest) isEmptyLogExpected() bool { + return len(t.LoggerRegex) == 1 && t.LoggerRegex[0] == "" +} + +// ArgumentSuite defines a suite of tests for a single ArgumentTest. +type ArgumentSuite struct { + Cmd string // Cmd appends to the safecli.Builder before test if not empty. + Arguments []ArgumentTest // Tests to run. +} + +// TestArguments runs all tests in the suite. +func (s *ArgumentSuite) TestArguments(c *check.C) { + for _, arg := range s.Arguments { + arg.Test(c, s.Cmd) + } +} diff --git a/pkg/kopia/cli/repository/storage/s3/s3.go b/pkg/kopia/cli/repository/storage/s3/s3.go new file mode 100644 index 0000000000..7a5489755b --- /dev/null +++ b/pkg/kopia/cli/repository/storage/s3/s3.go @@ -0,0 +1,63 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3 + +import ( + "strings" + + "github.com/kanisterio/safecli/command" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" + intlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" + "github.com/kanisterio/kanister/pkg/log" +) + +// New creates a new subcommand for the S3 storage. +func New(location internal.Location, repoPathPrefix string, logger log.Logger) command.Applier { + if logger == nil { + logger = intlog.NopLogger{} + } + endpoint := resolveS3Endpoint(location.Endpoint(), logger) + prefix := internal.GenerateFullRepoPath(location.Prefix(), repoPathPrefix) + return command.NewArguments(subcmdS3, + optRegion(location.Region()), + optBucket(location.BucketName()), + optEndpoint(endpoint), + optPrefix(prefix), + optDisableTLS(location.IsInsecureEndpoint()), + optDisableTLSVerify(location.HasSkipSSLVerify()), + ) +} + +// resolveS3Endpoint removes the trailing slash and +// protocol from provided endpoint and +// returns the absolute endpoint string. +func resolveS3Endpoint(endpoint string, logger log.Logger) string { + if endpoint == "" { + return "" + } + + if strings.HasSuffix(endpoint, "/") { + logger.Print("Removing trailing slashes from the endpoint") + endpoint = strings.TrimRight(endpoint, "/") + } + + sp := strings.SplitN(endpoint, "://", 2) + if len(sp) > 1 { + logger.Print("Removing leading protocol from the endpoint") + } + + return sp[len(sp)-1] +} diff --git a/pkg/kopia/cli/repository/storage/s3/s3_opts.go b/pkg/kopia/cli/repository/storage/s3/s3_opts.go new file mode 100644 index 0000000000..131c5803ad --- /dev/null +++ b/pkg/kopia/cli/repository/storage/s3/s3_opts.go @@ -0,0 +1,68 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3 + +import ( + "github.com/kanisterio/safecli/command" + + "github.com/kanisterio/kanister/pkg/kopia/cli" +) + +var ( + subcmdS3 = command.NewArgument("s3") +) + +// optBucket creates a new bucket option with a given name. +// If the name is empty, it returns ErrInvalidBucketName. +func optBucket(name string) command.Applier { + if name == "" { + return command.NewErrorArgument(cli.ErrInvalidBucketName) + } + return command.NewOptionWithArgument("--bucket", name) +} + +// optEndpoint creates a new endpoint option with a given endpoint. +// If the endpoint is empty, the endpoint option is not set. +func optEndpoint(endpoint string) command.Applier { + if endpoint == "" { + return command.NewNoopArgument() + } + return command.NewOptionWithArgument("--endpoint", endpoint) +} + +// optPrefix creates a new prefix option with a given prefix. +// If the prefix is empty, the prefix option is not set. +func optPrefix(prefix string) command.Applier { + return command.NewOptionWithArgument("--prefix", prefix) +} + +// optRegion creates a new region option with a given region. +// If the region is empty, the region option is not set. +func optRegion(region string) command.Applier { + if region == "" { + return command.NewNoopArgument() + } + return command.NewOptionWithArgument("--region", region) +} + +// optDisableTLS creates a new disable TLS option with a given value. +func optDisableTLS(disable bool) command.Applier { + return command.NewOption("--disable-tls", disable) +} + +// optDisableTLSVerify creates a new disable TLS verification option with a given value. +func optDisableTLSVerify(disable bool) command.Applier { + return command.NewOption("--disable-tls-verification", disable) +} diff --git a/pkg/kopia/cli/repository/storage/s3/s3_opts_test.go b/pkg/kopia/cli/repository/storage/s3/s3_opts_test.go new file mode 100644 index 0000000000..a903634819 --- /dev/null +++ b/pkg/kopia/cli/repository/storage/s3/s3_opts_test.go @@ -0,0 +1,65 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3 + +import ( + "testing" + + "github.com/kanisterio/safecli/command" + "github.com/kanisterio/safecli/test" + "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli" +) + +func TestS3Options(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ + { + Name: "optRegion", + Argument: command.NewArguments(optRegion("region"), optRegion("")), + ExpectedCLI: []string{"cmd", "--region=region"}, + }, + { + Name: "optBucket with bucketname should return option", + Argument: optBucket("bucketname"), + ExpectedCLI: []string{"cmd", "--bucket=bucketname"}, + }, + { + Name: "optBucket with empty bucketname should return error", + Argument: optBucket(""), + ExpectedErr: cli.ErrInvalidBucketName, + }, + { + Name: "optEndpoint", + Argument: command.NewArguments(optEndpoint("endpoint"), optEndpoint("")), + ExpectedCLI: []string{"cmd", "--endpoint=endpoint"}, + }, + { + Name: "optPrefix", + Argument: command.NewArguments(optPrefix("prefix"), optPrefix("")), + ExpectedCLI: []string{"cmd", "--prefix=prefix", "--prefix="}, + }, + { + Name: "optDisableTLS", + Argument: command.NewArguments(optDisableTLS(true), optDisableTLS(false)), + ExpectedCLI: []string{"cmd", "--disable-tls"}, + }, + { + Name: "optDisableTLSVerify", + Argument: command.NewArguments(optDisableTLSVerify(true), optDisableTLSVerify(false)), + ExpectedCLI: []string{"cmd", "--disable-tls-verification"}, + }, +}}) diff --git a/pkg/kopia/cli/repository/storage/s3/s3_test.go b/pkg/kopia/cli/repository/storage/s3/s3_test.go new file mode 100644 index 0000000000..1bb9c77cf4 --- /dev/null +++ b/pkg/kopia/cli/repository/storage/s3/s3_test.go @@ -0,0 +1,164 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3 + +import ( + "strconv" + "testing" + + "github.com/kanisterio/safecli/test" + "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" + intlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" + inttest "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" + "github.com/kanisterio/kanister/pkg/log" +) + +func TestNewS3(t *testing.T) { check.TestingT(t) } + +func newLocation(prefix, endpoint, region, bucket string, skipSSLVerify bool) internal.Location { + return internal.Location{ + "prefix": []byte(prefix), + "endpoint": []byte(endpoint), + "region": []byte(region), + "bucket": []byte(bucket), + "skipSSLVerify": []byte(strconv.FormatBool(skipSSLVerify)), + } +} + +// s3test is a test case for NewS3. +type s3test struct { + Name string + Location internal.Location + RepoPath string + ExpectedCLI []string + ExpectedErr error + Logger log.Logger + LoggerRegex []string +} + +// newS3Test creates a new test case for NewS3. +func newS3Test(s3t s3test) inttest.ArgumentTest { + return inttest.ArgumentTest{ + ArgumentTest: test.ArgumentTest{ + Name: s3t.Name, + Argument: New(s3t.Location, s3t.RepoPath, s3t.Logger), + ExpectedCLI: s3t.ExpectedCLI, + ExpectedErr: s3t.ExpectedErr, + }, + Logger: s3t.Logger, + LoggerRegex: s3t.LoggerRegex, + } +} + +// toArgTests converts a list of s3tests to a list of ArgumentTests. +func toArgTests(s3tests []s3test) []inttest.ArgumentTest { + argTests := make([]inttest.ArgumentTest, len(s3tests)) + for i, s3t := range s3tests { + argTests[i] = newS3Test(s3t) + } + return argTests +} + +var _ = check.Suite(&inttest.ArgumentSuite{Cmd: "cmd", Arguments: toArgTests([]s3test{ + { + Name: "NewS3", + Location: newLocation("prefix", "http://endpoint/path/", "region", "bucket", true), + RepoPath: "repoPath", + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--endpoint=endpoint/path", + "--prefix=prefix/repoPath/", + "--disable-tls", + "--disable-tls-verification", + }, + Logger: &intlog.StringLogger{}, + LoggerRegex: []string{ + "Removing leading", + "Removing trailing", + }, + }, + { + Name: "NewS3 w/o logger should not panic", + Location: newLocation("prefix", "http://endpoint/path/", "region", "bucket", true), + RepoPath: "repoPath", + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--endpoint=endpoint/path", + "--prefix=prefix/repoPath/", + "--disable-tls", + "--disable-tls-verification", + }, + Logger: &intlog.StringLogger{}, + }, + { + Name: "NewS3 with empty repoPath and https endpoint", + Location: newLocation("prefix", "https://endpoint/path/", "region", "bucket", false), + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--endpoint=endpoint/path", + "--prefix=prefix/", + }, + Logger: &intlog.StringLogger{}, + LoggerRegex: []string{ + "Removing leading", + "Removing trailing", + }, + }, + { + Name: "NewS3 with empty repoPath and endpoint", + Location: newLocation("prefix", "", "region", "bucket", true), + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--prefix=prefix/", + "--disable-tls-verification", + }, + Logger: &intlog.StringLogger{}, + LoggerRegex: []string{""}, // no output expected + }, + { + Name: "NewS3 with empty repoPath, prefix and endpoint", + Location: newLocation("", "", "region", "bucket", true), + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--prefix=", + "--disable-tls-verification", + }, + Logger: &intlog.StringLogger{}, + LoggerRegex: []string{""}, // no output expected + }, + { + Name: "NewS3 with empty repoPath, prefix, endpoint and bucket", + ExpectedErr: cli.ErrInvalidBucketName, + Logger: &intlog.StringLogger{}, + LoggerRegex: []string{""}, // no output expected + }, + { + Name: "NewS3 with empty logger should not panic", + Location: newLocation("", "https://endpoint/path/", "", "bucket", false), + ExpectedCLI: []string{"cmd", "s3", + "--bucket=bucket", + "--endpoint=endpoint/path", + "--prefix=", + }, + }, +})})