From 680e782a072c4174ddd1f25837488a49d5c7803f Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 2 Feb 2024 18:08:02 -0800 Subject: [PATCH 01/73] Add safecli dependency --- go.mod | 2 ++ go.sum | 2 ++ 2 files changed, 4 insertions(+) diff --git a/go.mod b/go.mod index bb49e9f3df..0b81ec2854 100644 --- a/go.mod +++ b/go.mod @@ -216,6 +216,8 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) +require github.com/kanisterio/safecli v0.0.3 + require ( github.com/Azure/go-autorest/autorest v0.11.27 // indirect github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect diff --git a/go.sum b/go.sum index 9793c1f653..fed0f3304c 100644 --- a/go.sum +++ b/go.sum @@ -359,6 +359,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kanisterio/safecli v0.0.3 h1:ts3oRVSoRexFBv9pOdWYZsN4k068pWx5Cl4zN54mQro= +github.com/kanisterio/safecli v0.0.3/go.mod h1:fK3Mcbeiso+NtkUdhGugK0Vf4S2l8ObvFf564ry1W5A= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734 h1:qulsCaCv+O2y9/sQ9nd5KChnAgFOWakTHQ9ZADjs6DQ= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734/go.mod h1:rdqSnvOJuKCPFW/h2rVLuXOAkRnHHdp9PZcKx4HCoDM= github.com/kastenhq/stow v0.2.6-kasten.1.0.20231101232131-9321daa23aae h1:2cl4yuAJpdmLCx7G8eIsfNlQBLEfw0JDj6mTTyqc5qg= From 72011e75bc8478d1856b1e86a8f6bee896f55635 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 2 Feb 2024 18:09:47 -0800 Subject: [PATCH 02/73] add new flag implementations based on the safecli package for the Kopia CLI --- pkg/kopia/cli/errors.go | 25 +++ pkg/kopia/cli/internal/flag/bool_flag.go | 45 ++++++ pkg/kopia/cli/internal/flag/flag.go | 81 ++++++++++ pkg/kopia/cli/internal/flag/flag_test.go | 171 +++++++++++++++++++++ pkg/kopia/cli/internal/flag/string_flag.go | 78 ++++++++++ pkg/kopia/cli/internal/test/flag_suite.go | 112 ++++++++++++++ 6 files changed, 512 insertions(+) create mode 100644 pkg/kopia/cli/errors.go create mode 100644 pkg/kopia/cli/internal/flag/bool_flag.go create mode 100644 pkg/kopia/cli/internal/flag/flag.go create mode 100644 pkg/kopia/cli/internal/flag/flag_test.go create mode 100644 pkg/kopia/cli/internal/flag/string_flag.go create mode 100644 pkg/kopia/cli/internal/test/flag_suite.go diff --git a/pkg/kopia/cli/errors.go b/pkg/kopia/cli/errors.go new file mode 100644 index 0000000000..946dd04e87 --- /dev/null +++ b/pkg/kopia/cli/errors.go @@ -0,0 +1,25 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cli + +import ( + "github.com/pkg/errors" +) + +// flag errors +var ( + // ErrInvalidFlag is returned when the flag name is empty. + ErrInvalidFlag = errors.New("invalid flag") +) diff --git a/pkg/kopia/cli/internal/flag/bool_flag.go b/pkg/kopia/cli/internal/flag/bool_flag.go new file mode 100644 index 0000000000..4eea665cb2 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/bool_flag.go @@ -0,0 +1,45 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flag + +import ( + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/kopia/cli" +) + +// boolFlag defines a boolean flag with a given flag name. +// If enabled is set to true, the flag is applied; otherwise, it is not. +type boolFlag struct { + flag string + enabled bool +} + +// Apply appends the flag to the command if the flag is enabled. +func (f boolFlag) Apply(cli safecli.CommandAppender) error { + if f.enabled { + cli.AppendLoggable(f.flag) + } + return nil +} + +// NewBoolFlag creates a new bool flag with a given flag name. +// If the flag name is empty, cli.ErrInvalidFlag is returned. +func NewBoolFlag(flag string, enabled bool) Applier { + if flag == "" { + return ErrorFlag(cli.ErrInvalidFlag) + } + return boolFlag{flag, enabled} +} diff --git a/pkg/kopia/cli/internal/flag/flag.go b/pkg/kopia/cli/internal/flag/flag.go new file mode 100644 index 0000000000..898ae31e16 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/flag.go @@ -0,0 +1,81 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flag + +import ( + "github.com/kanisterio/safecli" +) + +// Applier applies flags/args to the command. +type Applier interface { + // Apply applies the flags/args to the command. + Apply(cli safecli.CommandAppender) error +} + +// Apply appends multiple flags to the CLI. +// If any of the flags encounter an error during the Apply process, +// the error is returned and no changes are made to the CLI. +// If no error, the flags are appended to the CLI. +func Apply(cli safecli.CommandAppender, flags ...Applier) error { + // create a new sub builder which will be used to apply the flags + // to avoid mutating the CLI if an error is encountered. + sub := safecli.NewBuilder() + for _, flag := range flags { + if flag == nil { // if the flag is nil, skip it + continue + } + if err := flag.Apply(cli); err != nil { + return err + } + } + cli.Append(sub) + return nil +} + +// flags defines a collection of Flags. +type flags []Applier + +// Apply applies the flags to the CLI. +func (flags flags) Apply(cli safecli.CommandAppender) error { + return Apply(cli, flags...) +} + +// NewFlags creates a new collection of flags. +func NewFlags(fs ...Applier) Applier { + return flags(fs) +} + +// simpleFlag is a simple implementation of the Applier interface. +type simpleFlag struct { + err error +} + +// Apply does nothing except return an error if one is set. +func (f simpleFlag) Apply(safecli.CommandAppender) error { + return f.err +} + +// EmptyFlag creates a new flag that does nothing. +// It is useful for creating a no-op flag when a condition is not met +// but Applier interface is required. +func EmptyFlag() Applier { + return simpleFlag{} +} + +// ErrorFlag creates a new flag that returns an error when applied. +// It is useful for creating a flag validation if a condition is not met. +func ErrorFlag(err error) Applier { + return simpleFlag{err} +} diff --git a/pkg/kopia/cli/internal/flag/flag_test.go b/pkg/kopia/cli/internal/flag/flag_test.go new file mode 100644 index 0000000000..c20a46b450 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/flag_test.go @@ -0,0 +1,171 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flag_test + +import ( + "errors" + "testing" + + "gopkg.in/check.v1" + + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" +) + +var ( + ErrFlag = errors.New("flag error") +) + +// MockFlagApplier is a mock implementation of the FlagApplier interface. +type MockFlagApplier struct { + flagName string + applyErr error +} + +func (m *MockFlagApplier) Apply(cli safecli.CommandAppender) error { + cli.AppendLoggable(m.flagName) + return m.applyErr +} + +func TestApply(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(&test.FlagSuite{Cmd: "cmd", Tests: []test.FlagTest{ + { + Name: "Apply with no flags should generate only the command", + ExpectedCLI: []string{"cmd"}, + }, + { + Name: "Apply with nil flags should generate only the command", + Flag: flag.NewFlags(nil, nil), + ExpectedCLI: []string{"cmd"}, + }, + { + Name: "Apply with flags should generate the command and flags", + Flag: flag.NewFlags( + &MockFlagApplier{flagName: "--flag1", applyErr: nil}, + &MockFlagApplier{flagName: "--flag2", applyErr: nil}, + ), + ExpectedCLI: []string{"cmd", "--flag1", "--flag2"}, + }, + { + Name: "Apply with one error flag should not modify the command and return the error", + Flag: flag.NewFlags( + &MockFlagApplier{flagName: "flag1", applyErr: nil}, + &MockFlagApplier{flagName: "flag2", applyErr: ErrFlag}, + ), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: ErrFlag, + }, + { + Name: "NewBoolFlag", + Flag: flag.NewFlags( + flag.NewBoolFlag("--flag1", true), + flag.NewBoolFlag("--flag2", false), + ), + ExpectedCLI: []string{"cmd", "--flag1"}, + }, + { + Name: "NewBoolFlag with empty flag name should return an error", + Flag: flag.NewFlags( + flag.NewBoolFlag("", true), + ), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: cli.ErrInvalidFlag, + }, + { + Name: "NewStringFlag", + Flag: flag.NewFlags( + flag.NewStringFlag("--flag1", "value1"), + flag.NewStringFlag("--flag2", ""), + ), + ExpectedCLI: []string{"cmd", "--flag1=value1"}, + }, + { + Name: "NewStringFlag with all empty values should return an error", + Flag: flag.NewFlags( + flag.NewStringFlag("--flag1", "value1"), + flag.NewStringFlag("", ""), + ), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: cli.ErrInvalidFlag, + }, + { + Name: "NewRedactedStringFlag", + Flag: flag.NewFlags( + flag.NewRedactedStringFlag("--flag1", "value1"), + flag.NewRedactedStringFlag("--flag2", ""), + flag.NewRedactedStringFlag("", "value3"), + ), + ExpectedCLI: []string{"cmd", "--flag1=value1", "value3"}, + ExpectedLog: "cmd --flag1=<****> <****>", + }, + { + Name: "NewRedactedStringFlag with all empty values should return an error", + Flag: flag.NewFlags( + flag.NewRedactedStringFlag("--flag1", "value1"), + flag.NewRedactedStringFlag("", ""), + ), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: cli.ErrInvalidFlag, + }, + { + Name: "NewStringValue", + Flag: flag.NewFlags( + flag.NewStringArgument("value1"), + ), + ExpectedCLI: []string{"cmd", "value1"}, + }, + { + Name: "NewStringValue with empty value should return an error", + Flag: flag.NewFlags( + flag.NewStringArgument(""), + ), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: cli.ErrInvalidFlag, + }, + { + Name: "NewFlags should generate multiple flags", + Flag: flag.NewFlags( + flag.NewStringFlag("--flag1", "value1"), + flag.NewRedactedStringFlag("--flag2", "value2"), + flag.NewStringArgument("value3"), + ), + ExpectedCLI: []string{"cmd", "--flag1=value1", "--flag2=value2", "value3"}, + ExpectedLog: "cmd --flag1=value1 --flag2=<****> value3", + }, + { + Name: "NewFlags should generate no flags if one of them returns an error", + Flag: flag.NewFlags( + flag.NewStringFlag("--flag1", "value1"), + &MockFlagApplier{flagName: "flag2", applyErr: ErrFlag}, + ), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: ErrFlag, + }, + { + Name: "EmptyFlag should not generate any flags", + Flag: flag.EmptyFlag(), + ExpectedCLI: []string{"cmd"}, + }, + { + Name: "ErrorFlag should return an error", + Flag: flag.ErrorFlag(ErrFlag), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: ErrFlag, + }, +}}) diff --git a/pkg/kopia/cli/internal/flag/string_flag.go b/pkg/kopia/cli/internal/flag/string_flag.go new file mode 100644 index 0000000000..a94c29722c --- /dev/null +++ b/pkg/kopia/cli/internal/flag/string_flag.go @@ -0,0 +1,78 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flag + +import ( + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/kopia/cli" +) + +// stringFlag defines a string flag with a given flag name and value. +// If the value is empty, the flag is not applied. +type stringFlag struct { + flag string // flag name + value string // flag value + redacted bool // output the value as redacted +} + +// appenderFunc is a function that appends strings to a command. +type appenderFunc func(...string) *safecli.Builder + +// Apply appends the flag to the command if the value is not empty. +// If the value is redacted, it is appended as redacted. +func (f stringFlag) Apply(cli safecli.CommandAppender) error { + if f.value == "" { + return nil + } + appendValue, appendFlagValue := f.selectAppenderFuncs(cli) + if f.flag == "" { + appendValue(f.value) + } else { + appendFlagValue(f.flag, f.value) + } + return nil +} + +// selectAppenderFuncs returns the appropriate appender functions based on the redacted flag. +func (f stringFlag) selectAppenderFuncs(cli safecli.CommandAppender) (appenderFunc, appenderFunc) { + if f.redacted { + return cli.AppendRedacted, cli.AppendRedactedKV + } + return cli.AppendLoggable, cli.AppendLoggableKV +} + +// newStringFlag creates a new string flag with a given flag name and value. +func newStringFlag(flag, val string, redacted bool) Applier { + if flag == "" && val == "" { + return ErrorFlag(cli.ErrInvalidFlag) + } + return stringFlag{flag: flag, value: val, redacted: redacted} +} + +// NewStringFlag creates a new string flag with a given flag name and value. +func NewStringFlag(flag, val string) Applier { + return newStringFlag(flag, val, false) +} + +// NewRedactedStringFlag creates a new string flag with a given flag name and value. +func NewRedactedStringFlag(flag, val string) Applier { + return newStringFlag(flag, val, true) +} + +// NewStringArgument creates a new string argument with a given value. +func NewStringArgument(val string) Applier { + return newStringFlag("", val, false) +} diff --git a/pkg/kopia/cli/internal/test/flag_suite.go b/pkg/kopia/cli/internal/test/flag_suite.go new file mode 100644 index 0000000000..7c41a1ea3a --- /dev/null +++ b/pkg/kopia/cli/internal/test/flag_suite.go @@ -0,0 +1,112 @@ +package test + +import ( + "strings" + + "gopkg.in/check.v1" + + "github.com/pkg/errors" + + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" +) + +// FlagTest defines a single test for a flag. +type FlagTest struct { + // Name of the test. (required) + Name string + + // Flag to test. (required) + Flag flag.Applier + + // Expected CLI arguments. (optional) + ExpectedCLI []string + + // Expected log output. (optional) + // if empty, it will be set to ExpectedCLI joined with space. + // if empty and ExpectedCLI is empty, it will be ignored. + ExpectedLog string + + // Expected error. (optional) + // If nil, no error is expected and + // ExpectedCLI and ExpectedLog are checked. + ExpectedErr error +} + +// CheckCommentString implements check.CommentInterface +func (t *FlagTest) CheckCommentString() string { + return t.Name +} + +// setDefaultExpectedLog sets the default value for ExpectedLog based on ExpectedCLI. +func (t *FlagTest) setDefaultExpectedLog() { + if len(t.ExpectedLog) == 0 && len(t.ExpectedCLI) > 0 { + t.ExpectedLog = strings.Join(t.ExpectedCLI, " ") + } +} + +// assertError checks the error against ExpectedErr. +func (t *FlagTest) assertError(c *check.C, err error) { + if actualErr := errors.Cause(err); actualErr != nil { + c.Assert(actualErr, check.Equals, t.ExpectedErr, t) + } else { + c.Assert(err, check.Equals, t.ExpectedErr, t) + } +} + +// assertNoError makes sure there is no error. +func (t *FlagTest) assertNoError(c *check.C, err error) { + c.Assert(err, check.IsNil, t) +} + +// assertCLI asserts the builder's CLI output against ExpectedCLI. +func (t *FlagTest) assertCLI(c *check.C, b *safecli.Builder) { + c.Check(b.Build(), check.DeepEquals, t.ExpectedCLI, t) +} + +// assertLog asserts the builder's log output against ExpectedLog. +func (t *FlagTest) assertLog(c *check.C, b *safecli.Builder) { + t.setDefaultExpectedLog() + c.Check(b.String(), check.Equals, t.ExpectedLog, t) +} + +// Test runs the flag test. +func (ft *FlagTest) Test(c *check.C, b *safecli.Builder) { + err := flag.Apply(b, ft.Flag) + if ft.ExpectedErr != nil { + ft.assertError(c, err) + } else { + ft.assertNoError(c, err) + ft.assertCLI(c, b) + ft.assertLog(c, b) + } +} + +// FlagSuite defines a test suite for flags. +type FlagSuite struct { + Cmd string // Cmd appends to the safecli.Builder before test if not empty. + Tests []FlagTest // Tests to run. +} + +// TestFlags runs all tests in the flag suite. +func (s *FlagSuite) TestFlags(c *check.C) { + for _, test := range s.Tests { + b := newBuilder(s.Cmd) + test.Test(c, b) + } +} + +// NewFlagSuite creates a new FlagSuite. +func NewFlagSuite(tests []FlagTest) *FlagSuite { + return &FlagSuite{Tests: tests} +} + +// newBuilder creates a new safecli.Builder with the given command. +func newBuilder(cmd string) *safecli.Builder { + builder := safecli.NewBuilder() + if cmd != "" { + builder.AppendLoggable(cmd) + } + return builder +} From 0f635ad33638621c3e81849de07983c60192c73e Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Sat, 3 Feb 2024 16:03:14 -0800 Subject: [PATCH 03/73] apply go fmt Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/flag/flag.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/kopia/cli/internal/flag/flag.go b/pkg/kopia/cli/internal/flag/flag.go index 898ae31e16..66148034a1 100644 --- a/pkg/kopia/cli/internal/flag/flag.go +++ b/pkg/kopia/cli/internal/flag/flag.go @@ -68,7 +68,7 @@ func (f simpleFlag) Apply(safecli.CommandAppender) error { } // EmptyFlag creates a new flag that does nothing. -// It is useful for creating a no-op flag when a condition is not met +// It is useful for creating a no-op flag when a condition is not met // but Applier interface is required. func EmptyFlag() Applier { return simpleFlag{} From d0a6dd13afeb5f54cff4448c74131522680dda5c Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Mon, 5 Feb 2024 17:51:24 -0800 Subject: [PATCH 04/73] Add common Kopia args and flags --- pkg/kopia/cli/args.go | 34 +++ pkg/kopia/cli/errors.go | 6 + .../cli/internal/flag/common/common_flags.go | 205 ++++++++++++++++++ .../internal/flag/common/common_flags_test.go | 196 +++++++++++++++++ pkg/kopia/cli/internal/test/flag_suite.go | 25 +-- .../cli/internal/test/flag_suite_test.go | 146 +++++++++++++ pkg/kopia/cli/internal/test/redact.go | 47 ++++ pkg/kopia/cli/internal/test/redact_test.go | 51 +++++ 8 files changed, 695 insertions(+), 15 deletions(-) create mode 100644 pkg/kopia/cli/args.go create mode 100644 pkg/kopia/cli/internal/flag/common/common_flags.go create mode 100644 pkg/kopia/cli/internal/flag/common/common_flags_test.go create mode 100644 pkg/kopia/cli/internal/test/flag_suite_test.go create mode 100644 pkg/kopia/cli/internal/test/redact.go create mode 100644 pkg/kopia/cli/internal/test/redact_test.go diff --git a/pkg/kopia/cli/args.go b/pkg/kopia/cli/args.go new file mode 100644 index 0000000000..78b697b7a9 --- /dev/null +++ b/pkg/kopia/cli/args.go @@ -0,0 +1,34 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cli + +// The common arguments for Kopia CLI. + +// CommonArgs provides the common arguments for Kopia CLI. +type CommonArgs struct { + ConfigFilePath string // the path to the config file. + LogDirectory string // the directory where logs are stored. + LogLevel string // the level of logging. Default is "error". + RepoPassword string // the password for the repository. +} + +// CacheArgs provides the cache arguments for Kopia CLI. +type CacheArgs struct { + CacheDirectory string // the directory where cache is stored. Default is "/tmp/kopia-cache". + ContentCacheSizeMB int // the size of the content cache in MB. + ContentCacheSizeLimitMB int // the maximum size of the content cache in MB. + MetadataCacheSizeMB int // the size of the metadata cache in MB. + MetadataCacheSizeLimitMB int // the maximum size of the metadata cache in MB. +} diff --git a/pkg/kopia/cli/errors.go b/pkg/kopia/cli/errors.go index 946dd04e87..265c694d29 100644 --- a/pkg/kopia/cli/errors.go +++ b/pkg/kopia/cli/errors.go @@ -22,4 +22,10 @@ import ( var ( // ErrInvalidFlag is returned when the flag name is empty. ErrInvalidFlag = errors.New("invalid flag") + // ErrInvalidCommonArgs is returned when the common flag expects at most one cli.CommonArgs argument. + ErrInvalidCommonArgs = errors.New("common flag expects at most one cli.CommonArgs argument") + // ErrInvalidCacheArgs is returned when the cache flag expects at most one cli.CacheArgs argument. + ErrInvalidCacheArgs = errors.New("cache flag expects at most one cli.CacheArgs argument") + // ErrInvalidID is returned when the ID is empty. + ErrInvalidID = errors.New("invalid ID") ) diff --git a/pkg/kopia/cli/internal/flag/common/common_flags.go b/pkg/kopia/cli/internal/flag/common/common_flags.go new file mode 100644 index 0000000000..6df9b1bb5c --- /dev/null +++ b/pkg/kopia/cli/internal/flag/common/common_flags.go @@ -0,0 +1,205 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "strconv" + + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" +) + +// Flags without conditions which applied to different kopia commands. +var ( + All = flag.NewBoolFlag("--all", true) + Delta = flag.NewBoolFlag("--delta", true) + ShowIdentical = flag.NewBoolFlag("--show-identical", true) + NoGRPC = flag.NewBoolFlag("--no-grpc", true) +) + +// predefined flags +var ( + CheckForUpdates = checkForUpdates{CheckForUpdates: true} + NoCheckForUpdates = checkForUpdates{CheckForUpdates: false} +) + +// flag defaults +var ( + defaultLogLevel = "error" + defaultCacheDirectory = "/tmp/kopia-cache" +) + +// LogDirectory creates a new log directory flag with a given directory. +func LogDirectory(dir string) flag.Applier { + return flag.NewStringFlag("--log-dir", dir) +} + +// LogLevel creates a new log level flag with a given level. +// If the level is empty, the default log level is used. +func LogLevel(level string) flag.Applier { + if level == "" { + level = defaultLogLevel + } + return flag.NewStringFlag("--log-level", level) +} + +// CacheDirectory creates a new cache directory flag with a given directory. +// If the directory is empty, the default cache directory is used. +func CacheDirectory(dir string) flag.Applier { + if dir == "" { + dir = defaultCacheDirectory + } + return flag.NewStringFlag("--cache-directory", dir) +} + +// ConfigFilePath creates a new config file path flag with a given path. +func ConfigFilePath(path string) flag.Applier { + return flag.NewStringFlag("--config-file", path) +} + +// RepoPassword creates a new repository password flag with a given password. +func RepoPassword(password string) flag.Applier { + return flag.NewRedactedStringFlag("--password", password) +} + +// checkForUpdates is the flag for checking for updates. +type checkForUpdates struct { + CheckForUpdates bool +} + +func (f checkForUpdates) Flag() string { + if f.CheckForUpdates { + return "--check-for-updates" + } + return "--no-check-for-updates" +} + +func (f checkForUpdates) Apply(cli safecli.CommandAppender) error { + cli.AppendLoggable(f.Flag()) + return nil +} + +// ReadOnly creates a new read only flag. +func ReadOnly(readOnly bool) flag.Applier { + return flag.NewBoolFlag("--readonly", readOnly) +} + +// ContentCacheSizeLimitMB creates a new content cache size flag with a given size. +func ContentCacheSizeLimitMB(size int) flag.Applier { + val := strconv.Itoa(size) + return flag.NewStringFlag("--content-cache-size-limit-mb", val) +} + +// ContentCacheSizeMB creates a new content cache size flag with a given size. +func ContentCacheSizeMB(size int) flag.Applier { + val := strconv.Itoa(size) + return flag.NewStringFlag("--content-cache-size-mb", val) +} + +// MetadataCacheSizeLimitMB creates a new metadata cache size flag with a given size. +func MetadataCacheSizeLimitMB(size int) flag.Applier { + val := strconv.Itoa(size) + return flag.NewStringFlag("--metadata-cache-size-limit-mb", val) +} + +// MetadataCacheSizeMB creates a new metadata cache size flag with a given size. +func MetadataCacheSizeMB(size int) flag.Applier { + val := strconv.Itoa(size) + return flag.NewStringFlag("--metadata-cache-size-mb", val) +} + +// common are the global flags for Kopia. +type common struct { + cli.CommonArgs +} + +// Apply applies the global flags to the command. +func (f common) Apply(cmd safecli.CommandAppender) error { + return flag.Apply(cmd, + ConfigFilePath(f.ConfigFilePath), + LogLevel(f.LogLevel), + LogDirectory(f.LogDirectory), + RepoPassword(f.RepoPassword), + ) +} + +// Common creates a new common flag. +// If no arguments are provided, the default common flags are used. +// If one argument is provided, the common flags are used. +// If more than one argument is provided, ErrInvalidCommonArgs is returned. +func Common(args ...cli.CommonArgs) flag.Applier { + switch len(args) { + case 0: + return common{cli.CommonArgs{}} + case 1: + return common{args[0]} + default: + return flag.ErrorFlag(cli.ErrInvalidCommonArgs) + } +} + +// cache defines cache flags and implements Applier interface for the cache flags. +type cache struct { + cli.CacheArgs +} + +// Apply applies the cache flags to the command. +func (f cache) Apply(cmd safecli.CommandAppender) error { + return flag.Apply(cmd, + CacheDirectory(f.CacheDirectory), + ContentCacheSizeLimitMB(f.ContentCacheSizeLimitMB), + MetadataCacheSizeLimitMB(f.MetadataCacheSizeLimitMB), + // ContentCacheSizeMB(f.ContentCacheSizeMB), + // MetadataCacheSizeMB(f.MetadataCacheSizeMB), + ) +} + +// Cache creates a new cache flag. +// If no arguments are provided, the default cache flags are used. +// If one argument is provided, the cache flags are used. +// If more than one argument is provided, ErrInvalidCacheArgs is returned. +func Cache(args ...cli.CacheArgs) flag.Applier { + switch len(args) { + case 0: + return cache{cli.CacheArgs{}} + case 1: + return cache{args[0]} + default: + return flag.ErrorFlag(cli.ErrInvalidCacheArgs) + } +} + +// JSONOutput creates a new JSON output flag. +func JSONOutput(enable bool) flag.Applier { + return flag.NewBoolFlag("--json", enable) +} + +// JSON flag enables JSON output for different kopia commands. +var JSON = JSONOutput(true) + +// Delete creates a new delete flag. +func Delete(enable bool) flag.Applier { + return flag.NewBoolFlag("--delete", enable) +} + +// ID create the Kopia ID argument for different commands. +func ID(id string) flag.Applier { + if id == "" { + return flag.ErrorFlag(cli.ErrInvalidID) + } + return flag.NewStringArgument(id) +} diff --git a/pkg/kopia/cli/internal/flag/common/common_flags_test.go b/pkg/kopia/cli/internal/flag/common/common_flags_test.go new file mode 100644 index 0000000000..2e8ac8cfc1 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/common/common_flags_test.go @@ -0,0 +1,196 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "fmt" + "testing" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" + "gopkg.in/check.v1" +) + +func TestCommonFlags(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(test.NewFlagSuite([]test.FlagTest{ + { + Name: "Empty LogDirectory should generate a flag with default value", + Flag: LogDirectory(""), + }, + { + Name: "LogDirectory with value should generate a flag with the given directory", + Flag: LogDirectory("/path/to/logs"), + ExpectedCLI: []string{"--log-dir=/path/to/logs"}, + }, + { + Name: "Empty LogLevel should generate a flag with default value", + Flag: LogLevel(""), + ExpectedCLI: []string{fmt.Sprintf("--log-level=%s", defaultLogLevel)}, + }, + { + Name: "LogLevel with value should generate a flag with the given level", + Flag: LogLevel("info"), + ExpectedCLI: []string{"--log-level=info"}, + }, + { + Name: "Empty CacheDirectory should generate a flag with default value", + Flag: CacheDirectory(""), + ExpectedCLI: []string{fmt.Sprintf("--cache-directory=%s", defaultCacheDirectory)}, + }, + { + Name: "CacheDirectory with value should generate a flag with the given directory", + Flag: CacheDirectory("/home/user/.cache/kopia"), + ExpectedCLI: []string{"--cache-directory=/home/user/.cache/kopia"}, + }, + { + Name: "Empty ConfigFilePath should not generate a flag", + Flag: ConfigFilePath(""), + }, + { + Name: "ConfigFilePath with value should generate a flag with the given config file path", + Flag: ConfigFilePath("/var/kopia/config"), + ExpectedCLI: []string{"--config-file=/var/kopia/config"}, + }, + { + Name: "Empty RepoPassword should not generate a flag", + Flag: RepoPassword(""), + }, + { + Name: "RepoPassword with value should generate a flag with the given value and redact it for logs", + Flag: RepoPassword("pass12345"), + ExpectedCLI: []string{"--password=pass12345"}, + }, + { + Name: "CheckForUpdates should always generate a flag", + Flag: CheckForUpdates, + ExpectedCLI: []string{"--check-for-updates"}, + }, + { + Name: "NoCheckForUpdates should always generate a flag", + Flag: NoCheckForUpdates, + ExpectedCLI: []string{"--no-check-for-updates"}, + }, + { + Name: "ReadOnly(false)should not generate a flag", + Flag: ReadOnly(false), + }, + { + Name: "ReadOnly(true) should generate a flag", + Flag: ReadOnly(true), + ExpectedCLI: []string{"--readonly"}, + }, + { + Name: "NoGRPC should always generate '--no-grpc' flag", + Flag: NoGRPC, + ExpectedCLI: []string{"--no-grpc"}, + }, + { + Name: "JSON should always generate a flag", + Flag: JSON, + ExpectedCLI: []string{"--json"}, + }, + { + Name: "ContentCacheSizeLimitMB with value should generate a flag with the given value", + Flag: ContentCacheSizeLimitMB(1024), + ExpectedCLI: []string{"--content-cache-size-limit-mb=1024"}, + }, + { + Name: "ContentCacheSizeMB with value should generate a flag with the given value", + Flag: ContentCacheSizeMB(1024), + ExpectedCLI: []string{"--content-cache-size-mb=1024"}, + }, + { + Name: "MetadataCacheSizeLimitMB with value should generate a flag with the given value", + Flag: MetadataCacheSizeLimitMB(1024), + ExpectedCLI: []string{"--metadata-cache-size-limit-mb=1024"}, + }, + { + Name: "MetadataCacheSizeMB with value should generate a flag with the given value", + Flag: MetadataCacheSizeMB(1024), + ExpectedCLI: []string{"--metadata-cache-size-mb=1024"}, + }, + { + Name: "Empty Common should generate a flag with default value(s)", + Flag: Common(), + ExpectedCLI: []string{"--log-level=error"}, + }, + { + Name: "Common with more than one cli.CommonArgs should generate an error", + Flag: Common(cli.CommonArgs{}, cli.CommonArgs{}), + ExpectedErr: cli.ErrInvalidCommonArgs, + }, + { + Name: "Common with values should generate multiple flags with the given values and redact password for logs", + Flag: Common(cli.CommonArgs{ + ConfigFilePath: "/var/kopia/config", + LogDirectory: "/var/log/kopia", + LogLevel: "info", + RepoPassword: "pass12345", + }), + ExpectedCLI: []string{ + "--config-file=/var/kopia/config", + "--log-level=info", + "--log-dir=/var/log/kopia", + "--password=pass12345", + }, + }, + { + Name: "Empty FlagCacheArgs should generate multiple flags with default values", + Flag: Cache(), + ExpectedCLI: []string{ + "--cache-directory=/tmp/kopia-cache", + "--content-cache-size-limit-mb=0", + "--metadata-cache-size-limit-mb=0", + }, + }, + { + Name: "Cache with more than one cli.CacheArgs should generate an error", + Flag: Cache(cli.CacheArgs{}, cli.CacheArgs{}), + ExpectedErr: cli.ErrInvalidCacheArgs, + }, + { + Name: "Cache with CacheArgs should generate multiple cache related flags", + Flag: Cache(cli.CacheArgs{ + CacheDirectory: "/home/user/.cache/kopia", + ContentCacheSizeLimitMB: 1024, + MetadataCacheSizeLimitMB: 2048, + }), + ExpectedCLI: []string{ + "--cache-directory=/home/user/.cache/kopia", + "--content-cache-size-limit-mb=1024", + "--metadata-cache-size-limit-mb=2048", + }, + }, + { + Name: "Delete(false) should not generate a flag", + Flag: Delete(false), + }, + { + Name: "Delete(true) should generate a flag", + Flag: Delete(true), + ExpectedCLI: []string{"--delete"}, + }, + { + Name: "Empty ID should generate an ErrInvalidID error", + Flag: ID(""), + ExpectedErr: cli.ErrInvalidID, + }, + { + Name: "ID with value should generate an argument with the given value", + Flag: ID("id12345"), + ExpectedCLI: []string{"id12345"}, + }, +})) diff --git a/pkg/kopia/cli/internal/test/flag_suite.go b/pkg/kopia/cli/internal/test/flag_suite.go index 7c41a1ea3a..c930573466 100644 --- a/pkg/kopia/cli/internal/test/flag_suite.go +++ b/pkg/kopia/cli/internal/test/flag_suite.go @@ -1,8 +1,6 @@ package test import ( - "strings" - "gopkg.in/check.v1" "github.com/pkg/errors" @@ -42,17 +40,14 @@ func (t *FlagTest) CheckCommentString() string { // setDefaultExpectedLog sets the default value for ExpectedLog based on ExpectedCLI. func (t *FlagTest) setDefaultExpectedLog() { if len(t.ExpectedLog) == 0 && len(t.ExpectedCLI) > 0 { - t.ExpectedLog = strings.Join(t.ExpectedCLI, " ") + t.ExpectedLog = RedactCLI(t.ExpectedCLI) } } // assertError checks the error against ExpectedErr. func (t *FlagTest) assertError(c *check.C, err error) { - if actualErr := errors.Cause(err); actualErr != nil { - c.Assert(actualErr, check.Equals, t.ExpectedErr, t) - } else { - c.Assert(err, check.Equals, t.ExpectedErr, t) - } + actualErr := errors.Cause(err) + c.Assert(actualErr, check.Equals, t.ExpectedErr, t) } // assertNoError makes sure there is no error. @@ -72,14 +67,14 @@ func (t *FlagTest) assertLog(c *check.C, b *safecli.Builder) { } // Test runs the flag test. -func (ft *FlagTest) Test(c *check.C, b *safecli.Builder) { - err := flag.Apply(b, ft.Flag) - if ft.ExpectedErr != nil { - ft.assertError(c, err) +func (t *FlagTest) Test(c *check.C, b *safecli.Builder) { + err := flag.Apply(b, t.Flag) + if t.ExpectedErr != nil { + t.assertError(c, err) } else { - ft.assertNoError(c, err) - ft.assertCLI(c, b) - ft.assertLog(c, b) + t.assertNoError(c, err) + t.assertCLI(c, b) + t.assertLog(c, b) } } diff --git a/pkg/kopia/cli/internal/test/flag_suite_test.go b/pkg/kopia/cli/internal/test/flag_suite_test.go new file mode 100644 index 0000000000..307ff62533 --- /dev/null +++ b/pkg/kopia/cli/internal/test/flag_suite_test.go @@ -0,0 +1,146 @@ +package test_test + +import ( + "strings" + "testing" + + "github.com/pkg/errors" + + "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" + "github.com/kanisterio/safecli" +) + +func TestCustomFlag(t *testing.T) { check.TestingT(t) } + +// CustomFlagTest is a test for FlagTest. +// it has a custom flag that can be used to test the flag. +// and implements flag.Applier. +type CustomFlagTest struct { + name string + flag string + flagErr error + expectedErr error +} + +func (t *CustomFlagTest) Apply(cli safecli.CommandAppender) error { + if t.flagErr == nil { + cli.AppendLoggable(t.flag) + } + return t.flagErr +} + +func (t *CustomFlagTest) Test(c *check.C) { + flagTest := test.FlagTest{ + Name: t.name, + Flag: t, + ExpectedErr: t.expectedErr, + } + if t.flag != "" { + flagTest.ExpectedCLI = []string{t.flag} + } + b := safecli.NewBuilder() + flagTest.Test(c, b) +} + +type CustomFlagSuite struct { + cmd string + tests []test.FlagTest +} + +func (s *CustomFlagSuite) Test(c *check.C) { + suite := test.NewFlagSuite(s.tests) + suite.Cmd = s.cmd + suite.TestFlags(c) +} + +// TestRunnerWithConfig is a test suite for CustomFlagTest. +type TestRunnerWithConfig struct { + out strings.Builder // output buffer for the test results + cfg *check.RunConf // custom test configuration +} + +// register the test suite +var _ = check.Suite(&TestRunnerWithConfig{}) + +// SetUpTest sets up the test suite for running. +// it initializes the output buffer and the test configuration. +func (s *TestRunnerWithConfig) SetUpTest(c *check.C) { + s.out = strings.Builder{} + s.cfg = &check.RunConf{ + Output: &s.out, + Verbose: true, + } +} + +// TestFlagTestOK tests the FlagTest with no errors. +func (s *TestRunnerWithConfig) TestFlagTestOK(c *check.C) { + cft := CustomFlagTest{ + name: "TestFlagOK", + flag: "--test", + } + res := check.Run(&cft, s.cfg) + c.Assert(s.out.String(), check.Matches, "PASS: .*CustomFlagTest\\.Test.*\n") + c.Assert(res.Passed(), check.Equals, true) +} + +// TestFlagTestErr tests the FlagTest with an error. +func (s *TestRunnerWithConfig) TestFlagTestErr(c *check.C) { + err := errors.New("test error") + cft := CustomFlagTest{ + name: "TestFlagErr", + flagErr: err, + expectedErr: err, + } + res := check.Run(&cft, s.cfg) + c.Assert(s.out.String(), check.Matches, "PASS: .*CustomFlagTest\\.Test.*\n") + c.Assert(res.Passed(), check.Equals, true) +} + +// TestFlagTestWrapperErr tests the FlagTest with a wrapped error. +func (s *TestRunnerWithConfig) TestFlagTestWrapperErr(c *check.C) { + err := errors.New("test error") + werr := errors.Wrap(err, "wrapper error") + cft := CustomFlagTest{ + name: "TestFlagTestWrapperErr", + flagErr: werr, + expectedErr: err, + } + res := check.Run(&cft, s.cfg) + c.Assert(s.out.String(), check.Matches, "PASS: .*CustomFlagTest\\.Test.*\n") + c.Assert(res.Passed(), check.Equals, true) +} + +// TestFlagTestUnexpectedErr tests the FlagTest with an unexpected error. +func (s *TestRunnerWithConfig) TestFlagTestUnexpectedErr(c *check.C) { + err := errors.New("test error") + cft := CustomFlagTest{ + name: "TestFlagUnexpectedErr", + flag: "--test", + flagErr: err, + expectedErr: nil, + } + res := check.Run(&cft, s.cfg) + ss := s.out.String() + c.Assert(strings.Contains(ss, "TestFlagUnexpectedErr"), check.Equals, true) + c.Assert(strings.Contains(ss, "test error"), check.Equals, true) + c.Assert(res.Passed(), check.Equals, false) +} + +// TestFlagSuiteOK tests the FlagSuite with no errors. +func (s *TestRunnerWithConfig) TestFlagSuiteOK(c *check.C) { + cfs := CustomFlagSuite{ + cmd: "cmd", + tests: []test.FlagTest{ + { + Name: "TestFlagOK", + Flag: &CustomFlagTest{name: "TestFlagOK", flag: "--test"}, + ExpectedCLI: []string{"cmd", "--test"}, + }, + }, + } + res := check.Run(&cfs, s.cfg) + c.Assert(s.out.String(), check.Matches, "PASS: .*CustomFlagSuite\\.Test.*\n") + c.Assert(res.Passed(), check.Equals, true) +} diff --git a/pkg/kopia/cli/internal/test/redact.go b/pkg/kopia/cli/internal/test/redact.go new file mode 100644 index 0000000000..2622877d13 --- /dev/null +++ b/pkg/kopia/cli/internal/test/redact.go @@ -0,0 +1,47 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "fmt" + "strings" +) + +const ( + redactField = "<****>" +) + +var redactedFlags = []string{ + "--password", + "--user-password", + "--server-password", + "--server-control-password", + "--server-cert-fingerprint", +} + +// RedactCLI redacts sensitive information from the CLI command for tests. +func RedactCLI(cli []string) string { + redactedCLI := make([]string, len(cli)) + for i, arg := range cli { + redactedCLI[i] = arg + for _, flag := range redactedFlags { + if strings.HasPrefix(arg, flag+"=") { + redactedCLI[i] = fmt.Sprintf("%s=%s", flag, redactField) + break // redacted flag found, no need to check further + } + } + } + return strings.Join(redactedCLI, " ") +} \ No newline at end of file diff --git a/pkg/kopia/cli/internal/test/redact_test.go b/pkg/kopia/cli/internal/test/redact_test.go new file mode 100644 index 0000000000..4bce54b009 --- /dev/null +++ b/pkg/kopia/cli/internal/test/redact_test.go @@ -0,0 +1,51 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "strings" + "testing" + + "gopkg.in/check.v1" +) + +func TestRedactCLI(t *testing.T) { check.TestingT(t) } + +type RedactSuite struct{} + +var _ = check.Suite(&RedactSuite{}) + +func (s *RedactSuite) TestRedactCLI(c *check.C) { + cli := []string{ + "--password=secret", + "--user-password=123456", + "--server-password=pass123", + "--server-control-password=abc123", + "--server-cert-fingerprint=abcd1234", + "--other-flag=value", + "argument", + } + expected := []string{ + "--password=<****>", + "--user-password=<****>", + "--server-password=<****>", + "--server-control-password=<****>", + "--server-cert-fingerprint=<****>", + "--other-flag=value", + "argument", + } + result := RedactCLI(cli) + c.Assert(result, check.Equals, strings.Join(expected, " ")) +} From ca853c1700d714f5615ad6bb7015d042b0bf315b Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Mon, 5 Feb 2024 18:14:49 -0800 Subject: [PATCH 05/73] Add Kopia storage core flags --- pkg/kopia/cli/errors.go | 6 ++ .../internal/flag/storage/model/factory.go | 56 +++++++++++++ .../flag/storage/model/factory_test.go | 81 +++++++++++++++++++ .../internal/flag/storage/model/location.go | 61 ++++++++++++++ .../flag/storage/model/location_test.go | 81 +++++++++++++++++++ .../cli/internal/flag/storage/model/path.go | 28 +++++++ .../internal/flag/storage/model/path_test.go | 50 ++++++++++++ .../flag/storage/model/storage_flag.go | 61 ++++++++++++++ .../flag/storage/model/storage_flag_test.go | 69 ++++++++++++++++ pkg/kopia/cli/internal/log/log.go | 45 +++++++++++ 10 files changed, 538 insertions(+) create mode 100644 pkg/kopia/cli/internal/flag/storage/model/factory.go create mode 100644 pkg/kopia/cli/internal/flag/storage/model/factory_test.go create mode 100644 pkg/kopia/cli/internal/flag/storage/model/location.go create mode 100644 pkg/kopia/cli/internal/flag/storage/model/location_test.go create mode 100644 pkg/kopia/cli/internal/flag/storage/model/path.go create mode 100644 pkg/kopia/cli/internal/flag/storage/model/path_test.go create mode 100644 pkg/kopia/cli/internal/flag/storage/model/storage_flag.go create mode 100644 pkg/kopia/cli/internal/flag/storage/model/storage_flag_test.go create mode 100644 pkg/kopia/cli/internal/log/log.go diff --git a/pkg/kopia/cli/errors.go b/pkg/kopia/cli/errors.go index 265c694d29..0f42f4c37d 100644 --- a/pkg/kopia/cli/errors.go +++ b/pkg/kopia/cli/errors.go @@ -29,3 +29,9 @@ var ( // ErrInvalidID is returned when the ID is empty. ErrInvalidID = errors.New("invalid ID") ) + +// storage errors +var ( + // ErrUnsupportedStorage is returned when the storage is not supported. + ErrUnsupportedStorage = errors.New("unsupported storage") +) diff --git a/pkg/kopia/cli/internal/flag/storage/model/factory.go b/pkg/kopia/cli/internal/flag/storage/model/factory.go new file mode 100644 index 0000000000..8a77fbf7f6 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/model/factory.go @@ -0,0 +1,56 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "fmt" + + "github.com/pkg/errors" + + "github.com/kanisterio/safecli" + + rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" + + "github.com/kanisterio/kanister/pkg/kopia/cli" +) + +// StorageBuilder defines a function that creates +// a safecli.Builder for the storage sub command. +type StorageBuilder func(StorageFlag) (*safecli.Builder, error) + +// StorageBuilderFactory defines a factory interface +// for creating a StorageBuilder by type. +type StorageBuilderFactory interface { + Create(rs.LocType) StorageBuilder +} + +// BuildersFactory defines a map of StorageBuilder by LocType. +type BuildersFactory map[rs.LocType]StorageBuilder + +// Create returns a StorageBuilder by LocType and +// implements the StorageBuilderFactory interface. +func (sb BuildersFactory) Create(locType rs.LocType) StorageBuilder { + if b, found := sb[locType]; found { + return b + } + return sb.unsupportedStorageType(locType) +} + +// unsupportedStorageType returns an error for an unsupported location type. +func (sb BuildersFactory) unsupportedStorageType(locType rs.LocType) StorageBuilder { + return func(StorageFlag) (*safecli.Builder, error) { + return nil, errors.Wrap(cli.ErrUnsupportedStorage, fmt.Sprintf("unsupported location type: '%v'", locType)) + } +} diff --git a/pkg/kopia/cli/internal/flag/storage/model/factory_test.go b/pkg/kopia/cli/internal/flag/storage/model/factory_test.go new file mode 100644 index 0000000000..e0bee04d65 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/model/factory_test.go @@ -0,0 +1,81 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "testing" + + "gopkg.in/check.v1" + + "github.com/kanisterio/safecli" + + rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" +) + +func TestFactory(t *testing.T) { check.TestingT(t) } + +type FactorySuite struct{} + +var ( + _ = check.Suite(&FactorySuite{}) + ltBlue = rs.LocType("blue") + ltRed = rs.LocType("red") + ltUnknown = rs.LocType("unknown") +) + +type mockBuilder struct { + cmd string + err error +} + +func (m *mockBuilder) New() StorageBuilder { + return func(sf StorageFlag) (*safecli.Builder, error) { + if m.err != nil { + return nil, m.err + } + b := safecli.NewBuilder() + b.AppendLoggable(m.cmd) + return b, nil + } +} + +func mockFactory() StorageBuilderFactory { + factory := make(BuildersFactory) + + blue := mockBuilder{cmd: "blue"} + red := mockBuilder{cmd: "red"} + + factory[ltBlue] = blue.New() + factory[ltRed] = red.New() + return factory +} + +func (s *FactorySuite) TestBuildersFactory(c *check.C) { + factory := mockFactory() + + b, err := factory.Create(ltBlue)(StorageFlag{}) + c.Assert(err, check.IsNil) + c.Check(b, check.NotNil) + c.Check(b.Build(), check.DeepEquals, []string{"blue"}) + + b, err = factory.Create(ltRed)(StorageFlag{}) + c.Assert(err, check.IsNil) + c.Check(b, check.NotNil) + c.Check(b.Build(), check.DeepEquals, []string{"red"}) + + b, err = factory.Create(ltUnknown)(StorageFlag{}) + c.Assert(err, check.ErrorMatches, ".*unsupported location type: 'unknown'.*") + c.Check(b, check.IsNil) +} diff --git a/pkg/kopia/cli/internal/flag/storage/model/location.go b/pkg/kopia/cli/internal/flag/storage/model/location.go new file mode 100644 index 0000000000..3a9a4e4fc9 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/model/location.go @@ -0,0 +1,61 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "strconv" + "strings" + + rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" +) + +// Location is a map of key-value pairs that represent different storage properties. +type Location map[string][]byte + +// Type returns the location type. +func (l Location) Type() rs.LocType { + return rs.LocType(string(l[rs.TypeKey])) +} + +// Region returns the location region. +func (l Location) Region() string { + return string(l[rs.RegionKey]) +} + +// BucketName returns the location bucket name. +func (l Location) BucketName() string { + return string(l[rs.BucketKey]) +} + +// Endpoint returns the location endpoint. +func (l Location) Endpoint() string { + return string(l[rs.EndpointKey]) +} + +// Prefix returns the location prefix. +func (l Location) Prefix() string { + return string(l[rs.PrefixKey]) +} + +// IsInsecureEndpoint returns true if the location endpoint is insecure/http. +func (l Location) IsInsecureEndpoint() bool { + return strings.HasPrefix(l.Endpoint(), "http:") +} + +// HasSkipSSLVerify returns true if the location has skip SSL verification. +func (l Location) HasSkipSSLVerify() bool { + v, _ := strconv.ParseBool(string(l[rs.SkipSSLVerifyKey])) + return v +} diff --git a/pkg/kopia/cli/internal/flag/storage/model/location_test.go b/pkg/kopia/cli/internal/flag/storage/model/location_test.go new file mode 100644 index 0000000000..1051bce642 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/model/location_test.go @@ -0,0 +1,81 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "testing" + + rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" + "gopkg.in/check.v1" +) + +func TestLocation(t *testing.T) { check.TestingT(t) } + +type LocationSuite struct{} + +var _ = check.Suite(&LocationSuite{}) + +func (s *LocationSuite) TestLocation(c *check.C) { + type expected struct { + Type rs.LocType + Region string + BucketName string + Endpoint string + Prefix string + IsInsecure bool + HasSkipSSLVerify bool + } + + tests := []struct { + name string + location Location + expected expected + }{ + { + name: "Test with no fields", + location: Location{}, + expected: expected{}, + }, + { + name: "Test with all fields", + location: Location{ + rs.TypeKey: []byte("Type1"), + rs.RegionKey: []byte("Region1"), + rs.BucketKey: []byte("Bucket1"), + rs.EndpointKey: []byte("http://Endpoint1"), + rs.PrefixKey: []byte("Prefix1"), + rs.SkipSSLVerifyKey: []byte("true"), + }, + expected: expected{ + Type: "Type1", + Region: "Region1", + BucketName: "Bucket1", + Endpoint: "http://Endpoint1", + Prefix: "Prefix1", + IsInsecure: true, + HasSkipSSLVerify: true, + }, + }, + } + for _, test := range tests { + c.Check(test.location.Type(), check.Equals, test.expected.Type) + c.Check(test.location.Region(), check.Equals, test.expected.Region) + c.Check(test.location.BucketName(), check.Equals, test.expected.BucketName) + c.Check(test.location.Endpoint(), check.Equals, test.expected.Endpoint) + c.Check(test.location.Prefix(), check.Equals, test.expected.Prefix) + c.Check(test.location.IsInsecureEndpoint(), check.Equals, test.expected.IsInsecure) + c.Check(test.location.HasSkipSSLVerify(), check.Equals, test.expected.HasSkipSSLVerify) + } +} diff --git a/pkg/kopia/cli/internal/flag/storage/model/path.go b/pkg/kopia/cli/internal/flag/storage/model/path.go new file mode 100644 index 0000000000..5e18da2cb8 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/model/path.go @@ -0,0 +1,28 @@ +// Copyright 2022 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "path" +) + +// GenerateFullRepoPath generates the full repository path. +// If the location-specific prefix is empty, the repository-specific prefix is returned. +func GenerateFullRepoPath(locPrefix, repoPathPrefix string) string { + if locPrefix != "" { + return path.Join(locPrefix, repoPathPrefix) + "/" + } + return repoPathPrefix +} diff --git a/pkg/kopia/cli/internal/flag/storage/model/path_test.go b/pkg/kopia/cli/internal/flag/storage/model/path_test.go new file mode 100644 index 0000000000..2fd1762362 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/model/path_test.go @@ -0,0 +1,50 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "testing" + + "gopkg.in/check.v1" +) + +func TestPath(t *testing.T) { check.TestingT(t) } + +type PathSuite struct{} + +var _ = check.Suite(&PathSuite{}) + +func (s *PathSuite) TestGenerateFullRepoPath(c *check.C) { + tests := []struct { + locPrefix string + repoPathPrefix string + expected string + }{ + { + locPrefix: "", + repoPathPrefix: "repo", + expected: "repo", + }, + { + locPrefix: "loc", + repoPathPrefix: "repo", + expected: "loc/repo/", + }, + } + for _, test := range tests { + got := GenerateFullRepoPath(test.locPrefix, test.repoPathPrefix) + c.Check(got, check.Equals, test.expected) + } +} diff --git a/pkg/kopia/cli/internal/flag/storage/model/storage_flag.go b/pkg/kopia/cli/internal/flag/storage/model/storage_flag.go new file mode 100644 index 0000000000..89ac32cb6f --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/model/storage_flag.go @@ -0,0 +1,61 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "github.com/pkg/errors" + + "github.com/kanisterio/safecli" + + cmdlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" + log "github.com/kanisterio/kanister/pkg/log" +) + +var ( + // ErrInvalidFactory is returned when the factory is nil. + ErrInvalidFactory = errors.New("factory cannot be nil") +) + +// StorageFlag is a set of flags that are used to create a StorageFlag sub command. +type StorageFlag struct { + Location Location + RepoPathPrefix string + + Factory StorageBuilderFactory + Logger log.Logger +} + +// GetLogger returns the logger. +// If the logger is nil, it returns a NopLogger. +func (s StorageFlag) GetLogger() log.Logger { + if s.Logger == nil { + s.Logger = &cmdlog.NopLogger{} + } + return s.Logger +} + +// Apply applies the storage flags to the command. +func (s StorageFlag) Apply(cli safecli.CommandAppender) error { + if s.Factory == nil { + return ErrInvalidFactory + } + storageBuilder := s.Factory.Create(s.Location.Type()) + storageCLI, err := storageBuilder(s) + if err != nil { + return errors.Wrap(err, "failed to apply storage args") + } + cli.Append(storageCLI) + return nil +} diff --git a/pkg/kopia/cli/internal/flag/storage/model/storage_flag_test.go b/pkg/kopia/cli/internal/flag/storage/model/storage_flag_test.go new file mode 100644 index 0000000000..dd211eef4a --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/model/storage_flag_test.go @@ -0,0 +1,69 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "testing" + + "gopkg.in/check.v1" + + "github.com/kanisterio/safecli" + + rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" +) + +func TestStorageFlag(t *testing.T) { check.TestingT(t) } + +type StorageFlagSuite struct{} + +var _ = check.Suite(&StorageFlagSuite{}) + +func (s *StorageFlagSuite) TestGetLogger(c *check.C) { + sf := StorageFlag{} + c.Check(sf.GetLogger(), check.NotNil) + sf.Logger = nil + c.Check(sf.GetLogger(), check.NotNil) +} + +func (s *StorageFlagSuite) TestApplyNoFactory(c *check.C) { + sf := StorageFlag{} + err := sf.Apply(nil) + c.Check(err, check.Equals, ErrInvalidFactory) +} + +func (s *StorageFlagSuite) TestApply(c *check.C) { + sf := StorageFlag{ + Location: Location{ + rs.TypeKey: []byte("blue"), + }, + Factory: mockFactory(), + } + b := safecli.NewBuilder() + err := sf.Apply(b) + c.Check(err, check.IsNil) + c.Check(b.Build(), check.DeepEquals, []string{"blue"}) +} + +func (s *StorageFlagSuite) TestApplyUnknowType(c *check.C) { + sf := StorageFlag{ + Location: Location{ + rs.TypeKey: []byte("unknow"), + }, + Factory: mockFactory(), + } + b := safecli.NewBuilder() + err := sf.Apply(b) + c.Check(err, check.ErrorMatches, ".*failed to apply storage args.*") +} diff --git a/pkg/kopia/cli/internal/log/log.go b/pkg/kopia/cli/internal/log/log.go new file mode 100644 index 0000000000..4d2b29ffe3 --- /dev/null +++ b/pkg/kopia/cli/internal/log/log.go @@ -0,0 +1,45 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "context" + "io" + + "github.com/kanisterio/kanister/pkg/field" + "github.com/kanisterio/kanister/pkg/log" +) + +// NopLogger is a logger that does nothing. +// TODO: Move to log package +type NopLogger struct{} + +// Print does nothing. +func (NopLogger) Print(msg string, fields ...field.M) { +} + +// PrintTo does nothing. +func (NopLogger) PrintTo(w io.Writer, msg string, fields ...field.M) { +} + +// WithContext does nothing. +func (NopLogger) WithContext(ctx context.Context) log.Logger { + return &NopLogger{} +} + +// WithError does nothing. +func (NopLogger) WithError(err error) log.Logger { + return &NopLogger{} +} From 2d44961f212e55a2c25efb5b1f08ddd510576e5c Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Tue, 6 Feb 2024 14:51:59 -0800 Subject: [PATCH 06/73] Add kopia filesystem storage flags --- pkg/kopia/cli/errors.go | 2 + pkg/kopia/cli/internal/command/command.go | 45 ++++ .../cli/internal/command/command_test.go | 97 ++++++++ pkg/kopia/cli/internal/command/commands.go | 6 + pkg/kopia/cli/internal/flag/storage/fs/fs.go | 39 +++ .../cli/internal/flag/storage/fs/fs_flags.go | 26 ++ .../internal/flag/storage/fs/fs_flags_test.go | 36 +++ .../cli/internal/flag/storage/fs/fs_test.go | 58 +++++ .../cli/internal/flag/storage/storage.go | 70 ++++++ .../cli/internal/flag/storage/storage_test.go | 231 ++++++++++++++++++ pkg/kopia/cli/internal/test/command_suite.go | 134 ++++++++++ pkg/kopia/cli/internal/test/string_logger.go | 58 +++++ 12 files changed, 802 insertions(+) create mode 100644 pkg/kopia/cli/internal/command/command.go create mode 100644 pkg/kopia/cli/internal/command/command_test.go create mode 100644 pkg/kopia/cli/internal/command/commands.go create mode 100644 pkg/kopia/cli/internal/flag/storage/fs/fs.go create mode 100644 pkg/kopia/cli/internal/flag/storage/fs/fs_flags.go create mode 100644 pkg/kopia/cli/internal/flag/storage/fs/fs_flags_test.go create mode 100644 pkg/kopia/cli/internal/flag/storage/fs/fs_test.go create mode 100644 pkg/kopia/cli/internal/flag/storage/storage.go create mode 100644 pkg/kopia/cli/internal/flag/storage/storage_test.go create mode 100644 pkg/kopia/cli/internal/test/command_suite.go create mode 100644 pkg/kopia/cli/internal/test/string_logger.go diff --git a/pkg/kopia/cli/errors.go b/pkg/kopia/cli/errors.go index 0f42f4c37d..c4cc8393f1 100644 --- a/pkg/kopia/cli/errors.go +++ b/pkg/kopia/cli/errors.go @@ -28,6 +28,8 @@ var ( ErrInvalidCacheArgs = errors.New("cache flag expects at most one cli.CacheArgs argument") // ErrInvalidID is returned when the ID is empty. ErrInvalidID = errors.New("invalid ID") + // ErrInvalidCommand is returned when the command is empty. + ErrInvalidCommand = errors.New("invalid command") ) // storage errors diff --git a/pkg/kopia/cli/internal/command/command.go b/pkg/kopia/cli/internal/command/command.go new file mode 100644 index 0000000000..04acbaecd4 --- /dev/null +++ b/pkg/kopia/cli/internal/command/command.go @@ -0,0 +1,45 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +import ( + "github.com/kanisterio/safecli" + + clierrors "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" +) + +// Command is a CLI command/subcommand. +type Command struct { + name string +} + +// Apply applies the command to the CLI. +func (c Command) Apply(cli safecli.CommandAppender) error { + if len(c.name) == 0 { + return clierrors.ErrInvalidCommand + } + cli.AppendLoggable(c.name) + return nil +} + +// NewCommandBuilder returns a new safecli.Builder for the storage sub command. +func NewCommandBuilder(cmd flag.Applier, flags ...flag.Applier) (*safecli.Builder, error) { + b := safecli.NewBuilder() + if err := flag.Apply(b, append([]flag.Applier{cmd}, flags...)...); err != nil { + return nil, err + } + return b, nil +} diff --git a/pkg/kopia/cli/internal/command/command_test.go b/pkg/kopia/cli/internal/command/command_test.go new file mode 100644 index 0000000000..f4ae074d37 --- /dev/null +++ b/pkg/kopia/cli/internal/command/command_test.go @@ -0,0 +1,97 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +import ( + "errors" + "testing" + + "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/safecli" +) + +func TestCommand(t *testing.T) { check.TestingT(t) } + +type CommandSuite struct{} + +var _ = check.Suite(&CommandSuite{}) + +var ( + errInvalidCommand = errors.New("invalid command") + errInvalidFlag = errors.New("invalid flag") +) + +type mockCommandAndFlag struct { + flagName string + err error +} + +func (m *mockCommandAndFlag) Apply(cli safecli.CommandAppender) error { + if m.err == nil { + cli.AppendLoggable(m.flagName) + } + return m.err +} + +func (s *CommandSuite) TestCommand(c *check.C) { + b := safecli.NewBuilder() + cmd := Command{"cmd"} + err := cmd.Apply(b) + c.Assert(err, check.IsNil) + c.Check(b.Build(), check.DeepEquals, []string{"cmd"}) +} + +func (s *CommandSuite) TestEmptyCommand(c *check.C) { + b := safecli.NewBuilder() + cmd := Command{} + err := cmd.Apply(b) + c.Assert(err, check.Equals, cli.ErrInvalidCommand) +} + +func (s *CommandSuite) TestNewCommandBuilderWithFailedCommand(c *check.C) { + // test if command is invalid + b, err := NewCommandBuilder( + &mockCommandAndFlag{err: errInvalidCommand}, + ) + c.Assert(b, check.IsNil) + c.Assert(err, check.Equals, errInvalidCommand) +} + +func (s *CommandSuite) TestNewCommandBuilderWithFailedFlag(c *check.C) { + // test if flag is invalid + b, err := NewCommandBuilder( + &mockCommandAndFlag{flagName: "cmd"}, + &mockCommandAndFlag{err: errInvalidFlag}, + ) + c.Assert(b, check.IsNil) + c.Assert(err, check.Equals, errInvalidFlag) +} + +func (s *CommandSuite) TestNewCommandBuilder(c *check.C) { + // test if command and flag are valid + b, err := NewCommandBuilder( + &mockCommandAndFlag{flagName: "cmd"}, + &mockCommandAndFlag{flagName: "--flag1"}, + &mockCommandAndFlag{flagName: "--flag2"}, + ) + c.Assert(err, check.IsNil) + c.Check(b.Build(), check.DeepEquals, []string{ + "cmd", + "--flag1", + "--flag2", + }) +} diff --git a/pkg/kopia/cli/internal/command/commands.go b/pkg/kopia/cli/internal/command/commands.go new file mode 100644 index 0000000000..17fa517524 --- /dev/null +++ b/pkg/kopia/cli/internal/command/commands.go @@ -0,0 +1,6 @@ +package command + +// Repository storage sub commands. +var ( + FileSystem = Command{"filesystem"} +) diff --git a/pkg/kopia/cli/internal/flag/storage/fs/fs.go b/pkg/kopia/cli/internal/flag/storage/fs/fs.go new file mode 100644 index 0000000000..20a8b0fac7 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/fs/fs.go @@ -0,0 +1,39 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/command" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" +) + +const ( + // DefaultFSMountPath is the default mount path for the filesystem subcommand storage. + DefaultFSMountPath = "/mnt/data" +) + +// New returns a builder for the filesystem subcommand storage. +func New(f model.StorageFlag) (*safecli.Builder, error) { + path := generateFileSystemMountPath(f.Location.Prefix(), f.RepoPathPrefix) + return command.NewCommandBuilder(command.FileSystem, + Path(path), + ) +} + +func generateFileSystemMountPath(locPrefix, repoPathPrefix string) string { + return DefaultFSMountPath + "/" + model.GenerateFullRepoPath(locPrefix, repoPathPrefix) +} diff --git a/pkg/kopia/cli/internal/flag/storage/fs/fs_flags.go b/pkg/kopia/cli/internal/flag/storage/fs/fs_flags.go new file mode 100644 index 0000000000..67c252e1e4 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/fs/fs_flags.go @@ -0,0 +1,26 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" + +// +// Filestore flags. +// + +// Path creates a new path flag with a given path. +func Path(path string) flag.Applier { + return flag.NewStringFlag("--path", path) +} diff --git a/pkg/kopia/cli/internal/flag/storage/fs/fs_flags_test.go b/pkg/kopia/cli/internal/flag/storage/fs/fs_flags_test.go new file mode 100644 index 0000000000..c0631f0a75 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/fs/fs_flags_test.go @@ -0,0 +1,36 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "testing" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" + "gopkg.in/check.v1" +) + +func TestFilestoreFlags(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(test.NewFlagSuite([]test.FlagTest{ + { + Name: "Empty Path should not generate a flag", + Flag: Path(""), + }, + { + Name: "Path with value should generate a flag with the given value", + Flag: Path("/path/to/file"), + ExpectedCLI: []string{"--path=/path/to/file"}, + }, +})) diff --git a/pkg/kopia/cli/internal/flag/storage/fs/fs_test.go b/pkg/kopia/cli/internal/flag/storage/fs/fs_test.go new file mode 100644 index 0000000000..13d992abdb --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/fs/fs_test.go @@ -0,0 +1,58 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "testing" + + "gopkg.in/check.v1" + + "github.com/kanisterio/safecli" + + rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" +) + +func TestStorageFS(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ + { + Name: "Empty FS storage flag should generate subcommand with default flags", + CLI: func() (safecli.CommandBuilder, error) { + return New(model.StorageFlag{}) + }, + ExpectedCLI: []string{ + "filesystem", + "--path=/mnt/data/", + }, + }, + { + Name: "FS with values should generate subcommand with specific flags", + CLI: func() (safecli.CommandBuilder, error) { + return New(model.StorageFlag{ + RepoPathPrefix: "repo/path/prefix", + Location: model.Location{ + rs.PrefixKey: []byte("prefix"), + }, + }) + }, + ExpectedCLI: []string{ + "filesystem", + "--path=/mnt/data/prefix/repo/path/prefix/", + }, + }, +})) diff --git a/pkg/kopia/cli/internal/flag/storage/storage.go b/pkg/kopia/cli/internal/flag/storage/storage.go new file mode 100644 index 0000000000..48431e6a97 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/storage.go @@ -0,0 +1,70 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "sync" + + cmdlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" + "github.com/kanisterio/kanister/pkg/log" + rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/fs" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" +) + +// Option is a function that sets a storage option. +type Option func(*model.StorageFlag) + +// WithLogger sets the logger for the storage. +func WithLogger(logger log.Logger) Option { + return func(s *model.StorageFlag) { + s.Logger = logger + } +} + +// WithFactory sets the storage args builder factory for the storage. +func WithFactory(factory model.StorageBuilderFactory) Option { + return func(s *model.StorageFlag) { + s.Factory = factory + } +} + +var ( + // factoryOnce is used to initialize the factory once. + factoryOnce sync.Once + // factory creates a new StorageBuilder by LocType. + factory = model.BuildersFactory{} +) + +// Storage creates a new storage with the given location, repo path prefix and options. +func Storage(location model.Location, repoPathPrefix string, opts ...Option) model.StorageFlag { + factoryOnce.Do(func() { + // Register storage builders. + factory[rs.LocTypeFilestore] = fs.New + }) + // create a new storage with the given location, repo path prefix and defaults. + s := model.StorageFlag{ + Location: location, + RepoPathPrefix: repoPathPrefix, + Logger: &cmdlog.NopLogger{}, + Factory: &factory, + } + // apply storage options. + for _, opt := range opts { + opt(&s) + } + return s +} diff --git a/pkg/kopia/cli/internal/flag/storage/storage_test.go b/pkg/kopia/cli/internal/flag/storage/storage_test.go new file mode 100644 index 0000000000..3341156ac4 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/storage_test.go @@ -0,0 +1,231 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "fmt" + "testing" + + "gopkg.in/check.v1" + + "github.com/pkg/errors" + + "github.com/kanisterio/safecli" + + rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/command" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/fs" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" + cmdlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" +) + +func TestStorageFlags(t *testing.T) { check.TestingT(t) } + +type StorageSuite struct{} + +var _ = check.Suite(&StorageSuite{}) + +func (s *StorageSuite) TestLocationMethods(c *check.C) { + type expected struct { + Type rs.LocType + Region string + BucketName string + Endpoint string + Prefix string + IsInsecure bool + HasSkipSSLVerify bool + } + + tests := []struct { + name string + location model.Location + expected expected + }{ + { + name: "Test1", + location: model.Location{ + rs.TypeKey: []byte("Type1"), + rs.RegionKey: []byte("Region1"), + rs.BucketKey: []byte("Bucket1"), + rs.EndpointKey: []byte("http://Endpoint1"), + rs.PrefixKey: []byte("Prefix1"), + rs.SkipSSLVerifyKey: []byte("true"), + }, + expected: expected{ + Type: "Type1", + Region: "Region1", + BucketName: "Bucket1", + Endpoint: "http://Endpoint1", + Prefix: "Prefix1", + IsInsecure: true, + HasSkipSSLVerify: true, + }, + }, + { + name: "Test2", + location: model.Location{ + rs.TypeKey: []byte("Type2"), + rs.RegionKey: []byte("Region2"), + rs.BucketKey: []byte("Bucket2"), + rs.EndpointKey: []byte("https://Endpoint2"), + rs.PrefixKey: []byte("Prefix2"), + rs.SkipSSLVerifyKey: []byte("false"), + }, + expected: expected{ + Type: "Type2", + Region: "Region2", + BucketName: "Bucket2", + Endpoint: "https://Endpoint2", + Prefix: "Prefix2", + IsInsecure: false, + HasSkipSSLVerify: false, + }, + }, + } + + for _, tt := range tests { + c.Assert(tt.location.Type(), check.Equals, tt.expected.Type) + c.Assert(tt.location.Region(), check.Equals, tt.expected.Region) + c.Assert(tt.location.BucketName(), check.Equals, tt.expected.BucketName) + c.Assert(tt.location.Endpoint(), check.Equals, tt.expected.Endpoint) + c.Assert(tt.location.Prefix(), check.Equals, tt.expected.Prefix) + c.Assert(tt.location.IsInsecureEndpoint(), check.Equals, tt.expected.IsInsecure) + c.Assert(tt.location.HasSkipSSLVerify(), check.Equals, tt.expected.HasSkipSSLVerify) + } +} + +func (s *StorageSuite) TestStorageFlag(c *check.C) { + tests := []struct { + name string + storage flag.Applier + expCLI []string + err error + errMsg string + }{ + { + name: "Empty Storage should generate an error", + storage: Storage(nil, ""), + err: cli.ErrUnsupportedStorage, + }, + { + name: "Filesystem without prefix and with repo path should generate repo path", + storage: Storage( + model.Location{ + rs.TypeKey: []byte("filestore"), + rs.PrefixKey: []byte(""), + }, + "dir1/subdir/", + ), + expCLI: []string{ + "filesystem", + fmt.Sprintf("--path=%s/dir1/subdir/", fs.DefaultFSMountPath), + }, + }, + { + name: "Filesystem with prefix and repo path should generate merged prefix and repo path", + storage: Storage( + model.Location{ + rs.TypeKey: []byte("filestore"), + rs.PrefixKey: []byte("test-prefix"), + }, + "dir1/subdir/", + ), + expCLI: []string{ + "filesystem", + fmt.Sprintf("--path=%s/test-prefix/dir1/subdir/", fs.DefaultFSMountPath), + }, + }, + { + name: "Unsupported storage type should generate an error", + storage: Storage( + model.Location{ + rs.TypeKey: []byte("ftp"), + }, + "prefixfs", + ), + errMsg: "failed to apply storage args: unsupported location type: 'ftp': unsupported storage", + err: cli.ErrUnsupportedStorage, + }, + } + + for _, tt := range tests { + b := safecli.NewBuilder() + err := tt.storage.Apply(b) + + cmt := check.Commentf("FAIL: %v", tt.name) + if tt.errMsg != "" { + c.Assert(err.Error(), check.Equals, tt.errMsg, cmt) + } + + if tt.err == nil { + c.Assert(err, check.IsNil, cmt) + } else { + if errors.Cause(err) != nil { + c.Assert(errors.Cause(err), check.DeepEquals, tt.err, cmt) + } else { + c.Assert(err, check.Equals, tt.err, cmt) + } + } + c.Assert(b.Build(), check.DeepEquals, tt.expCLI, cmt) + } +} + +type MockFlagWithError struct{} + +func (f MockFlagWithError) Flag() string { + return "mock" +} + +var errMock = fmt.Errorf("mock error") + +func (f MockFlagWithError) Apply(cli safecli.CommandAppender) error { + return errMock +} + +func (s *StorageSuite) TestNewStorageBuilderWithErrorFlag(c *check.C) { + b, err := command.NewCommandBuilder(command.FileSystem, MockFlagWithError{}) + c.Assert(b, check.IsNil) + c.Assert(err, check.NotNil) + c.Assert(err, check.DeepEquals, errMock) +} + +func (s *StorageSuite) TestStorageGetLogger(c *check.C) { + storage := Storage(nil, "prefix") + c.Assert(storage.GetLogger(), check.NotNil) + + nopLog := &cmdlog.NopLogger{} + storage = Storage(nil, "prefix", WithLogger(nopLog)) + c.Assert(storage.GetLogger(), check.Equals, nopLog) +} + +type MockFactory struct{} + +func (f MockFactory) Create(locType rs.LocType) model.StorageBuilder { + return func(s model.StorageFlag) (*safecli.Builder, error) { + return safecli.NewBuilder("mock"), nil + } +} + +func (s *StorageSuite) TestStorageFactory(c *check.C) { + storage := Storage(nil, "prefix") + c.Assert(storage.GetLogger(), check.NotNil) + + mockFactory := &MockFactory{} + storage = Storage(nil, "prefix", WithFactory(mockFactory)) + c.Assert(storage.Factory, check.Equals, mockFactory) +} diff --git a/pkg/kopia/cli/internal/test/command_suite.go b/pkg/kopia/cli/internal/test/command_suite.go new file mode 100644 index 0000000000..480c1fcba5 --- /dev/null +++ b/pkg/kopia/cli/internal/test/command_suite.go @@ -0,0 +1,134 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "fmt" + + "gopkg.in/check.v1" + + "github.com/pkg/errors" + + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/log" + + "github.com/kanisterio/kanister/pkg/kopia/cli" +) + +// CommonArgs is a set of common arguments for the tests. +var CommonArgs = cli.CommonArgs{ + RepoPassword: "encr-key", + ConfigFilePath: "path/kopia.config", + LogDirectory: "cache/log", +} + +// CommandTest defines a single test for a command. +type CommandTest struct { + // Name of the test. (required) + Name string + + // CLI to test. (required) + CLI func() (safecli.CommandBuilder, error) + + // Expected CLI arguments. (optional) + ExpectedCLI []string + + // Expected log output. (optional) + // if empty, it will be derived from ExpectedCLI by redacting sensitive information. + // if empty and ExpectedCLI is empty, it will be ignored. + ExpectedLog string + + // Expected error. (optional) + // If nil, no error is expected and + // ExpectedCLI and ExpectedLog are checked. + ExpectedErr error + + // LoggerRegex is a list of regular expressions to match against the log output. (optional) + Logger log.Logger + LoggerRegex []string +} + +// CheckCommentString implements check.CommentInterface +func (t *CommandTest) CheckCommentString() string { + return t.Name +} + +func (t *CommandTest) setDefaultExpectedLog() { + if len(t.ExpectedLog) == 0 && len(t.ExpectedCLI) > 0 { + t.ExpectedLog = RedactCLI(t.ExpectedCLI) + } +} + +func (t *CommandTest) assertError(c *check.C, err error) { + actualErr := errors.Cause(err) + c.Assert(actualErr, check.Equals, t.ExpectedErr, t) +} + +func (t *CommandTest) assertNoError(c *check.C, err error) { + c.Assert(err, check.IsNil, t) +} + +func (t *CommandTest) assertCLI(c *check.C, b safecli.CommandBuilder) { + c.Check(b.Build(), check.DeepEquals, t.ExpectedCLI, t) +} + +func (t *CommandTest) assertLog(c *check.C, b safecli.CommandBuilder) { + t.setDefaultExpectedLog() + c.Check(fmt.Sprint(b), check.Equals, t.ExpectedLog, t) +} + +func (t *CommandTest) assertLogger(c *check.C) { + log, ok := t.Logger.(*StringLogger) + if !ok { + c.Fatalf("t.Logger is not a StringLogger") + } + cmtLog := check.Commentf("FAIL: %v\nlog %#v expected to match %#v", t.Name, log, t.LoggerRegex) + for _, regex := range t.LoggerRegex { + c.Assert(log.MatchString(regex), check.Equals, true, cmtLog) + } +} + +// Test runs the command test. +func (t *CommandTest) Test(c *check.C) { + b, err := t.CLI() + if t.ExpectedErr == nil { + t.assertNoError(c, err) + t.assertCLI(c, b) + t.assertLog(c, b) + } else { + t.assertError(c, err) + } + if t.Logger != nil { + t.assertLogger(c) + } +} + +// CommandSuite defines a test suite for commands. +type CommandSuite struct { + Tests []CommandTest +} + +// TestCommands runs all tests in the suite. +func (s *CommandSuite) TestCommands(c *check.C) { + for _, test := range s.Tests { + test.Test(c) + } +} + +// NewCommandSuite creates a new CommandSuite. +func NewCommandSuite(tests []CommandTest) *CommandSuite { + return &CommandSuite{Tests: tests} +} diff --git a/pkg/kopia/cli/internal/test/string_logger.go b/pkg/kopia/cli/internal/test/string_logger.go new file mode 100644 index 0000000000..1a55f46220 --- /dev/null +++ b/pkg/kopia/cli/internal/test/string_logger.go @@ -0,0 +1,58 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "context" + "io" + "regexp" + + "github.com/kanisterio/kanister/pkg/field" + "github.com/kanisterio/kanister/pkg/log" +) + +// StringLogger implements log.Logger and stores log messages in a slice of strings. +// It is useful for testing. +type StringLogger []string + +// Print appends the message to the slice. +func (l *StringLogger) Print(msg string, fields ...field.M) { + *l = append(*l, msg) +} + +// PrintTo appends the message to the slice. +func (l *StringLogger) PrintTo(w io.Writer, msg string, fields ...field.M) { + *l = append(*l, msg) +} + +// WithContext does nothing. +func (l *StringLogger) WithContext(ctx context.Context) log.Logger { + return l +} + +// WithError does nothing. +func (l *StringLogger) WithError(err error) log.Logger { + return l +} + +// MatchString returns true if any of the log messages match the pattern. +func (l *StringLogger) MatchString(pattern string) bool { + for _, line := range *l { + if found, _ := regexp.MatchString(pattern, line); found { + return true + } + } + return false +} From 8f363ed89950664f41a4739860e132aeb1cc85b0 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Tue, 6 Feb 2024 15:05:59 -0800 Subject: [PATCH 07/73] cleanup storage tests --- .../cli/internal/flag/storage/storage_test.go | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/pkg/kopia/cli/internal/flag/storage/storage_test.go b/pkg/kopia/cli/internal/flag/storage/storage_test.go index 3341156ac4..96fe150d5a 100644 --- a/pkg/kopia/cli/internal/flag/storage/storage_test.go +++ b/pkg/kopia/cli/internal/flag/storage/storage_test.go @@ -169,28 +169,22 @@ func (s *StorageSuite) TestStorageFlag(c *check.C) { cmt := check.Commentf("FAIL: %v", tt.name) if tt.errMsg != "" { + c.Assert(err, check.NotNil, cmt) c.Assert(err.Error(), check.Equals, tt.errMsg, cmt) } if tt.err == nil { c.Assert(err, check.IsNil, cmt) } else { - if errors.Cause(err) != nil { - c.Assert(errors.Cause(err), check.DeepEquals, tt.err, cmt) - } else { - c.Assert(err, check.Equals, tt.err, cmt) - } + c.Assert(errors.Cause(err), check.Equals, tt.err, cmt) } c.Assert(b.Build(), check.DeepEquals, tt.expCLI, cmt) } } +// MockFlagWithError is a mock flag that always returns an error type MockFlagWithError struct{} -func (f MockFlagWithError) Flag() string { - return "mock" -} - var errMock = fmt.Errorf("mock error") func (f MockFlagWithError) Apply(cli safecli.CommandAppender) error { @@ -200,8 +194,7 @@ func (f MockFlagWithError) Apply(cli safecli.CommandAppender) error { func (s *StorageSuite) TestNewStorageBuilderWithErrorFlag(c *check.C) { b, err := command.NewCommandBuilder(command.FileSystem, MockFlagWithError{}) c.Assert(b, check.IsNil) - c.Assert(err, check.NotNil) - c.Assert(err, check.DeepEquals, errMock) + c.Assert(err, check.Equals, errMock) } func (s *StorageSuite) TestStorageGetLogger(c *check.C) { @@ -213,11 +206,12 @@ func (s *StorageSuite) TestStorageGetLogger(c *check.C) { c.Assert(storage.GetLogger(), check.Equals, nopLog) } +// MockFactory is a mock storage factory type MockFactory struct{} func (f MockFactory) Create(locType rs.LocType) model.StorageBuilder { return func(s model.StorageFlag) (*safecli.Builder, error) { - return safecli.NewBuilder("mock"), nil + return safecli.NewBuilder("mockfactory"), nil } } @@ -228,4 +222,8 @@ func (s *StorageSuite) TestStorageFactory(c *check.C) { mockFactory := &MockFactory{} storage = Storage(nil, "prefix", WithFactory(mockFactory)) c.Assert(storage.Factory, check.Equals, mockFactory) + b, err := storage.Factory.Create("anything")(model.StorageFlag{}) + c.Assert(b, check.NotNil) + c.Assert(err, check.IsNil) + c.Assert(b.Build(), check.DeepEquals, []string{"mockfactory"}) } From 0eb826a093e7dd492822118d5d6a272e99eb537b Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Tue, 6 Feb 2024 15:15:21 -0800 Subject: [PATCH 08/73] Add kopia GCS storage flags --- pkg/kopia/cli/internal/command/commands.go | 1 + .../cli/internal/flag/storage/gcs/gcs.go | 34 +++++++++++ .../internal/flag/storage/gcs/gcs_flags.go | 36 +++++++++++ .../flag/storage/gcs/gcs_flags_test.go | 36 +++++++++++ .../cli/internal/flag/storage/gcs/gcs_test.go | 61 +++++++++++++++++++ .../cli/internal/flag/storage/storage.go | 5 +- .../cli/internal/flag/storage/storage_test.go | 17 ++++++ 7 files changed, 189 insertions(+), 1 deletion(-) create mode 100644 pkg/kopia/cli/internal/flag/storage/gcs/gcs.go create mode 100644 pkg/kopia/cli/internal/flag/storage/gcs/gcs_flags.go create mode 100644 pkg/kopia/cli/internal/flag/storage/gcs/gcs_flags_test.go create mode 100644 pkg/kopia/cli/internal/flag/storage/gcs/gcs_test.go diff --git a/pkg/kopia/cli/internal/command/commands.go b/pkg/kopia/cli/internal/command/commands.go index 17fa517524..e0136a4c71 100644 --- a/pkg/kopia/cli/internal/command/commands.go +++ b/pkg/kopia/cli/internal/command/commands.go @@ -3,4 +3,5 @@ package command // Repository storage sub commands. var ( FileSystem = Command{"filesystem"} + GCS = Command{"gcs"} ) diff --git a/pkg/kopia/cli/internal/flag/storage/gcs/gcs.go b/pkg/kopia/cli/internal/flag/storage/gcs/gcs.go new file mode 100644 index 0000000000..076c9beda7 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/gcs/gcs.go @@ -0,0 +1,34 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcs + +import ( + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/consts" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/command" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" +) + +// New returns a builder for the GCS subcommand storage. +func New(s model.StorageFlag) (*safecli.Builder, error) { + prefix := model.GenerateFullRepoPath(s.Location.Prefix(), s.RepoPathPrefix) + return command.NewCommandBuilder(command.GCS, + Bucket(s.Location.BucketName()), + CredentialsFile(consts.GoogleCloudCredsFilePath), + Prefix(prefix), + ) +} diff --git a/pkg/kopia/cli/internal/flag/storage/gcs/gcs_flags.go b/pkg/kopia/cli/internal/flag/storage/gcs/gcs_flags.go new file mode 100644 index 0000000000..8800b8ce0a --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/gcs/gcs_flags.go @@ -0,0 +1,36 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcs + +import "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" + +// +// GCS flags. +// + +// Bucket creates a new GCS bucket flag with a given bucket name. +func Bucket(bucket string) flag.Applier { + return flag.NewStringFlag("--bucket", bucket) +} + +// Prefix creates a new GCS prefix flag with a given prefix. +func Prefix(prefix string) flag.Applier { + return flag.NewStringFlag("--prefix", prefix) +} + +// CredentialsFile creates a new GCS credentials file flag with a given file path. +func CredentialsFile(filePath string) flag.Applier { + return flag.NewStringFlag("--credentials-file", filePath) +} diff --git a/pkg/kopia/cli/internal/flag/storage/gcs/gcs_flags_test.go b/pkg/kopia/cli/internal/flag/storage/gcs/gcs_flags_test.go new file mode 100644 index 0000000000..4a6fadf6c6 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/gcs/gcs_flags_test.go @@ -0,0 +1,36 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcs + +import ( + "testing" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" + "gopkg.in/check.v1" +) + +func TestStorageGCSFlags(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(test.NewFlagSuite([]test.FlagTest{ + { + Name: "Empty CredentialsFile should not generate a flag", + Flag: CredentialsFile(""), + }, + { + Name: "CredentialsFile with value should generate a flag with the given value", + Flag: CredentialsFile("/path/to/credentials"), + ExpectedCLI: []string{"--credentials-file=/path/to/credentials"}, + }, +})) diff --git a/pkg/kopia/cli/internal/flag/storage/gcs/gcs_test.go b/pkg/kopia/cli/internal/flag/storage/gcs/gcs_test.go new file mode 100644 index 0000000000..c2f2fe9476 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/gcs/gcs_test.go @@ -0,0 +1,61 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcs + +import ( + "testing" + + "gopkg.in/check.v1" + + "github.com/kanisterio/safecli" + + rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" +) + +func TestStorageGCS(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ + { + Name: "Empty GCS storage flag should generate subcommand with default flags", + CLI: func() (safecli.CommandBuilder, error) { + return New(model.StorageFlag{}) + }, + ExpectedCLI: []string{ + "gcs", + "--credentials-file=/tmp/creds.txt", + }, + }, + { + Name: "GCS with values should generate subcommand with specific flags", + CLI: func() (safecli.CommandBuilder, error) { + return New(model.StorageFlag{ + RepoPathPrefix: "repo/path/prefix", + Location: model.Location{ + rs.PrefixKey: []byte("prefix"), + rs.BucketKey: []byte("bucket"), + }, + }) + }, + ExpectedCLI: []string{ + "gcs", + "--bucket=bucket", + "--credentials-file=/tmp/creds.txt", + "--prefix=prefix/repo/path/prefix/", + }, + }, +})) diff --git a/pkg/kopia/cli/internal/flag/storage/storage.go b/pkg/kopia/cli/internal/flag/storage/storage.go index 48431e6a97..2b0b564ac6 100644 --- a/pkg/kopia/cli/internal/flag/storage/storage.go +++ b/pkg/kopia/cli/internal/flag/storage/storage.go @@ -17,11 +17,13 @@ package storage import ( "sync" - cmdlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" "github.com/kanisterio/kanister/pkg/log" rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" + cmdlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/fs" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/gcs" "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" ) @@ -54,6 +56,7 @@ func Storage(location model.Location, repoPathPrefix string, opts ...Option) mod factoryOnce.Do(func() { // Register storage builders. factory[rs.LocTypeFilestore] = fs.New + factory[rs.LocTypeGCS] = gcs.New }) // create a new storage with the given location, repo path prefix and defaults. s := model.StorageFlag{ diff --git a/pkg/kopia/cli/internal/flag/storage/storage_test.go b/pkg/kopia/cli/internal/flag/storage/storage_test.go index 96fe150d5a..2164afc7ed 100644 --- a/pkg/kopia/cli/internal/flag/storage/storage_test.go +++ b/pkg/kopia/cli/internal/flag/storage/storage_test.go @@ -161,6 +161,23 @@ func (s *StorageSuite) TestStorageFlag(c *check.C) { errMsg: "failed to apply storage args: unsupported location type: 'ftp': unsupported storage", err: cli.ErrUnsupportedStorage, }, + { + name: "GCS should generate gcs args", + storage: Storage( + model.Location{ + rs.TypeKey: []byte("gcs"), + rs.BucketKey: []byte("bucket"), + rs.PrefixKey: []byte("/path/to/prefix"), + }, + "prefixfs", + ), + expCLI: []string{ + "gcs", + "--bucket=bucket", + "--credentials-file=/tmp/creds.txt", + "--prefix=/path/to/prefix/prefixfs/", + }, + }, } for _, tt := range tests { From 13266243d33561e74fd9f90b61e167503f18d13a Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Tue, 6 Feb 2024 15:21:57 -0800 Subject: [PATCH 09/73] add gcs flag tests --- .../flag/storage/gcs/gcs_flags_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pkg/kopia/cli/internal/flag/storage/gcs/gcs_flags_test.go b/pkg/kopia/cli/internal/flag/storage/gcs/gcs_flags_test.go index 4a6fadf6c6..45eea13c0f 100644 --- a/pkg/kopia/cli/internal/flag/storage/gcs/gcs_flags_test.go +++ b/pkg/kopia/cli/internal/flag/storage/gcs/gcs_flags_test.go @@ -24,6 +24,24 @@ import ( func TestStorageGCSFlags(t *testing.T) { check.TestingT(t) } var _ = check.Suite(test.NewFlagSuite([]test.FlagTest{ + { + Name: "Empty Bucket should not generate a flag", + Flag: Bucket(""), + }, + { + Name: "Bucket with value should generate a flag with the given value", + Flag: Bucket("bucket"), + ExpectedCLI: []string{"--bucket=bucket"}, + }, + { + Name: "Empty Prefix should not generate a flag", + Flag: Prefix(""), + }, + { + Name: "Prefix with value should generate a flag with the given value", + Flag: Prefix("prefix"), + ExpectedCLI: []string{"--prefix=prefix"}, + }, { Name: "Empty CredentialsFile should not generate a flag", Flag: CredentialsFile(""), From c3ba3eb5567ffdd720d0746c1edbb9cddc6f4905 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Tue, 6 Feb 2024 15:27:31 -0800 Subject: [PATCH 10/73] Add kopia azure storage flags --- pkg/kopia/cli/internal/command/commands.go | 1 + .../cli/internal/flag/storage/azure/azure.go | 31 ++++++++++ .../flag/storage/azure/azure_flags.go | 31 ++++++++++ .../flag/storage/azure/azure_flags_test.go | 45 +++++++++++++++ .../internal/flag/storage/azure/azure_test.go | 57 +++++++++++++++++++ .../cli/internal/flag/storage/storage.go | 2 + .../cli/internal/flag/storage/storage_test.go | 16 ++++++ 7 files changed, 183 insertions(+) create mode 100644 pkg/kopia/cli/internal/flag/storage/azure/azure.go create mode 100644 pkg/kopia/cli/internal/flag/storage/azure/azure_flags.go create mode 100644 pkg/kopia/cli/internal/flag/storage/azure/azure_flags_test.go create mode 100644 pkg/kopia/cli/internal/flag/storage/azure/azure_test.go diff --git a/pkg/kopia/cli/internal/command/commands.go b/pkg/kopia/cli/internal/command/commands.go index e0136a4c71..3108e93630 100644 --- a/pkg/kopia/cli/internal/command/commands.go +++ b/pkg/kopia/cli/internal/command/commands.go @@ -4,4 +4,5 @@ package command var ( FileSystem = Command{"filesystem"} GCS = Command{"gcs"} + Azure = Command{"azure"} ) diff --git a/pkg/kopia/cli/internal/flag/storage/azure/azure.go b/pkg/kopia/cli/internal/flag/storage/azure/azure.go new file mode 100644 index 0000000000..0fb6e8387f --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/azure/azure.go @@ -0,0 +1,31 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azure + +import ( + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/command" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" +) + +// New returns a builder for the Azure subcommand storage. +func New(f model.StorageFlag) (*safecli.Builder, error) { + prefix := model.GenerateFullRepoPath(f.Location.Prefix(), f.RepoPathPrefix) + return command.NewCommandBuilder(command.Azure, + Countainer(f.Location.BucketName()), + Prefix(prefix), + ) +} diff --git a/pkg/kopia/cli/internal/flag/storage/azure/azure_flags.go b/pkg/kopia/cli/internal/flag/storage/azure/azure_flags.go new file mode 100644 index 0000000000..a9943e617a --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/azure/azure_flags.go @@ -0,0 +1,31 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azure + +import "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" + +// +// Azure flags. +// + +// Prefix creates a new Azure prefix flag with a given prefix. +func Prefix(prefix string) flag.Applier { + return flag.NewStringFlag("--prefix", prefix) +} + +// Countainer creates a new Azure container flag with a given container name. +func Countainer(name string) flag.Applier { + return flag.NewStringFlag("--container", name) +} diff --git a/pkg/kopia/cli/internal/flag/storage/azure/azure_flags_test.go b/pkg/kopia/cli/internal/flag/storage/azure/azure_flags_test.go new file mode 100644 index 0000000000..6d44ed325d --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/azure/azure_flags_test.go @@ -0,0 +1,45 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azure + +import ( + "testing" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" + "gopkg.in/check.v1" +) + +func TestStorageAzureFlags(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(test.NewFlagSuite([]test.FlagTest{ + { + Name: "Empty Prefix should not generate a flag", + Flag: Prefix(""), + }, + { + Name: "Prefix with value should generate a flag with the given value", + Flag: Prefix("prefix"), + ExpectedCLI: []string{"--prefix=prefix"}, + }, + { + Name: "Empty AzureCountainer should not generate a flag", + Flag: Countainer(""), + }, + { + Name: "AzureCountainer with value should generate a flag with the given value", + Flag: Countainer("container"), + ExpectedCLI: []string{"--container=container"}, + }, +})) diff --git a/pkg/kopia/cli/internal/flag/storage/azure/azure_test.go b/pkg/kopia/cli/internal/flag/storage/azure/azure_test.go new file mode 100644 index 0000000000..3e16ac8f40 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/azure/azure_test.go @@ -0,0 +1,57 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azure + +import ( + "testing" + + "gopkg.in/check.v1" + + "github.com/kanisterio/safecli" + + rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" +) + +func TestStorageAzure(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ + { + Name: "Empty Azure storage flag should generate subcommand with default flags", + CLI: func() (safecli.CommandBuilder, error) { + return New(model.StorageFlag{}) + }, + ExpectedCLI: []string{"azure"}, + }, + { + Name: "Azure with values should generate subcommand with specific flags", + CLI: func() (safecli.CommandBuilder, error) { + return New(model.StorageFlag{ + RepoPathPrefix: "repo/path/prefix", + Location: model.Location{ + rs.PrefixKey: []byte("prefix"), + rs.BucketKey: []byte("container"), + }, + }) + }, + ExpectedCLI: []string{ + "azure", + "--container=container", + "--prefix=prefix/repo/path/prefix/", + }, + }, +})) diff --git a/pkg/kopia/cli/internal/flag/storage/storage.go b/pkg/kopia/cli/internal/flag/storage/storage.go index 2b0b564ac6..08e36d8897 100644 --- a/pkg/kopia/cli/internal/flag/storage/storage.go +++ b/pkg/kopia/cli/internal/flag/storage/storage.go @@ -22,6 +22,7 @@ import ( cmdlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/azure" "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/fs" "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/gcs" "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" @@ -57,6 +58,7 @@ func Storage(location model.Location, repoPathPrefix string, opts ...Option) mod // Register storage builders. factory[rs.LocTypeFilestore] = fs.New factory[rs.LocTypeGCS] = gcs.New + factory[rs.LocTypeAzure] = azure.New }) // create a new storage with the given location, repo path prefix and defaults. s := model.StorageFlag{ diff --git a/pkg/kopia/cli/internal/flag/storage/storage_test.go b/pkg/kopia/cli/internal/flag/storage/storage_test.go index 2164afc7ed..09031550b4 100644 --- a/pkg/kopia/cli/internal/flag/storage/storage_test.go +++ b/pkg/kopia/cli/internal/flag/storage/storage_test.go @@ -178,6 +178,22 @@ func (s *StorageSuite) TestStorageFlag(c *check.C) { "--prefix=/path/to/prefix/prefixfs/", }, }, + { + name: "Azure should generate azure args", + storage: Storage( + model.Location{ + rs.TypeKey: []byte("azure"), + rs.BucketKey: []byte("bucket"), + rs.PrefixKey: []byte("/path/to/prefix"), + }, + "prefixfs", + ), + expCLI: []string{ + "azure", + "--container=bucket", + "--prefix=/path/to/prefix/prefixfs/", + }, + }, } for _, tt := range tests { From 0c7c121f4f57663766819cc06632469e12b93585 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Tue, 6 Feb 2024 15:39:17 -0800 Subject: [PATCH 11/73] Add kopia s3 and s3 compliant storage flags --- pkg/kopia/cli/internal/command/commands.go | 1 + pkg/kopia/cli/internal/flag/storage/s3/s3.go | 61 ++++++++++ .../cli/internal/flag/storage/s3/s3_flags.go | 53 +++++++++ .../internal/flag/storage/s3/s3_flags_test.go | 81 +++++++++++++ .../cli/internal/flag/storage/s3/s3_test.go | 75 +++++++++++++ .../cli/internal/flag/storage/storage.go | 3 + .../cli/internal/flag/storage/storage_test.go | 106 ++++++++++++++++++ 7 files changed, 380 insertions(+) create mode 100644 pkg/kopia/cli/internal/flag/storage/s3/s3.go create mode 100644 pkg/kopia/cli/internal/flag/storage/s3/s3_flags.go create mode 100644 pkg/kopia/cli/internal/flag/storage/s3/s3_flags_test.go create mode 100644 pkg/kopia/cli/internal/flag/storage/s3/s3_test.go diff --git a/pkg/kopia/cli/internal/command/commands.go b/pkg/kopia/cli/internal/command/commands.go index 3108e93630..ec4718dbf5 100644 --- a/pkg/kopia/cli/internal/command/commands.go +++ b/pkg/kopia/cli/internal/command/commands.go @@ -5,4 +5,5 @@ var ( FileSystem = Command{"filesystem"} GCS = Command{"gcs"} Azure = Command{"azure"} + S3 = Command{"s3"} ) diff --git a/pkg/kopia/cli/internal/flag/storage/s3/s3.go b/pkg/kopia/cli/internal/flag/storage/s3/s3.go new file mode 100644 index 0000000000..887e1e3584 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/s3/s3.go @@ -0,0 +1,61 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3 + +import ( + "strings" + + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/log" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/command" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" +) + +// New returns a builder for the S3 subcommand storage. +func New(s model.StorageFlag) (*safecli.Builder, error) { + endpoint := resolveS3Endpoint(s.Location.Endpoint(), s.GetLogger()) + prefix := model.GenerateFullRepoPath(s.Location.Prefix(), s.RepoPathPrefix) + return command.NewCommandBuilder(command.S3, + Region(s.Location.Region()), + Bucket(s.Location.BucketName()), + Endpoint(endpoint), + Prefix(prefix), + DisableTLS(s.Location.IsInsecureEndpoint()), + DisableTLSVerify(s.Location.HasSkipSSLVerify()), + ) +} + +// resolveS3Endpoint removes the trailing slash and +// protocol from provided endpoint and +// returns the absolute endpoint string. +func resolveS3Endpoint(endpoint string, logger log.Logger) string { + if endpoint == "" { + return "" + } + + if strings.HasSuffix(endpoint, "/") { + logger.Print("Removing trailing slashes from the endpoint") + endpoint = strings.TrimRight(endpoint, "/") + } + + sp := strings.SplitN(endpoint, "://", 2) + if len(sp) > 1 { + logger.Print("Removing leading protocol from the endpoint") + } + + return sp[len(sp)-1] +} diff --git a/pkg/kopia/cli/internal/flag/storage/s3/s3_flags.go b/pkg/kopia/cli/internal/flag/storage/s3/s3_flags.go new file mode 100644 index 0000000000..9688b81155 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/s3/s3_flags.go @@ -0,0 +1,53 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3 + +import ( + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" +) + +// +// S3 flags. +// + +// Bucket creates a new S3 bucket flag with a given bucket name. +func Bucket(bucket string) flag.Applier { + return flag.NewStringFlag("--bucket", bucket) +} + +// Endpoint creates a new S3 endpoint flag with a given endpoint. +func Endpoint(endpoint string) flag.Applier { + return flag.NewStringFlag("--endpoint", endpoint) +} + +// Prefix creates a new S3 prefix flag with a given prefix. +func Prefix(prefix string) flag.Applier { + return flag.NewStringFlag("--prefix", prefix) +} + +// Region creates a new S3 region flag with a given region. +func Region(region string) flag.Applier { + return flag.NewStringFlag("--region", region) +} + +// DisableTLS creates a new S3 disable TLS flag. +func DisableTLS(disable bool) flag.Applier { + return flag.NewBoolFlag("--disable-tls", disable) +} + +// DisableTLSVerify creates a new S3 disable TLS verification flag. +func DisableTLSVerify(disable bool) flag.Applier { + return flag.NewBoolFlag("--disable-tls-verification", disable) +} diff --git a/pkg/kopia/cli/internal/flag/storage/s3/s3_flags_test.go b/pkg/kopia/cli/internal/flag/storage/s3/s3_flags_test.go new file mode 100644 index 0000000000..f77ebafa5d --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/s3/s3_flags_test.go @@ -0,0 +1,81 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3 + +import ( + "testing" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" + "gopkg.in/check.v1" +) + +func TestStorageS3Flags(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(test.NewFlagSuite([]test.FlagTest{ + { + Name: "Empty Bucket should not generate a flag", + Flag: Bucket(""), + }, + { + Name: "Bucket with value should generate a flag with the given value", + Flag: Bucket("bucket"), + ExpectedCLI: []string{"--bucket=bucket"}, + }, + { + Name: "Empty Endpoint should not generate a flag", + Flag: Endpoint(""), + }, + { + Name: "Endpoint with value should generate a flag with the given value", + Flag: Endpoint("endpoint"), + ExpectedCLI: []string{"--endpoint=endpoint"}, + }, + { + Name: "Empty Prefix should not generate a flag", + Flag: Prefix(""), + }, + { + Name: "Prefix with value should generate a flag with the given value", + Flag: Prefix("prefix"), + ExpectedCLI: []string{"--prefix=prefix"}, + }, + { + Name: "Empty Region should not generate a flag", + Flag: Region(""), + }, + { + Name: "Region with value should generate a flag with the given value", + Flag: Region("region"), + ExpectedCLI: []string{"--region=region"}, + }, + { + Name: "DisableTLS(false) should not generate a flag", + Flag: DisableTLS(false), + }, + { + Name: "DisableTLS(true) should generate a flag", + Flag: DisableTLS(true), + ExpectedCLI: []string{"--disable-tls"}, + }, + { + Name: "DisableTLSVerify(false) should not generate a flag", + Flag: DisableTLSVerify(false), + }, + { + Name: "DisableTLSVerify(true) should generate a flag", + Flag: DisableTLSVerify(true), + ExpectedCLI: []string{"--disable-tls-verification"}, + }, +})) diff --git a/pkg/kopia/cli/internal/flag/storage/s3/s3_test.go b/pkg/kopia/cli/internal/flag/storage/s3/s3_test.go new file mode 100644 index 0000000000..131660c880 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/storage/s3/s3_test.go @@ -0,0 +1,75 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3 + +import ( + "testing" + + "gopkg.in/check.v1" + + "github.com/kanisterio/safecli" + + rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" +) + +func TestStorageS3(t *testing.T) { check.TestingT(t) } + +var logger = &test.StringLogger{} + +var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ + { + Name: "Empty S3 storage flag should generate subcommand with default flags", + CLI: func() (safecli.CommandBuilder, error) { + return New(model.StorageFlag{}) + }, + ExpectedCLI: []string{ + "s3", + }, + }, + { + Name: "S3 with values should generate subcommand with specific flags", + CLI: func() (safecli.CommandBuilder, error) { + return New(model.StorageFlag{ + RepoPathPrefix: "repo/path/prefix", + Location: model.Location{ + rs.PrefixKey: []byte("prefix"), + rs.EndpointKey: []byte("http://endpoint/path/"), + rs.RegionKey: []byte("region"), + rs.BucketKey: []byte("bucket"), + rs.SkipSSLVerifyKey: []byte("true"), + }, + Logger: logger, + }) + }, + ExpectedCLI: []string{ + "s3", + "--region=region", + "--bucket=bucket", + "--endpoint=endpoint/path", + "--prefix=prefix/repo/path/prefix/", + "--disable-tls", + "--disable-tls-verification", + }, + + Logger: logger, + LoggerRegex: []string{ + "Removing leading", + "Removing trailing", + }, + }, +})) diff --git a/pkg/kopia/cli/internal/flag/storage/storage.go b/pkg/kopia/cli/internal/flag/storage/storage.go index 08e36d8897..9c21745a3e 100644 --- a/pkg/kopia/cli/internal/flag/storage/storage.go +++ b/pkg/kopia/cli/internal/flag/storage/storage.go @@ -26,6 +26,7 @@ import ( "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/fs" "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/gcs" "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/s3" ) // Option is a function that sets a storage option. @@ -59,6 +60,8 @@ func Storage(location model.Location, repoPathPrefix string, opts ...Option) mod factory[rs.LocTypeFilestore] = fs.New factory[rs.LocTypeGCS] = gcs.New factory[rs.LocTypeAzure] = azure.New + factory[rs.LocTypeS3] = s3.New + factory[rs.LocTypes3Compliant] = s3.New }) // create a new storage with the given location, repo path prefix and defaults. s := model.StorageFlag{ diff --git a/pkg/kopia/cli/internal/flag/storage/storage_test.go b/pkg/kopia/cli/internal/flag/storage/storage_test.go index 09031550b4..468e65f8e8 100644 --- a/pkg/kopia/cli/internal/flag/storage/storage_test.go +++ b/pkg/kopia/cli/internal/flag/storage/storage_test.go @@ -194,6 +194,112 @@ func (s *StorageSuite) TestStorageFlag(c *check.C) { "--prefix=/path/to/prefix/prefixfs/", }, }, + { + name: "S3 should generate s3 args", + storage: Storage( + model.Location{ + rs.TypeKey: []byte("s3"), + rs.EndpointKey: []byte("http://endpoint.com"), // disable TLS + rs.PrefixKey: []byte("/path/to/prefix"), + rs.RegionKey: []byte("us-east-1"), + rs.BucketKey: []byte("bucket"), + rs.SkipSSLVerifyKey: []byte("true"), + }, + "prefixfs", + ), + expCLI: []string{ + "s3", + "--region=us-east-1", + "--bucket=bucket", + "--endpoint=endpoint.com", + "--prefix=/path/to/prefix/prefixfs/", + "--disable-tls", + "--disable-tls-verification", + }, + }, + { + name: "S3 with no prefix should use onlu repo path prefix", + storage: Storage( + model.Location{ + rs.TypeKey: []byte("s3"), + rs.EndpointKey: []byte("http://endpoint.com"), // disable TLS + rs.RegionKey: []byte("us-east-1"), + rs.BucketKey: []byte("bucket"), + rs.SkipSSLVerifyKey: []byte("true"), + }, + "prefixfs", + ), + expCLI: []string{ + "s3", + "--region=us-east-1", + "--bucket=bucket", + "--endpoint=endpoint.com", + "--prefix=prefixfs", + "--disable-tls", + "--disable-tls-verification", + }, + }, + { + name: "S3 with no endpoint should omit endpoint flag", + storage: Storage( + model.Location{ + rs.TypeKey: []byte("s3"), + rs.PrefixKey: []byte("/path/to/prefix"), + rs.RegionKey: []byte("us-east-1"), + rs.BucketKey: []byte("bucket"), + }, + "prefixfs", + ), + expCLI: []string{ + "s3", + "--region=us-east-1", + "--bucket=bucket", + "--prefix=/path/to/prefix/prefixfs/", + }, + }, + { + name: "S3 endpoint with trailing slashes should be trimmed", + storage: Storage( + model.Location{ + rs.TypeKey: []byte("s3"), + rs.EndpointKey: []byte("https://endpoint.com//////"), // slashes will be trimmed + rs.PrefixKey: []byte("/path/to/prefix"), + rs.RegionKey: []byte("us-east-1"), + rs.BucketKey: []byte("bucket"), + }, + "prefixfs", + ), + expCLI: []string{ + "s3", + "--region=us-east-1", + "--bucket=bucket", + "--endpoint=endpoint.com", + "--prefix=/path/to/prefix/prefixfs/", + }, + }, + { + name: "S3 compliant should generate s3 args", + storage: Storage( + model.Location{ + rs.TypeKey: []byte("s3Compliant"), + rs.EndpointKey: []byte("http://endpoint.com"), // disable TLS + rs.PrefixKey: []byte("/path/to/prefix"), + rs.RegionKey: []byte("us-east-1"), + rs.BucketKey: []byte("bucket"), + rs.SkipSSLVerifyKey: []byte("true"), + }, + "prefixfs", + ), + expCLI: []string{ + "s3", + "--region=us-east-1", + "--bucket=bucket", + "--endpoint=endpoint.com", + "--prefix=/path/to/prefix/prefixfs/", + "--disable-tls", + "--disable-tls-verification", + }, + }, } for _, tt := range tests { From 03df94c783ed0e81398ad3dceb6ee5a45b3dbd4f Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Tue, 6 Feb 2024 15:53:50 -0800 Subject: [PATCH 12/73] Use test.FlagSuite for storage tests --- .../cli/internal/flag/storage/storage_test.go | 364 ++++++++---------- pkg/kopia/cli/internal/test/flag_suite.go | 12 + 2 files changed, 180 insertions(+), 196 deletions(-) diff --git a/pkg/kopia/cli/internal/flag/storage/storage_test.go b/pkg/kopia/cli/internal/flag/storage/storage_test.go index 468e65f8e8..4bf1ce416f 100644 --- a/pkg/kopia/cli/internal/flag/storage/storage_test.go +++ b/pkg/kopia/cli/internal/flag/storage/storage_test.go @@ -20,18 +20,16 @@ import ( "gopkg.in/check.v1" - "github.com/pkg/errors" - "github.com/kanisterio/safecli" rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/kanister/pkg/kopia/cli/internal/command" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/fs" "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" cmdlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" ) func TestStorageFlags(t *testing.T) { check.TestingT(t) } @@ -109,217 +107,191 @@ func (s *StorageSuite) TestLocationMethods(c *check.C) { } } -func (s *StorageSuite) TestStorageFlag(c *check.C) { - tests := []struct { - name string - storage flag.Applier - expCLI []string - err error - errMsg string - }{ - { - name: "Empty Storage should generate an error", - storage: Storage(nil, ""), - err: cli.ErrUnsupportedStorage, - }, - { - name: "Filesystem without prefix and with repo path should generate repo path", - storage: Storage( - model.Location{ - rs.TypeKey: []byte("filestore"), - rs.PrefixKey: []byte(""), - }, - "dir1/subdir/", - ), - expCLI: []string{ - "filesystem", - fmt.Sprintf("--path=%s/dir1/subdir/", fs.DefaultFSMountPath), +var _ = check.Suite(test.NewFlagSuite([]test.FlagTest{ + { + Name: "Empty Storage should generate an error", + Flag: Storage(nil, ""), + ExpectedErr: cli.ErrUnsupportedStorage, + }, + { + Name: "Filesystem without prefix and with repo path should generate repo path", + Flag: Storage( + model.Location{ + rs.TypeKey: []byte("filestore"), + rs.PrefixKey: []byte(""), }, + "dir1/subdir/", + ), + ExpectedCLI: []string{ + "filesystem", + fmt.Sprintf("--path=%s/dir1/subdir/", fs.DefaultFSMountPath), }, - { - name: "Filesystem with prefix and repo path should generate merged prefix and repo path", - storage: Storage( - model.Location{ - rs.TypeKey: []byte("filestore"), - rs.PrefixKey: []byte("test-prefix"), - }, - "dir1/subdir/", - ), - expCLI: []string{ - "filesystem", - fmt.Sprintf("--path=%s/test-prefix/dir1/subdir/", fs.DefaultFSMountPath), + }, + { + Name: "Filesystem with prefix and repo path should generate merged prefix and repo path", + Flag: Storage( + model.Location{ + rs.TypeKey: []byte("filestore"), + rs.PrefixKey: []byte("test-prefix"), }, + "dir1/subdir/", + ), + ExpectedCLI: []string{ + "filesystem", + fmt.Sprintf("--path=%s/test-prefix/dir1/subdir/", fs.DefaultFSMountPath), }, - { - name: "Unsupported storage type should generate an error", - storage: Storage( - model.Location{ - rs.TypeKey: []byte("ftp"), - }, - "prefixfs", - ), - errMsg: "failed to apply storage args: unsupported location type: 'ftp': unsupported storage", - err: cli.ErrUnsupportedStorage, - }, - { - name: "GCS should generate gcs args", - storage: Storage( - model.Location{ - rs.TypeKey: []byte("gcs"), - rs.BucketKey: []byte("bucket"), - rs.PrefixKey: []byte("/path/to/prefix"), - }, - "prefixfs", - ), - expCLI: []string{ - "gcs", - "--bucket=bucket", - "--credentials-file=/tmp/creds.txt", - "--prefix=/path/to/prefix/prefixfs/", + }, + { + Name: "Unsupported storage type should generate an error", + Flag: Storage( + model.Location{ + rs.TypeKey: []byte("ftp"), + }, + "prefixfs", + ), + ExpectedErr: cli.ErrUnsupportedStorage, + ExpectedErrMsg: "failed to apply storage args: unsupported location type: 'ftp': unsupported storage", + }, + { + Name: "GCS should generate gcs args", + Flag: Storage( + model.Location{ + rs.TypeKey: []byte("gcs"), + rs.BucketKey: []byte("bucket"), + rs.PrefixKey: []byte("/path/to/prefix"), }, + "prefixfs", + ), + ExpectedCLI: []string{ + "gcs", + "--bucket=bucket", + "--credentials-file=/tmp/creds.txt", + "--prefix=/path/to/prefix/prefixfs/", }, - { - name: "Azure should generate azure args", - storage: Storage( - model.Location{ - rs.TypeKey: []byte("azure"), - rs.BucketKey: []byte("bucket"), - rs.PrefixKey: []byte("/path/to/prefix"), - }, - "prefixfs", - ), - expCLI: []string{ - "azure", - "--container=bucket", - "--prefix=/path/to/prefix/prefixfs/", + }, + { + Name: "Azure should generate azure args", + Flag: Storage( + model.Location{ + rs.TypeKey: []byte("azure"), + rs.BucketKey: []byte("bucket"), + rs.PrefixKey: []byte("/path/to/prefix"), }, + "prefixfs", + ), + ExpectedCLI: []string{ + "azure", + "--container=bucket", + "--prefix=/path/to/prefix/prefixfs/", }, - { - name: "S3 should generate s3 args", - storage: Storage( - model.Location{ - rs.TypeKey: []byte("s3"), - rs.EndpointKey: []byte("http://endpoint.com"), // disable TLS - rs.PrefixKey: []byte("/path/to/prefix"), - rs.RegionKey: []byte("us-east-1"), - rs.BucketKey: []byte("bucket"), - rs.SkipSSLVerifyKey: []byte("true"), - }, - "prefixfs", - ), - expCLI: []string{ - "s3", - "--region=us-east-1", - "--bucket=bucket", - "--endpoint=endpoint.com", - "--prefix=/path/to/prefix/prefixfs/", - "--disable-tls", - "--disable-tls-verification", + }, + { + Name: "S3 should generate s3 args", + Flag: Storage( + model.Location{ + rs.TypeKey: []byte("s3"), + rs.EndpointKey: []byte("http://endpoint.com"), // disable TLS + rs.PrefixKey: []byte("/path/to/prefix"), + rs.RegionKey: []byte("us-east-1"), + rs.BucketKey: []byte("bucket"), + rs.SkipSSLVerifyKey: []byte("true"), }, + "prefixfs", + ), + ExpectedCLI: []string{ + "s3", + "--region=us-east-1", + "--bucket=bucket", + "--endpoint=endpoint.com", + "--prefix=/path/to/prefix/prefixfs/", + "--disable-tls", + "--disable-tls-verification", }, - { - name: "S3 with no prefix should use onlu repo path prefix", - storage: Storage( - model.Location{ - rs.TypeKey: []byte("s3"), - rs.EndpointKey: []byte("http://endpoint.com"), // disable TLS - rs.RegionKey: []byte("us-east-1"), - rs.BucketKey: []byte("bucket"), - rs.SkipSSLVerifyKey: []byte("true"), - }, - "prefixfs", - ), - expCLI: []string{ - "s3", - "--region=us-east-1", - "--bucket=bucket", - "--endpoint=endpoint.com", - "--prefix=prefixfs", - "--disable-tls", - "--disable-tls-verification", + }, + { + Name: "S3 with no prefix should use onlu repo path prefix", + Flag: Storage( + model.Location{ + rs.TypeKey: []byte("s3"), + rs.EndpointKey: []byte("http://endpoint.com"), // disable TLS + rs.RegionKey: []byte("us-east-1"), + rs.BucketKey: []byte("bucket"), + rs.SkipSSLVerifyKey: []byte("true"), }, + "prefixfs", + ), + ExpectedCLI: []string{ + "s3", + "--region=us-east-1", + "--bucket=bucket", + "--endpoint=endpoint.com", + "--prefix=prefixfs", + "--disable-tls", + "--disable-tls-verification", }, - { - name: "S3 with no endpoint should omit endpoint flag", - storage: Storage( - model.Location{ - rs.TypeKey: []byte("s3"), - rs.PrefixKey: []byte("/path/to/prefix"), - rs.RegionKey: []byte("us-east-1"), - rs.BucketKey: []byte("bucket"), - }, - "prefixfs", - ), - expCLI: []string{ - "s3", - "--region=us-east-1", - "--bucket=bucket", - "--prefix=/path/to/prefix/prefixfs/", + }, + { + Name: "S3 with no endpoint should omit endpoint flag", + Flag: Storage( + model.Location{ + rs.TypeKey: []byte("s3"), + rs.PrefixKey: []byte("/path/to/prefix"), + rs.RegionKey: []byte("us-east-1"), + rs.BucketKey: []byte("bucket"), }, + "prefixfs", + ), + ExpectedCLI: []string{ + "s3", + "--region=us-east-1", + "--bucket=bucket", + "--prefix=/path/to/prefix/prefixfs/", }, - { - name: "S3 endpoint with trailing slashes should be trimmed", - storage: Storage( - model.Location{ - rs.TypeKey: []byte("s3"), - rs.EndpointKey: []byte("https://endpoint.com//////"), // slashes will be trimmed - rs.PrefixKey: []byte("/path/to/prefix"), - rs.RegionKey: []byte("us-east-1"), - rs.BucketKey: []byte("bucket"), - }, - "prefixfs", - ), - expCLI: []string{ - "s3", - "--region=us-east-1", - "--bucket=bucket", - "--endpoint=endpoint.com", - "--prefix=/path/to/prefix/prefixfs/", + }, + { + Name: "S3 endpoint with trailing slashes should be trimmed", + Flag: Storage( + model.Location{ + rs.TypeKey: []byte("s3"), + rs.EndpointKey: []byte("https://endpoint.com//////"), // slashes will be trimmed + rs.PrefixKey: []byte("/path/to/prefix"), + rs.RegionKey: []byte("us-east-1"), + rs.BucketKey: []byte("bucket"), }, + "prefixfs", + ), + ExpectedCLI: []string{ + "s3", + "--region=us-east-1", + "--bucket=bucket", + "--endpoint=endpoint.com", + "--prefix=/path/to/prefix/prefixfs/", }, - { - name: "S3 compliant should generate s3 args", - storage: Storage( - model.Location{ - rs.TypeKey: []byte("s3Compliant"), - rs.EndpointKey: []byte("http://endpoint.com"), // disable TLS - rs.PrefixKey: []byte("/path/to/prefix"), - rs.RegionKey: []byte("us-east-1"), - rs.BucketKey: []byte("bucket"), - rs.SkipSSLVerifyKey: []byte("true"), - }, - "prefixfs", - ), - expCLI: []string{ - "s3", - "--region=us-east-1", - "--bucket=bucket", - "--endpoint=endpoint.com", - "--prefix=/path/to/prefix/prefixfs/", - "--disable-tls", - "--disable-tls-verification", + }, + { + Name: "S3 compliant should generate s3 args", + Flag: Storage( + model.Location{ + rs.TypeKey: []byte("s3Compliant"), + rs.EndpointKey: []byte("http://endpoint.com"), // disable TLS + rs.PrefixKey: []byte("/path/to/prefix"), + rs.RegionKey: []byte("us-east-1"), + rs.BucketKey: []byte("bucket"), + rs.SkipSSLVerifyKey: []byte("true"), }, + "prefixfs", + ), + ExpectedCLI: []string{ + "s3", + "--region=us-east-1", + "--bucket=bucket", + "--endpoint=endpoint.com", + "--prefix=/path/to/prefix/prefixfs/", + "--disable-tls", + "--disable-tls-verification", }, - } - - for _, tt := range tests { - b := safecli.NewBuilder() - err := tt.storage.Apply(b) - - cmt := check.Commentf("FAIL: %v", tt.name) - if tt.errMsg != "" { - c.Assert(err, check.NotNil, cmt) - c.Assert(err.Error(), check.Equals, tt.errMsg, cmt) - } - - if tt.err == nil { - c.Assert(err, check.IsNil, cmt) - } else { - c.Assert(errors.Cause(err), check.Equals, tt.err, cmt) - } - c.Assert(b.Build(), check.DeepEquals, tt.expCLI, cmt) - } -} + }, +})) // MockFlagWithError is a mock flag that always returns an error type MockFlagWithError struct{} diff --git a/pkg/kopia/cli/internal/test/flag_suite.go b/pkg/kopia/cli/internal/test/flag_suite.go index c930573466..fa651114ae 100644 --- a/pkg/kopia/cli/internal/test/flag_suite.go +++ b/pkg/kopia/cli/internal/test/flag_suite.go @@ -30,6 +30,10 @@ type FlagTest struct { // If nil, no error is expected and // ExpectedCLI and ExpectedLog are checked. ExpectedErr error + + // Expected error message. (optional) + // If empty, it will be ignored. + ExpectedErrMsg string } // CheckCommentString implements check.CommentInterface @@ -50,6 +54,13 @@ func (t *FlagTest) assertError(c *check.C, err error) { c.Assert(actualErr, check.Equals, t.ExpectedErr, t) } +// assertErrorMsg checks the error message against ExpectedErrMsg. +func (t *FlagTest) assertErrorMsg(c *check.C, err error) { + if t.ExpectedErrMsg != "" { + c.Assert(err.Error(), check.Equals, t.ExpectedErrMsg, t) + } +} + // assertNoError makes sure there is no error. func (t *FlagTest) assertNoError(c *check.C, err error) { c.Assert(err, check.IsNil, t) @@ -71,6 +82,7 @@ func (t *FlagTest) Test(c *check.C, b *safecli.Builder) { err := flag.Apply(b, t.Flag) if t.ExpectedErr != nil { t.assertError(c, err) + t.assertErrorMsg(c, err) } else { t.assertNoError(c, err) t.assertCLI(c, b) From 6224599421fe65ddac8d49a185854f2ebdafce5e Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Tue, 6 Feb 2024 15:57:56 -0800 Subject: [PATCH 13/73] Fix typo --- pkg/kopia/cli/internal/flag/storage/storage_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/kopia/cli/internal/flag/storage/storage_test.go b/pkg/kopia/cli/internal/flag/storage/storage_test.go index 4bf1ce416f..54d6704c25 100644 --- a/pkg/kopia/cli/internal/flag/storage/storage_test.go +++ b/pkg/kopia/cli/internal/flag/storage/storage_test.go @@ -209,7 +209,7 @@ var _ = check.Suite(test.NewFlagSuite([]test.FlagTest{ }, }, { - Name: "S3 with no prefix should use onlu repo path prefix", + Name: "S3 with no prefix should use only repo path prefix", Flag: Storage( model.Location{ rs.TypeKey: []byte("s3"), From 02f0449a3dd3221a970d3178179e4d51092c41ac Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Tue, 6 Feb 2024 16:19:04 -0800 Subject: [PATCH 14/73] Add kopia CLI repository connect command --- pkg/kopia/cli/internal/command/command.go | 8 + .../cli/internal/command/command_test.go | 11 + pkg/kopia/cli/internal/command/commands.go | 25 ++ .../flag/repository/repository_flags.go | 65 +++++ .../flag/repository/repository_flags_test.go | 107 +++++++ pkg/kopia/cli/repository/repository_create.go | 61 ++++ pkg/kopia/cli/repository/repository_test.go | 275 ++++++++++++++++++ 7 files changed, 552 insertions(+) create mode 100644 pkg/kopia/cli/internal/flag/repository/repository_flags.go create mode 100644 pkg/kopia/cli/internal/flag/repository/repository_flags_test.go create mode 100644 pkg/kopia/cli/repository/repository_create.go create mode 100644 pkg/kopia/cli/repository/repository_test.go diff --git a/pkg/kopia/cli/internal/command/command.go b/pkg/kopia/cli/internal/command/command.go index 04acbaecd4..68c7d531e3 100644 --- a/pkg/kopia/cli/internal/command/command.go +++ b/pkg/kopia/cli/internal/command/command.go @@ -17,8 +17,10 @@ package command import ( "github.com/kanisterio/safecli" + "github.com/kanisterio/kanister/pkg/kopia/cli" clierrors "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" + flagcommon "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/common" ) // Command is a CLI command/subcommand. @@ -43,3 +45,9 @@ func NewCommandBuilder(cmd flag.Applier, flags ...flag.Applier) (*safecli.Builde } return b, nil } + +// NewKopiaCommandBuilder returns a new Kopia command builder. +func NewKopiaCommandBuilder(args cli.CommonArgs, flags ...flag.Applier) (*safecli.Builder, error) { + flags = append([]flag.Applier{flagcommon.Common(args)}, flags...) + return NewCommandBuilder(KopiaBinaryName, flags...) +} diff --git a/pkg/kopia/cli/internal/command/command_test.go b/pkg/kopia/cli/internal/command/command_test.go index f4ae074d37..70e6a34981 100644 --- a/pkg/kopia/cli/internal/command/command_test.go +++ b/pkg/kopia/cli/internal/command/command_test.go @@ -95,3 +95,14 @@ func (s *CommandSuite) TestNewCommandBuilder(c *check.C) { "--flag2", }) } + +func (s *CommandSuite) TestNewKopiaCommandBuilder(c *check.C) { + b, err := NewKopiaCommandBuilder(cli.CommonArgs{}, &mockCommandAndFlag{flagName: "--flag1"}, &mockCommandAndFlag{flagName: "--flag2"}) + c.Assert(err, check.IsNil) + c.Check(b.Build(), check.DeepEquals, []string{ + "kopia", + "--log-level=error", + "--flag1", + "--flag2", + }) +} diff --git a/pkg/kopia/cli/internal/command/commands.go b/pkg/kopia/cli/internal/command/commands.go index ec4718dbf5..73052ac336 100644 --- a/pkg/kopia/cli/internal/command/commands.go +++ b/pkg/kopia/cli/internal/command/commands.go @@ -1,5 +1,30 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package command +// KopiaBinaryName is the name of the Kopia binary. +var ( + KopiaBinaryName = Command{"kopia"} +) + +// Repository commands. +var ( + Repository = Command{"repository"} + Create = Command{"create"} +) + // Repository storage sub commands. var ( FileSystem = Command{"filesystem"} diff --git a/pkg/kopia/cli/internal/flag/repository/repository_flags.go b/pkg/kopia/cli/internal/flag/repository/repository_flags.go new file mode 100644 index 0000000000..bd586781cd --- /dev/null +++ b/pkg/kopia/cli/internal/flag/repository/repository_flags.go @@ -0,0 +1,65 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repository + +import ( + "time" + + "github.com/go-openapi/strfmt" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" +) + +// Hostname creates a new host flag with a given hostname. +func Hostname(hostname string) flag.Applier { + return flag.NewStringFlag("--override-hostname", hostname) +} + +// Username creates a new username flag with a given username. +func Username(username string) flag.Applier { + return flag.NewStringFlag("--override-username", username) +} + +// BlobRetention creates a new blob retention flag with a given mode and period. +// If mode is empty, the flag will be a no-op. +func BlobRetention(mode string, period time.Duration) flag.Applier { + if mode == "" { + return flag.EmptyFlag() + } + return flag.NewFlags( + flag.NewStringFlag("--retention-mode", mode), + flag.NewStringFlag("--retention-period", period.String()), + ) +} + +// PIT creates a new point-in-time flag with a given point-in-time. +// If pit is zero, the flag will be a no-op. +func PIT(pit strfmt.DateTime) flag.Applier { + dt := strfmt.DateTime(pit) + if time.Time(dt).IsZero() { + return flag.EmptyFlag() + } + return flag.NewStringFlag("--point-in-time", dt.String()) +} + +// ServerURL creates a new server URL flag with a given server URL. +func ServerURL(serverURL string) flag.Applier { + return flag.NewStringFlag("--url", serverURL) +} + +// ServerCertFingerprint creates a new server certificate fingerprint flag with a given fingerprint. +func ServerCertFingerprint(fingerprint string) flag.Applier { + return flag.NewRedactedStringFlag("--server-cert-fingerprint", fingerprint) +} diff --git a/pkg/kopia/cli/internal/flag/repository/repository_flags_test.go b/pkg/kopia/cli/internal/flag/repository/repository_flags_test.go new file mode 100644 index 0000000000..cb50443d85 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/repository/repository_flags_test.go @@ -0,0 +1,107 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repository + +import ( + "testing" + "time" + + "github.com/go-openapi/strfmt" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" + "gopkg.in/check.v1" +) + +func TestRepositoryFlags(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(test.NewFlagSuite([]test.FlagTest{ + { + Name: "Empty Hostname should not generate a flag", + Flag: Hostname(""), + }, + { + Name: "Hostname with value should generate a flag with given value", + Flag: Hostname("hostname"), + ExpectedCLI: []string{ + "--override-hostname=hostname", + }, + }, + { + Name: "Empty Username should not generate a flag", + Flag: Username(""), + }, + { + Name: "Username with value should generate a flag with given value", + Flag: Username("username"), + ExpectedCLI: []string{ + "--override-username=username", + }, + }, + { + Name: "Empty BlobRetention should not generate a flag", + Flag: BlobRetention("", time.Duration(0)), + }, + { + Name: "BlobRetention with values should generate multiple flags with given values", + Flag: BlobRetention("mode", 24*time.Hour), + ExpectedCLI: []string{ + "--retention-mode=mode", + "--retention-period=24h0m0s", + }, + }, + { + Name: "BlobRetention with RetentionMode only should generate mode flag with zero period", + Flag: BlobRetention("mode", 0), + ExpectedCLI: []string{ + "--retention-mode=mode", + "--retention-period=0s", + }, + }, + { + Name: "Empty PIT should not generate a flag", + Flag: PIT(strfmt.DateTime{}), + }, + { + Name: "PIT with value should generate a flag with given value", + Flag: PIT(func() strfmt.DateTime { + dt, _ := strfmt.ParseDateTime("2024-01-02T03:04:05.678Z") + return dt + }()), + ExpectedCLI: []string{ + "--point-in-time=2024-01-02T03:04:05.678Z", + }, + }, + { + Name: "Empty ServerURL should not generate a flag", + Flag: ServerURL(""), + }, + { + Name: "ServerURL with value should generate a flag with given value", + Flag: ServerURL("ServerURL"), + ExpectedCLI: []string{ + "--url=ServerURL", + }, + }, + { + Name: "Empty ServerCertFingerprint should not generate a flag", + Flag: ServerCertFingerprint(""), + }, + { + Name: "ServerCertFingerprint with value should generate a flag with given value and redact fingerprint for logs", + Flag: ServerCertFingerprint("ServerCertFingerprint"), + ExpectedCLI: []string{ + "--server-cert-fingerprint=ServerCertFingerprint", + }, + }, +})) diff --git a/pkg/kopia/cli/repository/repository_create.go b/pkg/kopia/cli/repository/repository_create.go new file mode 100644 index 0000000000..90291380c2 --- /dev/null +++ b/pkg/kopia/cli/repository/repository_create.go @@ -0,0 +1,61 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repository + +import ( + "time" + + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/log" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/command" + flagcommon "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/common" + flagrepo "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/repository" + flagstorage "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage" +) + +// CreateArgs defines the arguments for the `kopia repository create` command. +type CreateArgs struct { + cli.CommonArgs + cli.CacheArgs + + Hostname string // the hostname of the repository + Username string // the username of the repository + Location map[string][]byte // the location of the repository + RepoPathPrefix string // the prefix of the repository path + RetentionMode string // retention mode for supported storage backends + RetentionPeriod time.Duration // retention period for supported storage backends + + Logger log.Logger +} + +// Create creates a new `kopia repository create ...` command. +func Create(args CreateArgs) (safecli.CommandBuilder, error) { + return command.NewKopiaCommandBuilder(args.CommonArgs, + command.Repository, command.Create, + flagcommon.NoCheckForUpdates, + flagcommon.Cache(args.CacheArgs), + flagrepo.Hostname(args.Hostname), + flagrepo.Username(args.Username), + flagrepo.BlobRetention(args.RetentionMode, args.RetentionPeriod), + flagstorage.Storage( + args.Location, + args.RepoPathPrefix, + flagstorage.WithLogger(args.Logger), // log.Debug for old output + ), + ) +} diff --git a/pkg/kopia/cli/repository/repository_test.go b/pkg/kopia/cli/repository/repository_test.go new file mode 100644 index 0000000000..bd9ad2b720 --- /dev/null +++ b/pkg/kopia/cli/repository/repository_test.go @@ -0,0 +1,275 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repository + +import ( + "testing" + "time" + + "gopkg.in/check.v1" + + "github.com/kanisterio/safecli" + + rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" +) + +func TestRepositoryCommands(t *testing.T) { check.TestingT(t) } + +var ( + cacheArgs = cli.CacheArgs{ + CacheDirectory: "/tmp/cache.dir", + ContentCacheSizeLimitMB: 0, + MetadataCacheSizeLimitMB: 0, + } + + retentionMode = "Locked" + retentionPeriod = 15 * time.Minute + + locFS = model.Location{ + rs.TypeKey: []byte("filestore"), + rs.PrefixKey: []byte("test-prefix"), + } + + locAzure = model.Location{ + rs.TypeKey: []byte("azure"), + rs.BucketKey: []byte("test-bucket"), + rs.PrefixKey: []byte("test-prefix"), + } + + locGCS = model.Location{ + rs.TypeKey: []byte("gcs"), + rs.BucketKey: []byte("test-bucket"), + rs.PrefixKey: []byte("test-prefix"), + } + + locS3 = model.Location{ + rs.TypeKey: []byte("s3"), + rs.EndpointKey: []byte("test-endpoint"), + rs.RegionKey: []byte("test-region"), + rs.BucketKey: []byte("test-bucket"), + rs.PrefixKey: []byte("test-prefix"), + rs.SkipSSLVerifyKey: []byte("false"), + } + + locS3Compliant = model.Location{ + rs.TypeKey: []byte("s3Compliant"), + rs.EndpointKey: []byte("test-endpoint"), + rs.RegionKey: []byte("test-region"), + rs.BucketKey: []byte("test-bucket"), + rs.PrefixKey: []byte("test-prefix"), + rs.SkipSSLVerifyKey: []byte("false"), + } +) + +// Test Repository Create command +var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ + { + Name: "repository create with no storage", + CLI: func() (safecli.CommandBuilder, error) { + args := CreateArgs{ + CommonArgs: test.CommonArgs, + CacheArgs: cacheArgs, + Hostname: "test-hostname", + Username: "test-username", + RepoPathPrefix: "test-path/prefix", + } + return Create(args) + }, + ExpectedErr: cli.ErrUnsupportedStorage, + }, + { + Name: "repository create with filestore location", + CLI: func() (safecli.CommandBuilder, error) { + args := CreateArgs{ + CommonArgs: test.CommonArgs, + CacheArgs: cacheArgs, + Hostname: "test-hostname", + Username: "test-username", + RepoPathPrefix: "test-path/prefix", + Location: locFS, + RetentionMode: retentionMode, + RetentionPeriod: retentionPeriod, + } + return Create(args) + }, + ExpectedCLI: []string{"kopia", + "--config-file=path/kopia.config", + "--log-level=error", + "--log-dir=cache/log", + "--password=encr-key", + "repository", + "create", + "--no-check-for-updates", + "--cache-directory=/tmp/cache.dir", + "--content-cache-size-limit-mb=0", + "--metadata-cache-size-limit-mb=0", + "--override-hostname=test-hostname", + "--override-username=test-username", + "--retention-mode=Locked", + "--retention-period=15m0s", + "filesystem", + "--path=/mnt/data/test-prefix/test-path/prefix/", + }, + }, + { + Name: "repository create with azure location", + CLI: func() (safecli.CommandBuilder, error) { + args := CreateArgs{ + CommonArgs: test.CommonArgs, + CacheArgs: cacheArgs, + Hostname: "test-hostname", + Username: "test-username", + RepoPathPrefix: "test-path/prefix", + Location: locAzure, + RetentionMode: retentionMode, + RetentionPeriod: retentionPeriod, + } + return Create(args) + }, + ExpectedCLI: []string{"kopia", + "--config-file=path/kopia.config", + "--log-level=error", + "--log-dir=cache/log", + "--password=encr-key", + "repository", + "create", + "--no-check-for-updates", + "--cache-directory=/tmp/cache.dir", + "--content-cache-size-limit-mb=0", + "--metadata-cache-size-limit-mb=0", + "--override-hostname=test-hostname", + "--override-username=test-username", + "--retention-mode=Locked", + "--retention-period=15m0s", + "azure", + "--container=test-bucket", + "--prefix=test-prefix/test-path/prefix/", + }, + }, + { + Name: "repository create with gcs location", + CLI: func() (safecli.CommandBuilder, error) { + args := CreateArgs{ + CommonArgs: test.CommonArgs, + CacheArgs: cacheArgs, + Hostname: "test-hostname", + Username: "test-username", + RepoPathPrefix: "test-path/prefix", + Location: locGCS, + RetentionMode: retentionMode, + RetentionPeriod: retentionPeriod, + } + return Create(args) + }, + ExpectedCLI: []string{"kopia", + "--config-file=path/kopia.config", + "--log-level=error", + "--log-dir=cache/log", + "--password=encr-key", + "repository", + "create", + "--no-check-for-updates", + "--cache-directory=/tmp/cache.dir", + "--content-cache-size-limit-mb=0", + "--metadata-cache-size-limit-mb=0", + "--override-hostname=test-hostname", + "--override-username=test-username", + "--retention-mode=Locked", + "--retention-period=15m0s", + "gcs", + "--bucket=test-bucket", + "--credentials-file=/tmp/creds.txt", + "--prefix=test-prefix/test-path/prefix/", + }, + }, + { + Name: "repository create with s3 location", + CLI: func() (safecli.CommandBuilder, error) { + args := CreateArgs{ + CommonArgs: test.CommonArgs, + CacheArgs: cacheArgs, + Hostname: "test-hostname", + Username: "test-username", + RepoPathPrefix: "test-path/prefix", + Location: locS3, + RetentionMode: retentionMode, + RetentionPeriod: retentionPeriod, + } + return Create(args) + }, + ExpectedCLI: []string{"kopia", + "--config-file=path/kopia.config", + "--log-level=error", + "--log-dir=cache/log", + "--password=encr-key", + "repository", + "create", + "--no-check-for-updates", + "--cache-directory=/tmp/cache.dir", + "--content-cache-size-limit-mb=0", + "--metadata-cache-size-limit-mb=0", + "--override-hostname=test-hostname", + "--override-username=test-username", + "--retention-mode=Locked", + "--retention-period=15m0s", + "s3", + "--region=test-region", + "--bucket=test-bucket", + "--endpoint=test-endpoint", + "--prefix=test-prefix/test-path/prefix/", + }, + }, + { + Name: "repository create with s3 compliant location", + CLI: func() (safecli.CommandBuilder, error) { + args := CreateArgs{ + CommonArgs: test.CommonArgs, + CacheArgs: cacheArgs, + Hostname: "test-hostname", + Username: "test-username", + RepoPathPrefix: "test-path/prefix", + Location: locS3Compliant, + RetentionMode: retentionMode, + RetentionPeriod: retentionPeriod, + } + return Create(args) + }, + ExpectedCLI: []string{"kopia", + "--config-file=path/kopia.config", + "--log-level=error", + "--log-dir=cache/log", + "--password=encr-key", + "repository", + "create", + "--no-check-for-updates", + "--cache-directory=/tmp/cache.dir", + "--content-cache-size-limit-mb=0", + "--metadata-cache-size-limit-mb=0", + "--override-hostname=test-hostname", + "--override-username=test-username", + "--retention-mode=Locked", + "--retention-period=15m0s", + "s3", + "--region=test-region", + "--bucket=test-bucket", + "--endpoint=test-endpoint", + "--prefix=test-prefix/test-path/prefix/", + }, + }, +})) From 067ecc3c285f3711265fe0fc5d28a9cb773e3bf1 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Tue, 6 Feb 2024 16:28:41 -0800 Subject: [PATCH 15/73] Add kopia CLI repository connect command --- pkg/kopia/cli/internal/command/commands.go | 1 + .../cli/repository/repository_connect.go | 57 +++++++++++++++ pkg/kopia/cli/repository/repository_test.go | 70 +++++++++++++++++++ 3 files changed, 128 insertions(+) create mode 100644 pkg/kopia/cli/repository/repository_connect.go diff --git a/pkg/kopia/cli/internal/command/commands.go b/pkg/kopia/cli/internal/command/commands.go index 73052ac336..fb54d5dbd2 100644 --- a/pkg/kopia/cli/internal/command/commands.go +++ b/pkg/kopia/cli/internal/command/commands.go @@ -23,6 +23,7 @@ var ( var ( Repository = Command{"repository"} Create = Command{"create"} + Connect = Command{"connect"} ) // Repository storage sub commands. diff --git a/pkg/kopia/cli/repository/repository_connect.go b/pkg/kopia/cli/repository/repository_connect.go new file mode 100644 index 0000000000..5b4e20dae4 --- /dev/null +++ b/pkg/kopia/cli/repository/repository_connect.go @@ -0,0 +1,57 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repository + +import ( + "github.com/go-openapi/strfmt" + + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/command" + flagcommon "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/common" + flagrepo "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/repository" + flagstorage "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage" + "github.com/kanisterio/kanister/pkg/log" +) + +// ConnectArgs defines the arguments for the `kopia repository connect` command. +type ConnectArgs struct { + cli.CommonArgs + cli.CacheArgs + + Hostname string // the hostname of the repository + Username string // the username of the repository + Location map[string][]byte // the location of the repository + RepoPathPrefix string // the prefix of the repository path + ReadOnly bool // connect to a repository in read-only mode + PointInTime strfmt.DateTime // connect to a repository as it was at a specific point in time + + Logger log.Logger +} + +// Connect creates a new `kopia repository connect ...` command. +func Connect(args ConnectArgs) (safecli.CommandBuilder, error) { + return command.NewKopiaCommandBuilder(args.CommonArgs, + command.Repository, command.Connect, + flagcommon.NoCheckForUpdates, + flagcommon.ReadOnly(args.ReadOnly), + flagcommon.Cache(args.CacheArgs), + flagrepo.Hostname(args.Hostname), + flagrepo.Username(args.Username), + flagstorage.Storage(args.Location, args.RepoPathPrefix, flagstorage.WithLogger(args.Logger)), + flagrepo.PIT(args.PointInTime), + ) +} diff --git a/pkg/kopia/cli/repository/repository_test.go b/pkg/kopia/cli/repository/repository_test.go index bd9ad2b720..ace602952b 100644 --- a/pkg/kopia/cli/repository/repository_test.go +++ b/pkg/kopia/cli/repository/repository_test.go @@ -20,6 +20,7 @@ import ( "gopkg.in/check.v1" + "github.com/go-openapi/strfmt" "github.com/kanisterio/safecli" rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" @@ -273,3 +274,72 @@ var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ }, }, })) + +// Test Repository Connect command +var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ + { + Name: "repository connect with default retention", + CLI: func() (safecli.CommandBuilder, error) { + args := ConnectArgs{ + CommonArgs: test.CommonArgs, + CacheArgs: cacheArgs, + Hostname: "test-hostname", + Username: "test-username", + RepoPathPrefix: "test-path/prefix", + Location: locFS, + } + return Connect(args) + }, + ExpectedCLI: []string{"kopia", + "--config-file=path/kopia.config", + "--log-level=error", + "--log-dir=cache/log", + "--password=encr-key", + "repository", + "connect", + "--no-check-for-updates", + "--cache-directory=/tmp/cache.dir", + "--content-cache-size-limit-mb=0", + "--metadata-cache-size-limit-mb=0", + "--override-hostname=test-hostname", + "--override-username=test-username", + "filesystem", + "--path=/mnt/data/test-prefix/test-path/prefix/", + }, + }, + { + Name: "repository connect with PIT and ReadOnly", + CLI: func() (safecli.CommandBuilder, error) { + pit, _ := strfmt.ParseDateTime("2021-02-03T01:02:03Z") + args := ConnectArgs{ + CommonArgs: test.CommonArgs, + CacheArgs: cacheArgs, + Hostname: "test-hostname", + Username: "test-username", + RepoPathPrefix: "test-path/prefix", + Location: locFS, + PointInTime: pit, + ReadOnly: true, + } + return Connect(args) + }, + ExpectedCLI: []string{"kopia", + "--config-file=path/kopia.config", + "--log-level=error", + "--log-dir=cache/log", + "--password=encr-key", + "repository", + "connect", + "--no-check-for-updates", + "--readonly", + "--cache-directory=/tmp/cache.dir", + "--content-cache-size-limit-mb=0", + "--metadata-cache-size-limit-mb=0", + "--override-hostname=test-hostname", + "--override-username=test-username", + "filesystem", + "--path=/mnt/data/test-prefix/test-path/prefix/", + "--point-in-time=2021-02-03T01:02:03.000Z", + }, + }, +})) From bb8f0d581f954d412092a293a45dda295b800cae Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Tue, 6 Feb 2024 17:23:59 -0800 Subject: [PATCH 16/73] Add kopia CLI repository connect server command --- pkg/kopia/cli/internal/command/commands.go | 1 + .../repository/repository_connect_server.go | 55 +++++++++++++++++++ pkg/kopia/cli/repository/repository_test.go | 38 +++++++++++++ 3 files changed, 94 insertions(+) create mode 100644 pkg/kopia/cli/repository/repository_connect_server.go diff --git a/pkg/kopia/cli/internal/command/commands.go b/pkg/kopia/cli/internal/command/commands.go index fb54d5dbd2..f5d6ce2a5c 100644 --- a/pkg/kopia/cli/internal/command/commands.go +++ b/pkg/kopia/cli/internal/command/commands.go @@ -24,6 +24,7 @@ var ( Repository = Command{"repository"} Create = Command{"create"} Connect = Command{"connect"} + Server = Command{"server"} ) // Repository storage sub commands. diff --git a/pkg/kopia/cli/repository/repository_connect_server.go b/pkg/kopia/cli/repository/repository_connect_server.go new file mode 100644 index 0000000000..8286d6d008 --- /dev/null +++ b/pkg/kopia/cli/repository/repository_connect_server.go @@ -0,0 +1,55 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repository + +import ( + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/log" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/command" + flagcommon "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/common" + flagrepo "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/repository" +) + +// ConnectServerArgs defines the arguments for the `kopia repository connect server` command. +type ConnectServerArgs struct { + cli.CommonArgs + cli.CacheArgs + + Hostname string // hostname of the repository + Username string // username of the repository + ServerURL string // URL of the Kopia Repository API server + Fingerprint string // fingerprint of the server's TLS certificate + ReadOnly bool // connect to a repository in read-only mode + + Logger log.Logger +} + +// ConnectServer creates a new `kopia repository connect server...` command. +func ConnectServer(args ConnectServerArgs) (safecli.CommandBuilder, error) { + return command.NewKopiaCommandBuilder(args.CommonArgs, + command.Repository, command.Connect, command.Server, + flagcommon.NoCheckForUpdates, + flagcommon.NoGRPC, + flagcommon.ReadOnly(args.ReadOnly), + flagcommon.Cache(args.CacheArgs), + flagrepo.Hostname(args.Hostname), + flagrepo.Username(args.Username), + flagrepo.ServerURL(args.ServerURL), + flagrepo.ServerCertFingerprint(args.Fingerprint), + ) +} diff --git a/pkg/kopia/cli/repository/repository_test.go b/pkg/kopia/cli/repository/repository_test.go index ace602952b..76b3418bc4 100644 --- a/pkg/kopia/cli/repository/repository_test.go +++ b/pkg/kopia/cli/repository/repository_test.go @@ -343,3 +343,41 @@ var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ }, }, })) + +// Test Repository Connect Server command +var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ + { + Name: "repository connect server", + CLI: func() (safecli.CommandBuilder, error) { + args := ConnectServerArgs{ + CommonArgs: test.CommonArgs, + CacheArgs: cacheArgs, + Hostname: "test-hostname", + Username: "test-username", + ServerURL: "http://test-server", + Fingerprint: "test-fingerprint", + ReadOnly: true, + } + return ConnectServer(args) + }, + ExpectedCLI: []string{"kopia", + "--config-file=path/kopia.config", + "--log-level=error", + "--log-dir=cache/log", + "--password=encr-key", + "repository", + "connect", + "server", + "--no-check-for-updates", + "--no-grpc", + "--readonly", + "--cache-directory=/tmp/cache.dir", + "--content-cache-size-limit-mb=0", + "--metadata-cache-size-limit-mb=0", + "--override-hostname=test-hostname", + "--override-username=test-username", + "--url=http://test-server", + "--server-cert-fingerprint=test-fingerprint", + }, + }, +})) From c5da31730d19963617fc808d3318ffe5b98703d9 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Tue, 6 Feb 2024 17:34:15 -0800 Subject: [PATCH 17/73] Add kopia CLI repository set-parameters command --- pkg/kopia/cli/internal/command/commands.go | 9 +-- .../repository/repository_set_parameters.go | 45 ++++++++++++++ pkg/kopia/cli/repository/repository_test.go | 62 +++++++++++++++++++ 3 files changed, 112 insertions(+), 4 deletions(-) create mode 100644 pkg/kopia/cli/repository/repository_set_parameters.go diff --git a/pkg/kopia/cli/internal/command/commands.go b/pkg/kopia/cli/internal/command/commands.go index f5d6ce2a5c..73ed075440 100644 --- a/pkg/kopia/cli/internal/command/commands.go +++ b/pkg/kopia/cli/internal/command/commands.go @@ -21,10 +21,11 @@ var ( // Repository commands. var ( - Repository = Command{"repository"} - Create = Command{"create"} - Connect = Command{"connect"} - Server = Command{"server"} + Repository = Command{"repository"} + Create = Command{"create"} + Connect = Command{"connect"} + Server = Command{"server"} + SetParameters = Command{"set-parameters"} ) // Repository storage sub commands. diff --git a/pkg/kopia/cli/repository/repository_set_parameters.go b/pkg/kopia/cli/repository/repository_set_parameters.go new file mode 100644 index 0000000000..80bc98d6bb --- /dev/null +++ b/pkg/kopia/cli/repository/repository_set_parameters.go @@ -0,0 +1,45 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repository + +import ( + "time" + + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/log" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/command" + flagrepo "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/repository" +) + +// SetParametersArgs defines the arguments for the `kopia repository set-parameters ...` command. +type SetParametersArgs struct { + cli.CommonArgs + + RetentionMode string // retention mode for supported storage backends + RetentionPeriod time.Duration // retention period for supported storage backends + + Logger log.Logger +} + +// SetParameters creates a new `kopia repository set-parameters ...` command. +func SetParameters(args SetParametersArgs) (safecli.CommandBuilder, error) { + return command.NewKopiaCommandBuilder(args.CommonArgs, + command.Repository, command.SetParameters, + flagrepo.BlobRetention(args.RetentionMode, args.RetentionPeriod), + ) +} diff --git a/pkg/kopia/cli/repository/repository_test.go b/pkg/kopia/cli/repository/repository_test.go index 76b3418bc4..0e4aba9b1a 100644 --- a/pkg/kopia/cli/repository/repository_test.go +++ b/pkg/kopia/cli/repository/repository_test.go @@ -381,3 +381,65 @@ var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ }, }, })) + +// Test Repository Set Parameters command +var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ + { + Name: "repository set-parameters with default retention", + CLI: func() (safecli.CommandBuilder, error) { + args := SetParametersArgs{ + CommonArgs: test.CommonArgs, + } + return SetParameters(args) + }, + ExpectedCLI: []string{"kopia", + "--config-file=path/kopia.config", + "--log-level=error", + "--log-dir=cache/log", + "--password=encr-key", + "repository", + "set-parameters", + }, + }, + { + Name: "repository set-parameters with custom retention args", + CLI: func() (safecli.CommandBuilder, error) { + args := SetParametersArgs{ + CommonArgs: test.CommonArgs, + RetentionMode: retentionMode, + RetentionPeriod: retentionPeriod, + } + return SetParameters(args) + }, + ExpectedCLI: []string{"kopia", + "--config-file=path/kopia.config", + "--log-level=error", + "--log-dir=cache/log", + "--password=encr-key", + "repository", + "set-parameters", + "--retention-mode=Locked", + "--retention-period=15m0s", + }, + }, + { + Name: "repository set-parameters with custom retention mode only", + CLI: func() (safecli.CommandBuilder, error) { + args := SetParametersArgs{ + CommonArgs: test.CommonArgs, + RetentionMode: retentionMode, + } + return SetParameters(args) + }, + ExpectedCLI: []string{"kopia", + "--config-file=path/kopia.config", + "--log-level=error", + "--log-dir=cache/log", + "--password=encr-key", + "repository", + "set-parameters", + "--retention-mode=Locked", + "--retention-period=0s", + }, + }, +})) From eec4a162a5263a2fccf3d7e0bc57a7613c8b47d6 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Tue, 6 Feb 2024 17:43:51 -0800 Subject: [PATCH 18/73] Add kopia CLI repository status command --- pkg/kopia/cli/internal/command/commands.go | 1 + pkg/kopia/cli/repository/repository_status.go | 42 +++++++++++++++++++ pkg/kopia/cli/repository/repository_test.go | 40 ++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 pkg/kopia/cli/repository/repository_status.go diff --git a/pkg/kopia/cli/internal/command/commands.go b/pkg/kopia/cli/internal/command/commands.go index 73ed075440..ba71923a9f 100644 --- a/pkg/kopia/cli/internal/command/commands.go +++ b/pkg/kopia/cli/internal/command/commands.go @@ -26,6 +26,7 @@ var ( Connect = Command{"connect"} Server = Command{"server"} SetParameters = Command{"set-parameters"} + Status = Command{"status"} ) // Repository storage sub commands. diff --git a/pkg/kopia/cli/repository/repository_status.go b/pkg/kopia/cli/repository/repository_status.go new file mode 100644 index 0000000000..fe2ad99906 --- /dev/null +++ b/pkg/kopia/cli/repository/repository_status.go @@ -0,0 +1,42 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repository + +import ( + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/log" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/command" + flagcommon "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/common" +) + +// StatusArgs defines the arguments for the `kopia repository status ...` command. +type StatusArgs struct { + cli.CommonArgs + + JSONOutput bool // shows the output in JSON format + + Logger log.Logger +} + +// Status creates a new `kopia repository status ...` command. +func Status(args StatusArgs) (safecli.CommandBuilder, error) { + return command.NewKopiaCommandBuilder(args.CommonArgs, + command.Repository, command.Status, + flagcommon.JSONOutput(args.JSONOutput), + ) +} diff --git a/pkg/kopia/cli/repository/repository_test.go b/pkg/kopia/cli/repository/repository_test.go index 0e4aba9b1a..8d0c26ce99 100644 --- a/pkg/kopia/cli/repository/repository_test.go +++ b/pkg/kopia/cli/repository/repository_test.go @@ -443,3 +443,43 @@ var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ }, }, })) + +// Test Repository Status command +var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ + { + Name: "repository status with default args", + CLI: func() (safecli.CommandBuilder, error) { + args := StatusArgs{ + CommonArgs: test.CommonArgs, + } + return Status(args) + }, + ExpectedCLI: []string{"kopia", + "--config-file=path/kopia.config", + "--log-level=error", + "--log-dir=cache/log", + "--password=encr-key", + "repository", + "status", + }, + }, + { + Name: "repository status with JSON output", + CLI: func() (safecli.CommandBuilder, error) { + args := StatusArgs{ + CommonArgs: test.CommonArgs, + JSONOutput: true, + } + return Status(args) + }, + ExpectedCLI: []string{"kopia", + "--config-file=path/kopia.config", + "--log-level=error", + "--log-dir=cache/log", + "--password=encr-key", + "repository", + "status", + "--json", + }, + }, +})) From 9850f4a450fe5a0ce37ab18189987b88d05dce00 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Mon, 12 Feb 2024 12:11:54 -0800 Subject: [PATCH 19/73] Fix Apply and test.Suit Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/flag/flag.go | 2 +- pkg/kopia/cli/internal/test/flag_suite.go | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/kopia/cli/internal/flag/flag.go b/pkg/kopia/cli/internal/flag/flag.go index 66148034a1..39a357be53 100644 --- a/pkg/kopia/cli/internal/flag/flag.go +++ b/pkg/kopia/cli/internal/flag/flag.go @@ -36,7 +36,7 @@ func Apply(cli safecli.CommandAppender, flags ...Applier) error { if flag == nil { // if the flag is nil, skip it continue } - if err := flag.Apply(cli); err != nil { + if err := flag.Apply(sub); err != nil { return err } } diff --git a/pkg/kopia/cli/internal/test/flag_suite.go b/pkg/kopia/cli/internal/test/flag_suite.go index 7c41a1ea3a..7fe60d909c 100644 --- a/pkg/kopia/cli/internal/test/flag_suite.go +++ b/pkg/kopia/cli/internal/test/flag_suite.go @@ -62,7 +62,9 @@ func (t *FlagTest) assertNoError(c *check.C, err error) { // assertCLI asserts the builder's CLI output against ExpectedCLI. func (t *FlagTest) assertCLI(c *check.C, b *safecli.Builder) { - c.Check(b.Build(), check.DeepEquals, t.ExpectedCLI, t) + if t.ExpectedCLI != nil { + c.Check(b.Build(), check.DeepEquals, t.ExpectedCLI, t) + } } // assertLog asserts the builder's log output against ExpectedLog. @@ -74,11 +76,11 @@ func (t *FlagTest) assertLog(c *check.C, b *safecli.Builder) { // Test runs the flag test. func (ft *FlagTest) Test(c *check.C, b *safecli.Builder) { err := flag.Apply(b, ft.Flag) + ft.assertCLI(c, b) if ft.ExpectedErr != nil { ft.assertError(c, err) } else { ft.assertNoError(c, err) - ft.assertCLI(c, b) ft.assertLog(c, b) } } From 246e1c1aac53b73be773197f8df1a9748a76970a Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Mon, 12 Feb 2024 13:21:04 -0800 Subject: [PATCH 20/73] Remove variadic args for Common and Cache flags Signed-off-by: pavel.larkin --- .../cli/internal/flag/common/common_flags.go | 28 +++---------------- .../internal/flag/common/common_flags_test.go | 14 ++-------- 2 files changed, 6 insertions(+), 36 deletions(-) diff --git a/pkg/kopia/cli/internal/flag/common/common_flags.go b/pkg/kopia/cli/internal/flag/common/common_flags.go index 6df9b1bb5c..3c3000885f 100644 --- a/pkg/kopia/cli/internal/flag/common/common_flags.go +++ b/pkg/kopia/cli/internal/flag/common/common_flags.go @@ -138,18 +138,8 @@ func (f common) Apply(cmd safecli.CommandAppender) error { } // Common creates a new common flag. -// If no arguments are provided, the default common flags are used. -// If one argument is provided, the common flags are used. -// If more than one argument is provided, ErrInvalidCommonArgs is returned. -func Common(args ...cli.CommonArgs) flag.Applier { - switch len(args) { - case 0: - return common{cli.CommonArgs{}} - case 1: - return common{args[0]} - default: - return flag.ErrorFlag(cli.ErrInvalidCommonArgs) - } +func Common(args cli.CommonArgs) flag.Applier { + return common{args} } // cache defines cache flags and implements Applier interface for the cache flags. @@ -169,18 +159,8 @@ func (f cache) Apply(cmd safecli.CommandAppender) error { } // Cache creates a new cache flag. -// If no arguments are provided, the default cache flags are used. -// If one argument is provided, the cache flags are used. -// If more than one argument is provided, ErrInvalidCacheArgs is returned. -func Cache(args ...cli.CacheArgs) flag.Applier { - switch len(args) { - case 0: - return cache{cli.CacheArgs{}} - case 1: - return cache{args[0]} - default: - return flag.ErrorFlag(cli.ErrInvalidCacheArgs) - } +func Cache(args cli.CacheArgs) flag.Applier { + return cache{args} } // JSONOutput creates a new JSON output flag. diff --git a/pkg/kopia/cli/internal/flag/common/common_flags_test.go b/pkg/kopia/cli/internal/flag/common/common_flags_test.go index 2e8ac8cfc1..a7d03f571f 100644 --- a/pkg/kopia/cli/internal/flag/common/common_flags_test.go +++ b/pkg/kopia/cli/internal/flag/common/common_flags_test.go @@ -124,14 +124,9 @@ var _ = check.Suite(test.NewFlagSuite([]test.FlagTest{ }, { Name: "Empty Common should generate a flag with default value(s)", - Flag: Common(), + Flag: Common(cli.CommonArgs{}), ExpectedCLI: []string{"--log-level=error"}, }, - { - Name: "Common with more than one cli.CommonArgs should generate an error", - Flag: Common(cli.CommonArgs{}, cli.CommonArgs{}), - ExpectedErr: cli.ErrInvalidCommonArgs, - }, { Name: "Common with values should generate multiple flags with the given values and redact password for logs", Flag: Common(cli.CommonArgs{ @@ -149,18 +144,13 @@ var _ = check.Suite(test.NewFlagSuite([]test.FlagTest{ }, { Name: "Empty FlagCacheArgs should generate multiple flags with default values", - Flag: Cache(), + Flag: Cache(cli.CacheArgs{}), ExpectedCLI: []string{ "--cache-directory=/tmp/kopia-cache", "--content-cache-size-limit-mb=0", "--metadata-cache-size-limit-mb=0", }, }, - { - Name: "Cache with more than one cli.CacheArgs should generate an error", - Flag: Cache(cli.CacheArgs{}, cli.CacheArgs{}), - ExpectedErr: cli.ErrInvalidCacheArgs, - }, { Name: "Cache with CacheArgs should generate multiple cache related flags", Flag: Cache(cli.CacheArgs{ From 24707e513c3322a79d1e61c0a24d814153b1de8d Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 16 Feb 2024 16:51:58 -0800 Subject: [PATCH 21/73] pkg/kopia/cli/internal/flag is implemented in the safecli@v0.0.4 now Signed-off-by: pavel.larkin --- go.mod | 2 +- go.sum | 2 + pkg/kopia/cli/errors.go | 25 --- pkg/kopia/cli/internal/flag/bool_flag.go | 45 ------ pkg/kopia/cli/internal/flag/flag.go | 81 ---------- pkg/kopia/cli/internal/flag/flag_test.go | 171 --------------------- pkg/kopia/cli/internal/flag/string_flag.go | 78 ---------- pkg/kopia/cli/internal/test/flag_suite.go | 114 -------------- 8 files changed, 3 insertions(+), 515 deletions(-) delete mode 100644 pkg/kopia/cli/errors.go delete mode 100644 pkg/kopia/cli/internal/flag/bool_flag.go delete mode 100644 pkg/kopia/cli/internal/flag/flag.go delete mode 100644 pkg/kopia/cli/internal/flag/flag_test.go delete mode 100644 pkg/kopia/cli/internal/flag/string_flag.go delete mode 100644 pkg/kopia/cli/internal/test/flag_suite.go diff --git a/go.mod b/go.mod index 0b81ec2854..afa1376f0e 100644 --- a/go.mod +++ b/go.mod @@ -216,7 +216,7 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) -require github.com/kanisterio/safecli v0.0.3 +require github.com/kanisterio/safecli v0.0.4 require ( github.com/Azure/go-autorest/autorest v0.11.27 // indirect diff --git a/go.sum b/go.sum index fed0f3304c..8247482a7e 100644 --- a/go.sum +++ b/go.sum @@ -361,6 +361,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kanisterio/safecli v0.0.3 h1:ts3oRVSoRexFBv9pOdWYZsN4k068pWx5Cl4zN54mQro= github.com/kanisterio/safecli v0.0.3/go.mod h1:fK3Mcbeiso+NtkUdhGugK0Vf4S2l8ObvFf564ry1W5A= +github.com/kanisterio/safecli v0.0.4 h1:8pO7Zhm9YeW47XQZNRt/AaXj7kSPeIbaV7aRDRtHs4k= +github.com/kanisterio/safecli v0.0.4/go.mod h1:KBraqj8mdv2cwAr9wecknGUb8jztTzUik0r7uE6yRA8= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734 h1:qulsCaCv+O2y9/sQ9nd5KChnAgFOWakTHQ9ZADjs6DQ= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734/go.mod h1:rdqSnvOJuKCPFW/h2rVLuXOAkRnHHdp9PZcKx4HCoDM= github.com/kastenhq/stow v0.2.6-kasten.1.0.20231101232131-9321daa23aae h1:2cl4yuAJpdmLCx7G8eIsfNlQBLEfw0JDj6mTTyqc5qg= diff --git a/pkg/kopia/cli/errors.go b/pkg/kopia/cli/errors.go deleted file mode 100644 index 946dd04e87..0000000000 --- a/pkg/kopia/cli/errors.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cli - -import ( - "github.com/pkg/errors" -) - -// flag errors -var ( - // ErrInvalidFlag is returned when the flag name is empty. - ErrInvalidFlag = errors.New("invalid flag") -) diff --git a/pkg/kopia/cli/internal/flag/bool_flag.go b/pkg/kopia/cli/internal/flag/bool_flag.go deleted file mode 100644 index 4eea665cb2..0000000000 --- a/pkg/kopia/cli/internal/flag/bool_flag.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package flag - -import ( - "github.com/kanisterio/safecli" - - "github.com/kanisterio/kanister/pkg/kopia/cli" -) - -// boolFlag defines a boolean flag with a given flag name. -// If enabled is set to true, the flag is applied; otherwise, it is not. -type boolFlag struct { - flag string - enabled bool -} - -// Apply appends the flag to the command if the flag is enabled. -func (f boolFlag) Apply(cli safecli.CommandAppender) error { - if f.enabled { - cli.AppendLoggable(f.flag) - } - return nil -} - -// NewBoolFlag creates a new bool flag with a given flag name. -// If the flag name is empty, cli.ErrInvalidFlag is returned. -func NewBoolFlag(flag string, enabled bool) Applier { - if flag == "" { - return ErrorFlag(cli.ErrInvalidFlag) - } - return boolFlag{flag, enabled} -} diff --git a/pkg/kopia/cli/internal/flag/flag.go b/pkg/kopia/cli/internal/flag/flag.go deleted file mode 100644 index 39a357be53..0000000000 --- a/pkg/kopia/cli/internal/flag/flag.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package flag - -import ( - "github.com/kanisterio/safecli" -) - -// Applier applies flags/args to the command. -type Applier interface { - // Apply applies the flags/args to the command. - Apply(cli safecli.CommandAppender) error -} - -// Apply appends multiple flags to the CLI. -// If any of the flags encounter an error during the Apply process, -// the error is returned and no changes are made to the CLI. -// If no error, the flags are appended to the CLI. -func Apply(cli safecli.CommandAppender, flags ...Applier) error { - // create a new sub builder which will be used to apply the flags - // to avoid mutating the CLI if an error is encountered. - sub := safecli.NewBuilder() - for _, flag := range flags { - if flag == nil { // if the flag is nil, skip it - continue - } - if err := flag.Apply(sub); err != nil { - return err - } - } - cli.Append(sub) - return nil -} - -// flags defines a collection of Flags. -type flags []Applier - -// Apply applies the flags to the CLI. -func (flags flags) Apply(cli safecli.CommandAppender) error { - return Apply(cli, flags...) -} - -// NewFlags creates a new collection of flags. -func NewFlags(fs ...Applier) Applier { - return flags(fs) -} - -// simpleFlag is a simple implementation of the Applier interface. -type simpleFlag struct { - err error -} - -// Apply does nothing except return an error if one is set. -func (f simpleFlag) Apply(safecli.CommandAppender) error { - return f.err -} - -// EmptyFlag creates a new flag that does nothing. -// It is useful for creating a no-op flag when a condition is not met -// but Applier interface is required. -func EmptyFlag() Applier { - return simpleFlag{} -} - -// ErrorFlag creates a new flag that returns an error when applied. -// It is useful for creating a flag validation if a condition is not met. -func ErrorFlag(err error) Applier { - return simpleFlag{err} -} diff --git a/pkg/kopia/cli/internal/flag/flag_test.go b/pkg/kopia/cli/internal/flag/flag_test.go deleted file mode 100644 index c20a46b450..0000000000 --- a/pkg/kopia/cli/internal/flag/flag_test.go +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package flag_test - -import ( - "errors" - "testing" - - "gopkg.in/check.v1" - - "github.com/kanisterio/safecli" - - "github.com/kanisterio/kanister/pkg/kopia/cli" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" -) - -var ( - ErrFlag = errors.New("flag error") -) - -// MockFlagApplier is a mock implementation of the FlagApplier interface. -type MockFlagApplier struct { - flagName string - applyErr error -} - -func (m *MockFlagApplier) Apply(cli safecli.CommandAppender) error { - cli.AppendLoggable(m.flagName) - return m.applyErr -} - -func TestApply(t *testing.T) { check.TestingT(t) } - -var _ = check.Suite(&test.FlagSuite{Cmd: "cmd", Tests: []test.FlagTest{ - { - Name: "Apply with no flags should generate only the command", - ExpectedCLI: []string{"cmd"}, - }, - { - Name: "Apply with nil flags should generate only the command", - Flag: flag.NewFlags(nil, nil), - ExpectedCLI: []string{"cmd"}, - }, - { - Name: "Apply with flags should generate the command and flags", - Flag: flag.NewFlags( - &MockFlagApplier{flagName: "--flag1", applyErr: nil}, - &MockFlagApplier{flagName: "--flag2", applyErr: nil}, - ), - ExpectedCLI: []string{"cmd", "--flag1", "--flag2"}, - }, - { - Name: "Apply with one error flag should not modify the command and return the error", - Flag: flag.NewFlags( - &MockFlagApplier{flagName: "flag1", applyErr: nil}, - &MockFlagApplier{flagName: "flag2", applyErr: ErrFlag}, - ), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: ErrFlag, - }, - { - Name: "NewBoolFlag", - Flag: flag.NewFlags( - flag.NewBoolFlag("--flag1", true), - flag.NewBoolFlag("--flag2", false), - ), - ExpectedCLI: []string{"cmd", "--flag1"}, - }, - { - Name: "NewBoolFlag with empty flag name should return an error", - Flag: flag.NewFlags( - flag.NewBoolFlag("", true), - ), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: cli.ErrInvalidFlag, - }, - { - Name: "NewStringFlag", - Flag: flag.NewFlags( - flag.NewStringFlag("--flag1", "value1"), - flag.NewStringFlag("--flag2", ""), - ), - ExpectedCLI: []string{"cmd", "--flag1=value1"}, - }, - { - Name: "NewStringFlag with all empty values should return an error", - Flag: flag.NewFlags( - flag.NewStringFlag("--flag1", "value1"), - flag.NewStringFlag("", ""), - ), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: cli.ErrInvalidFlag, - }, - { - Name: "NewRedactedStringFlag", - Flag: flag.NewFlags( - flag.NewRedactedStringFlag("--flag1", "value1"), - flag.NewRedactedStringFlag("--flag2", ""), - flag.NewRedactedStringFlag("", "value3"), - ), - ExpectedCLI: []string{"cmd", "--flag1=value1", "value3"}, - ExpectedLog: "cmd --flag1=<****> <****>", - }, - { - Name: "NewRedactedStringFlag with all empty values should return an error", - Flag: flag.NewFlags( - flag.NewRedactedStringFlag("--flag1", "value1"), - flag.NewRedactedStringFlag("", ""), - ), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: cli.ErrInvalidFlag, - }, - { - Name: "NewStringValue", - Flag: flag.NewFlags( - flag.NewStringArgument("value1"), - ), - ExpectedCLI: []string{"cmd", "value1"}, - }, - { - Name: "NewStringValue with empty value should return an error", - Flag: flag.NewFlags( - flag.NewStringArgument(""), - ), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: cli.ErrInvalidFlag, - }, - { - Name: "NewFlags should generate multiple flags", - Flag: flag.NewFlags( - flag.NewStringFlag("--flag1", "value1"), - flag.NewRedactedStringFlag("--flag2", "value2"), - flag.NewStringArgument("value3"), - ), - ExpectedCLI: []string{"cmd", "--flag1=value1", "--flag2=value2", "value3"}, - ExpectedLog: "cmd --flag1=value1 --flag2=<****> value3", - }, - { - Name: "NewFlags should generate no flags if one of them returns an error", - Flag: flag.NewFlags( - flag.NewStringFlag("--flag1", "value1"), - &MockFlagApplier{flagName: "flag2", applyErr: ErrFlag}, - ), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: ErrFlag, - }, - { - Name: "EmptyFlag should not generate any flags", - Flag: flag.EmptyFlag(), - ExpectedCLI: []string{"cmd"}, - }, - { - Name: "ErrorFlag should return an error", - Flag: flag.ErrorFlag(ErrFlag), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: ErrFlag, - }, -}}) diff --git a/pkg/kopia/cli/internal/flag/string_flag.go b/pkg/kopia/cli/internal/flag/string_flag.go deleted file mode 100644 index a94c29722c..0000000000 --- a/pkg/kopia/cli/internal/flag/string_flag.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package flag - -import ( - "github.com/kanisterio/safecli" - - "github.com/kanisterio/kanister/pkg/kopia/cli" -) - -// stringFlag defines a string flag with a given flag name and value. -// If the value is empty, the flag is not applied. -type stringFlag struct { - flag string // flag name - value string // flag value - redacted bool // output the value as redacted -} - -// appenderFunc is a function that appends strings to a command. -type appenderFunc func(...string) *safecli.Builder - -// Apply appends the flag to the command if the value is not empty. -// If the value is redacted, it is appended as redacted. -func (f stringFlag) Apply(cli safecli.CommandAppender) error { - if f.value == "" { - return nil - } - appendValue, appendFlagValue := f.selectAppenderFuncs(cli) - if f.flag == "" { - appendValue(f.value) - } else { - appendFlagValue(f.flag, f.value) - } - return nil -} - -// selectAppenderFuncs returns the appropriate appender functions based on the redacted flag. -func (f stringFlag) selectAppenderFuncs(cli safecli.CommandAppender) (appenderFunc, appenderFunc) { - if f.redacted { - return cli.AppendRedacted, cli.AppendRedactedKV - } - return cli.AppendLoggable, cli.AppendLoggableKV -} - -// newStringFlag creates a new string flag with a given flag name and value. -func newStringFlag(flag, val string, redacted bool) Applier { - if flag == "" && val == "" { - return ErrorFlag(cli.ErrInvalidFlag) - } - return stringFlag{flag: flag, value: val, redacted: redacted} -} - -// NewStringFlag creates a new string flag with a given flag name and value. -func NewStringFlag(flag, val string) Applier { - return newStringFlag(flag, val, false) -} - -// NewRedactedStringFlag creates a new string flag with a given flag name and value. -func NewRedactedStringFlag(flag, val string) Applier { - return newStringFlag(flag, val, true) -} - -// NewStringArgument creates a new string argument with a given value. -func NewStringArgument(val string) Applier { - return newStringFlag("", val, false) -} diff --git a/pkg/kopia/cli/internal/test/flag_suite.go b/pkg/kopia/cli/internal/test/flag_suite.go deleted file mode 100644 index 7fe60d909c..0000000000 --- a/pkg/kopia/cli/internal/test/flag_suite.go +++ /dev/null @@ -1,114 +0,0 @@ -package test - -import ( - "strings" - - "gopkg.in/check.v1" - - "github.com/pkg/errors" - - "github.com/kanisterio/safecli" - - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" -) - -// FlagTest defines a single test for a flag. -type FlagTest struct { - // Name of the test. (required) - Name string - - // Flag to test. (required) - Flag flag.Applier - - // Expected CLI arguments. (optional) - ExpectedCLI []string - - // Expected log output. (optional) - // if empty, it will be set to ExpectedCLI joined with space. - // if empty and ExpectedCLI is empty, it will be ignored. - ExpectedLog string - - // Expected error. (optional) - // If nil, no error is expected and - // ExpectedCLI and ExpectedLog are checked. - ExpectedErr error -} - -// CheckCommentString implements check.CommentInterface -func (t *FlagTest) CheckCommentString() string { - return t.Name -} - -// setDefaultExpectedLog sets the default value for ExpectedLog based on ExpectedCLI. -func (t *FlagTest) setDefaultExpectedLog() { - if len(t.ExpectedLog) == 0 && len(t.ExpectedCLI) > 0 { - t.ExpectedLog = strings.Join(t.ExpectedCLI, " ") - } -} - -// assertError checks the error against ExpectedErr. -func (t *FlagTest) assertError(c *check.C, err error) { - if actualErr := errors.Cause(err); actualErr != nil { - c.Assert(actualErr, check.Equals, t.ExpectedErr, t) - } else { - c.Assert(err, check.Equals, t.ExpectedErr, t) - } -} - -// assertNoError makes sure there is no error. -func (t *FlagTest) assertNoError(c *check.C, err error) { - c.Assert(err, check.IsNil, t) -} - -// assertCLI asserts the builder's CLI output against ExpectedCLI. -func (t *FlagTest) assertCLI(c *check.C, b *safecli.Builder) { - if t.ExpectedCLI != nil { - c.Check(b.Build(), check.DeepEquals, t.ExpectedCLI, t) - } -} - -// assertLog asserts the builder's log output against ExpectedLog. -func (t *FlagTest) assertLog(c *check.C, b *safecli.Builder) { - t.setDefaultExpectedLog() - c.Check(b.String(), check.Equals, t.ExpectedLog, t) -} - -// Test runs the flag test. -func (ft *FlagTest) Test(c *check.C, b *safecli.Builder) { - err := flag.Apply(b, ft.Flag) - ft.assertCLI(c, b) - if ft.ExpectedErr != nil { - ft.assertError(c, err) - } else { - ft.assertNoError(c, err) - ft.assertLog(c, b) - } -} - -// FlagSuite defines a test suite for flags. -type FlagSuite struct { - Cmd string // Cmd appends to the safecli.Builder before test if not empty. - Tests []FlagTest // Tests to run. -} - -// TestFlags runs all tests in the flag suite. -func (s *FlagSuite) TestFlags(c *check.C) { - for _, test := range s.Tests { - b := newBuilder(s.Cmd) - test.Test(c, b) - } -} - -// NewFlagSuite creates a new FlagSuite. -func NewFlagSuite(tests []FlagTest) *FlagSuite { - return &FlagSuite{Tests: tests} -} - -// newBuilder creates a new safecli.Builder with the given command. -func newBuilder(cmd string) *safecli.Builder { - builder := safecli.NewBuilder() - if cmd != "" { - builder.AppendLoggable(cmd) - } - return builder -} From 550d124c4d159e0009b2c6bca5b05c54e8368c13 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 16 Feb 2024 20:40:54 -0800 Subject: [PATCH 22/73] Add pkg/kopia/cli package Signed-off-by: pavel.larkin --- pkg/kopia/cli/doc.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 pkg/kopia/cli/doc.go diff --git a/pkg/kopia/cli/doc.go b/pkg/kopia/cli/doc.go new file mode 100644 index 0000000000..6f0681452f --- /dev/null +++ b/pkg/kopia/cli/doc.go @@ -0,0 +1,21 @@ +package cli + +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import ( + _ "github.com/kanisterio/safecli" +) + +// This package contains the implementation of the Kopia CLI using github.com/kanisterio/safecli. From fc918a0846cd6f4594da3536d0cd8982e84f2161 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 16 Feb 2024 20:41:22 -0800 Subject: [PATCH 23/73] go mod tidy Signed-off-by: pavel.larkin --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 233bc702f1..9e093b449a 100644 --- a/go.sum +++ b/go.sum @@ -362,8 +362,6 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/kanisterio/safecli v0.0.3 h1:ts3oRVSoRexFBv9pOdWYZsN4k068pWx5Cl4zN54mQro= -github.com/kanisterio/safecli v0.0.3/go.mod h1:fK3Mcbeiso+NtkUdhGugK0Vf4S2l8ObvFf564ry1W5A= github.com/kanisterio/safecli v0.0.4 h1:8pO7Zhm9YeW47XQZNRt/AaXj7kSPeIbaV7aRDRtHs4k= github.com/kanisterio/safecli v0.0.4/go.mod h1:KBraqj8mdv2cwAr9wecknGUb8jztTzUik0r7uE6yRA8= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734 h1:qulsCaCv+O2y9/sQ9nd5KChnAgFOWakTHQ9ZADjs6DQ= From 9fdf94bdfc30503fcc98e7163da50e5360f6839b Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 16 Feb 2024 21:37:45 -0800 Subject: [PATCH 24/73] Add Kopia storage helpers Signed-off-by: pavel.larkin --- .../internal/flag/storage/model/factory.go | 56 ------------- .../flag/storage/model/factory_test.go | 81 ------------------- .../flag/storage/model/storage_flag.go | 61 -------------- .../flag/storage/model/storage_flag_test.go | 69 ---------------- .../{flag/storage/model => }/location.go | 2 +- .../{flag/storage/model => }/location_test.go | 9 ++- .../internal/{flag/storage/model => }/path.go | 2 +- .../{flag/storage/model => }/path_test.go | 5 +- 8 files changed, 10 insertions(+), 275 deletions(-) delete mode 100644 pkg/kopia/cli/internal/flag/storage/model/factory.go delete mode 100644 pkg/kopia/cli/internal/flag/storage/model/factory_test.go delete mode 100644 pkg/kopia/cli/internal/flag/storage/model/storage_flag.go delete mode 100644 pkg/kopia/cli/internal/flag/storage/model/storage_flag_test.go rename pkg/kopia/cli/internal/{flag/storage/model => }/location.go (99%) rename pkg/kopia/cli/internal/{flag/storage/model => }/location_test.go (93%) rename pkg/kopia/cli/internal/{flag/storage/model => }/path.go (98%) rename pkg/kopia/cli/internal/{flag/storage/model => }/path_test.go (88%) diff --git a/pkg/kopia/cli/internal/flag/storage/model/factory.go b/pkg/kopia/cli/internal/flag/storage/model/factory.go deleted file mode 100644 index 8a77fbf7f6..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/model/factory.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package model - -import ( - "fmt" - - "github.com/pkg/errors" - - "github.com/kanisterio/safecli" - - rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" - - "github.com/kanisterio/kanister/pkg/kopia/cli" -) - -// StorageBuilder defines a function that creates -// a safecli.Builder for the storage sub command. -type StorageBuilder func(StorageFlag) (*safecli.Builder, error) - -// StorageBuilderFactory defines a factory interface -// for creating a StorageBuilder by type. -type StorageBuilderFactory interface { - Create(rs.LocType) StorageBuilder -} - -// BuildersFactory defines a map of StorageBuilder by LocType. -type BuildersFactory map[rs.LocType]StorageBuilder - -// Create returns a StorageBuilder by LocType and -// implements the StorageBuilderFactory interface. -func (sb BuildersFactory) Create(locType rs.LocType) StorageBuilder { - if b, found := sb[locType]; found { - return b - } - return sb.unsupportedStorageType(locType) -} - -// unsupportedStorageType returns an error for an unsupported location type. -func (sb BuildersFactory) unsupportedStorageType(locType rs.LocType) StorageBuilder { - return func(StorageFlag) (*safecli.Builder, error) { - return nil, errors.Wrap(cli.ErrUnsupportedStorage, fmt.Sprintf("unsupported location type: '%v'", locType)) - } -} diff --git a/pkg/kopia/cli/internal/flag/storage/model/factory_test.go b/pkg/kopia/cli/internal/flag/storage/model/factory_test.go deleted file mode 100644 index e0bee04d65..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/model/factory_test.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package model - -import ( - "testing" - - "gopkg.in/check.v1" - - "github.com/kanisterio/safecli" - - rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" -) - -func TestFactory(t *testing.T) { check.TestingT(t) } - -type FactorySuite struct{} - -var ( - _ = check.Suite(&FactorySuite{}) - ltBlue = rs.LocType("blue") - ltRed = rs.LocType("red") - ltUnknown = rs.LocType("unknown") -) - -type mockBuilder struct { - cmd string - err error -} - -func (m *mockBuilder) New() StorageBuilder { - return func(sf StorageFlag) (*safecli.Builder, error) { - if m.err != nil { - return nil, m.err - } - b := safecli.NewBuilder() - b.AppendLoggable(m.cmd) - return b, nil - } -} - -func mockFactory() StorageBuilderFactory { - factory := make(BuildersFactory) - - blue := mockBuilder{cmd: "blue"} - red := mockBuilder{cmd: "red"} - - factory[ltBlue] = blue.New() - factory[ltRed] = red.New() - return factory -} - -func (s *FactorySuite) TestBuildersFactory(c *check.C) { - factory := mockFactory() - - b, err := factory.Create(ltBlue)(StorageFlag{}) - c.Assert(err, check.IsNil) - c.Check(b, check.NotNil) - c.Check(b.Build(), check.DeepEquals, []string{"blue"}) - - b, err = factory.Create(ltRed)(StorageFlag{}) - c.Assert(err, check.IsNil) - c.Check(b, check.NotNil) - c.Check(b.Build(), check.DeepEquals, []string{"red"}) - - b, err = factory.Create(ltUnknown)(StorageFlag{}) - c.Assert(err, check.ErrorMatches, ".*unsupported location type: 'unknown'.*") - c.Check(b, check.IsNil) -} diff --git a/pkg/kopia/cli/internal/flag/storage/model/storage_flag.go b/pkg/kopia/cli/internal/flag/storage/model/storage_flag.go deleted file mode 100644 index 89ac32cb6f..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/model/storage_flag.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package model - -import ( - "github.com/pkg/errors" - - "github.com/kanisterio/safecli" - - cmdlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" - log "github.com/kanisterio/kanister/pkg/log" -) - -var ( - // ErrInvalidFactory is returned when the factory is nil. - ErrInvalidFactory = errors.New("factory cannot be nil") -) - -// StorageFlag is a set of flags that are used to create a StorageFlag sub command. -type StorageFlag struct { - Location Location - RepoPathPrefix string - - Factory StorageBuilderFactory - Logger log.Logger -} - -// GetLogger returns the logger. -// If the logger is nil, it returns a NopLogger. -func (s StorageFlag) GetLogger() log.Logger { - if s.Logger == nil { - s.Logger = &cmdlog.NopLogger{} - } - return s.Logger -} - -// Apply applies the storage flags to the command. -func (s StorageFlag) Apply(cli safecli.CommandAppender) error { - if s.Factory == nil { - return ErrInvalidFactory - } - storageBuilder := s.Factory.Create(s.Location.Type()) - storageCLI, err := storageBuilder(s) - if err != nil { - return errors.Wrap(err, "failed to apply storage args") - } - cli.Append(storageCLI) - return nil -} diff --git a/pkg/kopia/cli/internal/flag/storage/model/storage_flag_test.go b/pkg/kopia/cli/internal/flag/storage/model/storage_flag_test.go deleted file mode 100644 index dd211eef4a..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/model/storage_flag_test.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package model - -import ( - "testing" - - "gopkg.in/check.v1" - - "github.com/kanisterio/safecli" - - rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" -) - -func TestStorageFlag(t *testing.T) { check.TestingT(t) } - -type StorageFlagSuite struct{} - -var _ = check.Suite(&StorageFlagSuite{}) - -func (s *StorageFlagSuite) TestGetLogger(c *check.C) { - sf := StorageFlag{} - c.Check(sf.GetLogger(), check.NotNil) - sf.Logger = nil - c.Check(sf.GetLogger(), check.NotNil) -} - -func (s *StorageFlagSuite) TestApplyNoFactory(c *check.C) { - sf := StorageFlag{} - err := sf.Apply(nil) - c.Check(err, check.Equals, ErrInvalidFactory) -} - -func (s *StorageFlagSuite) TestApply(c *check.C) { - sf := StorageFlag{ - Location: Location{ - rs.TypeKey: []byte("blue"), - }, - Factory: mockFactory(), - } - b := safecli.NewBuilder() - err := sf.Apply(b) - c.Check(err, check.IsNil) - c.Check(b.Build(), check.DeepEquals, []string{"blue"}) -} - -func (s *StorageFlagSuite) TestApplyUnknowType(c *check.C) { - sf := StorageFlag{ - Location: Location{ - rs.TypeKey: []byte("unknow"), - }, - Factory: mockFactory(), - } - b := safecli.NewBuilder() - err := sf.Apply(b) - c.Check(err, check.ErrorMatches, ".*failed to apply storage args.*") -} diff --git a/pkg/kopia/cli/internal/flag/storage/model/location.go b/pkg/kopia/cli/internal/location.go similarity index 99% rename from pkg/kopia/cli/internal/flag/storage/model/location.go rename to pkg/kopia/cli/internal/location.go index 3a9a4e4fc9..8b56714487 100644 --- a/pkg/kopia/cli/internal/flag/storage/model/location.go +++ b/pkg/kopia/cli/internal/location.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package model +package internal import ( "strconv" diff --git a/pkg/kopia/cli/internal/flag/storage/model/location_test.go b/pkg/kopia/cli/internal/location_test.go similarity index 93% rename from pkg/kopia/cli/internal/flag/storage/model/location_test.go rename to pkg/kopia/cli/internal/location_test.go index 1051bce642..f74ca6be71 100644 --- a/pkg/kopia/cli/internal/flag/storage/model/location_test.go +++ b/pkg/kopia/cli/internal/location_test.go @@ -12,11 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -package model +package internal_test import ( "testing" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" "gopkg.in/check.v1" ) @@ -40,17 +41,17 @@ func (s *LocationSuite) TestLocation(c *check.C) { tests := []struct { name string - location Location + location internal.Location expected expected }{ { name: "Test with no fields", - location: Location{}, + location: internal.Location{}, expected: expected{}, }, { name: "Test with all fields", - location: Location{ + location: internal.Location{ rs.TypeKey: []byte("Type1"), rs.RegionKey: []byte("Region1"), rs.BucketKey: []byte("Bucket1"), diff --git a/pkg/kopia/cli/internal/flag/storage/model/path.go b/pkg/kopia/cli/internal/path.go similarity index 98% rename from pkg/kopia/cli/internal/flag/storage/model/path.go rename to pkg/kopia/cli/internal/path.go index 5e18da2cb8..9085ebca4c 100644 --- a/pkg/kopia/cli/internal/flag/storage/model/path.go +++ b/pkg/kopia/cli/internal/path.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package model +package internal import ( "path" diff --git a/pkg/kopia/cli/internal/flag/storage/model/path_test.go b/pkg/kopia/cli/internal/path_test.go similarity index 88% rename from pkg/kopia/cli/internal/flag/storage/model/path_test.go rename to pkg/kopia/cli/internal/path_test.go index 2fd1762362..4ddfeabed1 100644 --- a/pkg/kopia/cli/internal/flag/storage/model/path_test.go +++ b/pkg/kopia/cli/internal/path_test.go @@ -12,11 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -package model +package internal_test import ( "testing" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" "gopkg.in/check.v1" ) @@ -44,7 +45,7 @@ func (s *PathSuite) TestGenerateFullRepoPath(c *check.C) { }, } for _, test := range tests { - got := GenerateFullRepoPath(test.locPrefix, test.repoPathPrefix) + got := internal.GenerateFullRepoPath(test.locPrefix, test.repoPathPrefix) c.Check(got, check.Equals, test.expected) } } From 57adfc2f41db0efb90c51079a538197c30618f41 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 16 Feb 2024 23:24:23 -0800 Subject: [PATCH 25/73] Implement Kopia storage Filesystem opts Signed-off-by: pavel.larkin --- pkg/kopia/cli/errors.go | 2 + pkg/kopia/cli/internal/command/command.go | 45 ---- .../cli/internal/command/command_test.go | 97 -------- pkg/kopia/cli/internal/command/commands.go | 6 - pkg/kopia/cli/internal/flag/storage/fs/fs.go | 39 --- .../cli/internal/flag/storage/fs/fs_flags.go | 26 -- .../cli/internal/flag/storage/fs/fs_test.go | 58 ----- .../cli/internal/flag/storage/storage.go | 70 ------ .../cli/internal/flag/storage/storage_test.go | 229 ------------------ pkg/kopia/cli/internal/test/command_suite.go | 134 ---------- pkg/kopia/cli/internal/test/string_logger.go | 58 ----- pkg/kopia/cli/repository/storage/fs/fs.go | 31 +++ .../cli/repository/storage/fs/fs_opts.go | 19 ++ .../storage/fs/fs_opts_test.go} | 21 +- .../cli/repository/storage/fs/fs_test.go | 39 +++ 15 files changed, 103 insertions(+), 771 deletions(-) delete mode 100644 pkg/kopia/cli/internal/command/command.go delete mode 100644 pkg/kopia/cli/internal/command/command_test.go delete mode 100644 pkg/kopia/cli/internal/command/commands.go delete mode 100644 pkg/kopia/cli/internal/flag/storage/fs/fs.go delete mode 100644 pkg/kopia/cli/internal/flag/storage/fs/fs_flags.go delete mode 100644 pkg/kopia/cli/internal/flag/storage/fs/fs_test.go delete mode 100644 pkg/kopia/cli/internal/flag/storage/storage.go delete mode 100644 pkg/kopia/cli/internal/flag/storage/storage_test.go delete mode 100644 pkg/kopia/cli/internal/test/command_suite.go delete mode 100644 pkg/kopia/cli/internal/test/string_logger.go create mode 100644 pkg/kopia/cli/repository/storage/fs/fs.go create mode 100644 pkg/kopia/cli/repository/storage/fs/fs_opts.go rename pkg/kopia/cli/{internal/flag/storage/fs/fs_flags_test.go => repository/storage/fs/fs_opts_test.go} (54%) create mode 100644 pkg/kopia/cli/repository/storage/fs/fs_test.go diff --git a/pkg/kopia/cli/errors.go b/pkg/kopia/cli/errors.go index 79658eb3a5..97a71e4dce 100644 --- a/pkg/kopia/cli/errors.go +++ b/pkg/kopia/cli/errors.go @@ -30,4 +30,6 @@ var ( var ( // ErrUnsupportedStorage is returned when the storage is not supported. ErrUnsupportedStorage = errors.New("unsupported storage") + // ErrInvalidRepoPath is returned when the repoPath is empty. + ErrInvalidRepoPath = errors.New("repository path cannot be empty") ) diff --git a/pkg/kopia/cli/internal/command/command.go b/pkg/kopia/cli/internal/command/command.go deleted file mode 100644 index 04acbaecd4..0000000000 --- a/pkg/kopia/cli/internal/command/command.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package command - -import ( - "github.com/kanisterio/safecli" - - clierrors "github.com/kanisterio/kanister/pkg/kopia/cli" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" -) - -// Command is a CLI command/subcommand. -type Command struct { - name string -} - -// Apply applies the command to the CLI. -func (c Command) Apply(cli safecli.CommandAppender) error { - if len(c.name) == 0 { - return clierrors.ErrInvalidCommand - } - cli.AppendLoggable(c.name) - return nil -} - -// NewCommandBuilder returns a new safecli.Builder for the storage sub command. -func NewCommandBuilder(cmd flag.Applier, flags ...flag.Applier) (*safecli.Builder, error) { - b := safecli.NewBuilder() - if err := flag.Apply(b, append([]flag.Applier{cmd}, flags...)...); err != nil { - return nil, err - } - return b, nil -} diff --git a/pkg/kopia/cli/internal/command/command_test.go b/pkg/kopia/cli/internal/command/command_test.go deleted file mode 100644 index f4ae074d37..0000000000 --- a/pkg/kopia/cli/internal/command/command_test.go +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package command - -import ( - "errors" - "testing" - - "gopkg.in/check.v1" - - "github.com/kanisterio/kanister/pkg/kopia/cli" - "github.com/kanisterio/safecli" -) - -func TestCommand(t *testing.T) { check.TestingT(t) } - -type CommandSuite struct{} - -var _ = check.Suite(&CommandSuite{}) - -var ( - errInvalidCommand = errors.New("invalid command") - errInvalidFlag = errors.New("invalid flag") -) - -type mockCommandAndFlag struct { - flagName string - err error -} - -func (m *mockCommandAndFlag) Apply(cli safecli.CommandAppender) error { - if m.err == nil { - cli.AppendLoggable(m.flagName) - } - return m.err -} - -func (s *CommandSuite) TestCommand(c *check.C) { - b := safecli.NewBuilder() - cmd := Command{"cmd"} - err := cmd.Apply(b) - c.Assert(err, check.IsNil) - c.Check(b.Build(), check.DeepEquals, []string{"cmd"}) -} - -func (s *CommandSuite) TestEmptyCommand(c *check.C) { - b := safecli.NewBuilder() - cmd := Command{} - err := cmd.Apply(b) - c.Assert(err, check.Equals, cli.ErrInvalidCommand) -} - -func (s *CommandSuite) TestNewCommandBuilderWithFailedCommand(c *check.C) { - // test if command is invalid - b, err := NewCommandBuilder( - &mockCommandAndFlag{err: errInvalidCommand}, - ) - c.Assert(b, check.IsNil) - c.Assert(err, check.Equals, errInvalidCommand) -} - -func (s *CommandSuite) TestNewCommandBuilderWithFailedFlag(c *check.C) { - // test if flag is invalid - b, err := NewCommandBuilder( - &mockCommandAndFlag{flagName: "cmd"}, - &mockCommandAndFlag{err: errInvalidFlag}, - ) - c.Assert(b, check.IsNil) - c.Assert(err, check.Equals, errInvalidFlag) -} - -func (s *CommandSuite) TestNewCommandBuilder(c *check.C) { - // test if command and flag are valid - b, err := NewCommandBuilder( - &mockCommandAndFlag{flagName: "cmd"}, - &mockCommandAndFlag{flagName: "--flag1"}, - &mockCommandAndFlag{flagName: "--flag2"}, - ) - c.Assert(err, check.IsNil) - c.Check(b.Build(), check.DeepEquals, []string{ - "cmd", - "--flag1", - "--flag2", - }) -} diff --git a/pkg/kopia/cli/internal/command/commands.go b/pkg/kopia/cli/internal/command/commands.go deleted file mode 100644 index 17fa517524..0000000000 --- a/pkg/kopia/cli/internal/command/commands.go +++ /dev/null @@ -1,6 +0,0 @@ -package command - -// Repository storage sub commands. -var ( - FileSystem = Command{"filesystem"} -) diff --git a/pkg/kopia/cli/internal/flag/storage/fs/fs.go b/pkg/kopia/cli/internal/flag/storage/fs/fs.go deleted file mode 100644 index 20a8b0fac7..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/fs/fs.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package fs - -import ( - "github.com/kanisterio/safecli" - - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/command" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" -) - -const ( - // DefaultFSMountPath is the default mount path for the filesystem subcommand storage. - DefaultFSMountPath = "/mnt/data" -) - -// New returns a builder for the filesystem subcommand storage. -func New(f model.StorageFlag) (*safecli.Builder, error) { - path := generateFileSystemMountPath(f.Location.Prefix(), f.RepoPathPrefix) - return command.NewCommandBuilder(command.FileSystem, - Path(path), - ) -} - -func generateFileSystemMountPath(locPrefix, repoPathPrefix string) string { - return DefaultFSMountPath + "/" + model.GenerateFullRepoPath(locPrefix, repoPathPrefix) -} diff --git a/pkg/kopia/cli/internal/flag/storage/fs/fs_flags.go b/pkg/kopia/cli/internal/flag/storage/fs/fs_flags.go deleted file mode 100644 index 67c252e1e4..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/fs/fs_flags.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package fs - -import "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" - -// -// Filestore flags. -// - -// Path creates a new path flag with a given path. -func Path(path string) flag.Applier { - return flag.NewStringFlag("--path", path) -} diff --git a/pkg/kopia/cli/internal/flag/storage/fs/fs_test.go b/pkg/kopia/cli/internal/flag/storage/fs/fs_test.go deleted file mode 100644 index 13d992abdb..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/fs/fs_test.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package fs - -import ( - "testing" - - "gopkg.in/check.v1" - - "github.com/kanisterio/safecli" - - rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" - - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" -) - -func TestStorageFS(t *testing.T) { check.TestingT(t) } - -var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ - { - Name: "Empty FS storage flag should generate subcommand with default flags", - CLI: func() (safecli.CommandBuilder, error) { - return New(model.StorageFlag{}) - }, - ExpectedCLI: []string{ - "filesystem", - "--path=/mnt/data/", - }, - }, - { - Name: "FS with values should generate subcommand with specific flags", - CLI: func() (safecli.CommandBuilder, error) { - return New(model.StorageFlag{ - RepoPathPrefix: "repo/path/prefix", - Location: model.Location{ - rs.PrefixKey: []byte("prefix"), - }, - }) - }, - ExpectedCLI: []string{ - "filesystem", - "--path=/mnt/data/prefix/repo/path/prefix/", - }, - }, -})) diff --git a/pkg/kopia/cli/internal/flag/storage/storage.go b/pkg/kopia/cli/internal/flag/storage/storage.go deleted file mode 100644 index 48431e6a97..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/storage.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import ( - "sync" - - cmdlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" - "github.com/kanisterio/kanister/pkg/log" - rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" - - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/fs" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" -) - -// Option is a function that sets a storage option. -type Option func(*model.StorageFlag) - -// WithLogger sets the logger for the storage. -func WithLogger(logger log.Logger) Option { - return func(s *model.StorageFlag) { - s.Logger = logger - } -} - -// WithFactory sets the storage args builder factory for the storage. -func WithFactory(factory model.StorageBuilderFactory) Option { - return func(s *model.StorageFlag) { - s.Factory = factory - } -} - -var ( - // factoryOnce is used to initialize the factory once. - factoryOnce sync.Once - // factory creates a new StorageBuilder by LocType. - factory = model.BuildersFactory{} -) - -// Storage creates a new storage with the given location, repo path prefix and options. -func Storage(location model.Location, repoPathPrefix string, opts ...Option) model.StorageFlag { - factoryOnce.Do(func() { - // Register storage builders. - factory[rs.LocTypeFilestore] = fs.New - }) - // create a new storage with the given location, repo path prefix and defaults. - s := model.StorageFlag{ - Location: location, - RepoPathPrefix: repoPathPrefix, - Logger: &cmdlog.NopLogger{}, - Factory: &factory, - } - // apply storage options. - for _, opt := range opts { - opt(&s) - } - return s -} diff --git a/pkg/kopia/cli/internal/flag/storage/storage_test.go b/pkg/kopia/cli/internal/flag/storage/storage_test.go deleted file mode 100644 index 96fe150d5a..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/storage_test.go +++ /dev/null @@ -1,229 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import ( - "fmt" - "testing" - - "gopkg.in/check.v1" - - "github.com/pkg/errors" - - "github.com/kanisterio/safecli" - - rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" - - "github.com/kanisterio/kanister/pkg/kopia/cli" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/command" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/fs" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" - cmdlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" -) - -func TestStorageFlags(t *testing.T) { check.TestingT(t) } - -type StorageSuite struct{} - -var _ = check.Suite(&StorageSuite{}) - -func (s *StorageSuite) TestLocationMethods(c *check.C) { - type expected struct { - Type rs.LocType - Region string - BucketName string - Endpoint string - Prefix string - IsInsecure bool - HasSkipSSLVerify bool - } - - tests := []struct { - name string - location model.Location - expected expected - }{ - { - name: "Test1", - location: model.Location{ - rs.TypeKey: []byte("Type1"), - rs.RegionKey: []byte("Region1"), - rs.BucketKey: []byte("Bucket1"), - rs.EndpointKey: []byte("http://Endpoint1"), - rs.PrefixKey: []byte("Prefix1"), - rs.SkipSSLVerifyKey: []byte("true"), - }, - expected: expected{ - Type: "Type1", - Region: "Region1", - BucketName: "Bucket1", - Endpoint: "http://Endpoint1", - Prefix: "Prefix1", - IsInsecure: true, - HasSkipSSLVerify: true, - }, - }, - { - name: "Test2", - location: model.Location{ - rs.TypeKey: []byte("Type2"), - rs.RegionKey: []byte("Region2"), - rs.BucketKey: []byte("Bucket2"), - rs.EndpointKey: []byte("https://Endpoint2"), - rs.PrefixKey: []byte("Prefix2"), - rs.SkipSSLVerifyKey: []byte("false"), - }, - expected: expected{ - Type: "Type2", - Region: "Region2", - BucketName: "Bucket2", - Endpoint: "https://Endpoint2", - Prefix: "Prefix2", - IsInsecure: false, - HasSkipSSLVerify: false, - }, - }, - } - - for _, tt := range tests { - c.Assert(tt.location.Type(), check.Equals, tt.expected.Type) - c.Assert(tt.location.Region(), check.Equals, tt.expected.Region) - c.Assert(tt.location.BucketName(), check.Equals, tt.expected.BucketName) - c.Assert(tt.location.Endpoint(), check.Equals, tt.expected.Endpoint) - c.Assert(tt.location.Prefix(), check.Equals, tt.expected.Prefix) - c.Assert(tt.location.IsInsecureEndpoint(), check.Equals, tt.expected.IsInsecure) - c.Assert(tt.location.HasSkipSSLVerify(), check.Equals, tt.expected.HasSkipSSLVerify) - } -} - -func (s *StorageSuite) TestStorageFlag(c *check.C) { - tests := []struct { - name string - storage flag.Applier - expCLI []string - err error - errMsg string - }{ - { - name: "Empty Storage should generate an error", - storage: Storage(nil, ""), - err: cli.ErrUnsupportedStorage, - }, - { - name: "Filesystem without prefix and with repo path should generate repo path", - storage: Storage( - model.Location{ - rs.TypeKey: []byte("filestore"), - rs.PrefixKey: []byte(""), - }, - "dir1/subdir/", - ), - expCLI: []string{ - "filesystem", - fmt.Sprintf("--path=%s/dir1/subdir/", fs.DefaultFSMountPath), - }, - }, - { - name: "Filesystem with prefix and repo path should generate merged prefix and repo path", - storage: Storage( - model.Location{ - rs.TypeKey: []byte("filestore"), - rs.PrefixKey: []byte("test-prefix"), - }, - "dir1/subdir/", - ), - expCLI: []string{ - "filesystem", - fmt.Sprintf("--path=%s/test-prefix/dir1/subdir/", fs.DefaultFSMountPath), - }, - }, - { - name: "Unsupported storage type should generate an error", - storage: Storage( - model.Location{ - rs.TypeKey: []byte("ftp"), - }, - "prefixfs", - ), - errMsg: "failed to apply storage args: unsupported location type: 'ftp': unsupported storage", - err: cli.ErrUnsupportedStorage, - }, - } - - for _, tt := range tests { - b := safecli.NewBuilder() - err := tt.storage.Apply(b) - - cmt := check.Commentf("FAIL: %v", tt.name) - if tt.errMsg != "" { - c.Assert(err, check.NotNil, cmt) - c.Assert(err.Error(), check.Equals, tt.errMsg, cmt) - } - - if tt.err == nil { - c.Assert(err, check.IsNil, cmt) - } else { - c.Assert(errors.Cause(err), check.Equals, tt.err, cmt) - } - c.Assert(b.Build(), check.DeepEquals, tt.expCLI, cmt) - } -} - -// MockFlagWithError is a mock flag that always returns an error -type MockFlagWithError struct{} - -var errMock = fmt.Errorf("mock error") - -func (f MockFlagWithError) Apply(cli safecli.CommandAppender) error { - return errMock -} - -func (s *StorageSuite) TestNewStorageBuilderWithErrorFlag(c *check.C) { - b, err := command.NewCommandBuilder(command.FileSystem, MockFlagWithError{}) - c.Assert(b, check.IsNil) - c.Assert(err, check.Equals, errMock) -} - -func (s *StorageSuite) TestStorageGetLogger(c *check.C) { - storage := Storage(nil, "prefix") - c.Assert(storage.GetLogger(), check.NotNil) - - nopLog := &cmdlog.NopLogger{} - storage = Storage(nil, "prefix", WithLogger(nopLog)) - c.Assert(storage.GetLogger(), check.Equals, nopLog) -} - -// MockFactory is a mock storage factory -type MockFactory struct{} - -func (f MockFactory) Create(locType rs.LocType) model.StorageBuilder { - return func(s model.StorageFlag) (*safecli.Builder, error) { - return safecli.NewBuilder("mockfactory"), nil - } -} - -func (s *StorageSuite) TestStorageFactory(c *check.C) { - storage := Storage(nil, "prefix") - c.Assert(storage.GetLogger(), check.NotNil) - - mockFactory := &MockFactory{} - storage = Storage(nil, "prefix", WithFactory(mockFactory)) - c.Assert(storage.Factory, check.Equals, mockFactory) - b, err := storage.Factory.Create("anything")(model.StorageFlag{}) - c.Assert(b, check.NotNil) - c.Assert(err, check.IsNil) - c.Assert(b.Build(), check.DeepEquals, []string{"mockfactory"}) -} diff --git a/pkg/kopia/cli/internal/test/command_suite.go b/pkg/kopia/cli/internal/test/command_suite.go deleted file mode 100644 index 480c1fcba5..0000000000 --- a/pkg/kopia/cli/internal/test/command_suite.go +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package test - -import ( - "fmt" - - "gopkg.in/check.v1" - - "github.com/pkg/errors" - - "github.com/kanisterio/safecli" - - "github.com/kanisterio/kanister/pkg/log" - - "github.com/kanisterio/kanister/pkg/kopia/cli" -) - -// CommonArgs is a set of common arguments for the tests. -var CommonArgs = cli.CommonArgs{ - RepoPassword: "encr-key", - ConfigFilePath: "path/kopia.config", - LogDirectory: "cache/log", -} - -// CommandTest defines a single test for a command. -type CommandTest struct { - // Name of the test. (required) - Name string - - // CLI to test. (required) - CLI func() (safecli.CommandBuilder, error) - - // Expected CLI arguments. (optional) - ExpectedCLI []string - - // Expected log output. (optional) - // if empty, it will be derived from ExpectedCLI by redacting sensitive information. - // if empty and ExpectedCLI is empty, it will be ignored. - ExpectedLog string - - // Expected error. (optional) - // If nil, no error is expected and - // ExpectedCLI and ExpectedLog are checked. - ExpectedErr error - - // LoggerRegex is a list of regular expressions to match against the log output. (optional) - Logger log.Logger - LoggerRegex []string -} - -// CheckCommentString implements check.CommentInterface -func (t *CommandTest) CheckCommentString() string { - return t.Name -} - -func (t *CommandTest) setDefaultExpectedLog() { - if len(t.ExpectedLog) == 0 && len(t.ExpectedCLI) > 0 { - t.ExpectedLog = RedactCLI(t.ExpectedCLI) - } -} - -func (t *CommandTest) assertError(c *check.C, err error) { - actualErr := errors.Cause(err) - c.Assert(actualErr, check.Equals, t.ExpectedErr, t) -} - -func (t *CommandTest) assertNoError(c *check.C, err error) { - c.Assert(err, check.IsNil, t) -} - -func (t *CommandTest) assertCLI(c *check.C, b safecli.CommandBuilder) { - c.Check(b.Build(), check.DeepEquals, t.ExpectedCLI, t) -} - -func (t *CommandTest) assertLog(c *check.C, b safecli.CommandBuilder) { - t.setDefaultExpectedLog() - c.Check(fmt.Sprint(b), check.Equals, t.ExpectedLog, t) -} - -func (t *CommandTest) assertLogger(c *check.C) { - log, ok := t.Logger.(*StringLogger) - if !ok { - c.Fatalf("t.Logger is not a StringLogger") - } - cmtLog := check.Commentf("FAIL: %v\nlog %#v expected to match %#v", t.Name, log, t.LoggerRegex) - for _, regex := range t.LoggerRegex { - c.Assert(log.MatchString(regex), check.Equals, true, cmtLog) - } -} - -// Test runs the command test. -func (t *CommandTest) Test(c *check.C) { - b, err := t.CLI() - if t.ExpectedErr == nil { - t.assertNoError(c, err) - t.assertCLI(c, b) - t.assertLog(c, b) - } else { - t.assertError(c, err) - } - if t.Logger != nil { - t.assertLogger(c) - } -} - -// CommandSuite defines a test suite for commands. -type CommandSuite struct { - Tests []CommandTest -} - -// TestCommands runs all tests in the suite. -func (s *CommandSuite) TestCommands(c *check.C) { - for _, test := range s.Tests { - test.Test(c) - } -} - -// NewCommandSuite creates a new CommandSuite. -func NewCommandSuite(tests []CommandTest) *CommandSuite { - return &CommandSuite{Tests: tests} -} diff --git a/pkg/kopia/cli/internal/test/string_logger.go b/pkg/kopia/cli/internal/test/string_logger.go deleted file mode 100644 index 1a55f46220..0000000000 --- a/pkg/kopia/cli/internal/test/string_logger.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package test - -import ( - "context" - "io" - "regexp" - - "github.com/kanisterio/kanister/pkg/field" - "github.com/kanisterio/kanister/pkg/log" -) - -// StringLogger implements log.Logger and stores log messages in a slice of strings. -// It is useful for testing. -type StringLogger []string - -// Print appends the message to the slice. -func (l *StringLogger) Print(msg string, fields ...field.M) { - *l = append(*l, msg) -} - -// PrintTo appends the message to the slice. -func (l *StringLogger) PrintTo(w io.Writer, msg string, fields ...field.M) { - *l = append(*l, msg) -} - -// WithContext does nothing. -func (l *StringLogger) WithContext(ctx context.Context) log.Logger { - return l -} - -// WithError does nothing. -func (l *StringLogger) WithError(err error) log.Logger { - return l -} - -// MatchString returns true if any of the log messages match the pattern. -func (l *StringLogger) MatchString(pattern string) bool { - for _, line := range *l { - if found, _ := regexp.MatchString(pattern, line); found { - return true - } - } - return false -} diff --git a/pkg/kopia/cli/repository/storage/fs/fs.go b/pkg/kopia/cli/repository/storage/fs/fs.go new file mode 100644 index 0000000000..d93cb339dd --- /dev/null +++ b/pkg/kopia/cli/repository/storage/fs/fs.go @@ -0,0 +1,31 @@ +package fs + +import ( + "github.com/kanisterio/safecli/command" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" + "github.com/kanisterio/kanister/pkg/log" +) + +const ( + defaultFSMountPath = "/mnt/data" +) + +// New creates a new subcommand for the filesystem storage. +func New(location internal.Location, repoPathPrefix string, _ log.Logger) command.Applier { + path, err := generateFileSystemMountPath(location.Prefix(), repoPathPrefix) + if err != nil { + return command.NewErrorArgument(err) + } + return command.NewArguments(subcmdFilesystem, optRepoPath(path)) +} + +// generateFileSystemMountPath generates the mount path for the filesystem storage. +func generateFileSystemMountPath(locPrefix, repoPrefix string) (string, error) { + fullRepoPath := internal.GenerateFullRepoPath(locPrefix, repoPrefix) + if fullRepoPath == "" { + return "", cli.ErrInvalidRepoPath + } + return defaultFSMountPath + "/" + fullRepoPath, nil +} diff --git a/pkg/kopia/cli/repository/storage/fs/fs_opts.go b/pkg/kopia/cli/repository/storage/fs/fs_opts.go new file mode 100644 index 0000000000..953b471057 --- /dev/null +++ b/pkg/kopia/cli/repository/storage/fs/fs_opts.go @@ -0,0 +1,19 @@ +package fs + +import ( + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/safecli/command" +) + +var ( + subcmdFilesystem = command.NewArgument("filesystem") +) + +// optRepoPath creates a new path option with a given path. +// If the path is empty, it returns an error. +func optRepoPath(path string) command.Applier { + if path == "" { + return command.NewErrorArgument(cli.ErrInvalidRepoPath) + } + return command.NewOptionWithArgument("--path", path) +} diff --git a/pkg/kopia/cli/internal/flag/storage/fs/fs_flags_test.go b/pkg/kopia/cli/repository/storage/fs/fs_opts_test.go similarity index 54% rename from pkg/kopia/cli/internal/flag/storage/fs/fs_flags_test.go rename to pkg/kopia/cli/repository/storage/fs/fs_opts_test.go index c0631f0a75..8bacac053b 100644 --- a/pkg/kopia/cli/internal/flag/storage/fs/fs_flags_test.go +++ b/pkg/kopia/cli/repository/storage/fs/fs_opts_test.go @@ -17,20 +17,23 @@ package fs import ( "testing" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/safecli/command" + "github.com/kanisterio/safecli/test" "gopkg.in/check.v1" ) -func TestFilestoreFlags(t *testing.T) { check.TestingT(t) } +func TestFilesystemOptions(t *testing.T) { check.TestingT(t) } -var _ = check.Suite(test.NewFlagSuite([]test.FlagTest{ +var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ { - Name: "Empty Path should not generate a flag", - Flag: Path(""), + Name: "optRepoPath", + Argument: command.NewArguments(optRepoPath("/path/to/repo")), + ExpectedCLI: []string{"cmd", "--path=/path/to/repo"}, }, { - Name: "Path with value should generate a flag with the given value", - Flag: Path("/path/to/file"), - ExpectedCLI: []string{"--path=/path/to/file"}, + Name: "Invalid RepoPath", + Argument: command.NewArguments(optRepoPath("")), + ExpectedErr: cli.ErrInvalidRepoPath, }, -})) +}}) diff --git a/pkg/kopia/cli/repository/storage/fs/fs_test.go b/pkg/kopia/cli/repository/storage/fs/fs_test.go new file mode 100644 index 0000000000..91bc9c183f --- /dev/null +++ b/pkg/kopia/cli/repository/storage/fs/fs_test.go @@ -0,0 +1,39 @@ +package fs + +import ( + "testing" + + "github.com/kanisterio/safecli/command" + "github.com/kanisterio/safecli/test" + "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" +) + +func TestNewFilesystem(t *testing.T) { check.TestingT(t) } + +func newFilesystem(prefix, repoPath string) command.Applier { + l := internal.Location{ + "prefix": []byte(prefix), + } + return New(l, repoPath, nil) +} + +var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ + { + Name: "NewFilesystem", + Argument: newFilesystem("prefix", "repoPath"), + ExpectedCLI: []string{"cmd", "filesystem", "--path=/mnt/data/prefix/repoPath/"}, + }, + { + Name: "NewFilesystem with empty repoPath", + Argument: newFilesystem("prefix", ""), + ExpectedCLI: []string{"cmd", "filesystem", "--path=/mnt/data/prefix/"}, + }, + { + Name: "NewFilesystem with empty local prefix and repo prefix should return error", + Argument: newFilesystem("", ""), + ExpectedErr: cli.ErrInvalidRepoPath, + }, +}}) From 18d8dd6804ee70753f7e997a0c167e6e3ca0ae74 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 16 Feb 2024 23:26:31 -0800 Subject: [PATCH 26/73] Add (c) headers Signed-off-by: pavel.larkin --- pkg/kopia/cli/repository/storage/fs/fs.go | 14 ++++++++++++++ pkg/kopia/cli/repository/storage/fs/fs_opts.go | 14 ++++++++++++++ pkg/kopia/cli/repository/storage/fs/fs_test.go | 14 ++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/pkg/kopia/cli/repository/storage/fs/fs.go b/pkg/kopia/cli/repository/storage/fs/fs.go index d93cb339dd..5683c1ab85 100644 --- a/pkg/kopia/cli/repository/storage/fs/fs.go +++ b/pkg/kopia/cli/repository/storage/fs/fs.go @@ -1,3 +1,17 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package fs import ( diff --git a/pkg/kopia/cli/repository/storage/fs/fs_opts.go b/pkg/kopia/cli/repository/storage/fs/fs_opts.go index 953b471057..4968c33045 100644 --- a/pkg/kopia/cli/repository/storage/fs/fs_opts.go +++ b/pkg/kopia/cli/repository/storage/fs/fs_opts.go @@ -1,3 +1,17 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package fs import ( diff --git a/pkg/kopia/cli/repository/storage/fs/fs_test.go b/pkg/kopia/cli/repository/storage/fs/fs_test.go index 91bc9c183f..9f65dcdbe5 100644 --- a/pkg/kopia/cli/repository/storage/fs/fs_test.go +++ b/pkg/kopia/cli/repository/storage/fs/fs_test.go @@ -1,3 +1,17 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package fs import ( From e1bf966696aa7d9b42a8ae8432eaeebf0f844fd5 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 16 Feb 2024 23:27:18 -0800 Subject: [PATCH 27/73] Remove unused error Signed-off-by: pavel.larkin --- pkg/kopia/cli/errors.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/kopia/cli/errors.go b/pkg/kopia/cli/errors.go index 97a71e4dce..93afdca4e9 100644 --- a/pkg/kopia/cli/errors.go +++ b/pkg/kopia/cli/errors.go @@ -22,8 +22,6 @@ import ( var ( // ErrInvalidID is returned when the ID is empty. ErrInvalidID = errors.New("invalid ID") - // ErrInvalidCommand is returned when the command is empty. - ErrInvalidCommand = errors.New("invalid command") ) // storage errors From e51868b7d21001f4716a4b00fc6483070b928ad6 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 16 Feb 2024 23:29:25 -0800 Subject: [PATCH 28/73] Reorganize imports Signed-off-by: pavel.larkin --- pkg/kopia/cli/repository/storage/fs/fs_opts.go | 3 ++- pkg/kopia/cli/repository/storage/fs/fs_opts_test.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/kopia/cli/repository/storage/fs/fs_opts.go b/pkg/kopia/cli/repository/storage/fs/fs_opts.go index 4968c33045..23b3ba7a5c 100644 --- a/pkg/kopia/cli/repository/storage/fs/fs_opts.go +++ b/pkg/kopia/cli/repository/storage/fs/fs_opts.go @@ -15,8 +15,9 @@ package fs import ( - "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/safecli/command" + + "github.com/kanisterio/kanister/pkg/kopia/cli" ) var ( diff --git a/pkg/kopia/cli/repository/storage/fs/fs_opts_test.go b/pkg/kopia/cli/repository/storage/fs/fs_opts_test.go index 8bacac053b..c84d90c138 100644 --- a/pkg/kopia/cli/repository/storage/fs/fs_opts_test.go +++ b/pkg/kopia/cli/repository/storage/fs/fs_opts_test.go @@ -17,10 +17,11 @@ package fs import ( "testing" - "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/safecli/command" "github.com/kanisterio/safecli/test" "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli" ) func TestFilesystemOptions(t *testing.T) { check.TestingT(t) } From ff01936e31e71439d9952a436faeca8da13736db Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 16 Feb 2024 23:54:20 -0800 Subject: [PATCH 29/73] Add Kopia GCS storage opts Signed-off-by: pavel.larkin --- pkg/kopia/cli/repository/storage/gcs/gcs.go | 37 +++++++++++++ .../cli/repository/storage/gcs/gcs_opts.go | 38 +++++++++++++ .../repository/storage/gcs/gcs_opts_test.go | 43 +++++++++++++++ .../cli/repository/storage/gcs/gcs_test.go | 53 +++++++++++++++++++ 4 files changed, 171 insertions(+) create mode 100644 pkg/kopia/cli/repository/storage/gcs/gcs.go create mode 100644 pkg/kopia/cli/repository/storage/gcs/gcs_opts.go create mode 100644 pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go create mode 100644 pkg/kopia/cli/repository/storage/gcs/gcs_test.go diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs.go b/pkg/kopia/cli/repository/storage/gcs/gcs.go new file mode 100644 index 0000000000..884f67ff99 --- /dev/null +++ b/pkg/kopia/cli/repository/storage/gcs/gcs.go @@ -0,0 +1,37 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcs + +import ( + "github.com/kanisterio/safecli/command" + + "github.com/kanisterio/kanister/pkg/consts" + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" + "github.com/kanisterio/kanister/pkg/log" +) + +// New creates a new subcommand for the GCS storage. +func New(location internal.Location, repoPathPrefix string, _ log.Logger) command.Applier { + prefix := internal.GenerateFullRepoPath(location.Prefix(), repoPathPrefix) + if prefix == "" { + return command.NewErrorArgument(cli.ErrInvalidRepoPath) + } + return command.NewArguments(subcmdGCS, + optBucket(location.BucketName()), + optCredentialsFile(consts.GoogleCloudCredsFilePath), + optPrefix(prefix), + ) +} diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs_opts.go b/pkg/kopia/cli/repository/storage/gcs/gcs_opts.go new file mode 100644 index 0000000000..1ddc62702d --- /dev/null +++ b/pkg/kopia/cli/repository/storage/gcs/gcs_opts.go @@ -0,0 +1,38 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcs + +import ( + "github.com/kanisterio/safecli/command" +) + +var ( + subcmdGCS = command.NewArgument("gcs") +) + +// optBucket creates a new bucket option with a given name. +func optBucket(name string) command.Applier { + return command.NewOptionWithArgument("--bucket", name) +} + +// optPrefix creates a new prefix option with a given prefix. +func optPrefix(prefix string) command.Applier { + return command.NewOptionWithArgument("--prefix", prefix) +} + +// optCredentialsFile creates a new GCS credentials file option with a given file path. +func optCredentialsFile(path string) command.Applier { + return command.NewOptionWithArgument("--credentials-file", path) +} diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go b/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go new file mode 100644 index 0000000000..65fff33f5c --- /dev/null +++ b/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go @@ -0,0 +1,43 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcs + +import ( + "testing" + + "github.com/kanisterio/safecli/command" + "github.com/kanisterio/safecli/test" + "gopkg.in/check.v1" +) + +func TestGCSOptions(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ + { + Name: "optBucket", + Argument: command.NewArguments(optBucket("bucketname"), optBucket("")), + ExpectedCLI: []string{"cmd", "--bucket=bucketname"}, + }, + { + Name: "optPrefix", + Argument: command.NewArguments(optPrefix("prefix"), optPrefix("")), + ExpectedCLI: []string{"cmd", "--prefix=prefix"}, + }, + { + Name: "optCredentialsFile", + Argument: command.NewArguments(optCredentialsFile("/tmp/file.creds"), optCredentialsFile("")), + ExpectedCLI: []string{"cmd", "--credentials-file=/tmp/file.creds"}, + }, +}}) diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs_test.go b/pkg/kopia/cli/repository/storage/gcs/gcs_test.go new file mode 100644 index 0000000000..181e4427cb --- /dev/null +++ b/pkg/kopia/cli/repository/storage/gcs/gcs_test.go @@ -0,0 +1,53 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gcs + +import ( + "testing" + + "github.com/kanisterio/safecli/command" + "github.com/kanisterio/safecli/test" + "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" +) + +func TestNewGCS(t *testing.T) { check.TestingT(t) } + +func newGCS(prefix, repoPath string) command.Applier { + l := internal.Location{ + "prefix": []byte(prefix), + } + return New(l, repoPath, nil) +} + +var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ + { + Name: "NewGCS", + Argument: newGCS("prefix", "repoPath"), + ExpectedCLI: []string{"cmd", "gcs", "--credentials-file=/tmp/creds.txt", "--prefix=prefix/repoPath/"}, + }, + { + Name: "NewGCS with empty repoPath", + Argument: newGCS("prefix", ""), + ExpectedCLI: []string{"cmd", "gcs", "--credentials-file=/tmp/creds.txt", "--prefix=prefix/"}, + }, + { + Name: "NewGCS with empty local prefix and repo prefix should return error", + Argument: newGCS("", ""), + ExpectedErr: cli.ErrInvalidRepoPath, + }, +}}) From c3cd83cf9e8ff992485ec05a196264e9ba96bf3d Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 16 Feb 2024 23:55:42 -0800 Subject: [PATCH 30/73] Reorganize imports Signed-off-by: pavel.larkin --- pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go b/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go index 65fff33f5c..a0737a94cd 100644 --- a/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go +++ b/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go @@ -17,9 +17,10 @@ package gcs import ( "testing" + "gopkg.in/check.v1" + "github.com/kanisterio/safecli/command" "github.com/kanisterio/safecli/test" - "gopkg.in/check.v1" ) func TestGCSOptions(t *testing.T) { check.TestingT(t) } From 78c7092af6694e839f0582dd3781d7e6971e3ca7 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Sat, 17 Feb 2024 00:10:15 -0800 Subject: [PATCH 31/73] Add Kopia Azure storage opts Signed-off-by: pavel.larkin --- .../flag/storage/azure/azure_flags_test.go | 45 --------------- .../internal/flag/storage/azure/azure_test.go | 57 ------------------- .../storage/azure/azure.go | 22 ++++--- .../storage/azure/azure_opts.go} | 22 +++---- .../storage/azure/azure_opts_test.go | 39 +++++++++++++ .../repository/storage/azure/azure_test.go | 54 ++++++++++++++++++ 6 files changed, 118 insertions(+), 121 deletions(-) delete mode 100644 pkg/kopia/cli/internal/flag/storage/azure/azure_flags_test.go delete mode 100644 pkg/kopia/cli/internal/flag/storage/azure/azure_test.go rename pkg/kopia/cli/{internal/flag => repository}/storage/azure/azure.go (51%) rename pkg/kopia/cli/{internal/flag/storage/azure/azure_flags.go => repository/storage/azure/azure_opts.go} (57%) create mode 100644 pkg/kopia/cli/repository/storage/azure/azure_opts_test.go create mode 100644 pkg/kopia/cli/repository/storage/azure/azure_test.go diff --git a/pkg/kopia/cli/internal/flag/storage/azure/azure_flags_test.go b/pkg/kopia/cli/internal/flag/storage/azure/azure_flags_test.go deleted file mode 100644 index 6d44ed325d..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/azure/azure_flags_test.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package azure - -import ( - "testing" - - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" - "gopkg.in/check.v1" -) - -func TestStorageAzureFlags(t *testing.T) { check.TestingT(t) } - -var _ = check.Suite(test.NewFlagSuite([]test.FlagTest{ - { - Name: "Empty Prefix should not generate a flag", - Flag: Prefix(""), - }, - { - Name: "Prefix with value should generate a flag with the given value", - Flag: Prefix("prefix"), - ExpectedCLI: []string{"--prefix=prefix"}, - }, - { - Name: "Empty AzureCountainer should not generate a flag", - Flag: Countainer(""), - }, - { - Name: "AzureCountainer with value should generate a flag with the given value", - Flag: Countainer("container"), - ExpectedCLI: []string{"--container=container"}, - }, -})) diff --git a/pkg/kopia/cli/internal/flag/storage/azure/azure_test.go b/pkg/kopia/cli/internal/flag/storage/azure/azure_test.go deleted file mode 100644 index 3e16ac8f40..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/azure/azure_test.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package azure - -import ( - "testing" - - "gopkg.in/check.v1" - - "github.com/kanisterio/safecli" - - rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" - - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" -) - -func TestStorageAzure(t *testing.T) { check.TestingT(t) } - -var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ - { - Name: "Empty Azure storage flag should generate subcommand with default flags", - CLI: func() (safecli.CommandBuilder, error) { - return New(model.StorageFlag{}) - }, - ExpectedCLI: []string{"azure"}, - }, - { - Name: "Azure with values should generate subcommand with specific flags", - CLI: func() (safecli.CommandBuilder, error) { - return New(model.StorageFlag{ - RepoPathPrefix: "repo/path/prefix", - Location: model.Location{ - rs.PrefixKey: []byte("prefix"), - rs.BucketKey: []byte("container"), - }, - }) - }, - ExpectedCLI: []string{ - "azure", - "--container=container", - "--prefix=prefix/repo/path/prefix/", - }, - }, -})) diff --git a/pkg/kopia/cli/internal/flag/storage/azure/azure.go b/pkg/kopia/cli/repository/storage/azure/azure.go similarity index 51% rename from pkg/kopia/cli/internal/flag/storage/azure/azure.go rename to pkg/kopia/cli/repository/storage/azure/azure.go index 0fb6e8387f..a00233d604 100644 --- a/pkg/kopia/cli/internal/flag/storage/azure/azure.go +++ b/pkg/kopia/cli/repository/storage/azure/azure.go @@ -15,17 +15,21 @@ package azure import ( - "github.com/kanisterio/safecli" + "github.com/kanisterio/safecli/command" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/command" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" + "github.com/kanisterio/kanister/pkg/log" ) -// New returns a builder for the Azure subcommand storage. -func New(f model.StorageFlag) (*safecli.Builder, error) { - prefix := model.GenerateFullRepoPath(f.Location.Prefix(), f.RepoPathPrefix) - return command.NewCommandBuilder(command.Azure, - Countainer(f.Location.BucketName()), - Prefix(prefix), +// New creates a new subcommand for the Azure storage. +func New(location internal.Location, repoPathPrefix string, _ log.Logger) command.Applier { + prefix := internal.GenerateFullRepoPath(location.Prefix(), repoPathPrefix) + if prefix == "" { + return command.NewErrorArgument(cli.ErrInvalidRepoPath) + } + return command.NewArguments(subcmdAzure, + optContainer(location.BucketName()), + optPrefix(prefix), ) } diff --git a/pkg/kopia/cli/internal/flag/storage/azure/azure_flags.go b/pkg/kopia/cli/repository/storage/azure/azure_opts.go similarity index 57% rename from pkg/kopia/cli/internal/flag/storage/azure/azure_flags.go rename to pkg/kopia/cli/repository/storage/azure/azure_opts.go index a9943e617a..5618d090eb 100644 --- a/pkg/kopia/cli/internal/flag/storage/azure/azure_flags.go +++ b/pkg/kopia/cli/repository/storage/azure/azure_opts.go @@ -14,18 +14,20 @@ package azure -import "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" +import ( + "github.com/kanisterio/safecli/command" +) -// -// Azure flags. -// +var ( + subcmdAzure = command.NewArgument("azure") +) -// Prefix creates a new Azure prefix flag with a given prefix. -func Prefix(prefix string) flag.Applier { - return flag.NewStringFlag("--prefix", prefix) +// optPrefix creates a new prefix option with a given prefix. +func optPrefix(prefix string) command.Applier { + return command.NewOptionWithArgument("--prefix", prefix) } -// Countainer creates a new Azure container flag with a given container name. -func Countainer(name string) flag.Applier { - return flag.NewStringFlag("--container", name) +// optContainer creates a new container option with a given container name. +func optContainer(name string) command.Applier { + return command.NewOptionWithArgument("--container", name) } diff --git a/pkg/kopia/cli/repository/storage/azure/azure_opts_test.go b/pkg/kopia/cli/repository/storage/azure/azure_opts_test.go new file mode 100644 index 0000000000..f85a3e58da --- /dev/null +++ b/pkg/kopia/cli/repository/storage/azure/azure_opts_test.go @@ -0,0 +1,39 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azure + +import ( + "testing" + + "gopkg.in/check.v1" + + "github.com/kanisterio/safecli/command" + "github.com/kanisterio/safecli/test" +) + +func TestAzureOptions(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ + { + Name: "optContainer", + Argument: command.NewArguments(optContainer("containername"), optContainer("")), + ExpectedCLI: []string{"cmd", "--container=containername"}, + }, + { + Name: "optPrefix", + Argument: command.NewArguments(optPrefix("prefix"), optPrefix("")), + ExpectedCLI: []string{"cmd", "--prefix=prefix"}, + }, +}}) diff --git a/pkg/kopia/cli/repository/storage/azure/azure_test.go b/pkg/kopia/cli/repository/storage/azure/azure_test.go new file mode 100644 index 0000000000..942db5e698 --- /dev/null +++ b/pkg/kopia/cli/repository/storage/azure/azure_test.go @@ -0,0 +1,54 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package azure + +import ( + "testing" + + "github.com/kanisterio/safecli/command" + "github.com/kanisterio/safecli/test" + "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" +) + +func TestNewAzure(t *testing.T) { check.TestingT(t) } + +func newAzure(prefix, repoPath string) command.Applier { + l := internal.Location{ + "prefix": []byte(prefix), + "bucket": []byte("bucket"), + } + return New(l, repoPath, nil) +} + +var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ + { + Name: "NewAzure", + Argument: newAzure("prefix", "repoPath"), + ExpectedCLI: []string{"cmd", "azure", "--container=bucket", "--prefix=prefix/repoPath/"}, + }, + { + Name: "NewAzure with empty repoPath", + Argument: newAzure("prefix", ""), + ExpectedCLI: []string{"cmd", "azure", "--container=bucket", "--prefix=prefix/"}, + }, + { + Name: "NewAzure with empty local prefix and repo prefix should return error", + Argument: newAzure("", ""), + ExpectedErr: cli.ErrInvalidRepoPath, + }, +}}) From b2f956acb2dae4be5f7473d6583c337579e73bc1 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Sat, 17 Feb 2024 00:12:09 -0800 Subject: [PATCH 32/73] Fix gcs test Signed-off-by: pavel.larkin --- pkg/kopia/cli/repository/storage/gcs/gcs_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs_test.go b/pkg/kopia/cli/repository/storage/gcs/gcs_test.go index 181e4427cb..7a522e31f3 100644 --- a/pkg/kopia/cli/repository/storage/gcs/gcs_test.go +++ b/pkg/kopia/cli/repository/storage/gcs/gcs_test.go @@ -30,6 +30,7 @@ func TestNewGCS(t *testing.T) { check.TestingT(t) } func newGCS(prefix, repoPath string) command.Applier { l := internal.Location{ "prefix": []byte(prefix), + "bucket": []byte("bucket"), } return New(l, repoPath, nil) } @@ -38,12 +39,12 @@ var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTe { Name: "NewGCS", Argument: newGCS("prefix", "repoPath"), - ExpectedCLI: []string{"cmd", "gcs", "--credentials-file=/tmp/creds.txt", "--prefix=prefix/repoPath/"}, + ExpectedCLI: []string{"cmd", "gcs", "--bucket=bucket", "--credentials-file=/tmp/creds.txt", "--prefix=prefix/repoPath/"}, }, { Name: "NewGCS with empty repoPath", Argument: newGCS("prefix", ""), - ExpectedCLI: []string{"cmd", "gcs", "--credentials-file=/tmp/creds.txt", "--prefix=prefix/"}, + ExpectedCLI: []string{"cmd", "gcs", "--bucket=bucket", "--credentials-file=/tmp/creds.txt", "--prefix=prefix/"}, }, { Name: "NewGCS with empty local prefix and repo prefix should return error", From b75596634b348d3c3013a6dcef341f62a543b16a Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Sat, 17 Feb 2024 00:38:20 -0800 Subject: [PATCH 33/73] Add Kopia S3 and S3 compliant storage opts Signed-off-by: pavel.larkin --- .../cli/internal/flag/storage/s3/s3_flags.go | 53 ------------ .../internal/flag/storage/s3/s3_flags_test.go | 81 ------------------- .../cli/internal/flag/storage/s3/s3_test.go | 75 ----------------- .../flag => repository}/storage/s3/s3.go | 36 +++++---- .../cli/repository/storage/s3/s3_opts.go | 53 ++++++++++++ .../cli/repository/storage/s3/s3_opts_test.go | 59 ++++++++++++++ .../cli/repository/storage/s3/s3_test.go | 80 ++++++++++++++++++ 7 files changed, 213 insertions(+), 224 deletions(-) delete mode 100644 pkg/kopia/cli/internal/flag/storage/s3/s3_flags.go delete mode 100644 pkg/kopia/cli/internal/flag/storage/s3/s3_flags_test.go delete mode 100644 pkg/kopia/cli/internal/flag/storage/s3/s3_test.go rename pkg/kopia/cli/{internal/flag => repository}/storage/s3/s3.go (58%) create mode 100644 pkg/kopia/cli/repository/storage/s3/s3_opts.go create mode 100644 pkg/kopia/cli/repository/storage/s3/s3_opts_test.go create mode 100644 pkg/kopia/cli/repository/storage/s3/s3_test.go diff --git a/pkg/kopia/cli/internal/flag/storage/s3/s3_flags.go b/pkg/kopia/cli/internal/flag/storage/s3/s3_flags.go deleted file mode 100644 index 9688b81155..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/s3/s3_flags.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package s3 - -import ( - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" -) - -// -// S3 flags. -// - -// Bucket creates a new S3 bucket flag with a given bucket name. -func Bucket(bucket string) flag.Applier { - return flag.NewStringFlag("--bucket", bucket) -} - -// Endpoint creates a new S3 endpoint flag with a given endpoint. -func Endpoint(endpoint string) flag.Applier { - return flag.NewStringFlag("--endpoint", endpoint) -} - -// Prefix creates a new S3 prefix flag with a given prefix. -func Prefix(prefix string) flag.Applier { - return flag.NewStringFlag("--prefix", prefix) -} - -// Region creates a new S3 region flag with a given region. -func Region(region string) flag.Applier { - return flag.NewStringFlag("--region", region) -} - -// DisableTLS creates a new S3 disable TLS flag. -func DisableTLS(disable bool) flag.Applier { - return flag.NewBoolFlag("--disable-tls", disable) -} - -// DisableTLSVerify creates a new S3 disable TLS verification flag. -func DisableTLSVerify(disable bool) flag.Applier { - return flag.NewBoolFlag("--disable-tls-verification", disable) -} diff --git a/pkg/kopia/cli/internal/flag/storage/s3/s3_flags_test.go b/pkg/kopia/cli/internal/flag/storage/s3/s3_flags_test.go deleted file mode 100644 index f77ebafa5d..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/s3/s3_flags_test.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package s3 - -import ( - "testing" - - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" - "gopkg.in/check.v1" -) - -func TestStorageS3Flags(t *testing.T) { check.TestingT(t) } - -var _ = check.Suite(test.NewFlagSuite([]test.FlagTest{ - { - Name: "Empty Bucket should not generate a flag", - Flag: Bucket(""), - }, - { - Name: "Bucket with value should generate a flag with the given value", - Flag: Bucket("bucket"), - ExpectedCLI: []string{"--bucket=bucket"}, - }, - { - Name: "Empty Endpoint should not generate a flag", - Flag: Endpoint(""), - }, - { - Name: "Endpoint with value should generate a flag with the given value", - Flag: Endpoint("endpoint"), - ExpectedCLI: []string{"--endpoint=endpoint"}, - }, - { - Name: "Empty Prefix should not generate a flag", - Flag: Prefix(""), - }, - { - Name: "Prefix with value should generate a flag with the given value", - Flag: Prefix("prefix"), - ExpectedCLI: []string{"--prefix=prefix"}, - }, - { - Name: "Empty Region should not generate a flag", - Flag: Region(""), - }, - { - Name: "Region with value should generate a flag with the given value", - Flag: Region("region"), - ExpectedCLI: []string{"--region=region"}, - }, - { - Name: "DisableTLS(false) should not generate a flag", - Flag: DisableTLS(false), - }, - { - Name: "DisableTLS(true) should generate a flag", - Flag: DisableTLS(true), - ExpectedCLI: []string{"--disable-tls"}, - }, - { - Name: "DisableTLSVerify(false) should not generate a flag", - Flag: DisableTLSVerify(false), - }, - { - Name: "DisableTLSVerify(true) should generate a flag", - Flag: DisableTLSVerify(true), - ExpectedCLI: []string{"--disable-tls-verification"}, - }, -})) diff --git a/pkg/kopia/cli/internal/flag/storage/s3/s3_test.go b/pkg/kopia/cli/internal/flag/storage/s3/s3_test.go deleted file mode 100644 index 131660c880..0000000000 --- a/pkg/kopia/cli/internal/flag/storage/s3/s3_test.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package s3 - -import ( - "testing" - - "gopkg.in/check.v1" - - "github.com/kanisterio/safecli" - - rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" - - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" -) - -func TestStorageS3(t *testing.T) { check.TestingT(t) } - -var logger = &test.StringLogger{} - -var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ - { - Name: "Empty S3 storage flag should generate subcommand with default flags", - CLI: func() (safecli.CommandBuilder, error) { - return New(model.StorageFlag{}) - }, - ExpectedCLI: []string{ - "s3", - }, - }, - { - Name: "S3 with values should generate subcommand with specific flags", - CLI: func() (safecli.CommandBuilder, error) { - return New(model.StorageFlag{ - RepoPathPrefix: "repo/path/prefix", - Location: model.Location{ - rs.PrefixKey: []byte("prefix"), - rs.EndpointKey: []byte("http://endpoint/path/"), - rs.RegionKey: []byte("region"), - rs.BucketKey: []byte("bucket"), - rs.SkipSSLVerifyKey: []byte("true"), - }, - Logger: logger, - }) - }, - ExpectedCLI: []string{ - "s3", - "--region=region", - "--bucket=bucket", - "--endpoint=endpoint/path", - "--prefix=prefix/repo/path/prefix/", - "--disable-tls", - "--disable-tls-verification", - }, - - Logger: logger, - LoggerRegex: []string{ - "Removing leading", - "Removing trailing", - }, - }, -})) diff --git a/pkg/kopia/cli/internal/flag/storage/s3/s3.go b/pkg/kopia/cli/repository/storage/s3/s3.go similarity index 58% rename from pkg/kopia/cli/internal/flag/storage/s3/s3.go rename to pkg/kopia/cli/repository/storage/s3/s3.go index 887e1e3584..610e03cc1e 100644 --- a/pkg/kopia/cli/internal/flag/storage/s3/s3.go +++ b/pkg/kopia/cli/repository/storage/s3/s3.go @@ -17,25 +17,31 @@ package s3 import ( "strings" - "github.com/kanisterio/safecli" + "github.com/kanisterio/safecli/command" + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" + intlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" "github.com/kanisterio/kanister/pkg/log" - - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/command" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" ) -// New returns a builder for the S3 subcommand storage. -func New(s model.StorageFlag) (*safecli.Builder, error) { - endpoint := resolveS3Endpoint(s.Location.Endpoint(), s.GetLogger()) - prefix := model.GenerateFullRepoPath(s.Location.Prefix(), s.RepoPathPrefix) - return command.NewCommandBuilder(command.S3, - Region(s.Location.Region()), - Bucket(s.Location.BucketName()), - Endpoint(endpoint), - Prefix(prefix), - DisableTLS(s.Location.IsInsecureEndpoint()), - DisableTLSVerify(s.Location.HasSkipSSLVerify()), +// New creates a new subcommand for the S3 storage. +func New(location internal.Location, repoPathPrefix string, logger log.Logger) command.Applier { + if logger == nil { + logger = intlog.NopLogger{} + } + endpoint := resolveS3Endpoint(location.Endpoint(), logger) + prefix := internal.GenerateFullRepoPath(location.Prefix(), repoPathPrefix) + if prefix == "" { + return command.NewErrorArgument(cli.ErrInvalidRepoPath) + } + return command.NewArguments(subcmdS3, + optRegion(location.Region()), + optBucket(location.BucketName()), + optEndpoint(endpoint), + optPrefix(prefix), + optDisableTLS(location.IsInsecureEndpoint()), + optDisableTLSVerify(location.HasSkipSSLVerify()), ) } diff --git a/pkg/kopia/cli/repository/storage/s3/s3_opts.go b/pkg/kopia/cli/repository/storage/s3/s3_opts.go new file mode 100644 index 0000000000..980537f24f --- /dev/null +++ b/pkg/kopia/cli/repository/storage/s3/s3_opts.go @@ -0,0 +1,53 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3 + +import ( + "github.com/kanisterio/safecli/command" +) + +var ( + subcmdS3 = command.NewArgument("s3") +) + +// optBucket creates a new bucket option with a given name. +func optBucket(name string) command.Applier { + return command.NewOptionWithArgument("--bucket", name) +} + +// optEndpoint creates a new endpoint option with a given endpoint. +func optEndpoint(endpoint string) command.Applier { + return command.NewOptionWithArgument("--endpoint", endpoint) +} + +// optPrefix creates a new prefix option with a given prefix. +func optPrefix(prefix string) command.Applier { + return command.NewOptionWithArgument("--prefix", prefix) +} + +// optRegion creates a new region option with a given region. +func optRegion(region string) command.Applier { + return command.NewOptionWithArgument("--region", region) +} + +// optDisableTLS creates a new disable TLS option with a given value. +func optDisableTLS(disable bool) command.Applier { + return command.NewOption("--disable-tls", disable) +} + +// optDisableTLSVerify creates a new disable TLS verification option with a given value. +func optDisableTLSVerify(disable bool) command.Applier { + return command.NewOption("--disable-tls-verification", disable) +} diff --git a/pkg/kopia/cli/repository/storage/s3/s3_opts_test.go b/pkg/kopia/cli/repository/storage/s3/s3_opts_test.go new file mode 100644 index 0000000000..44108bbf32 --- /dev/null +++ b/pkg/kopia/cli/repository/storage/s3/s3_opts_test.go @@ -0,0 +1,59 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3 + +import ( + "testing" + + "gopkg.in/check.v1" + + "github.com/kanisterio/safecli/command" + "github.com/kanisterio/safecli/test" +) + +func TestS3Options(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ + { + Name: "optBucket", + Argument: command.NewArguments(optBucket("bucketname"), optBucket("")), + ExpectedCLI: []string{"cmd", "--bucket=bucketname"}, + }, + { + Name: "optEndpoint", + Argument: command.NewArguments(optEndpoint("endpoint"), optEndpoint("")), + ExpectedCLI: []string{"cmd", "--endpoint=endpoint"}, + }, + { + Name: "optPrefix", + Argument: command.NewArguments(optPrefix("prefix"), optPrefix("")), + ExpectedCLI: []string{"cmd", "--prefix=prefix"}, + }, + { + Name: "optRegion", + Argument: command.NewArguments(optRegion("region"), optRegion("")), + ExpectedCLI: []string{"cmd", "--region=region"}, + }, + { + Name: "optDisableTLS", + Argument: command.NewArguments(optDisableTLS(true), optDisableTLS(false)), + ExpectedCLI: []string{"cmd", "--disable-tls"}, + }, + { + Name: "optDisableTLSVerify", + Argument: command.NewArguments(optDisableTLSVerify(true), optDisableTLSVerify(false)), + ExpectedCLI: []string{"cmd", "--disable-tls-verification"}, + }, +}}) diff --git a/pkg/kopia/cli/repository/storage/s3/s3_test.go b/pkg/kopia/cli/repository/storage/s3/s3_test.go new file mode 100644 index 0000000000..8e566a514f --- /dev/null +++ b/pkg/kopia/cli/repository/storage/s3/s3_test.go @@ -0,0 +1,80 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3 + +import ( + "testing" + + "github.com/kanisterio/safecli/command" + "github.com/kanisterio/safecli/test" + "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" +) + +func TestNewS3(t *testing.T) { check.TestingT(t) } + +func newS3(prefix, repoPath, endpoint string) command.Applier { + l := internal.Location{ + "prefix": []byte(prefix), + "endpoint": []byte(endpoint), + "region": []byte("region"), + "bucket": []byte("bucket"), + "skipSSLVerify": []byte("true"), + } + return New(l, repoPath, nil) +} + +var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ + { + Name: "NewS3", + Argument: newS3("prefix", "repoPath", "http://endpoint/path/"), + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--endpoint=endpoint/path", + "--prefix=prefix/repoPath/", + "--disable-tls", + "--disable-tls-verification", + }, + }, + { + Name: "NewS3 with empty repoPath and https endpoint", + Argument: newS3("prefix", "", "https://endpoint/path/"), + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--endpoint=endpoint/path", + "--prefix=prefix/", + "--disable-tls-verification", + }, + }, + { + Name: "NewS3 with empty repoPath and endpoint", + Argument: newS3("prefix", "", ""), + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--prefix=prefix/", + "--disable-tls-verification", + }, + }, + { + Name: "NewS3 with empty local prefix and repo prefix should return error", + Argument: newS3("", "", ""), + ExpectedErr: cli.ErrInvalidRepoPath, + }, +}}) From 0764e2839f75554ffe424a3beb5987e340d9c1fa Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Sat, 17 Feb 2024 01:49:25 -0800 Subject: [PATCH 34/73] Add Kopia S3 and S3 compliant storage opts Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/log/log.go | 29 +++ .../cli/repository/storage/s3/s3_test.go | 183 ++++++++++++++---- 2 files changed, 175 insertions(+), 37 deletions(-) diff --git a/pkg/kopia/cli/internal/log/log.go b/pkg/kopia/cli/internal/log/log.go index 4d2b29ffe3..e3d2ea2544 100644 --- a/pkg/kopia/cli/internal/log/log.go +++ b/pkg/kopia/cli/internal/log/log.go @@ -17,6 +17,7 @@ package storage import ( "context" "io" + "regexp" "github.com/kanisterio/kanister/pkg/field" "github.com/kanisterio/kanister/pkg/log" @@ -43,3 +44,31 @@ func (NopLogger) WithContext(ctx context.Context) log.Logger { func (NopLogger) WithError(err error) log.Logger { return &NopLogger{} } + +// StringLogger is a logger that stores log messages in a slice of strings. +type StringLogger []string + +func (l *StringLogger) Print(msg string, fields ...field.M) { + *l = append(*l, msg) +} + +func (l *StringLogger) PrintTo(w io.Writer, msg string, fields ...field.M) { + *l = append(*l, msg) +} + +func (l *StringLogger) WithContext(ctx context.Context) log.Logger { + return l +} + +func (l *StringLogger) WithError(err error) log.Logger { + return l +} + +func (l *StringLogger) MatchString(pattern string) bool { + for _, line := range *l { + if found, _ := regexp.MatchString(pattern, line); found { + return true + } + } + return false +} diff --git a/pkg/kopia/cli/repository/storage/s3/s3_test.go b/pkg/kopia/cli/repository/storage/s3/s3_test.go index 8e566a514f..2f67b61fb4 100644 --- a/pkg/kopia/cli/repository/storage/s3/s3_test.go +++ b/pkg/kopia/cli/repository/storage/s3/s3_test.go @@ -17,64 +17,173 @@ package s3 import ( "testing" - "github.com/kanisterio/safecli/command" "github.com/kanisterio/safecli/test" "gopkg.in/check.v1" "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/kanister/pkg/kopia/cli/internal" + intlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" + "github.com/kanisterio/kanister/pkg/log" ) func TestNewS3(t *testing.T) { check.TestingT(t) } -func newS3(prefix, repoPath, endpoint string) command.Applier { - l := internal.Location{ - "prefix": []byte(prefix), - "endpoint": []byte(endpoint), - "region": []byte("region"), - "bucket": []byte("bucket"), - "skipSSLVerify": []byte("true"), +// ArgTest extends test.ArgumentTest to include logger tests. +type ArgTest struct { + test test.ArgumentTest + + location internal.Location // location is the location to use for the test. + repoPath string // repoPath is the repository path to use for the test. + Logger log.Logger // Logger is the logger to use for the test. (optional) + LoggerRegex []string // LoggerRegex is a list of regexs to match against the log output. (optional) +} + +// Test runs the test with the given command and checks the log output. +func (t *ArgTest) Test(c *check.C, cmd string) { + t.test.Argument = New(t.location, t.repoPath, t.Logger) + t.test.Test(c, cmd) + t.assertLog(c) +} + +// assertLog checks the log output against the expected regexs. +func (t *ArgTest) assertLog(c *check.C) { + if t.Logger == nil { + return + } + log := t.Logger.(*intlog.StringLogger) + if len(t.LoggerRegex) == 1 && t.LoggerRegex[0] == "" { + cmtLog := check.Commentf("FAIL: log should be empty but got %#v", log) + c.Assert(len([]string(*log)), check.Equals, 0, cmtLog) + } else { + for _, regex := range t.LoggerRegex { + cmtLog := check.Commentf("FAIL: %v\nlog %#v expected to match %#v", t.test.Name, log, regex) + c.Assert(log.MatchString(regex), check.Equals, true, cmtLog) + } + + } +} + +// ArgSuite defines a suite of tests for a single ArgTest. +type ArgSuite struct { + Cmd string // Cmd appends to the safecli.Builder before test if not empty. + Arguments []ArgTest // Tests to run. +} + +// TestArguments runs all tests in the suite. +func (s *ArgSuite) TestArguments(c *check.C) { + for _, arg := range s.Arguments { + arg.Test(c, s.Cmd) } - return New(l, repoPath, nil) } -var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ +var _ = check.Suite(&ArgSuite{Cmd: "cmd", Arguments: []ArgTest{ { - Name: "NewS3", - Argument: newS3("prefix", "repoPath", "http://endpoint/path/"), - ExpectedCLI: []string{"cmd", "s3", - "--region=region", - "--bucket=bucket", - "--endpoint=endpoint/path", - "--prefix=prefix/repoPath/", - "--disable-tls", - "--disable-tls-verification", + test: test.ArgumentTest{ + Name: "NewS3", + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--endpoint=endpoint/path", + "--prefix=prefix/repoPath/", + "--disable-tls", + "--disable-tls-verification", + }, + }, + location: internal.Location{ + "prefix": []byte("prefix"), + "endpoint": []byte("http://endpoint/path/"), + "region": []byte("region"), + "bucket": []byte("bucket"), + "skipSSLVerify": []byte("true"), + }, + repoPath: "repoPath", + Logger: &intlog.StringLogger{}, + LoggerRegex: []string{ + "Removing leading", + "Removing trailing", }, }, { - Name: "NewS3 with empty repoPath and https endpoint", - Argument: newS3("prefix", "", "https://endpoint/path/"), - ExpectedCLI: []string{"cmd", "s3", - "--region=region", - "--bucket=bucket", - "--endpoint=endpoint/path", - "--prefix=prefix/", - "--disable-tls-verification", + test: test.ArgumentTest{ + Name: "NewS3 w/o logger should not panic", + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--endpoint=endpoint/path", + "--prefix=prefix/repoPath/", + "--disable-tls", + "--disable-tls-verification", + }, + }, + location: internal.Location{ + "prefix": []byte("prefix"), + "endpoint": []byte("http://endpoint/path/"), + "region": []byte("region"), + "bucket": []byte("bucket"), + "skipSSLVerify": []byte("true"), }, + repoPath: "repoPath", }, { - Name: "NewS3 with empty repoPath and endpoint", - Argument: newS3("prefix", "", ""), - ExpectedCLI: []string{"cmd", "s3", - "--region=region", - "--bucket=bucket", - "--prefix=prefix/", - "--disable-tls-verification", + test: test.ArgumentTest{ + Name: "NewS3 with empty repoPath and https endpoint", + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--endpoint=endpoint/path", + "--prefix=prefix/", + "--disable-tls-verification", + }, + }, + location: internal.Location{ + "prefix": []byte("prefix"), + "endpoint": []byte("https://endpoint/path/"), + "region": []byte("region"), + "bucket": []byte("bucket"), + "skipSSLVerify": []byte("true"), + }, + repoPath: "", + Logger: &intlog.StringLogger{}, + LoggerRegex: []string{ + "Removing leading", + "Removing trailing", }, }, { - Name: "NewS3 with empty local prefix and repo prefix should return error", - Argument: newS3("", "", ""), - ExpectedErr: cli.ErrInvalidRepoPath, + test: test.ArgumentTest{ + Name: "NewS3 with empty repoPath and endpoint", + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--prefix=prefix/", + "--disable-tls-verification", + }, + }, + location: internal.Location{ + "prefix": []byte("prefix"), + "endpoint": []byte(""), + "region": []byte("region"), + "bucket": []byte("bucket"), + "skipSSLVerify": []byte("true"), + }, + repoPath: "", + Logger: &intlog.StringLogger{}, + LoggerRegex: []string{""}, // no output expected + }, + { + test: test.ArgumentTest{ + Name: "NewS3 with empty repoPath, prefix and endpoint", + ExpectedErr: cli.ErrInvalidRepoPath, + }, + location: internal.Location{ + "prefix": []byte(""), + "endpoint": []byte(""), + "region": []byte("region"), + "bucket": []byte("bucket"), + "skipSSLVerify": []byte("true"), + }, + repoPath: "", + Logger: &intlog.StringLogger{}, + LoggerRegex: []string{""}, // no output expected }, }}) From 03607618b67820073ca87f0ecdeab841f0a85bc4 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Sat, 17 Feb 2024 01:53:54 -0800 Subject: [PATCH 35/73] Cleanup tests Signed-off-by: pavel.larkin --- .../cli/repository/storage/s3/s3_test.go | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/pkg/kopia/cli/repository/storage/s3/s3_test.go b/pkg/kopia/cli/repository/storage/s3/s3_test.go index 2f67b61fb4..0d8cd65d04 100644 --- a/pkg/kopia/cli/repository/storage/s3/s3_test.go +++ b/pkg/kopia/cli/repository/storage/s3/s3_test.go @@ -45,21 +45,30 @@ func (t *ArgTest) Test(c *check.C, cmd string) { t.assertLog(c) } +func (t *ArgTest) isEmptyLogExpected() bool { + return len(t.LoggerRegex) == 1 && t.LoggerRegex[0] == "" +} + // assertLog checks the log output against the expected regexs. func (t *ArgTest) assertLog(c *check.C) { if t.Logger == nil { return } - log := t.Logger.(*intlog.StringLogger) - if len(t.LoggerRegex) == 1 && t.LoggerRegex[0] == "" { + + log, ok := t.Logger.(*intlog.StringLogger) + if !ok { + c.Fatalf("t.Logger is not *intlog.StringLogger") + } + if t.isEmptyLogExpected() { cmtLog := check.Commentf("FAIL: log should be empty but got %#v", log) c.Assert(len([]string(*log)), check.Equals, 0, cmtLog) - } else { - for _, regex := range t.LoggerRegex { - cmtLog := check.Commentf("FAIL: %v\nlog %#v expected to match %#v", t.test.Name, log, regex) - c.Assert(log.MatchString(regex), check.Equals, true, cmtLog) - } + return + } + // Check each regex. + for _, regex := range t.LoggerRegex { + cmtLog := check.Commentf("FAIL: %v\nlog %#v expected to match %#v", t.test.Name, log, regex) + c.Assert(log.MatchString(regex), check.Equals, true, cmtLog) } } From be4031d37595569244239a5068af77470b83ca97 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Sat, 17 Feb 2024 02:55:48 -0800 Subject: [PATCH 36/73] Add Kopia repository create command Signed-off-by: pavel.larkin --- .../flag/repository/repository_flags.go | 65 ----------- .../flag/repository/repository_flags_test.go | 107 ------------------ pkg/kopia/cli/internal/kopia.go | 29 +++++ pkg/kopia/cli/internal/test/command_suite.go | 96 ++++++++++++++++ pkg/kopia/cli/repository/opts.go | 84 ++++++++++++++ pkg/kopia/cli/repository/repository_create.go | 34 +++--- pkg/kopia/cli/repository/repository_test.go | 73 ++++++------ 7 files changed, 264 insertions(+), 224 deletions(-) delete mode 100644 pkg/kopia/cli/internal/flag/repository/repository_flags.go delete mode 100644 pkg/kopia/cli/internal/flag/repository/repository_flags_test.go create mode 100644 pkg/kopia/cli/internal/kopia.go create mode 100644 pkg/kopia/cli/internal/test/command_suite.go create mode 100644 pkg/kopia/cli/repository/opts.go diff --git a/pkg/kopia/cli/internal/flag/repository/repository_flags.go b/pkg/kopia/cli/internal/flag/repository/repository_flags.go deleted file mode 100644 index bd586781cd..0000000000 --- a/pkg/kopia/cli/internal/flag/repository/repository_flags.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package repository - -import ( - "time" - - "github.com/go-openapi/strfmt" - - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" -) - -// Hostname creates a new host flag with a given hostname. -func Hostname(hostname string) flag.Applier { - return flag.NewStringFlag("--override-hostname", hostname) -} - -// Username creates a new username flag with a given username. -func Username(username string) flag.Applier { - return flag.NewStringFlag("--override-username", username) -} - -// BlobRetention creates a new blob retention flag with a given mode and period. -// If mode is empty, the flag will be a no-op. -func BlobRetention(mode string, period time.Duration) flag.Applier { - if mode == "" { - return flag.EmptyFlag() - } - return flag.NewFlags( - flag.NewStringFlag("--retention-mode", mode), - flag.NewStringFlag("--retention-period", period.String()), - ) -} - -// PIT creates a new point-in-time flag with a given point-in-time. -// If pit is zero, the flag will be a no-op. -func PIT(pit strfmt.DateTime) flag.Applier { - dt := strfmt.DateTime(pit) - if time.Time(dt).IsZero() { - return flag.EmptyFlag() - } - return flag.NewStringFlag("--point-in-time", dt.String()) -} - -// ServerURL creates a new server URL flag with a given server URL. -func ServerURL(serverURL string) flag.Applier { - return flag.NewStringFlag("--url", serverURL) -} - -// ServerCertFingerprint creates a new server certificate fingerprint flag with a given fingerprint. -func ServerCertFingerprint(fingerprint string) flag.Applier { - return flag.NewRedactedStringFlag("--server-cert-fingerprint", fingerprint) -} diff --git a/pkg/kopia/cli/internal/flag/repository/repository_flags_test.go b/pkg/kopia/cli/internal/flag/repository/repository_flags_test.go deleted file mode 100644 index cb50443d85..0000000000 --- a/pkg/kopia/cli/internal/flag/repository/repository_flags_test.go +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package repository - -import ( - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" - "gopkg.in/check.v1" -) - -func TestRepositoryFlags(t *testing.T) { check.TestingT(t) } - -var _ = check.Suite(test.NewFlagSuite([]test.FlagTest{ - { - Name: "Empty Hostname should not generate a flag", - Flag: Hostname(""), - }, - { - Name: "Hostname with value should generate a flag with given value", - Flag: Hostname("hostname"), - ExpectedCLI: []string{ - "--override-hostname=hostname", - }, - }, - { - Name: "Empty Username should not generate a flag", - Flag: Username(""), - }, - { - Name: "Username with value should generate a flag with given value", - Flag: Username("username"), - ExpectedCLI: []string{ - "--override-username=username", - }, - }, - { - Name: "Empty BlobRetention should not generate a flag", - Flag: BlobRetention("", time.Duration(0)), - }, - { - Name: "BlobRetention with values should generate multiple flags with given values", - Flag: BlobRetention("mode", 24*time.Hour), - ExpectedCLI: []string{ - "--retention-mode=mode", - "--retention-period=24h0m0s", - }, - }, - { - Name: "BlobRetention with RetentionMode only should generate mode flag with zero period", - Flag: BlobRetention("mode", 0), - ExpectedCLI: []string{ - "--retention-mode=mode", - "--retention-period=0s", - }, - }, - { - Name: "Empty PIT should not generate a flag", - Flag: PIT(strfmt.DateTime{}), - }, - { - Name: "PIT with value should generate a flag with given value", - Flag: PIT(func() strfmt.DateTime { - dt, _ := strfmt.ParseDateTime("2024-01-02T03:04:05.678Z") - return dt - }()), - ExpectedCLI: []string{ - "--point-in-time=2024-01-02T03:04:05.678Z", - }, - }, - { - Name: "Empty ServerURL should not generate a flag", - Flag: ServerURL(""), - }, - { - Name: "ServerURL with value should generate a flag with given value", - Flag: ServerURL("ServerURL"), - ExpectedCLI: []string{ - "--url=ServerURL", - }, - }, - { - Name: "Empty ServerCertFingerprint should not generate a flag", - Flag: ServerCertFingerprint(""), - }, - { - Name: "ServerCertFingerprint with value should generate a flag with given value and redact fingerprint for logs", - Flag: ServerCertFingerprint("ServerCertFingerprint"), - ExpectedCLI: []string{ - "--server-cert-fingerprint=ServerCertFingerprint", - }, - }, -})) diff --git a/pkg/kopia/cli/internal/kopia.go b/pkg/kopia/cli/internal/kopia.go new file mode 100644 index 0000000000..5af2df455d --- /dev/null +++ b/pkg/kopia/cli/internal/kopia.go @@ -0,0 +1,29 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "github.com/kanisterio/safecli" + "github.com/kanisterio/safecli/command" +) + +const ( + kopiaBinName = "kopia" +) + +// NewKopiaCommand creates a new safecli.Builder for the kopia command. +func NewKopiaCommand(args ...command.Applier) (*safecli.Builder, error) { + return command.New(kopiaBinName, args...) +} diff --git a/pkg/kopia/cli/internal/test/command_suite.go b/pkg/kopia/cli/internal/test/command_suite.go new file mode 100644 index 0000000000..0eebf793ff --- /dev/null +++ b/pkg/kopia/cli/internal/test/command_suite.go @@ -0,0 +1,96 @@ +package test + +import ( + "github.com/kanisterio/safecli" + "github.com/kanisterio/safecli/test" + "github.com/pkg/errors" + "gopkg.in/check.v1" +) + +// CommandTest defines a single test for a command. +type CommandTest struct { + // Name of the test. (required) + Name string + + // Command to test. (required) + Command func() (*safecli.Builder, error) + + // Expected CLI arguments. (optional) + ExpectedCLI []string + + // Expected log output. (optional) + // if empty, it will be set to ExpectedCLI joined with space. + // if empty and ExpectedCLI is empty, it will be ignored. + ExpectedLog string + + // Expected error. (optional) + // If nil, no error is expected and + // ExpectedCLI and ExpectedLog are checked. + ExpectedErr error +} + +// CheckCommentString implements check.CommentInterface +func (t *CommandTest) CheckCommentString() string { + return t.Name +} + +// setDefaultExpectedLog sets the default value for ExpectedLog based on ExpectedCLI. +func (t *CommandTest) setDefaultExpectedLog() { + if len(t.ExpectedLog) == 0 && len(t.ExpectedCLI) > 0 { + t.ExpectedLog = test.RedactCLI(t.ExpectedCLI) + } +} + +// assertNoError makes sure there is no error. +func (t *CommandTest) assertNoError(c *check.C, err error) { + c.Assert(err, check.IsNil, t) +} + +// assertError checks the error against ExpectedErr. +func (t *CommandTest) assertError(c *check.C, err error) { + actualErr := errors.Cause(err) + c.Assert(actualErr, check.Equals, t.ExpectedErr, t) +} + +// assertCLI asserts the builder's CLI output against ExpectedCLI. +func (t *CommandTest) assertCLI(c *check.C, b *safecli.Builder) { + if t.ExpectedCLI != nil { + c.Check(b.Build(), check.DeepEquals, t.ExpectedCLI, t) + } +} + +// assertLog asserts the builder's log output against ExpectedLog. +func (t *CommandTest) assertLog(c *check.C, b *safecli.Builder) { + if t.ExpectedCLI != nil { + t.setDefaultExpectedLog() + c.Check(b.String(), check.Equals, t.ExpectedLog, t) + } +} + +func (t *CommandTest) Test(c *check.C) { + cmd, err := t.Command() + if t.ExpectedErr == nil { + t.assertNoError(c, err) + } else { + t.assertError(c, err) + } + t.assertCLI(c, cmd) + t.assertLog(c, cmd) +} + +// CommandSuite defines a test suite for commands. +type CommandSuite struct { + Commands []CommandTest +} + +// TestCommands runs all tests in the suite. +func (s *CommandSuite) TestCommands(c *check.C) { + for _, cmd := range s.Commands { + cmd.Test(c) + } +} + +// NewCommandSuite creates a new CommandSuite. +func NewCommandSuite(commands []CommandTest) *CommandSuite { + return &CommandSuite{Commands: commands} +} diff --git a/pkg/kopia/cli/repository/opts.go b/pkg/kopia/cli/repository/opts.go new file mode 100644 index 0000000000..7a44b7ea72 --- /dev/null +++ b/pkg/kopia/cli/repository/opts.go @@ -0,0 +1,84 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repository + +import ( + "time" + + "github.com/kanisterio/safecli/command" + "github.com/pkg/errors" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" + "github.com/kanisterio/kanister/pkg/kopia/cli/repository/storage/azure" + "github.com/kanisterio/kanister/pkg/kopia/cli/repository/storage/fs" + "github.com/kanisterio/kanister/pkg/kopia/cli/repository/storage/gcs" + "github.com/kanisterio/kanister/pkg/kopia/cli/repository/storage/s3" + "github.com/kanisterio/kanister/pkg/log" + rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" +) + +var ( + cmdRepository = command.NewArgument("repository") + + subcmdCreate = command.NewArgument("create") + subcmdConnect = command.NewArgument("connect") +) + +// optHostname creates a new option for the hostname of the repository. +func optHostname(h string) command.Applier { + return command.NewOptionWithArgument("--override-hostname", h) +} + +// optUsername creates a new option for the username of the repository. +func optUsername(u string) command.Applier { + return command.NewOptionWithArgument("--override-username", u) +} + +// optBlobRetention creates new blob retention options with a given mode and period. +// If mode is empty, the retention is disabled. +func optBlobRetention(mode string, period time.Duration) command.Applier { + if mode == "" { + return command.NewNoopArgument() + } + return command.NewArguments( + command.NewOptionWithArgument("--retention-mode", mode), + command.NewOptionWithArgument("--retention-period", period.String()), + ) +} + +type storageBuilder func(internal.Location, string, log.Logger) command.Applier + +var storageBuilders = map[rs.LocType]storageBuilder{ + rs.LocTypeFilestore: fs.New, + rs.LocTypeAzure: azure.New, + rs.LocTypeS3: s3.New, + rs.LocTypes3Compliant: s3.New, + rs.LocTypeGCS: gcs.New, +} + +// optStorage creates a list of options for the specified storage location. +func optStorage(l internal.Location, repoPathPrefix string, logger log.Logger) command.Applier { + sb := storageBuilders[l.Type()] + if sb == nil { + return errUnsupportedStorageType(l.Type()) + } + return sb(l, repoPathPrefix, logger) +} + +func errUnsupportedStorageType(t rs.LocType) command.Applier { + err := errors.Wrapf(cli.ErrUnsupportedStorage, "storage location: %v", t) + return command.NewErrorArgument(err) +} diff --git a/pkg/kopia/cli/repository/repository_create.go b/pkg/kopia/cli/repository/repository_create.go index 90291380c2..f7efd91d54 100644 --- a/pkg/kopia/cli/repository/repository_create.go +++ b/pkg/kopia/cli/repository/repository_create.go @@ -19,19 +19,16 @@ import ( "github.com/kanisterio/safecli" + "github.com/kanisterio/kanister/pkg/kopia/cli/args" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/opts" "github.com/kanisterio/kanister/pkg/log" - - "github.com/kanisterio/kanister/pkg/kopia/cli" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/command" - flagcommon "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/common" - flagrepo "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/repository" - flagstorage "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage" ) // CreateArgs defines the arguments for the `kopia repository create` command. type CreateArgs struct { - cli.CommonArgs - cli.CacheArgs + args.Common // embeds common arguments + args.Cache // embeds cache arguments Hostname string // the hostname of the repository Username string // the username of the repository @@ -44,18 +41,19 @@ type CreateArgs struct { } // Create creates a new `kopia repository create ...` command. -func Create(args CreateArgs) (safecli.CommandBuilder, error) { - return command.NewKopiaCommandBuilder(args.CommonArgs, - command.Repository, command.Create, - flagcommon.NoCheckForUpdates, - flagcommon.Cache(args.CacheArgs), - flagrepo.Hostname(args.Hostname), - flagrepo.Username(args.Username), - flagrepo.BlobRetention(args.RetentionMode, args.RetentionPeriod), - flagstorage.Storage( +func Create(args CreateArgs) (*safecli.Builder, error) { + return internal.NewKopiaCommand( + opts.Common(args.Common), + cmdRepository, subcmdCreate, + opts.CheckForUpdates(false), + opts.Cache(args.Cache), + optHostname(args.Hostname), + optUsername(args.Username), + optBlobRetention(args.RetentionMode, args.RetentionPeriod), + optStorage( args.Location, args.RepoPathPrefix, - flagstorage.WithLogger(args.Logger), // log.Debug for old output + args.Logger, ), ) } diff --git a/pkg/kopia/cli/repository/repository_test.go b/pkg/kopia/cli/repository/repository_test.go index bd9ad2b720..66d4356721 100644 --- a/pkg/kopia/cli/repository/repository_test.go +++ b/pkg/kopia/cli/repository/repository_test.go @@ -18,21 +18,26 @@ import ( "testing" "time" - "gopkg.in/check.v1" - "github.com/kanisterio/safecli" - - rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" + "gopkg.in/check.v1" "github.com/kanisterio/kanister/pkg/kopia/cli" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage/model" + "github.com/kanisterio/kanister/pkg/kopia/cli/args" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" + rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" ) func TestRepositoryCommands(t *testing.T) { check.TestingT(t) } var ( - cacheArgs = cli.CacheArgs{ + common = args.Common{ + RepoPassword: "encr-key", + ConfigFilePath: "path/kopia.config", + LogDirectory: "cache/log", + } + + cache = args.Cache{ CacheDirectory: "/tmp/cache.dir", ContentCacheSizeLimitMB: 0, MetadataCacheSizeLimitMB: 0, @@ -41,24 +46,24 @@ var ( retentionMode = "Locked" retentionPeriod = 15 * time.Minute - locFS = model.Location{ + locFS = internal.Location{ rs.TypeKey: []byte("filestore"), rs.PrefixKey: []byte("test-prefix"), } - locAzure = model.Location{ + locAzure = internal.Location{ rs.TypeKey: []byte("azure"), rs.BucketKey: []byte("test-bucket"), rs.PrefixKey: []byte("test-prefix"), } - locGCS = model.Location{ + locGCS = internal.Location{ rs.TypeKey: []byte("gcs"), rs.BucketKey: []byte("test-bucket"), rs.PrefixKey: []byte("test-prefix"), } - locS3 = model.Location{ + locS3 = internal.Location{ rs.TypeKey: []byte("s3"), rs.EndpointKey: []byte("test-endpoint"), rs.RegionKey: []byte("test-region"), @@ -67,7 +72,7 @@ var ( rs.SkipSSLVerifyKey: []byte("false"), } - locS3Compliant = model.Location{ + locS3Compliant = internal.Location{ rs.TypeKey: []byte("s3Compliant"), rs.EndpointKey: []byte("test-endpoint"), rs.RegionKey: []byte("test-region"), @@ -81,10 +86,10 @@ var ( var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ { Name: "repository create with no storage", - CLI: func() (safecli.CommandBuilder, error) { + Command: func() (*safecli.Builder, error) { args := CreateArgs{ - CommonArgs: test.CommonArgs, - CacheArgs: cacheArgs, + Common: common, + Cache: cache, Hostname: "test-hostname", Username: "test-username", RepoPathPrefix: "test-path/prefix", @@ -95,10 +100,10 @@ var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ }, { Name: "repository create with filestore location", - CLI: func() (safecli.CommandBuilder, error) { + Command: func() (*safecli.Builder, error) { args := CreateArgs{ - CommonArgs: test.CommonArgs, - CacheArgs: cacheArgs, + Common: common, + Cache: cache, Hostname: "test-hostname", Username: "test-username", RepoPathPrefix: "test-path/prefix", @@ -110,8 +115,8 @@ var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ }, ExpectedCLI: []string{"kopia", "--config-file=path/kopia.config", - "--log-level=error", "--log-dir=cache/log", + "--log-level=error", "--password=encr-key", "repository", "create", @@ -129,10 +134,10 @@ var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ }, { Name: "repository create with azure location", - CLI: func() (safecli.CommandBuilder, error) { + Command: func() (*safecli.Builder, error) { args := CreateArgs{ - CommonArgs: test.CommonArgs, - CacheArgs: cacheArgs, + Common: common, + Cache: cache, Hostname: "test-hostname", Username: "test-username", RepoPathPrefix: "test-path/prefix", @@ -144,8 +149,8 @@ var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ }, ExpectedCLI: []string{"kopia", "--config-file=path/kopia.config", - "--log-level=error", "--log-dir=cache/log", + "--log-level=error", "--password=encr-key", "repository", "create", @@ -164,10 +169,10 @@ var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ }, { Name: "repository create with gcs location", - CLI: func() (safecli.CommandBuilder, error) { + Command: func() (*safecli.Builder, error) { args := CreateArgs{ - CommonArgs: test.CommonArgs, - CacheArgs: cacheArgs, + Common: common, + Cache: cache, Hostname: "test-hostname", Username: "test-username", RepoPathPrefix: "test-path/prefix", @@ -179,8 +184,8 @@ var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ }, ExpectedCLI: []string{"kopia", "--config-file=path/kopia.config", - "--log-level=error", "--log-dir=cache/log", + "--log-level=error", "--password=encr-key", "repository", "create", @@ -200,10 +205,10 @@ var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ }, { Name: "repository create with s3 location", - CLI: func() (safecli.CommandBuilder, error) { + Command: func() (*safecli.Builder, error) { args := CreateArgs{ - CommonArgs: test.CommonArgs, - CacheArgs: cacheArgs, + Common: common, + Cache: cache, Hostname: "test-hostname", Username: "test-username", RepoPathPrefix: "test-path/prefix", @@ -215,8 +220,8 @@ var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ }, ExpectedCLI: []string{"kopia", "--config-file=path/kopia.config", - "--log-level=error", "--log-dir=cache/log", + "--log-level=error", "--password=encr-key", "repository", "create", @@ -237,10 +242,10 @@ var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ }, { Name: "repository create with s3 compliant location", - CLI: func() (safecli.CommandBuilder, error) { + Command: func() (*safecli.Builder, error) { args := CreateArgs{ - CommonArgs: test.CommonArgs, - CacheArgs: cacheArgs, + Common: common, + Cache: cache, Hostname: "test-hostname", Username: "test-username", RepoPathPrefix: "test-path/prefix", @@ -252,8 +257,8 @@ var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ }, ExpectedCLI: []string{"kopia", "--config-file=path/kopia.config", - "--log-level=error", "--log-dir=cache/log", + "--log-level=error", "--password=encr-key", "repository", "create", From 130747e340613bfed9027880059188e8121237c6 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Sat, 17 Feb 2024 02:59:58 -0800 Subject: [PATCH 37/73] Cleanup Signed-off-by: pavel.larkin --- pkg/kopia/cli/repository/opts.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/kopia/cli/repository/opts.go b/pkg/kopia/cli/repository/opts.go index 7a44b7ea72..04a63002b4 100644 --- a/pkg/kopia/cli/repository/opts.go +++ b/pkg/kopia/cli/repository/opts.go @@ -33,8 +33,7 @@ import ( var ( cmdRepository = command.NewArgument("repository") - subcmdCreate = command.NewArgument("create") - subcmdConnect = command.NewArgument("connect") + subcmdCreate = command.NewArgument("create") ) // optHostname creates a new option for the hostname of the repository. From 74a6567d75d5aaaf634d05324aa183fa07dec0d7 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Sat, 17 Feb 2024 15:43:46 -0800 Subject: [PATCH 38/73] Reorganize tests Signed-off-by: pavel.larkin --- pkg/kopia/cli/repository/data_test.go | 67 ++++++++++++++++ pkg/kopia/cli/repository/opts_test.go | 79 +++++++++++++++++++ ...tory_test.go => repository_create_test.go} | 58 +------------- 3 files changed, 147 insertions(+), 57 deletions(-) create mode 100644 pkg/kopia/cli/repository/data_test.go create mode 100644 pkg/kopia/cli/repository/opts_test.go rename pkg/kopia/cli/repository/{repository_test.go => repository_create_test.go} (80%) diff --git a/pkg/kopia/cli/repository/data_test.go b/pkg/kopia/cli/repository/data_test.go new file mode 100644 index 0000000000..0b6ef70b29 --- /dev/null +++ b/pkg/kopia/cli/repository/data_test.go @@ -0,0 +1,67 @@ +package repository + +import ( + "time" + + "github.com/kanisterio/kanister/pkg/kopia/cli/args" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" + rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" +) + +var ( + common = args.Common{ + RepoPassword: "encr-key", + ConfigFilePath: "path/kopia.config", + LogDirectory: "cache/log", + } + + cache = args.Cache{ + CacheDirectory: "/tmp/cache.dir", + ContentCacheSizeLimitMB: 0, + MetadataCacheSizeLimitMB: 0, + } +) + +var ( + retentionMode = "Locked" + retentionPeriod = 15 * time.Minute + + locFS = internal.Location{ + rs.TypeKey: []byte("filestore"), + rs.PrefixKey: []byte("test-prefix"), + } + + locAzure = internal.Location{ + rs.TypeKey: []byte("azure"), + rs.BucketKey: []byte("test-bucket"), + rs.PrefixKey: []byte("test-prefix"), + } + + locGCS = internal.Location{ + rs.TypeKey: []byte("gcs"), + rs.BucketKey: []byte("test-bucket"), + rs.PrefixKey: []byte("test-prefix"), + } + + locS3 = internal.Location{ + rs.TypeKey: []byte("s3"), + rs.EndpointKey: []byte("test-endpoint"), + rs.RegionKey: []byte("test-region"), + rs.BucketKey: []byte("test-bucket"), + rs.PrefixKey: []byte("test-prefix"), + rs.SkipSSLVerifyKey: []byte("false"), + } + + locS3Compliant = internal.Location{ + rs.TypeKey: []byte("s3Compliant"), + rs.EndpointKey: []byte("test-endpoint"), + rs.RegionKey: []byte("test-region"), + rs.BucketKey: []byte("test-bucket"), + rs.PrefixKey: []byte("test-prefix"), + rs.SkipSSLVerifyKey: []byte("false"), + } + + locFTP = internal.Location{ + rs.TypeKey: []byte("ftp"), + } +) diff --git a/pkg/kopia/cli/repository/opts_test.go b/pkg/kopia/cli/repository/opts_test.go new file mode 100644 index 0000000000..3dd798291e --- /dev/null +++ b/pkg/kopia/cli/repository/opts_test.go @@ -0,0 +1,79 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repository + +import ( + "testing" + + "github.com/kanisterio/safecli/command" + "github.com/kanisterio/safecli/test" + "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli" +) + +func TestRepositoryOptions(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ + { + Name: "optHostname", + Argument: command.NewArguments( + optHostname("host"), + optHostname(""), // no output + ), + ExpectedCLI: []string{"cmd", "--override-hostname=host"}, + }, + { + Name: "optUsername", + Argument: command.NewArguments( + optUsername("user"), + optUsername(""), // no output + ), + ExpectedCLI: []string{"cmd", "--override-username=user"}, + }, + { + Name: "optBlobRetention", + Argument: command.NewArguments( + optBlobRetention(retentionMode, retentionPeriod), + optBlobRetention("", 0), // no output + ), + ExpectedCLI: []string{"cmd", "--retention-mode=Locked", "--retention-period=15m0s"}, + }, + { + Name: "optStorage FS", + Argument: optStorage(locFS, "repoPathPrefix", nil), + ExpectedCLI: []string{"cmd", "filesystem", "--path=/mnt/data/test-prefix/repoPathPrefix/"}, + }, + { + Name: "optStorage Azure", + Argument: optStorage(locAzure, "repoPathPrefix", nil), + ExpectedCLI: []string{"cmd", "azure", "--container=test-bucket", "--prefix=test-prefix/repoPathPrefix/"}, + }, + { + Name: "optStorage S3", + Argument: optStorage(locS3, "repoPathPrefix", nil), + ExpectedCLI: []string{"cmd", "s3", "--region=test-region", "--bucket=test-bucket", "--endpoint=test-endpoint", "--prefix=test-prefix/repoPathPrefix/"}, + }, + { + Name: "optStorage S3Compliant", + Argument: optStorage(locS3Compliant, "repoPathPrefix", nil), + ExpectedCLI: []string{"cmd", "s3", "--region=test-region", "--bucket=test-bucket", "--endpoint=test-endpoint", "--prefix=test-prefix/repoPathPrefix/"}, + }, + { + Name: "optStorage FTP Unsupported", + Argument: optStorage(locFTP, "repoPathPrefix", nil), + ExpectedErr: cli.ErrUnsupportedStorage, + }, +}}) diff --git a/pkg/kopia/cli/repository/repository_test.go b/pkg/kopia/cli/repository/repository_create_test.go similarity index 80% rename from pkg/kopia/cli/repository/repository_test.go rename to pkg/kopia/cli/repository/repository_create_test.go index 66d4356721..7a9001932c 100644 --- a/pkg/kopia/cli/repository/repository_test.go +++ b/pkg/kopia/cli/repository/repository_create_test.go @@ -16,71 +16,15 @@ package repository import ( "testing" - "time" "github.com/kanisterio/safecli" "gopkg.in/check.v1" "github.com/kanisterio/kanister/pkg/kopia/cli" - "github.com/kanisterio/kanister/pkg/kopia/cli/args" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal" "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" - rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" ) -func TestRepositoryCommands(t *testing.T) { check.TestingT(t) } - -var ( - common = args.Common{ - RepoPassword: "encr-key", - ConfigFilePath: "path/kopia.config", - LogDirectory: "cache/log", - } - - cache = args.Cache{ - CacheDirectory: "/tmp/cache.dir", - ContentCacheSizeLimitMB: 0, - MetadataCacheSizeLimitMB: 0, - } - - retentionMode = "Locked" - retentionPeriod = 15 * time.Minute - - locFS = internal.Location{ - rs.TypeKey: []byte("filestore"), - rs.PrefixKey: []byte("test-prefix"), - } - - locAzure = internal.Location{ - rs.TypeKey: []byte("azure"), - rs.BucketKey: []byte("test-bucket"), - rs.PrefixKey: []byte("test-prefix"), - } - - locGCS = internal.Location{ - rs.TypeKey: []byte("gcs"), - rs.BucketKey: []byte("test-bucket"), - rs.PrefixKey: []byte("test-prefix"), - } - - locS3 = internal.Location{ - rs.TypeKey: []byte("s3"), - rs.EndpointKey: []byte("test-endpoint"), - rs.RegionKey: []byte("test-region"), - rs.BucketKey: []byte("test-bucket"), - rs.PrefixKey: []byte("test-prefix"), - rs.SkipSSLVerifyKey: []byte("false"), - } - - locS3Compliant = internal.Location{ - rs.TypeKey: []byte("s3Compliant"), - rs.EndpointKey: []byte("test-endpoint"), - rs.RegionKey: []byte("test-region"), - rs.BucketKey: []byte("test-bucket"), - rs.PrefixKey: []byte("test-prefix"), - rs.SkipSSLVerifyKey: []byte("false"), - } -) +func TestRepositoryCreateCommand(t *testing.T) { check.TestingT(t) } // Test Repository Create command var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ From 37991ae4d44ce336c14e5a3d5ab76e519b026d02 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Sat, 17 Feb 2024 16:07:04 -0800 Subject: [PATCH 39/73] Add Kopia repository connect command Signed-off-by: pavel.larkin --- pkg/kopia/cli/repository/opts.go | 17 +++- .../cli/repository/repository_connect.go | 40 ++++----- .../cli/repository/repository_connect_test.go | 82 +++++++++++++++++++ .../cli/repository/repository_create_test.go | 70 ---------------- 4 files changed, 119 insertions(+), 90 deletions(-) create mode 100644 pkg/kopia/cli/repository/repository_connect_test.go diff --git a/pkg/kopia/cli/repository/opts.go b/pkg/kopia/cli/repository/opts.go index 04a63002b4..2f7e12db7e 100644 --- a/pkg/kopia/cli/repository/opts.go +++ b/pkg/kopia/cli/repository/opts.go @@ -17,6 +17,7 @@ package repository import ( "time" + "github.com/go-openapi/strfmt" "github.com/kanisterio/safecli/command" "github.com/pkg/errors" @@ -33,7 +34,8 @@ import ( var ( cmdRepository = command.NewArgument("repository") - subcmdCreate = command.NewArgument("create") + subcmdCreate = command.NewArgument("create") + subcmdConnect = command.NewArgument("connect") ) // optHostname creates a new option for the hostname of the repository. @@ -81,3 +83,16 @@ func errUnsupportedStorageType(t rs.LocType) command.Applier { err := errors.Wrapf(cli.ErrUnsupportedStorage, "storage location: %v", t) return command.NewErrorArgument(err) } + +// optReadOnly creates a new option for the read-only mode of the repository. +func optReadOnly(readOnly bool) command.Applier { + return command.NewOption("--readonly", readOnly) +} + +// optPointInTime creates a new option for the point-in-time of the repository. +func optPointInTime(tm strfmt.DateTime) command.Applier { + if time.Time(tm).IsZero() { + return command.NewNoopArgument() + } + return command.NewOptionWithArgument("--point-in-time", tm.String()) +} diff --git a/pkg/kopia/cli/repository/repository_connect.go b/pkg/kopia/cli/repository/repository_connect.go index 5b4e20dae4..2bb6143c63 100644 --- a/pkg/kopia/cli/repository/repository_connect.go +++ b/pkg/kopia/cli/repository/repository_connect.go @@ -17,20 +17,17 @@ package repository import ( "github.com/go-openapi/strfmt" - "github.com/kanisterio/safecli" - - "github.com/kanisterio/kanister/pkg/kopia/cli" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/command" - flagcommon "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/common" - flagrepo "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/repository" - flagstorage "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag/storage" + "github.com/kanisterio/kanister/pkg/kopia/cli/args" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/opts" "github.com/kanisterio/kanister/pkg/log" + "github.com/kanisterio/safecli" ) // ConnectArgs defines the arguments for the `kopia repository connect` command. type ConnectArgs struct { - cli.CommonArgs - cli.CacheArgs + args.Common + args.Cache Hostname string // the hostname of the repository Username string // the username of the repository @@ -43,15 +40,20 @@ type ConnectArgs struct { } // Connect creates a new `kopia repository connect ...` command. -func Connect(args ConnectArgs) (safecli.CommandBuilder, error) { - return command.NewKopiaCommandBuilder(args.CommonArgs, - command.Repository, command.Connect, - flagcommon.NoCheckForUpdates, - flagcommon.ReadOnly(args.ReadOnly), - flagcommon.Cache(args.CacheArgs), - flagrepo.Hostname(args.Hostname), - flagrepo.Username(args.Username), - flagstorage.Storage(args.Location, args.RepoPathPrefix, flagstorage.WithLogger(args.Logger)), - flagrepo.PIT(args.PointInTime), +func Connect(args ConnectArgs) (*safecli.Builder, error) { + return internal.NewKopiaCommand( + opts.Common(args.Common), + cmdRepository, subcmdConnect, + opts.CheckForUpdates(false), + optReadOnly(args.ReadOnly), + opts.Cache(args.Cache), + optHostname(args.Hostname), + optUsername(args.Username), + optStorage( + args.Location, + args.RepoPathPrefix, + args.Logger, + ), + optPointInTime(args.PointInTime), ) } diff --git a/pkg/kopia/cli/repository/repository_connect_test.go b/pkg/kopia/cli/repository/repository_connect_test.go new file mode 100644 index 0000000000..a84ce059ec --- /dev/null +++ b/pkg/kopia/cli/repository/repository_connect_test.go @@ -0,0 +1,82 @@ +package repository + +import ( + "testing" + + "github.com/go-openapi/strfmt" + "github.com/kanisterio/safecli" + "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" +) + +func TestRepositoryConnectCommand(t *testing.T) { check.TestingT(t) } + +// Test Repository Connect command +var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ + { + Name: "repository connect with default retention", + Command: func() (*safecli.Builder, error) { + args := ConnectArgs{ + Common: common, + Cache: cache, + Hostname: "test-hostname", + Username: "test-username", + RepoPathPrefix: "test-path/prefix", + Location: locFS, + } + return Connect(args) + }, + ExpectedCLI: []string{"kopia", + "--config-file=path/kopia.config", + "--log-dir=cache/log", + "--log-level=error", + "--password=encr-key", + "repository", + "connect", + "--no-check-for-updates", + "--cache-directory=/tmp/cache.dir", + "--content-cache-size-limit-mb=0", + "--metadata-cache-size-limit-mb=0", + "--override-hostname=test-hostname", + "--override-username=test-username", + "filesystem", + "--path=/mnt/data/test-prefix/test-path/prefix/", + }, + }, + { + Name: "repository connect with PIT and ReadOnly", + Command: func() (*safecli.Builder, error) { + pit, _ := strfmt.ParseDateTime("2021-02-03T01:02:03Z") + args := ConnectArgs{ + Common: common, + Cache: cache, + Hostname: "test-hostname", + Username: "test-username", + RepoPathPrefix: "test-path/prefix", + Location: locFS, + PointInTime: pit, + ReadOnly: true, + } + return Connect(args) + }, + ExpectedCLI: []string{"kopia", + "--config-file=path/kopia.config", + "--log-dir=cache/log", + "--log-level=error", + "--password=encr-key", + "repository", + "connect", + "--no-check-for-updates", + "--readonly", + "--cache-directory=/tmp/cache.dir", + "--content-cache-size-limit-mb=0", + "--metadata-cache-size-limit-mb=0", + "--override-hostname=test-hostname", + "--override-username=test-username", + "filesystem", + "--path=/mnt/data/test-prefix/test-path/prefix/", + "--point-in-time=2021-02-03T01:02:03.000Z", + }, + }, +})) diff --git a/pkg/kopia/cli/repository/repository_create_test.go b/pkg/kopia/cli/repository/repository_create_test.go index c566d30935..7a9001932c 100644 --- a/pkg/kopia/cli/repository/repository_create_test.go +++ b/pkg/kopia/cli/repository/repository_create_test.go @@ -17,7 +17,6 @@ package repository import ( "testing" - "github.com/go-openapi/strfmt" "github.com/kanisterio/safecli" "gopkg.in/check.v1" @@ -223,72 +222,3 @@ var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ }, }, })) - -// Test Repository Connect command -var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ - { - Name: "repository connect with default retention", - CLI: func() (safecli.CommandBuilder, error) { - args := ConnectArgs{ - CommonArgs: test.CommonArgs, - CacheArgs: cacheArgs, - Hostname: "test-hostname", - Username: "test-username", - RepoPathPrefix: "test-path/prefix", - Location: locFS, - } - return Connect(args) - }, - ExpectedCLI: []string{"kopia", - "--config-file=path/kopia.config", - "--log-level=error", - "--log-dir=cache/log", - "--password=encr-key", - "repository", - "connect", - "--no-check-for-updates", - "--cache-directory=/tmp/cache.dir", - "--content-cache-size-limit-mb=0", - "--metadata-cache-size-limit-mb=0", - "--override-hostname=test-hostname", - "--override-username=test-username", - "filesystem", - "--path=/mnt/data/test-prefix/test-path/prefix/", - }, - }, - { - Name: "repository connect with PIT and ReadOnly", - CLI: func() (safecli.CommandBuilder, error) { - pit, _ := strfmt.ParseDateTime("2021-02-03T01:02:03Z") - args := ConnectArgs{ - CommonArgs: test.CommonArgs, - CacheArgs: cacheArgs, - Hostname: "test-hostname", - Username: "test-username", - RepoPathPrefix: "test-path/prefix", - Location: locFS, - PointInTime: pit, - ReadOnly: true, - } - return Connect(args) - }, - ExpectedCLI: []string{"kopia", - "--config-file=path/kopia.config", - "--log-level=error", - "--log-dir=cache/log", - "--password=encr-key", - "repository", - "connect", - "--no-check-for-updates", - "--readonly", - "--cache-directory=/tmp/cache.dir", - "--content-cache-size-limit-mb=0", - "--metadata-cache-size-limit-mb=0", - "--override-hostname=test-hostname", - "--override-username=test-username", - "filesystem", - "--path=/mnt/data/test-prefix/test-path/prefix/", - "--point-in-time=2021-02-03T01:02:03.000Z", - }, - }, -})) From 209e87bdfd8fe2847caadc206bb7b68281ce007b Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Sat, 17 Feb 2024 16:11:22 -0800 Subject: [PATCH 40/73] Add opts tests Signed-off-by: pavel.larkin --- pkg/kopia/cli/repository/opts_test.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pkg/kopia/cli/repository/opts_test.go b/pkg/kopia/cli/repository/opts_test.go index 3dd798291e..40f4cf625e 100644 --- a/pkg/kopia/cli/repository/opts_test.go +++ b/pkg/kopia/cli/repository/opts_test.go @@ -17,6 +17,7 @@ package repository import ( "testing" + "github.com/go-openapi/strfmt" "github.com/kanisterio/safecli/command" "github.com/kanisterio/safecli/test" "gopkg.in/check.v1" @@ -76,4 +77,23 @@ var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTe Argument: optStorage(locFTP, "repoPathPrefix", nil), ExpectedErr: cli.ErrUnsupportedStorage, }, + { + Name: "optReadOnly", + Argument: command.NewArguments( + optReadOnly(true), + optReadOnly(false), // + ), + ExpectedCLI: []string{"cmd", "--readonly"}, + }, + { + Name: "optPointInTime", + Argument: command.NewArguments( + optPointInTime(func() strfmt.DateTime { + t, _ := strfmt.ParseDateTime("2021-02-03T01:02:03.000Z") + return t + }()), + optPointInTime(strfmt.DateTime{}), // no output + ), + ExpectedCLI: []string{"cmd", "--point-in-time=2021-02-03T01:02:03.000Z"}, + }, }}) From f00879c8c279584884ced227389f00b8753851de Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Sat, 17 Feb 2024 16:11:43 -0800 Subject: [PATCH 41/73] Add opts tests Signed-off-by: pavel.larkin --- pkg/kopia/cli/repository/opts_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/kopia/cli/repository/opts_test.go b/pkg/kopia/cli/repository/opts_test.go index 40f4cf625e..af595e996c 100644 --- a/pkg/kopia/cli/repository/opts_test.go +++ b/pkg/kopia/cli/repository/opts_test.go @@ -81,7 +81,7 @@ var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTe Name: "optReadOnly", Argument: command.NewArguments( optReadOnly(true), - optReadOnly(false), // + optReadOnly(false), // no output ), ExpectedCLI: []string{"cmd", "--readonly"}, }, From 2c6cb6e1b3dadf3a80e3355f9900ba3eddb23771 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Mon, 26 Feb 2024 15:38:03 -0800 Subject: [PATCH 42/73] Convert common flags from vars to funcs Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/opts/opts.go | 19 ++++++++++++++----- pkg/kopia/cli/internal/opts/opts_test.go | 15 +++++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/pkg/kopia/cli/internal/opts/opts.go b/pkg/kopia/cli/internal/opts/opts.go index bee0ece9d1..fb53f5d679 100644 --- a/pkg/kopia/cli/internal/opts/opts.go +++ b/pkg/kopia/cli/internal/opts/opts.go @@ -16,11 +16,20 @@ package opts import "github.com/kanisterio/safecli/command" -var ( - All = command.NewOption("--all", true) - Delta = command.NewOption("--delta", true) - ShowIdentical = command.NewOption("--show-identical", true) -) +// All creates a new all option. +func All(enabled bool) command.Applier { + return command.NewOption("--all", enabled) +} + +// Delta creates a new delta option. +func Delta(enabled bool) command.Applier { + return command.NewOption("--delta", enabled) +} + +// ShowIdentical creates a new show identical option. +func ShowIdentical(enabled bool) command.Applier { + return command.NewOption("--show-identical", enabled) +} // ReadOnly creates a new read only option. func ReadOnly(enabled bool) command.Applier { diff --git a/pkg/kopia/cli/internal/opts/opts_test.go b/pkg/kopia/cli/internal/opts/opts_test.go index 8ea81c5e25..f7aa7178ee 100644 --- a/pkg/kopia/cli/internal/opts/opts_test.go +++ b/pkg/kopia/cli/internal/opts/opts_test.go @@ -26,6 +26,21 @@ import ( func TestOptions(t *testing.T) { check.TestingT(t) } var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ + { + Name: "All", + Argument: command.NewArguments(opts.All(true), opts.All(false)), + ExpectedCLI: []string{"cmd", "--all"}, + }, + { + Name: "Delta", + Argument: command.NewArguments(opts.Delta(true), opts.Delta(false)), + ExpectedCLI: []string{"cmd", "--delta"}, + }, + { + Name: "ShowIdentical", + Argument: command.NewArguments(opts.ShowIdentical(true), opts.ShowIdentical(false)), + ExpectedCLI: []string{"cmd", "--show-identical"}, + }, { Name: "Readonly", Argument: command.NewArguments(opts.ReadOnly(true), opts.ReadOnly(false)), From 1a3ee2df4c0422ab5c87e742c5c804d2fae495dd Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 2 Feb 2024 18:08:02 -0800 Subject: [PATCH 43/73] Add safecli dependency --- go.mod | 2 ++ go.sum | 2 ++ 2 files changed, 4 insertions(+) diff --git a/go.mod b/go.mod index 629fe7c11c..b00d7b6b8a 100644 --- a/go.mod +++ b/go.mod @@ -216,6 +216,8 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) +require github.com/kanisterio/safecli v0.0.3 + require ( github.com/Azure/go-autorest/autorest v0.11.27 // indirect github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect diff --git a/go.sum b/go.sum index 8e946fc5ff..69a0e01e4b 100644 --- a/go.sum +++ b/go.sum @@ -359,6 +359,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kanisterio/safecli v0.0.3 h1:ts3oRVSoRexFBv9pOdWYZsN4k068pWx5Cl4zN54mQro= +github.com/kanisterio/safecli v0.0.3/go.mod h1:fK3Mcbeiso+NtkUdhGugK0Vf4S2l8ObvFf564ry1W5A= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734 h1:qulsCaCv+O2y9/sQ9nd5KChnAgFOWakTHQ9ZADjs6DQ= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734/go.mod h1:rdqSnvOJuKCPFW/h2rVLuXOAkRnHHdp9PZcKx4HCoDM= github.com/kastenhq/stow v0.2.6-kasten.1.0.20231101232131-9321daa23aae h1:2cl4yuAJpdmLCx7G8eIsfNlQBLEfw0JDj6mTTyqc5qg= From 386e7e942d477e80d19d6a33081c883c88142b0e Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 2 Feb 2024 18:09:47 -0800 Subject: [PATCH 44/73] add new flag implementations based on the safecli package for the Kopia CLI --- pkg/kopia/cli/errors.go | 25 +++ pkg/kopia/cli/internal/flag/bool_flag.go | 45 ++++++ pkg/kopia/cli/internal/flag/flag.go | 81 ++++++++++ pkg/kopia/cli/internal/flag/flag_test.go | 171 +++++++++++++++++++++ pkg/kopia/cli/internal/flag/string_flag.go | 78 ++++++++++ pkg/kopia/cli/internal/test/flag_suite.go | 112 ++++++++++++++ 6 files changed, 512 insertions(+) create mode 100644 pkg/kopia/cli/errors.go create mode 100644 pkg/kopia/cli/internal/flag/bool_flag.go create mode 100644 pkg/kopia/cli/internal/flag/flag.go create mode 100644 pkg/kopia/cli/internal/flag/flag_test.go create mode 100644 pkg/kopia/cli/internal/flag/string_flag.go create mode 100644 pkg/kopia/cli/internal/test/flag_suite.go diff --git a/pkg/kopia/cli/errors.go b/pkg/kopia/cli/errors.go new file mode 100644 index 0000000000..946dd04e87 --- /dev/null +++ b/pkg/kopia/cli/errors.go @@ -0,0 +1,25 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cli + +import ( + "github.com/pkg/errors" +) + +// flag errors +var ( + // ErrInvalidFlag is returned when the flag name is empty. + ErrInvalidFlag = errors.New("invalid flag") +) diff --git a/pkg/kopia/cli/internal/flag/bool_flag.go b/pkg/kopia/cli/internal/flag/bool_flag.go new file mode 100644 index 0000000000..4eea665cb2 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/bool_flag.go @@ -0,0 +1,45 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flag + +import ( + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/kopia/cli" +) + +// boolFlag defines a boolean flag with a given flag name. +// If enabled is set to true, the flag is applied; otherwise, it is not. +type boolFlag struct { + flag string + enabled bool +} + +// Apply appends the flag to the command if the flag is enabled. +func (f boolFlag) Apply(cli safecli.CommandAppender) error { + if f.enabled { + cli.AppendLoggable(f.flag) + } + return nil +} + +// NewBoolFlag creates a new bool flag with a given flag name. +// If the flag name is empty, cli.ErrInvalidFlag is returned. +func NewBoolFlag(flag string, enabled bool) Applier { + if flag == "" { + return ErrorFlag(cli.ErrInvalidFlag) + } + return boolFlag{flag, enabled} +} diff --git a/pkg/kopia/cli/internal/flag/flag.go b/pkg/kopia/cli/internal/flag/flag.go new file mode 100644 index 0000000000..898ae31e16 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/flag.go @@ -0,0 +1,81 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flag + +import ( + "github.com/kanisterio/safecli" +) + +// Applier applies flags/args to the command. +type Applier interface { + // Apply applies the flags/args to the command. + Apply(cli safecli.CommandAppender) error +} + +// Apply appends multiple flags to the CLI. +// If any of the flags encounter an error during the Apply process, +// the error is returned and no changes are made to the CLI. +// If no error, the flags are appended to the CLI. +func Apply(cli safecli.CommandAppender, flags ...Applier) error { + // create a new sub builder which will be used to apply the flags + // to avoid mutating the CLI if an error is encountered. + sub := safecli.NewBuilder() + for _, flag := range flags { + if flag == nil { // if the flag is nil, skip it + continue + } + if err := flag.Apply(cli); err != nil { + return err + } + } + cli.Append(sub) + return nil +} + +// flags defines a collection of Flags. +type flags []Applier + +// Apply applies the flags to the CLI. +func (flags flags) Apply(cli safecli.CommandAppender) error { + return Apply(cli, flags...) +} + +// NewFlags creates a new collection of flags. +func NewFlags(fs ...Applier) Applier { + return flags(fs) +} + +// simpleFlag is a simple implementation of the Applier interface. +type simpleFlag struct { + err error +} + +// Apply does nothing except return an error if one is set. +func (f simpleFlag) Apply(safecli.CommandAppender) error { + return f.err +} + +// EmptyFlag creates a new flag that does nothing. +// It is useful for creating a no-op flag when a condition is not met +// but Applier interface is required. +func EmptyFlag() Applier { + return simpleFlag{} +} + +// ErrorFlag creates a new flag that returns an error when applied. +// It is useful for creating a flag validation if a condition is not met. +func ErrorFlag(err error) Applier { + return simpleFlag{err} +} diff --git a/pkg/kopia/cli/internal/flag/flag_test.go b/pkg/kopia/cli/internal/flag/flag_test.go new file mode 100644 index 0000000000..c20a46b450 --- /dev/null +++ b/pkg/kopia/cli/internal/flag/flag_test.go @@ -0,0 +1,171 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flag_test + +import ( + "errors" + "testing" + + "gopkg.in/check.v1" + + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" +) + +var ( + ErrFlag = errors.New("flag error") +) + +// MockFlagApplier is a mock implementation of the FlagApplier interface. +type MockFlagApplier struct { + flagName string + applyErr error +} + +func (m *MockFlagApplier) Apply(cli safecli.CommandAppender) error { + cli.AppendLoggable(m.flagName) + return m.applyErr +} + +func TestApply(t *testing.T) { check.TestingT(t) } + +var _ = check.Suite(&test.FlagSuite{Cmd: "cmd", Tests: []test.FlagTest{ + { + Name: "Apply with no flags should generate only the command", + ExpectedCLI: []string{"cmd"}, + }, + { + Name: "Apply with nil flags should generate only the command", + Flag: flag.NewFlags(nil, nil), + ExpectedCLI: []string{"cmd"}, + }, + { + Name: "Apply with flags should generate the command and flags", + Flag: flag.NewFlags( + &MockFlagApplier{flagName: "--flag1", applyErr: nil}, + &MockFlagApplier{flagName: "--flag2", applyErr: nil}, + ), + ExpectedCLI: []string{"cmd", "--flag1", "--flag2"}, + }, + { + Name: "Apply with one error flag should not modify the command and return the error", + Flag: flag.NewFlags( + &MockFlagApplier{flagName: "flag1", applyErr: nil}, + &MockFlagApplier{flagName: "flag2", applyErr: ErrFlag}, + ), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: ErrFlag, + }, + { + Name: "NewBoolFlag", + Flag: flag.NewFlags( + flag.NewBoolFlag("--flag1", true), + flag.NewBoolFlag("--flag2", false), + ), + ExpectedCLI: []string{"cmd", "--flag1"}, + }, + { + Name: "NewBoolFlag with empty flag name should return an error", + Flag: flag.NewFlags( + flag.NewBoolFlag("", true), + ), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: cli.ErrInvalidFlag, + }, + { + Name: "NewStringFlag", + Flag: flag.NewFlags( + flag.NewStringFlag("--flag1", "value1"), + flag.NewStringFlag("--flag2", ""), + ), + ExpectedCLI: []string{"cmd", "--flag1=value1"}, + }, + { + Name: "NewStringFlag with all empty values should return an error", + Flag: flag.NewFlags( + flag.NewStringFlag("--flag1", "value1"), + flag.NewStringFlag("", ""), + ), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: cli.ErrInvalidFlag, + }, + { + Name: "NewRedactedStringFlag", + Flag: flag.NewFlags( + flag.NewRedactedStringFlag("--flag1", "value1"), + flag.NewRedactedStringFlag("--flag2", ""), + flag.NewRedactedStringFlag("", "value3"), + ), + ExpectedCLI: []string{"cmd", "--flag1=value1", "value3"}, + ExpectedLog: "cmd --flag1=<****> <****>", + }, + { + Name: "NewRedactedStringFlag with all empty values should return an error", + Flag: flag.NewFlags( + flag.NewRedactedStringFlag("--flag1", "value1"), + flag.NewRedactedStringFlag("", ""), + ), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: cli.ErrInvalidFlag, + }, + { + Name: "NewStringValue", + Flag: flag.NewFlags( + flag.NewStringArgument("value1"), + ), + ExpectedCLI: []string{"cmd", "value1"}, + }, + { + Name: "NewStringValue with empty value should return an error", + Flag: flag.NewFlags( + flag.NewStringArgument(""), + ), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: cli.ErrInvalidFlag, + }, + { + Name: "NewFlags should generate multiple flags", + Flag: flag.NewFlags( + flag.NewStringFlag("--flag1", "value1"), + flag.NewRedactedStringFlag("--flag2", "value2"), + flag.NewStringArgument("value3"), + ), + ExpectedCLI: []string{"cmd", "--flag1=value1", "--flag2=value2", "value3"}, + ExpectedLog: "cmd --flag1=value1 --flag2=<****> value3", + }, + { + Name: "NewFlags should generate no flags if one of them returns an error", + Flag: flag.NewFlags( + flag.NewStringFlag("--flag1", "value1"), + &MockFlagApplier{flagName: "flag2", applyErr: ErrFlag}, + ), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: ErrFlag, + }, + { + Name: "EmptyFlag should not generate any flags", + Flag: flag.EmptyFlag(), + ExpectedCLI: []string{"cmd"}, + }, + { + Name: "ErrorFlag should return an error", + Flag: flag.ErrorFlag(ErrFlag), + ExpectedCLI: []string{"cmd"}, + ExpectedErr: ErrFlag, + }, +}}) diff --git a/pkg/kopia/cli/internal/flag/string_flag.go b/pkg/kopia/cli/internal/flag/string_flag.go new file mode 100644 index 0000000000..a94c29722c --- /dev/null +++ b/pkg/kopia/cli/internal/flag/string_flag.go @@ -0,0 +1,78 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package flag + +import ( + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/kopia/cli" +) + +// stringFlag defines a string flag with a given flag name and value. +// If the value is empty, the flag is not applied. +type stringFlag struct { + flag string // flag name + value string // flag value + redacted bool // output the value as redacted +} + +// appenderFunc is a function that appends strings to a command. +type appenderFunc func(...string) *safecli.Builder + +// Apply appends the flag to the command if the value is not empty. +// If the value is redacted, it is appended as redacted. +func (f stringFlag) Apply(cli safecli.CommandAppender) error { + if f.value == "" { + return nil + } + appendValue, appendFlagValue := f.selectAppenderFuncs(cli) + if f.flag == "" { + appendValue(f.value) + } else { + appendFlagValue(f.flag, f.value) + } + return nil +} + +// selectAppenderFuncs returns the appropriate appender functions based on the redacted flag. +func (f stringFlag) selectAppenderFuncs(cli safecli.CommandAppender) (appenderFunc, appenderFunc) { + if f.redacted { + return cli.AppendRedacted, cli.AppendRedactedKV + } + return cli.AppendLoggable, cli.AppendLoggableKV +} + +// newStringFlag creates a new string flag with a given flag name and value. +func newStringFlag(flag, val string, redacted bool) Applier { + if flag == "" && val == "" { + return ErrorFlag(cli.ErrInvalidFlag) + } + return stringFlag{flag: flag, value: val, redacted: redacted} +} + +// NewStringFlag creates a new string flag with a given flag name and value. +func NewStringFlag(flag, val string) Applier { + return newStringFlag(flag, val, false) +} + +// NewRedactedStringFlag creates a new string flag with a given flag name and value. +func NewRedactedStringFlag(flag, val string) Applier { + return newStringFlag(flag, val, true) +} + +// NewStringArgument creates a new string argument with a given value. +func NewStringArgument(val string) Applier { + return newStringFlag("", val, false) +} diff --git a/pkg/kopia/cli/internal/test/flag_suite.go b/pkg/kopia/cli/internal/test/flag_suite.go new file mode 100644 index 0000000000..7c41a1ea3a --- /dev/null +++ b/pkg/kopia/cli/internal/test/flag_suite.go @@ -0,0 +1,112 @@ +package test + +import ( + "strings" + + "gopkg.in/check.v1" + + "github.com/pkg/errors" + + "github.com/kanisterio/safecli" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" +) + +// FlagTest defines a single test for a flag. +type FlagTest struct { + // Name of the test. (required) + Name string + + // Flag to test. (required) + Flag flag.Applier + + // Expected CLI arguments. (optional) + ExpectedCLI []string + + // Expected log output. (optional) + // if empty, it will be set to ExpectedCLI joined with space. + // if empty and ExpectedCLI is empty, it will be ignored. + ExpectedLog string + + // Expected error. (optional) + // If nil, no error is expected and + // ExpectedCLI and ExpectedLog are checked. + ExpectedErr error +} + +// CheckCommentString implements check.CommentInterface +func (t *FlagTest) CheckCommentString() string { + return t.Name +} + +// setDefaultExpectedLog sets the default value for ExpectedLog based on ExpectedCLI. +func (t *FlagTest) setDefaultExpectedLog() { + if len(t.ExpectedLog) == 0 && len(t.ExpectedCLI) > 0 { + t.ExpectedLog = strings.Join(t.ExpectedCLI, " ") + } +} + +// assertError checks the error against ExpectedErr. +func (t *FlagTest) assertError(c *check.C, err error) { + if actualErr := errors.Cause(err); actualErr != nil { + c.Assert(actualErr, check.Equals, t.ExpectedErr, t) + } else { + c.Assert(err, check.Equals, t.ExpectedErr, t) + } +} + +// assertNoError makes sure there is no error. +func (t *FlagTest) assertNoError(c *check.C, err error) { + c.Assert(err, check.IsNil, t) +} + +// assertCLI asserts the builder's CLI output against ExpectedCLI. +func (t *FlagTest) assertCLI(c *check.C, b *safecli.Builder) { + c.Check(b.Build(), check.DeepEquals, t.ExpectedCLI, t) +} + +// assertLog asserts the builder's log output against ExpectedLog. +func (t *FlagTest) assertLog(c *check.C, b *safecli.Builder) { + t.setDefaultExpectedLog() + c.Check(b.String(), check.Equals, t.ExpectedLog, t) +} + +// Test runs the flag test. +func (ft *FlagTest) Test(c *check.C, b *safecli.Builder) { + err := flag.Apply(b, ft.Flag) + if ft.ExpectedErr != nil { + ft.assertError(c, err) + } else { + ft.assertNoError(c, err) + ft.assertCLI(c, b) + ft.assertLog(c, b) + } +} + +// FlagSuite defines a test suite for flags. +type FlagSuite struct { + Cmd string // Cmd appends to the safecli.Builder before test if not empty. + Tests []FlagTest // Tests to run. +} + +// TestFlags runs all tests in the flag suite. +func (s *FlagSuite) TestFlags(c *check.C) { + for _, test := range s.Tests { + b := newBuilder(s.Cmd) + test.Test(c, b) + } +} + +// NewFlagSuite creates a new FlagSuite. +func NewFlagSuite(tests []FlagTest) *FlagSuite { + return &FlagSuite{Tests: tests} +} + +// newBuilder creates a new safecli.Builder with the given command. +func newBuilder(cmd string) *safecli.Builder { + builder := safecli.NewBuilder() + if cmd != "" { + builder.AppendLoggable(cmd) + } + return builder +} From b815f633208dd54aab0963b69d2298263190d4a6 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Sat, 3 Feb 2024 16:03:14 -0800 Subject: [PATCH 45/73] apply go fmt Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/flag/flag.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/kopia/cli/internal/flag/flag.go b/pkg/kopia/cli/internal/flag/flag.go index 898ae31e16..66148034a1 100644 --- a/pkg/kopia/cli/internal/flag/flag.go +++ b/pkg/kopia/cli/internal/flag/flag.go @@ -68,7 +68,7 @@ func (f simpleFlag) Apply(safecli.CommandAppender) error { } // EmptyFlag creates a new flag that does nothing. -// It is useful for creating a no-op flag when a condition is not met +// It is useful for creating a no-op flag when a condition is not met // but Applier interface is required. func EmptyFlag() Applier { return simpleFlag{} From bcafabb2ba3c8c1cc71cb31c8417728cdd3e4610 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Mon, 12 Feb 2024 12:11:54 -0800 Subject: [PATCH 46/73] Fix Apply and test.Suit Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/flag/flag.go | 2 +- pkg/kopia/cli/internal/test/flag_suite.go | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/kopia/cli/internal/flag/flag.go b/pkg/kopia/cli/internal/flag/flag.go index 66148034a1..39a357be53 100644 --- a/pkg/kopia/cli/internal/flag/flag.go +++ b/pkg/kopia/cli/internal/flag/flag.go @@ -36,7 +36,7 @@ func Apply(cli safecli.CommandAppender, flags ...Applier) error { if flag == nil { // if the flag is nil, skip it continue } - if err := flag.Apply(cli); err != nil { + if err := flag.Apply(sub); err != nil { return err } } diff --git a/pkg/kopia/cli/internal/test/flag_suite.go b/pkg/kopia/cli/internal/test/flag_suite.go index 7c41a1ea3a..7fe60d909c 100644 --- a/pkg/kopia/cli/internal/test/flag_suite.go +++ b/pkg/kopia/cli/internal/test/flag_suite.go @@ -62,7 +62,9 @@ func (t *FlagTest) assertNoError(c *check.C, err error) { // assertCLI asserts the builder's CLI output against ExpectedCLI. func (t *FlagTest) assertCLI(c *check.C, b *safecli.Builder) { - c.Check(b.Build(), check.DeepEquals, t.ExpectedCLI, t) + if t.ExpectedCLI != nil { + c.Check(b.Build(), check.DeepEquals, t.ExpectedCLI, t) + } } // assertLog asserts the builder's log output against ExpectedLog. @@ -74,11 +76,11 @@ func (t *FlagTest) assertLog(c *check.C, b *safecli.Builder) { // Test runs the flag test. func (ft *FlagTest) Test(c *check.C, b *safecli.Builder) { err := flag.Apply(b, ft.Flag) + ft.assertCLI(c, b) if ft.ExpectedErr != nil { ft.assertError(c, err) } else { ft.assertNoError(c, err) - ft.assertCLI(c, b) ft.assertLog(c, b) } } From 6d61bb4c3e5a533f98bcea127655c7b5eba6e677 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 16 Feb 2024 16:51:58 -0800 Subject: [PATCH 47/73] pkg/kopia/cli/internal/flag is implemented in the safecli@v0.0.4 now Signed-off-by: pavel.larkin --- go.mod | 2 +- go.sum | 2 + pkg/kopia/cli/errors.go | 25 --- pkg/kopia/cli/internal/flag/bool_flag.go | 45 ------ pkg/kopia/cli/internal/flag/flag.go | 81 ---------- pkg/kopia/cli/internal/flag/flag_test.go | 171 --------------------- pkg/kopia/cli/internal/flag/string_flag.go | 78 ---------- pkg/kopia/cli/internal/test/flag_suite.go | 114 -------------- 8 files changed, 3 insertions(+), 515 deletions(-) delete mode 100644 pkg/kopia/cli/errors.go delete mode 100644 pkg/kopia/cli/internal/flag/bool_flag.go delete mode 100644 pkg/kopia/cli/internal/flag/flag.go delete mode 100644 pkg/kopia/cli/internal/flag/flag_test.go delete mode 100644 pkg/kopia/cli/internal/flag/string_flag.go delete mode 100644 pkg/kopia/cli/internal/test/flag_suite.go diff --git a/go.mod b/go.mod index b00d7b6b8a..fb6a6abd8d 100644 --- a/go.mod +++ b/go.mod @@ -216,7 +216,7 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) -require github.com/kanisterio/safecli v0.0.3 +require github.com/kanisterio/safecli v0.0.4 require ( github.com/Azure/go-autorest/autorest v0.11.27 // indirect diff --git a/go.sum b/go.sum index 69a0e01e4b..5ebbafea17 100644 --- a/go.sum +++ b/go.sum @@ -361,6 +361,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kanisterio/safecli v0.0.3 h1:ts3oRVSoRexFBv9pOdWYZsN4k068pWx5Cl4zN54mQro= github.com/kanisterio/safecli v0.0.3/go.mod h1:fK3Mcbeiso+NtkUdhGugK0Vf4S2l8ObvFf564ry1W5A= +github.com/kanisterio/safecli v0.0.4 h1:8pO7Zhm9YeW47XQZNRt/AaXj7kSPeIbaV7aRDRtHs4k= +github.com/kanisterio/safecli v0.0.4/go.mod h1:KBraqj8mdv2cwAr9wecknGUb8jztTzUik0r7uE6yRA8= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734 h1:qulsCaCv+O2y9/sQ9nd5KChnAgFOWakTHQ9ZADjs6DQ= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734/go.mod h1:rdqSnvOJuKCPFW/h2rVLuXOAkRnHHdp9PZcKx4HCoDM= github.com/kastenhq/stow v0.2.6-kasten.1.0.20231101232131-9321daa23aae h1:2cl4yuAJpdmLCx7G8eIsfNlQBLEfw0JDj6mTTyqc5qg= diff --git a/pkg/kopia/cli/errors.go b/pkg/kopia/cli/errors.go deleted file mode 100644 index 946dd04e87..0000000000 --- a/pkg/kopia/cli/errors.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cli - -import ( - "github.com/pkg/errors" -) - -// flag errors -var ( - // ErrInvalidFlag is returned when the flag name is empty. - ErrInvalidFlag = errors.New("invalid flag") -) diff --git a/pkg/kopia/cli/internal/flag/bool_flag.go b/pkg/kopia/cli/internal/flag/bool_flag.go deleted file mode 100644 index 4eea665cb2..0000000000 --- a/pkg/kopia/cli/internal/flag/bool_flag.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package flag - -import ( - "github.com/kanisterio/safecli" - - "github.com/kanisterio/kanister/pkg/kopia/cli" -) - -// boolFlag defines a boolean flag with a given flag name. -// If enabled is set to true, the flag is applied; otherwise, it is not. -type boolFlag struct { - flag string - enabled bool -} - -// Apply appends the flag to the command if the flag is enabled. -func (f boolFlag) Apply(cli safecli.CommandAppender) error { - if f.enabled { - cli.AppendLoggable(f.flag) - } - return nil -} - -// NewBoolFlag creates a new bool flag with a given flag name. -// If the flag name is empty, cli.ErrInvalidFlag is returned. -func NewBoolFlag(flag string, enabled bool) Applier { - if flag == "" { - return ErrorFlag(cli.ErrInvalidFlag) - } - return boolFlag{flag, enabled} -} diff --git a/pkg/kopia/cli/internal/flag/flag.go b/pkg/kopia/cli/internal/flag/flag.go deleted file mode 100644 index 39a357be53..0000000000 --- a/pkg/kopia/cli/internal/flag/flag.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package flag - -import ( - "github.com/kanisterio/safecli" -) - -// Applier applies flags/args to the command. -type Applier interface { - // Apply applies the flags/args to the command. - Apply(cli safecli.CommandAppender) error -} - -// Apply appends multiple flags to the CLI. -// If any of the flags encounter an error during the Apply process, -// the error is returned and no changes are made to the CLI. -// If no error, the flags are appended to the CLI. -func Apply(cli safecli.CommandAppender, flags ...Applier) error { - // create a new sub builder which will be used to apply the flags - // to avoid mutating the CLI if an error is encountered. - sub := safecli.NewBuilder() - for _, flag := range flags { - if flag == nil { // if the flag is nil, skip it - continue - } - if err := flag.Apply(sub); err != nil { - return err - } - } - cli.Append(sub) - return nil -} - -// flags defines a collection of Flags. -type flags []Applier - -// Apply applies the flags to the CLI. -func (flags flags) Apply(cli safecli.CommandAppender) error { - return Apply(cli, flags...) -} - -// NewFlags creates a new collection of flags. -func NewFlags(fs ...Applier) Applier { - return flags(fs) -} - -// simpleFlag is a simple implementation of the Applier interface. -type simpleFlag struct { - err error -} - -// Apply does nothing except return an error if one is set. -func (f simpleFlag) Apply(safecli.CommandAppender) error { - return f.err -} - -// EmptyFlag creates a new flag that does nothing. -// It is useful for creating a no-op flag when a condition is not met -// but Applier interface is required. -func EmptyFlag() Applier { - return simpleFlag{} -} - -// ErrorFlag creates a new flag that returns an error when applied. -// It is useful for creating a flag validation if a condition is not met. -func ErrorFlag(err error) Applier { - return simpleFlag{err} -} diff --git a/pkg/kopia/cli/internal/flag/flag_test.go b/pkg/kopia/cli/internal/flag/flag_test.go deleted file mode 100644 index c20a46b450..0000000000 --- a/pkg/kopia/cli/internal/flag/flag_test.go +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package flag_test - -import ( - "errors" - "testing" - - "gopkg.in/check.v1" - - "github.com/kanisterio/safecli" - - "github.com/kanisterio/kanister/pkg/kopia/cli" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" -) - -var ( - ErrFlag = errors.New("flag error") -) - -// MockFlagApplier is a mock implementation of the FlagApplier interface. -type MockFlagApplier struct { - flagName string - applyErr error -} - -func (m *MockFlagApplier) Apply(cli safecli.CommandAppender) error { - cli.AppendLoggable(m.flagName) - return m.applyErr -} - -func TestApply(t *testing.T) { check.TestingT(t) } - -var _ = check.Suite(&test.FlagSuite{Cmd: "cmd", Tests: []test.FlagTest{ - { - Name: "Apply with no flags should generate only the command", - ExpectedCLI: []string{"cmd"}, - }, - { - Name: "Apply with nil flags should generate only the command", - Flag: flag.NewFlags(nil, nil), - ExpectedCLI: []string{"cmd"}, - }, - { - Name: "Apply with flags should generate the command and flags", - Flag: flag.NewFlags( - &MockFlagApplier{flagName: "--flag1", applyErr: nil}, - &MockFlagApplier{flagName: "--flag2", applyErr: nil}, - ), - ExpectedCLI: []string{"cmd", "--flag1", "--flag2"}, - }, - { - Name: "Apply with one error flag should not modify the command and return the error", - Flag: flag.NewFlags( - &MockFlagApplier{flagName: "flag1", applyErr: nil}, - &MockFlagApplier{flagName: "flag2", applyErr: ErrFlag}, - ), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: ErrFlag, - }, - { - Name: "NewBoolFlag", - Flag: flag.NewFlags( - flag.NewBoolFlag("--flag1", true), - flag.NewBoolFlag("--flag2", false), - ), - ExpectedCLI: []string{"cmd", "--flag1"}, - }, - { - Name: "NewBoolFlag with empty flag name should return an error", - Flag: flag.NewFlags( - flag.NewBoolFlag("", true), - ), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: cli.ErrInvalidFlag, - }, - { - Name: "NewStringFlag", - Flag: flag.NewFlags( - flag.NewStringFlag("--flag1", "value1"), - flag.NewStringFlag("--flag2", ""), - ), - ExpectedCLI: []string{"cmd", "--flag1=value1"}, - }, - { - Name: "NewStringFlag with all empty values should return an error", - Flag: flag.NewFlags( - flag.NewStringFlag("--flag1", "value1"), - flag.NewStringFlag("", ""), - ), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: cli.ErrInvalidFlag, - }, - { - Name: "NewRedactedStringFlag", - Flag: flag.NewFlags( - flag.NewRedactedStringFlag("--flag1", "value1"), - flag.NewRedactedStringFlag("--flag2", ""), - flag.NewRedactedStringFlag("", "value3"), - ), - ExpectedCLI: []string{"cmd", "--flag1=value1", "value3"}, - ExpectedLog: "cmd --flag1=<****> <****>", - }, - { - Name: "NewRedactedStringFlag with all empty values should return an error", - Flag: flag.NewFlags( - flag.NewRedactedStringFlag("--flag1", "value1"), - flag.NewRedactedStringFlag("", ""), - ), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: cli.ErrInvalidFlag, - }, - { - Name: "NewStringValue", - Flag: flag.NewFlags( - flag.NewStringArgument("value1"), - ), - ExpectedCLI: []string{"cmd", "value1"}, - }, - { - Name: "NewStringValue with empty value should return an error", - Flag: flag.NewFlags( - flag.NewStringArgument(""), - ), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: cli.ErrInvalidFlag, - }, - { - Name: "NewFlags should generate multiple flags", - Flag: flag.NewFlags( - flag.NewStringFlag("--flag1", "value1"), - flag.NewRedactedStringFlag("--flag2", "value2"), - flag.NewStringArgument("value3"), - ), - ExpectedCLI: []string{"cmd", "--flag1=value1", "--flag2=value2", "value3"}, - ExpectedLog: "cmd --flag1=value1 --flag2=<****> value3", - }, - { - Name: "NewFlags should generate no flags if one of them returns an error", - Flag: flag.NewFlags( - flag.NewStringFlag("--flag1", "value1"), - &MockFlagApplier{flagName: "flag2", applyErr: ErrFlag}, - ), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: ErrFlag, - }, - { - Name: "EmptyFlag should not generate any flags", - Flag: flag.EmptyFlag(), - ExpectedCLI: []string{"cmd"}, - }, - { - Name: "ErrorFlag should return an error", - Flag: flag.ErrorFlag(ErrFlag), - ExpectedCLI: []string{"cmd"}, - ExpectedErr: ErrFlag, - }, -}}) diff --git a/pkg/kopia/cli/internal/flag/string_flag.go b/pkg/kopia/cli/internal/flag/string_flag.go deleted file mode 100644 index a94c29722c..0000000000 --- a/pkg/kopia/cli/internal/flag/string_flag.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2024 The Kanister Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package flag - -import ( - "github.com/kanisterio/safecli" - - "github.com/kanisterio/kanister/pkg/kopia/cli" -) - -// stringFlag defines a string flag with a given flag name and value. -// If the value is empty, the flag is not applied. -type stringFlag struct { - flag string // flag name - value string // flag value - redacted bool // output the value as redacted -} - -// appenderFunc is a function that appends strings to a command. -type appenderFunc func(...string) *safecli.Builder - -// Apply appends the flag to the command if the value is not empty. -// If the value is redacted, it is appended as redacted. -func (f stringFlag) Apply(cli safecli.CommandAppender) error { - if f.value == "" { - return nil - } - appendValue, appendFlagValue := f.selectAppenderFuncs(cli) - if f.flag == "" { - appendValue(f.value) - } else { - appendFlagValue(f.flag, f.value) - } - return nil -} - -// selectAppenderFuncs returns the appropriate appender functions based on the redacted flag. -func (f stringFlag) selectAppenderFuncs(cli safecli.CommandAppender) (appenderFunc, appenderFunc) { - if f.redacted { - return cli.AppendRedacted, cli.AppendRedactedKV - } - return cli.AppendLoggable, cli.AppendLoggableKV -} - -// newStringFlag creates a new string flag with a given flag name and value. -func newStringFlag(flag, val string, redacted bool) Applier { - if flag == "" && val == "" { - return ErrorFlag(cli.ErrInvalidFlag) - } - return stringFlag{flag: flag, value: val, redacted: redacted} -} - -// NewStringFlag creates a new string flag with a given flag name and value. -func NewStringFlag(flag, val string) Applier { - return newStringFlag(flag, val, false) -} - -// NewRedactedStringFlag creates a new string flag with a given flag name and value. -func NewRedactedStringFlag(flag, val string) Applier { - return newStringFlag(flag, val, true) -} - -// NewStringArgument creates a new string argument with a given value. -func NewStringArgument(val string) Applier { - return newStringFlag("", val, false) -} diff --git a/pkg/kopia/cli/internal/test/flag_suite.go b/pkg/kopia/cli/internal/test/flag_suite.go deleted file mode 100644 index 7fe60d909c..0000000000 --- a/pkg/kopia/cli/internal/test/flag_suite.go +++ /dev/null @@ -1,114 +0,0 @@ -package test - -import ( - "strings" - - "gopkg.in/check.v1" - - "github.com/pkg/errors" - - "github.com/kanisterio/safecli" - - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/flag" -) - -// FlagTest defines a single test for a flag. -type FlagTest struct { - // Name of the test. (required) - Name string - - // Flag to test. (required) - Flag flag.Applier - - // Expected CLI arguments. (optional) - ExpectedCLI []string - - // Expected log output. (optional) - // if empty, it will be set to ExpectedCLI joined with space. - // if empty and ExpectedCLI is empty, it will be ignored. - ExpectedLog string - - // Expected error. (optional) - // If nil, no error is expected and - // ExpectedCLI and ExpectedLog are checked. - ExpectedErr error -} - -// CheckCommentString implements check.CommentInterface -func (t *FlagTest) CheckCommentString() string { - return t.Name -} - -// setDefaultExpectedLog sets the default value for ExpectedLog based on ExpectedCLI. -func (t *FlagTest) setDefaultExpectedLog() { - if len(t.ExpectedLog) == 0 && len(t.ExpectedCLI) > 0 { - t.ExpectedLog = strings.Join(t.ExpectedCLI, " ") - } -} - -// assertError checks the error against ExpectedErr. -func (t *FlagTest) assertError(c *check.C, err error) { - if actualErr := errors.Cause(err); actualErr != nil { - c.Assert(actualErr, check.Equals, t.ExpectedErr, t) - } else { - c.Assert(err, check.Equals, t.ExpectedErr, t) - } -} - -// assertNoError makes sure there is no error. -func (t *FlagTest) assertNoError(c *check.C, err error) { - c.Assert(err, check.IsNil, t) -} - -// assertCLI asserts the builder's CLI output against ExpectedCLI. -func (t *FlagTest) assertCLI(c *check.C, b *safecli.Builder) { - if t.ExpectedCLI != nil { - c.Check(b.Build(), check.DeepEquals, t.ExpectedCLI, t) - } -} - -// assertLog asserts the builder's log output against ExpectedLog. -func (t *FlagTest) assertLog(c *check.C, b *safecli.Builder) { - t.setDefaultExpectedLog() - c.Check(b.String(), check.Equals, t.ExpectedLog, t) -} - -// Test runs the flag test. -func (ft *FlagTest) Test(c *check.C, b *safecli.Builder) { - err := flag.Apply(b, ft.Flag) - ft.assertCLI(c, b) - if ft.ExpectedErr != nil { - ft.assertError(c, err) - } else { - ft.assertNoError(c, err) - ft.assertLog(c, b) - } -} - -// FlagSuite defines a test suite for flags. -type FlagSuite struct { - Cmd string // Cmd appends to the safecli.Builder before test if not empty. - Tests []FlagTest // Tests to run. -} - -// TestFlags runs all tests in the flag suite. -func (s *FlagSuite) TestFlags(c *check.C) { - for _, test := range s.Tests { - b := newBuilder(s.Cmd) - test.Test(c, b) - } -} - -// NewFlagSuite creates a new FlagSuite. -func NewFlagSuite(tests []FlagTest) *FlagSuite { - return &FlagSuite{Tests: tests} -} - -// newBuilder creates a new safecli.Builder with the given command. -func newBuilder(cmd string) *safecli.Builder { - builder := safecli.NewBuilder() - if cmd != "" { - builder.AppendLoggable(cmd) - } - return builder -} From e6ddb8f6bef447dfed85029a4b6f4632c3d874d5 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 16 Feb 2024 20:40:54 -0800 Subject: [PATCH 48/73] Add pkg/kopia/cli package Signed-off-by: pavel.larkin --- pkg/kopia/cli/doc.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 pkg/kopia/cli/doc.go diff --git a/pkg/kopia/cli/doc.go b/pkg/kopia/cli/doc.go new file mode 100644 index 0000000000..6f0681452f --- /dev/null +++ b/pkg/kopia/cli/doc.go @@ -0,0 +1,21 @@ +package cli + +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import ( + _ "github.com/kanisterio/safecli" +) + +// This package contains the implementation of the Kopia CLI using github.com/kanisterio/safecli. From 4e7ffd787fc799d5869d2cbd083bd2b8f591ee52 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 16 Feb 2024 20:41:22 -0800 Subject: [PATCH 49/73] go mod tidy Signed-off-by: pavel.larkin --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 5ebbafea17..67c3138454 100644 --- a/go.sum +++ b/go.sum @@ -359,8 +359,6 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/kanisterio/safecli v0.0.3 h1:ts3oRVSoRexFBv9pOdWYZsN4k068pWx5Cl4zN54mQro= -github.com/kanisterio/safecli v0.0.3/go.mod h1:fK3Mcbeiso+NtkUdhGugK0Vf4S2l8ObvFf564ry1W5A= github.com/kanisterio/safecli v0.0.4 h1:8pO7Zhm9YeW47XQZNRt/AaXj7kSPeIbaV7aRDRtHs4k= github.com/kanisterio/safecli v0.0.4/go.mod h1:KBraqj8mdv2cwAr9wecknGUb8jztTzUik0r7uE6yRA8= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734 h1:qulsCaCv+O2y9/sQ9nd5KChnAgFOWakTHQ9ZADjs6DQ= From f16aea76d2e21eab94df31c1a689eeba7d57e375 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Tue, 27 Feb 2024 13:33:47 -0800 Subject: [PATCH 50/73] Update safecli to v0.0.5 Signed-off-by: pavel.larkin --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fb6a6abd8d..95b851aa8f 100644 --- a/go.mod +++ b/go.mod @@ -216,7 +216,7 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) -require github.com/kanisterio/safecli v0.0.4 +require github.com/kanisterio/safecli v0.0.5 require ( github.com/Azure/go-autorest/autorest v0.11.27 // indirect diff --git a/go.sum b/go.sum index 67c3138454..26b154c2a6 100644 --- a/go.sum +++ b/go.sum @@ -359,8 +359,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/kanisterio/safecli v0.0.4 h1:8pO7Zhm9YeW47XQZNRt/AaXj7kSPeIbaV7aRDRtHs4k= -github.com/kanisterio/safecli v0.0.4/go.mod h1:KBraqj8mdv2cwAr9wecknGUb8jztTzUik0r7uE6yRA8= +github.com/kanisterio/safecli v0.0.5 h1:1B9JkmmE4YYCIj4eYMVUXJXQpbpQ+GSOrGoqacfi58Q= +github.com/kanisterio/safecli v0.0.5/go.mod h1:KBraqj8mdv2cwAr9wecknGUb8jztTzUik0r7uE6yRA8= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734 h1:qulsCaCv+O2y9/sQ9nd5KChnAgFOWakTHQ9ZADjs6DQ= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734/go.mod h1:rdqSnvOJuKCPFW/h2rVLuXOAkRnHHdp9PZcKx4HCoDM= github.com/kastenhq/stow v0.2.6-kasten.1.0.20231101232131-9321daa23aae h1:2cl4yuAJpdmLCx7G8eIsfNlQBLEfw0JDj6mTTyqc5qg= From dcd6425038f059ea148871fcd8ab1e44b754d417 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Tue, 27 Feb 2024 16:25:09 -0800 Subject: [PATCH 51/73] Update safecli to v0.0.6 Signed-off-by: pavel.larkin --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 95b851aa8f..f655f747a9 100644 --- a/go.mod +++ b/go.mod @@ -216,7 +216,7 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) -require github.com/kanisterio/safecli v0.0.5 +require github.com/kanisterio/safecli v0.0.6 require ( github.com/Azure/go-autorest/autorest v0.11.27 // indirect diff --git a/go.sum b/go.sum index 26b154c2a6..0d88ed3030 100644 --- a/go.sum +++ b/go.sum @@ -359,8 +359,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/kanisterio/safecli v0.0.5 h1:1B9JkmmE4YYCIj4eYMVUXJXQpbpQ+GSOrGoqacfi58Q= -github.com/kanisterio/safecli v0.0.5/go.mod h1:KBraqj8mdv2cwAr9wecknGUb8jztTzUik0r7uE6yRA8= +github.com/kanisterio/safecli v0.0.6 h1:Mq99jK7A/SBiHKZalUIlsk4qaSVZJNbBWb8rjaJv/Jk= +github.com/kanisterio/safecli v0.0.6/go.mod h1:KBraqj8mdv2cwAr9wecknGUb8jztTzUik0r7uE6yRA8= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734 h1:qulsCaCv+O2y9/sQ9nd5KChnAgFOWakTHQ9ZADjs6DQ= github.com/kastenhq/check v0.0.0-20180626002341-0264cfcea734/go.mod h1:rdqSnvOJuKCPFW/h2rVLuXOAkRnHHdp9PZcKx4HCoDM= github.com/kastenhq/stow v0.2.6-kasten.1.0.20231101232131-9321daa23aae h1:2cl4yuAJpdmLCx7G8eIsfNlQBLEfw0JDj6mTTyqc5qg= From b093421406784745d822f1fc8c92402803230d25 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Wed, 28 Feb 2024 14:58:18 -0800 Subject: [PATCH 52/73] Fix tests Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/opts/common_opts.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/kopia/cli/internal/opts/common_opts.go b/pkg/kopia/cli/internal/opts/common_opts.go index 4eb8b2b101..641de33528 100644 --- a/pkg/kopia/cli/internal/opts/common_opts.go +++ b/pkg/kopia/cli/internal/opts/common_opts.go @@ -24,7 +24,11 @@ const ( ) // LogDirectory creates a new log directory option with a given directory. +// if the directory is empty, the log directory option is not set. func LogDirectory(dir string) command.Applier { + if dir == "" { + return command.NewNoopArgument() + } return command.NewOptionWithArgument("--log-dir", dir) } @@ -38,12 +42,20 @@ func LogLevel(level string) command.Applier { } // ConfigFilePath creates a new config file path option with a given path. +// If the path is empty, the config file path option is not set. func ConfigFilePath(path string) command.Applier { + if path == "" { + return command.NewNoopArgument() + } return command.NewOptionWithArgument("--config-file", path) } // RepoPassword creates a new repository password option with a given password. +// If the password is empty, the repository password option is not set. func RepoPassword(password string) command.Applier { + if password == "" { + return command.NewNoopArgument() + } return command.NewOptionWithRedactedArgument("--password", password) } From de3a52bf38e787bf52470b0d81f4043929eb86e3 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Wed, 28 Feb 2024 15:03:11 -0800 Subject: [PATCH 53/73] Add Location.IsPointInTypeSupported Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/location.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkg/kopia/cli/internal/location.go b/pkg/kopia/cli/internal/location.go index 8b56714487..1d01cfbae6 100644 --- a/pkg/kopia/cli/internal/location.go +++ b/pkg/kopia/cli/internal/location.go @@ -59,3 +59,14 @@ func (l Location) HasSkipSSLVerify() bool { v, _ := strconv.ParseBool(string(l[rs.SkipSSLVerifyKey])) return v } + +// IsPointInTimeSupported returns true if the location supports point-in-time recovery. +// Currently, only S3 and Azure support point-in-time recovery. +func (l Location) IsPointInTypeSupported() bool { + switch l.Type() { + case rs.LocTypeAzure, rs.LocTypeS3, rs.LocTypes3Compliant: + return true + default: + return false + } +} From 722b27915574ed350edf3ccf4436051bfd02bdf4 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Wed, 28 Feb 2024 15:12:55 -0800 Subject: [PATCH 54/73] Add tests for Location.IsPointInTypeSupported Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/location_test.go | 52 +++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/pkg/kopia/cli/internal/location_test.go b/pkg/kopia/cli/internal/location_test.go index f74ca6be71..20d8e979ac 100644 --- a/pkg/kopia/cli/internal/location_test.go +++ b/pkg/kopia/cli/internal/location_test.go @@ -37,6 +37,7 @@ func (s *LocationSuite) TestLocation(c *check.C) { Prefix string IsInsecure bool HasSkipSSLVerify bool + IsPITSupported bool } tests := []struct { @@ -69,6 +70,56 @@ func (s *LocationSuite) TestLocation(c *check.C) { HasSkipSSLVerify: true, }, }, + { + name: "Test PIT Support for S3 Compliant", + location: internal.Location{ + rs.TypeKey: []byte(rs.LocTypes3Compliant), + }, + expected: expected{ + Type: "s3Compliant", + IsPITSupported: true, + }, + }, + { + name: "Test PIT Support for S3", + location: internal.Location{ + rs.TypeKey: []byte(rs.LocTypeS3), + }, + expected: expected{ + Type: "s3", + IsPITSupported: true, + }, + }, + { + name: "Test PIT Support for Azure", + location: internal.Location{ + rs.TypeKey: []byte(rs.LocTypeAzure), + }, + expected: expected{ + Type: "azure", + IsPITSupported: true, + }, + }, + { + name: "Test No PIT Support for GCS", + location: internal.Location{ + rs.TypeKey: []byte(rs.LocTypeGCS), + }, + expected: expected{ + Type: "gcs", + IsPITSupported: false, + }, + }, + { + name: "Test No PIT Support for FS", + location: internal.Location{ + rs.TypeKey: []byte(rs.LocTypeFilestore), + }, + expected: expected{ + Type: "filestore", + IsPITSupported: false, + }, + }, } for _, test := range tests { c.Check(test.location.Type(), check.Equals, test.expected.Type) @@ -78,5 +129,6 @@ func (s *LocationSuite) TestLocation(c *check.C) { c.Check(test.location.Prefix(), check.Equals, test.expected.Prefix) c.Check(test.location.IsInsecureEndpoint(), check.Equals, test.expected.IsInsecure) c.Check(test.location.HasSkipSSLVerify(), check.Equals, test.expected.HasSkipSSLVerify) + c.Check(test.location.IsPointInTypeSupported(), check.Equals, test.expected.IsPITSupported) } } From db06c4007fa45a7d047f83ddbc6fbae1056271dc Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Wed, 28 Feb 2024 15:31:56 -0800 Subject: [PATCH 55/73] Fix s3 options Signed-off-by: pavel.larkin --- pkg/kopia/cli/repository/storage/s3/s3_opts.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/kopia/cli/repository/storage/s3/s3_opts.go b/pkg/kopia/cli/repository/storage/s3/s3_opts.go index 980537f24f..8097a78ba4 100644 --- a/pkg/kopia/cli/repository/storage/s3/s3_opts.go +++ b/pkg/kopia/cli/repository/storage/s3/s3_opts.go @@ -28,7 +28,11 @@ func optBucket(name string) command.Applier { } // optEndpoint creates a new endpoint option with a given endpoint. +// If the endpoint is empty, the endpoint option is not set. func optEndpoint(endpoint string) command.Applier { + if endpoint == "" { + return command.NewNoopArgument() + } return command.NewOptionWithArgument("--endpoint", endpoint) } @@ -38,7 +42,11 @@ func optPrefix(prefix string) command.Applier { } // optRegion creates a new region option with a given region. +// If the region is empty, the region option is not set. func optRegion(region string) command.Applier { + if region == "" { + return command.NewNoopArgument() + } return command.NewOptionWithArgument("--region", region) } From 0962e3bd3066858781871d68c19530c0f5ddd022 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Wed, 28 Feb 2024 15:32:19 -0800 Subject: [PATCH 56/73] Fix s3 options Signed-off-by: pavel.larkin --- pkg/kopia/cli/repository/storage/s3/s3_opts.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/kopia/cli/repository/storage/s3/s3_opts.go b/pkg/kopia/cli/repository/storage/s3/s3_opts.go index 8097a78ba4..4c5b3da64c 100644 --- a/pkg/kopia/cli/repository/storage/s3/s3_opts.go +++ b/pkg/kopia/cli/repository/storage/s3/s3_opts.go @@ -23,7 +23,11 @@ var ( ) // optBucket creates a new bucket option with a given name. +// If the name is empty, the bucket option is not set. func optBucket(name string) command.Applier { + if name == "" { + return command.NewNoopArgument() + } return command.NewOptionWithArgument("--bucket", name) } @@ -37,7 +41,11 @@ func optEndpoint(endpoint string) command.Applier { } // optPrefix creates a new prefix option with a given prefix. +// If the prefix is empty, the prefix option is not set. func optPrefix(prefix string) command.Applier { + if prefix == "" { + return command.NewNoopArgument() + } return command.NewOptionWithArgument("--prefix", prefix) } From a0053897672e19e1fd82c08c9927195734a71408 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Wed, 28 Feb 2024 17:01:47 -0800 Subject: [PATCH 57/73] Fix options to return errors for empty args Signed-off-by: pavel.larkin --- pkg/kopia/cli/errors.go | 8 +++-- pkg/kopia/cli/repository/storage/gcs/gcs.go | 4 --- .../cli/repository/storage/gcs/gcs_opts.go | 13 +++++++++ .../repository/storage/gcs/gcs_opts_test.go | 29 ++++++++++++++----- .../cli/repository/storage/gcs/gcs_test.go | 2 +- 5 files changed, 42 insertions(+), 14 deletions(-) diff --git a/pkg/kopia/cli/errors.go b/pkg/kopia/cli/errors.go index 93afdca4e9..0b0b6db69c 100644 --- a/pkg/kopia/cli/errors.go +++ b/pkg/kopia/cli/errors.go @@ -28,6 +28,10 @@ var ( var ( // ErrUnsupportedStorage is returned when the storage is not supported. ErrUnsupportedStorage = errors.New("unsupported storage") - // ErrInvalidRepoPath is returned when the repoPath is empty. - ErrInvalidRepoPath = errors.New("repository path cannot be empty") + // ErrInvalidPrefix is returned when the prefix is empty. + ErrInvalidPrefix = errors.New("prefix cannot be empty") + // ErrInvalidBucketName is returned when the bucketName is empty. + ErrInvalidBucketName = errors.New("bucket name cannot be empty") + // ErrInvalidCredentialsFile is returned when the credentials file is empty. + ErrInvalidCredentialsFile = errors.New("credentials file cannot be empty") ) diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs.go b/pkg/kopia/cli/repository/storage/gcs/gcs.go index 884f67ff99..a430d3a95a 100644 --- a/pkg/kopia/cli/repository/storage/gcs/gcs.go +++ b/pkg/kopia/cli/repository/storage/gcs/gcs.go @@ -18,7 +18,6 @@ import ( "github.com/kanisterio/safecli/command" "github.com/kanisterio/kanister/pkg/consts" - "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/kanister/pkg/kopia/cli/internal" "github.com/kanisterio/kanister/pkg/log" ) @@ -26,9 +25,6 @@ import ( // New creates a new subcommand for the GCS storage. func New(location internal.Location, repoPathPrefix string, _ log.Logger) command.Applier { prefix := internal.GenerateFullRepoPath(location.Prefix(), repoPathPrefix) - if prefix == "" { - return command.NewErrorArgument(cli.ErrInvalidRepoPath) - } return command.NewArguments(subcmdGCS, optBucket(location.BucketName()), optCredentialsFile(consts.GoogleCloudCredsFilePath), diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs_opts.go b/pkg/kopia/cli/repository/storage/gcs/gcs_opts.go index 1ddc62702d..690ce4a03b 100644 --- a/pkg/kopia/cli/repository/storage/gcs/gcs_opts.go +++ b/pkg/kopia/cli/repository/storage/gcs/gcs_opts.go @@ -15,6 +15,7 @@ package gcs import ( + "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/safecli/command" ) @@ -23,16 +24,28 @@ var ( ) // optBucket creates a new bucket option with a given name. +// If the name is empty, it returns ErrInvalidBucketName. func optBucket(name string) command.Applier { + if name == "" { + return command.NewErrorArgument(cli.ErrInvalidBucketName) + } return command.NewOptionWithArgument("--bucket", name) } // optPrefix creates a new prefix option with a given prefix. +// If the prefix is empty, it returns ErrInvalidPrefix. func optPrefix(prefix string) command.Applier { + if prefix == "" { + return command.NewErrorArgument(cli.ErrInvalidPrefix) + } return command.NewOptionWithArgument("--prefix", prefix) } // optCredentialsFile creates a new GCS credentials file option with a given file path. +// If the file path is empty, it returns ErrInvalidCredentialsFile. func optCredentialsFile(path string) command.Applier { + if path == "" { + return command.NewErrorArgument(cli.ErrInvalidCredentialsFile) + } return command.NewOptionWithArgument("--credentials-file", path) } diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go b/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go index a0737a94cd..2d4dd99525 100644 --- a/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go +++ b/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go @@ -19,7 +19,7 @@ import ( "gopkg.in/check.v1" - "github.com/kanisterio/safecli/command" + "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/safecli/test" ) @@ -27,18 +27,33 @@ func TestGCSOptions(t *testing.T) { check.TestingT(t) } var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ { - Name: "optBucket", - Argument: command.NewArguments(optBucket("bucketname"), optBucket("")), + Name: "optBucket with bucketname should return option", + Argument: optBucket("bucketname"), ExpectedCLI: []string{"cmd", "--bucket=bucketname"}, }, { - Name: "optPrefix", - Argument: command.NewArguments(optPrefix("prefix"), optPrefix("")), + Name: "optBucket with empty bucketname should return error", + Argument: optBucket(""), + ExpectedErr: cli.ErrInvalidBucketName, + }, + { + Name: "optPrefix with prefix should return option", + Argument: optPrefix("prefix"), ExpectedCLI: []string{"cmd", "--prefix=prefix"}, }, { - Name: "optCredentialsFile", - Argument: command.NewArguments(optCredentialsFile("/tmp/file.creds"), optCredentialsFile("")), + Name: "optPrefix with empty prefix should return error", + Argument: optPrefix(""), + ExpectedErr: cli.ErrInvalidPrefix, + }, + { + Name: "optCredentialsFile with path should return option", + Argument: optCredentialsFile("/tmp/file.creds"), ExpectedCLI: []string{"cmd", "--credentials-file=/tmp/file.creds"}, }, + { + Name: "optCredentialsFile with empty path should return error", + Argument: optCredentialsFile(""), + ExpectedErr: cli.ErrInvalidCredentialsFile, + }, }}) diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs_test.go b/pkg/kopia/cli/repository/storage/gcs/gcs_test.go index 7a522e31f3..f27bf1cbbc 100644 --- a/pkg/kopia/cli/repository/storage/gcs/gcs_test.go +++ b/pkg/kopia/cli/repository/storage/gcs/gcs_test.go @@ -49,6 +49,6 @@ var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTe { Name: "NewGCS with empty local prefix and repo prefix should return error", Argument: newGCS("", ""), - ExpectedErr: cli.ErrInvalidRepoPath, + ExpectedErr: cli.ErrInvalidPrefix, }, }}) From 7cabf8a8b2e7cdc203dbe24a0d745fc8b6dd285f Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Wed, 28 Feb 2024 17:04:15 -0800 Subject: [PATCH 58/73] Fix options to return errors for empty args Signed-off-by: pavel.larkin --- pkg/kopia/cli/errors.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/kopia/cli/errors.go b/pkg/kopia/cli/errors.go index 0b0b6db69c..b6eea2f9ab 100644 --- a/pkg/kopia/cli/errors.go +++ b/pkg/kopia/cli/errors.go @@ -28,6 +28,8 @@ var ( var ( // ErrUnsupportedStorage is returned when the storage is not supported. ErrUnsupportedStorage = errors.New("unsupported storage") + // ErrInvalidRepoPath is returned when the repoPath is empty. + ErrInvalidRepoPath = errors.New("repository path cannot be empty") // ErrInvalidPrefix is returned when the prefix is empty. ErrInvalidPrefix = errors.New("prefix cannot be empty") // ErrInvalidBucketName is returned when the bucketName is empty. From 8e1c1f7142e4d4ae23e565e595bc46ee4ab8a352 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Wed, 28 Feb 2024 17:11:02 -0800 Subject: [PATCH 59/73] Fix options to return errors for empty args Signed-off-by: pavel.larkin --- pkg/kopia/cli/errors.go | 2 ++ .../cli/repository/storage/azure/azure.go | 4 ---- .../repository/storage/azure/azure_opts.go | 10 +++++++++ .../storage/azure/azure_opts_test.go | 22 ++++++++++++++----- .../repository/storage/azure/azure_test.go | 2 +- 5 files changed, 29 insertions(+), 11 deletions(-) diff --git a/pkg/kopia/cli/errors.go b/pkg/kopia/cli/errors.go index b6eea2f9ab..483302b57f 100644 --- a/pkg/kopia/cli/errors.go +++ b/pkg/kopia/cli/errors.go @@ -36,4 +36,6 @@ var ( ErrInvalidBucketName = errors.New("bucket name cannot be empty") // ErrInvalidCredentialsFile is returned when the credentials file is empty. ErrInvalidCredentialsFile = errors.New("credentials file cannot be empty") + // ErrInvalidContainerName is returned when the containerName is empty. + ErrInvalidContainerName = errors.New("container name cannot be empty") ) diff --git a/pkg/kopia/cli/repository/storage/azure/azure.go b/pkg/kopia/cli/repository/storage/azure/azure.go index a00233d604..5929ff4e27 100644 --- a/pkg/kopia/cli/repository/storage/azure/azure.go +++ b/pkg/kopia/cli/repository/storage/azure/azure.go @@ -17,7 +17,6 @@ package azure import ( "github.com/kanisterio/safecli/command" - "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/kanister/pkg/kopia/cli/internal" "github.com/kanisterio/kanister/pkg/log" ) @@ -25,9 +24,6 @@ import ( // New creates a new subcommand for the Azure storage. func New(location internal.Location, repoPathPrefix string, _ log.Logger) command.Applier { prefix := internal.GenerateFullRepoPath(location.Prefix(), repoPathPrefix) - if prefix == "" { - return command.NewErrorArgument(cli.ErrInvalidRepoPath) - } return command.NewArguments(subcmdAzure, optContainer(location.BucketName()), optPrefix(prefix), diff --git a/pkg/kopia/cli/repository/storage/azure/azure_opts.go b/pkg/kopia/cli/repository/storage/azure/azure_opts.go index 5618d090eb..bff5920b52 100644 --- a/pkg/kopia/cli/repository/storage/azure/azure_opts.go +++ b/pkg/kopia/cli/repository/storage/azure/azure_opts.go @@ -16,6 +16,8 @@ package azure import ( "github.com/kanisterio/safecli/command" + + "github.com/kanisterio/kanister/pkg/kopia/cli" ) var ( @@ -23,11 +25,19 @@ var ( ) // optPrefix creates a new prefix option with a given prefix. +// If the prefix is empty, it returns ErrInvalidPrefix. func optPrefix(prefix string) command.Applier { + if prefix == "" { + return command.NewErrorArgument(cli.ErrInvalidPrefix) + } return command.NewOptionWithArgument("--prefix", prefix) } // optContainer creates a new container option with a given container name. +// If the name is empty, it returns ErrInvalidContainerName. func optContainer(name string) command.Applier { + if name == "" { + return command.NewErrorArgument(cli.ErrInvalidContainerName) + } return command.NewOptionWithArgument("--container", name) } diff --git a/pkg/kopia/cli/repository/storage/azure/azure_opts_test.go b/pkg/kopia/cli/repository/storage/azure/azure_opts_test.go index f85a3e58da..cdaffb7ced 100644 --- a/pkg/kopia/cli/repository/storage/azure/azure_opts_test.go +++ b/pkg/kopia/cli/repository/storage/azure/azure_opts_test.go @@ -17,23 +17,33 @@ package azure import ( "testing" + "github.com/kanisterio/safecli/test" "gopkg.in/check.v1" - "github.com/kanisterio/safecli/command" - "github.com/kanisterio/safecli/test" + "github.com/kanisterio/kanister/pkg/kopia/cli" ) func TestAzureOptions(t *testing.T) { check.TestingT(t) } var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ { - Name: "optContainer", - Argument: command.NewArguments(optContainer("containername"), optContainer("")), + Name: "optContainer with containername should return option", + Argument: optContainer("containername"), ExpectedCLI: []string{"cmd", "--container=containername"}, }, { - Name: "optPrefix", - Argument: command.NewArguments(optPrefix("prefix"), optPrefix("")), + Name: "optContainer with empty containername should return error", + Argument: optContainer(""), + ExpectedErr: cli.ErrInvalidContainerName, + }, + { + Name: "optPrefix with prefix should return option", + Argument: optPrefix("prefix"), ExpectedCLI: []string{"cmd", "--prefix=prefix"}, }, + { + Name: "optPrefix with empty prefix should return error", + Argument: optPrefix(""), + ExpectedErr: cli.ErrInvalidPrefix, + }, }}) diff --git a/pkg/kopia/cli/repository/storage/azure/azure_test.go b/pkg/kopia/cli/repository/storage/azure/azure_test.go index 942db5e698..f2ec9095aa 100644 --- a/pkg/kopia/cli/repository/storage/azure/azure_test.go +++ b/pkg/kopia/cli/repository/storage/azure/azure_test.go @@ -49,6 +49,6 @@ var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTe { Name: "NewAzure with empty local prefix and repo prefix should return error", Argument: newAzure("", ""), - ExpectedErr: cli.ErrInvalidRepoPath, + ExpectedErr: cli.ErrInvalidPrefix, }, }}) From 9500b329f18e4fcb106b8213d27c1aff012b212b Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Wed, 28 Feb 2024 17:45:17 -0800 Subject: [PATCH 60/73] Support empty prefix Signed-off-by: pavel.larkin --- pkg/kopia/cli/repository/storage/gcs/gcs_opts.go | 4 ---- pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go | 4 ++-- pkg/kopia/cli/repository/storage/gcs/gcs_test.go | 3 +-- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs_opts.go b/pkg/kopia/cli/repository/storage/gcs/gcs_opts.go index 690ce4a03b..d04c89480c 100644 --- a/pkg/kopia/cli/repository/storage/gcs/gcs_opts.go +++ b/pkg/kopia/cli/repository/storage/gcs/gcs_opts.go @@ -33,11 +33,7 @@ func optBucket(name string) command.Applier { } // optPrefix creates a new prefix option with a given prefix. -// If the prefix is empty, it returns ErrInvalidPrefix. func optPrefix(prefix string) command.Applier { - if prefix == "" { - return command.NewErrorArgument(cli.ErrInvalidPrefix) - } return command.NewOptionWithArgument("--prefix", prefix) } diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go b/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go index 2d4dd99525..46d79a7d6e 100644 --- a/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go +++ b/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go @@ -42,9 +42,9 @@ var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTe ExpectedCLI: []string{"cmd", "--prefix=prefix"}, }, { - Name: "optPrefix with empty prefix should return error", + Name: "optPrefix with empty prefix should return option with empty string", Argument: optPrefix(""), - ExpectedErr: cli.ErrInvalidPrefix, + ExpectedCLI: []string{"cmd", "--prefix="}, }, { Name: "optCredentialsFile with path should return option", diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs_test.go b/pkg/kopia/cli/repository/storage/gcs/gcs_test.go index f27bf1cbbc..2bc9280c8b 100644 --- a/pkg/kopia/cli/repository/storage/gcs/gcs_test.go +++ b/pkg/kopia/cli/repository/storage/gcs/gcs_test.go @@ -21,7 +21,6 @@ import ( "github.com/kanisterio/safecli/test" "gopkg.in/check.v1" - "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/kanister/pkg/kopia/cli/internal" ) @@ -49,6 +48,6 @@ var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTe { Name: "NewGCS with empty local prefix and repo prefix should return error", Argument: newGCS("", ""), - ExpectedErr: cli.ErrInvalidPrefix, + ExpectedCLI: []string{"cmd", "gcs", "--bucket=bucket", "--credentials-file=/tmp/creds.txt", "--prefix="}, }, }}) From 777209609104e63a95a0aa163b82e41033a46c69 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Wed, 28 Feb 2024 17:55:25 -0800 Subject: [PATCH 61/73] Support empty prefix Signed-off-by: pavel.larkin --- .../cli/repository/storage/azure/azure_opts.go | 4 ---- .../repository/storage/azure/azure_opts_test.go | 14 +++++--------- .../cli/repository/storage/azure/azure_test.go | 17 +++++++++++------ 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/pkg/kopia/cli/repository/storage/azure/azure_opts.go b/pkg/kopia/cli/repository/storage/azure/azure_opts.go index bff5920b52..1433388c09 100644 --- a/pkg/kopia/cli/repository/storage/azure/azure_opts.go +++ b/pkg/kopia/cli/repository/storage/azure/azure_opts.go @@ -25,11 +25,7 @@ var ( ) // optPrefix creates a new prefix option with a given prefix. -// If the prefix is empty, it returns ErrInvalidPrefix. func optPrefix(prefix string) command.Applier { - if prefix == "" { - return command.NewErrorArgument(cli.ErrInvalidPrefix) - } return command.NewOptionWithArgument("--prefix", prefix) } diff --git a/pkg/kopia/cli/repository/storage/azure/azure_opts_test.go b/pkg/kopia/cli/repository/storage/azure/azure_opts_test.go index cdaffb7ced..bc46a8b7a0 100644 --- a/pkg/kopia/cli/repository/storage/azure/azure_opts_test.go +++ b/pkg/kopia/cli/repository/storage/azure/azure_opts_test.go @@ -17,6 +17,7 @@ package azure import ( "testing" + "github.com/kanisterio/safecli/command" "github.com/kanisterio/safecli/test" "gopkg.in/check.v1" @@ -27,7 +28,7 @@ func TestAzureOptions(t *testing.T) { check.TestingT(t) } var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ { - Name: "optContainer with containername should return option", + Name: "optContainer", Argument: optContainer("containername"), ExpectedCLI: []string{"cmd", "--container=containername"}, }, @@ -37,13 +38,8 @@ var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTe ExpectedErr: cli.ErrInvalidContainerName, }, { - Name: "optPrefix with prefix should return option", - Argument: optPrefix("prefix"), - ExpectedCLI: []string{"cmd", "--prefix=prefix"}, - }, - { - Name: "optPrefix with empty prefix should return error", - Argument: optPrefix(""), - ExpectedErr: cli.ErrInvalidPrefix, + Name: "optPrefix", + Argument: command.NewArguments(optPrefix("prefix"), optPrefix("")), + ExpectedCLI: []string{"cmd", "--prefix=prefix", "--prefix="}, }, }}) diff --git a/pkg/kopia/cli/repository/storage/azure/azure_test.go b/pkg/kopia/cli/repository/storage/azure/azure_test.go index f2ec9095aa..6f426edf46 100644 --- a/pkg/kopia/cli/repository/storage/azure/azure_test.go +++ b/pkg/kopia/cli/repository/storage/azure/azure_test.go @@ -27,10 +27,10 @@ import ( func TestNewAzure(t *testing.T) { check.TestingT(t) } -func newAzure(prefix, repoPath string) command.Applier { +func newAzure(prefix, repoPath, bucket string) command.Applier { l := internal.Location{ "prefix": []byte(prefix), - "bucket": []byte("bucket"), + "bucket": []byte(bucket), } return New(l, repoPath, nil) } @@ -38,17 +38,22 @@ func newAzure(prefix, repoPath string) command.Applier { var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ { Name: "NewAzure", - Argument: newAzure("prefix", "repoPath"), + Argument: newAzure("prefix", "repoPath", "bucket"), ExpectedCLI: []string{"cmd", "azure", "--container=bucket", "--prefix=prefix/repoPath/"}, }, { Name: "NewAzure with empty repoPath", - Argument: newAzure("prefix", ""), + Argument: newAzure("prefix", "", "bucket"), ExpectedCLI: []string{"cmd", "azure", "--container=bucket", "--prefix=prefix/"}, }, { Name: "NewAzure with empty local prefix and repo prefix should return error", - Argument: newAzure("", ""), - ExpectedErr: cli.ErrInvalidPrefix, + Argument: newAzure("", "", "bucket"), + ExpectedCLI: []string{"cmd", "azure", "--container=bucket", "--prefix="}, + }, + { + Name: "NewAzure with empty bucket should return ErrInvalidContainerName", + Argument: newAzure("", "", ""), + ExpectedErr: cli.ErrInvalidContainerName, }, }}) From a4ddad67f0f8135fac7570a8c850e625b44a39e6 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Wed, 28 Feb 2024 17:57:21 -0800 Subject: [PATCH 62/73] Support empty prefix Signed-off-by: pavel.larkin --- .../cli/repository/storage/gcs/gcs_opts_test.go | 12 ++++-------- pkg/kopia/cli/repository/storage/gcs/gcs_test.go | 16 +++++++++++----- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go b/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go index 46d79a7d6e..d45adab333 100644 --- a/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go +++ b/pkg/kopia/cli/repository/storage/gcs/gcs_opts_test.go @@ -20,6 +20,7 @@ import ( "gopkg.in/check.v1" "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/safecli/command" "github.com/kanisterio/safecli/test" ) @@ -37,14 +38,9 @@ var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTe ExpectedErr: cli.ErrInvalidBucketName, }, { - Name: "optPrefix with prefix should return option", - Argument: optPrefix("prefix"), - ExpectedCLI: []string{"cmd", "--prefix=prefix"}, - }, - { - Name: "optPrefix with empty prefix should return option with empty string", - Argument: optPrefix(""), - ExpectedCLI: []string{"cmd", "--prefix="}, + Name: "optPrefix", + Argument: command.NewArguments(optPrefix("prefix"), optPrefix("")), + ExpectedCLI: []string{"cmd", "--prefix=prefix", "--prefix="}, }, { Name: "optCredentialsFile with path should return option", diff --git a/pkg/kopia/cli/repository/storage/gcs/gcs_test.go b/pkg/kopia/cli/repository/storage/gcs/gcs_test.go index 2bc9280c8b..09356db69d 100644 --- a/pkg/kopia/cli/repository/storage/gcs/gcs_test.go +++ b/pkg/kopia/cli/repository/storage/gcs/gcs_test.go @@ -21,15 +21,16 @@ import ( "github.com/kanisterio/safecli/test" "gopkg.in/check.v1" + "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/kanister/pkg/kopia/cli/internal" ) func TestNewGCS(t *testing.T) { check.TestingT(t) } -func newGCS(prefix, repoPath string) command.Applier { +func newGCS(prefix, repoPath, bucket string) command.Applier { l := internal.Location{ "prefix": []byte(prefix), - "bucket": []byte("bucket"), + "bucket": []byte(bucket), } return New(l, repoPath, nil) } @@ -37,17 +38,22 @@ func newGCS(prefix, repoPath string) command.Applier { var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ { Name: "NewGCS", - Argument: newGCS("prefix", "repoPath"), + Argument: newGCS("prefix", "repoPath", "bucket"), ExpectedCLI: []string{"cmd", "gcs", "--bucket=bucket", "--credentials-file=/tmp/creds.txt", "--prefix=prefix/repoPath/"}, }, { Name: "NewGCS with empty repoPath", - Argument: newGCS("prefix", ""), + Argument: newGCS("prefix", "", "bucket"), ExpectedCLI: []string{"cmd", "gcs", "--bucket=bucket", "--credentials-file=/tmp/creds.txt", "--prefix=prefix/"}, }, { Name: "NewGCS with empty local prefix and repo prefix should return error", - Argument: newGCS("", ""), + Argument: newGCS("", "", "bucket"), ExpectedCLI: []string{"cmd", "gcs", "--bucket=bucket", "--credentials-file=/tmp/creds.txt", "--prefix="}, }, + { + Name: "NewGCS with empty bucket should return ErrInvalidBucketName", + Argument: newGCS("", "", ""), + ExpectedErr: cli.ErrInvalidBucketName, + }, }}) From 37e1dc7903c0ec0ae620335fa286d2a871bd74ab Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Wed, 28 Feb 2024 18:21:49 -0800 Subject: [PATCH 63/73] Support empty prefix Signed-off-by: pavel.larkin --- .../cli/repository/storage/s3/arg_test.go | 68 ++++++++++ pkg/kopia/cli/repository/storage/s3/s3.go | 4 - .../cli/repository/storage/s3/s3_opts.go | 9 +- .../cli/repository/storage/s3/s3_opts_test.go | 21 +-- .../cli/repository/storage/s3/s3_test.go | 121 +++++------------- 5 files changed, 112 insertions(+), 111 deletions(-) create mode 100644 pkg/kopia/cli/repository/storage/s3/arg_test.go diff --git a/pkg/kopia/cli/repository/storage/s3/arg_test.go b/pkg/kopia/cli/repository/storage/s3/arg_test.go new file mode 100644 index 0000000000..90776f5a4f --- /dev/null +++ b/pkg/kopia/cli/repository/storage/s3/arg_test.go @@ -0,0 +1,68 @@ +package s3 + +import ( + "github.com/kanisterio/safecli/test" + "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" + intlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" + "github.com/kanisterio/kanister/pkg/log" +) + +// ArgTest extends test.ArgumentTest to include logger tests. +type ArgTest struct { + test test.ArgumentTest + + location internal.Location // location is the location to use for the test. + repoPath string // repoPath is the repository path to use for the test. + Logger log.Logger // Logger is the logger to use for the test. (optional) + LoggerRegex []string // LoggerRegex is a list of regexs to match against the log output. (optional) +} + +// Test runs the test with the given command and checks the log output. +func (t *ArgTest) Test(c *check.C, cmd string) { + t.test.Argument = New(t.location, t.repoPath, t.Logger) + t.test.Test(c, cmd) + t.assertLog(c) +} + +// assertLog checks the log output against the expected regexs. +func (t *ArgTest) assertLog(c *check.C) { + if t.Logger == nil { + return + } + + log, ok := t.Logger.(*intlog.StringLogger) + if !ok { + c.Fatalf("t.Logger is not *intlog.StringLogger") + } + if t.isEmptyLogExpected() { + cmtLog := check.Commentf("FAIL: log should be empty but got %#v", log) + c.Assert(len([]string(*log)), check.Equals, 0, cmtLog) + return + } + + // Check each regex. + for _, regex := range t.LoggerRegex { + cmtLog := check.Commentf("FAIL: %v\nlog %#v expected to match %#v", t.test.Name, log, regex) + c.Assert(log.MatchString(regex), check.Equals, true, cmtLog) + } +} + +// isEmptyLogExpected returns true if the test expects an empty log. +func (t *ArgTest) isEmptyLogExpected() bool { + return len(t.LoggerRegex) == 1 && t.LoggerRegex[0] == "" +} + +// ArgSuite defines a suite of tests for a single ArgTest. +type ArgSuite struct { + Cmd string // Cmd appends to the safecli.Builder before test if not empty. + Arguments []ArgTest // Tests to run. +} + +// TestArguments runs all tests in the suite. +func (s *ArgSuite) TestArguments(c *check.C) { + for _, arg := range s.Arguments { + arg.Test(c, s.Cmd) + } +} diff --git a/pkg/kopia/cli/repository/storage/s3/s3.go b/pkg/kopia/cli/repository/storage/s3/s3.go index 610e03cc1e..7a5489755b 100644 --- a/pkg/kopia/cli/repository/storage/s3/s3.go +++ b/pkg/kopia/cli/repository/storage/s3/s3.go @@ -19,7 +19,6 @@ import ( "github.com/kanisterio/safecli/command" - "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/kanister/pkg/kopia/cli/internal" intlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" "github.com/kanisterio/kanister/pkg/log" @@ -32,9 +31,6 @@ func New(location internal.Location, repoPathPrefix string, logger log.Logger) c } endpoint := resolveS3Endpoint(location.Endpoint(), logger) prefix := internal.GenerateFullRepoPath(location.Prefix(), repoPathPrefix) - if prefix == "" { - return command.NewErrorArgument(cli.ErrInvalidRepoPath) - } return command.NewArguments(subcmdS3, optRegion(location.Region()), optBucket(location.BucketName()), diff --git a/pkg/kopia/cli/repository/storage/s3/s3_opts.go b/pkg/kopia/cli/repository/storage/s3/s3_opts.go index 4c5b3da64c..131c5803ad 100644 --- a/pkg/kopia/cli/repository/storage/s3/s3_opts.go +++ b/pkg/kopia/cli/repository/storage/s3/s3_opts.go @@ -16,6 +16,8 @@ package s3 import ( "github.com/kanisterio/safecli/command" + + "github.com/kanisterio/kanister/pkg/kopia/cli" ) var ( @@ -23,10 +25,10 @@ var ( ) // optBucket creates a new bucket option with a given name. -// If the name is empty, the bucket option is not set. +// If the name is empty, it returns ErrInvalidBucketName. func optBucket(name string) command.Applier { if name == "" { - return command.NewNoopArgument() + return command.NewErrorArgument(cli.ErrInvalidBucketName) } return command.NewOptionWithArgument("--bucket", name) } @@ -43,9 +45,6 @@ func optEndpoint(endpoint string) command.Applier { // optPrefix creates a new prefix option with a given prefix. // If the prefix is empty, the prefix option is not set. func optPrefix(prefix string) command.Applier { - if prefix == "" { - return command.NewNoopArgument() - } return command.NewOptionWithArgument("--prefix", prefix) } diff --git a/pkg/kopia/cli/repository/storage/s3/s3_opts_test.go b/pkg/kopia/cli/repository/storage/s3/s3_opts_test.go index 44108bbf32..3485f852d2 100644 --- a/pkg/kopia/cli/repository/storage/s3/s3_opts_test.go +++ b/pkg/kopia/cli/repository/storage/s3/s3_opts_test.go @@ -17,20 +17,26 @@ package s3 import ( "testing" - "gopkg.in/check.v1" - "github.com/kanisterio/safecli/command" "github.com/kanisterio/safecli/test" + "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli" ) func TestS3Options(t *testing.T) { check.TestingT(t) } var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ { - Name: "optBucket", - Argument: command.NewArguments(optBucket("bucketname"), optBucket("")), + Name: "optBucket with bucketname should return option", + Argument: optBucket("bucketname"), ExpectedCLI: []string{"cmd", "--bucket=bucketname"}, }, + { + Name: "optBucket with empty bucketname should return error", + Argument: optBucket(""), + ExpectedErr: cli.ErrInvalidBucketName, + }, { Name: "optEndpoint", Argument: command.NewArguments(optEndpoint("endpoint"), optEndpoint("")), @@ -39,12 +45,7 @@ var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTe { Name: "optPrefix", Argument: command.NewArguments(optPrefix("prefix"), optPrefix("")), - ExpectedCLI: []string{"cmd", "--prefix=prefix"}, - }, - { - Name: "optRegion", - Argument: command.NewArguments(optRegion("region"), optRegion("")), - ExpectedCLI: []string{"cmd", "--region=region"}, + ExpectedCLI: []string{"cmd", "--prefix=prefix", "--prefix="}, }, { Name: "optDisableTLS", diff --git a/pkg/kopia/cli/repository/storage/s3/s3_test.go b/pkg/kopia/cli/repository/storage/s3/s3_test.go index 0d8cd65d04..8f688b2309 100644 --- a/pkg/kopia/cli/repository/storage/s3/s3_test.go +++ b/pkg/kopia/cli/repository/storage/s3/s3_test.go @@ -15,6 +15,7 @@ package s3 import ( + "strconv" "testing" "github.com/kanisterio/safecli/test" @@ -23,65 +24,17 @@ import ( "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/kanister/pkg/kopia/cli/internal" intlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" - "github.com/kanisterio/kanister/pkg/log" ) func TestNewS3(t *testing.T) { check.TestingT(t) } -// ArgTest extends test.ArgumentTest to include logger tests. -type ArgTest struct { - test test.ArgumentTest - - location internal.Location // location is the location to use for the test. - repoPath string // repoPath is the repository path to use for the test. - Logger log.Logger // Logger is the logger to use for the test. (optional) - LoggerRegex []string // LoggerRegex is a list of regexs to match against the log output. (optional) -} - -// Test runs the test with the given command and checks the log output. -func (t *ArgTest) Test(c *check.C, cmd string) { - t.test.Argument = New(t.location, t.repoPath, t.Logger) - t.test.Test(c, cmd) - t.assertLog(c) -} - -func (t *ArgTest) isEmptyLogExpected() bool { - return len(t.LoggerRegex) == 1 && t.LoggerRegex[0] == "" -} - -// assertLog checks the log output against the expected regexs. -func (t *ArgTest) assertLog(c *check.C) { - if t.Logger == nil { - return - } - - log, ok := t.Logger.(*intlog.StringLogger) - if !ok { - c.Fatalf("t.Logger is not *intlog.StringLogger") - } - if t.isEmptyLogExpected() { - cmtLog := check.Commentf("FAIL: log should be empty but got %#v", log) - c.Assert(len([]string(*log)), check.Equals, 0, cmtLog) - return - } - - // Check each regex. - for _, regex := range t.LoggerRegex { - cmtLog := check.Commentf("FAIL: %v\nlog %#v expected to match %#v", t.test.Name, log, regex) - c.Assert(log.MatchString(regex), check.Equals, true, cmtLog) - } -} - -// ArgSuite defines a suite of tests for a single ArgTest. -type ArgSuite struct { - Cmd string // Cmd appends to the safecli.Builder before test if not empty. - Arguments []ArgTest // Tests to run. -} - -// TestArguments runs all tests in the suite. -func (s *ArgSuite) TestArguments(c *check.C) { - for _, arg := range s.Arguments { - arg.Test(c, s.Cmd) +func newLocation(prefix, endpoint, region, bucket string, skipSSLVerify bool) internal.Location { + return internal.Location{ + "prefix": []byte(prefix), + "endpoint": []byte(endpoint), + "region": []byte(region), + "bucket": []byte(bucket), + "skipSSLVerify": []byte(strconv.FormatBool(skipSSLVerify)), } } @@ -98,13 +51,7 @@ var _ = check.Suite(&ArgSuite{Cmd: "cmd", Arguments: []ArgTest{ "--disable-tls-verification", }, }, - location: internal.Location{ - "prefix": []byte("prefix"), - "endpoint": []byte("http://endpoint/path/"), - "region": []byte("region"), - "bucket": []byte("bucket"), - "skipSSLVerify": []byte("true"), - }, + location: newLocation("prefix", "http://endpoint/path/", "region", "bucket", true), repoPath: "repoPath", Logger: &intlog.StringLogger{}, LoggerRegex: []string{ @@ -124,13 +71,7 @@ var _ = check.Suite(&ArgSuite{Cmd: "cmd", Arguments: []ArgTest{ "--disable-tls-verification", }, }, - location: internal.Location{ - "prefix": []byte("prefix"), - "endpoint": []byte("http://endpoint/path/"), - "region": []byte("region"), - "bucket": []byte("bucket"), - "skipSSLVerify": []byte("true"), - }, + location: newLocation("prefix", "http://endpoint/path/", "region", "bucket", true), repoPath: "repoPath", }, { @@ -141,16 +82,9 @@ var _ = check.Suite(&ArgSuite{Cmd: "cmd", Arguments: []ArgTest{ "--bucket=bucket", "--endpoint=endpoint/path", "--prefix=prefix/", - "--disable-tls-verification", }, }, - location: internal.Location{ - "prefix": []byte("prefix"), - "endpoint": []byte("https://endpoint/path/"), - "region": []byte("region"), - "bucket": []byte("bucket"), - "skipSSLVerify": []byte("true"), - }, + location: newLocation("prefix", "https://endpoint/path/", "region", "bucket", false), repoPath: "", Logger: &intlog.StringLogger{}, LoggerRegex: []string{ @@ -168,29 +102,32 @@ var _ = check.Suite(&ArgSuite{Cmd: "cmd", Arguments: []ArgTest{ "--disable-tls-verification", }, }, - location: internal.Location{ - "prefix": []byte("prefix"), - "endpoint": []byte(""), - "region": []byte("region"), - "bucket": []byte("bucket"), - "skipSSLVerify": []byte("true"), - }, + location: newLocation("prefix", "", "region", "bucket", true), repoPath: "", Logger: &intlog.StringLogger{}, LoggerRegex: []string{""}, // no output expected }, { test: test.ArgumentTest{ - Name: "NewS3 with empty repoPath, prefix and endpoint", - ExpectedErr: cli.ErrInvalidRepoPath, + Name: "NewS3 with empty repoPath, prefix and endpoint", + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--prefix=", + "--disable-tls-verification", + }, }, - location: internal.Location{ - "prefix": []byte(""), - "endpoint": []byte(""), - "region": []byte("region"), - "bucket": []byte("bucket"), - "skipSSLVerify": []byte("true"), + location: newLocation("", "", "region", "bucket", true), + repoPath: "", + Logger: &intlog.StringLogger{}, + LoggerRegex: []string{""}, // no output expected + }, + { + test: test.ArgumentTest{ + Name: "NewS3 with empty repoPath, prefix, endpoint and bucket", + ExpectedErr: cli.ErrInvalidBucketName, }, + location: internal.Location{}, repoPath: "", Logger: &intlog.StringLogger{}, LoggerRegex: []string{""}, // no output expected From 8a33fda6a2a9780d661d82c493c03a995891eaf0 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Wed, 28 Feb 2024 18:28:53 -0800 Subject: [PATCH 64/73] Support empty argument for hostname and username options Signed-off-by: pavel.larkin --- pkg/kopia/cli/repository/opts.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/kopia/cli/repository/opts.go b/pkg/kopia/cli/repository/opts.go index 04a63002b4..81409c99a0 100644 --- a/pkg/kopia/cli/repository/opts.go +++ b/pkg/kopia/cli/repository/opts.go @@ -37,12 +37,20 @@ var ( ) // optHostname creates a new option for the hostname of the repository. +// If the hostname is empty, the hostname option is not set. func optHostname(h string) command.Applier { + if h == "" { + return command.NewNoopArgument() + } return command.NewOptionWithArgument("--override-hostname", h) } // optUsername creates a new option for the username of the repository. +// If the username is empty, the username option is not set. func optUsername(u string) command.Applier { + if u == "" { + return command.NewNoopArgument() + } return command.NewOptionWithArgument("--override-username", u) } From 378d25536d7adba7c45de50aaae60d15576a412f Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Wed, 28 Feb 2024 18:41:35 -0800 Subject: [PATCH 65/73] Support PIT option for Azure and S3 only Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/kopia_test.go | 21 ++++++++++ pkg/kopia/cli/repository/opts.go | 6 +-- pkg/kopia/cli/repository/opts_test.go | 34 +++++++++++---- .../cli/repository/repository_connect.go | 2 +- .../cli/repository/repository_connect_test.go | 41 +++++++++++++++++-- 5 files changed, 88 insertions(+), 16 deletions(-) create mode 100644 pkg/kopia/cli/internal/kopia_test.go diff --git a/pkg/kopia/cli/internal/kopia_test.go b/pkg/kopia/cli/internal/kopia_test.go new file mode 100644 index 0000000000..c168843d35 --- /dev/null +++ b/pkg/kopia/cli/internal/kopia_test.go @@ -0,0 +1,21 @@ +package internal_test + +import ( + "testing" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/opts" + "gopkg.in/check.v1" +) + +func TestNewKopiaCommand(t *testing.T) { check.TestingT(t) } + +type NewKopiaCommandSuite struct{} + +var _ = check.Suite(&NewKopiaCommandSuite{}) + +func (s *PathSuite) TestNewKopiaCommandSuite(c *check.C) { + cmd, err := internal.NewKopiaCommand(opts.JSON(true)) + c.Check(err, check.IsNil) + c.Check(cmd.Build(), check.DeepEquals, []string{"kopia", "--json"}) +} diff --git a/pkg/kopia/cli/repository/opts.go b/pkg/kopia/cli/repository/opts.go index 07d9ad06b9..16581c427e 100644 --- a/pkg/kopia/cli/repository/opts.go +++ b/pkg/kopia/cli/repository/opts.go @@ -98,9 +98,9 @@ func optReadOnly(readOnly bool) command.Applier { } // optPointInTime creates a new option for the point-in-time of the repository. -func optPointInTime(tm strfmt.DateTime) command.Applier { - if time.Time(tm).IsZero() { +func optPointInTime(l internal.Location, pit strfmt.DateTime) command.Applier { + if !l.IsPointInTypeSupported() || time.Time(pit).IsZero() { return command.NewNoopArgument() } - return command.NewOptionWithArgument("--point-in-time", tm.String()) + return command.NewOptionWithArgument("--point-in-time", pit.String()) } diff --git a/pkg/kopia/cli/repository/opts_test.go b/pkg/kopia/cli/repository/opts_test.go index af595e996c..f5cc578d1a 100644 --- a/pkg/kopia/cli/repository/opts_test.go +++ b/pkg/kopia/cli/repository/opts_test.go @@ -15,6 +15,7 @@ package repository import ( + "fmt" "testing" "github.com/go-openapi/strfmt" @@ -23,6 +24,7 @@ import ( "gopkg.in/check.v1" "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" ) func TestRepositoryOptions(t *testing.T) { check.TestingT(t) } @@ -86,14 +88,28 @@ var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTe ExpectedCLI: []string{"cmd", "--readonly"}, }, { - Name: "optPointInTime", - Argument: command.NewArguments( - optPointInTime(func() strfmt.DateTime { - t, _ := strfmt.ParseDateTime("2021-02-03T01:02:03.000Z") - return t - }()), - optPointInTime(strfmt.DateTime{}), // no output - ), - ExpectedCLI: []string{"cmd", "--point-in-time=2021-02-03T01:02:03.000Z"}, + Name: "optPointInTime supported only for azure and s3", + Argument: func() command.Arguments { + locations := []internal.Location{ + locFS, // no output + locAzure, // idx: 1 + locGCS, // no output + locS3, // idx: 3 + locS3Compliant, // idx: 4 + } + var args command.Arguments + for idx, l := range locations { + t, _ := strfmt.ParseDateTime( + fmt.Sprintf("2021-02-%02dT01:02:03.000Z", idx), + ) + args = append(args, optPointInTime(l, t)) + } + return args + }(), + ExpectedCLI: []string{"cmd", + "--point-in-time=2021-02-01T01:02:03.000Z", + "--point-in-time=2021-02-03T01:02:03.000Z", + "--point-in-time=2021-02-04T01:02:03.000Z", + }, }, }}) diff --git a/pkg/kopia/cli/repository/repository_connect.go b/pkg/kopia/cli/repository/repository_connect.go index 2bb6143c63..a64639ac48 100644 --- a/pkg/kopia/cli/repository/repository_connect.go +++ b/pkg/kopia/cli/repository/repository_connect.go @@ -54,6 +54,6 @@ func Connect(args ConnectArgs) (*safecli.Builder, error) { args.RepoPathPrefix, args.Logger, ), - optPointInTime(args.PointInTime), + optPointInTime(args.Location, args.PointInTime), ) } diff --git a/pkg/kopia/cli/repository/repository_connect_test.go b/pkg/kopia/cli/repository/repository_connect_test.go index a84ce059ec..d3f8f338ea 100644 --- a/pkg/kopia/cli/repository/repository_connect_test.go +++ b/pkg/kopia/cli/repository/repository_connect_test.go @@ -45,9 +45,8 @@ var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ }, }, { - Name: "repository connect with PIT and ReadOnly", + Name: "repository connect with ReadOnly", Command: func() (*safecli.Builder, error) { - pit, _ := strfmt.ParseDateTime("2021-02-03T01:02:03Z") args := ConnectArgs{ Common: common, Cache: cache, @@ -55,7 +54,6 @@ var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ Username: "test-username", RepoPathPrefix: "test-path/prefix", Location: locFS, - PointInTime: pit, ReadOnly: true, } return Connect(args) @@ -76,6 +74,43 @@ var _ = check.Suite(test.NewCommandSuite([]test.CommandTest{ "--override-username=test-username", "filesystem", "--path=/mnt/data/test-prefix/test-path/prefix/", + }, + }, + { + Name: "repository connect with PIT and ReadOnly", + Command: func() (*safecli.Builder, error) { + pit, _ := strfmt.ParseDateTime("2021-02-03T01:02:03Z") + args := ConnectArgs{ + Common: common, + Cache: cache, + Hostname: "test-hostname", + Username: "test-username", + RepoPathPrefix: "path/prefix", + Location: locS3, + PointInTime: pit, + ReadOnly: true, + } + return Connect(args) + }, + ExpectedCLI: []string{"kopia", + "--config-file=path/kopia.config", + "--log-dir=cache/log", + "--log-level=error", + "--password=encr-key", + "repository", + "connect", + "--no-check-for-updates", + "--readonly", + "--cache-directory=/tmp/cache.dir", + "--content-cache-size-limit-mb=0", + "--metadata-cache-size-limit-mb=0", + "--override-hostname=test-hostname", + "--override-username=test-username", + "s3", + "--region=test-region", + "--bucket=test-bucket", + "--endpoint=test-endpoint", + "--prefix=test-prefix/path/prefix/", "--point-in-time=2021-02-03T01:02:03.000Z", }, }, From 241e6938ea5661bf3057ee39b821b657517245e4 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Wed, 28 Feb 2024 18:49:28 -0800 Subject: [PATCH 66/73] Server URL option is required Signed-off-by: pavel.larkin --- pkg/kopia/cli/errors.go | 2 ++ pkg/kopia/cli/repository/opts.go | 6 ++++++ pkg/kopia/cli/repository/opts_test.go | 12 +++++++----- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/pkg/kopia/cli/errors.go b/pkg/kopia/cli/errors.go index 483302b57f..68153212d5 100644 --- a/pkg/kopia/cli/errors.go +++ b/pkg/kopia/cli/errors.go @@ -38,4 +38,6 @@ var ( ErrInvalidCredentialsFile = errors.New("credentials file cannot be empty") // ErrInvalidContainerName is returned when the containerName is empty. ErrInvalidContainerName = errors.New("container name cannot be empty") + // ErrInvalidServerURL is returned when the serverURL is empty. + ErrInvalidServerURL = errors.New("server URL cannot be empty") ) diff --git a/pkg/kopia/cli/repository/opts.go b/pkg/kopia/cli/repository/opts.go index 3a65b8cb1e..fdea7e3394 100644 --- a/pkg/kopia/cli/repository/opts.go +++ b/pkg/kopia/cli/repository/opts.go @@ -108,10 +108,16 @@ func optPointInTime(l internal.Location, pit strfmt.DateTime) command.Applier { // optServerURL creates a new server URL flag with a given server URL. func optServerURL(serverURL string) command.Applier { + if serverURL == "" { + return command.NewErrorArgument(cli.ErrInvalidServerURL) + } return command.NewOptionWithArgument("--url", serverURL) } // optServerCertFingerprint creates a new server certificate fingerprint flag with a given fingerprint. func optServerCertFingerprint(fingerprint string) command.Applier { + if fingerprint == "" { + return command.NewNoopArgument() + } return command.NewOptionWithRedactedArgument("--server-cert-fingerprint", fingerprint) } diff --git a/pkg/kopia/cli/repository/opts_test.go b/pkg/kopia/cli/repository/opts_test.go index a3b5510173..0228834edc 100644 --- a/pkg/kopia/cli/repository/opts_test.go +++ b/pkg/kopia/cli/repository/opts_test.go @@ -113,13 +113,15 @@ var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTe }, }, { - Name: "optServerURL", - Argument: command.NewArguments( - optServerURL("http://test-server"), - optServerURL(""), // no output - ), + Name: "optServerURL with ServerURL should return option", + Argument: optServerURL("http://test-server"), ExpectedCLI: []string{"cmd", "--url=http://test-server"}, }, + { + Name: "optServerURL with empty ServerURL should return error", + Argument: optServerURL(""), + ExpectedErr: cli.ErrInvalidServerURL, + }, { Name: "optServerCertFingerprint", Argument: command.NewArguments( From 44c133ec05dae55112f85ae614237cc19882092d Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Thu, 29 Feb 2024 13:54:31 -0800 Subject: [PATCH 67/73] Fix formatting Signed-off-by: pavel.larkin --- pkg/kopia/cli/doc.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/kopia/cli/doc.go b/pkg/kopia/cli/doc.go index 6f0681452f..75425c26e4 100644 --- a/pkg/kopia/cli/doc.go +++ b/pkg/kopia/cli/doc.go @@ -1,5 +1,3 @@ -package cli - // Copyright 2024 The Kanister Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,8 +12,10 @@ package cli // See the License for the specific language governing permissions and // limitations under the License. +// This package contains the implementation of the Kopia CLI using github.com/kanisterio/safecli. + +package cli + import ( _ "github.com/kanisterio/safecli" ) - -// This package contains the implementation of the Kopia CLI using github.com/kanisterio/safecli. From 33b1342f23779161506b26d7f65cf81732cb4b7f Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Thu, 29 Feb 2024 15:16:35 -0800 Subject: [PATCH 68/73] organize imports Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/args/args.go | 3 ++- pkg/kopia/cli/internal/args/args_test.go | 5 +++-- pkg/kopia/cli/internal/opts/cache_opts.go | 3 ++- pkg/kopia/cli/internal/opts/cache_opts_test.go | 5 +++-- pkg/kopia/cli/internal/opts/common_opts.go | 3 ++- pkg/kopia/cli/internal/opts/common_opts_test.go | 5 +++-- pkg/kopia/cli/internal/opts/opts_test.go | 3 ++- 7 files changed, 17 insertions(+), 10 deletions(-) diff --git a/pkg/kopia/cli/internal/args/args.go b/pkg/kopia/cli/internal/args/args.go index a6a8223b6d..cad732dc9d 100644 --- a/pkg/kopia/cli/internal/args/args.go +++ b/pkg/kopia/cli/internal/args/args.go @@ -15,8 +15,9 @@ package args import ( - "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/safecli/command" + + "github.com/kanisterio/kanister/pkg/kopia/cli" ) // ID creates a new ID argument. diff --git a/pkg/kopia/cli/internal/args/args_test.go b/pkg/kopia/cli/internal/args/args_test.go index ea6fac26ff..c8362009e8 100644 --- a/pkg/kopia/cli/internal/args/args_test.go +++ b/pkg/kopia/cli/internal/args/args_test.go @@ -17,10 +17,11 @@ package args_test import ( "testing" - "github.com/kanisterio/kanister/pkg/kopia/cli" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/args" "github.com/kanisterio/safecli/test" "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/args" ) func TestArgs(t *testing.T) { check.TestingT(t) } diff --git a/pkg/kopia/cli/internal/opts/cache_opts.go b/pkg/kopia/cli/internal/opts/cache_opts.go index 75fa0b8f89..fb1222b4af 100644 --- a/pkg/kopia/cli/internal/opts/cache_opts.go +++ b/pkg/kopia/cli/internal/opts/cache_opts.go @@ -17,8 +17,9 @@ package opts import ( "strconv" - "github.com/kanisterio/kanister/pkg/kopia/cli/args" "github.com/kanisterio/safecli/command" + + "github.com/kanisterio/kanister/pkg/kopia/cli/args" ) const ( diff --git a/pkg/kopia/cli/internal/opts/cache_opts_test.go b/pkg/kopia/cli/internal/opts/cache_opts_test.go index 40c56d8966..ee87091144 100644 --- a/pkg/kopia/cli/internal/opts/cache_opts_test.go +++ b/pkg/kopia/cli/internal/opts/cache_opts_test.go @@ -17,11 +17,12 @@ package opts_test import ( "testing" - "github.com/kanisterio/kanister/pkg/kopia/cli/args" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/opts" "github.com/kanisterio/safecli/command" "github.com/kanisterio/safecli/test" "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli/args" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/opts" ) func TestCacheOptions(t *testing.T) { check.TestingT(t) } diff --git a/pkg/kopia/cli/internal/opts/common_opts.go b/pkg/kopia/cli/internal/opts/common_opts.go index 641de33528..d17f71e5c3 100644 --- a/pkg/kopia/cli/internal/opts/common_opts.go +++ b/pkg/kopia/cli/internal/opts/common_opts.go @@ -15,8 +15,9 @@ package opts import ( - "github.com/kanisterio/kanister/pkg/kopia/cli/args" "github.com/kanisterio/safecli/command" + + "github.com/kanisterio/kanister/pkg/kopia/cli/args" ) const ( diff --git a/pkg/kopia/cli/internal/opts/common_opts_test.go b/pkg/kopia/cli/internal/opts/common_opts_test.go index f3f24131a4..28560a9537 100644 --- a/pkg/kopia/cli/internal/opts/common_opts_test.go +++ b/pkg/kopia/cli/internal/opts/common_opts_test.go @@ -17,11 +17,12 @@ package opts_test import ( "testing" - "github.com/kanisterio/kanister/pkg/kopia/cli/args" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/opts" "github.com/kanisterio/safecli/command" "github.com/kanisterio/safecli/test" "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli/args" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/opts" ) func TestCommonOptions(t *testing.T) { check.TestingT(t) } diff --git a/pkg/kopia/cli/internal/opts/opts_test.go b/pkg/kopia/cli/internal/opts/opts_test.go index f7aa7178ee..c0e955fd85 100644 --- a/pkg/kopia/cli/internal/opts/opts_test.go +++ b/pkg/kopia/cli/internal/opts/opts_test.go @@ -17,10 +17,11 @@ package opts_test import ( "testing" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal/opts" "github.com/kanisterio/safecli/command" "github.com/kanisterio/safecli/test" "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal/opts" ) func TestOptions(t *testing.T) { check.TestingT(t) } From 516a14c56b6d20ce6bbc8e1b9db40a5f65a22581 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Thu, 29 Feb 2024 15:20:39 -0800 Subject: [PATCH 69/73] organize imports Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/location_test.go | 3 ++- pkg/kopia/cli/internal/path_test.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/kopia/cli/internal/location_test.go b/pkg/kopia/cli/internal/location_test.go index 20d8e979ac..d28defd5ed 100644 --- a/pkg/kopia/cli/internal/location_test.go +++ b/pkg/kopia/cli/internal/location_test.go @@ -17,9 +17,10 @@ package internal_test import ( "testing" + "gopkg.in/check.v1" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" - "gopkg.in/check.v1" ) func TestLocation(t *testing.T) { check.TestingT(t) } diff --git a/pkg/kopia/cli/internal/path_test.go b/pkg/kopia/cli/internal/path_test.go index 4ddfeabed1..e52b0d2c52 100644 --- a/pkg/kopia/cli/internal/path_test.go +++ b/pkg/kopia/cli/internal/path_test.go @@ -17,8 +17,9 @@ package internal_test import ( "testing" - "github.com/kanisterio/kanister/pkg/kopia/cli/internal" "gopkg.in/check.v1" + + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" ) func TestPath(t *testing.T) { check.TestingT(t) } From 3030d2f682b09b09a141aeb76c3f21b3843a774a Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Thu, 29 Feb 2024 15:29:02 -0800 Subject: [PATCH 70/73] organize imports Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/kopia_test.go | 3 ++- pkg/kopia/cli/repository/repository_connect.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/kopia/cli/internal/kopia_test.go b/pkg/kopia/cli/internal/kopia_test.go index c168843d35..7d4c2602e4 100644 --- a/pkg/kopia/cli/internal/kopia_test.go +++ b/pkg/kopia/cli/internal/kopia_test.go @@ -3,9 +3,10 @@ package internal_test import ( "testing" + "gopkg.in/check.v1" + "github.com/kanisterio/kanister/pkg/kopia/cli/internal" "github.com/kanisterio/kanister/pkg/kopia/cli/internal/opts" - "gopkg.in/check.v1" ) func TestNewKopiaCommand(t *testing.T) { check.TestingT(t) } diff --git a/pkg/kopia/cli/repository/repository_connect.go b/pkg/kopia/cli/repository/repository_connect.go index a64639ac48..db03f9c5a4 100644 --- a/pkg/kopia/cli/repository/repository_connect.go +++ b/pkg/kopia/cli/repository/repository_connect.go @@ -17,11 +17,12 @@ package repository import ( "github.com/go-openapi/strfmt" + "github.com/kanisterio/safecli" + "github.com/kanisterio/kanister/pkg/kopia/cli/args" "github.com/kanisterio/kanister/pkg/kopia/cli/internal" "github.com/kanisterio/kanister/pkg/kopia/cli/internal/opts" "github.com/kanisterio/kanister/pkg/log" - "github.com/kanisterio/safecli" ) // ConnectArgs defines the arguments for the `kopia repository connect` command. From 868f5ee86bcb2b8ea3ac47e22eca94682fd69906 Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 1 Mar 2024 10:11:11 -0800 Subject: [PATCH 71/73] Add missed copyright headers Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/kopia_test.go | 14 ++++++++++++++ pkg/kopia/cli/internal/test/command_suite.go | 14 ++++++++++++++ pkg/kopia/cli/repository/data_test.go | 14 ++++++++++++++ .../repository/repository_connect_server_test.go | 14 ++++++++++++++ .../cli/repository/repository_connect_test.go | 14 ++++++++++++++ .../repository/repository_set_parameters_test.go | 14 ++++++++++++++ pkg/kopia/cli/repository/repository_status_test.go | 14 ++++++++++++++ pkg/kopia/cli/repository/storage/s3/arg_test.go | 14 ++++++++++++++ 8 files changed, 112 insertions(+) diff --git a/pkg/kopia/cli/internal/kopia_test.go b/pkg/kopia/cli/internal/kopia_test.go index 7d4c2602e4..32e5d7d67b 100644 --- a/pkg/kopia/cli/internal/kopia_test.go +++ b/pkg/kopia/cli/internal/kopia_test.go @@ -1,3 +1,17 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package internal_test import ( diff --git a/pkg/kopia/cli/internal/test/command_suite.go b/pkg/kopia/cli/internal/test/command_suite.go index 0eebf793ff..8ce690be20 100644 --- a/pkg/kopia/cli/internal/test/command_suite.go +++ b/pkg/kopia/cli/internal/test/command_suite.go @@ -1,3 +1,17 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package test import ( diff --git a/pkg/kopia/cli/repository/data_test.go b/pkg/kopia/cli/repository/data_test.go index 0b6ef70b29..6ad10df1a3 100644 --- a/pkg/kopia/cli/repository/data_test.go +++ b/pkg/kopia/cli/repository/data_test.go @@ -1,3 +1,17 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package repository import ( diff --git a/pkg/kopia/cli/repository/repository_connect_server_test.go b/pkg/kopia/cli/repository/repository_connect_server_test.go index 43a188637e..b50a03b0f6 100644 --- a/pkg/kopia/cli/repository/repository_connect_server_test.go +++ b/pkg/kopia/cli/repository/repository_connect_server_test.go @@ -1,3 +1,17 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package repository import ( diff --git a/pkg/kopia/cli/repository/repository_connect_test.go b/pkg/kopia/cli/repository/repository_connect_test.go index d3f8f338ea..9f8f0d7067 100644 --- a/pkg/kopia/cli/repository/repository_connect_test.go +++ b/pkg/kopia/cli/repository/repository_connect_test.go @@ -1,3 +1,17 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package repository import ( diff --git a/pkg/kopia/cli/repository/repository_set_parameters_test.go b/pkg/kopia/cli/repository/repository_set_parameters_test.go index bf9d460e61..5f7fe01a08 100644 --- a/pkg/kopia/cli/repository/repository_set_parameters_test.go +++ b/pkg/kopia/cli/repository/repository_set_parameters_test.go @@ -1,3 +1,17 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package repository import ( diff --git a/pkg/kopia/cli/repository/repository_status_test.go b/pkg/kopia/cli/repository/repository_status_test.go index 583d83dc92..68a11cfdd4 100644 --- a/pkg/kopia/cli/repository/repository_status_test.go +++ b/pkg/kopia/cli/repository/repository_status_test.go @@ -1,3 +1,17 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package repository import ( diff --git a/pkg/kopia/cli/repository/storage/s3/arg_test.go b/pkg/kopia/cli/repository/storage/s3/arg_test.go index 90776f5a4f..53f0a858ff 100644 --- a/pkg/kopia/cli/repository/storage/s3/arg_test.go +++ b/pkg/kopia/cli/repository/storage/s3/arg_test.go @@ -1,3 +1,17 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package s3 import ( From da5058df57c56cd0d7051996615878bf3a2bd7bc Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Fri, 1 Mar 2024 13:30:24 -0800 Subject: [PATCH 72/73] Fix s3 tests Signed-off-by: pavel.larkin --- pkg/kopia/cli/internal/test/arg_suit.go | 81 +++++++++ .../cli/repository/storage/s3/arg_test.go | 68 -------- .../cli/repository/storage/s3/s3_opts_test.go | 5 + .../cli/repository/storage/s3/s3_test.go | 157 +++++++++++------- 4 files changed, 179 insertions(+), 132 deletions(-) create mode 100644 pkg/kopia/cli/internal/test/arg_suit.go delete mode 100644 pkg/kopia/cli/repository/storage/s3/arg_test.go diff --git a/pkg/kopia/cli/internal/test/arg_suit.go b/pkg/kopia/cli/internal/test/arg_suit.go new file mode 100644 index 0000000000..21ccbf70e4 --- /dev/null +++ b/pkg/kopia/cli/internal/test/arg_suit.go @@ -0,0 +1,81 @@ +// Copyright 2024 The Kanister Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "github.com/kanisterio/safecli/test" + "gopkg.in/check.v1" + + intlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" + "github.com/kanisterio/kanister/pkg/log" +) + +// ArgumentTest extends test.ArgumentTest to include logger tests. +type ArgumentTest struct { + test.ArgumentTest + + Logger log.Logger // Logger is the logger to use for the test. (optional) + LoggerRegex []string // LoggerRegex is a list of regexs to match against the log output. (optional) +} + +// Test runs the test with the given command and checks the log output. +func (t *ArgumentTest) Test(c *check.C, cmd string) { + t.ArgumentTest.Test(c, cmd) + t.assertLog(c) +} + +// assertLog checks the log output against the expected regexs. +func (t *ArgumentTest) assertLog(c *check.C) { + if t.Logger == nil { + if len(t.LoggerRegex) > 0 { + c.Fatalf("t.Logger is nil but t.LoggerRegex is %#v", t.LoggerRegex) + } + return + } + + log, ok := t.Logger.(*intlog.StringLogger) + if !ok { + c.Fatalf("t.Logger is not *intlog.StringLogger") + } + if t.isEmptyLogExpected() { + cmtLog := check.Commentf("FAIL: log should be empty but got %#v", log) + c.Assert(len([]string(*log)), check.Equals, 0, cmtLog) + return + } + + // Check each regex. + for _, regex := range t.LoggerRegex { + cmtLog := check.Commentf("FAIL: %v\nlog %#v expected to match %#v", t.ArgumentTest.Name, log, regex) + c.Assert(log.MatchString(regex), check.Equals, true, cmtLog) + } +} + +// isEmptyLogExpected returns true if the test expects an empty log. +func (t *ArgumentTest) isEmptyLogExpected() bool { + return len(t.LoggerRegex) == 1 && t.LoggerRegex[0] == "" +} + +// ArgumentSuite defines a suite of tests for a single ArgumentTest. +type ArgumentSuite struct { + Cmd string // Cmd appends to the safecli.Builder before test if not empty. + Arguments []ArgumentTest // Tests to run. +} + +// TestArguments runs all tests in the suite. +func (s *ArgumentSuite) TestArguments(c *check.C) { + for _, arg := range s.Arguments { + arg.Test(c, s.Cmd) + } +} diff --git a/pkg/kopia/cli/repository/storage/s3/arg_test.go b/pkg/kopia/cli/repository/storage/s3/arg_test.go deleted file mode 100644 index 90776f5a4f..0000000000 --- a/pkg/kopia/cli/repository/storage/s3/arg_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package s3 - -import ( - "github.com/kanisterio/safecli/test" - "gopkg.in/check.v1" - - "github.com/kanisterio/kanister/pkg/kopia/cli/internal" - intlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" - "github.com/kanisterio/kanister/pkg/log" -) - -// ArgTest extends test.ArgumentTest to include logger tests. -type ArgTest struct { - test test.ArgumentTest - - location internal.Location // location is the location to use for the test. - repoPath string // repoPath is the repository path to use for the test. - Logger log.Logger // Logger is the logger to use for the test. (optional) - LoggerRegex []string // LoggerRegex is a list of regexs to match against the log output. (optional) -} - -// Test runs the test with the given command and checks the log output. -func (t *ArgTest) Test(c *check.C, cmd string) { - t.test.Argument = New(t.location, t.repoPath, t.Logger) - t.test.Test(c, cmd) - t.assertLog(c) -} - -// assertLog checks the log output against the expected regexs. -func (t *ArgTest) assertLog(c *check.C) { - if t.Logger == nil { - return - } - - log, ok := t.Logger.(*intlog.StringLogger) - if !ok { - c.Fatalf("t.Logger is not *intlog.StringLogger") - } - if t.isEmptyLogExpected() { - cmtLog := check.Commentf("FAIL: log should be empty but got %#v", log) - c.Assert(len([]string(*log)), check.Equals, 0, cmtLog) - return - } - - // Check each regex. - for _, regex := range t.LoggerRegex { - cmtLog := check.Commentf("FAIL: %v\nlog %#v expected to match %#v", t.test.Name, log, regex) - c.Assert(log.MatchString(regex), check.Equals, true, cmtLog) - } -} - -// isEmptyLogExpected returns true if the test expects an empty log. -func (t *ArgTest) isEmptyLogExpected() bool { - return len(t.LoggerRegex) == 1 && t.LoggerRegex[0] == "" -} - -// ArgSuite defines a suite of tests for a single ArgTest. -type ArgSuite struct { - Cmd string // Cmd appends to the safecli.Builder before test if not empty. - Arguments []ArgTest // Tests to run. -} - -// TestArguments runs all tests in the suite. -func (s *ArgSuite) TestArguments(c *check.C) { - for _, arg := range s.Arguments { - arg.Test(c, s.Cmd) - } -} diff --git a/pkg/kopia/cli/repository/storage/s3/s3_opts_test.go b/pkg/kopia/cli/repository/storage/s3/s3_opts_test.go index 3485f852d2..a903634819 100644 --- a/pkg/kopia/cli/repository/storage/s3/s3_opts_test.go +++ b/pkg/kopia/cli/repository/storage/s3/s3_opts_test.go @@ -27,6 +27,11 @@ import ( func TestS3Options(t *testing.T) { check.TestingT(t) } var _ = check.Suite(&test.ArgumentSuite{Cmd: "cmd", Arguments: []test.ArgumentTest{ + { + Name: "optRegion", + Argument: command.NewArguments(optRegion("region"), optRegion("")), + ExpectedCLI: []string{"cmd", "--region=region"}, + }, { Name: "optBucket with bucketname should return option", Argument: optBucket("bucketname"), diff --git a/pkg/kopia/cli/repository/storage/s3/s3_test.go b/pkg/kopia/cli/repository/storage/s3/s3_test.go index 8f688b2309..1bb9c77cf4 100644 --- a/pkg/kopia/cli/repository/storage/s3/s3_test.go +++ b/pkg/kopia/cli/repository/storage/s3/s3_test.go @@ -24,6 +24,8 @@ import ( "github.com/kanisterio/kanister/pkg/kopia/cli" "github.com/kanisterio/kanister/pkg/kopia/cli/internal" intlog "github.com/kanisterio/kanister/pkg/kopia/cli/internal/log" + inttest "github.com/kanisterio/kanister/pkg/kopia/cli/internal/test" + "github.com/kanisterio/kanister/pkg/log" ) func TestNewS3(t *testing.T) { check.TestingT(t) } @@ -38,98 +40,125 @@ func newLocation(prefix, endpoint, region, bucket string, skipSSLVerify bool) in } } -var _ = check.Suite(&ArgSuite{Cmd: "cmd", Arguments: []ArgTest{ +// s3test is a test case for NewS3. +type s3test struct { + Name string + Location internal.Location + RepoPath string + ExpectedCLI []string + ExpectedErr error + Logger log.Logger + LoggerRegex []string +} + +// newS3Test creates a new test case for NewS3. +func newS3Test(s3t s3test) inttest.ArgumentTest { + return inttest.ArgumentTest{ + ArgumentTest: test.ArgumentTest{ + Name: s3t.Name, + Argument: New(s3t.Location, s3t.RepoPath, s3t.Logger), + ExpectedCLI: s3t.ExpectedCLI, + ExpectedErr: s3t.ExpectedErr, + }, + Logger: s3t.Logger, + LoggerRegex: s3t.LoggerRegex, + } +} + +// toArgTests converts a list of s3tests to a list of ArgumentTests. +func toArgTests(s3tests []s3test) []inttest.ArgumentTest { + argTests := make([]inttest.ArgumentTest, len(s3tests)) + for i, s3t := range s3tests { + argTests[i] = newS3Test(s3t) + } + return argTests +} + +var _ = check.Suite(&inttest.ArgumentSuite{Cmd: "cmd", Arguments: toArgTests([]s3test{ { - test: test.ArgumentTest{ - Name: "NewS3", - ExpectedCLI: []string{"cmd", "s3", - "--region=region", - "--bucket=bucket", - "--endpoint=endpoint/path", - "--prefix=prefix/repoPath/", - "--disable-tls", - "--disable-tls-verification", - }, + Name: "NewS3", + Location: newLocation("prefix", "http://endpoint/path/", "region", "bucket", true), + RepoPath: "repoPath", + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--endpoint=endpoint/path", + "--prefix=prefix/repoPath/", + "--disable-tls", + "--disable-tls-verification", }, - location: newLocation("prefix", "http://endpoint/path/", "region", "bucket", true), - repoPath: "repoPath", - Logger: &intlog.StringLogger{}, + Logger: &intlog.StringLogger{}, LoggerRegex: []string{ "Removing leading", "Removing trailing", }, }, { - test: test.ArgumentTest{ - Name: "NewS3 w/o logger should not panic", - ExpectedCLI: []string{"cmd", "s3", - "--region=region", - "--bucket=bucket", - "--endpoint=endpoint/path", - "--prefix=prefix/repoPath/", - "--disable-tls", - "--disable-tls-verification", - }, + Name: "NewS3 w/o logger should not panic", + Location: newLocation("prefix", "http://endpoint/path/", "region", "bucket", true), + RepoPath: "repoPath", + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--endpoint=endpoint/path", + "--prefix=prefix/repoPath/", + "--disable-tls", + "--disable-tls-verification", }, - location: newLocation("prefix", "http://endpoint/path/", "region", "bucket", true), - repoPath: "repoPath", + Logger: &intlog.StringLogger{}, }, { - test: test.ArgumentTest{ - Name: "NewS3 with empty repoPath and https endpoint", - ExpectedCLI: []string{"cmd", "s3", - "--region=region", - "--bucket=bucket", - "--endpoint=endpoint/path", - "--prefix=prefix/", - }, + Name: "NewS3 with empty repoPath and https endpoint", + Location: newLocation("prefix", "https://endpoint/path/", "region", "bucket", false), + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--endpoint=endpoint/path", + "--prefix=prefix/", }, - location: newLocation("prefix", "https://endpoint/path/", "region", "bucket", false), - repoPath: "", - Logger: &intlog.StringLogger{}, + Logger: &intlog.StringLogger{}, LoggerRegex: []string{ "Removing leading", "Removing trailing", }, }, { - test: test.ArgumentTest{ - Name: "NewS3 with empty repoPath and endpoint", - ExpectedCLI: []string{"cmd", "s3", - "--region=region", - "--bucket=bucket", - "--prefix=prefix/", - "--disable-tls-verification", - }, + Name: "NewS3 with empty repoPath and endpoint", + Location: newLocation("prefix", "", "region", "bucket", true), + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--prefix=prefix/", + "--disable-tls-verification", }, - location: newLocation("prefix", "", "region", "bucket", true), - repoPath: "", Logger: &intlog.StringLogger{}, LoggerRegex: []string{""}, // no output expected }, { - test: test.ArgumentTest{ - Name: "NewS3 with empty repoPath, prefix and endpoint", - ExpectedCLI: []string{"cmd", "s3", - "--region=region", - "--bucket=bucket", - "--prefix=", - "--disable-tls-verification", - }, + Name: "NewS3 with empty repoPath, prefix and endpoint", + Location: newLocation("", "", "region", "bucket", true), + ExpectedCLI: []string{"cmd", "s3", + "--region=region", + "--bucket=bucket", + "--prefix=", + "--disable-tls-verification", }, - location: newLocation("", "", "region", "bucket", true), - repoPath: "", Logger: &intlog.StringLogger{}, LoggerRegex: []string{""}, // no output expected }, { - test: test.ArgumentTest{ - Name: "NewS3 with empty repoPath, prefix, endpoint and bucket", - ExpectedErr: cli.ErrInvalidBucketName, - }, - location: internal.Location{}, - repoPath: "", + Name: "NewS3 with empty repoPath, prefix, endpoint and bucket", + ExpectedErr: cli.ErrInvalidBucketName, Logger: &intlog.StringLogger{}, LoggerRegex: []string{""}, // no output expected }, -}}) + { + Name: "NewS3 with empty logger should not panic", + Location: newLocation("", "https://endpoint/path/", "", "bucket", false), + ExpectedCLI: []string{"cmd", "s3", + "--bucket=bucket", + "--endpoint=endpoint/path", + "--prefix=", + }, + }, +})}) From 259d333790b16376b4d8fb1c1289a58af2cec45c Mon Sep 17 00:00:00 2001 From: "pavel.larkin" Date: Tue, 5 Mar 2024 15:33:24 -0800 Subject: [PATCH 73/73] Fix imports Signed-off-by: pavel.larkin --- pkg/kopia/cli/repository/repository_status.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/kopia/cli/repository/repository_status.go b/pkg/kopia/cli/repository/repository_status.go index fa74990de5..c13dfd37ad 100644 --- a/pkg/kopia/cli/repository/repository_status.go +++ b/pkg/kopia/cli/repository/repository_status.go @@ -17,11 +17,10 @@ package repository import ( "github.com/kanisterio/safecli" - "github.com/kanisterio/kanister/pkg/log" - "github.com/kanisterio/kanister/pkg/kopia/cli/args" "github.com/kanisterio/kanister/pkg/kopia/cli/internal" "github.com/kanisterio/kanister/pkg/kopia/cli/internal/opts" + "github.com/kanisterio/kanister/pkg/log" ) // StatusArgs defines the arguments for the `kopia repository status ...` command.