Add kopia repository storage args generation utilities (#1696)

* Add utility to read file and directory from a pod

* Add copyright to new files

* Address review comments - fix error message and add comments

* Add Environment to PodOptions

* Add SecretMounts to PodOptions

* Add tests for new fields added to PodOptions

* Add function comment

* Revert file reader and SecretMounts implementation

* Remove stale test

* Add kopia repository storage args generation utilities

* Address review comments, add copyright to new files

* Address review comments - Update tests to use flags

* Use map[string][]byte instead of map[string]string

* Add test for ResolveS3Endpoint

* Remove unused field

* Address review comments - rename variables and functions

* Address review commnents - add needed comment and rename

* Merge common flags, remove unused flags

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

pkg/kopia/command/storage/azure_args.go

@@ -0,0 +1,35 @@
+// Copyright 2022 The Kanister Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"github.com/kanisterio/kanister/pkg/logsafe"
+)
+
+const (
+	azureSubCommand    = "azure"
+	azureContainerFlag = "--container"
+)
+
+func azureArgs(location map[string][]byte, repoPathPrefix string) logsafe.Cmd {
+	// Append prefix from the location to the repository path prefix, if specified
+	fullRepoPathPrefix := GenerateFullRepoPath(getPrefixFromMap(location), repoPathPrefix)
+
+	args := logsafe.NewLoggable(azureSubCommand)
+	args = args.AppendLoggableKV(azureContainerFlag, getBucketNameFromMap(location))
+	args = args.AppendLoggableKV(prefixFlag, fullRepoPathPrefix)
+
+	return args
+}

pkg/kopia/command/storage/azure_args_test.go

@@ -0,0 +1,43 @@
+// Copyright 2022 The Kanister Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"fmt"
+
+	"gopkg.in/check.v1"
+)
+
+func (s *StorageUtilsSuite) TestAzureArgsUtil(c *check.C) {
+	repoPathPrefix := "dir/sub-dir"
+	for _, tc := range []struct {
+		location        map[string][]byte
+		expectedCommand string
+	}{
+		{
+			location: map[string][]byte{
+				bucketKey: []byte("test-bucket"),
+				prefixKey: []byte("test-prefix"),
+			},
+			expectedCommand: fmt.Sprint(azureSubCommand,
+				fmt.Sprintf(" %s=%s ", azureContainerFlag, "test-bucket"),
+				fmt.Sprintf("%s=%s", prefixFlag, fmt.Sprintf("test-prefix/%s/", repoPathPrefix)),
+			),
+		},
+	} {
+		cmd := azureArgs(tc.location, repoPathPrefix)
+		c.Assert(cmd.String(), check.Equals, tc.expectedCommand)
+	}
+}

pkg/kopia/command/storage/filesystem_args.go

@@ -0,0 +1,34 @@
+// Copyright 2022 The Kanister Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"github.com/kanisterio/kanister/pkg/logsafe"
+)
+
+const (
+	filesystemSubCommand = "filesystem"
+	pathFlag             = "--path"
+	// DefaultFSMountPath is the mount path for the file store PVC on Kopia API server
+	DefaultFSMountPath = "/mnt/data"
+)
+
+func filesystemArgs(location map[string][]byte, repoPathPrefix string) logsafe.Cmd {
+	// Append prefix from the location to the repository path prefix, if specified
+	fullRepoPathPrefix := GenerateFullRepoPath(getPrefixFromMap(location), repoPathPrefix)
+
+	args := logsafe.NewLoggable(filesystemSubCommand)
+	return args.AppendLoggableKV(pathFlag, DefaultFSMountPath+"/"+fullRepoPathPrefix)
+}

pkg/kopia/command/storage/filesystem_args_test.go

@@ -0,0 +1,49 @@
+// Copyright 2022 The Kanister Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"fmt"
+
+	"gopkg.in/check.v1"
+)
+
+func (s *StorageUtilsSuite) TestFilesystemArgsUtil(c *check.C) {
+	for _, tc := range []struct {
+		prefix         string
+		repoPathPrefix string
+		expectedPath   string
+	}{
+		{
+			prefix:         "",
+			repoPathPrefix: "dir1/subdir/",
+			expectedPath:   fmt.Sprintf("%s/dir1/subdir/", DefaultFSMountPath),
+		},
+		{
+			prefix:         "test-prefix",
+			repoPathPrefix: "dir1/subdir/",
+			expectedPath:   fmt.Sprintf("%s/test-prefix/dir1/subdir/", DefaultFSMountPath),
+		},
+	} {
+		sec := map[string][]byte{
+			prefixKey: []byte(tc.prefix),
+		}
+		args := filesystemArgs(sec, tc.repoPathPrefix)
+		expectedValue := fmt.Sprint(
+			filesystemSubCommand,
+			fmt.Sprintf(" %s=%s", pathFlag, tc.expectedPath))
+		c.Assert(args.String(), check.Equals, expectedValue)
+	}
+}

pkg/kopia/command/storage/gcs_args.go

@@ -0,0 +1,35 @@
+// Copyright 2022 The Kanister Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"github.com/kanisterio/kanister/pkg/consts"
+	"github.com/kanisterio/kanister/pkg/logsafe"
+)
+
+const (
+	gcsSubCommand       = "gcs"
+	credentialsFileFlag = "--credentials-file"
+)
+
+func gcsArgs(location map[string][]byte, repoPathPrefix string) logsafe.Cmd {
+	// Append prefix from the location to the repository path prefix, if specified
+	fullRepoPathPrefix := GenerateFullRepoPath(getPrefixFromMap(location), repoPathPrefix)
+
+	args := logsafe.NewLoggable(gcsSubCommand)
+	args = args.AppendLoggableKV(bucketFlag, getBucketNameFromMap(location))
+	args = args.AppendLoggableKV(credentialsFileFlag, consts.GoogleCloudCredsFilePath)
+	return args.AppendLoggableKV(prefixFlag, fullRepoPathPrefix)
+}

pkg/kopia/command/storage/gcs_args_test.go

@@ -0,0 +1,36 @@
+// Copyright 2022 The Kanister Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"fmt"
+
+	"gopkg.in/check.v1"
+)
+
+func (s *StorageUtilsSuite) TestGCSArgsUtil(c *check.C) {
+	locSecret := map[string][]byte{
+		prefixKey: []byte("test-prefix"),
+		bucketKey: []byte("test-bucket"),
+	}
+	repoPathPrefix := "dir/sub-dir"
+	cmd := gcsArgs(locSecret, repoPathPrefix)
+	c.Assert(cmd.String(), check.Equals, fmt.Sprint(
+		gcsSubCommand,
+		fmt.Sprintf(" --%s=%s", bucketKey, locSecret[bucketKey]),
+		fmt.Sprintf(" %s=/tmp/creds.txt", credentialsFileFlag),
+		fmt.Sprintf(" --%s=%s/%s/", prefixKey, locSecret[prefixKey], repoPathPrefix),
+	))
+}

pkg/kopia/command/storage/s3_args.go

@@ -0,0 +1,80 @@
+// Copyright 2022 The Kanister Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"strings"
+
+	"github.com/kanisterio/kanister/pkg/log"
+	"github.com/kanisterio/kanister/pkg/logsafe"
+)
+
+const (
+	s3SubCommand           = "s3"
+	s3DisableTLSFlag       = "--disable-tls"
+	s3DisableTLSVerifyFlag = "--disable-tls-verification"
+	s3EndpointFlag         = "--endpoint"
+	s3RegionFlag           = "--region"
+)
+
+func s3Args(location map[string][]byte, repoPathPrefix string) logsafe.Cmd {
+	args := logsafe.NewLoggable(s3SubCommand)
+	args = args.AppendLoggableKV(bucketFlag, getBucketNameFromMap(location))
+
+	e := getEndpointFromMap(location)
+	if e != "" {
+		s3Endpoint := ResolveS3Endpoint(e)
+		args = args.AppendLoggableKV(s3EndpointFlag, s3Endpoint)
+
+		if HttpInsecureEndpoint(e) {
+			args = args.AppendLoggable(s3DisableTLSFlag)
+		}
+	}
+
+	// Append prefix from the location to the repository path prefix, if specified
+	fullRepoPathPrefix := GenerateFullRepoPath(getPrefixFromMap(location), repoPathPrefix)
+
+	args = args.AppendLoggableKV(prefixFlag, fullRepoPathPrefix)
+
+	if checkSkipSSLVerifyFromMap(location) {
+		args = args.AppendLoggable(s3DisableTLSVerifyFlag)
+	}
+
+	region := getRegionFromMap(location)
+	if region != "" {
+		args = args.AppendLoggableKV(s3RegionFlag, region)
+	}
+
+	return args
+}
+
+// ResolveS3Endpoint removes the trailing slash and
+// protocol from provided endpoint and returns the absolute
+// endpoint string
+func ResolveS3Endpoint(endpoint string) string {
+	if strings.HasSuffix(endpoint, "/") {
+		log.Debug().Print("Removing trailing slashes from the endpoint")
+		endpoint = strings.TrimRight(endpoint, "/")
+	}
+	sp := strings.SplitN(endpoint, "://", 2)
+	if len(sp) > 1 {
+		log.Debug().Print("Removing leading protocol from the endpoint")
+	}
+	return sp[len(sp)-1]
+}
+
+func HttpInsecureEndpoint(endpoint string) bool {
+	return strings.HasPrefix(endpoint, "http:")
+}