-
Notifications
You must be signed in to change notification settings - Fork 152
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add kopia repository storage args generation utilities (#1696)
* Add utility to read file and directory from a pod * Add copyright to new files * Address review comments - fix error message and add comments * Add Environment to PodOptions * Add SecretMounts to PodOptions * Add tests for new fields added to PodOptions * Add function comment * Revert file reader and SecretMounts implementation * Remove stale test * Add kopia repository storage args generation utilities * Address review comments, add copyright to new files * Address review comments - Update tests to use flags * Use map[string][]byte instead of map[string]string * Add test for ResolveS3Endpoint * Remove unused field * Address review comments - rename variables and functions * Address review commnents - add needed comment and rename * Merge common flags, remove unused flags Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
- Loading branch information
1 parent
734eb66
commit f65bc3c
Showing
15 changed files
with
885 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,35 @@ | ||
// Copyright 2022 The Kanister Authors. | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package storage | ||
|
||
import ( | ||
"github.com/kanisterio/kanister/pkg/logsafe" | ||
) | ||
|
||
const ( | ||
azureSubCommand = "azure" | ||
azureContainerFlag = "--container" | ||
) | ||
|
||
func azureArgs(location map[string][]byte, repoPathPrefix string) logsafe.Cmd { | ||
// Append prefix from the location to the repository path prefix, if specified | ||
fullRepoPathPrefix := GenerateFullRepoPath(getPrefixFromMap(location), repoPathPrefix) | ||
|
||
args := logsafe.NewLoggable(azureSubCommand) | ||
args = args.AppendLoggableKV(azureContainerFlag, getBucketNameFromMap(location)) | ||
args = args.AppendLoggableKV(prefixFlag, fullRepoPathPrefix) | ||
|
||
return args | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,43 @@ | ||
// Copyright 2022 The Kanister Authors. | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package storage | ||
|
||
import ( | ||
"fmt" | ||
|
||
"gopkg.in/check.v1" | ||
) | ||
|
||
func (s *StorageUtilsSuite) TestAzureArgsUtil(c *check.C) { | ||
repoPathPrefix := "dir/sub-dir" | ||
for _, tc := range []struct { | ||
location map[string][]byte | ||
expectedCommand string | ||
}{ | ||
{ | ||
location: map[string][]byte{ | ||
bucketKey: []byte("test-bucket"), | ||
prefixKey: []byte("test-prefix"), | ||
}, | ||
expectedCommand: fmt.Sprint(azureSubCommand, | ||
fmt.Sprintf(" %s=%s ", azureContainerFlag, "test-bucket"), | ||
fmt.Sprintf("%s=%s", prefixFlag, fmt.Sprintf("test-prefix/%s/", repoPathPrefix)), | ||
), | ||
}, | ||
} { | ||
cmd := azureArgs(tc.location, repoPathPrefix) | ||
c.Assert(cmd.String(), check.Equals, tc.expectedCommand) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
// Copyright 2022 The Kanister Authors. | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package storage | ||
|
||
import ( | ||
"github.com/kanisterio/kanister/pkg/logsafe" | ||
) | ||
|
||
const ( | ||
filesystemSubCommand = "filesystem" | ||
pathFlag = "--path" | ||
// DefaultFSMountPath is the mount path for the file store PVC on Kopia API server | ||
DefaultFSMountPath = "/mnt/data" | ||
) | ||
|
||
func filesystemArgs(location map[string][]byte, repoPathPrefix string) logsafe.Cmd { | ||
// Append prefix from the location to the repository path prefix, if specified | ||
fullRepoPathPrefix := GenerateFullRepoPath(getPrefixFromMap(location), repoPathPrefix) | ||
|
||
args := logsafe.NewLoggable(filesystemSubCommand) | ||
return args.AppendLoggableKV(pathFlag, DefaultFSMountPath+"/"+fullRepoPathPrefix) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,49 @@ | ||
// Copyright 2022 The Kanister Authors. | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package storage | ||
|
||
import ( | ||
"fmt" | ||
|
||
"gopkg.in/check.v1" | ||
) | ||
|
||
func (s *StorageUtilsSuite) TestFilesystemArgsUtil(c *check.C) { | ||
for _, tc := range []struct { | ||
prefix string | ||
repoPathPrefix string | ||
expectedPath string | ||
}{ | ||
{ | ||
prefix: "", | ||
repoPathPrefix: "dir1/subdir/", | ||
expectedPath: fmt.Sprintf("%s/dir1/subdir/", DefaultFSMountPath), | ||
}, | ||
{ | ||
prefix: "test-prefix", | ||
repoPathPrefix: "dir1/subdir/", | ||
expectedPath: fmt.Sprintf("%s/test-prefix/dir1/subdir/", DefaultFSMountPath), | ||
}, | ||
} { | ||
sec := map[string][]byte{ | ||
prefixKey: []byte(tc.prefix), | ||
} | ||
args := filesystemArgs(sec, tc.repoPathPrefix) | ||
expectedValue := fmt.Sprint( | ||
filesystemSubCommand, | ||
fmt.Sprintf(" %s=%s", pathFlag, tc.expectedPath)) | ||
c.Assert(args.String(), check.Equals, expectedValue) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,35 @@ | ||
// Copyright 2022 The Kanister Authors. | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package storage | ||
|
||
import ( | ||
"github.com/kanisterio/kanister/pkg/consts" | ||
"github.com/kanisterio/kanister/pkg/logsafe" | ||
) | ||
|
||
const ( | ||
gcsSubCommand = "gcs" | ||
credentialsFileFlag = "--credentials-file" | ||
) | ||
|
||
func gcsArgs(location map[string][]byte, repoPathPrefix string) logsafe.Cmd { | ||
// Append prefix from the location to the repository path prefix, if specified | ||
fullRepoPathPrefix := GenerateFullRepoPath(getPrefixFromMap(location), repoPathPrefix) | ||
|
||
args := logsafe.NewLoggable(gcsSubCommand) | ||
args = args.AppendLoggableKV(bucketFlag, getBucketNameFromMap(location)) | ||
args = args.AppendLoggableKV(credentialsFileFlag, consts.GoogleCloudCredsFilePath) | ||
return args.AppendLoggableKV(prefixFlag, fullRepoPathPrefix) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,36 @@ | ||
// Copyright 2022 The Kanister Authors. | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package storage | ||
|
||
import ( | ||
"fmt" | ||
|
||
"gopkg.in/check.v1" | ||
) | ||
|
||
func (s *StorageUtilsSuite) TestGCSArgsUtil(c *check.C) { | ||
locSecret := map[string][]byte{ | ||
prefixKey: []byte("test-prefix"), | ||
bucketKey: []byte("test-bucket"), | ||
} | ||
repoPathPrefix := "dir/sub-dir" | ||
cmd := gcsArgs(locSecret, repoPathPrefix) | ||
c.Assert(cmd.String(), check.Equals, fmt.Sprint( | ||
gcsSubCommand, | ||
fmt.Sprintf(" --%s=%s", bucketKey, locSecret[bucketKey]), | ||
fmt.Sprintf(" %s=/tmp/creds.txt", credentialsFileFlag), | ||
fmt.Sprintf(" --%s=%s/%s/", prefixKey, locSecret[prefixKey], repoPathPrefix), | ||
)) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,80 @@ | ||
// Copyright 2022 The Kanister Authors. | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package storage | ||
|
||
import ( | ||
"strings" | ||
|
||
"github.com/kanisterio/kanister/pkg/log" | ||
"github.com/kanisterio/kanister/pkg/logsafe" | ||
) | ||
|
||
const ( | ||
s3SubCommand = "s3" | ||
s3DisableTLSFlag = "--disable-tls" | ||
s3DisableTLSVerifyFlag = "--disable-tls-verification" | ||
s3EndpointFlag = "--endpoint" | ||
s3RegionFlag = "--region" | ||
) | ||
|
||
func s3Args(location map[string][]byte, repoPathPrefix string) logsafe.Cmd { | ||
args := logsafe.NewLoggable(s3SubCommand) | ||
args = args.AppendLoggableKV(bucketFlag, getBucketNameFromMap(location)) | ||
|
||
e := getEndpointFromMap(location) | ||
if e != "" { | ||
s3Endpoint := ResolveS3Endpoint(e) | ||
args = args.AppendLoggableKV(s3EndpointFlag, s3Endpoint) | ||
|
||
if HttpInsecureEndpoint(e) { | ||
args = args.AppendLoggable(s3DisableTLSFlag) | ||
} | ||
} | ||
|
||
// Append prefix from the location to the repository path prefix, if specified | ||
fullRepoPathPrefix := GenerateFullRepoPath(getPrefixFromMap(location), repoPathPrefix) | ||
|
||
args = args.AppendLoggableKV(prefixFlag, fullRepoPathPrefix) | ||
|
||
if checkSkipSSLVerifyFromMap(location) { | ||
args = args.AppendLoggable(s3DisableTLSVerifyFlag) | ||
} | ||
|
||
region := getRegionFromMap(location) | ||
if region != "" { | ||
args = args.AppendLoggableKV(s3RegionFlag, region) | ||
} | ||
|
||
return args | ||
} | ||
|
||
// ResolveS3Endpoint removes the trailing slash and | ||
// protocol from provided endpoint and returns the absolute | ||
// endpoint string | ||
func ResolveS3Endpoint(endpoint string) string { | ||
if strings.HasSuffix(endpoint, "/") { | ||
log.Debug().Print("Removing trailing slashes from the endpoint") | ||
endpoint = strings.TrimRight(endpoint, "/") | ||
} | ||
sp := strings.SplitN(endpoint, "://", 2) | ||
if len(sp) > 1 { | ||
log.Debug().Print("Removing leading protocol from the endpoint") | ||
} | ||
return sp[len(sp)-1] | ||
} | ||
|
||
func HttpInsecureEndpoint(endpoint string) bool { | ||
return strings.HasPrefix(endpoint, "http:") | ||
} |
Oops, something went wrong.