-
Notifications
You must be signed in to change notification settings - Fork 152
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Implement Kopia storage helpers #2655
Changes from all commits
680e782
72011e7
0f635ad
d0a6dd1
ca853c1
9850f4a
246e1c1
419652c
24707e5
c029fc1
cff7220
550d124
fc918a0
9530c91
75fb036
9fdf94b
2c6cb6e
1a3ee2d
386e7e9
b815f63
bcafabb
6d61bb4
e6ddb8f
4e7ffd7
f16aea7
dcd6425
d3dfb31
d2ac987
b093421
6e71143
de3a52b
722b279
44c133e
1897a1e
33b1342
83292af
516a14c
cd03d1e
8cf36c7
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,72 @@ | ||
// Copyright 2024 The Kanister Authors. | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package internal | ||
|
||
import ( | ||
"strconv" | ||
"strings" | ||
|
||
rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" | ||
) | ||
|
||
// Location is a map of key-value pairs that represent different storage properties. | ||
type Location map[string][]byte | ||
|
||
// Type returns the location type. | ||
func (l Location) Type() rs.LocType { | ||
return rs.LocType(string(l[rs.TypeKey])) | ||
} | ||
|
||
// Region returns the location region. | ||
func (l Location) Region() string { | ||
return string(l[rs.RegionKey]) | ||
} | ||
|
||
// BucketName returns the location bucket name. | ||
func (l Location) BucketName() string { | ||
return string(l[rs.BucketKey]) | ||
} | ||
|
||
// Endpoint returns the location endpoint. | ||
func (l Location) Endpoint() string { | ||
return string(l[rs.EndpointKey]) | ||
} | ||
|
||
// Prefix returns the location prefix. | ||
func (l Location) Prefix() string { | ||
return string(l[rs.PrefixKey]) | ||
} | ||
|
||
// IsInsecureEndpoint returns true if the location endpoint is insecure/http. | ||
func (l Location) IsInsecureEndpoint() bool { | ||
return strings.HasPrefix(l.Endpoint(), "http:") | ||
} | ||
|
||
// HasSkipSSLVerify returns true if the location has skip SSL verification. | ||
func (l Location) HasSkipSSLVerify() bool { | ||
v, _ := strconv.ParseBool(string(l[rs.SkipSSLVerifyKey])) | ||
return v | ||
} | ||
|
||
// IsPointInTimeSupported returns true if the location supports point-in-time recovery. | ||
// Currently, only S3 and Azure support point-in-time recovery. | ||
func (l Location) IsPointInTypeSupported() bool { | ||
switch l.Type() { | ||
case rs.LocTypeAzure, rs.LocTypeS3, rs.LocTypes3Compliant: | ||
return true | ||
default: | ||
return false | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,135 @@ | ||
// Copyright 2024 The Kanister Authors. | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package internal_test | ||
|
||
import ( | ||
"testing" | ||
|
||
"gopkg.in/check.v1" | ||
|
||
"github.com/kanisterio/kanister/pkg/kopia/cli/internal" | ||
rs "github.com/kanisterio/kanister/pkg/secrets/repositoryserver" | ||
) | ||
|
||
func TestLocation(t *testing.T) { check.TestingT(t) } | ||
|
||
type LocationSuite struct{} | ||
|
||
var _ = check.Suite(&LocationSuite{}) | ||
|
||
func (s *LocationSuite) TestLocation(c *check.C) { | ||
type expected struct { | ||
Type rs.LocType | ||
Region string | ||
BucketName string | ||
Endpoint string | ||
Prefix string | ||
IsInsecure bool | ||
HasSkipSSLVerify bool | ||
IsPITSupported bool | ||
} | ||
|
||
tests := []struct { | ||
name string | ||
location internal.Location | ||
expected expected | ||
}{ | ||
{ | ||
name: "Test with no fields", | ||
location: internal.Location{}, | ||
expected: expected{}, | ||
}, | ||
{ | ||
name: "Test with all fields", | ||
location: internal.Location{ | ||
rs.TypeKey: []byte("Type1"), | ||
rs.RegionKey: []byte("Region1"), | ||
rs.BucketKey: []byte("Bucket1"), | ||
rs.EndpointKey: []byte("http://Endpoint1"), | ||
rs.PrefixKey: []byte("Prefix1"), | ||
rs.SkipSSLVerifyKey: []byte("true"), | ||
}, | ||
expected: expected{ | ||
Type: "Type1", | ||
Region: "Region1", | ||
BucketName: "Bucket1", | ||
Endpoint: "http://Endpoint1", | ||
Prefix: "Prefix1", | ||
IsInsecure: true, | ||
HasSkipSSLVerify: true, | ||
}, | ||
}, | ||
{ | ||
name: "Test PIT Support for S3 Compliant", | ||
location: internal.Location{ | ||
rs.TypeKey: []byte(rs.LocTypes3Compliant), | ||
}, | ||
expected: expected{ | ||
Type: "s3Compliant", | ||
IsPITSupported: true, | ||
}, | ||
}, | ||
{ | ||
name: "Test PIT Support for S3", | ||
location: internal.Location{ | ||
rs.TypeKey: []byte(rs.LocTypeS3), | ||
}, | ||
expected: expected{ | ||
Type: "s3", | ||
IsPITSupported: true, | ||
}, | ||
}, | ||
{ | ||
name: "Test PIT Support for Azure", | ||
location: internal.Location{ | ||
rs.TypeKey: []byte(rs.LocTypeAzure), | ||
}, | ||
expected: expected{ | ||
Type: "azure", | ||
IsPITSupported: true, | ||
}, | ||
}, | ||
{ | ||
name: "Test No PIT Support for GCS", | ||
location: internal.Location{ | ||
rs.TypeKey: []byte(rs.LocTypeGCS), | ||
}, | ||
expected: expected{ | ||
Type: "gcs", | ||
IsPITSupported: false, | ||
}, | ||
}, | ||
{ | ||
name: "Test No PIT Support for FS", | ||
location: internal.Location{ | ||
rs.TypeKey: []byte(rs.LocTypeFilestore), | ||
}, | ||
expected: expected{ | ||
Type: "filestore", | ||
IsPITSupported: false, | ||
}, | ||
}, | ||
} | ||
for _, test := range tests { | ||
c.Check(test.location.Type(), check.Equals, test.expected.Type) | ||
c.Check(test.location.Region(), check.Equals, test.expected.Region) | ||
c.Check(test.location.BucketName(), check.Equals, test.expected.BucketName) | ||
c.Check(test.location.Endpoint(), check.Equals, test.expected.Endpoint) | ||
c.Check(test.location.Prefix(), check.Equals, test.expected.Prefix) | ||
c.Check(test.location.IsInsecureEndpoint(), check.Equals, test.expected.IsInsecure) | ||
c.Check(test.location.HasSkipSSLVerify(), check.Equals, test.expected.HasSkipSSLVerify) | ||
c.Check(test.location.IsPointInTypeSupported(), check.Equals, test.expected.IsPITSupported) | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,45 @@ | ||
// Copyright 2024 The Kanister Authors. | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package storage | ||
|
||
import ( | ||
"context" | ||
"io" | ||
|
||
"github.com/kanisterio/kanister/pkg/field" | ||
"github.com/kanisterio/kanister/pkg/log" | ||
) | ||
|
||
// NopLogger is a logger that does nothing. | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why do we need this? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This Nop logger is used inside s3 storage as a default logger if no logger is provided. You don't need to write something like below:
and can simply use:
|
||
// TODO: Move to log package | ||
hairyhum marked this conversation as resolved.
Show resolved
Hide resolved
|
||
type NopLogger struct{} | ||
|
||
// Print does nothing. | ||
func (NopLogger) Print(msg string, fields ...field.M) { | ||
} | ||
|
||
// PrintTo does nothing. | ||
func (NopLogger) PrintTo(w io.Writer, msg string, fields ...field.M) { | ||
} | ||
|
||
// WithContext does nothing. | ||
func (NopLogger) WithContext(ctx context.Context) log.Logger { | ||
return &NopLogger{} | ||
} | ||
|
||
// WithError does nothing. | ||
func (NopLogger) WithError(err error) log.Logger { | ||
return &NopLogger{} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,28 @@ | ||
// Copyright 2022 The Kanister Authors. | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package internal | ||
|
||
import ( | ||
"path" | ||
) | ||
|
||
// GenerateFullRepoPath generates the full repository path. | ||
// If the location-specific prefix is empty, the repository-specific prefix is returned. | ||
func GenerateFullRepoPath(locPrefix, repoPathPrefix string) string { | ||
if locPrefix != "" { | ||
return path.Join(locPrefix, repoPathPrefix) + "/" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is trailing There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This code is copied from There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think it's needed, but will let @pavannd1 confirm. Anyways it's existing behaviour so should be good to merge, we can remove it if not required later? |
||
} | ||
return repoPathPrefix | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,52 @@ | ||
// Copyright 2024 The Kanister Authors. | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package internal_test | ||
|
||
import ( | ||
"testing" | ||
|
||
"gopkg.in/check.v1" | ||
|
||
"github.com/kanisterio/kanister/pkg/kopia/cli/internal" | ||
) | ||
|
||
func TestPath(t *testing.T) { check.TestingT(t) } | ||
|
||
type PathSuite struct{} | ||
|
||
var _ = check.Suite(&PathSuite{}) | ||
|
||
func (s *PathSuite) TestGenerateFullRepoPath(c *check.C) { | ||
tests := []struct { | ||
locPrefix string | ||
repoPathPrefix string | ||
expected string | ||
}{ | ||
{ | ||
locPrefix: "", | ||
repoPathPrefix: "repo", | ||
expected: "repo", | ||
}, | ||
{ | ||
locPrefix: "loc", | ||
repoPathPrefix: "repo", | ||
expected: "loc/repo/", | ||
}, | ||
} | ||
for _, test := range tests { | ||
got := internal.GenerateFullRepoPath(test.locPrefix, test.repoPathPrefix) | ||
c.Check(got, check.Equals, test.expected) | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Shouldn't we handle error?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I've implemented the same logic as it is in the current implementation:
strconv.ParseBool
returnsfalse
in the case of error.