-
Notifications
You must be signed in to change notification settings - Fork 152
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #189 from kanisterio/sync
Add Chronicle CLI; RestoreData in parallel; Add Kind to Makefile
- Loading branch information
Showing
21 changed files
with
776 additions
and
128 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
package chronicle | ||
|
||
import ( | ||
"bytes" | ||
"context" | ||
"io" | ||
"io/ioutil" | ||
|
||
"github.com/kanisterio/kanister/pkg/location" | ||
"github.com/kanisterio/kanister/pkg/param" | ||
"github.com/pkg/errors" | ||
) | ||
|
||
func Pull(ctx context.Context, target io.Writer, p param.Profile, manifest string) error { | ||
// Read manifest | ||
buf := bytes.NewBuffer(nil) | ||
location.Read(ctx, buf, p, manifest) | ||
// Read Data | ||
data, err := ioutil.ReadAll(buf) | ||
if err != nil { | ||
return errors.Wrap(err, "Could not read chronicle manifest") | ||
} | ||
return location.Read(ctx, target, p, string(data)) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,147 @@ | ||
package chronicle | ||
|
||
import ( | ||
"context" | ||
"encoding/json" | ||
"fmt" | ||
"io/ioutil" | ||
"os" | ||
"os/exec" | ||
"os/signal" | ||
"strings" | ||
"syscall" | ||
"time" | ||
|
||
"github.com/pkg/errors" | ||
log "github.com/sirupsen/logrus" | ||
|
||
"github.com/kanisterio/kanister/pkg/envdir" | ||
"github.com/kanisterio/kanister/pkg/location" | ||
"github.com/kanisterio/kanister/pkg/param" | ||
) | ||
|
||
type PushParams struct { | ||
ProfilePath string | ||
ArtifactPath string | ||
Frequency time.Duration | ||
EnvDir string | ||
Command []string | ||
} | ||
|
||
func (p PushParams) Validate() error { | ||
return nil | ||
} | ||
|
||
func Push(p PushParams) error { | ||
log.Infof("%#v", p) | ||
ctx := setupSignalHandler(context.Background()) | ||
var i int | ||
for { | ||
start := time.Now().UTC() | ||
|
||
if err := push(ctx, p, i); err != nil { | ||
return err | ||
} | ||
|
||
end := time.Now().UTC() | ||
sleep := p.Frequency - end.Sub(start) | ||
select { | ||
case <-ctx.Done(): | ||
return nil | ||
case <-time.After(sleep): | ||
} | ||
i++ | ||
} | ||
} | ||
|
||
func setupSignalHandler(ctx context.Context) context.Context { | ||
var can context.CancelFunc | ||
ctx, can = context.WithCancel(ctx) | ||
c := make(chan os.Signal, 2) | ||
signal.Notify(c, os.Interrupt, syscall.SIGTERM) | ||
go func() { | ||
<-c | ||
log.Infof("Shutting down process") | ||
can() | ||
<-c | ||
log.Infof("Killing process") | ||
os.Exit(1) | ||
}() | ||
return ctx | ||
} | ||
|
||
func push(ctx context.Context, p PushParams, ord int) error { | ||
// Read profile. | ||
prof, ok, err := readProfile(p.ProfilePath) | ||
if !ok || err != nil { | ||
return errors.Wrap(err, "") | ||
} | ||
|
||
// Get envdir values if set. | ||
var env []string | ||
if p.EnvDir != "" { | ||
var err error | ||
envdir.EnvDir(p.EnvDir) | ||
if err != nil { | ||
return err | ||
} | ||
} | ||
|
||
// Chronicle command w/ piped output. | ||
cmd := exec.CommandContext(ctx, "sh", "-c", strings.Join(p.Command, " ")) | ||
cmd.Env = append(cmd.Env, env...) | ||
out, err := cmd.StdoutPipe() | ||
if err != nil { | ||
return errors.Wrap(err, "Failed to open command pipe") | ||
} | ||
cmd.Stderr = os.Stderr | ||
cur := fmt.Sprintf("%s-%d", p.ArtifactPath, ord) | ||
// Write data to object store | ||
if err := cmd.Start(); err != nil { | ||
return errors.Wrap(err, "Failed to start chronicle pipe command") | ||
} | ||
if err := location.Write(ctx, out, prof, cur); err != nil { | ||
return errors.Wrap(err, "Failed to write command output to object storage") | ||
} | ||
if err := cmd.Wait(); err != nil { | ||
return errors.Wrap(err, "Chronicle pipe command failed") | ||
} | ||
|
||
// Write manifest pointing to new data | ||
man := strings.NewReader(cur) | ||
if err := location.Write(ctx, man, prof, p.ArtifactPath); err != nil { | ||
return errors.Wrap(err, "Failed to write command output to object storage") | ||
} | ||
// Delete old data | ||
prev := fmt.Sprintf("%s-%d", p.ArtifactPath, ord-1) | ||
location.Delete(ctx, prof, prev) | ||
return nil | ||
} | ||
|
||
func readProfile(path string) (p param.Profile, ok bool, err error) { | ||
var buf []byte | ||
buf, err = ioutil.ReadFile(path) | ||
switch { | ||
case os.IsNotExist(err): | ||
ok = true | ||
err = nil | ||
return | ||
case err != nil: | ||
err = errors.Wrap(err, "Failed to read profile") | ||
return | ||
} | ||
if err = json.Unmarshal(buf, &p); err != nil { | ||
err = errors.Wrap(err, "Failed to unmarshal profile") | ||
} else { | ||
ok = true | ||
} | ||
return | ||
} | ||
|
||
func writeProfile(path string, p param.Profile) error { | ||
buf, err := json.Marshal(p) | ||
if err != nil { | ||
return errors.Wrap(err, "Failed to write profile") | ||
} | ||
return ioutil.WriteFile(path, buf, os.ModePerm) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,39 @@ | ||
package chronicle | ||
|
||
import ( | ||
"context" | ||
"path/filepath" | ||
|
||
. "gopkg.in/check.v1" | ||
"k8s.io/apimachinery/pkg/util/rand" | ||
|
||
crv1alpha1 "github.com/kanisterio/kanister/pkg/apis/cr/v1alpha1" | ||
"github.com/kanisterio/kanister/pkg/objectstore" | ||
"github.com/kanisterio/kanister/pkg/testutil" | ||
) | ||
|
||
type ChroniclePushSuite struct{} | ||
|
||
var _ = Suite(&ChroniclePushSuite{}) | ||
|
||
func (s *ChroniclePushSuite) TestPush(c *C) { | ||
osType := objectstore.ProviderTypeS3 | ||
loc := crv1alpha1.Location{ | ||
Type: crv1alpha1.LocationTypeS3Compliant, | ||
Region: testutil.TestS3Region, | ||
Bucket: testutil.TestS3BucketName, | ||
} | ||
prof := *testutil.ObjectStoreProfileOrSkip(c, osType, loc) | ||
pp := filepath.Join(c.MkDir(), "profile.json") | ||
err := writeProfile(pp, prof) | ||
c.Assert(err, IsNil) | ||
|
||
p := PushParams{ | ||
ProfilePath: pp, | ||
ArtifactPath: rand.String(10), | ||
Command: []string{"echo hello"}, | ||
} | ||
ctx := context.Background() | ||
err = push(ctx, p, 0) | ||
c.Assert(err, IsNil) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,61 @@ | ||
package chronicle | ||
|
||
import ( | ||
"bytes" | ||
"context" | ||
"io/ioutil" | ||
"path/filepath" | ||
"strconv" | ||
"strings" | ||
"testing" | ||
|
||
. "gopkg.in/check.v1" | ||
"k8s.io/apimachinery/pkg/util/rand" | ||
|
||
crv1alpha1 "github.com/kanisterio/kanister/pkg/apis/cr/v1alpha1" | ||
"github.com/kanisterio/kanister/pkg/objectstore" | ||
"github.com/kanisterio/kanister/pkg/testutil" | ||
) | ||
|
||
// Hook up gocheck into the "go test" runner. | ||
func Test(t *testing.T) { TestingT(t) } | ||
|
||
type ChronicleSuite struct{} | ||
|
||
var _ = Suite(&ChronicleSuite{}) | ||
|
||
func (s *ChronicleSuite) TestPushPull(c *C) { | ||
osType := objectstore.ProviderTypeS3 | ||
loc := crv1alpha1.Location{ | ||
Type: crv1alpha1.LocationTypeS3Compliant, | ||
Region: testutil.TestS3Region, | ||
Bucket: testutil.TestS3BucketName, | ||
} | ||
prof := *testutil.ObjectStoreProfileOrSkip(c, osType, loc) | ||
pp := filepath.Join(c.MkDir(), "profile.json") | ||
err := writeProfile(pp, prof) | ||
c.Assert(err, IsNil) | ||
|
||
p := PushParams{ | ||
ProfilePath: pp, | ||
ArtifactPath: rand.String(10), | ||
} | ||
ctx := context.Background() | ||
|
||
for i := range make([]struct{}, 5) { | ||
// Write i to bucket | ||
p.Command = []string{"echo", strconv.Itoa(i)} | ||
err = push(ctx, p, i) | ||
c.Assert(err, IsNil) | ||
|
||
// Pull and check that we still get i | ||
buf := bytes.NewBuffer(nil) | ||
err = Pull(ctx, buf, prof, p.ArtifactPath) | ||
c.Assert(err, IsNil) | ||
s, err := ioutil.ReadAll(buf) | ||
c.Assert(err, IsNil) | ||
// Remove additional '\n' | ||
t := strings.TrimSuffix(string(s), "\n") | ||
c.Assert(t, Equals, strconv.Itoa(i)) | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.