Merge pull request #136 from kanisterio/sync

Bump Kanister version to 0.13.0; Object Store enhancements; BackupData: Capture and return SnapshotID from logs

README.md

@@ -114,7 +114,7 @@ found [here](https://hub.kubeapps.com/charts/stable/kanister-operator) and can
 be installed by specifying the `stable` chart prefix.
 
 ```bash
-helm install --name myrelease --namespace kanister stable/kanister-operator --set image.tag=0.12.0
+helm install --name myrelease --namespace kanister stable/kanister-operator --set image.tag=0.13.0
 ```
 
 > **Warning**: The Kanister chart found in `stable` may be behind the one in

docker/postgres-kanister-tools/Dockerfile

@@ -11,6 +11,6 @@ RUN apk -v --update add --no-cache python py-pip groff less && \
     apk -v --purge del py-pip && \
     rm -f /var/cache/apk/*
 
-COPY --from=kanisterio/kanister-tools:0.12.0 /usr/local/bin/kando /usr/local/bin/kando
+COPY --from=kanisterio/kanister-tools:0.13.0 /usr/local/bin/kando /usr/local/bin/kando
 
 CMD ["tail", "-f", "/dev/null"]

docs/functions.rst

@@ -351,7 +351,7 @@ Example:
     args:
       namespace: "{{ .Deployment.Namespace }}"
       pod: "{{ index .Deployment.Pods 0 }}"
-      image: kanisterio/kanister-tools:0.12.0
+      image: kanisterio/kanister-tools:0.13.0
       backupArtifactPrefix: s3-bucket/path/artifactPrefix
       backupIdentifier: "{{ .Time }}"
 

docs/helm.rst

@@ -26,7 +26,7 @@ You also need to install the Kanister controller
 
 .. code-block:: console
 
-   $ helm install --name myrelease --namespace kanister kanister/kanister-operator --set image.tag=0.12.0
+   $ helm install --name myrelease --namespace kanister kanister/kanister-operator --set image.tag=0.13.0
 
 Kanister-Enabled Applications
 =============================

docs/index.rst

@@ -66,7 +66,7 @@ and backup to an AWS S3 bucket.
   helm repo add kanister http://charts.kanister.io
 
   # Install the Kanister Controller
-  helm install --name myrelease --namespace kanister kanister/kanister-operator --set image.tag=0.12.0
+  helm install --name myrelease --namespace kanister kanister/kanister-operator --set image.tag=0.13.0
 
   # Install MySQL and configure its Kanister Blueprint.
   # Also create a Profile CR that can be used in ActionSets

docs/install.rst

@@ -31,7 +31,7 @@ This will install the Kanister controller in the `kanister` namespace
    $ helm repo add kanister https://charts.kanister.io/
 
    # Install the Kanister operator controller using helm
-   $ helm install --name myrelease --namespace kanister kanister/kanister-operator --set image.tag=0.12.0
+   $ helm install --name myrelease --namespace kanister kanister/kanister-operator --set image.tag=0.13.0
 
 
 Building and Deploying from Source

docs/tooling.rst

@@ -384,7 +384,7 @@ These tools, especially `kando` are meant to be invoked inside containers via
 Blueprints. Although suggest using the released image when possible, we've also
 made it simple to add these tools to your container.
 
-The released image, `kanisterio/kanister-tools:0.12.0`, is hosted by
+The released image, `kanisterio/kanister-tools:0.13.0`, is hosted by
 `dockerhub <https://cloud.docker.com/swarm/kanisterio/repository/docker/kanisterio/kanister-tools/general>`_.
 
 The Dockerfile for this image is in the

examples/helm/kanister/kanister-elasticsearch/image/Dockerfile

@@ -1,5 +1,5 @@
 
-FROM kanisterio/kanister-tools:0.12.0
+FROM kanisterio/kanister-tools:0.13.0
 MAINTAINER Supriya Kharade <supriya@kasten.io>
 
 COPY esdump-setup.sh /esdump-setup.sh

examples/helm/kanister/kanister-elasticsearch/kanister/elasticsearch-blueprint.yaml

@@ -10,7 +10,7 @@ actions:
       name: takeDataSnapshot
       args:
         namespace: "{{ .StatefulSet.Namespace }}"
-        image: "kanisterio/es-sidecar:0.12.0"
+        image: "kanisterio/es-sidecar:0.13.0"
         command:
           - bash
           - -o
@@ -37,7 +37,7 @@ actions:
       name: restoreSnapshots
       args:
         namespace: "{{ .StatefulSet.Namespace }}"
-        image: "kanisterio/es-sidecar:0.12.0"
+        image: "kanisterio/es-sidecar:0.13.0"
         command:
           - bash
           - -o
@@ -63,7 +63,7 @@ actions:
       name: deleteSnapshots
       args:
         namespace: "{{ .StatefulSet.Namespace }}"
-        image: "kanisterio/es-sidecar:0.12.0"
+        image: "kanisterio/es-sidecar:0.13.0"
         command:
           - bash
           - -o

examples/helm/kanister/kanister-mongodb-replicaset/image/Dockerfile

@@ -7,6 +7,6 @@ ADD . /kanister
 
 RUN /kanister/install.sh && rm -rf /kanister && rm -rf /tmp && mkdir /tmp
 
-COPY --from=kanisterio/kanister-tools:0.12.0 /usr/local/bin/kando /usr/local/bin/kando
+COPY --from=kanisterio/kanister-tools:0.13.0 /usr/local/bin/kando /usr/local/bin/kando
 
 CMD ["tail", "-f", "/dev/null"]

examples/helm/kanister/kanister-mongodb-replicaset/templates/mongodb-statefulset.yaml

@@ -139,7 +139,7 @@ spec:
               mountPath: /data/db
       containers:
         - name: kanister-sidecar
-          image: kanisterio/mongo-sidecar:0.12.0
+          image: kanisterio/mongo-sidecar:0.13.0
           imagePullPolicy: Always
           env:
           {{- if .Values.auth.enabled }}

examples/helm/kanister/kanister-mysql/image/Dockerfile

@@ -7,6 +7,6 @@ ADD . /kanister
 
 RUN /kanister/install.sh && rm -rf /kanister && rm -rf /tmp && mkdir /tmp
 
-COPY --from=kanisterio/kanister-tools:0.12.0 /usr/local/bin/kando /usr/local/bin/kando
+COPY --from=kanisterio/kanister-tools:0.13.0 /usr/local/bin/kando /usr/local/bin/kando
 
 CMD ["tail", "-f", "/dev/null"]

examples/helm/kanister/kanister-mysql/templates/deployment.yaml

@@ -100,7 +100,7 @@ spec:
           mountPath: /etc/mysql/conf.d
         {{- end }}
       - name: kanister-sidecar
-        image: kanisterio/mysql-sidecar:0.12.0
+        image: kanisterio/mysql-sidecar:0.13.0
         imagePullPolicy: Always
         env:
         {{- if .Values.mysqlAllowEmptyPassword }}

examples/time-log/blueprint.yaml

@@ -41,7 +41,7 @@ actions:
       args:
         namespace: "{{ .Deployment.Namespace }}"
         pod: "{{ index .Deployment.Pods 0 }}"
-        image: kanisterio/kanister-tools:0.12.0
+        image: kanisterio/kanister-tools:0.13.0
         backupArtifactPrefix: "{{ .ArtifactsIn.timeLog.KeyValue.path }}"
         backupIdentifier: "{{ .ArtifactsIn.backupIdentifier.KeyValue.id }}"
     - func: ScaleWorkload

examples/time-log/time-logger-deployment.yaml

@@ -27,7 +27,7 @@ spec:
     spec:
       containers:
       - name: test-container
-        image: kanisterio/kanister-tools:0.12.0
+        image: kanisterio/kanister-tools:0.13.0
         command: ["sh", "-c"]
         args: ["while true; do for x in $(seq 1200); do date >> /var/log/time.log; sleep 1; done; truncate /var/log/time.log --size 0; done"]
         volumeMounts:

helm/kanister-operator/values.yaml

@@ -3,7 +3,7 @@
 # Declare variables to be passed into your templates.
 image:
   repository: kanisterio/controller
-  tag: 0.12.0
+  tag: 0.13.0
   pullPolicy: IfNotPresent
 rbac:
   create: true

pkg/function/backup_data.go

@@ -2,6 +2,7 @@ package function
 
 import (
 	"context"
+	"regexp"
 
 	"github.com/pkg/errors"
 
@@ -52,6 +53,22 @@ func validateProfile(profile *param.Profile) error {
 	return nil
 }
 
+func getSnapshotIDFromLog(output string) string {
+	if output == "" {
+		return ""
+	}
+	logs := regexp.MustCompile("[\n]").Split(output, -1)
+	for _, l := range logs {
+		// Log should contain "snapshot ABC123 saved"
+		pattern := regexp.MustCompile(`snapshot\s(.*?)\ssaved$`)
+		match := pattern.FindAllStringSubmatch(l, 1)
+		if match != nil {
+			return match[0][1]
+		}
+	}
+	return ""
+}
+
 func (*backupDataFunc) Exec(ctx context.Context, tp param.TemplateParams, args map[string]interface{}) (map[string]interface{}, error) {
 	var namespace, pod, container, includePath, backupArtifactPrefix, backupIdentifier, encryptionKey string
 	var err error
@@ -97,6 +114,10 @@ func (*backupDataFunc) Exec(ctx context.Context, tp param.TemplateParams, args m
 	if err != nil {
 		return nil, errors.Wrapf(err, "Failed to create and upload backup")
 	}
+	// Get the snapshot ID from log
+	if snapID := getSnapshotIDFromLog(stdout); snapID != "" {
+		return map[string]interface{}{"snapshotID": snapID}, nil
+	}
 	return nil, nil
 }
 

pkg/function/backup_data_test.go

@@ -7,10 +7,10 @@ import (
 	"github.com/kanisterio/kanister/pkg/param"
 )
 
-type ValidateProfileSuite struct {
+type BackupDataSuite struct {
 }
 
-var _ = Suite(&ValidateProfileSuite{})
+var _ = Suite(&BackupDataSuite{})
 
 func newValidProfile() *param.Profile {
 	return &param.Profile{
@@ -56,7 +56,7 @@ func newInvalidProfile() *param.Profile {
 	}
 }
 
-func (s *ValidateProfileSuite) TestValidateProfile(c *C) {
+func (s *BackupDataSuite) TestValidateProfile(c *C) {
 	testCases := []struct {
 		name       string
 		profile    *param.Profile
@@ -71,3 +71,18 @@ func (s *ValidateProfileSuite) TestValidateProfile(c *C) {
 		c.Check(err, tc.errChecker, Commentf("Test %s Failed", tc.name))
 	}
 }
+
+func (s *BackupDataSuite) TestGetSnapshotID(c *C) {
+	for _, tc := range []struct {
+		log      string
+		expected string
+	}{
+		{"snapshot 1a2b3c4d saved", "1a2b3c4d"},
+		{"snapshot 123abcd", ""},
+		{"Invalid message", ""},
+		{"snapshot abc123\n saved", ""},
+	} {
+		id := getSnapshotIDFromLog(tc.log)
+		c.Check(id, Equals, tc.expected, Commentf("Failed for log: %s", tc.log))
+	}
+}

pkg/function/copy_volume_data.go

@@ -17,7 +17,7 @@ import (
 )
 
 const (
-	kanisterToolsImage                         = "kanisterio/kanister-tools:0.12.0"
+	kanisterToolsImage                         = "kanisterio/kanister-tools:0.13.0"
 	copyVolumeDataMountPoint                   = "/mnt/vol_data/%s"
 	copyVolumeDataJobPrefix                    = "copy-vol-data-"
 	CopyVolumeDataNamespaceArg                 = "namespace"

pkg/function/data_test.go

@@ -80,7 +80,7 @@ func newRestoreDataBlueprint(pvc string) *crv1alpha1.Blueprint {
 						Func: "RestoreData",
 						Args: map[string]interface{}{
 							RestoreDataNamespaceArg:            "{{ .StatefulSet.Namespace }}",
-							RestoreDataImageArg:                "kanisterio/kanister-tools:0.12.0",
+							RestoreDataImageArg:                "kanisterio/kanister-tools:0.13.0",
 							RestoreDataBackupArtifactPrefixArg: "{{ .Profile.Location.S3Compliant.Bucket }}/{{ .Profile.Location.S3Compliant.Prefix }}",
 							RestoreDataRestorePathArg:          "/",
 							RestoreDataBackupIdentifierArg:     "{{ .Time }}",
@@ -176,7 +176,10 @@ func (s *DataSuite) TestBackupRestoreData(c *C) {
 		phases, err := kanister.GetPhases(bp, actionName, *tp)
 		c.Assert(err, IsNil)
 		for _, p := range phases {
-			_, err = p.Exec(context.Background(), bp, actionName, *tp)
+			out, err := p.Exec(context.Background(), bp, actionName, *tp)
+			if out != nil {
+				c.Assert(out["snapshotID"], NotNil)
+			}
 			c.Assert(err, IsNil)
 		}
 	}
@@ -222,7 +225,7 @@ func newCopyDataTestBlueprint() crv1alpha1.Blueprint {
 						Func: "RestoreData",
 						Args: map[string]interface{}{
 							RestoreDataNamespaceArg:            "{{ .PVC.Namespace }}",
-							RestoreDataImageArg:                "kanisterio/kanister-tools:0.12.0",
+							RestoreDataImageArg:                "kanisterio/kanister-tools:0.13.0",
 							RestoreDataBackupArtifactPrefixArg: fmt.Sprintf("{{ .Options.%s }}", CopyVolumeDataOutputBackupArtifactLocation),
 							RestoreDataBackupIdentifierArg:     fmt.Sprintf("{{ .Options.%s }}", CopyVolumeDataOutputBackupID),
 							RestoreDataVolsArg: map[string]string{

pkg/function/delete_data.go

@@ -63,7 +63,7 @@ func (*deleteDataFunc) Exec(ctx context.Context, tp param.TemplateParams, args m
 	// Generate delete command
 	cmd := generateDeleteCommand(artifact, tp.Profile)
 	// Use KubeTask to delete the artifact
-	return nil, kubeTask(ctx, namespace, "kanisterio/kanister-tools:0.12.0", cmd)
+	return nil, kubeTask(ctx, namespace, "kanisterio/kanister-tools:0.13.0", cmd)
 }
 
 func (*deleteDataFunc) RequiredArgs() []string {

pkg/function/kube_exec_test.go

@@ -157,11 +157,11 @@ func (s *KubeExecTest) TestParseLogAndCreateOutput(c *C) {
 		errChecker Checker
 		outChecker Checker
 	}{
-		{"###Phase-output###: {\"key\":\"version\",\"value\":\"0.12.0\"}", map[string]interface{}{"version": "0.12.0"}, IsNil, NotNil},
-		{"###Phase-output###: {\"key\":\"version\",\"value\":\"0.12.0\"}\n###Phase-output###: {\"key\":\"path\",\"value\":\"/backup/path\"}",
-			map[string]interface{}{"version": "0.12.0", "path": "/backup/path"}, IsNil, NotNil},
-		{"Random message ###Phase-output###: {\"key\":\"version\",\"value\":\"0.12.0\"}", map[string]interface{}{"version": "0.12.0"}, IsNil, NotNil},
-		{"Random message with newline \n###Phase-output###: {\"key\":\"version\",\"value\":\"0.12.0\"}", map[string]interface{}{"version": "0.12.0"}, IsNil, NotNil},
+		{"###Phase-output###: {\"key\":\"version\",\"value\":\"0.13.0\"}", map[string]interface{}{"version": "0.13.0"}, IsNil, NotNil},
+		{"###Phase-output###: {\"key\":\"version\",\"value\":\"0.13.0\"}\n###Phase-output###: {\"key\":\"path\",\"value\":\"/backup/path\"}",
+			map[string]interface{}{"version": "0.13.0", "path": "/backup/path"}, IsNil, NotNil},
+		{"Random message ###Phase-output###: {\"key\":\"version\",\"value\":\"0.13.0\"}", map[string]interface{}{"version": "0.13.0"}, IsNil, NotNil},
+		{"Random message with newline \n###Phase-output###: {\"key\":\"version\",\"value\":\"0.13.0\"}", map[string]interface{}{"version": "0.13.0"}, IsNil, NotNil},
 		{"###Phase-output###: Invalid message", nil, NotNil, IsNil},
 		{"Random message", nil, IsNil, IsNil},
 	} {

pkg/kube/pod_test.go

@@ -50,7 +50,7 @@ func (s *PodSuite) TestPod(c *C) {
 	pod, err := CreatePod(ctx, s.cli, &PodOptions{
 		Namespace:    s.namespace,
 		GenerateName: "test-",
-		Image:        "kanisterio/kanister-tools:0.12.0",
+		Image:        "kanisterio/kanister-tools:0.13.0",
 		Command:      []string{"sh", "-c", "tail -f /dev/null"},
 	})
 	c.Assert(err, IsNil)
@@ -76,7 +76,7 @@ func (s *PodSuite) TestPodWithVolumes(c *C) {
 	pod, err := CreatePod(ctx, cli, &PodOptions{
 		Namespace:    s.namespace,
 		GenerateName: "test-",
-		Image:        "kanisterio/kanister-tools:0.12.0",
+		Image:        "kanisterio/kanister-tools:0.13.0",
 		Command:      []string{"sh", "-c", "tail -f /dev/null"},
 		Volumes:      vols,
 	})

pkg/objectstore/aws.go

@@ -2,6 +2,7 @@ package objectstore
 
 import (
 	"context"
+	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
@@ -14,6 +15,7 @@ import (
 const (
 	bucketNotFound = "NotFound"
 	noSuchBucket   = s3.ErrCodeNoSuchBucket
+	gcsS3NotFound  = "not found"
 )
 
 func config(region string) *aws.Config {
@@ -25,11 +27,14 @@ func config(region string) *aws.Config {
 }
 
 func isBucketNotFoundError(err error) bool {
+	if err == nil {
+		return false
+	}
 	if awsErr, ok := errors.Cause(err).(awserr.Error); ok {
 		code := awsErr.Code()
 		return code == bucketNotFound || code == noSuchBucket
 	}
-	return false
+	return strings.Contains(err.Error(), gcsS3NotFound)
 }
 
 func GetS3BucketRegion(ctx context.Context, bucketName, regionHint string) (string, error) {