Merge pull request #149 from kanisterio/sync

GCP location and credentials for Kanister, Prefer the get script to install kanister tools, Bump tool script to 0.16.0, Skip flaky scale test
kanisterio · Jan 3, 2019 · a6db8ad · a6db8ad
2 parents f803628 + 7030294
commit a6db8ad
Show file tree

Hide file tree

Showing 26 changed files with 110 additions and 150 deletions.
diff --git a/docker/postgres-kanister-tools/Dockerfile b/docker/postgres-kanister-tools/Dockerfile
@@ -11,6 +11,6 @@ RUN apk -v --update add --no-cache python py-pip groff less && \
     apk -v --purge del py-pip && \
     rm -f /var/cache/apk/*
 
-COPY --from=kanisterio/kanister-tools:0.15.0 /usr/local/bin/kando /usr/local/bin/kando
+RUN curl https://raw.githubusercontent.com/kanisterio/kanister/master/scripts/get.sh | bash
 
 CMD ["tail", "-f", "/dev/null"]
diff --git a/examples/helm/kanister/kanister-mongodb-replicaset/image/Dockerfile b/examples/helm/kanister/kanister-mongodb-replicaset/image/Dockerfile
@@ -7,6 +7,6 @@ ADD . /kanister
 
 RUN /kanister/install.sh && rm -rf /kanister && rm -rf /tmp && mkdir /tmp
 
-COPY --from=kanisterio/kanister-tools:0.15.0 /usr/local/bin/kando /usr/local/bin/kando
+RUN curl https://raw.githubusercontent.com/kanisterio/kanister/master/scripts/get.sh | bash
 
 CMD ["tail", "-f", "/dev/null"]
diff --git a/examples/helm/kanister/kanister-mysql/image/Dockerfile b/examples/helm/kanister/kanister-mysql/image/Dockerfile
@@ -7,6 +7,6 @@ ADD . /kanister
 
 RUN /kanister/install.sh && rm -rf /kanister && rm -rf /tmp && mkdir /tmp
 
-COPY --from=kanisterio/kanister-tools:0.15.0 /usr/local/bin/kando /usr/local/bin/kando
+RUN curl https://raw.githubusercontent.com/kanisterio/kanister/master/scripts/get.sh | bash
 
 CMD ["tail", "-f", "/dev/null"]
diff --git a/examples/helm/kanister/kanister-postgresql/kanister/postgres-blueprint.yaml b/examples/helm/kanister/kanister-postgresql/kanister/postgres-blueprint.yaml
@@ -34,7 +34,7 @@ actions:
             then
               # Setup wal-e s3 connection parameters.
               timeline={{ toDate "2006-01-02T15:04:05.999999999Z07:00" .Time  | date "2006-01-02T15-04-05" }}
-              wale_s3_prefix="s3://{{ .Profile.Location.S3Compliant.Bucket }}/postgres-backups/{{ .Deployment.Name }}/${timeline}"
+              wale_s3_prefix="s3://{{ .Profile.Location.Bucket }}/postgres-backups/{{ .Deployment.Name }}/${timeline}"
               echo "${wale_s3_prefix}" > "${env_wal_prefix}"
             fi
 
@@ -44,8 +44,8 @@ actions:
             env_wal_access_key_id="${env_dir}/AWS_ACCESS_KEY_ID"
             env_wal_secret_access_key="${env_dir}/AWS_SECRET_ACCESS_KEY"
 
-            {{- if .Profile.Location.S3Compliant.Endpoint }}
-              wale_s3_endpoint="{{ .Profile.Location.S3Compliant.Endpoint | quote }}"
+            {{- if .Profile.Location.Endpoint }}
+              wale_s3_endpoint="{{ .Profile.Location.Endpoint | quote }}"
               wale_s3_endpoint=${wale_s3_endpoint,,}
               {{- if .Profile.SkipSSLVerify }}
                 # Since wal-e does not support skip-ssl-verify switch to http://
@@ -63,8 +63,8 @@ actions:
             {{- else }}
               # Region is required when no endpoint is used (AWS S3).
               wale_s3_region="us-east-1"
-              {{- if .Profile.Location.S3Compliant.Region }}
-                wale_s3_region="{{ .Profile.Location.S3Compliant.Region | quote}}"
+              {{- if .Profile.Location.Region }}
+                wale_s3_region="{{ .Profile.Location.Region | quote}}"
               {{- end }}
               echo "${wale_s3_region}" > "${env_wal_region}"
             {{- end }}
@@ -83,13 +83,13 @@ actions:
             {{- if .Profile.SkipSSLVerify }}
               s3_cmd+=("--no-verify-ssl")
             {{- end }}
-            {{- if .Profile.Location.S3Compliant.Endpoint }}
-              s3_cmd+=(--endpoint "{{ .Profile.Location.S3Compliant.Endpoint }}")
+            {{- if .Profile.Location.Endpoint }}
+              s3_cmd+=(--endpoint "{{ .Profile.Location.Endpoint }}")
             {{- end }}
-            {{- if .Profile.Location.S3Compliant.Region }}
-              s3_cmd+=(--region "{{ .Profile.Location.S3Compliant.Region | quote}}")
+            {{- if .Profile.Location.Region }}
+              s3_cmd+=(--region "{{ .Profile.Location.Region | quote}}")
             {{- end }}
-            s3_path="s3://{{ .Profile.Location.S3Compliant.Bucket }}/postgres-backups/{{ .Deployment.Name }}/{{ toDate "2006-01-02T15:04:05.999999999Z07:00" .Time  | date "2006-01-02T15-04-05" }}/manifest.txt"
+            s3_path="s3://{{ .Profile.Location.Bucket }}/postgres-backups/{{ .Deployment.Name }}/{{ toDate "2006-01-02T15:04:05.999999999Z07:00" .Time  | date "2006-01-02T15-04-05" }}/manifest.txt"
             s3_cmd+=(s3 cp - "${s3_path}")
 
             set +o xtrace
@@ -147,13 +147,13 @@ actions:
             {{- if .Profile.SkipSSLVerify }}
               s3_cmd+=(" --no-verify-ssl")
             {{- end }}
-            {{- if .Profile.Location.S3Compliant.Endpoint }}
-              s3_cmd+=(--endpoint "{{ .Profile.Location.S3Compliant.Endpoint }}")
+            {{- if .Profile.Location.Endpoint }}
+              s3_cmd+=(--endpoint "{{ .Profile.Location.Endpoint }}")
             {{- end }}
-            {{- if .Profile.Location.S3Compliant.Region }}
-              s3_cmd+=(--region "{{ .Profile.Location.S3Compliant.Region | quote}}")
+            {{- if .Profile.Location.Region }}
+              s3_cmd+=(--region "{{ .Profile.Location.Region | quote}}")
             {{- end }}
-            s3_path="s3://{{ .Profile.Location.S3Compliant.Bucket }}/{{ .ArtifactsIn.manifest.KeyValue.path }}"
+            s3_path="s3://{{ .Profile.Location.Bucket }}/{{ .ArtifactsIn.manifest.KeyValue.path }}"
             s3_cmd+=(s3 cp "${s3_path}" -)
 
             set +o xtrace
@@ -166,8 +166,8 @@ actions:
 
             # Fetch base backup using the old WALE_S3_PREFIX.
             # First need to setup wal-e conf as env vars
-            {{- if .Profile.Location.S3Compliant.Endpoint }}
-              wale_s3_endpoint="{{ .Profile.Location.S3Compliant.Endpoint | quote}}"
+            {{- if .Profile.Location.Endpoint }}
+              wale_s3_endpoint="{{ .Profile.Location.Endpoint | quote}}"
               wale_s3_endpoint=${wale_s3_endpoint,,}
               {{- if .Profile.SkipSSLVerify }}
                 # Since wal-e does not support skip-ssl-verify switch to http://
@@ -183,8 +183,8 @@ actions:
               # Region will be ignored for S3 compatible object store so skipping.
             {{- else }}
               # Region is required when no endpoint is used (AWS S3).
-              {{- if .Profile.Location.S3Compliant.Region }}
-                wale_s3_region="{{ .Profile.Location.S3Compliant.Region | quote}}"
+              {{- if .Profile.Location.Region }}
+                wale_s3_region="{{ .Profile.Location.Region | quote}}"
               {{- else }}
                 wale_s3_region="us-east-1"
               {{- end }}
@@ -226,7 +226,7 @@ actions:
             # Recovery is now complete and can switch to new WAL timeline
             env_wal_prefix="${pgdata}/env/WALE_S3_PREFIX"
             timeline={{ toDate "2006-01-02T15:04:05.999999999Z07:00" .Time  | date "2006-01-02T15-04-05" }}
-            wale_s3_prefix="s3://{{ .Profile.Location.S3Compliant.Bucket }}/{{ .ArtifactsIn.manifest.KeyValue.prefix }}/${timeline}"
+            wale_s3_prefix="s3://{{ .Profile.Location.Bucket }}/{{ .ArtifactsIn.manifest.KeyValue.prefix }}/${timeline}"
             echo "${wale_s3_prefix}" > "${env_wal_prefix}"
     - func: ScaleWorkload
       name: restartPod
@@ -267,13 +267,13 @@ actions:
             {{- if .Profile.SkipSSLVerify }}
               aws_args+=(" --no-verify-ssl")
             {{- end }}
-            {{- if .Profile.Location.S3Compliant.Endpoint }}
-              aws_args+=(--endpoint "{{ .Profile.Location.S3Compliant.Endpoint }}")
+            {{- if .Profile.Location.Endpoint }}
+              aws_args+=(--endpoint "{{ .Profile.Location.Endpoint }}")
             {{- end }}
-            {{- if .Profile.Location.S3Compliant.Region }}
-              aws_args+=(--region "{{ .Profile.Location.S3Compliant.Region | quote}}")
+            {{- if .Profile.Location.Region }}
+              aws_args+=(--region "{{ .Profile.Location.Region | quote}}")
             {{- end }}
-            s3_path="s3://{{ .Profile.Location.S3Compliant.Bucket }}/{{ .ArtifactsIn.manifest.KeyValue.path }}"
+            s3_path="s3://{{ .Profile.Location.Bucket }}/{{ .ArtifactsIn.manifest.KeyValue.path }}"
 
 
             # Get and parse artifact manifest to discover the timeline and the base-backup name.
@@ -286,8 +286,8 @@ actions:
             aws "${aws_args[@]}" s3 rm --recursive "${base_backup_path}"
 
             # Setup configuration for wal-e.
-            {{- if .Profile.Location.S3Compliant.Endpoint }}
-              wale_s3_endpoint="{{ .Profile.Location.S3Compliant.Endpoint | quote}}"
+            {{- if .Profile.Location.Endpoint }}
+              wale_s3_endpoint="{{ .Profile.Location.Endpoint | quote}}"
               wale_s3_endpoint-${wale_s3_endpoint,,}
               {{- if .Profile.SkipSSLVerify }}
                 # Since wal-e does not support skip-ssl-verify switch to http://
@@ -303,8 +303,8 @@ actions:
               # Region will be ignored for S3 compatible object store so skipping.
             {{- else }}
               # Region is required when no endpoint is used (AWS S3).
-              {{- if .Profile.Location.S3Compliant.Region }}
-                wale_s3_region="{{ .Profile.Location.S3Compliant.Region | quote}}"
+              {{- if .Profile.Location.Region }}
+                wale_s3_region="{{ .Profile.Location.Region | quote}}"
               {{- else }}
                 wale_s3_region="us-east-1"
               {{- end }}

diff --git a/examples/time-log/blueprint.yaml b/examples/time-log/blueprint.yaml
@@ -9,7 +9,7 @@ actions:
     outputArtifacts:
       timeLog:
         keyValue:
-          path: '{{ .Profile.Location.S3Compliant.Bucket }}/time-log'
+          path: '{{ .Profile.Location.Bucket }}/time-log'
       backupIdentifier:
         keyValue:
           id: '{{ toDate "2006-01-02T15:04:05.999999999Z07:00" .Time | date "2006-01-02" }}'
@@ -21,7 +21,7 @@ actions:
         pod: "{{ index .Deployment.Pods 0 }}"
         container: test-container
         includePath: /var/log
-        backupArtifactPrefix: "{{ .Profile.Location.S3Compliant.Bucket }}/time-log"
+        backupArtifactPrefix: "{{ .Profile.Location.Bucket }}/time-log"
         backupIdentifier: '{{ toDate "2006-01-02T15:04:05.999999999Z07:00" .Time | date "2006-01-02" }}'
   restore:
     type: Deployment

diff --git a/examples/time-log/s3-profile.yaml b/examples/time-log/s3-profile.yaml
@@ -6,11 +6,10 @@ metadata:
 # Note: Add details of an existing S3 compliant bucket below
 location:
   type: s3Compliant
-  s3Compliant:
-    bucket: XXXX
-    endpoint: XXXX
-    prefix: XXXX
-    region: XXXX
+  bucket: XXXX
+  endpoint: XXXX
+  prefix: XXXX
+  region: XXXX
 credential:
   type: keyPair
   keyPair:

diff --git a/helm/profile/templates/profile.yaml b/helm/profile/templates/profile.yaml
@@ -19,11 +19,10 @@ metadata:
 {{ include "profile.helmLabels" . | indent 4 }}
 location:
   type: s3Compliant
-  s3Compliant:
-    bucket: {{ required "S3 compatible bucket is required when configuring a profile." .Values.s3.bucket | quote }}
-    endpoint: {{ .Values.s3.endpoint | quote }}
-    prefix: {{ .Values.s3.prefix | quote }}
-    region: {{ .Values.s3.region | quote }}
+  bucket: {{ required "S3 compatible bucket is required when configuring a profile." .Values.s3.bucket | quote }}
+  endpoint: {{ .Values.s3.endpoint | quote }}
+  prefix: {{ .Values.s3.prefix | quote }}
+  region: {{ .Values.s3.region | quote }}
 credential:
   type: keyPair
   keyPair:

diff --git a/pkg/apis/cr/v1alpha1/types.go b/pkg/apis/cr/v1alpha1/types.go
@@ -228,21 +228,17 @@ type Profile struct {
 type LocationType string
 
 const (
+	LocationTypeGCS         LocationType = "gcs"
 	LocationTypeS3Compliant LocationType = "s3Compliant"
 )
 
 // Location
 type Location struct {
-	Type        LocationType         `json:"type"`
-	S3Compliant *S3CompliantLocation `json:"s3Compliant"`
-}
-
-// S3Compliant
-type S3CompliantLocation struct {
-	Bucket   string `json:"bucket"`
-	Endpoint string `json:"endpoint"`
-	Prefix   string `json:"prefix"`
-	Region   string `json:"region"`
+	Type     LocationType `json:"type"`
+	Bucket   string       `json:"bucket"`
+	Endpoint string       `json:"endpoint"`
+	Prefix   string       `json:"prefix"`
+	Region   string       `json:"region"`
 }
 
 // CredentialType

diff --git a/pkg/apis/cr/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/cr/v1alpha1/zz_generated.deepcopy.go
diff --git a/pkg/function/backup_data_test.go b/pkg/function/backup_data_test.go
@@ -15,13 +15,11 @@ var _ = Suite(&BackupDataSuite{})
 func newValidProfile() *param.Profile {
 	return &param.Profile{
 		Location: crv1alpha1.Location{
-			Type: crv1alpha1.LocationTypeS3Compliant,
-			S3Compliant: &crv1alpha1.S3CompliantLocation{
-				Bucket:   "test-bucket",
-				Endpoint: "",
-				Prefix:   "",
-				Region:   "us-west-1",
-			},
+			Type:     crv1alpha1.LocationTypeS3Compliant,
+			Bucket:   "test-bucket",
+			Endpoint: "",
+			Prefix:   "",
+			Region:   "us-west-1",
 		},
 		Credential: param.Credential{
 			Type: param.CredentialTypeKeyPair,
@@ -37,13 +35,11 @@ func newValidProfile() *param.Profile {
 func newInvalidProfile() *param.Profile {
 	return &param.Profile{
 		Location: crv1alpha1.Location{
-			Type: "foo-type",
-			S3Compliant: &crv1alpha1.S3CompliantLocation{
-				Bucket:   "test-bucket",
-				Endpoint: "",
-				Prefix:   "",
-				Region:   "us-west-1",
-			},
+			Type:     "foo-type",
+			Bucket:   "test-bucket",
+			Endpoint: "",
+			Prefix:   "",
+			Region:   "us-west-1",
 		},
 		Credential: param.Credential{
 			Type: param.CredentialTypeKeyPair,

diff --git a/pkg/function/create_volume_from_snapshot_test.go b/pkg/function/create_volume_from_snapshot_test.go
@@ -28,9 +28,8 @@ func (s *CreateVolumeFromSnapshotTestSuite) TestCreateVolumeFromSnapshot(c *C) {
 	mockGetter := mockblockstorage.NewGetter()
 	profile := &param.Profile{
 		Location: crv1alpha1.Location{
-			Type: crv1alpha1.LocationTypeS3Compliant,
-			S3Compliant: &crv1alpha1.S3CompliantLocation{
-				Region: "us-west-2"},
+			Type:   crv1alpha1.LocationTypeS3Compliant,
+			Region: "us-west-2",
 		},
 		Credential: param.Credential{
 			Type: param.CredentialTypeKeyPair,

diff --git a/pkg/function/create_volume_snapshot.go b/pkg/function/create_volume_snapshot.go
@@ -69,7 +69,7 @@ func ValidateProfile(profile *param.Profile) error {
 	if profile.Location.Type != crv1alpha1.LocationTypeS3Compliant {
 		return errors.New("Location type not supported")
 	}
-	if len(profile.Location.S3Compliant.Region) == 0 {
+	if len(profile.Location.Region) == 0 {
 		return errors.New("Region is not set")
 	}
 	if profile.Credential.Type != param.CredentialTypeKeyPair {

diff --git a/pkg/function/create_volume_snapshot_test.go b/pkg/function/create_volume_snapshot_test.go
@@ -27,9 +27,8 @@ func (s *CreateVolumeSnapshotTestSuite) TestGetPVCInfo(c *C) {
 	tp := param.TemplateParams{
 		Profile: &param.Profile{
 			Location: crv1alpha1.Location{
-				Type: crv1alpha1.LocationTypeS3Compliant,
-				S3Compliant: &crv1alpha1.S3CompliantLocation{
-					Region: "us-west-2"},
+				Type:   crv1alpha1.LocationTypeS3Compliant,
+				Region: "us-west-2",
 			},
 			Credential: param.Credential{
 				Type: param.CredentialTypeKeyPair,