Merge pull request #143 from kanisterio/sync

Add volume support to RBAC; Improve function signature; Update Docs

bundle.yaml

@@ -53,6 +53,47 @@ rules:
   - "*"
   verbs:
   - "*"
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  verbs:
+  - get
+  - list
+  - update
+  - watch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - storageclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
 ---
 apiVersion: v1
 kind: ServiceAccount

docs/install.rst

@@ -17,6 +17,10 @@ Prerequisites
 
 * `helm <https://helm.sh>`_ installed and initialized using the command `helm init`
 
+* :ref:`kanctl <tooling>` installed
+
+* Access to an S3 compatible bucket and credentials.
+
 * Docker (for source-based installs only)
 
 
@@ -33,6 +37,12 @@ This will install the Kanister controller in the `kanister` namespace
    # Install the Kanister operator controller using helm
    $ helm install --name myrelease --namespace kanister kanister/kanister-operator --set image.tag=0.14.0
 
+   # Create an S3 Compliant Kanister profile using kanctl
+   $ kanctl create profile s3compliant --bucket <bucket> --access-key ${AWS_ACCESS_KEY_ID} \
+                                       --secret-key ${AWS_SECRET_ACCESS_KEY}               \
+                                       --region <region>                                   \
+                                       --namespace kanister
+
 
 Building and Deploying from Source
 ==================================
@@ -63,5 +73,5 @@ namespace.
 
 .. code-block:: bash
 
-   # Deploy controller version 0.10.0 to Kubernetes
-   $ make deploy VERSION="0.10.0"
+   # Deploy controller version 0.14.0 to Kubernetes
+   $ make deploy VERSION="0.14.0"

docs/tooling.rst

@@ -223,9 +223,9 @@ A new S3Compliant profile can be created using the s3compliant subcommand
 
 .. code-block:: bash
 
-  $ kanctl create profile s3compliant --bucket <bucket> --access-key $AWS_ACCESS_KEY_ID \
-                                      --secret-key $AWS_SECRET_ACCESS_KEY               \
-                                      --region us-west-1                                \
+  $ kanctl create profile s3compliant --bucket <bucket> --access-key ${AWS_ACCESS_KEY_ID} \
+                                      --secret-key ${AWS_SECRET_ACCESS_KEY}               \
+                                      --region us-west-1                                  \
                                       --namespace kanister
   secret 's3-secret-chst2' created
   profile 's3-profile-5mmkj' created
@@ -377,6 +377,20 @@ The following snippet is an example of using kando from inside a Blueprint.
 
   kando output version 0.14.0
 
+Install the tools
+=================
+
+Installation of the tools requires `Go <https://golang.org/doc/install>`_ to be installed
+
+.. code-block:: bash
+
+  # Installing kanctl
+  $ go install -v github.com/kanisterio/kanister/cmd/kanctl
+
+  # Installing kando
+  $ go install -v github.com/kanisterio/kanister/cmd/kando
+
+
 Docker Image
 ============
 

examples/helm/kanister/kanister-mongodb-replicaset/image/install.sh

@@ -19,7 +19,7 @@ mv 90forceyes /etc/apt/apt.conf.d/
 
 echo "================= Installing basic packages ==================="
 apt-get update && \
-apt-get install wget musl-dev python -y
+apt-get install wget musl-dev python rsync -y
 
 echo "================= Installing Python packages ==================="
 wget --progress=dot:mega https://bootstrap.pypa.io/get-pip.py

helm/kanister-operator/Chart.yaml

@@ -9,5 +9,5 @@ maintainers:
   - email: tom@kasten.io
     name: tdmanv
 icon: https://kasten.io/assets/img/kanister-logo.png
-appVersion: 0.10.0
+appVersion: 0.14.0
 source: https://github.com/kanisterio/kanister

helm/kanister-operator/templates/rbac.yaml

@@ -55,3 +55,18 @@ subjects:
   name: {{ template "kanister-operator.serviceAccountName" . }}
   namespace: {{ .Release.Namespace }}
 {{- end }}
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  labels:
+{{ include "kanister-operator.helmLabels" . | indent 4 }}
+  name: {{ template "kanister-operator.fullname" . }}-pv-provisioner
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:persistent-volume-provisioner
+subjects:
+- kind: ServiceAccount
+  name: {{ template "kanister-operator.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}

pkg/function/create_volume_from_snapshot.go

@@ -28,6 +28,7 @@ var (
 const (
 	CreateVolumeFromSnapshotNamespaceArg = "namespace"
 	CreateVolumeFromSnapshotManifestArg  = "snapshots"
+	CreateVolumeFromSnapshotPVCNamesArg  = "pvcNames"
 )
 
 type createVolumeFromSnapshotFunc struct{}
@@ -36,15 +37,22 @@ func (*createVolumeFromSnapshotFunc) Name() string {
 	return "CreateVolumeFromSnapshot"
 }
 
-func createVolumeFromSnapshot(ctx context.Context, cli kubernetes.Interface, namespace, snapshotinfo string, profile *param.Profile, getter getter.Getter) (map[string]blockstorage.Provider, error) {
+func createVolumeFromSnapshot(ctx context.Context, cli kubernetes.Interface, namespace, snapshotinfo string, pvcNames []string, profile *param.Profile, getter getter.Getter) (map[string]blockstorage.Provider, error) {
 	PVCData := []VolumeSnapshotInfo{}
 	err := json.Unmarshal([]byte(snapshotinfo), &PVCData)
 	if err != nil {
 		return nil, errors.Wrapf(err, "Could not decode JSON data")
 	}
+	if len(pvcNames) > 0 && len(pvcNames) != len(PVCData) {
+		return nil, errors.New("Invalid number of PVC names provided")
+	}
 	// providerList required for unit testing
 	providerList := make(map[string]blockstorage.Provider)
-	for _, pvcInfo := range PVCData {
+	for i, pvcInfo := range PVCData {
+		pvcName := pvcInfo.PVCName
+		if len(pvcNames) > 0 {
+			pvcName = pvcNames[i]
+		}
 		config := make(map[string]string)
 		switch pvcInfo.Type {
 		case blockstorage.TypeEBS:
@@ -59,18 +67,19 @@ func createVolumeFromSnapshot(ctx context.Context, cli kubernetes.Interface, nam
 		if err != nil {
 			return nil, errors.Wrapf(err, "Could not get storage provider %v", pvcInfo.Type)
 		}
-		_, err = cli.Core().PersistentVolumeClaims(namespace).Get(pvcInfo.PVCName, metav1.GetOptions{})
+		_, err = cli.Core().PersistentVolumeClaims(namespace).Get(pvcName, metav1.GetOptions{})
 		if err == nil {
-			if err = kube.DeletePVC(cli, namespace, pvcInfo.PVCName); err != nil {
+			if err = kube.DeletePVC(cli, namespace, pvcName); err != nil {
 				return nil, err
 			}
 		}
 		snapshot, err := provider.SnapshotGet(ctx, pvcInfo.SnapshotID)
 		if err != nil {
 			return nil, errors.Wrapf(err, "Failed to get Snapshot from Provider")
 		}
+
 		tags := map[string]string{
-			"pvcname": pvcInfo.PVCName,
+			"pvcname": pvcName,
 		}
 		snapshot.Volume.VolumeType = pvcInfo.VolumeType
 		snapshot.Volume.Az = pvcInfo.Az
@@ -81,7 +90,7 @@ func createVolumeFromSnapshot(ctx context.Context, cli kubernetes.Interface, nam
 		}
 
 		annotations := map[string]string{}
-		pvc, err := kube.CreatePVC(ctx, cli, namespace, pvcInfo.PVCName, vol.Size, vol.ID, annotations)
+		pvc, err := kube.CreatePVC(ctx, cli, namespace, pvcName, vol.Size, vol.ID, annotations)
 		if err != nil {
 			return nil, errors.Wrapf(err, "Unable to create PVC for volume %v", *vol)
 		}
@@ -101,13 +110,17 @@ func (kef *createVolumeFromSnapshotFunc) Exec(ctx context.Context, tp param.Temp
 		return nil, errors.Wrapf(err, "Failed to create Kubernetes client")
 	}
 	var namespace, snapshotinfo string
+	var pvcNames []string
 	if err = Arg(args, CreateVolumeFromSnapshotNamespaceArg, &namespace); err != nil {
 		return nil, err
 	}
 	if err = Arg(args, CreateVolumeFromSnapshotManifestArg, &snapshotinfo); err != nil {
 		return nil, err
 	}
-	_, err = createVolumeFromSnapshot(ctx, cli, namespace, snapshotinfo, tp.Profile, getter.New())
+	if err = OptArg(args, CreateVolumeFromSnapshotPVCNamesArg, &pvcNames, nil); err != nil {
+		return nil, err
+	}
+	_, err = createVolumeFromSnapshot(ctx, cli, namespace, snapshotinfo, pvcNames, tp.Profile, getter.New())
 	return nil, err
 }
 

pkg/function/create_volume_from_snapshot_test.go

@@ -75,13 +75,20 @@ func (s *CreateVolumeFromSnapshotTestSuite) TestCreateVolumeFromSnapshot(c *C) {
 	for _, tc := range []struct {
 		snapshotinfo string
 		check        Checker
+		newPVCs      []string
 	}{
 		{
 			snapshotinfo: snapinfo,
 			check:        IsNil,
+			newPVCs:      nil,
+		},
+		{
+			snapshotinfo: snapinfo,
+			check:        IsNil,
+			newPVCs:      []string{"newpvc-1", "newpvc-2"},
 		},
 	} {
-		providerList, err := createVolumeFromSnapshot(ctx, cli, ns, tc.snapshotinfo, profile, mockGetter)
+		providerList, err := createVolumeFromSnapshot(ctx, cli, ns, tc.snapshotinfo, tc.newPVCs, profile, mockGetter)
 		c.Assert(providerList, Not(Equals), tc.check)
 		c.Assert(err, tc.check)
 		if err != nil {
@@ -100,9 +107,16 @@ func (s *CreateVolumeFromSnapshotTestSuite) TestCreateVolumeFromSnapshot(c *C) {
 		c.Assert(mockblockstorage.CheckID("snap-2", provider.(*mockblockstorage.Provider).SnapIDList), Equals, true)
 		c.Assert(len(provider.(*mockblockstorage.Provider).VolIDList) == 1, Equals, true)
 
-		_, err = cli.Core().PersistentVolumeClaims(ns).Get("pvc-1", metav1.GetOptions{})
-		c.Assert(err, IsNil)
-		_, err = cli.Core().PersistentVolumeClaims(ns).Get("pvc-2", metav1.GetOptions{})
-		c.Assert(err, IsNil)
+		if tc.newPVCs != nil {
+			_, err = cli.Core().PersistentVolumeClaims(ns).Get("newpvc-1", metav1.GetOptions{})
+			c.Assert(err, IsNil)
+			_, err = cli.Core().PersistentVolumeClaims(ns).Get("newpvc-2", metav1.GetOptions{})
+			c.Assert(err, IsNil)
+		} else {
+			_, err = cli.Core().PersistentVolumeClaims(ns).Get("pvc-1", metav1.GetOptions{})
+			c.Assert(err, IsNil)
+			_, err = cli.Core().PersistentVolumeClaims(ns).Get("pvc-2", metav1.GetOptions{})
+			c.Assert(err, IsNil)
+		}
 	}
 }

pkg/function/kube_exec_all.go

@@ -55,16 +55,17 @@ func (*kubeExecAllFunc) Exec(ctx context.Context, tp param.TemplateParams, args
 	}
 	ps := strings.Fields(pods)
 	cs := strings.Fields(containers)
-	return nil, execAll(cli, namespace, ps, cs, cmd)
+	return execAll(cli, namespace, ps, cs, cmd)
 }
 
 func (*kubeExecAllFunc) RequiredArgs() []string {
 	return []string{KubeExecAllNamespaceArg, KubeExecAllPodsNameArg, KubeExecAllContainersNameArg, KubeExecAllCommandArg}
 }
 
-func execAll(cli kubernetes.Interface, namespace string, ps []string, cs []string, cmd []string) error {
+func execAll(cli kubernetes.Interface, namespace string, ps []string, cs []string, cmd []string) (map[string]interface{}, error) {
 	numContainers := len(ps) * len(cs)
 	errChan := make(chan error, numContainers)
+	output := ""
 	// Run the command
 	for _, p := range ps {
 		for _, c := range cs {
@@ -73,6 +74,7 @@ func execAll(cli kubernetes.Interface, namespace string, ps []string, cs []strin
 				format.Log(p, c, stdout)
 				format.Log(p, c, stderr)
 				errChan <- err
+				output = output + "\n" + stdout
 			}(p, c)
 		}
 	}
@@ -84,7 +86,11 @@ func execAll(cli kubernetes.Interface, namespace string, ps []string, cs []strin
 		}
 	}
 	if len(errs) != 0 {
-		return errors.New(strings.Join(errs, "\n"))
+		return nil, errors.New(strings.Join(errs, "\n"))
 	}
-	return nil
+	out, err := parseLogAndCreateOutput(output)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
 }