Add new integration test for CSI Snapshot Blueprint (#1208)

* Added new function CreateCSISnapshot * Fixed the instantiated values for kubeCli and dynCli * Added waitForReady=true as the default * Added apiGroup snapshot.storage.k8s.io in cluster-role for kanister-operator * Added documentation for Create CSI VolumeSnapshot Func * Changed namespace to a required arg * Rasults of running gofmt * Reverting changes to return type of snapshotter.Create() * Added new function RestoreCSISnapshot * Changed snapShotter -> snapshotter * grouped imports and added the comment for 'waitForReady' * Updates suggested in docs/functions.rst * Changed snapshot name argument to optional argument and added a default value * reviewer comments * Changes suggested in reviewer comments * gofmt update * gofmt update * Fixed failed to render outputartifact template 'snapshotInfo' issue * Updated const comments and function description in documentation * Updated const comments * Updated the return obj for the function * Declared and used const for outputartifact keys * Added documentation for RestoreCSISnapshot function * Minor changes in docs/functions.rst * Updated the rules in clusterrole * Updated the rules in clusterrole * added asterisk again in helm/kanister-operator/templates/rbac.yaml * Updated rules in clusterrole to check Travis CI failure * Created a test suite for CreateCSISanpshot func * Updated the testcase * Resolved the error in testcase * Updated the create snapshot testcase * removed AddKnownTypeWithName * Added test suite for RestoreCSISnapshot func * Updated the CreateCSISnapshot testcase * gofmt update * golang-ci: fixed whitespace * Reviewer comments * Reviewer comments * Updated the testcase for RestoreCSISnapshot function * Updated the testcase for CreateCSiSnapshot function * Updated the createPVC method in testcase * Fixed RBAC YAML * Fixed RBAC YAML * Added the DeleteCSISnapshot function * Added RBAC updates * Added testcase for DeleteCSISnapshot function * gofmt fix * Added documentation for DeleteCSISnapshot function * fixed issue in docs/functions.rst * Updated the namespace in testcase * Added default context timeout * Fixed the time.Minute issue * Fixed the time.Minute issue * Updated the defer cancel() * Added a custom error for create * Returned the custom error * Updated the error msg * Removed the timeout code snippet * Added validations for volumeMode and accessMode * Removed unnecessary const for default timeout and import for time package * Added the DeadlineExceeded error check and corrected the imports * Updated the logic to validate volume mode * context update * context update * Fixed the validateVolumeModeArg method * Updated the testcase for validate methods * Added testcase for arg validation * Added blank spaces * gofmt updates * Added Note in the docs * Added table driven testcases for validate methods * Removed multiple context.Background() calls and removed the validate method calls in main test case * style: Add whitespace in error messages * refactor: Update const names in test files * refactor: Update length of random alphanumeric suffix to 5 characters * refactor: Update const in test file * style: Add CSI Snapshot funcs MySQL example blueprint * style: Updates in the example README * style: Update title * style: typo in Prerequisites * style: Fix multiple typos * style: Update helm command * style: Fix title, blueprint note and 'Backup application data' descripion * style: Remove unnecessary quotation marks * style: Update blueprint note * refactor: Update argument validation methods * style: Fix typo in a comment * fix: Update create blueprint command * style: fix grammatical typos and readme header syntax * style: Fix create blueprint note with correct blueprint file name * style: Update README as per reviewer comments * style: Explain --pvc argument * style: Update introduction as per reviewer comment * refactor: Update call to validate methods in restore unit test * refactor: Update restore test case as per reviewer comment * refactor: Update CreateCSiSnapshot function and the test case * refactor: Revert random alphanumeric string length * fix: gofmt error * refactor: Update CreateCSiSnapshot function and the test case * refactor: Update CreateCSiSnapshot function and the test case * refactor: Update CreateCSiSnapshot function and the test case * refactor: Moved code out of Exec method * chore: Update signature of the restoreCSISnapshot method * refactor: Revert CreateCSISnapshot Func changes in this PR * refactor: Update create snapshot unit test * fix: CI failure * chore: Comment update * Update example README as per reviewer comments * Update KubeOps function to allow create & delete for core group resources * Add test case for KubeOps Delete core group resource * Remove core group permissions from clusterrole * Update KubeOps Delete operation to wait for resource to be deleted * Update testcase for kubeops delete on core group resource * Update KubeOps test to use existing svc spec * Update poll.Wait logic for waiting on delete operation to complete * Fix gofmt issues * Update wait.Poll to poll.Wait method & pass context from parent * Update as per reviewer comments * Update example with Time Log application * Replace namespace mysql with time-log * Handle error from Delete() and add comment for using poll.Wait() * Add waitForResourceDeletion function * Remove redundant check for error * Add integration test for CSI snapshot blueprint * Fix curl not found and unused string return from execCommmand method * Update base build image * Merge kubectl apply statements * Update comment in integration_register.go * Revert createPhase method in kubeops_test.go * Update KubeOps test * Update README as per reviewer comments * Add comment to explain the waitForResourceDeletion function * Separate CSI driver setup in new target * Update in install_csi_hostpath_driver method * Update comment for waitForResourceDeletion * Remove default storageclass patch * Update the default storageclass * Move default storageclass setup from shell script to integration test * Add storageClassName field in PVC manifest of CSISnapshot integration test * Remove patch steps from CSISnapshot integration test * Run CSI driver install before minio setup in Travis CI * Install CSI hostpath driver in kind cluster that we setup in CI (#1212) * Install CSI driver * Update in install_csi_hostpath_driver method * Remove default storageclass patch * Update the default storageclass * Run CSI driver install before minio setup in Travis CI * Remove patch steps from CSISnapshot integration test * Rename test app & convert raw string manifests to Go structs * Add comment for app struct Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
kanisterio · Feb 4, 2022 · 307b053 · 307b053
1 parent 67c25b5
commit 307b053
Show file tree

Hide file tree

Showing 8 changed files with 355 additions and 4 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -46,6 +46,7 @@ jobs:
       before_script:
       - docker pull kindest/node:v1.21.1
       - make start-kind
+      - make install-csi-hostpath-driver
       - make install-minio
       script:
       - make integration-test
@@ -67,5 +68,5 @@ jobs:
         if [[ "$EXIT_STATUS" -eq 0 ]] && [[ "$TRAVIS_BRANCH" == "master" ]] && [[ "$TRAVIS_PULL_REQUEST" == "false" ]]; then
           sudo chown "$USER":"$USER" "$HOME"/.docker -R
           docker login ghcr.io -u "$GITHUB_USERNAME" -p "$GITHUB_TOKEN"
-          ./build/push_images.sh 
+          ./build/push_images.sh
         fi
diff --git a/Makefile b/Makefile
@@ -57,7 +57,7 @@ IMAGE_NAME := $(BIN)
 
 IMAGE := $(REGISTRY)/$(IMAGE_NAME)
 
-BUILD_IMAGE ?= ghcr.io/kanisterio/build:v0.0.16
+BUILD_IMAGE ?= ghcr.io/kanisterio/build:v0.0.17
 
 # tag 0.1.0 is, 0.0.1 (latest) + gh + aws + helm binary
 DOCS_BUILD_IMAGE ?= ghcr.io/kanisterio/docker-sphinx:0.2.0
@@ -265,6 +265,9 @@ tiller:
 install-minio:
 	@$(MAKE) run CMD='-c "./build/minio.sh install_minio"'
 
+install-csi-hostpath-driver:
+	@$(MAKE) run CMD='-c "./build/local_kubernetes.sh install_csi_hostpath_driver"'
+
 uninstall-minio:
 	@$(MAKE) run CMD='-c "./build/minio.sh uninstall_minio"'
 

diff --git a/build/integration-test.sh b/build/integration-test.sh
@@ -25,7 +25,7 @@ TEST_TIMEOUT="30m"
 # Set default options
 TEST_OPTIONS="-tags=integration -timeout ${TEST_TIMEOUT} -check.suitep ${DOP}"
 # Regex to match apps to run in short mode
-SHORT_APPS="^PostgreSQL$|^MySQL$|Elasticsearch|^MongoDB$|Maria|^MSSQL$"
+SHORT_APPS="^PostgreSQL$|^MySQL$|Elasticsearch|^MongoDB$|Maria|^MSSQL$|^TimeLogCSI$"
 # OCAPPS has all the apps that are to be tested against openshift cluster
 OC_APPS3_11="MysqlDBDepConfig$|MongoDBDepConfig$|PostgreSQLDepConfig$"
 OC_APPS4_4="MysqlDBDepConfig4_4|MongoDBDepConfig4_4|PostgreSQLDepConfig4_4"

diff --git a/build/local_kubernetes.sh b/build/local_kubernetes.sh
@@ -19,6 +19,7 @@ export KUBE_VERSION=${KUBE_VERSION:-"v1.21.1"}
 export KIND_VERSION=${KIND_VERSION:-"v0.11.1"}
 export LOCAL_CLUSTER_NAME=${LOCAL_CLUSTER_NAME:-"kanister"}
 export LOCAL_PATH_PROV_VERSION="v0.0.11"
+export SNAPSHOTTER_VERSION="v5.0.0"
 declare -a REQUIRED_BINS=( docker jq go )
 
 if command -v apt-get
@@ -61,6 +62,23 @@ start_localkube() {
     wait_for_pods
 }
 
+install_csi_hostpath_driver() {
+    # Install VolumeSnapshot CRDs
+    kubectl apply -fhttps://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${SNAPSHOTTER_VERSION}/client/config/crd/snapshot.storage.k8s.io_{volumesnapshots.yaml,volumesnapshotclasses.yaml,volumesnapshotcontents.yaml}
+
+    # Create snapshot controller
+    kubectl apply -fhttps://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${SNAPSHOTTER_VERSION}/deploy/kubernetes/snapshot-controller/{rbac-snapshot-controller.yaml,setup-snapshot-controller.yaml}
+
+    # Deploy the CSI Hostpath Driver
+    cd /tmp
+    git clone https://github.com/kubernetes-csi/csi-driver-host-path.git
+    cd csi-driver-host-path
+    ./deploy/kubernetes-1.21/deploy.sh
+
+    # Create StorageClass
+    kubectl apply -f ./examples/csi-storageclass.yaml
+}
+
 stop_localkube() {
     if ! command -v kind
     then
@@ -127,6 +145,7 @@ Where operation is one of the following:
   get_localkube: installs kind
   start_localkube : localkube start
   stop_localkube : localkube stop
+  install_csi_hostpath_driver : installs CSI hostpath driver
 EOM
     exit 1
 }
@@ -138,6 +157,9 @@ case "${1}" in
         get_localkube)
             time -p get_localkube
             ;;
+        install_csi_hostpath_driver)
+            time -p install_csi_hostpath_driver
+            ;;
         start_localkube)
             time -p start_localkube
             ;;

diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile
@@ -2,7 +2,7 @@ FROM alpine:3.10.5
 
 LABEL maintainer="Tom Manville<tom@kasten.io>"
 
-RUN apk add --update --no-cache ca-certificates bash git docker jq \
+RUN apk add --update --no-cache ca-certificates bash git docker curl jq \
     && update-ca-certificates \
     && rm -rf /var/cache/apk/*
 

diff --git a/pkg/app/csi-snapshot.go b/pkg/app/csi-snapshot.go
@@ -0,0 +1,241 @@
+// Copyright 2022 The Kanister Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package app
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+	appsv1 "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+
+	crv1alpha1 "github.com/kanisterio/kanister/pkg/apis/cr/v1alpha1"
+	"github.com/kanisterio/kanister/pkg/field"
+	"github.com/kanisterio/kanister/pkg/kube"
+	"github.com/kanisterio/kanister/pkg/log"
+)
+
+// Integration test app for CSI Snapshot functions
+type TimeLogCSI struct {
+	cli        kubernetes.Interface
+	namespace  string
+	name       string
+	deployment appsv1.Deployment
+	pvc        v1.PersistentVolumeClaim
+}
+
+// NewTimeLogCSI instantiates the TimeLogCSI integration test app
+func NewTimeLogCSI(name string) App {
+	return &TimeLogCSI{
+		name: name,
+	}
+}
+
+// Init initialises kubernetes CLI
+func (tlc *TimeLogCSI) Init(ctx context.Context) error {
+	cfg, err := kube.LoadConfig()
+	if err != nil {
+		return err
+	}
+	tlc.cli, err = kubernetes.NewForConfig(cfg)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Install deploys the TimeLogCSI App Deployment and PVC
+func (tlc *TimeLogCSI) Install(ctx context.Context, namespace string) error {
+	tlc.namespace = namespace
+	pvcObj := tlc.getAppPersistentVolumeClaimObj()
+	pvc, err := tlc.cli.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, pvcObj, metav1.CreateOptions{})
+	if err != nil {
+		return err
+	}
+	log.Print("PVC created successfully", field.M{"app": tlc.name, "pvc": pvc.Name})
+	tlc.pvc = *pvc
+	deploymentObj := tlc.getAppDeploymentObj()
+	deployment, err := tlc.cli.AppsV1().Deployments(namespace).Create(ctx, deploymentObj, metav1.CreateOptions{})
+	if err != nil {
+		return err
+	}
+	tlc.deployment = *deployment
+	log.Print("Deployment created successfully", field.M{"app": tlc.name, "deployment": deployment.Name})
+	return nil
+}
+
+// Reset deletes the log file present at the volume mount path
+func (tlc *TimeLogCSI) Reset(ctx context.Context) error {
+	log.Print("Resetting the application.", field.M{"app": tlc.name})
+
+	removeLogFileCmd := []string{"sh", "-c", "rm /var/log/time.log"}
+	stderr, err := tlc.execCommand(ctx, removeLogFileCmd)
+	if err != nil {
+		return errors.Wrapf(err, "Error while deleting log file: %s", stderr)
+	}
+
+	log.Print("Reset of the application was successful.", field.M{"app": tlc.name})
+	return nil
+}
+
+// IsReady waits for the App Deployment to be in 'Ready' state
+func (tlc *TimeLogCSI) IsReady(ctx context.Context) (bool, error) {
+	log.Print("Waiting for the application to be ready.", field.M{"app": tlc.name})
+	ctx, cancel := context.WithTimeout(ctx, mssqlWaitTimeout)
+	defer cancel()
+
+	err := kube.WaitOnDeploymentReady(ctx, tlc.cli, tlc.namespace, tlc.deployment.Name)
+	if err != nil {
+		return false, err
+	}
+	log.Print("Application instance is ready.", field.M{"app": tlc.name})
+	return true, nil
+}
+
+// Object defines the objectReference that will be used to create actions in blueprint
+func (tlc *TimeLogCSI) Object() crv1alpha1.ObjectReference {
+	return crv1alpha1.ObjectReference{
+		Kind:      "deployment",
+		Name:      tlc.name,
+		Namespace: tlc.namespace,
+	}
+}
+
+// Uninstall removes the TimeLogCSI app from the cluster
+func (tlc *TimeLogCSI) Uninstall(ctx context.Context) error {
+	err := tlc.cli.CoreV1().Namespaces().Delete(ctx, tlc.namespace, metav1.DeleteOptions{})
+	if err != nil {
+		return err
+	}
+	log.Print("Namespace deleted successfully", field.M{"app": tlc.name})
+	return nil
+}
+
+// Ping is used to check the connection with Deployment Pod
+func (tlc *TimeLogCSI) Ping(ctx context.Context) error {
+	log.Print("Pinging the application.", field.M{"app": tlc.name})
+
+	listDirectories := []string{"sh", "-c", "ls /var/log"}
+	stderr, err := tlc.execCommand(ctx, listDirectories)
+	if err != nil {
+		return errors.Wrapf(err, "Error while Pinging the application %s", stderr)
+	}
+
+	log.Print("Ping to the application was success.", field.M{"app": tlc.name})
+	return nil
+}
+
+func (tlc *TimeLogCSI) Insert(ctx context.Context) error {
+	return nil
+}
+
+func (tlc *TimeLogCSI) Initialize(ctx context.Context) error {
+	return nil
+}
+
+func (tlc *TimeLogCSI) Count(ctx context.Context) (int, error) {
+	return 3, nil
+}
+
+func (tlc *TimeLogCSI) Secrets() map[string]crv1alpha1.ObjectReference {
+	return nil
+}
+
+func (tlc *TimeLogCSI) ConfigMaps() map[string]crv1alpha1.ObjectReference {
+	return nil
+}
+
+func (tlc *TimeLogCSI) GetClusterScopedResources(ctx context.Context) []crv1alpha1.ObjectReference {
+	return nil
+}
+
+func (tlc *TimeLogCSI) execCommand(ctx context.Context, command []string) (string, error) {
+	podname, containername, err := kube.GetPodContainerFromDeployment(ctx, tlc.cli, tlc.namespace, tlc.name)
+	if err != nil || podname == "" {
+		return "", errors.Wrapf(err, "Error getting pod and containername %s.", tlc.name)
+	}
+	_, stderr, err := kube.Exec(tlc.cli, tlc.namespace, podname, containername, command, nil)
+	return stderr, err
+}
+
+func (tlc TimeLogCSI) getAppDeploymentObj() *appsv1.Deployment {
+	depReplicas := int32(1)
+	deployment := &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: tlc.name,
+		},
+		Spec: appsv1.DeploymentSpec{
+			Replicas: &depReplicas,
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{"app": tlc.name},
+			},
+			Template: v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{"app": tlc.name},
+				},
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{
+						{
+							Name:    "test-container",
+							Image:   "ghcr.io/kanisterio/kanister-tools:0.72.0",
+							Command: []string{"sh", "-c"},
+							Args:    []string{"while true; do for x in $(seq 1200); do date >> /var/log/time.log; sleep 1; done; truncate /var/log/time.log --size 0; done"},
+							VolumeMounts: []v1.VolumeMount{
+								{
+									Name:      "data",
+									MountPath: "/var/log",
+								},
+							},
+						},
+					},
+					Volumes: []v1.Volume{
+						{
+							Name: "data",
+							VolumeSource: v1.VolumeSource{
+								PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
+									ClaimName: "time-log-pvc",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	return deployment
+}
+
+func (tlc TimeLogCSI) getAppPersistentVolumeClaimObj() *v1.PersistentVolumeClaim {
+	storageClassName := "csi-hostpath-sc"
+	pvc := &v1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   "time-log-pvc",
+			Labels: map[string]string{"app": tlc.name},
+		},
+		Spec: v1.PersistentVolumeClaimSpec{
+			StorageClassName: &storageClassName,
+			AccessModes:      []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
+			Resources: v1.ResourceRequirements{
+				Requests: v1.ResourceList{
+					v1.ResourceStorage: *resource.NewQuantity(1073741824, resource.BinarySI),
+				},
+			},
+		},
+	}
+	return pvc
+}