Skip to content

Commit

Permalink
Adding a Kanister function to restore CSI VolumeSnapshot (#1166)
Browse files Browse the repository at this point in the history
* Added new function RestoreCSISnapshot

* reviewer comments

* Fixed failed to render outputartifact template 'snapshotInfo' issue

* Updated const comments

* Updated the return obj for the function

* Added documentation for RestoreCSISnapshot function

* Added test suite for RestoreCSISnapshot func

* gofmt update

* Reviewer comments

* Updated the testcase for RestoreCSISnapshot function

* Updated the createPVC method in testcase

* Fixed RBAC YAML

* Added validations for volumeMode and accessMode

* Updated the logic to validate volume mode

* Fixed the validateVolumeModeArg method

* Added testcase for arg validation

* gofmt updates

* Added table driven testcases for validate methods

* Removed multiple context.Background() calls and removed the validate method calls in main test case

* style: Add whitespace in error messages

* refactor: Update const names in test files

* refactor: Update length of random alphanumeric suffix to 5 characters

* refactor: Update argument validation methods

* style: Fix typo in a comment

* refactor: Update call to validate methods in restore unit test

* refactor: Update restore test case as per reviewer comment

* refactor: Update CreateCSiSnapshot function and the test case

* refactor: Revert random alphanumeric string length

* fix: gofmt error

* refactor: Update CreateCSiSnapshot function and the test case

* refactor: Update CreateCSiSnapshot function and the test case

* refactor: Update CreateCSiSnapshot function and the test case

* chore: Update signature of the restoreCSISnapshot method

* refactor: Revert CreateCSISnapshot Func changes in this PR

* refactor: Update create snapshot unit test

* fix: CI failure

* chore: Comment update

* fix: golangci-lint fix

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
  • Loading branch information
Shlok Chaudhari and mergify[bot] authored Jan 17, 2022
1 parent 0362c2f commit fe499ac
Show file tree
Hide file tree
Showing 5 changed files with 466 additions and 29 deletions.
46 changes: 46 additions & 0 deletions docs/functions.rst
Original file line number Diff line number Diff line change
Expand Up @@ -1328,6 +1328,52 @@ Example:
snapshotClass: do-block-storage
RestoreCSISnapshot
------------------

This function restores a new PersistentVolumeClaim using CSI VolumeSnapshot.

Arguments:

.. csv-table::
:header: "Argument", "Required", "Type", "Description"
:align: left
:widths: 5,5,5,15

`name`, Yes, `string`, name of the VolumeSnapshot
`pvc`, Yes, `string`, name of the new PVC
`namespace`, Yes, `string`, namespace of the VolumeSnapshot and resultant PersistentVolumeClaim
`storageClass`, Yes, `string`, name of the StorageClass
`restoreSize`, Yes, `string`, required memory size to restore PVC
`accessModes`, No, `[]string`, access modes for the underlying PV (Default is ``[]{"ReadWriteOnce"}```)
`volumeMode`, No, `string`, mode of volume (Default is ``"Filesystem"```)
`labels`, No, `map[string]string`, optional labels for the PersistentVolumeClaim

.. note::
Output artifact ``snapshotInfo`` from ``CreateCSISnapshot`` function can be used as an input artifact in this function.

Example:

.. code-block:: yaml
:linenos:
actions:
restore:
inputArtifactNames:
- snapshotInfo
phases:
- func: RestoreCSISnapshot
name: restoreCSISnapshot
args:
name: "{{ .ArtifactsIn.snapshotInfo.KeyValue.name }}"
pvc: "{{ .ArtifactsIn.snapshotInfo.KeyValue.pvc }}-restored"
namespace: "{{ .ArtifactsIn.snapshotInfo.KeyValue.namespace }}"
storageClass: do-block-storage
restoreSize: "{{ .ArtifactsIn.snapshotInfo.KeyValue.restoreSize }}"
accessModes: ["ReadWriteOnce"]
volumeMode: "Filesystem"
Registering Functions
---------------------

Expand Down
21 changes: 15 additions & 6 deletions pkg/function/create_csi_snapshot.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"context"
"fmt"

v1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
"github.com/pkg/errors"
"k8s.io/apimachinery/pkg/util/rand"

Expand Down Expand Up @@ -73,7 +74,7 @@ func (*createCSISnapshotFunc) Exec(ctx context.Context, tp param.TemplateParams,
if err := Arg(args, CreateCSISnapshotSnapshotClassArg, &snapshotClass); err != nil {
return nil, err
}
if err := OptArg(args, CreateCSISnapshotNameArg, &name, defaultSnapshotName(pvc, 20)); err != nil {
if err := OptArg(args, CreateCSISnapshotNameArg, &name, defaultSnapshotName(pvc, 5)); err != nil {
return nil, err
}
if err := OptArg(args, CreateCSISnapshotLabelsArg, &labels, map[string]string{}); err != nil {
Expand All @@ -98,10 +99,7 @@ func (*createCSISnapshotFunc) Exec(ctx context.Context, tp param.TemplateParams,
}
// waitForReady is set to true by default because snapshot information is needed as output artifacts
waitForReady := true
if err := snapshotter.Create(ctx, name, namespace, pvc, &snapshotClass, waitForReady, labels); err != nil {
return nil, err
}
vs, err := snapshotter.Get(ctx, name, namespace)
vs, err := createCSISnapshot(ctx, snapshotter, name, namespace, pvc, snapshotClass, waitForReady, labels)
if err != nil {
return nil, err
}
Expand All @@ -124,7 +122,18 @@ func (*createCSISnapshotFunc) RequiredArgs() []string {
}
}

// defaultSnapshotName generates snapshot name using pvcName-snapshot-randomValue
func createCSISnapshot(ctx context.Context, snapshotter snapshot.Snapshotter, name, namespace, pvc, snapshotClass string, wait bool, labels map[string]string) (*v1.VolumeSnapshot, error) {
if err := snapshotter.Create(ctx, name, namespace, pvc, &snapshotClass, wait, labels); err != nil {
return nil, err
}
vs, err := snapshotter.Get(ctx, name, namespace)
if err != nil {
return nil, err
}
return vs, nil
}

// defaultSnapshotName generates snapshot name using <pvcName>-snapshot-<randomValue>
func defaultSnapshotName(pvcName string, len int) string {
return fmt.Sprintf("%s-snapshot-%s", pvcName, rand.String(len))
}
43 changes: 20 additions & 23 deletions pkg/function/create_csi_snapshot_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,16 +29,16 @@ import (
)

const (
// CreateCSISnapshotTestNamespace is the namespace where testing is done
CreateCSISnapshotTestNamespace = "test-create-csi-snapshot"
// CreateCSISnapshotPVCName is the name of the PVC that will be captured
CreateCSISnapshotPVCName = "test-pvc"
// CreateCSISnapshotSnapshotName is the name of the snapshot
CreateCSISnapshotSnapshotName = "test-snapshot"
// CreateCSISnapshotSnapshotClass is the fake snapshot class
CreateCSISnapshotSnapshotClass = "test-snapshot-class"
// CreateCSISnapshotStorageClass is the fake storage class
CreateCSISnapshotStorageClass = "test-storage-class"
// testCreateNamespace is the namespace where testing is done
testCreateNamespace = "test-create-csi-snapshot"
// pvcName is the name of the PVC that will be captured
pvcName = "test-pvc"
// snapshotName is the name of the snapshot
snapshotName = "test-snapshot"
// snapshotClass is the fake snapshot class
snapshotClass = "test-snapshot-class"
// storageClass is the fake storage class
storageClass = "test-storage-class"
)

type CreateCSISnapshotTestSuite struct {
Expand All @@ -52,11 +52,11 @@ type CreateCSISnapshotTestSuite struct {
var _ = Suite(&CreateCSISnapshotTestSuite{})

func (testSuite *CreateCSISnapshotTestSuite) SetUpSuite(c *C) {
testSuite.volumeSnapshotClass = CreateCSISnapshotSnapshotClass
testSuite.storageClass = CreateCSISnapshotStorageClass
testSuite.pvcName = CreateCSISnapshotPVCName
testSuite.snapName = CreateCSISnapshotSnapshotName
testSuite.namespace = CreateCSISnapshotTestNamespace
testSuite.volumeSnapshotClass = snapshotClass
testSuite.storageClass = storageClass
testSuite.pvcName = pvcName
testSuite.snapName = snapshotName
testSuite.namespace = testCreateNamespace
}

func (testSuite *CreateCSISnapshotTestSuite) TestCreateCSISnapshot(c *C) {
Expand All @@ -83,27 +83,24 @@ func (testSuite *CreateCSISnapshotTestSuite) TestCreateCSISnapshot(c *C) {
GroupVersion: "snapshot.storage.k8s.io/v1",
},
} {
ctx := context.Background()
fakeCli := fake.NewSimpleClientset()
fakeCli.Resources = []*metav1.APIResourceList{apiResourceList}

_, err := fakeCli.CoreV1().Namespaces().Create(context.TODO(), &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testSuite.namespace}}, metav1.CreateOptions{})
_, err := fakeCli.CoreV1().Namespaces().Create(ctx, &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: testSuite.namespace}}, metav1.CreateOptions{})
c.Assert(err, IsNil)

scheme := runtime.NewScheme()
fakeSnapshotter, err := snapshot.NewSnapshotter(fakeCli, dynfake.NewSimpleDynamicClient(scheme))
c.Assert(err, IsNil)

_, err = fakeCli.CoreV1().PersistentVolumeClaims(testSuite.namespace).Create(context.TODO(), getPVCManifest(testSuite.pvcName, testSuite.storageClass), metav1.CreateOptions{})
_, err = fakeCli.CoreV1().PersistentVolumeClaims(testSuite.namespace).Create(ctx, getPVCManifest(testSuite.pvcName, testSuite.storageClass), metav1.CreateOptions{})
c.Assert(err, IsNil)

err = fakeSnapshotter.Create(context.Background(), testSuite.snapName, testSuite.namespace, testSuite.pvcName, &testSuite.volumeSnapshotClass, false, nil)
_, err = createCSISnapshot(ctx, fakeSnapshotter, testSuite.snapName, testSuite.namespace, testSuite.pvcName, testSuite.volumeSnapshotClass, false, nil)
c.Assert(err, IsNil)

vs, err := fakeSnapshotter.Get(context.Background(), testSuite.snapName, testSuite.namespace)
c.Assert(err, IsNil)
c.Assert(vs.Name, Equals, testSuite.snapName)

err = fakeCli.CoreV1().Namespaces().Delete(context.Background(), testSuite.namespace, metav1.DeleteOptions{})
err = fakeCli.CoreV1().Namespaces().Delete(ctx, testSuite.namespace, metav1.DeleteOptions{})
c.Assert(err, IsNil)
}
}
Expand Down
201 changes: 201 additions & 0 deletions pkg/function/restore_csi_snapshot.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,201 @@
// Copyright 2022 The Kanister Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package function

import (
"context"
"errors"

v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"

kanister "github.com/kanisterio/kanister/pkg"
"github.com/kanisterio/kanister/pkg/kube"
"github.com/kanisterio/kanister/pkg/param"
)

func init() {
_ = kanister.Register(&restoreCSISnapshotFunc{})
}

var (
_ kanister.Func = (*restoreCSISnapshotFunc)(nil)
)

const (
// Snapshot API Group
SnapshotAPIGroup = "snapshot.storage.k8s.io"
// RestoreCSISnapshotFuncName gives the name of the function
RestoreCSISnapshotFuncName = "RestoreCSISnapshot"
// RestoreCSISnapshotNameArg provides name of the VolumeSnapshot
RestoreCSISnapshotNameArg = "name"
// RestoreCSISnapshotPVCNameArg gives the name of the newly restored PVC
RestoreCSISnapshotPVCNameArg = "pvc"
// RestoreCSISnapshotNamespaceArg mentions the namespace of the newly restored PVC
RestoreCSISnapshotNamespaceArg = "namespace"
// RestoreCSISnapshotStorageClassArg specifies the name of the StorageClass
RestoreCSISnapshotStorageClassArg = "storageClass"
// RestoreCSISnapshotStorageRequestArg provides the storage size to be requested for PV in PVC
RestoreCSISnapshotRestoreSizeArg = "restoreSize"
// RestoreCSISnapshotAccessModesArg lists down the accessmodes for the underlying PV
RestoreCSISnapshotAccessModesArg = "accessModes"
// RestoreCSISnapshotLabelsArg has labels that will be added to the newly restored PVC
RestoreCSISnapshotLabelsArg = "labels"
// RestoreCSISnapshotVolumeModeArg defines mode of volume
RestoreCSISnapshotVolumeModeArg = "volumeMode"
)

type restoreCSISnapshotFunc struct{}

type restoreCSISnapshotArgs struct {
Name string
PVC string
Namespace string
StorageClass string
RestoreSize *resource.Quantity
AccessModes []v1.PersistentVolumeAccessMode
Labels map[string]string
VolumeMode v1.PersistentVolumeMode
}

func (*restoreCSISnapshotFunc) Name() string {
return RestoreCSISnapshotFuncName
}

func (*restoreCSISnapshotFunc) Exec(ctx context.Context, tp param.TemplateParams, args map[string]interface{}) (map[string]interface{}, error) {
var restoreSize string
var restoreArgs restoreCSISnapshotArgs
if err := Arg(args, RestoreCSISnapshotNameArg, &restoreArgs.Name); err != nil {
return nil, err
}
if err := Arg(args, RestoreCSISnapshotPVCNameArg, &restoreArgs.PVC); err != nil {
return nil, err
}
if err := Arg(args, RestoreCSISnapshotNamespaceArg, &restoreArgs.Namespace); err != nil {
return nil, err
}
if err := Arg(args, RestoreCSISnapshotStorageClassArg, &restoreArgs.StorageClass); err != nil {
return nil, err
}
if err := Arg(args, RestoreCSISnapshotRestoreSizeArg, &restoreSize); err != nil {
return nil, err
}
if err := OptArg(args, RestoreCSISnapshotAccessModesArg, &restoreArgs.AccessModes, []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}); err != nil {
return nil, err
}
if err := validateVolumeAccessModesArg(restoreArgs.AccessModes); err != nil {
return nil, err
}
if err := OptArg(args, RestoreCSISnapshotVolumeModeArg, &restoreArgs.VolumeMode, v1.PersistentVolumeFilesystem); err != nil {
return nil, err
}
if err := validateVolumeModeArg(restoreArgs.VolumeMode); err != nil {
return nil, err
}
if err := OptArg(args, RestoreCSISnapshotLabelsArg, &restoreArgs.Labels, nil); err != nil {
return nil, err
}
size, err := resource.ParseQuantity(restoreSize)
if err != nil {
return nil, err
}
restoreArgs.RestoreSize = &size

kubeCli, err := getClient()
if err != nil {
return nil, err
}
if _, err := restoreCSISnapshot(ctx, kubeCli, restoreArgs); err != nil {
return nil, err
}
return nil, nil
}

func (*restoreCSISnapshotFunc) RequiredArgs() []string {
return []string{
RestoreCSISnapshotNameArg,
RestoreCSISnapshotPVCNameArg,
RestoreCSISnapshotNamespaceArg,
RestoreCSISnapshotStorageClassArg,
RestoreCSISnapshotRestoreSizeArg,
}
}

func getClient() (kubernetes.Interface, error) {
kubeCli, err := kube.NewClient()
return kubeCli, err
}

func restoreCSISnapshot(ctx context.Context, kubeCli kubernetes.Interface, args restoreCSISnapshotArgs) (*v1.PersistentVolumeClaim, error) {
pvc := newPVCManifest(args)
if _, err := kubeCli.CoreV1().PersistentVolumeClaims(args.Namespace).Create(ctx, pvc, metav1.CreateOptions{}); err != nil {
return nil, err
}
return pvc, nil
}

func newPVCManifest(args restoreCSISnapshotArgs) *v1.PersistentVolumeClaim {
snapshotAPIGroup := SnapshotAPIGroup
pvc := &v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: args.PVC,
Namespace: args.Namespace,
},
Spec: v1.PersistentVolumeClaimSpec{
AccessModes: args.AccessModes,
VolumeMode: &args.VolumeMode,
DataSource: &v1.TypedLocalObjectReference{
APIGroup: &snapshotAPIGroup,
Kind: "VolumeSnapshot",
Name: args.Name,
},
StorageClassName: &args.StorageClass,
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceStorage: *args.RestoreSize,
},
},
},
}
if args.Labels != nil {
pvc.ObjectMeta.Labels = args.Labels
}
return pvc
}

func validateVolumeModeArg(volumeMode v1.PersistentVolumeMode) error {
switch volumeMode {
case v1.PersistentVolumeFilesystem,
v1.PersistentVolumeBlock:
default:
return errors.New("Given volumeMode " + string(volumeMode) + " is invalid")
}
return nil
}

func validateVolumeAccessModesArg(accessModes []v1.PersistentVolumeAccessMode) error {
for _, accessModeInArg := range accessModes {
switch accessModeInArg {
case v1.ReadOnlyMany,
v1.ReadWriteMany,
v1.ReadWriteOnce:
default:
return errors.New("Given accessMode " + string(accessModeInArg) + " is invalid")
}
}
return nil
}
Loading

0 comments on commit fe499ac

Please sign in to comment.