diff --git a/e2e/cephfs.go b/e2e/cephfs.go index b3ae05864c2..9c336dd8f1f 100644 --- a/e2e/cephfs.go +++ b/e2e/cephfs.go @@ -2488,6 +2488,18 @@ var _ = Describe(cephfsType, func() { return } + By("Create volumeGroupSnapshot", func() { + scName := "csi-cephfs-sc" + base, err := newVolumeGroupSnapshotBase(f, scName, f.UniqueName, false, deployTimeout, 3) + if err != nil { + framework.Failf("failed to create volumeGroupSnapshot Base: %v", err) + } + snapshotter := newCephFSVolumeGroupSnapshot(f) + err = base.testVolumeGroupSnapshot(snapshotter) + if err != nil { + framework.Failf("failed to test volumeGroupSnapshot: %v", err) + } + }) // Make sure this should be last testcase in this file, because // it deletes pool By("Create a PVC and delete PVC when backend pool deleted", func() { diff --git a/e2e/volumegroupsnapshot.go b/e2e/volumegroupsnapshot.go new file mode 100644 index 00000000000..73e01a0365d --- /dev/null +++ b/e2e/volumegroupsnapshot.go @@ -0,0 +1,103 @@ +/* +Copyright 2024 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + + groupsnapapi "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/test/e2e/framework" +) + +type cephFS struct { + framework *framework.Framework +} + +var _ volumeGroupSnapshotter = &cephFS{} + +func newCephFSVolumeGroupSnapshot(f *framework.Framework) volumeGroupSnapshotter { + return &cephFS{ + framework: f, + } +} + +func (c *cephFS) GetVolumeGroupSnapshotClass() (*groupsnapapi.VolumeGroupSnapshotClass, error) { + vgscPath := fmt.Sprintf("%s/%s", cephFSExamplePath, "groupsnapshotclass.yaml") + vgsc := &groupsnapapi.VolumeGroupSnapshotClass{} + err := unmarshal(vgscPath, vgsc) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal VolumeGroupSnapshotClass: %w", err) + } + + vgsc.Parameters["csi.storage.k8s.io/group-snapshotter-secret-namespace"] = cephCSINamespace + vgsc.Parameters["csi.storage.k8s.io/group-snapshotter-secret-name"] = cephFSProvisionerSecretName + vgsc.Parameters["fsName"] = fileSystemName + + fsID, err := getClusterID(c.framework) + if err != nil { + return nil, fmt.Errorf("failed to get clusterID: %w", err) + } + vgsc.Parameters["clusterID"] = fsID + + return vgsc, nil +} + +func (c *cephFS) ValidateResourcesForCreate(vgs *groupsnapapi.VolumeGroupSnapshot) error { + metadataPool, err := getCephFSMetadataPoolName(c.framework, fileSystemName) + if err != nil { + return fmt.Errorf("failed getting cephFS metadata pool name: %w", err) + } + + sourcePVCCount := len(vgs.Status.PVCVolumeSnapshotRefList) + // we are creating clones for each source PVC + clonePVCCount := len(vgs.Status.PVCVolumeSnapshotRefList) + totalPVCCount := sourcePVCCount + clonePVCCount + validateSubvolumeCount(c.framework, totalPVCCount, fileSystemName, subvolumegroup) + + // we are creating 1 snapshot for each source PVC, validate the snapshot count + for _, pvcSnap := range vgs.Status.PVCVolumeSnapshotRefList { + pvc, err := c.framework.ClientSet.CoreV1().PersistentVolumeClaims(vgs.Namespace).Get(context.TODO(), + pvcSnap.PersistentVolumeClaimRef.Name, + metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get PVC: %w", err) + } + pv := pvc.Spec.VolumeName + pvObj, err := c.framework.ClientSet.CoreV1().PersistentVolumes().Get(context.TODO(), pv, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get PV: %w", err) + } + validateCephFSSnapshotCount(c.framework, 1, subvolumegroup, pvObj) + } + validateOmapCount(c.framework, totalPVCCount, cephfsType, metadataPool, volumesType) + validateOmapCount(c.framework, totalPVCCount, cephfsType, metadataPool, snapsType) + + return nil +} + +func (c *cephFS) ValidateResourcesForDelete() error { + metadataPool, err := getCephFSMetadataPoolName(c.framework, fileSystemName) + if err != nil { + return fmt.Errorf("failed getting cephFS metadata pool name: %w", err) + } + validateOmapCount(c.framework, 0, cephfsType, metadataPool, volumesType) + validateOmapCount(c.framework, 0, cephfsType, metadataPool, snapsType) + + return nil +} diff --git a/e2e/volumegroupsnapshot_base.go b/e2e/volumegroupsnapshot_base.go new file mode 100644 index 00000000000..561e9119f4d --- /dev/null +++ b/e2e/volumegroupsnapshot_base.go @@ -0,0 +1,409 @@ +/* +Copyright 2024 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "time" + + groupsnapapi "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1alpha1" + snapapi "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + groupsnapclient "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/typed/volumegroupsnapshot/v1alpha1" + v1 "k8s.io/api/core/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/kubernetes/test/e2e/framework" +) + +// VolumeGroupSnapshotter defines the common operations for handling +// volume group snapshots. +type VolumeGroupSnapshotter interface { + // CreateVolumeGroupSnapshotClass creates a volume group snapshot class. + CreateVolumeGroupSnapshotClass(vgsc *groupsnapapi.VolumeGroupSnapshotClass) error + // CreateVolumeGroupSnapshot creates a groupsnapshot with the specified name + // namespace and volume group snapshot class. + CreateVolumeGroupSnapshot(volumeGroupSnapshotName, + namespace, + volumeGroupSnapshotClassName string, + ) (*groupsnapapi.VolumeGroupSnapshot, error) + // DeleteVolumeGroupSnapshot deletes the specified volume + // group snapshot in the specified namespace. + DeleteVolumeGroupSnapshot(volumeGroupSnapshotName, namespace string) error + // DeleteVolumeGroupSnapshotClass deletes the specified volume + // group snapshot class. + DeleteVolumeGroupSnapshotClass(snapshotClassName string) error + // CreatePVCs creates PVCs with the specified namespace and labels. + CreatePVCs(namespace string, + labels map[string]string) ([]*v1.PersistentVolumeClaim, error) + // DeletePVCs deletes the specified PVCs. + DeletePVCs(pvcs []*v1.PersistentVolumeClaim) error + // CreatePVCClones creates pvcs from all the snapshots in VolumeGroupSnapshot. + CreatePVCClones(vgs *groupsnapapi.VolumeGroupSnapshot, + scName string) ([]*v1.PersistentVolumeClaim, error) +} + +// volumeGroupSnapshotter defines validation operations specific to each driver. +type volumeGroupSnapshotter interface { + // GetVolumeGroupSnapshotClass returns the volume group snapshot class. + GetVolumeGroupSnapshotClass() (*groupsnapapi.VolumeGroupSnapshotClass, error) + // ValidateResourcesForCreate validates the resources in the backend after + // creating clones. + ValidateResourcesForCreate(vgs *groupsnapapi.VolumeGroupSnapshot) error + // ValidateSnapshotsDeleted checks if all resources are deleted in the + // backend after all the resources are deleted. + ValidateResourcesForDelete() error +} + +type volumeGroupSnapshotterBase struct { + timeout int + framework *framework.Framework + groupclient *groupsnapclient.GroupsnapshotV1alpha1Client + storageClassName string + blockPVC bool + totalPVCCount int + namespace string +} + +func newVolumeGroupSnapshotBase(f *framework.Framework, namespace, + storageClass string, + blockPVC bool, + timeout, totalPVCCount int, +) (*volumeGroupSnapshotterBase, error) { + config, err := framework.LoadConfig() + if err != nil { + return nil, fmt.Errorf("error creating group snapshot client: %w", err) + } + c, err := groupsnapclient.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("error creating group snapshot client: %w", err) + } + + return &volumeGroupSnapshotterBase{ + framework: f, + groupclient: c, + namespace: namespace, + storageClassName: storageClass, + blockPVC: blockPVC, + timeout: timeout, + totalPVCCount: totalPVCCount, + }, err +} + +func (v *volumeGroupSnapshotterBase) CreatePVCs(namespace string, + labels map[string]string, +) ([]*v1.PersistentVolumeClaim, error) { + pvcs := make([]*v1.PersistentVolumeClaim, v.totalPVCCount) + for i := range v.totalPVCCount { + pvcs[i] = &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("pvc-%d", i), + Namespace: namespace, + }, + Spec: v1.PersistentVolumeClaimSpec{ + Resources: v1.VolumeResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: resource.MustParse("1Gi"), + }, + }, + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + StorageClassName: &v.storageClassName, + }, + } + if v.blockPVC { + volumeMode := v1.PersistentVolumeBlock + pvcs[i].Spec.VolumeMode = &volumeMode + } else { + volumeMode := v1.PersistentVolumeFilesystem + pvcs[i].Spec.VolumeMode = &volumeMode + } + pvcs[i].Labels = labels + err := createPVCAndvalidatePV(v.framework.ClientSet, pvcs[i], v.timeout) + if err != nil { + return nil, fmt.Errorf("failed to create PVC: %w", err) + } + } + + return pvcs, nil +} + +func (v *volumeGroupSnapshotterBase) DeletePVCs(pvcs []*v1.PersistentVolumeClaim) error { + for _, pvc := range pvcs { + err := deletePVCAndValidatePV(v.framework.ClientSet, pvc, v.timeout) + if err != nil { + return fmt.Errorf("failed to delete PVC: %w", err) + } + } + + return nil +} + +func (v *volumeGroupSnapshotterBase) CreatePVCClones(vgs *groupsnapapi.VolumeGroupSnapshot, + scName string, +) ([]*v1.PersistentVolumeClaim, error) { + pvcSnapRef := vgs.Status.PVCVolumeSnapshotRefList + namespace := vgs.Namespace + ctx := context.TODO() + pvcs := make([]*v1.PersistentVolumeClaim, len(pvcSnapRef)) + for i, pvcSnap := range pvcSnapRef { + pvc, err := v.framework.ClientSet.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, + pvcSnap.PersistentVolumeClaimRef.Name, + metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get PVC: %w", err) + } + pvcs[i] = pvc.DeepCopy() + pvcs[i].Name = fmt.Sprintf("%s-clone-%d", pvc.Name, i) + snap := pvcSnap.VolumeSnapshotRef + apiGroup := snapapi.GroupName + pvcs[i].Spec.DataSource = &v1.TypedLocalObjectReference{ + APIGroup: &apiGroup, + Kind: "VolumeSnapshot", + Name: snap.Name, + } + pvcs[i].Spec.StorageClassName = &scName + + err = createPVCAndvalidatePV(v.framework.ClientSet, pvc, v.timeout) + if err != nil { + return nil, fmt.Errorf("failed to create PVC: %w", err) + } + } + + return pvcs, nil +} + +func (v volumeGroupSnapshotterBase) CreateVolumeGroupSnapshotClass( + groupSnapshotClass *groupsnapapi.VolumeGroupSnapshotClass, +) error { + return wait.PollUntilContextTimeout( + context.TODO(), + poll, + time.Duration(v.timeout), + true, + func(ctx context.Context) (bool, error) { + _, err := v.groupclient.VolumeGroupSnapshotClasses().Create(ctx, groupSnapshotClass, metav1.CreateOptions{}) + if err != nil { + framework.Logf("error creating VolumeGroupSnapshotClass %q: %v", groupSnapshotClass.Name, err) + if isRetryableAPIError(err) { + return false, nil + } + + return false, fmt.Errorf("failed to create VolumeGroupSnapshotClass %q: %w", groupSnapshotClass.Name, err) + } + + return true, nil + }) +} + +func (v volumeGroupSnapshotterBase) CreateVolumeGroupSnapshot(name, + volumeGroupSnapshotClassName string, labels map[string]string, +) (*groupsnapapi.VolumeGroupSnapshot, error) { + namespace := v.namespace + groupSnapshot := &groupsnapapi.VolumeGroupSnapshot{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: groupsnapapi.VolumeGroupSnapshotSpec{ + Source: groupsnapapi.VolumeGroupSnapshotSource{ + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + }, + VolumeGroupSnapshotClassName: &volumeGroupSnapshotClassName, + }, + } + ctx := context.TODO() + _, err := v.groupclient.VolumeGroupSnapshots(namespace).Create(ctx, groupSnapshot, metav1.CreateOptions{}) + if err != nil { + + return nil, fmt.Errorf("failed to create VolumeGroupSnapshot %q: %w", name, err) + } + + framework.Logf("VolumeGroupSnapshot with name %v created in %v namespace", name, namespace) + + timeout := time.Duration(v.timeout) * time.Minute + start := time.Now() + framework.Logf("waiting for %+v to be in ready state", groupSnapshot) + + err = wait.PollUntilContextTimeout(ctx, poll, timeout, true, func(ctx context.Context) (bool, error) { + framework.Logf("waiting for VolumeGroupSnapshot %s (%d seconds elapsed)", name, int(time.Since(start).Seconds())) + groupSnapshot, err = v.groupclient.VolumeGroupSnapshots(namespace). + Get(ctx, name, metav1.GetOptions{}) + if err != nil { + framework.Logf("Error getting VolumeGroupSnapshot in namespace: '%s': %v", namespace, err) + if isRetryableAPIError(err) { + return false, nil + } + if apierrs.IsNotFound(err) { + return false, nil + } + + return false, fmt.Errorf("failed to get volumesnapshot: %w", err) + } + if groupSnapshot.Status == nil || groupSnapshot.Status.ReadyToUse == nil { + return false, nil + } + if *groupSnapshot.Status.ReadyToUse { + return true, nil + } + + framework.Logf("VolumeGroupSnapshot %s status %+v", groupSnapshot.Name, *groupSnapshot.Status) + + return false, nil + }) + if err != nil { + return nil, fmt.Errorf("failed to get VolumeGroupSnapshot %s: %w", name, err) + } + + return groupSnapshot, nil +} + +func (v volumeGroupSnapshotterBase) DeleteVolumeGroupSnapshot(volumeGroupSnapshotName, namespace string) error { + err := v.groupclient.VolumeGroupSnapshots(namespace).Delete( + context.TODO(), + volumeGroupSnapshotName, + metav1.DeleteOptions{}) + if err != nil { + return fmt.Errorf("failed to delete VolumeGroupSnapshot: %w", err) + } + start := time.Now() + framework.Logf("Waiting for VolumeGroupSnapshot %v to be deleted", volumeGroupSnapshotName) + timeout := time.Duration(v.timeout) * time.Minute + ctx := context.TODO() + + return wait.PollUntilContextTimeout( + ctx, + poll, + timeout, + true, + func(ctx context.Context) (bool, error) { + _, err := v.groupclient.VolumeGroupSnapshots(namespace).Get(ctx, volumeGroupSnapshotName, metav1.GetOptions{}) + if err != nil { + if isRetryableAPIError(err) { + return false, nil + } + if apierrs.IsNotFound(err) { + return true, nil + } + framework.Logf("%s VolumeGroupSnapshot to be deleted (%d seconds elapsed)", + volumeGroupSnapshotName, + int(time.Since(start).Seconds())) + + return false, fmt.Errorf("failed to get VolumeGroupSnapshot: %w", err) + } + + return false, nil + }) +} + +func (v volumeGroupSnapshotterBase) DeleteVolumeGroupSnapshotClass(groupSnapshotClassName string) error { + ctx := context.TODO() + err := v.groupclient.VolumeGroupSnapshotClasses().Delete( + ctx, groupSnapshotClassName, metav1.DeleteOptions{}) + if err != nil { + return fmt.Errorf("failed to delete VolumeGroupSnapshotClass: %w", err) + } + start := time.Now() + framework.Logf("Waiting for VolumeGroupSnapshotClass %v to be deleted", groupSnapshotClassName) + timeout := time.Duration(v.timeout) * time.Minute + + return wait.PollUntilContextTimeout( + ctx, + poll, + timeout, + true, + func(ctx context.Context) (bool, error) { + _, err := v.groupclient.VolumeGroupSnapshotClasses().Get(ctx, groupSnapshotClassName, metav1.GetOptions{}) + if err != nil { + if isRetryableAPIError(err) { + return false, nil + } + if apierrs.IsNotFound(err) { + return true, nil + } + framework.Logf("%s VolumeGroupSnapshotClass to be deleted (%d seconds elapsed)", + groupSnapshotClassName, + int(time.Since(start).Seconds())) + + return false, fmt.Errorf("failed to get VolumeGroupSnapshotClass: %w", err) + } + + return false, nil + }) +} + +func (v *volumeGroupSnapshotterBase) testVolumeGroupSnapshot(vol volumeGroupSnapshotter) error { + pvcLabels := map[string]string{"pvc": "vgsc"} + pvcs, err := v.CreatePVCs(v.namespace, pvcLabels) + if err != nil { + return fmt.Errorf("failed to create PVCs: %w", err) + } + + vgsc, err := vol.GetVolumeGroupSnapshotClass() + if err != nil { + return fmt.Errorf("failed to get volume group snapshot class: %w", err) + } + // Create a volume group snapshot class + vgscName := v.framework.Namespace.Name + "-vgsc" + vgsc.Name = vgscName + err = v.CreateVolumeGroupSnapshotClass(vgsc) + if err != nil { + return fmt.Errorf("failed to create volume group snapshot: %w", err) + } + vgsName := v.framework.Namespace.Name + "-vgs" + // Create a volume group snapshot + volumeGroupSnapshot, err := v.CreateVolumeGroupSnapshot(vgsName, vgscName, pvcLabels) + if err != nil { + return fmt.Errorf("failed to create volume group snapshot: %w", err) + } + + clonePVCs, err := v.CreatePVCClones(volumeGroupSnapshot, v.storageClassName) + if err != nil { + return fmt.Errorf("failed to create clones and pods: %w", err) + } + // validate the resources in the backend + err = vol.ValidateResourcesForCreate(volumeGroupSnapshot) + if err != nil { + return fmt.Errorf("failed to validate resources for create: %w", err) + } + + // Delete the clones + err = v.DeletePVCs(clonePVCs) + if err != nil { + return fmt.Errorf("failed to delete clones: %w", err) + } + // Delete the PVCs + err = v.DeletePVCs(pvcs) + if err != nil { + return fmt.Errorf("failed to delete PVCs: %w", err) + } + // Delete the volume group snapshot + err = v.DeleteVolumeGroupSnapshot(volumeGroupSnapshot.Name, "default") + if err != nil { + return fmt.Errorf("failed to delete volume group snapshot: %w", err) + } + // validate the resources in the backend after deleting the resources + err = vol.ValidateResourcesForDelete() + if err != nil { + return fmt.Errorf("failed to validate resources for delete: %w", err) + } + + return nil +}