From e9cb542d87aea724d6ee664937ee69bcc4d2f5db Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Fri, 15 Mar 2024 17:18:02 +0100 Subject: [PATCH] [WIP] cleanups and more function implementations --- internal/rbd/group_controller.go | 134 ++++++++++++-------- internal/rbd_group/snapshot.go | 19 ++- internal/rbd_group/volume_group.go | 105 +++++++++++++-- internal/rbd_group/volume_group_snapshot.go | 68 +++++++++- internal/rbd_types/group.go | 13 +- internal/rbd_types/snapshot.go | 7 +- 6 files changed, 270 insertions(+), 76 deletions(-) diff --git a/internal/rbd/group_controller.go b/internal/rbd/group_controller.go index 5fcecc638b8e..6b1e421475ad 100644 --- a/internal/rbd/group_controller.go +++ b/internal/rbd/group_controller.go @@ -31,11 +31,12 @@ import ( // cephConfig contains the configuration parameters for the Ceph cluster. type cephConfig struct { - clusterID string - mons string - pool string - journalPool string - namespace string + clusterID string + mons string + pool string + journalPool string + namespace string + groupNamePrefix string } func getCephConfig(ctx context.Context, params, secrets map[string]string) (*cephConfig, error) { @@ -64,12 +65,18 @@ func getCephConfig(ctx context.Context, params, secrets map[string]string) (*cep return nil, errors.New("missing required parameter: radosNamespace") } + namePrefix := params["groupNamePrefix"] + if namePrefix == "" { + return nil, errors.New("missing required parameter: groupNamePrefix") + } + return &cephConfig{ - clusterID: clusterID, - mons: mons, - pool: pool, - journalPool: journalPool, - namespace: namespace, + clusterID: clusterID, + mons: mons, + pool: pool, + journalPool: journalPool, + namespace: namespace, + groupNamePrefix: namePrefix, }, nil } @@ -105,6 +112,27 @@ func getVolumesForGroup(ctx context.Context, volumeIDs []string, secrets map[str return volumes, nil } +func initVolumeGroup(ctx context.Context, config *cephConfig, name string, secrets map[string]string) (types.VolumeGroup, error) { + group := rbd_group.NewVolumeGroup(ctx, name, config.clusterID, secrets) + + err := group.SetMonitors(ctx, config.mons) + if err != nil { + return nil, err + } + + err = group.SetPool(ctx, config.pool) + if err != nil { + return nil, err + } + + err = group.SetJournalNamespace(ctx, config.journalPool, config.namespace) + if err != nil { + return nil, err + } + + return group, nil +} + func (cs *ControllerServer) CreateVolumeGroupSnapshot(ctx context.Context, req *csi.CreateVolumeGroupSnapshotRequest) (*csi.CreateVolumeGroupSnapshotResponse, error) { // 1. resolve each rbd-image from the volume-id @@ -123,28 +151,26 @@ func (cs *ControllerServer) CreateVolumeGroupSnapshot(ctx context.Context, req * if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } - - group := rbd_group.NewVolumeGroup(ctx, req.GetName(), config.clusterID, req.GetSecrets()) - defer group.Destroy(ctx) - - err = group.SetMonitors(ctx, config.mons) - if err != nil { - return nil, status.Error(codes.Aborted, err.Error()) + for _, v := range volumes { + defer v.Destroy() } - err = group.SetPool(ctx, config.pool) + group, err := initVolumeGroup(ctx, config, req.GetName(), req.GetSecrets()) if err != nil { - return nil, status.Error(codes.Aborted, err.Error()) + return nil, status.Error(codes.InvalidArgument, err.Error()) } + defer group.Destroy(ctx) - err = group.SetJournalNamespace(ctx, config.journalPool, config.namespace) + // TODO: take a lock on the request + + err = group.Create(ctx, config.groupNamePrefix) if err != nil { return nil, status.Error(codes.Aborted, err.Error()) } - // TODO: add images to the group - for _, volume := range volumes { - err = group.AddVolume(ctx, volume) + // add images to the group + for _, v := range volumes { + err = group.AddVolume(ctx, v) if err != nil { return nil, status.Error(codes.Aborted, err.Error()) } @@ -156,41 +182,21 @@ func (cs *ControllerServer) CreateVolumeGroupSnapshot(ctx context.Context, req * } defer groupSnapshot.Destroy(ctx) - groupSnapshotID, err := groupSnapshot.GetID(ctx) - if err != nil { - return nil, status.Error(codes.Aborted, err.Error()) - } - - snapshots, err := groupSnapshot.ListSnapshots(ctx) - if err != nil { - return nil, status.Error(codes.Aborted, err.Error()) - } - - csiSnapshots := make([]*csi.Snapshot, len(snapshots)) - for i, snapshot := range snapshots { - csiSnapshot, err := snapshot.ToCSISnapshot(ctx) + // remove images from the group + for _, v := range volumes { + err = group.RemoveVolume(ctx, v) if err != nil { return nil, status.Error(codes.Aborted, err.Error()) } - - csiSnapshots[i] = csiSnapshot } - // TODO: remove images from the group - for _, volume := range volumes { - err = group.RemoveVolume(ctx, volume) - if err != nil { - return nil, status.Error(codes.Aborted, err.Error()) - } + csiGroupSnapshot, err := groupSnapshot.ToCSIVolumeGroupSnapshot(ctx) + if err != nil { + return nil, status.Error(codes.Aborted, err.Error()) } return &csi.CreateVolumeGroupSnapshotResponse{ - GroupSnapshot: &csi.VolumeGroupSnapshot{ - GroupSnapshotId: groupSnapshotID, - Snapshots: csiSnapshots, - CreationTime: nil, - ReadyToUse: groupSnapshot.GetReadyToUse(ctx), - }, + GroupSnapshot: csiGroupSnapshot, }, nil } @@ -199,11 +205,35 @@ func (cs *ControllerServer) DeleteVolumeGroupSnapshot(ctx context.Context, req * // 1. verify that all snapshots in the request are all snapshots in the group // 2. delete the group - return nil, nil + snapshot, err := rbd_group.GetVolumeGroupSnapshot(ctx, req.GetGroupSnapshotId(), req.GetSecrets()) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + defer snapshot.Destroy(ctx) + + err = snapshot.Delete(ctx) + if err != nil { + return nil, status.Error(codes.Aborted, err.Error()) + } + + return &csi.DeleteVolumeGroupSnapshotResponse{}, nil } // TODO // sortof optional, only used for static/pre-provisioned VolumeGroupSnapshots func (cs *ControllerServer) GetVolumeGroupSnapshot(ctx context.Context, req *csi.GetVolumeGroupSnapshotRequest) (*csi.GetVolumeGroupSnapshotResponse, error) { - return nil, nil + snapshot, err := rbd_group.GetVolumeGroupSnapshot(ctx, req.GetGroupSnapshotId(), req.GetSecrets()) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + defer snapshot.Destroy(ctx) + + csiGroupSnapshot, err := snapshot.ToCSIVolumeGroupSnapshot(ctx) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + + return &csi.GetVolumeGroupSnapshotResponse{ + GroupSnapshot: csiGroupSnapshot, + }, nil } diff --git a/internal/rbd_group/snapshot.go b/internal/rbd_group/snapshot.go index b80557bbb261..10eff8ef4dee 100644 --- a/internal/rbd_group/snapshot.go +++ b/internal/rbd_group/snapshot.go @@ -19,6 +19,7 @@ package rbd_group import ( "context" "fmt" + "time" "github.com/container-storage-interface/spec/lib/go/csi" @@ -55,6 +56,14 @@ func (rgs *rbdGroupSnapshot) String() string { return fmt.Sprintf("%s@%s", rgs.parent, rgs.snapName) } +func (rgs *rbdGroupSnapshot) GetCreationTime(ctx context.Context) (*time.Time, error) { + return nil, nil +} + +func (rgs *rbdGroupSnapshot) GetReadyToUse(ctx context.Context) (bool, error) { + return false, nil +} + func (rgs *rbdGroupSnapshot) ToCSISnapshot(ctx context.Context) (*csi.Snapshot, error) { parentID, err := rgs.parent.GetID(ctx) if err != nil { @@ -62,11 +71,11 @@ func (rgs *rbdGroupSnapshot) ToCSISnapshot(ctx context.Context) (*csi.Snapshot, } return &csi.Snapshot{ - SizeBytes: 0, - SnapshotId: "", - SourceVolumeId: "", - CreationTime: nil, - ReadyToUse: false, + SizeBytes: 0, + SnapshotId: "", + SourceVolumeId: "", + CreationTime: nil, + ReadyToUse: false, GroupSnapshotId: parentID, }, nil } diff --git a/internal/rbd_group/volume_group.go b/internal/rbd_group/volume_group.go index c55da8179cf8..dda9aab96469 100644 --- a/internal/rbd_group/volume_group.go +++ b/internal/rbd_group/volume_group.go @@ -30,7 +30,6 @@ import ( const ( groupSuffix = "rbd-group" - groupPrefix = "rbd-group" ) var ( @@ -71,6 +70,54 @@ func NewVolumeGroup(ctx context.Context, name, clusterID string, secrets map[str } } +func GetVolumeGroup(ctx context.Context, id string, secrets map[string]string) (types.VolumeGroup, error) { + csiID := util.CSIIdentifier{} + + err := csiID.DecomposeCSIID(id) + if err != nil { + return nil, err + } + + mons, _, err := util.GetMonsAndClusterID(ctx, csiID.ClusterID, false) + if err != nil { + return nil, err + } + + namespace, err := util.GetRadosNamespace(util.CsiConfigFile, csiID.ClusterID) + if err != nil { + return nil, err + } + + vg := &rbdVolumeGroup{ + clusterID: csiID.ClusterID, + monitors: mons, + secrets: secrets, + id: id, + } + defer func() { + if err != nil { + vg.Destroy(ctx) + } + }() + + vg.credentials, err = util.NewUserCredentials(secrets) + if err != nil { + return nil, err + } + + pool, err := util.GetPoolName(mons, vg.credentials, csiID.LocationID) + if err != nil { + return nil, err + } + + err = vg.SetJournalNamespace(ctx, pool, namespace) + if err != nil { + return nil, err + } + + return vg, nil +} + func (rvg *rbdVolumeGroup) validate() error { if rvg.ioctx == nil { return ErrRBDGroupNotConnected @@ -107,9 +154,9 @@ func (rvg *rbdVolumeGroup) Destroy(ctx context.Context) { } func (rvg *rbdVolumeGroup) GetID(ctx context.Context) (string, error) { - // FIXME: this should be the group-snapshot-handle - if rvg.id != "" { - return rvg.id, nil + if rvg.id == "" { + // FIXME: rbdVolumeGroup need have called .Create() or GetVolumeGroup() + return "", errors.New("BUG: rbdVolumeGroup does not have ID set") } return rvg.id, nil @@ -165,7 +212,7 @@ func (rvg *rbdVolumeGroup) SetJournalNamespace(ctx context.Context, pool, namesp return nil } -func (rvg *rbdVolumeGroup) Create(ctx context.Context) error { +func (rvg *rbdVolumeGroup) Create(ctx context.Context, prefix string) error { if err := rvg.validate(); err != nil { return err } @@ -185,7 +232,7 @@ func (rvg *rbdVolumeGroup) Create(ctx context.Context) error { rvg.journalPool, journalPoolID, rvg.name, - groupPrefix) + prefix) if err != nil { return err } @@ -210,7 +257,14 @@ func (rvg *rbdVolumeGroup) AddVolume(ctx context.Context, image types.Volume) er return err } - return image.AddToGroup(ctx, rvg.ioctx, rvg.name) + err := image.AddToGroup(ctx, rvg.ioctx, rvg.name) + if err != nil { + return err + } + + rvg.volumes = append(rvg.volumes, image) + + return nil } func (rvg *rbdVolumeGroup) RemoveVolume(ctx context.Context, image types.Volume) error { @@ -218,7 +272,42 @@ func (rvg *rbdVolumeGroup) RemoveVolume(ctx context.Context, image types.Volume) return err } - return image.RemoveFromGroup(ctx, rvg.ioctx, rvg.name) + // volume was already removed from the group + if len(rvg.volumes) == 0 { + return nil + } + + err := image.RemoveFromGroup(ctx, rvg.ioctx, rvg.name) + if err != nil { + return err + } + + // toRemove contain the ID of the volume that is removed from the group + toRemove, err := image.GetID(ctx) + if err != nil { + return err + } + + // volumes is the updated list, without the volume that was removed + volumes := make([]types.Volume, 0) + for _, v := range rvg.volumes { + id, err := v.GetID(ctx) + if err != nil { + return err + } + + if id == toRemove { + // do not add the volume to the list + continue + } + + volumes = append(volumes, v) + } + + // update the list of volumes + rvg.volumes = volumes + + return nil } func (rvg *rbdVolumeGroup) CreateSnapshot(ctx context.Context, snapName string) (types.VolumeGroupSnapshot, error) { diff --git a/internal/rbd_group/volume_group_snapshot.go b/internal/rbd_group/volume_group_snapshot.go index 56bb3b8f53c7..951896b95074 100644 --- a/internal/rbd_group/volume_group_snapshot.go +++ b/internal/rbd_group/volume_group_snapshot.go @@ -20,6 +20,8 @@ import ( "context" "time" + "github.com/container-storage-interface/spec/lib/go/csi" + types "github.com/ceph/ceph-csi/internal/rbd_types" ) @@ -44,11 +46,20 @@ func newVolumeGroupSnapshot(ctx context.Context, parent *rbdVolumeGroup, name st } } +func GetVolumeGroupSnapshot(ctx context.Context, id string, secrets map[string]string) (types.VolumeGroupSnapshot, error) { + // TODO: resolve the VolumeGroupSnapshot by id + return nil, nil +} + // Destroy frees the resources used by the rbdVolumeGroup. func (rvgs *rbdVolumeGroupSnapshot) Destroy(ctx context.Context) { // nothing to do (yet) } +func (rvgs *rbdVolumeGroupSnapshot) Delete(ctx context.Context) error { + return nil +} + func (rvgs *rbdVolumeGroupSnapshot) GetID(ctx context.Context) (string, error) { // FIXME: this should be the group-snapshot-handle return "", nil @@ -59,13 +70,62 @@ func (rvgs *rbdVolumeGroupSnapshot) ListSnapshots(ctx context.Context) ([]types. return nil, nil } -func (rvgs *rbdVolumeGroupSnapshot) GetCreationTime(ctx context.Context) *time.Time { +func (rvgs *rbdVolumeGroupSnapshot) GetCreationTime(ctx context.Context) (*time.Time, error) { // TODO: fetch the creation time of the group // A group snapshot does not seem to have its own creation time. Use // the time of the most recent created snapshot. - return nil + return nil, nil } -func (rvgs *rbdVolumeGroupSnapshot) GetReadyToUse(ctx context.Context) bool { - return true +// GetReadyToUse checks if all snapshots that are part if the group are ready +// to use. +func (rvgs *rbdVolumeGroupSnapshot) GetReadyToUse(ctx context.Context) (bool, error) { + for _, snapshot := range rvgs.snapshots { + ready, err := snapshot.GetReadyToUse(ctx) + if err != nil { + return false, err + } + + if !ready { + // if this snapshot is not ready, no need to check + // other snapshots + return false, nil + } + } + + return true, nil +} + +func (rvgs *rbdVolumeGroupSnapshot) ToCSIVolumeGroupSnapshot(ctx context.Context) (*csi.VolumeGroupSnapshot, error) { + groupSnapshotID, err := rvgs.GetID(ctx) + if err != nil { + return nil, err + } + + snapshots, err := rvgs.ListSnapshots(ctx) + if err != nil { + return nil, err + } + + csiSnapshots := make([]*csi.Snapshot, len(snapshots)) + for i, snapshot := range snapshots { + csiSnapshot, err := snapshot.ToCSISnapshot(ctx) + if err != nil { + return nil, err + } + + csiSnapshots[i] = csiSnapshot + } + + ready, err := rvgs.GetReadyToUse(ctx) + if err != nil { + return nil, err + } + + return &csi.VolumeGroupSnapshot{ + GroupSnapshotId: groupSnapshotID, + Snapshots: csiSnapshots, + CreationTime: nil, + ReadyToUse: ready, + }, nil } diff --git a/internal/rbd_types/group.go b/internal/rbd_types/group.go index a837c5a4e51e..5d31fda1a470 100644 --- a/internal/rbd_types/group.go +++ b/internal/rbd_types/group.go @@ -19,6 +19,8 @@ package rbd_types import ( "context" "time" + + "github.com/container-storage-interface/spec/lib/go/csi" ) // VolumeGroup contains a number of volumes, and can be used to create a @@ -37,14 +39,13 @@ type VolumeGroup interface { SetJournalNamespace(ctx context.Context, pool, namespace string) error - Create(ctx context.Context) error + Create(ctx context.Context, prefix string) error Delete(ctx context.Context) error AddVolume(ctx context.Context, volume Volume) error RemoveVolume(ctx context.Context, volume Volume) error CreateSnapshot(ctx context.Context, name string) (VolumeGroupSnapshot, error) - DeleteSnapshot(ctx context.Context, snapName string) error } // VolumeGroupSnapshot is an instance of a group of snapshots that was taken @@ -53,10 +54,14 @@ type VolumeGroupSnapshot interface { // Destroy frees the resources used by the VolumeGroupSnapshot. Destroy(ctx context.Context) + Delete(ctx context.Context) error + GetID(ctx context.Context) (string, error) ListSnapshots(ctx context.Context) ([]Snapshot, error) - GetCreationTime(ctx context.Context) *time.Time - GetReadyToUse(ctx context.Context) bool + GetCreationTime(ctx context.Context) (*time.Time, error) + GetReadyToUse(ctx context.Context) (bool, error) + + ToCSIVolumeGroupSnapshot(ctx context.Context) (*csi.VolumeGroupSnapshot, error) } diff --git a/internal/rbd_types/snapshot.go b/internal/rbd_types/snapshot.go index 594c8722b4c8..ab35eea78508 100644 --- a/internal/rbd_types/snapshot.go +++ b/internal/rbd_types/snapshot.go @@ -18,6 +18,7 @@ package rbd_types import ( "context" + "time" "github.com/container-storage-interface/spec/lib/go/csi" ) @@ -26,8 +27,8 @@ type Snapshot interface { // Destroy frees the resources used by the Snapshot. Destroy(ctx context.Context) - ToCSISnapshot(ctx context.Context) (*csi.Snapshot, error) + GetCreationTime(ctx context.Context) (*time.Time, error) + GetReadyToUse(ctx context.Context) (bool, error) - // Map the snapshot as an rbd device - Map(ctx context.Context) (string, error) // FIXME: just an example + ToCSISnapshot(ctx context.Context) (*csi.Snapshot, error) }