Skip to content

Commit

Permalink
Merge pull request #905 from huww98/disk-snapshot-tags
Browse files Browse the repository at this point in the history 
disk: add tags to ECS snapshot
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot committed Jan 31, 2024
2 parents 974b6fb + d3b6fd0 commit 659f8b2
Show file tree
Hide file tree
Showing 4 changed files with 148 additions and 112 deletions.
12 changes: 12 additions & 0 deletions pkg/common/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,12 @@ const (
PVNameKey = "csi.storage.k8s.io/pv/name"
)

// constants of keys in volume snapshot parameters
const (
VolumeSnapshotNamespaceKey = "csi.storage.k8s.io/volumesnapshot/namespace"
VolumeSnapshotNameKey = "csi.storage.k8s.io/volumesnapshot/name"
)

const (
// PVCNameTag is tag applied to provisioned alibaba cloud disk for compatibility
// with in-tree volume plugin. Value of the tag is PVC name. It is applied only when
Expand All @@ -36,3 +42,9 @@ const (
// Disk name have many restrictions, so we use this tag to store the original name
VolumeNameTag = "csi.alibabacloud.com/volume-name"
)

// Tags that will be added to ECS snapshots
const (
VolumeSnapshotNameTag = "csi.alibabacloud.com/snapshot/name"
VolumeSnapshotNamespaceTag = "csi.alibabacloud.com/snapshot/namespace"
)
42 changes: 26 additions & 16 deletions pkg/disk/cloud.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -775,27 +775,37 @@ func DescribeDiskInstanceEvents(instanceId string, ecsClient *ecs.Client) (event
return
}

func requestAndCreateSnapshot(ecsClient *ecs.Client, sourceVolumeID, snapshotName, resourceGroupID string, retentionDays, instantAccessRetentionDays int,
instantAccess bool) (*ecs.CreateSnapshotResponse, error) {
type createSnapshotParams struct {
SourceVolumeID string
SnapshotName string
ResourceGroupID string
RetentionDays int
InstantAccessRetentionDays int
InstantAccess bool
SnapshotTags []ecs.CreateSnapshotTag
}

func requestAndCreateSnapshot(ecsClient *ecs.Client, params *createSnapshotParams) (*ecs.CreateSnapshotResponse, error) {
// init createSnapshotRequest and parameters
createSnapshotRequest := ecs.CreateCreateSnapshotRequest()
createSnapshotRequest.DiskId = sourceVolumeID
createSnapshotRequest.SnapshotName = snapshotName
createSnapshotRequest.InstantAccess = requests.NewBoolean(instantAccess)
createSnapshotRequest.InstantAccessRetentionDays = requests.NewInteger(instantAccessRetentionDays)
if retentionDays != -1 {
createSnapshotRequest.RetentionDays = requests.NewInteger(retentionDays)
}
if resourceGroupID != "" {
createSnapshotRequest.ResourceGroupId = resourceGroupID
createSnapshotRequest.DiskId = params.SourceVolumeID
createSnapshotRequest.SnapshotName = params.SnapshotName
createSnapshotRequest.InstantAccess = requests.NewBoolean(params.InstantAccess)
createSnapshotRequest.InstantAccessRetentionDays = requests.NewInteger(params.InstantAccessRetentionDays)
if params.RetentionDays != 0 {
createSnapshotRequest.RetentionDays = requests.NewInteger(params.RetentionDays)
}
createSnapshotRequest.ResourceGroupId = params.ResourceGroupID

// Set tags
snapshotTags := []ecs.CreateSnapshotTag{}
tag1 := ecs.CreateSnapshotTag{Key: DISKTAGKEY2, Value: DISKTAGVALUE2}
snapshotTags = append(snapshotTags, tag1)
tag2 := ecs.CreateSnapshotTag{Key: SNAPSHOTTAGKEY1, Value: "true"}
snapshotTags = append(snapshotTags, tag2)
snapshotTags := []ecs.CreateSnapshotTag{
{Key: DISKTAGKEY2, Value: DISKTAGVALUE2},
{Key: SNAPSHOTTAGKEY1, Value: "true"},
}
if GlobalConfigVar.ClusterID != "" {
snapshotTags = append(snapshotTags, ecs.CreateSnapshotTag{Key: DISKTAGKEY3, Value: GlobalConfigVar.ClusterID})
}
snapshotTags = append(snapshotTags, params.SnapshotTags...)
createSnapshotRequest.Tag = &snapshotTags

// Do Snapshot create
Expand Down
25 changes: 12 additions & 13 deletions pkg/disk/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,22 +21,21 @@ const (
DISKTAGKEY3 = "ack.aliyun.com"
// ECS snapshot tag from old version, keep it for compatibility
SNAPSHOTTAGKEY1 = "force.delete.snapshot.k8s.aliyun.com"
// SNAPSHOTTYPE ...
SNAPSHOTTYPE = "snapshotType"
// INSTANTACCESS ...
INSTANTACCESS = "InstantAccess"
// RETENTIONDAYS ...
RETENTIONDAYS = "retentionDays"
// INSTANTACCESSRETENTIONDAYS ...
)

// keys used in CreateSnapshotRequest.Parameters
const (
SNAPSHOTTYPE = "snapshotType"
INSTANTACCESS = "InstantAccess"
RETENTIONDAYS = "retentionDays"
INSTANTACCESSRETENTIONDAYS = "instantAccessRetentionDays"
// SNAPSHOTRESOURCEGROUPID ...
SNAPSHOTRESOURCEGROUPID = "resourceGroupId"
SNAPSHOTRESOURCEGROUPID = "resourceGroupId"
SNAPSHOT_TAG_PREFIX = "snapshotTags/"
)

const (
// DiskSnapshotID means snapshot id
DiskSnapshotID = "csi.alibabacloud.com/disk-snapshot-id"
// VolumeSnapshotNamespace namespace
VolumeSnapshotNamespace = "csi.storage.k8s.io/volumesnapshot/namespace"
// VolumeSnapshotName tag
VolumeSnapshotName = "csi.storage.k8s.io/volumesnapshot/name"
// IAVolumeSnapshotKey tag
IAVolumeSnapshotKey = "csi.alibabacloud.com/snapshot-ia"
// SnapshotRequestTag interval limit
Expand Down
181 changes: 98 additions & 83 deletions pkg/disk/controllerserver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ import (
csicommon "github.com/kubernetes-csi/drivers/pkg/csi-common"
volumeSnasphotV1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
snapClientset "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned"
"github.com/kubernetes-sigs/alibaba-cloud-csi-driver/pkg/common"
"github.com/kubernetes-sigs/alibaba-cloud-csi-driver/pkg/disk/crds"
log "github.com/kubernetes-sigs/alibaba-cloud-csi-driver/pkg/log"
"github.com/kubernetes-sigs/alibaba-cloud-csi-driver/pkg/utils"
Expand Down Expand Up @@ -434,73 +435,102 @@ func (cs *controllerServer) ControllerUnpublishVolume(ctx context.Context, req *
return &csi.ControllerUnpublishVolumeResponse{}, nil
}

// if volumesnapshot have Annotations, use it first.
// storage.alibabacloud.com/snapshot-ttl
// storage.alibabacloud.com/ia-snapshot
// storage.alibabacloud.com/ia-snapshot-ttl
func getVolumeSnapshotConfig(req *csi.CreateSnapshotRequest) (int, bool, int, string, error) {
retentionDays := -1
useInstanceAccess := false
var instantAccessRetentionDays int

params := req.GetParameters()
if value, ok := params[SNAPSHOTTYPE]; ok && value == INSTANTACCESS {
useInstanceAccess = true
}
if value, ok := params[RETENTIONDAYS]; ok {
days, err := strconv.Atoi(value)
func getVolumeSnapshotConfig(req *csi.CreateSnapshotRequest) (*createSnapshotParams, error) {
var ecsParams createSnapshotParams
if req.Parameters != nil {
err := parseSnapshotParameters(req.Parameters, &ecsParams)
if err != nil {
err := status.Errorf(codes.InvalidArgument, "CreateSnapshot: retentiondays err %s", value)
return retentionDays, useInstanceAccess, instantAccessRetentionDays, "", err
log.Log.Errorf("CreateSnapshot:: Snapshot name[%s], parse config failed: %v", req.Name, err)
return nil, status.Errorf(codes.InvalidArgument, err.Error())
}
retentionDays = days
}
if value, ok := params[INSTANTACCESSRETENTIONDAYS]; ok {
days, err := strconv.Atoi(value)
if err != nil {
err := status.Errorf(codes.InvalidArgument, "CreateSnapshot: retentiondays err %s", value)
return retentionDays, useInstanceAccess, instantAccessRetentionDays, "", err
}
instantAccessRetentionDays = days
} else {
instantAccessRetentionDays = retentionDays
}

vsName := req.Parameters[VolumeSnapshotName]
vsNameSpace := req.Parameters[VolumeSnapshotNamespace]
vsName := req.Parameters[common.VolumeSnapshotNameKey]
vsNameSpace := req.Parameters[common.VolumeSnapshotNamespaceKey]
// volumesnapshot not in parameters, just retrun
if vsName == "" || vsNameSpace == "" {
return retentionDays, useInstanceAccess, instantAccessRetentionDays, "", nil
return &ecsParams, nil
}

volumeSnapshot, err := GlobalConfigVar.SnapClient.SnapshotV1().VolumeSnapshots(vsNameSpace).Get(context.Background(), vsName, metav1.GetOptions{})
if err != nil {
log.Log.Warnf("CreateSnapshot: cannot get volumeSnapshot: %s, with error: %v", req.Name, err.Error())
return retentionDays, useInstanceAccess, instantAccessRetentionDays, "", err
return nil, status.Errorf(codes.Internal, "failed to get VolumeSnapshot: %s/%s: %v", vsNameSpace, vsName, err)
}
err = parseSnapshotAnnotations(volumeSnapshot.Annotations, &ecsParams)
if err != nil {
log.Log.Errorf("CreateSnapshot:: Snapshot name[%s], parse annotation failed: %v", req.Name, err)
return nil, status.Errorf(codes.InvalidArgument, err.Error())
}
snapshotTTL := volumeSnapshot.Annotations["storage.alibabacloud.com/snapshot-ttl"]
iaEnable := volumeSnapshot.Annotations["storage.alibabacloud.com/ia-snapshot"]
iaTTL := volumeSnapshot.Annotations["storage.alibabacloud.com/ia-snapshot-ttl"]
return &ecsParams, nil
}

func parseSnapshotParameters(params map[string]string, ecsParams *createSnapshotParams) (err error) {
for k, v := range params {
switch k {
case SNAPSHOTTYPE:
if v == INSTANTACCESS {
ecsParams.InstantAccess = true
}
case RETENTIONDAYS:
ecsParams.RetentionDays, err = strconv.Atoi(v)
if err != nil {
return fmt.Errorf("failed to parse retentionDays: %w", err)
}
case INSTANTACCESSRETENTIONDAYS:
ecsParams.InstantAccessRetentionDays, err = strconv.Atoi(v)
if err != nil {
return fmt.Errorf("failed to parse instantAccessRetentionDays: %w", err)
}
case SNAPSHOTRESOURCEGROUPID:
ecsParams.ResourceGroupID = v
case common.VolumeSnapshotNameKey:
ecsParams.SnapshotTags = append(ecsParams.SnapshotTags, ecs.CreateSnapshotTag{
Key: common.VolumeSnapshotNameTag,
Value: v,
})
case common.VolumeSnapshotNamespaceKey:
ecsParams.SnapshotTags = append(ecsParams.SnapshotTags, ecs.CreateSnapshotTag{
Key: common.VolumeSnapshotNamespaceTag,
Value: v,
})
default:
if strings.HasPrefix(k, SNAPSHOT_TAG_PREFIX) {
ecsParams.SnapshotTags = append(ecsParams.SnapshotTags, ecs.CreateSnapshotTag{
Key: k[len(SNAPSHOT_TAG_PREFIX):],
Value: v,
})
}
}
}
return nil
}

// if volumesnapshot have Annotations, use it first.
// storage.alibabacloud.com/snapshot-ttl
// storage.alibabacloud.com/ia-snapshot
// storage.alibabacloud.com/ia-snapshot-ttl
func parseSnapshotAnnotations(anno map[string]string, ecsParams *createSnapshotParams) error {
snapshotTTL := anno["storage.alibabacloud.com/snapshot-ttl"]
iaEnable := anno["storage.alibabacloud.com/ia-snapshot"]
iaTTL := anno["storage.alibabacloud.com/ia-snapshot-ttl"]
var err error

if snapshotTTL != "" {
retentionDays, err = strconv.Atoi(snapshotTTL)
ecsParams.RetentionDays, err = strconv.Atoi(snapshotTTL)
if err != nil {
log.Log.Warnf("CreateSnapshot: Snapshot(%s) ttl format error: %v", req.Name, err.Error())
return retentionDays, useInstanceAccess, instantAccessRetentionDays, "", err
return fmt.Errorf("failed to parse annotation snapshot-ttl: %w", err)
}
}
if strings.ToLower(iaEnable) == "true" {
useInstanceAccess = true
ecsParams.InstantAccess = true
}
if iaTTL != "" {
instantAccessRetentionDays, err = strconv.Atoi(iaTTL)
ecsParams.InstantAccessRetentionDays, err = strconv.Atoi(iaTTL)
if err != nil {
log.Log.Warnf("CreateSnapshot: IA ttl(%s) format error: %v", req.Name, err.Error())
return retentionDays, useInstanceAccess, instantAccessRetentionDays, "", err
return fmt.Errorf("failed to parse annotation ia-snapshot-ttl: %w", err)
}
}
resourceGroupID := req.Parameters[SNAPSHOTRESOURCEGROUPID]
return retentionDays, useInstanceAccess, instantAccessRetentionDays, resourceGroupID, nil
return nil
}

// CreateSnapshot ...
Expand All @@ -516,8 +546,8 @@ func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS
SnapshotRequestMap[req.Name] = cur

// used for snapshot events
snapshotName := req.Parameters[VolumeSnapshotName]
snapshotNamespace := req.Parameters[VolumeSnapshotNamespace]
snapshotName := req.Parameters[common.VolumeSnapshotNameKey]
snapshotNamespace := req.Parameters[common.VolumeSnapshotNamespaceKey]

ref := &v1.ObjectReference{
Kind: "VolumeSnapshot",
Expand All @@ -526,10 +556,8 @@ func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS
Namespace: snapshotNamespace,
}

// parse snapshot Retention Days
retentionDays, useInstanceAccess, instantAccessRetentionDays, resourceGroupID, err := getVolumeSnapshotConfig(req)
params, err := getVolumeSnapshotConfig(req)
if err != nil {
log.Log.Errorf("CreateSnapshot:: Snapshot name[%s], parse retention days error: %v", req.Name, err)
return nil, err
}

Expand Down Expand Up @@ -608,14 +636,16 @@ func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS
// }

// if disk type is not essd and IA set disable
if useInstanceAccess && disks[0].Category != DiskESSD && disks[0].Category != DiskESSDAuto {
if params.InstantAccess && disks[0].Category != DiskESSD && disks[0].Category != DiskESSDAuto {
log.Log.Warnf("CreateSnapshot: Snapshot(%s) set as not IA type, because disk Category %s", req.Name, disks[0].Category)
useInstanceAccess = false
params.InstantAccess = false
}

// init createSnapshotRequest and parameters
createAt := ptypes.TimestampNow()
snapshotResponse, err := requestAndCreateSnapshot(ecsClient, sourceVolumeID, req.GetName(), resourceGroupID, retentionDays, instantAccessRetentionDays, useInstanceAccess)
params.SourceVolumeID = sourceVolumeID
params.SnapshotName = req.Name
snapshotResponse, err := requestAndCreateSnapshot(ecsClient, params)

if err != nil {
log.Log.Errorf("CreateSnapshot:: Snapshot create Failed: snapshotName[%s], sourceId[%s], error[%s]", req.Name, req.GetSourceVolumeId(), err.Error())
Expand All @@ -625,7 +655,7 @@ func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS

// if is IA snapshot, snapshot ready immediately
tmpReadyToUse := false
if useInstanceAccess {
if params.InstantAccess {
//updateSnapshotIAStatus(req, "completed")
tmpReadyToUse = true
delete(SnapshotRequestMap, req.Name)
Expand Down Expand Up @@ -672,7 +702,13 @@ func snapshotBeforeDelete(volumeID string, ecsClient *ecs.Client) error {
if value, ok := delVolumeSnap.Load(volumeID); ok {
return createStaticSnap(volumeID, value.(string), GlobalConfigVar.SnapClient)
}
resp, err := requestAndCreateSnapshot(ecsClient, volumeID, deleteVolumeSnapshotName, "", iValue, iValue, true)
resp, err := requestAndCreateSnapshot(ecsClient, &createSnapshotParams{
SourceVolumeID: volumeID,
SnapshotName: deleteVolumeSnapshotName,
RetentionDays: iValue,
InstantAccessRetentionDays: iValue,
InstantAccess: true,
})
if err != nil {
return err
}
Expand All @@ -684,33 +720,6 @@ func snapshotBeforeDelete(volumeID string, ecsClient *ecs.Client) error {
return createStaticSnap(volumeID, resp.SnapshotId, GlobalConfigVar.SnapClient)
}

func updateSnapshotIAStatus(req *csi.CreateSnapshotRequest, status string) error {
volumeSnapshotName := req.Parameters[VolumeSnapshotName]
volumeSnapshotNameSpace := req.Parameters[VolumeSnapshotNamespace]
if volumeSnapshotName == "" || volumeSnapshotNameSpace == "" {
log.Log.Infof("CreateSnapshot: cannot get volumesnapshot name and namespace: %s, %s, %s", volumeSnapshotName, volumeSnapshotNameSpace, req.Name)
return nil
}

volumeSnapshot, err := GlobalConfigVar.SnapClient.SnapshotV1().VolumeSnapshots(volumeSnapshotNameSpace).Get(context.Background(), volumeSnapshotName, metav1.GetOptions{})
if err != nil {
log.Log.Warnf("CreateSnapshot: get volumeSnapshot(%s/%s) labels error: %s", volumeSnapshotNameSpace, volumeSnapshotName, err.Error())
return err
}
if volumeSnapshot.Labels == nil {
volumeSnapshot.Labels = map[string]string{}
}
volumeSnapshot.Labels[IAVolumeSnapshotKey] = status

_, err = GlobalConfigVar.SnapClient.SnapshotV1().VolumeSnapshots(volumeSnapshotNameSpace).Update(context.Background(), volumeSnapshot, metav1.UpdateOptions{})
if err != nil {
log.Log.Warnf("CreateSnapshot: Update VolumeSnapshot(%s/%s) IA Status error: %s", volumeSnapshotNameSpace, volumeSnapshotName, err.Error())
return err
}
log.Log.Infof("CreateSnapshot: updateSnapshot(%s/%s) IA Status successful %s", volumeSnapshotNameSpace, volumeSnapshotName, req.Name)
return nil
}

// DeleteSnapshot ...
func (cs *controllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) {

Expand Down Expand Up @@ -1115,7 +1124,13 @@ func (cs *controllerServer) createVolumeExpandAutoSnapshot(ctx context.Context,

// init createSnapshotRequest and parameters
createAt := timestamppb.New(cur)
snapshotResponse, err := requestAndCreateSnapshot(ecsClient, sourceVolumeID, volumeExpandAutoSnapshotName, "", veasp.RetentionDays, veasp.InstanceAccessRetentionDays, veasp.InstantAccess)
snapshotResponse, err := requestAndCreateSnapshot(ecsClient, &createSnapshotParams{
SourceVolumeID: sourceVolumeID,
SnapshotName: volumeExpandAutoSnapshotName,
RetentionDays: veasp.RetentionDays,
InstantAccessRetentionDays: veasp.InstanceAccessRetentionDays,
InstantAccess: veasp.InstantAccess,
})
if err != nil {
log.Log.Errorf("ControllerExpandVolume:: volumeExpandAutoSnapshot create Failed: snapshotName[%s], sourceId[%s], error[%s]", volumeExpandAutoSnapshotName, sourceVolumeID, err.Error())
cs.recorder.Event(pvc, v1.EventTypeWarning, snapshotCreateError, err.Error())
Expand Down

0 comments on commit 659f8b2

Please sign in to comment.