From 91d1b84a991f117085d8cb96e4ae2ff63ade5322 Mon Sep 17 00:00:00 2001 From: andyzhangx Date: Mon, 7 Aug 2023 07:51:05 +0000 Subject: [PATCH] fix: stop resize file share when account limit is exceeded fix --- pkg/azurefile/azurefile.go | 6 ++++++ pkg/azurefile/controllerserver.go | 15 +++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/pkg/azurefile/azurefile.go b/pkg/azurefile/azurefile.go index 7fe62bd394..be7f55f434 100644 --- a/pkg/azurefile/azurefile.go +++ b/pkg/azurefile/azurefile.go @@ -250,6 +250,8 @@ type Driver struct { accountSearchCache azcache.Resource // a timed cache storing tag removing history (solve account update throttling issue) removeTagCache azcache.Resource + // a timed cache when resize file share failed due to account limit exceeded + resizeFileShareFailureCache azcache.Resource } // NewDriver Creates a NewCSIDriver object. Assumes vendor version is equal to driver version & @@ -303,6 +305,10 @@ func NewDriver(options *DriverOptions) *Driver { klog.Fatalf("%v", err) } + if driver.resizeFileShareFailureCache, err = azcache.NewTimedCache(3*time.Minute, getter, false); err != nil { + klog.Fatalf("%v", err) + } + return &driver } diff --git a/pkg/azurefile/controllerserver.go b/pkg/azurefile/controllerserver.go index 0806b1ac4c..3a8b3d710a 100644 --- a/pkg/azurefile/controllerserver.go +++ b/pkg/azurefile/controllerserver.go @@ -1039,6 +1039,16 @@ func (d *Driver) ControllerExpandVolume(ctx context.Context, req *csi.Controller subsID = d.cloud.SubscriptionID } + if accountName != "" { + cache, err := d.resizeFileShareFailureCache.Get(accountName, azcache.CacheReadTypeDefault) + if err != nil { + return nil, status.Errorf(codes.Internal, "resizeFileShareFailureCache(%s) failed with error: %v", accountName, err) + } + if cache != nil { + return nil, status.Errorf(codes.Internal, "account(%s) is in %s, wait for a few minutes to retry", accountName, accountLimitExceedManagementAPI) + } + } + mc := metrics.NewMetricContext(azureFileCSIDriverName, "controller_expand_volume", resourceGroupName, subsID, d.Name) isOperationSucceeded := false defer func() { @@ -1060,6 +1070,11 @@ func (d *Driver) ControllerExpandVolume(ctx context.Context, req *csi.Controller } if err = d.ResizeFileShare(ctx, subsID, resourceGroupName, accountName, fileShareName, int(requestGiB), secrets); err != nil { + if strings.Contains(err.Error(), accountLimitExceedManagementAPI) || strings.Contains(err.Error(), accountLimitExceedDataPlaneAPI) { + if accountName != "" { + d.resizeFileShareFailureCache.Set(accountName, "") + } + } return nil, status.Errorf(codes.Internal, "expand volume error: %v", err) }