From bb771cc84c47d41c9bf65e129d966047a8742b7e Mon Sep 17 00:00:00 2001 From: "Cuong. Duong Manh" Date: Tue, 4 Jun 2024 14:11:04 +0700 Subject: [PATCH] [feat] merge code with vngcloud manage csi driver --- cmd/vngcloud-blockstorage-csi-driver/main.go | 167 +++--- go.mod | 154 ++--- go.sum | 59 +- pkg/cloud/cloud.go | 592 +++++++++---------- pkg/cloud/cloud_interface.go | 24 - pkg/cloud/cloud_private.go | 202 +++++++ pkg/cloud/consts.go | 33 +- pkg/cloud/entity/snapshot.go | 15 + pkg/cloud/entity/volume.go | 69 +++ pkg/cloud/entity/volumetype.go | 7 + pkg/cloud/errors/errors.go | 11 + pkg/cloud/errors/ierrors.go | 7 + pkg/cloud/icloud.go | 26 + pkg/driver/constants.go | 15 +- pkg/driver/controller.go | 193 +++--- pkg/driver/driver.go | 24 +- pkg/driver/errors.go | 12 + pkg/driver/internal/inflight.go | 4 - pkg/driver/node.go | 53 +- pkg/driver/validation.go | 13 +- pkg/driver/volume_dto.go | 90 ++- 21 files changed, 1098 insertions(+), 672 deletions(-) delete mode 100644 pkg/cloud/cloud_interface.go create mode 100644 pkg/cloud/cloud_private.go create mode 100644 pkg/cloud/entity/snapshot.go create mode 100644 pkg/cloud/entity/volume.go create mode 100644 pkg/cloud/entity/volumetype.go create mode 100644 pkg/cloud/errors/errors.go create mode 100644 pkg/cloud/errors/ierrors.go create mode 100644 pkg/cloud/icloud.go diff --git a/cmd/vngcloud-blockstorage-csi-driver/main.go b/cmd/vngcloud-blockstorage-csi-driver/main.go index a3955cf..50596b8 100644 --- a/cmd/vngcloud-blockstorage-csi-driver/main.go +++ b/cmd/vngcloud-blockstorage-csi-driver/main.go @@ -1,83 +1,84 @@ package main import ( - "context" - "fmt" - flag "github.com/spf13/pflag" - "github.com/vngcloud/vngcloud-blockstorage-csi-driver/pkg/driver" - "github.com/vngcloud/vngcloud-blockstorage-csi-driver/pkg/metrics" - "k8s.io/component-base/featuregate" - "os" - "strings" - - logsapi "k8s.io/component-base/logs/api/v1" - "k8s.io/component-base/logs/json" - "k8s.io/klog/v2" - "time" + lctx "context" + lfmt "fmt" + los "os" + lstr "strings" + ltime "time" + + lflag "github.com/spf13/pflag" + lfeaturegate "k8s.io/component-base/featuregate" + llogApi "k8s.io/component-base/logs/api/v1" + ljson "k8s.io/component-base/logs/json" + llog "k8s.io/klog/v2" + + lsdriver "github.com/vngcloud/vngcloud-blockstorage-csi-driver/pkg/driver" + lsmetrics "github.com/vngcloud/vngcloud-blockstorage-csi-driver/pkg/metrics" ) func main() { - fs := flag.NewFlagSet("vngcloud-blockstorage-csi-driver", flag.ExitOnError) - if err := logsapi.RegisterLogFormat(logsapi.JSONLogFormat, json.Factory{}, logsapi.LoggingBetaOptions); err != nil { - klog.ErrorS(err, "failed to register JSON log format") + fs := lflag.NewFlagSet("vngcloud-blockstorage-csi-driver", lflag.ExitOnError) + if err := llogApi.RegisterLogFormat(llogApi.JSONLogFormat, ljson.Factory{}, llogApi.LoggingBetaOptions); err != nil { + llog.ErrorS(err, "failed to register JSON log format") } options := GetOptions(fs) // Start tracing as soon as possible if options.ServerOptions.EnableOtelTracing { - exporter, err := driver.InitOtelTracing() + exporter, err := lsdriver.InitOtelTracing() if err != nil { - klog.ErrorS(err, "failed to initialize otel tracing") - klog.FlushAndExit(klog.ExitFlushTimeout, 1) + llog.ErrorS(err, "failed to initialize otel tracing") + llog.FlushAndExit(llog.ExitFlushTimeout, 1) } // Exporter will flush traces on shutdown defer func() { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := lctx.WithTimeout(lctx.Background(), 10*ltime.Second) defer cancel() if err := exporter.Shutdown(ctx); err != nil { - klog.ErrorS(err, "could not shutdown otel exporter") + llog.ErrorS(err, "could not shutdown otel exporter") } }() } if options.ServerOptions.HttpEndpoint != "" { - r := metrics.InitializeRecorder() + r := lsmetrics.InitializeRecorder() r.InitializeMetricsHandler(options.ServerOptions.HttpEndpoint, "/metrics") } - drv, err := driver.NewDriver( - driver.WithClientID(options.Global.ClientID), - driver.WithClientSecret(options.Global.ClientSecret), - driver.WithIdentityURL(options.Global.IdentityURL), - driver.WithVServerURL(options.Global.VServerURL), - driver.WithEndpoint(options.ServerOptions.Endpoint), - driver.WithMode(options.DriverMode), - driver.WithOtelTracing(options.ServerOptions.EnableOtelTracing), - driver.WithModifyVolumeRequestHandlerTimeout(options.ControllerOptions.ModifyVolumeRequestHandlerTimeout), - driver.WithClusterID(options.ServerOptions.ClusterID), - driver.WithTagKeyLength(options.ServerOptions.TagKeyLength), - driver.WithTagValueLength(options.ServerOptions.TagValueLength), + drv, err := lsdriver.NewDriver( + lsdriver.WithClientID(options.Global.ClientID), + lsdriver.WithClientSecret(options.Global.ClientSecret), + lsdriver.WithIdentityURL(options.Global.IdentityURL), + lsdriver.WithVServerURL(options.Global.VServerURL), + lsdriver.WithEndpoint(options.ServerOptions.Endpoint), + lsdriver.WithMode(options.DriverMode), + lsdriver.WithOtelTracing(options.ServerOptions.EnableOtelTracing), + lsdriver.WithModifyVolumeRequestHandlerTimeout(options.ControllerOptions.ModifyVolumeRequestHandlerTimeout), + lsdriver.WithClusterID(options.ServerOptions.ClusterID), + lsdriver.WithTagKeyLength(options.ServerOptions.TagKeyLength), + lsdriver.WithTagValueLength(options.ServerOptions.TagValueLength), ) if err != nil { - klog.ErrorS(err, "failed to create driver") - klog.FlushAndExit(klog.ExitFlushTimeout, 1) + llog.ErrorS(err, "failed to create driver") + llog.FlushAndExit(llog.ExitFlushTimeout, 1) } if err := drv.Run(); err != nil { - klog.ErrorS(err, "failed to run driver") - klog.FlushAndExit(klog.ExitFlushTimeout, 1) + llog.ErrorS(err, "failed to run driver") + llog.FlushAndExit(llog.ExitFlushTimeout, 1) } } var ( - featureGate = featuregate.NewFeatureGate() + featureGate = lfeaturegate.NewFeatureGate() // used for testing - osExit = os.Exit + osExit = los.Exit ) type Options struct { - DriverMode driver.Mode + DriverMode lsdriver.Mode *ServerOptions *ControllerOptions @@ -99,44 +100,50 @@ type ServerOptions struct { EnableOtelTracing bool TagKeyLength int TagValueLength int + CacheUri string + AlertChannel string + AlertChannelSize int } -func (s *ServerOptions) AddFlags(fs *flag.FlagSet) { - fs.StringVar(&s.Endpoint, "endpoint", driver.DefaultCSIEndpoint, "Endpoint for the CSI driver server") +func (s *ServerOptions) AddFlags(fs *lflag.FlagSet) { + fs.StringVar(&s.Endpoint, "endpoint", lsdriver.DefaultCSIEndpoint, "Endpoint for the CSI driver server") fs.StringVar(&s.HttpEndpoint, "http-endpoint", "", "The TCP network address where the HTTP server for metrics will listen (example: `:8080`). The default is empty string, which means the server is disabled.") fs.BoolVar(&s.EnableOtelTracing, "enable-otel-tracing", false, "To enable opentelemetry tracing for the driver. The tracing is disabled by default. Configure the exporter endpoint with OTEL_EXPORTER_OTLP_ENDPOINT and other env variables, see https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/#general-sdk-configuration.") fs.StringVar(&s.ClusterID, "cluster-id", "", "The unique ID of the cluster. This is used to identify the cluster in the logs.") fs.IntVar(&s.TagKeyLength, "tag-key-length", 15, "The maximum length of the tag key.") fs.IntVar(&s.TagValueLength, "tag-value-length", 255, "The maximum length of the tag value.") + fs.StringVar(&s.CacheUri, "cache-uri", "", "The URI of the cache server.") + fs.StringVar(&s.AlertChannel, "alert-channel", "", "The alert channel to send alerts to.") + fs.IntVar(&s.AlertChannelSize, "alert-channel-size", 100, "The size of the alert channel.") } type ControllerOptions struct { - ModifyVolumeRequestHandlerTimeout time.Duration + ModifyVolumeRequestHandlerTimeout ltime.Duration UserAgentExtra string } -func (s *ControllerOptions) AddFlags(fs *flag.FlagSet) { - fs.DurationVar(&s.ModifyVolumeRequestHandlerTimeout, "modify-volume-request-handler-timeout", driver.DefaultModifyVolumeRequestHandlerTimeout, "Timeout for the window in which volume modification calls must be received in order for them to coalesce into a single volume modification call to AWS. This must be lower than the csi-resizer and volumemodifier timeouts") +func (s *ControllerOptions) AddFlags(fs *lflag.FlagSet) { + fs.DurationVar(&s.ModifyVolumeRequestHandlerTimeout, "modify-volume-request-handler-timeout", lsdriver.DefaultModifyVolumeRequestHandlerTimeout, "Timeout for the window in which volume modification calls must be received in order for them to coalesce into a single volume modification call to AWS. This must be lower than the csi-resizer and volumemodifier timeouts") fs.StringVar(&s.UserAgentExtra, "user-agent-extra", "", "Extra string appended to user agent.") } type NodeOptions struct{} -func (s *NodeOptions) AddFlags(fs *flag.FlagSet) { +func (s *NodeOptions) AddFlags(fs *lflag.FlagSet) { } func (s *NodeOptions) Validate() error { return nil } -func GetOptions(fs *flag.FlagSet) *Options { +func GetOptions(fs *lflag.FlagSet) *Options { var ( version = fs.Bool("version", false, "Print the version and exit.") toStderr = fs.Bool("logtostderr", false, "log to standard error instead of files. DEPRECATED: will be removed in a future release.") - args = os.Args[1:] - cmd = string(driver.AllMode) + args = los.Args[1:] + cmd = string(lsdriver.AllMode) serverOptions = ServerOptions{} controllerOptions = ControllerOptions{} @@ -144,75 +151,75 @@ func GetOptions(fs *flag.FlagSet) *Options { ) serverOptions.AddFlags(fs) - c := logsapi.NewLoggingConfiguration() + c := llogApi.NewLoggingConfiguration() - err := logsapi.AddFeatureGates(featureGate) + err := llogApi.AddFeatureGates(featureGate) if err != nil { - klog.ErrorS(err, "failed to add feature gates") + llog.ErrorS(err, "failed to add feature gates") } - logsapi.AddFlags(c, fs) + llogApi.AddFlags(c, fs) - if len(os.Args) > 1 && !strings.HasPrefix(os.Args[1], "-") { - cmd = os.Args[1] - args = os.Args[2:] + if len(los.Args) > 1 && !lstr.HasPrefix(los.Args[1], "-") { + cmd = los.Args[1] + args = los.Args[2:] } switch cmd { - case string(driver.ControllerMode): + case string(lsdriver.ControllerMode): controllerOptions.AddFlags(fs) - case string(driver.NodeMode): + case string(lsdriver.NodeMode): nodeOptions.AddFlags(fs) - case string(driver.AllMode): + case string(lsdriver.AllMode): controllerOptions.AddFlags(fs) nodeOptions.AddFlags(fs) default: - klog.Errorf("Unknown driver mode %s: Expected %s, %s, %s", cmd, driver.ControllerMode, driver.NodeMode, driver.AllMode) - klog.FlushAndExit(klog.ExitFlushTimeout, 0) + llog.Errorf("Unknown driver mode %s: Expected %s, %s, %s", cmd, lsdriver.ControllerMode, lsdriver.NodeMode, lsdriver.AllMode) + llog.FlushAndExit(llog.ExitFlushTimeout, 0) } if err = fs.Parse(args); err != nil { panic(err) } - if cmd != string(driver.ControllerMode) { + if cmd != string(lsdriver.ControllerMode) { // nodeOptions must have been populated from the cmdline, validate them. if err := nodeOptions.Validate(); err != nil { - klog.Error(err.Error()) - klog.FlushAndExit(klog.ExitFlushTimeout, 1) + llog.Error(err.Error()) + llog.FlushAndExit(llog.ExitFlushTimeout, 1) } } - err = logsapi.ValidateAndApply(c, featureGate) + err = llogApi.ValidateAndApply(c, featureGate) if err != nil { - klog.ErrorS(err, "failed to validate and apply logging configuration") + llog.ErrorS(err, "failed to validate and apply logging configuration") } if *version { - versionInfo, err := driver.GetVersionJSON() + versionInfo, err := lsdriver.GetVersionJSON() if err != nil { - klog.ErrorS(err, "failed to get version") - klog.FlushAndExit(klog.ExitFlushTimeout, 1) + llog.ErrorS(err, "failed to get version") + llog.FlushAndExit(llog.ExitFlushTimeout, 1) } - fmt.Println(versionInfo) + llog.InfoS(versionInfo) osExit(0) } if *toStderr { - klog.SetOutput(os.Stderr) + llog.SetOutput(los.Stderr) } config, err := getConfigFromEnv() if err != nil { - klog.Errorf("Failed to get config from files: %v", err) - klog.FlushAndExit(klog.ExitFlushTimeout, 1) + llog.Errorf("Failed to get config from files: %v", err) + llog.FlushAndExit(llog.ExitFlushTimeout, 1) } options := &Options{ - DriverMode: driver.Mode(cmd), + DriverMode: lsdriver.Mode(cmd), ServerOptions: &serverOptions, ControllerOptions: &controllerOptions, NodeOptions: &nodeOptions, @@ -223,13 +230,13 @@ func GetOptions(fs *flag.FlagSet) *Options { } func getConfigFromEnv() (*Global, error) { - clientID := os.Getenv("VNGCLOUD_ACCESS_KEY_ID") - clientSecret := os.Getenv("VNGCLOUD_SECRET_ACCESS_KEY") - identityEndpoint := os.Getenv("VNGCLOUD_IDENTITY_ENDPOINT") - vserverEndpoint := os.Getenv("VNGCLOUD_VSERVER_ENDPOINT") + clientID := los.Getenv("VNGCLOUD_ACCESS_KEY_ID") + clientSecret := los.Getenv("VNGCLOUD_SECRET_ACCESS_KEY") + identityEndpoint := los.Getenv("VNGCLOUD_IDENTITY_ENDPOINT") + vserverEndpoint := los.Getenv("VNGCLOUD_VSERVER_ENDPOINT") if clientID == "" || clientSecret == "" || identityEndpoint == "" || vserverEndpoint == "" { - return nil, fmt.Errorf("missing required environment variables") + return nil, lfmt.Errorf("missing required environment variables") } var cfg Global diff --git a/go.mod b/go.mod index 62877f8..0d296d1 100644 --- a/go.mod +++ b/go.mod @@ -1,42 +1,45 @@ module github.com/vngcloud/vngcloud-blockstorage-csi-driver -go 1.22 +go 1.22.3 require ( github.com/container-storage-interface/spec v1.9.0 - github.com/cuongpiger/joat v1.0.4 + github.com/cuongpiger/joat v1.0.10 github.com/spf13/pflag v1.0.5 - github.com/vngcloud/vngcloud-csi-volume-modifier v0.0.0 - github.com/vngcloud/vngcloud-go-sdk v1.0.2 - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 - go.opentelemetry.io/otel v1.24.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 - go.opentelemetry.io/otel/sdk v1.24.0 - golang.org/x/sys v0.18.0 - google.golang.org/grpc v1.62.1 - google.golang.org/protobuf v1.33.0 - k8s.io/api v0.29.3 - k8s.io/apimachinery v0.29.3 - k8s.io/client-go v1.5.2 - k8s.io/component-base v0.29.2 + github.com/vngcloud/vngcloud-csi-volume-modifier v1.0.2 + github.com/vngcloud/vngcloud-go-sdk/v2 v2.1.22 + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0 + go.opentelemetry.io/otel v1.26.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.26.0 + go.opentelemetry.io/otel/sdk v1.26.0 + golang.org/x/sync v0.7.0 + golang.org/x/sys v0.20.0 + google.golang.org/grpc v1.64.0 + google.golang.org/protobuf v1.34.1 + k8s.io/api v0.30.1 + k8s.io/apimachinery v0.30.1 + k8s.io/client-go v0.30.1 + k8s.io/component-base v0.30.1 k8s.io/klog/v2 v2.120.1 - k8s.io/kubernetes v1.29.2 - k8s.io/mount-utils v0.29.2 - k8s.io/utils v0.0.0-20240310230437-4693a0247e57 + k8s.io/kubernetes v1.30.1 + k8s.io/mount-utils v0.30.1 + k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 + ) require ( github.com/andybalholm/brotli v1.1.0 // indirect github.com/cloudflare/circl v1.3.7 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/imroc/req/v3 v3.43.3 // indirect + github.com/imroc/req/v3 v3.43.5 // indirect github.com/klauspost/compress v1.17.7 // indirect - github.com/onsi/ginkgo/v2 v2.17.0 // indirect - github.com/onsi/gomega v1.32.0 // indirect + github.com/onsi/ginkgo/v2 v2.17.3 // indirect + github.com/onsi/gomega v1.33.1 // indirect github.com/quic-go/qpack v0.4.0 // indirect github.com/quic-go/quic-go v0.41.0 // indirect github.com/refraction-networking/utls v1.6.3 // indirect @@ -44,11 +47,13 @@ require ( ) require ( + github.com/Microsoft/go-winio v0.6.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cenkalti/backoff/v4 v4.2.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/emicklei/go-restful/v3 v3.12.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/logr v1.4.1 // indirect @@ -57,13 +62,12 @@ require ( github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect - github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20240319011627-a57c5dfe54fd // indirect + github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -74,35 +78,35 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/selinux v1.11.0 // indirect - github.com/prometheus/client_golang v1.18.0 // indirect - github.com/prometheus/client_model v0.6.0 // indirect - github.com/prometheus/common v0.47.0 // indirect - github.com/prometheus/procfs v0.12.0 // indirect + github.com/prometheus/client_golang v1.19.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.53.0 // indirect + github.com/prometheus/procfs v0.14.0 // indirect github.com/spf13/cobra v1.8.0 // indirect - go.opentelemetry.io/otel/metric v1.24.0 // indirect - go.opentelemetry.io/otel/trace v1.24.0 // indirect - go.opentelemetry.io/proto/otlp v1.1.0 // indirect + github.com/technoweenie/multipartstreamer v1.0.1 // indirect + go.opentelemetry.io/otel/metric v1.26.0 // indirect + go.opentelemetry.io/otel/trace v1.26.0 // indirect + go.opentelemetry.io/proto/otlp v1.2.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.21.0 // indirect + golang.org/x/crypto v0.22.0 // indirect golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81 // indirect - golang.org/x/mod v0.16.0 // indirect - golang.org/x/net v0.22.0 // indirect - golang.org/x/oauth2 v0.18.0 // indirect - golang.org/x/term v0.18.0 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.24.0 // indirect + golang.org/x/oauth2 v0.19.0 // indirect + golang.org/x/term v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.19.0 // indirect - google.golang.org/appengine v1.6.8 // indirect + golang.org/x/tools v0.20.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.29.2 // indirect - k8s.io/apiserver v0.29.2 // indirect - k8s.io/cloud-provider v0.29.2 // indirect - k8s.io/kube-openapi v0.0.0-20240209001042-7a0d5b415232 // indirect + k8s.io/apiextensions-apiserver v0.30.0 // indirect + k8s.io/apiserver v0.30.1 // indirect + k8s.io/cloud-provider v0.30.0 // indirect + k8s.io/kube-openapi v0.0.0-20240411171206-dc4e619f62f3 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect @@ -110,35 +114,35 @@ require ( replace ( github.com/google/cel-go => github.com/google/cel-go v0.17.7 // Mitigate https://github.com/kubernetes/apiserver/issues/97 - k8s.io/api => k8s.io/api v0.29.2 - k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.29.2 - k8s.io/apimachinery => k8s.io/apimachinery v0.29.2 - k8s.io/apiserver => k8s.io/apiserver v0.29.2 - k8s.io/cli-runtime => k8s.io/cli-runtime v0.29.2 - k8s.io/client-go => k8s.io/client-go v0.29.2 - k8s.io/cloud-provider => k8s.io/cloud-provider v0.29.2 - k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.29.2 - k8s.io/code-generator => k8s.io/code-generator v0.29.2 - k8s.io/component-base => k8s.io/component-base v0.29.2 - k8s.io/component-helpers => k8s.io/component-helpers v0.29.2 - k8s.io/controller-manager => k8s.io/controller-manager v0.29.2 - k8s.io/cri-api => k8s.io/cri-api v0.29.2 - k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.29.2 - k8s.io/dynamic-resource-allocation => k8s.io/dynamic-resource-allocation v0.29.2 - k8s.io/endpointslice => k8s.io/endpointslice v0.29.2 - k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.29.2 - k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.29.2 - k8s.io/kube-proxy => k8s.io/kube-proxy v0.29.2 - k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.29.2 - k8s.io/kubectl => k8s.io/kubectl v0.29.2 - k8s.io/kubelet => k8s.io/kubelet v0.29.2 - k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.29.2 - k8s.io/metrics => k8s.io/metrics v0.29.2 - k8s.io/mount-utils => k8s.io/mount-utils v0.29.2 - k8s.io/node-api => k8s.io/node-api v0.29.2 - k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.29.2 - k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.29.2 - k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.29.2 - k8s.io/sample-controller => k8s.io/sample-controller v0.29.2 + k8s.io/api => k8s.io/api v0.30.1 + k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.30.1 + k8s.io/apimachinery => k8s.io/apimachinery v0.30.1 + k8s.io/apiserver => k8s.io/apiserver v0.30.1 + k8s.io/cli-runtime => k8s.io/cli-runtime v0.30.1 + k8s.io/client-go => k8s.io/client-go v0.30.1 + k8s.io/cloud-provider => k8s.io/cloud-provider v0.30.1 + k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.30.1 + k8s.io/code-generator => k8s.io/code-generator v0.30.1 + k8s.io/component-base => k8s.io/component-base v0.30.1 + k8s.io/component-helpers => k8s.io/component-helpers v0.30.1 + k8s.io/controller-manager => k8s.io/controller-manager v0.30.1 + k8s.io/cri-api => k8s.io/cri-api v0.30.1 + k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.30.1 + k8s.io/dynamic-resource-allocation => k8s.io/dynamic-resource-allocation v0.30.1 + k8s.io/endpointslice => k8s.io/endpointslice v0.30.1 + k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.30.1 + k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.30.1 + k8s.io/kube-proxy => k8s.io/kube-proxy v0.30.1 + k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.30.1 + k8s.io/kubectl => k8s.io/kubectl v0.30.1 + k8s.io/kubelet => k8s.io/kubelet v0.30.1 + k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.30.1 + k8s.io/metrics => k8s.io/metrics v0.30.1 + k8s.io/mount-utils => k8s.io/mount-utils v0.30.1 + k8s.io/node-api => k8s.io/node-api v0.30.1 + k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.30.1 + k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.30.1 + k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.30.1 + k8s.io/sample-controller => k8s.io/sample-controller v0.30.1 vbom.ml/util => github.com/fvbommel/util v0.0.2 // Mitigate https://github.com/fvbommel/util/issues/6 ) diff --git a/go.sum b/go.sum index 990e7fb..6761bcb 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -6,18 +7,21 @@ github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= github.com/container-storage-interface/spec v1.9.0 h1:zKtX4STsq31Knz3gciCYCi1SXtO2HJDecIjDVboYavY= github.com/container-storage-interface/spec v1.9.0/go.mod h1:ZfDu+3ZRyeVqxZM0Ds19MVLkN2d1XJ5MAfi1L3VjlT0= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/cuongpiger/joat v1.0.4 h1:8ZdrjAHZgCE5m47ILx2SKBp0ubf5yBey7UaidyOPbJM= -github.com/cuongpiger/joat v1.0.4/go.mod h1:x1yiXQOWGFMzR+UH7YLFAH1u5LiMGjorgJZRjSGkoZY= +github.com/cuongpiger/joat v1.0.10 h1:VGtyLaMw4tsUVZzMsLFa9CaZwTu8rbJQhm6IKFVT/Bw= +github.com/cuongpiger/joat v1.0.10/go.mod h1:x1yiXQOWGFMzR+UH7YLFAH1u5LiMGjorgJZRjSGkoZY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/emicklei/go-restful/v3 v3.12.0 h1:y2DdzBAURM29NFF94q6RaY4vjIH1rtwDapwQtU84iWk= github.com/emicklei/go-restful/v3 v3.12.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= @@ -37,6 +41,7 @@ github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+Gr github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= @@ -56,6 +61,7 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20240319011627-a57c5dfe54fd h1:LjW4RcTwfcqOYGmD7UpFrn1gfBZ9mgu7QN5mSeFkCog= github.com/google/pprof v0.0.0-20240319011627-a57c5dfe54fd/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 h1:/c3QmbOGMGTOumP2iT/rCwB7b0QDGLKzqOmktBjT+Is= @@ -65,8 +71,8 @@ github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/imroc/req/v3 v3.43.3 h1:WdZhpUev9THtuwEZsW2LOYacl12fm7IkB7OgACv40+k= -github.com/imroc/req/v3 v3.43.3/go.mod h1:SQIz5iYop16MJxbo8ib+4LnostGCok8NQf8ToyQc2xA= +github.com/imroc/req/v3 v3.43.5 h1:fL7dOEfld+iEv1rwnIxseJz2/Y7JZ/HgbAURLZkat80= +github.com/imroc/req/v3 v3.43.5/go.mod h1:SQIz5iYop16MJxbo8ib+4LnostGCok8NQf8ToyQc2xA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -94,20 +100,26 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/ginkgo/v2 v2.17.0 h1:kdnunFXpBjbzN56hcJHrXZ8M+LOkenKA7NnBzTNigTI= github.com/onsi/ginkgo/v2 v2.17.0/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= +github.com/onsi/ginkgo/v2 v2.17.3/go.mod h1:nP2DPOQoNsQmsVyv5rDA8JkXQoCs6goXIvr/PRJ1eCc= github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg= +github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.47.0 h1:p5Cz0FNHo7SnWOmWmoRozVcjEp0bIVU8cV7OShpjL1k= github.com/prometheus/common v0.47.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= +github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/prometheus/procfs v0.14.0/go.mod h1:XL+Iwz8k8ZabyZfMFHPiilCniixqQarAy5Mu67pHlNQ= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= github.com/quic-go/quic-go v0.41.0 h1:aD8MmHfgqTURWNJy48IYFg2OnxwHT3JL7ahGs73lb4k= @@ -126,29 +138,40 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/technoweenie/multipartstreamer v1.0.1/go.mod h1:jNVxdtShOxzAsukZwTSw6MDx5eUJoiEBsSvzDU9uzog= github.com/vngcloud/vngcloud-csi-volume-modifier v0.0.0 h1:CUfbN11yjZdndwOXtjWa/kAe8/KGaENnFojPVOUvsk8= github.com/vngcloud/vngcloud-csi-volume-modifier v0.0.0/go.mod h1:6fFAm1DK2qsdQmQub4Ps5jBFP/LQqLV/SEUf96j51u4= -github.com/vngcloud/vngcloud-go-sdk v1.0.2 h1:AJjqZ03bLY6bUkZpSL9QAx6w6sQ8IToam4WEVf8orXo= -github.com/vngcloud/vngcloud-go-sdk v1.0.2/go.mod h1:f7Pf1V3vhoKaf2B0c6JjTr8cYiohse2F5y/atIjNnrE= +github.com/vngcloud/vngcloud-csi-volume-modifier v1.0.2/go.mod h1:wkhxk+x3ILNOk0aUxGivvhdiOXxPOVNXeRDHUx/IR50= +github.com/vngcloud/vngcloud-go-sdk/v2 v2.1.22/go.mod h1:L/i9ahQv8G0pJuy+Y5seXqNNRyNwLic0oB41XyB7j90= +github.com/vngcloud/vngcloud-go-sdk/v2 v2.1.23 h1:CwxnZbdKplfcHyw6fZxe16e7pVudmNXsGFRJK4JhJN4= +github.com/vngcloud/vngcloud-go-sdk/v2 v2.1.23/go.mod h1:L/i9ahQv8G0pJuy+Y5seXqNNRyNwLic0oB41XyB7j90= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0/go.mod h1:27iA5uvhuRNmalO+iEUdVn5ZMj2qy10Mm+XRIpRmyuU= go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 h1:t6wl9SPayj+c7lEIFgm4ooDBZVb01IhLB4InpomhRw8= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0/go.mod h1:iSDOcsnSA5INXzZtwaBPrKp/lWu/V14Dd+llD0oI2EA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.26.0/go.mod h1:z46paqbJ9l7c9fIPCXTqTGwhQZ5XoTIsfeFYWboizjs= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 h1:Mw5xcxMwlqoJd97vwPxA8isEaIoxsta9/Q51+TTJLGE= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0/go.mod h1:CQNu9bj7o7mC6U7+CA/schKEYakYXWr79ucDHTMGhCM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.26.0/go.mod h1:wnJIG4fOqyynOnnQF/eQb4/16VlX2EJAHhHgqIqWfAo= go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= +go.opentelemetry.io/otel/sdk v1.26.0/go.mod h1:0p8MXpqLeJ0pzcszQQN4F0S5FVjBLgypeGSngLsmirs= go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI= go.opentelemetry.io/proto/otlp v1.1.0/go.mod h1:GpBHCBWiqvVLDqmHZsoMM3C5ySeKTC7ej/RNTae6MdY= +go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= @@ -163,6 +186,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81 h1:6R2FC06FonbXQ8pK11/PDFY6N6LWlf9KlzibaCapmqc= golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -170,6 +194,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -178,12 +203,17 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= +golang.org/x/oauth2 v0.19.0/go.mod h1:vYi7skDa1x015PmRRYZ7+s1cWyPgrPiSYRe4rnsexc8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -193,10 +223,12 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -212,6 +244,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= +golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -222,12 +255,15 @@ google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1: google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE= google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -241,18 +277,25 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.29.2 h1:hBC7B9+MU+ptchxEqTNW2DkUosJpp1P+Wn6YncZ474A= k8s.io/api v0.29.2/go.mod h1:sdIaaKuU7P44aoyyLlikSLayT6Vb7bvJNCX105xZXY0= +k8s.io/api v0.30.1/go.mod h1:ddbN2C0+0DIiPntan/bye3SW3PdwLa11/0yqwvuRrJM= k8s.io/apiextensions-apiserver v0.29.2 h1:UK3xB5lOWSnhaCk0RFZ0LUacPZz9RY4wi/yt2Iu+btg= k8s.io/apiextensions-apiserver v0.29.2/go.mod h1:aLfYjpA5p3OwtqNXQFkhJ56TB+spV8Gc4wfMhUA3/b8= +k8s.io/apiextensions-apiserver v0.30.1/go.mod h1:R4GuSrlhgq43oRY9sF2IToFh7PVlF1JjfWdoG3pixk4= k8s.io/apimachinery v0.29.2 h1:EWGpfJ856oj11C52NRCHuU7rFDwxev48z+6DSlGNsV8= k8s.io/apimachinery v0.29.2/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU= +k8s.io/apimachinery v0.30.1/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= k8s.io/apiserver v0.29.2 h1:+Z9S0dSNr+CjnVXQePG8TcBWHr3Q7BmAr7NraHvsMiQ= k8s.io/apiserver v0.29.2/go.mod h1:B0LieKVoyU7ykQvPFm7XSdIHaCHSzCzQWPFa5bqbeMQ= +k8s.io/apiserver v0.30.1/go.mod h1:i87ZnQ+/PGAmSbD/iEKM68bm1D5reX8fO4Ito4B01mo= k8s.io/client-go v0.29.2 h1:FEg85el1TeZp+/vYJM7hkDlSTFZ+c5nnK44DJ4FyoRg= k8s.io/client-go v0.29.2/go.mod h1:knlvFZE58VpqbQpJNbCbctTVXcd35mMyAAwBdpt4jrA= +k8s.io/client-go v0.30.1/go.mod h1:wrAqLNs2trwiCH/wxxmT/x3hKVH9PuV0GGW0oDoHVqc= k8s.io/cloud-provider v0.29.2 h1:ghKNXoQmeP8Fj/YTJNR6xQOzNrKXt6YZyy6mOEEa3yg= k8s.io/cloud-provider v0.29.2/go.mod h1:KAp+07AUGmxcLnoLY5FndU4hj6158KMbiviNgctNRUk= +k8s.io/cloud-provider v0.30.1/go.mod h1:1uZp+FSskXQoeAAIU91/XCO8X/9N1U3z5usYeSLT4MI= k8s.io/component-base v0.29.2 h1:lpiLyuvPA9yV1aQwGLENYyK7n/8t6l3nn3zAtFTJYe8= k8s.io/component-base v0.29.2/go.mod h1:BfB3SLrefbZXiBfbM+2H1dlat21Uewg/5qtKOl8degM= +k8s.io/component-base v0.30.1/go.mod h1:e/X9kDiOebwlI41AvBHuWdqFriSRrX50CdwA9TFaHLI= k8s.io/component-helpers v0.29.2 h1:1kTIanIdqUVG2nW3e2ENVEaYbZKphqPgEdCmJvk71aw= k8s.io/component-helpers v0.29.2/go.mod h1:gFc/p60rYtpD8UCcNfPCmbokHT2uy0yDpmr/KKUMNAw= k8s.io/csi-translation-lib v0.29.2 h1:TJVZTzR7gj6+HSb+jJxLUxnAuwrEy71IxhJ4nmTzyjE= @@ -261,12 +304,16 @@ k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240209001042-7a0d5b415232 h1:MMq4iF9pHuAz/9dLnHwBQKEoeigXClzs3MFh/seyqtA= k8s.io/kube-openapi v0.0.0-20240209001042-7a0d5b415232/go.mod h1:Pa1PvrP7ACSkuX6I7KYomY6cmMA0Tx86waBhDUgoKPw= +k8s.io/kube-openapi v0.0.0-20240411171206-dc4e619f62f3/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/kubernetes v1.29.2 h1:8hh1cntqdulanjQt7wSSSsJfBgOyx6fUdFWslvGL5m0= k8s.io/kubernetes v1.29.2/go.mod h1:xZPKU0yO0CBbLTnbd+XGyRmmtmaVuJykDb8gNCkeeUE= +k8s.io/kubernetes v1.30.1/go.mod h1:yPbIk3MhmhGigX62FLJm+CphNtjxqCvAIFQXup6RKS0= k8s.io/mount-utils v0.29.2 h1:FrUfgvOo63nqJRPXKoqN/DW1lMnR/y0pzpFErKh6p2o= k8s.io/mount-utils v0.29.2/go.mod h1:9IWJTMe8tG0MYMLEp60xK9GYVeCdA3g4LowmnVi+t9Y= +k8s.io/mount-utils v0.30.1/go.mod h1:9sCVmwGLcV1MPvbZ+rToMDnl1QcGozy+jBPd0MsQLIo= k8s.io/utils v0.0.0-20240310230437-4693a0247e57 h1:gbqbevonBh57eILzModw6mrkbwM0gQBEuevE/AaBsHY= k8s.io/utils v0.0.0-20240310230437-4693a0247e57/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go index 2aa1a31..62805c4 100644 --- a/pkg/cloud/cloud.go +++ b/pkg/cloud/cloud.go @@ -1,102 +1,58 @@ package cloud import ( - "fmt" - "strings" + lctx "context" + lfmt "fmt" - lset "github.com/cuongpiger/joat/data-structure/set" ljmath "github.com/cuongpiger/joat/math" - ljutils "github.com/cuongpiger/joat/utils" + ljtime "github.com/cuongpiger/joat/timer" ljwait "github.com/cuongpiger/joat/utils/exponential-backoff" - "github.com/vngcloud/vngcloud-go-sdk/client" - lsdkErr "github.com/vngcloud/vngcloud-go-sdk/error" - "github.com/vngcloud/vngcloud-go-sdk/vngcloud" - lerrEH "github.com/vngcloud/vngcloud-go-sdk/vngcloud/errors" - lsdkObj "github.com/vngcloud/vngcloud-go-sdk/vngcloud/objects" - lvolAct "github.com/vngcloud/vngcloud-go-sdk/vngcloud/services/blockstorage/v2/extensions/volume_actions" - lsnapshotV2 "github.com/vngcloud/vngcloud-go-sdk/vngcloud/services/blockstorage/v2/snapshot" - lvolV2 "github.com/vngcloud/vngcloud-go-sdk/vngcloud/services/blockstorage/v2/volume" - lvolAtch "github.com/vngcloud/vngcloud-go-sdk/vngcloud/services/compute/v2/extensions/volume_attach" - "github.com/vngcloud/vngcloud-go-sdk/vngcloud/services/identity/v2/extensions/oauth2" - "github.com/vngcloud/vngcloud-go-sdk/vngcloud/services/identity/v2/tokens" - lportal "github.com/vngcloud/vngcloud-go-sdk/vngcloud/services/portal/v1" - "k8s.io/klog/v2" + lsentity "github.com/vngcloud/vngcloud-blockstorage-csi-driver/pkg/cloud/entity" + lserr "github.com/vngcloud/vngcloud-blockstorage-csi-driver/pkg/cloud/errors" + lsdkClientV2 "github.com/vngcloud/vngcloud-go-sdk/v2/client" + lsdkEntity "github.com/vngcloud/vngcloud-go-sdk/v2/vngcloud/entity" + lsdkErrs "github.com/vngcloud/vngcloud-go-sdk/v2/vngcloud/sdk_error" + lsdkComputeV2 "github.com/vngcloud/vngcloud-go-sdk/v2/vngcloud/services/compute/v2" + lsdkPortalSvcV1 "github.com/vngcloud/vngcloud-go-sdk/v2/vngcloud/services/portal/v1" + lsdkVolumeV1 "github.com/vngcloud/vngcloud-go-sdk/v2/vngcloud/services/volume/v1" + lsdkVolumeV2 "github.com/vngcloud/vngcloud-go-sdk/v2/vngcloud/services/volume/v2" + llog "k8s.io/klog/v2" lsutil "github.com/vngcloud/vngcloud-blockstorage-csi-driver/pkg/util" ) func NewCloud(iamURL, vserverUrl, clientID, clientSecret string, metadataSvc MetadataService) (Cloud, error) { - vserverV1 := ljutils.NormalizeURL(vserverUrl) + "v1" - vserverV2 := ljutils.NormalizeURL(vserverUrl) + "v2" + projectID := metadataSvc.GetProjectID() + clientCfg := lsdkClientV2.NewSdkConfigure(). + WithClientId(clientID). + WithClientSecret(clientSecret). + WithIamEndpoint(iamURL). + WithVServerEndpoint(vserverUrl) - pc, err := newVngCloud(iamURL, clientID, clientSecret) - if err != nil { - return nil, err - } - - vserverV2Client, _ := vngcloud.NewServiceClient(vserverV2, pc, "vserver-v2") - vserverV1Client, _ := vngcloud.NewServiceClient(vserverV1, pc, "vserver-v1") - ei, err := setupPortalInfo(vserverV1Client, metadataSvc) - if err != nil { - return nil, err - } - - return &cloud{ - vServerV2: vserverV2Client, - vServerV1: vserverV1Client, - metadataService: metadataSvc, - extraInfo: ei, - }, nil -} + cloudClient := lsdkClientV2.NewClient(lctx.TODO()).Configure(clientCfg) -func newVngCloud(iamURL, clientID, clientSecret string) (*client.ProviderClient, error) { - identityUrl := ljutils.NormalizeURL(iamURL) + "v2" - provider, _ := vngcloud.NewClient(identityUrl) - err := vngcloud.Authenticate(provider, &oauth2.AuthOptions{ - ClientID: clientID, - ClientSecret: clientSecret, - AuthOptionsBuilder: &tokens.AuthOptions{ - IdentityEndpoint: iamURL, - }, - }) - - return provider, err -} + llog.V(5).InfoS("[DEBUG] - NodeGetInfo: Get the portal info and quota") + portal, sdkErr := cloudClient.VServerGateway().V1().PortalService(). + GetPortalInfo(lsdkPortalSvcV1.NewGetPortalInfoRequest(projectID)) -func setupPortalInfo(pportalClient *client.ServiceClient, pmetadataSvc MetadataService) (*extraInfa, error) { - projectID := pmetadataSvc.GetProjectID() - if len(projectID) < 1 { - return nil, fmt.Errorf("projectID is empty") + if sdkErr != nil { + llog.ErrorS(sdkErr.GetError(), "[ERROR] - NodeGetInfo; failed to get portal info") + return nil, sdkErr.GetError() } - klog.InfoS("setupPortalInfo", "projectID", projectID) + llog.InfoS("[INFO] - NodeGetInfo: Received the portal info successfully", "portal", portal) + cloudClient = cloudClient.WithProjectId(portal.ProjectID) - portalInfo, err := lportal.Get(pportalClient, projectID) - if err != nil { - return nil, err - } - - if portalInfo == nil { - return nil, fmt.Errorf("can not get portal information") - } - - return &extraInfa{ - ProjectID: portalInfo.ProjectID, - UserID: portalInfo.UserID, + return &cloud{ + metadataService: metadataSvc, + client: cloudClient, }, nil } type ( cloud struct { - vServerV1 *client.ServiceClient - vServerV2 *client.ServiceClient - extraInfo *extraInfa metadataService MetadataService - } - - extraInfa struct { - ProjectID string - UserID int64 + client lsdkClientV2.IClient } // ModifyDiskOptions represents parameters to modify a volume @@ -105,110 +61,175 @@ type ( } ) -func (s *cloud) CreateVolume(popts *lvolV2.CreateOpts) (*lsdkObj.Volume, error) { - opts := lvolV2.NewCreateOpts(s.getProjectID(), popts) - vol, err := lvolV2.Create(s.vServerV2, opts) +func (s *cloud) EitherCreateResizeVolume(preq lsdkVolumeV2.ICreateBlockVolumeRequest) (*lsentity.Volume, lserr.IError) { + var ( + vol, tmpVol *lsdkEntity.Volume + serr lserr.IError + sdkErr lsdkErrs.ISdkError + ) - if err != nil { - return nil, err + // Get the volume depend on the volume name + if preq.GetVolumeName() != "" { + llog.InfoS("[INFO] - EitherCreateResizeVolume: Get the volume by name", "volumeName", preq.GetVolumeName()) + vol, serr = s.getVolumeByName(preq.GetVolumeName()) + if serr != nil { + if !serr.IsError(lsdkErrs.EcVServerVolumeNotFound) { + llog.ErrorS(serr.GetError(), "[ERROR] - EitherCreateResizeVolume: Failed to get the volume by name", serr.GetListParameters()...) + return nil, serr + } + } } - vol, err = s.waitVolumeAchieveStatus(vol.VolumeId, volumeAvailableStatus) - if err != nil { - return nil, err - } + if vol != nil { + newSize := ljmath.MaxNumeric(vol.Size, uint64(preq.GetSize())) + newVolumeType := preq.GetVolumeType() + if vol.Size != newSize || vol.VolumeTypeID != newVolumeType { + llog.InfoS("[INFO] - EitherCreateResizeVolume: Resize the volume", "volumeID", vol.Id, "newSize", newSize, "newVolumeType", newVolumeType) + opt := lsdkVolumeV2.NewResizeBlockVolumeByIdRequest(newVolumeType, vol.Id, int(newSize)) + tmpVol, sdkErr = s.client.VServerGateway().V2().VolumeService().ResizeBlockVolumeById(opt) + if sdkErr != nil { + if sdkErr.IsError(lsdkErrs.EcVServerVolumeUnchanged) { + return &lsentity.Volume{Volume: tmpVol}, nil + } + + llog.ErrorS(sdkErr.GetError(), "[ERROR] - EitherCreateResizeVolume: Failed to resize the volume", sdkErr.GetListParameters()...) + return nil, lserr.NewError(sdkErr) + } - newSize := ljmath.MaxNumeric(vol.Size, popts.Size) - newVolumeType := popts.VolumeTypeId - if vol.Size != newSize || vol.VolumeTypeID != newVolumeType { - if err = s.ExpandVolume(vol.VolumeId, newVolumeType, newSize); err != nil { - return nil, err + vol = tmpVol } - vol.Size = newSize - vol.VolumeTypeID = newVolumeType + return &lsentity.Volume{Volume: vol}, nil } - return vol, err + llog.InfoS("[INFO] - EitherCreateResizeVolume: Create the volume", preq.GetListParameters()...) + vol, sdkErr = s.client.VServerGateway().V2().VolumeService().CreateBlockVolume(preq) + if sdkErr != nil { + llog.ErrorS(sdkErr.GetError(), "[ERROR] - EitherCreateResizeVolume: Failed to create the volume", sdkErr.GetListParameters()...) + return nil, lserr.NewError(sdkErr) + } + + llog.InfoS("[INFO] - EitherCreateResizeVolume: Created the volume successfully", "volumeID", vol.Id) + return &lsentity.Volume{ + Volume: vol, + }, nil } -func (s *cloud) GetVolumeByName(pvolName string) (*lsdkObj.VolumeList, error) { - opts := lvolV2.NewListOpts(s.getProjectID(), pvolName, defaultPage, defaultPageSize) - vl, err := lvolV2.List(s.vServerV2, opts) - if err != nil { - return nil, err.Error +func (s *cloud) GetVolumeByName(pvolName string) (*lsentity.Volume, lserr.IError) { + vol, serr := s.getVolumeByName(pvolName) + if serr != nil { + return nil, serr } - return vl, nil + return &lsentity.Volume{ + Volume: vol, + }, nil } -func (s *cloud) GetVolume(volumeID string) (*lsdkObj.Volume, *lsdkErr.SdkError) { - opts := lvolV2.NewGetOpts(s.getProjectID(), volumeID) - result, err := lvolV2.Get(s.vServerV2, opts) - return result, err +func (s *cloud) GetVolume(volumeID string) (*lsentity.Volume, lserr.IError) { + vol, serr := s.getVolumeById(volumeID) + if serr != nil { + return nil, serr + } + + return &lsentity.Volume{ + Volume: vol, + }, nil } func (s *cloud) DeleteVolume(volID string) error { - vol, err := s.GetVolume(volID) - if err != nil && err.Code == lerrEH.ErrCodeVolumeNotFound { + vol, ierr := s.GetVolume(volID) + if ierr != nil && ierr.IsError(lsdkErrs.EcVServerVolumeNotFound) { return nil } - if vol.Status == VolumeAvailableStatus { - return lvolV2.Delete(s.vServerV2, lvolV2.NewDeleteOpts(s.extraInfo.ProjectID, volID)) + if vol.CanDelete() { + _, err := s.waitVolumeAchieveStatus(volID, volumeAvailableStatus) + if err != nil { + return err + } + + opt := lsdkVolumeV2.NewDeleteBlockVolumeByIdRequest(volID) + sdkErr := s.client.VServerGateway().V2().VolumeService().DeleteBlockVolumeById(opt) + if sdkErr != nil { + if sdkErr.IsError(lsdkErrs.EcVServerVolumeNotFound) { + return nil + } + + return sdkErr.GetError() + } } return nil } -func (s *cloud) AttachVolume(instanceID, volumeID string) (string, error) { - vol, err := s.GetVolume(volumeID) - if err != nil { - return "", err.Error - } +func (s *cloud) AttachVolume(pinstanceId, pvolumeId string) (*lsentity.Volume, error) { + var ( + svol *lsentity.Volume + err error + ) - if vol == nil { - return "", fmt.Errorf("volume %s not found", volumeID) + svol, err = s.waitVolumeAchieveStatus(pvolumeId, volumeArchivedStatus) + if err != nil { + return nil, err } - if vol.VmId != nil && *vol.VmId != "-1" { - return "", fmt.Errorf("volume %s already attached to instance %s", volumeID, *vol.VmId) + if svol.AttachedTheInstance(pinstanceId) { + return svol, nil } - opts := lvolAtch.NewCreateOpts(s.getProjectID(), instanceID, volumeID) - _, err2 := lvolAtch.Attach(s.vServerV2, opts) - if err2 != nil { - if err2.Code == lerrEH.ErrCodeVolumeAlreadyAttached { - return vol.VolumeId, nil + opt := lsdkComputeV2.NewAttachBlockVolumeRequest(pinstanceId, pvolumeId) + sdkErr := s.client.VServerGateway().V2().ComputeService().AttachBlockVolume(opt) + if sdkErr != nil { + if !sdkErr.IsError(lsdkErrs.EcVServerVolumeAlreadyAttachedThisServer) { + return nil, sdkErr.GetError() } - - return "", err2.Error } - err3 := s.waitDiskAttached(instanceID, volumeID) + err = s.waitDiskAttached(pinstanceId, pvolumeId) + return svol, err - return vol.VolumeId, err3 } -func (s *cloud) DetachVolume(instanceID, volumeID string) error { - _, err := lvolAtch.Detach(s.vServerV2, lvolAtch.NewDeleteOpts(s.getProjectID(), instanceID, volumeID)) - // Disk has no attachments or not attached to the provided compute - if err != nil { - if errSetDetachIngore.ContainsOne(err.Code) { - return nil +func (s *cloud) DetachVolume(pinstanceId, pvolumeId string) error { + if err := ljwait.ExponentialBackoff(ljwait.NewBackOff(10, 10, true, ljtime.Minute(10)), func() (bool, error) { + _, sdkErr := s.getVolumeById(pvolumeId) + if sdkErr != nil { + if sdkErr.IsError(lsdkErrs.EcVServerVolumeNotFound) { + return true, nil + } + return false, sdkErr.GetError() + } + + opt := lsdkComputeV2.NewDetachBlockVolumeRequest(pinstanceId, pvolumeId) + sdkErr = s.client.VServerGateway().V2().ComputeService().DetachBlockVolume(opt) + if sdkErr != nil { + if errSetDetachIngore.ContainsOne(sdkErr.GetErrorCode()) { + return true, nil + } + + if sdkErr.IsError(lsdkErrs.EcVServerVolumeInProcess) { + return false, nil + } + + llog.ErrorS(sdkErr.GetError(), "[ERROR] - DetachVolume: Failed to detach the volume", sdkErr.GetListParameters()...) + return false, sdkErr.GetError() } - return err.Error + return false, nil + }); err != nil { + return err } - return s.waitDiskDetached(volumeID) + llog.V(5).InfoS("[DEBUG] - DetachVolume: Detached the volume successfully", "volumeId", pvolumeId, "instanceId", pinstanceId) + return nil } func (s *cloud) ResizeOrModifyDisk(volumeID string, newSizeBytes int64, options *ModifyDiskOptions) (newSize int64, err error) { newSizeGiB := uint64(lsutil.RoundUpGiB(newSizeBytes)) - volume, errSdk := s.GetVolume(volumeID) - if errSdk != nil { - return 0, errSdk.Error + volume, sdkErr := s.GetVolume(volumeID) + if sdkErr != nil { + return 0, sdkErr.GetError() } if newSizeGiB < volume.Size { @@ -219,15 +240,62 @@ func (s *cloud) ResizeOrModifyDisk(volumeID string, newSizeBytes int64, options options.VolumeType = volume.VolumeTypeID } + // Check that we need to modify this volume` needsModification, volumeSize, err := s.validateModifyVolume(volumeID, newSizeGiB, options) if err != nil || !needsModification { return volumeSize, err } - opts := lvolAct.NewResizeOpts(s.getProjectID(), options.VolumeType, volumeID, newSizeGiB) - _, errSdk = lvolAct.Resize(s.vServerV2, opts) - if errSdk != nil { - return 0, errSdk.Error + // The volume types are different => so please check the zone a same + same, sdkErr2 := s.checkSameZone(options.VolumeType, volume.VolumeTypeID) + if sdkErr2 != nil { + return 0, sdkErr2.GetError() + } else if !same && !volume.IsAttched() { + // In the case of these volume types are not in the same zone, we MUST migrate the volume to the target zone before resize this volume + err = ljwait.ExponentialBackoff(ljwait.NewBackOff(10, 10, true, ljtime.Minute(30)), func() (bool, error) { + volume, sdkErr = s.GetVolume(volumeID) + if sdkErr != nil { + return true, sdkErr.GetError() + } + + if volume.IsMigration() || volume.IsCreating() { + return false, nil + } else if volumeArchivedStatus.ContainsOne(volume.Status) && volume.VolumeTypeID == options.VolumeType { + // Check the volume status is in the archived status + return true, nil + } + + // So make a migration volume request to the VngCloud API + sdkErr2 = s.client.VServerGateway().V2().VolumeService(). + MigrateBlockVolumeById( + lsdkVolumeV2.NewMigrateBlockVolumeByIdRequest(volumeID, options.VolumeType). + WithConfirm(true)) + + // In the case of getting an error from the VngCloud API + if sdkErr2 != nil { + switch sdkErr2.GetErrorCode() { + case lsdkErrs.EcVServerVolumeMigrateInSameZone, lsdkErrs.EcVServerVolumeMigrateBeingProcess, lsdkErrs.EcVServerVolumeMigrateProcessingConfirm, lsdkErrs.EcVServerVolumeMigrateBeingMigrating, lsdkErrs.EcVServerVolumeMigrateBeingFinish: + // These statuses are used to indicate that the volume is being migrated => continue to wait + return false, nil + default: + // In some other cases, we need to handle the error + return true, sdkErr2.GetError() + } + } + + // Otherwise, we need to wait for the volume to be migrated + return false, nil + }) + + if err != nil { + return 0, err + } + } + + opt := lsdkVolumeV2.NewResizeBlockVolumeByIdRequest(volumeID, options.VolumeType, int(newSizeGiB)) + _, sdkErr = s.client.VServerGateway().V2().VolumeService().ResizeBlockVolumeById(opt) + if sdkErr != nil && !sdkErr.IsError(lsdkErrs.EcVServerVolumeUnchanged) { + return 0, sdkErr.GetError() } _, err = s.waitVolumeAchieveStatus(volumeID, volumeArchivedStatus) @@ -240,213 +308,119 @@ func (s *cloud) ResizeOrModifyDisk(volumeID string, newSizeBytes int64, options } func (s *cloud) ExpandVolume(volumeID, volumeTypeID string, newSize uint64) error { - opts := lvolAct.NewResizeOpts(s.extraInfo.ProjectID, volumeTypeID, volumeID, newSize) - _, errSdk := lvolAct.Resize(s.vServerV2, opts) - - if errSdk != nil { - if errSdk.Code == lerrEH.ErrCodeVolumeUnchanged { - return nil - } - - return errSdk.Error - } - - _, err := s.waitVolumeAchieveStatus(volumeID, volumeArchivedStatus) + //last := false + //err := ljwait.ExponentialBackoff(ljwait.NewBackOff(10, 10, true, ljtime.Minute(10)), func() (bool, error) { + // vol, err := s.getVolumeById(volumeID) + // if err != nil { + // lsalert.HandleIError(err) + // return true, err.GetError() + // } + // + // if volumeArchivedStatus.ContainsOne(vol.Status) { + // if last { + // return true, nil + // } + // + // opt := lsdkVolumeV2.NewResizeBlockVolumeByIdRequest(volumeID, volumeTypeID, int(newSize)) + // if _, sdkErr := s.client.VServerGateway().V2().VolumeService().ResizeBlockVolumeById(opt); sdkErr != nil { + // if sdkErr.IsError(lsdkErrs.EcVServerVolumeUnchanged) { + // return true, nil + // } + // + // lsalert.HandleSdkError(sdkErr) + // return true, sdkErr.GetError() + // } + // + // last = true + // } + // + // return false, nil + //}) + // + //return err + _, err := s.ResizeOrModifyDisk(volumeID, lsutil.GiBToBytes(int64(newSize)), &ModifyDiskOptions{ + VolumeType: volumeTypeID, + }) return err } func (s *cloud) GetDeviceDiskID(pvolID string) (string, error) { - opts := lvolAct.NewMappingOpts(s.getProjectID(), pvolID) - res, err := lvolAct.GetMappingVolume(s.vServerV2, opts) - - // Process error occurs + opts := lsdkVolumeV2.NewGetBlockVolumeByIdRequest(pvolID) + vol, err := s.client.VServerGateway().V2().VolumeService().GetUnderBlockVolumeId(opts) if err != nil { - return "", err + llog.ErrorS(err.GetError(), "[ERROR] - GetDeviceDiskID: Failed to get the device disk ID", err.GetListParameters()...) + return "", err.GetError() } - if len(res.UUID) < DefaultDiskSymbolIdLength { - return "", ErrDeviceVolumeIdNotFound - } - - // Truncate the UUID for the virtual disk - return res.UUID, nil + return vol.UnderId, nil } -func (s *cloud) GetVolumeSnapshotByName(pvolID, psnapshotName string) (*lsdkObj.Snapshot, error) { - res, err := lsnapshotV2.ListVolumeSnapshot( - s.vServerV2, lsnapshotV2.NewListVolumeOpts(s.getProjectID(), pvolID, defaultPage, defaultPageSize)) +func (s *cloud) GetVolumeSnapshotByName(pvolID, psnapshotName string) (*lsentity.Snapshot, error) { + opt := lsdkVolumeV2.NewListSnapshotsByBlockVolumeIdRequest(1, 10, pvolID) + res, err := s.client.VServerGateway().V2().VolumeService().ListSnapshotsByBlockVolumeId(opt) if err != nil { - return nil, err.Error + return nil, err.GetError() } for _, snap := range res.Items { - if snap.VolumeID == pvolID && snap.Name == psnapshotName { - return &snap, nil + if snap.VolumeId == pvolID && snap.Name == psnapshotName { + return &lsentity.Snapshot{Snapshot: snap}, nil } } return nil, ErrSnapshotNotFound } -func (s *cloud) CreateSnapshotFromVolume(pvolID string, popts *lsnapshotV2.CreateOpts) (*lsdkObj.Snapshot, error) { - resp, err := lsnapshotV2.Create(s.vServerV2, lsnapshotV2.NewCreateOpts(s.getProjectID(), pvolID, popts)) - if err != nil { - return nil, err +func (s *cloud) CreateSnapshotFromVolume(pclusterId, pvolId, psnapshotName string) (*lsentity.Snapshot, error) { + opt := lsdkVolumeV2.NewCreateSnapshotByBlockVolumeIdRequest(psnapshotName, pvolId). + WithPermanently(true). + WithDescription(lfmt.Sprintf(patternSnapshotDescription, pvolId, pclusterId)) + + snapshot, sdkErr := s.client.VServerGateway().V2().VolumeService().CreateSnapshotByBlockVolumeId(opt) + if sdkErr != nil { + return nil, sdkErr.GetError() } - err = s.waitSnapshotActive(popts.VolumeID, resp.Name) - return resp, err + err := s.waitSnapshotActive(pvolId, snapshot.Name) + return &lsentity.Snapshot{Snapshot: snapshot}, err } func (s *cloud) DeleteSnapshot(psnapshotID string) error { - err := lsnapshotV2.Delete(s.vServerV2, lsnapshotV2.NewDeleteOpts(s.getProjectID(), psnapshotID)) - if err != nil && err.Code != lerrEH.ErrCodeSnapshotNotFound { - return err.Error + opt := lsdkVolumeV2.NewDeleteSnapshotByIdRequest(psnapshotID) + sdkErr := s.client.VServerGateway().V2().VolumeService().DeleteSnapshotById(opt) + if sdkErr != nil { + if !sdkErr.IsError(lsdkErrs.EcVServerSnapshotNotFound) { + return sdkErr.GetError() + } } - return nil } -func (s *cloud) ListSnapshots(pvolID string, ppage int, ppageSize int) (*lsdkObj.SnapshotList, error) { - opts := lsnapshotV2.NewListVolumeOpts(s.getProjectID(), pvolID, ppage, ppageSize) - resp, err := lsnapshotV2.ListVolumeSnapshot(s.vServerV2, opts) - if err != nil { - return nil, err.Error - } - - return resp, nil -} - -func (s *cloud) waitVolumeAchieveStatus(pvolID string, pdesiredStatus lset.Set[string]) (*lsdkObj.Volume, error) { - var resVolume *lsdkObj.Volume - err := ljwait.ExponentialBackoff(ljwait.NewBackOff(waitVolumeActiveSteps, waitVolumeActiveDelay, true, waitVolumeActiveTimeout), func() (bool, error) { - vol, err := s.GetVolume(pvolID) - if err != nil { - return false, err.Error - } - - if pdesiredStatus.ContainsOne(vol.Status) { - resVolume = vol - return true, nil - } - - return false, nil - }) - - if err != nil { - return nil, err +func (s *cloud) ListSnapshots(pvolID string, ppage int, ppageSize int) (*lsentity.ListSnapshots, lserr.IError) { + opt := lsdkVolumeV2.NewListSnapshotsByBlockVolumeIdRequest(ppage, ppageSize, pvolID) + res, sdkErr := s.client.VServerGateway().V2().VolumeService().ListSnapshotsByBlockVolumeId(opt) + if sdkErr != nil { + return nil, lserr.NewError(sdkErr) } - return resVolume, nil -} - -func (s *cloud) waitSnapshotActive(pvolID, psnapshotName string) error { - return ljwait.ExponentialBackoff(ljwait.NewBackOff(waitSnapshotActiveSteps, waitSnapshotActiveDelay, true, waitSnapshotActiveTimeout), func() (bool, error) { - vol, err := s.GetVolumeSnapshotByName(pvolID, psnapshotName) - if err != nil { - return false, err - } - - if vol.Status == SnapshotActiveStatus { - return true, nil - } - - return false, nil - }) -} - -func (s *cloud) waitDiskDetached(volumeID string) error { - return ljwait.ExponentialBackoff( - ljwait.NewBackOff(waitVolumeDetachSteps, waitVolumeDetachDelay, true, waitVolumeDetachTimeout), - func() (bool, error) { - vol, err := s.GetVolume(volumeID) - if err != nil { - return false, err.Error - } - - if vol.Status == VolumeAvailableStatus || (vol.Status == VolumeInUseStatus && len(vol.AttachedMachine) > 0) { - return true, nil - } - - return false, nil - }, - ) -} - -func (s *cloud) waitDiskAttached(instanceID string, volumeID string) error { - return ljwait.ExponentialBackoff(ljwait.NewBackOff(waitVolumeAttachSteps, waitVolumeAttachDelay, true, waitVolumeAttachTimeout), func() (bool, error) { - vol, err := s.GetVolume(volumeID) - if err != nil { - return false, err.Error - } - - if vol.VmId != nil && *vol.VmId == instanceID && vol.Status == VolumeInUseStatus { - return true, nil - - } - - return false, nil - }) + return &lsentity.ListSnapshots{ListSnapshots: res}, nil } -func (s *cloud) getProjectID() string { - return s.extraInfo.ProjectID -} - -func (s *cloud) validateModifyVolume(volumeID string, newSizeGiB uint64, options *ModifyDiskOptions) (bool, int64, error) { - volume, err := s.GetVolume(volumeID) +func (s *cloud) GetVolumeTypeById(pvolTypeId string) (*lsentity.VolumeType, lserr.IError) { + opt := lsdkVolumeV1.NewGetVolumeTypeByIdRequest(pvolTypeId) + volType, err := s.client.VServerGateway().V1().VolumeService().GetVolumeTypeById(opt) if err != nil { - return true, 0, err.Error - } - - // At this point, we know we are starting a new volume modification - // If we're asked to modify a volume to its current state, ignore the request and immediately return a success - if !needsVolumeModification(volume, newSizeGiB, options) { - // Wait for any existing modifications to prevent race conditions where DescribeVolume(s) returns the new - // state before the volume is actually finished modifying - _, err2 := s.waitVolumeAchieveStatus(volumeID, volumeArchivedStatus) - if err != nil { - return true, int64(volume.Size), err2 - } - - returnGiB, returnErr := s.checkDesiredState(volumeID, newSizeGiB, options) - return false, returnGiB, returnErr + return nil, lserr.NewError(err) } - return true, 0, nil + return &lsentity.VolumeType{VolumeType: volType}, nil } -func (s *cloud) checkDesiredState(volumeID string, desiredSizeGiB uint64, options *ModifyDiskOptions) (int64, error) { - volume, err := s.GetVolume(volumeID) +func (s *cloud) GetDefaultVolumeType() (*lsentity.VolumeType, lserr.IError) { + volType, err := s.client.VServerGateway().V1().VolumeService().GetDefaultVolumeType() if err != nil { - return 0, err.Error - } - - // AWS resizes in chunks of GiB (not GB) - realSizeGiB := int64(volume.Size) - - // Check if there is a mismatch between the requested modification and the current volume - // If there is, the volume is still modifying and we should not return a success - if uint64(realSizeGiB) < desiredSizeGiB { - return realSizeGiB, fmt.Errorf("volume %q is still being expanded to %d size", volumeID, desiredSizeGiB) - } else if options.VolumeType != "" && !strings.EqualFold(volume.VolumeTypeID, options.VolumeType) { - return realSizeGiB, fmt.Errorf("volume %q is still being modified to type %q", volumeID, options.VolumeType) - } - - return realSizeGiB, nil -} - -func needsVolumeModification(volume *lsdkObj.Volume, newSizeGiB uint64, options *ModifyDiskOptions) bool { - oldSizeGiB := volume.Size - needsModification := false - - if oldSizeGiB < newSizeGiB { - needsModification = true - } - - if options.VolumeType != "" && !strings.EqualFold(volume.VolumeTypeID, options.VolumeType) { - needsModification = true + return nil, lserr.NewError(err) } - return needsModification + return &lsentity.VolumeType{VolumeType: volType}, nil } diff --git a/pkg/cloud/cloud_interface.go b/pkg/cloud/cloud_interface.go deleted file mode 100644 index 27f284f..0000000 --- a/pkg/cloud/cloud_interface.go +++ /dev/null @@ -1,24 +0,0 @@ -package cloud - -import ( - lsdkErr "github.com/vngcloud/vngcloud-go-sdk/error" - lsdkObj "github.com/vngcloud/vngcloud-go-sdk/vngcloud/objects" - lsnapshotV2 "github.com/vngcloud/vngcloud-go-sdk/vngcloud/services/blockstorage/v2/snapshot" - pvolv2 "github.com/vngcloud/vngcloud-go-sdk/vngcloud/services/blockstorage/v2/volume" -) - -type Cloud interface { - CreateVolume(popts *pvolv2.CreateOpts) (*lsdkObj.Volume, error) - GetVolumeByName(pvolName string) (*lsdkObj.VolumeList, error) - GetVolume(volumeID string) (*lsdkObj.Volume, *lsdkErr.SdkError) - DeleteVolume(volID string) error - AttachVolume(instanceID, volumeID string) (string, error) - DetachVolume(instanceID, volumeID string) error - ResizeOrModifyDisk(volumeID string, newSizeBytes int64, options *ModifyDiskOptions) (newSize int64, err error) - ExpandVolume(volumeID, volumeTypeID string, newSize uint64) error - GetDeviceDiskID(pvolID string) (string, error) - GetVolumeSnapshotByName(pvolID, psnapshotName string) (*lsdkObj.Snapshot, error) - CreateSnapshotFromVolume(pvolID string, popts *lsnapshotV2.CreateOpts) (*lsdkObj.Snapshot, error) - DeleteSnapshot(psnapshotID string) error - ListSnapshots(pvolID string, ppage int, ppageSize int) (*lsdkObj.SnapshotList, error) -} diff --git a/pkg/cloud/cloud_private.go b/pkg/cloud/cloud_private.go new file mode 100644 index 0000000..bf334d3 --- /dev/null +++ b/pkg/cloud/cloud_private.go @@ -0,0 +1,202 @@ +package cloud + +import ( + lctx "context" + lfmt "fmt" + lstr "strings" + + ljset "github.com/cuongpiger/joat/data-structure/set" + lset "github.com/cuongpiger/joat/data-structure/set" + ljwait "github.com/cuongpiger/joat/utils/exponential-backoff" + lsdkEntity "github.com/vngcloud/vngcloud-go-sdk/v2/vngcloud/entity" + lsdkErrs "github.com/vngcloud/vngcloud-go-sdk/v2/vngcloud/sdk_error" + lsdkVolume "github.com/vngcloud/vngcloud-go-sdk/v2/vngcloud/services/volume/v2" + lerrgroup "golang.org/x/sync/errgroup" + + lsentity "github.com/vngcloud/vngcloud-blockstorage-csi-driver/pkg/cloud/entity" + lserr "github.com/vngcloud/vngcloud-blockstorage-csi-driver/pkg/cloud/errors" +) + +func (s *cloud) getVolumeByName(pvolName string) (*lsdkEntity.Volume, lserr.IError) { + // The volume name is empty + if pvolName == "" { + return nil, new(lsdkErrs.SdkError).WithErrorCode(lsdkErrs.EcVServerVolumeNotFound) + } + + // Get the volume depends on name + opts := lsdkVolume.NewListBlockVolumesRequest(1, 10).WithName(pvolName) + vols, sdkErr := s.client.VServerGateway().V2().VolumeService().ListBlockVolumes(opts) + if sdkErr != nil { + return nil, lserr.NewError(sdkErr) + } + + // Get volume by name with greater than 1 vol + if vols.Len() != 1 { + return nil, new(lsdkErrs.SdkError).WithErrorCode(lsdkErrs.EcVServerVolumeNotFound) + } + + // If all valid, return the first item + return vols.Items[0], nil +} + +func (s *cloud) getVolumeById(pvolId string) (*lsdkEntity.Volume, lserr.IError) { + // Get the volume depends on id + opts := lsdkVolume.NewGetBlockVolumeByIdRequest(pvolId) + vol, sdkErr := s.client.VServerGateway().V2().VolumeService().GetBlockVolumeById(opts) + if sdkErr != nil { + return nil, lserr.NewError(sdkErr) + } + + return vol, nil +} + +func (s *cloud) waitSnapshotActive(pvolID, psnapshotName string) error { + return ljwait.ExponentialBackoff(ljwait.NewBackOff(waitSnapshotActiveSteps, waitSnapshotActiveDelay, true, waitSnapshotActiveTimeout), func() (bool, error) { + vol, err := s.GetVolumeSnapshotByName(pvolID, psnapshotName) + if err != nil { + return false, err + } + + if vol.Status == SnapshotActiveStatus { + return true, nil + } + + return false, nil + }) +} + +func (s *cloud) waitDiskAttached(instanceID string, volumeID string) error { + return ljwait.ExponentialBackoff(ljwait.NewBackOff(waitVolumeAttachSteps, waitVolumeAttachDelay, true, waitVolumeAttachTimeout), func() (bool, error) { + vol, err := s.getVolumeById(volumeID) + if err != nil { + return false, err.GetError() + } + + if vol.Status == VolumeErrorStatus { + return true, lfmt.Errorf("volume %q is in error state", volumeID) + } + + if vol.VmId != nil && *vol.VmId == instanceID && vol.Status == VolumeInUseStatus { + return true, nil + } + + return false, nil + }) +} + +func (s *cloud) validateModifyVolume(volumeID string, newSizeGiB uint64, options *ModifyDiskOptions) (bool, int64, error) { + volume, err := s.GetVolume(volumeID) + if err != nil { + return true, 0, err.GetError() + } + + // At this point, we know we are starting a new volume modification + // If we're asked to modify a volume to its current state, ignore the request and immediately return a success + if !needsVolumeModification(volume, newSizeGiB, options) { + // Wait for any existing modifications to prevent race conditions where DescribeVolume(s) returns the new + // state before the volume is actually finished modifying + _, err2 := s.waitVolumeAchieveStatus(volumeID, volumeArchivedStatus) + if err != nil { + return true, int64(volume.Size), err2 + } + + returnGiB, returnErr := s.checkDesiredState(volumeID, newSizeGiB, options) + return false, returnGiB, returnErr + } + + return true, 0, nil +} + +func (s *cloud) checkDesiredState(volumeID string, desiredSizeGiB uint64, options *ModifyDiskOptions) (int64, error) { + volume, err := s.GetVolume(volumeID) + if err != nil { + return 0, err.GetError() + } + + // AWS resizes in chunks of GiB (not GB) + realSizeGiB := int64(volume.Size) + + // Check if there is a mismatch between the requested modification and the current volume + // If there is, the volume is still modifying and we should not return a success + if uint64(realSizeGiB) < desiredSizeGiB { + return realSizeGiB, lfmt.Errorf("volume %q is still being expanded to %d size", volumeID, desiredSizeGiB) + } else if options.VolumeType != "" && !lstr.EqualFold(volume.VolumeTypeID, options.VolumeType) { + return realSizeGiB, lfmt.Errorf("volume %q is still being modified to type %q", volumeID, options.VolumeType) + } + + return realSizeGiB, nil +} + +func needsVolumeModification(volume *lsentity.Volume, newSizeGiB uint64, options *ModifyDiskOptions) bool { + oldSizeGiB := volume.Size + needsModification := false + + if oldSizeGiB < newSizeGiB { + needsModification = true + } + + if options.VolumeType != "" && !lstr.EqualFold(volume.VolumeTypeID, options.VolumeType) { + needsModification = true + } + + return needsModification +} + +func (s *cloud) waitVolumeAchieveStatus(pvolID string, pdesiredStatus lset.Set[string]) (*lsentity.Volume, error) { + var resVolume *lsentity.Volume + err := ljwait.ExponentialBackoff(ljwait.NewBackOff(waitVolumeActiveSteps, waitVolumeActiveDelay, true, waitVolumeActiveTimeout), func() (bool, error) { + vol, err := s.getVolumeById(pvolID) + if err != nil { + return false, err.GetError() + } + + if pdesiredStatus.ContainsOne(vol.Status) { + resVolume = &lsentity.Volume{Volume: vol} + return true, nil + } + + return false, nil + }) + + if err != nil { + return nil, err + } + + return resVolume, nil +} + +func (s *cloud) checkSameZone(pvolTypeA, pvolTypeB string) (bool, lsdkErrs.ISdkError) { + if pvolTypeA == pvolTypeB { + return true, nil + } + + var ( + zoneSet = ljset.NewSet[string]() + sdkErr lsdkErrs.ISdkError + ) + + group, _ := lerrgroup.WithContext(lctx.TODO()) + for _, volType := range []string{pvolTypeA, pvolTypeB} { + tmpVolType := volType + group.Go(func() error { + vol, err := s.GetVolumeTypeById(tmpVolType) + if err != nil { + sdkErr = err + return err.GetError() + } + + zoneSet.Add(vol.ZoneId) + return nil + }) + } + + if err := group.Wait(); err != nil { + return false, sdkErr + } + + if zoneSet.IsEmpty() || zoneSet.Cardinality() != 1 { + return false, nil + } + + return true, nil +} diff --git a/pkg/cloud/consts.go b/pkg/cloud/consts.go index 428c48a..46fbbbb 100644 --- a/pkg/cloud/consts.go +++ b/pkg/cloud/consts.go @@ -1,11 +1,12 @@ package cloud import ( + ltime "time" + lset "github.com/cuongpiger/joat/data-structure/set" - "github.com/vngcloud/vngcloud-blockstorage-csi-driver/pkg/util" - lsdkErr "github.com/vngcloud/vngcloud-go-sdk/error" - lsdkEH "github.com/vngcloud/vngcloud-go-sdk/vngcloud/errors" - "time" + lsdkErrs "github.com/vngcloud/vngcloud-go-sdk/v2/vngcloud/sdk_error" + + lsutil "github.com/vngcloud/vngcloud-blockstorage-csi-driver/pkg/util" ) const ( @@ -17,29 +18,20 @@ const ( ) const ( - DefaultDiskSymbolIdLength = 20 - // DefaultVolumeSize represents the default volume size. - DefaultVolumeSize int64 = 20 * util.GiB - - defaultPage = 1 - defaultPageSize = 100 + DefaultVolumeSize int64 = 5 * lsutil.GiB ) const ( - waitVolumeActiveTimeout = 5 * time.Minute + waitVolumeActiveTimeout = 5 * ltime.Minute waitVolumeActiveDelay = 10 waitVolumeActiveSteps = 5 - waitVolumeDetachTimeout = 7 * time.Minute - waitVolumeDetachDelay = 10 - waitVolumeDetachSteps = 5 - - waitVolumeAttachTimeout = 7 * time.Minute + waitVolumeAttachTimeout = 7 * ltime.Minute waitVolumeAttachDelay = 10 waitVolumeAttachSteps = 5 - waitSnapshotActiveTimeout = 5 * time.Minute + waitSnapshotActiveTimeout = 5 * ltime.Minute waitSnapshotActiveDelay = 10 waitSnapshotActiveSteps = 5 ) @@ -48,15 +40,20 @@ const ( VolumeAvailableStatus = "AVAILABLE" VolumeInUseStatus = "IN-USE" VolumeCreatingStatus = "CREATING" + VolumeErrorStatus = "ERROR" SnapshotActiveStatus = "ACTIVE" ) var ( - errSetDetachIngore = lset.NewSet[lsdkErr.ErrorCode](lsdkEH.ErrCodeVolumeAvailable, lsdkEH.ErrCodeVolumeNotFound) + errSetDetachIngore = lset.NewSet[lsdkErrs.ErrorCode](lsdkErrs.EcVServerVolumeNotFound, lsdkErrs.EcVServerVolumeAvailable) ) var ( volumeArchivedStatus = lset.NewSet[string](VolumeAvailableStatus, VolumeInUseStatus) volumeAvailableStatus = lset.NewSet[string](VolumeAvailableStatus) ) + +const ( + patternSnapshotDescription = "Snapshot of PersistentVolume %s for vKS cluster %s" +) diff --git a/pkg/cloud/entity/snapshot.go b/pkg/cloud/entity/snapshot.go new file mode 100644 index 0000000..3f1e99e --- /dev/null +++ b/pkg/cloud/entity/snapshot.go @@ -0,0 +1,15 @@ +package entity + +import lsdkEntity "github.com/vngcloud/vngcloud-go-sdk/v2/vngcloud/entity" + +type Snapshot struct { + *lsdkEntity.Snapshot +} + +type ListSnapshots struct { + *lsdkEntity.ListSnapshots +} + +func (s *ListSnapshots) Len() int { + return len(s.Items) +} diff --git a/pkg/cloud/entity/volume.go b/pkg/cloud/entity/volume.go new file mode 100644 index 0000000..a167438 --- /dev/null +++ b/pkg/cloud/entity/volume.go @@ -0,0 +1,69 @@ +package entity + +import lsdkEntity "github.com/vngcloud/vngcloud-go-sdk/v2/vngcloud/entity" + +const ( + VolumeInUseStatus = "IN-USE" +) + +type Volume struct { + *lsdkEntity.Volume +} + +func (s *Volume) IsError() bool { + return s.Status == "ERROR" +} + +func (s *Volume) IsMigration() bool { + return s.Status == "MIGRATING" +} + +func (s *Volume) IsCreating() bool { + return s.Status == "CREATING" || s.Status == "CREATING-BILLING" +} + +func (s *Volume) AttachedTheInstance(pinstanceId string) bool { + if s.VmId != nil && *s.VmId == pinstanceId { + return true + } + + for _, machineId := range s.AttachedMachine { + if machineId == pinstanceId { + return true + } + } + + return false +} + +func (s *Volume) CanDelete() bool { + if len(s.AttachedMachine) < 1 { + return true + } + + if s.VmId == nil { + return true + } + + if *s.VmId == "" { + return true + } + + return false +} + +func (s *Volume) IsAttched() bool { + if s.Status == VolumeInUseStatus { + return true + } + + if len(s.AttachedMachine) > 0 { + return true + } + + if s.VmId != nil && *s.VmId != "" { + return true + } + + return false +} diff --git a/pkg/cloud/entity/volumetype.go b/pkg/cloud/entity/volumetype.go new file mode 100644 index 0000000..723686f --- /dev/null +++ b/pkg/cloud/entity/volumetype.go @@ -0,0 +1,7 @@ +package entity + +import lsdkEntity "github.com/vngcloud/vngcloud-go-sdk/v2/vngcloud/entity" + +type VolumeType struct { + *lsdkEntity.VolumeType +} diff --git a/pkg/cloud/errors/errors.go b/pkg/cloud/errors/errors.go new file mode 100644 index 0000000..2e9c28e --- /dev/null +++ b/pkg/cloud/errors/errors.go @@ -0,0 +1,11 @@ +package errors + +import lsdkErrs "github.com/vngcloud/vngcloud-go-sdk/v2/vngcloud/sdk_error" + +type errors struct { + lsdkErrs.ISdkError +} + +func NewError(psdkErr lsdkErrs.ISdkError) IError { + return &errors{psdkErr} +} diff --git a/pkg/cloud/errors/ierrors.go b/pkg/cloud/errors/ierrors.go new file mode 100644 index 0000000..7fd40eb --- /dev/null +++ b/pkg/cloud/errors/ierrors.go @@ -0,0 +1,7 @@ +package errors + +import lsdkErrs "github.com/vngcloud/vngcloud-go-sdk/v2/vngcloud/sdk_error" + +type IError interface { + lsdkErrs.ISdkError +} diff --git a/pkg/cloud/icloud.go b/pkg/cloud/icloud.go new file mode 100644 index 0000000..3730829 --- /dev/null +++ b/pkg/cloud/icloud.go @@ -0,0 +1,26 @@ +package cloud + +import ( + lsdkVolumeV2 "github.com/vngcloud/vngcloud-go-sdk/v2/vngcloud/services/volume/v2" + + lsentity "github.com/vngcloud/vngcloud-blockstorage-csi-driver/pkg/cloud/entity" + lserr "github.com/vngcloud/vngcloud-blockstorage-csi-driver/pkg/cloud/errors" +) + +type Cloud interface { + EitherCreateResizeVolume(preq lsdkVolumeV2.ICreateBlockVolumeRequest) (*lsentity.Volume, lserr.IError) + GetVolumeByName(pvolName string) (*lsentity.Volume, lserr.IError) + GetVolume(volumeID string) (*lsentity.Volume, lserr.IError) + DeleteVolume(volID string) error + AttachVolume(instanceID, volumeID string) (*lsentity.Volume, error) + DetachVolume(instanceID, volumeID string) error + ResizeOrModifyDisk(volumeID string, newSizeBytes int64, options *ModifyDiskOptions) (newSize int64, err error) + ExpandVolume(volumeID, volumeTypeID string, newSize uint64) error + GetDeviceDiskID(pvolID string) (string, error) + GetVolumeSnapshotByName(pvolID, psnapshotName string) (*lsentity.Snapshot, error) + CreateSnapshotFromVolume(pclusterId, pvolId, psnapshotName string) (*lsentity.Snapshot, error) + DeleteSnapshot(psnapshotID string) error + ListSnapshots(pvolID string, ppage int, ppageSize int) (*lsentity.ListSnapshots, lserr.IError) + GetVolumeTypeById(pvolTypeId string) (*lsentity.VolumeType, lserr.IError) + GetDefaultVolumeType() (*lsentity.VolumeType, lserr.IError) +} diff --git a/pkg/driver/constants.go b/pkg/driver/constants.go index a7de62f..05bbd25 100644 --- a/pkg/driver/constants.go +++ b/pkg/driver/constants.go @@ -1,17 +1,18 @@ package driver import ( - lscloud "github.com/vngcloud/vngcloud-blockstorage-csi-driver/pkg/cloud" - "k8s.io/apimachinery/pkg/util/wait" ltime "time" lcsi "github.com/container-storage-interface/spec/lib/go/csi" + lwait "k8s.io/apimachinery/pkg/util/wait" + + lscloud "github.com/vngcloud/vngcloud-blockstorage-csi-driver/pkg/cloud" ) const ( DefaultCSIEndpoint = "unix://tmp/csi.sock" DefaultModifyVolumeRequestHandlerTimeout = 30 * ltime.Second - AgentNotReadyNodeTaintKey = "bs.csi.vngcloud.vn/agent-not-ready" + AgentNotReadyNodeTaintKey = "csi.vngcloud.vn/agent-not-ready" DefaultTimeoutModifyChannel = 10 * ltime.Minute @@ -19,12 +20,6 @@ const ( ZoneTopologyKey = "topology." + DriverName + "/zone" ) -const ( - // VolumeOperationAlreadyExists is message fmt returned to CO when there is another in-flight call on the given volumeID - volumeOperationAlreadyExists = "An operation with the given volume=%q is already in progress" - patternSnapshotDescription = "Snapshot of PersistentVolume %s for vKS cluster %s" -) - // constants of disk partition suffix const ( diskPartitionSuffix = "" @@ -166,7 +161,7 @@ var ( taintRemovalInitialDelay = 1 * ltime.Second // taintRemovalBackoff is the exponential backoff configuration for node taint removal - taintRemovalBackoff = wait.Backoff{ + taintRemovalBackoff = lwait.Backoff{ Duration: 500 * ltime.Millisecond, Factor: 2, Steps: 10, // Max delay = 0.5 * 2^9 = ~4 minutes diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go index 4964de8..28dac76 100644 --- a/pkg/driver/controller.go +++ b/pkg/driver/controller.go @@ -3,7 +3,6 @@ package driver import ( lctx "context" lerr "errors" - lfmt "fmt" lstrconv "strconv" lstr "strings" ltime "time" @@ -11,13 +10,15 @@ import ( lcsi "github.com/container-storage-interface/spec/lib/go/csi" ljoat "github.com/cuongpiger/joat/parser" lvmrpc "github.com/vngcloud/vngcloud-csi-volume-modifier/pkg/rpc" - lsdkEH "github.com/vngcloud/vngcloud-go-sdk/vngcloud/errors" - lsdkObj "github.com/vngcloud/vngcloud-go-sdk/vngcloud/objects" - lsdkSnapshotV2 "github.com/vngcloud/vngcloud-go-sdk/vngcloud/services/blockstorage/v2/snapshot" + lsdkEntity "github.com/vngcloud/vngcloud-go-sdk/v2/vngcloud/entity" + lsdkErrs "github.com/vngcloud/vngcloud-go-sdk/v2/vngcloud/sdk_error" lts "google.golang.org/protobuf/types/known/timestamppb" + lk8s "k8s.io/client-go/kubernetes" llog "k8s.io/klog/v2" lscloud "github.com/vngcloud/vngcloud-blockstorage-csi-driver/pkg/cloud" + lsentity "github.com/vngcloud/vngcloud-blockstorage-csi-driver/pkg/cloud/entity" + lserr "github.com/vngcloud/vngcloud-blockstorage-csi-driver/pkg/cloud/errors" lsinternal "github.com/vngcloud/vngcloud-blockstorage-csi-driver/pkg/driver/internal" lsutil "github.com/vngcloud/vngcloud-blockstorage-csi-driver/pkg/util" ) @@ -27,72 +28,76 @@ type controllerService struct { inFlight *lsinternal.InFlight modifyVolumeManager *modifyVolumeManager driverOptions *DriverOptions + k8sClient lk8s.Interface lvmrpc.UnimplementedModifyServer } // newControllerService creates a new controller service it panics if failed to create the service -func newControllerService(driverOptions *DriverOptions) controllerService { +func newControllerService(pdriOpts *DriverOptions) controllerService { metadata, err := NewMetadataFunc(lscloud.DefaultVServerMetadataClient) if err != nil { llog.ErrorS(err, "Could not determine the metadata information for the driver") panic(err) } - cloudSrv, err := NewCloudFunc(driverOptions.identityURL, driverOptions.vServerURL, driverOptions.clientID, driverOptions.clientSecret, metadata) + cloudSrv, err := NewCloudFunc(pdriOpts.identityURL, pdriOpts.vServerURL, pdriOpts.clientID, pdriOpts.clientSecret, metadata) if err != nil { panic(err) } + k8sClient, _ := lscloud.DefaultKubernetesAPIClient() return controllerService{ cloud: cloudSrv, inFlight: lsinternal.NewInFlight(), - driverOptions: driverOptions, + driverOptions: pdriOpts, modifyVolumeManager: newModifyVolumeManager(), + k8sClient: k8sClient, } } func (s *controllerService) CreateVolume(_ lctx.Context, preq *lcsi.CreateVolumeRequest) (*lcsi.CreateVolumeResponse, error) { - llog.V(5).InfoS("CreateVolume: called", "preq", *preq) + var ( + serr lserr.IError + ) + + llog.V(5).InfoS("[INFO] - CreateVolume: called", "preq", *preq) // Validate the create volume request if err := validateCreateVolumeRequest(preq); err != nil { - llog.ErrorS(err, "CreateVolume: invalid request") + llog.ErrorS(err, "[ERROR] - CreateVolume: invalid request") return nil, err } // Validate volume size, if volume size is less than the default volume size of cloud provider, set it to the default volume size - volSizeBytes := getVolSizeBytes(preq) + volSizeBytes, err := s.getVolSizeBytes(preq) + if err != nil { + llog.ErrorS(err, "[ERROR] - CreateVolume: failed to get volume size") + return nil, ErrFailedToValidateVolumeSize(preq.GetName(), err) + } + volName := preq.GetName() // get the name of the volume, always in the format of pvc- volCap := preq.GetVolumeCapabilities() // get volume capabilities multiAttach := isMultiAttach(volCap) // check if the volume is multi-attach, true if multi-attach, false otherwise // check if a request is already in-flight if ok := s.inFlight.Insert(volName); !ok { - llog.V(5).InfoS("CreateVolume: volume is already in-flight", "volumeName", volName) + llog.V(5).InfoS("[INFO] - CreateVolume: Volume is already in-flight", "volumeName", volName) return nil, ErrVolumeIsCreating(volName) } defer s.inFlight.Delete(volName) - vl, err := s.cloud.GetVolumeByName(volName) - if err != nil { - llog.ErrorS(err, "CreateVolume: failed to get volume", "volumeName", volName) - return nil, ErrFailedToListVolumeByName(volName) - } - - if vl != nil && len(vl.Items) == 1 { - if vl.Items[0].Status != lscloud.VolumeAvailableStatus { - llog.V(5).Infof("CreateVolume: volume %s is already in progress", volName) - return nil, ErrVolumeIsCreating(volName) + if _, serr = s.cloud.GetVolumeByName(volName); serr != nil { + if !serr.IsError(lsdkErrs.EcVServerVolumeNotFound) { + llog.ErrorS(serr.GetError(), "[ERROR] - CreateVolume: failed to get volume", "volumeName", volName) + return nil, ErrFailedToListVolumeByName(volName) } - } else if vl != nil && len(vl.Items) > 1 { - llog.Errorf("Multiple volumes found with the same name %s", volName) - return nil, ErrFailedToListVolumeByName(volName) } - cvr := NewCreateVolumeRequest() + cvr := NewCreateVolumeRequest().WithDriverOptions(s.driverOptions) parser, _ := ljoat.GetParser() for pk, pv := range preq.GetParameters() { + llog.InfoS("[INFO] - CreateVolume: parsing request parameters", "key", pk, "value", pv) switch lstr.ToLower(pk) { case VolumeTypeKey: cvr = cvr.WithVolumeTypeID(pv) @@ -138,7 +143,7 @@ func (s *controllerService) CreateVolume(_ lctx.Context, preq *lcsi.CreateVolume modifyOpts, err := parseModifyVolumeParameters(preq.GetMutableParameters()) if err != nil { - llog.ErrorS(err, "CreateVolume: invalid request") + llog.ErrorS(err, "[ERROR] - CreateVolume: invalid request") return nil, ErrModifyMutableParam } @@ -156,7 +161,7 @@ func (s *controllerService) CreateVolume(_ lctx.Context, preq *lcsi.CreateVolume respCtx, err := cvr.ToResponseContext(volCap) if err != nil { - llog.ErrorS(err, "CreateVolume: failed to parse response context", "volumeID", volName) + llog.ErrorS(err, "[ERROR] - CreateVolume: failed to parse response context", "volumeID", volName) return nil, err } @@ -166,18 +171,19 @@ func (s *controllerService) CreateVolume(_ lctx.Context, preq *lcsi.CreateVolume WithVolumeTypeID(modifyOpts.VolumeType). WithClusterID(s.getClusterID()) - resp, err := s.cloud.CreateVolume(cvr.ToSdkCreateVolumeOpts(s.driverOptions)) - if err != nil { - llog.ErrorS(err, "CreateVolume: failed to create volume", "volumeID", volName) - return nil, err + resp, sdkErr := s.cloud.EitherCreateResizeVolume(cvr.ToSdkCreateVolumeRequest()) + if sdkErr != nil { + llog.ErrorS(sdkErr.GetError(), "[ERROR] - CreateVolume: failed to create volume", sdkErr.GetErrorMessages()) + return nil, sdkErr.GetError() } return newCreateVolumeResponse(resp, cvr, respCtx), nil } func (s *controllerService) DeleteVolume(pctx lctx.Context, preq *lcsi.DeleteVolumeRequest) (*lcsi.DeleteVolumeResponse, error) { - llog.V(4).InfoS("DeleteVolume: called", "args", *preq) + llog.V(4).InfoS("[INFO] - DeleteVolume: called", "args", *preq) if err := validateDeleteVolumeRequest(preq); err != nil { + llog.Errorf("[ERROR] - DeleteVolume: invalid request") return nil, err } @@ -188,9 +194,21 @@ func (s *controllerService) DeleteVolume(pctx lctx.Context, preq *lcsi.DeleteVol } defer s.inFlight.Delete(volumeID) + // So the volume MUST NOT truly be deleted if it has at least one snapshot + lstSnapshots, ierr := s.cloud.ListSnapshots(volumeID, 1, 10) + if ierr != nil { + llog.ErrorS(ierr.GetError(), "[ERROR] - DeleteVolume: failed to list snapshots", "volumeID", volumeID) + return nil, ErrFailedToListSnapshot(volumeID) + } + + if lstSnapshots.Len() > 0 { + llog.ErrorS(nil, "[ERROR] - DeleteVolume: CANNOT delete this volume because of having snapshots", "volumeId", volumeID) + return nil, ErrDeleteVolumeHavingSnapshots(volumeID) + } + if err := s.cloud.DeleteVolume(volumeID); err != nil { if err != nil { - llog.ErrorS(err, "DeleteVolume: failed to delete volume", "volumeID", volumeID) + llog.ErrorS(err, "[ERROR] - DeleteVolume: failed to delete volume", "volumeID", volumeID) return nil, ErrFailedToDeleteVolume(volumeID) } } @@ -199,10 +217,10 @@ func (s *controllerService) DeleteVolume(pctx lctx.Context, preq *lcsi.DeleteVol } func (s *controllerService) ControllerPublishVolume(pctx lctx.Context, preq *lcsi.ControllerPublishVolumeRequest) (result *lcsi.ControllerPublishVolumeResponse, err error) { - llog.V(5).InfoS("ControllerPublishVolume: called", "preq", *preq) + llog.V(5).InfoS("[INFO] - ControllerPublishVolume: called with request", "preq", *preq) if err = validateControllerPublishVolumeRequest(preq); err != nil { - llog.ErrorS(err, "ControllerPublishVolume: invalid request") + llog.ErrorS(err, "[ERROR] - ControllerPublishVolume: invalid request") return nil, err } @@ -215,30 +233,30 @@ func (s *controllerService) ControllerPublishVolume(pctx lctx.Context, preq *lcs } defer s.inFlight.Delete(volumeID + nodeID) - llog.V(2).InfoS("ControllerPublishVolume: attaching volume into the instance", "volumeID", volumeID, "nodeID", nodeID) + llog.V(2).InfoS("[INFO] - ControllerPublishVolume: attaching volume into the instance", "volumeID", volumeID, "nodeID", nodeID) // Attach the volume and wait for it to be attached _, err = s.cloud.AttachVolume(nodeID, volumeID) if err != nil { - llog.ErrorS(err, "ControllerPublishVolume; failed to attach volume to instance", "volumeID", volumeID, "nodeID", nodeID) + llog.ErrorS(err, "[ERROR] - ControllerPublishVolume; failed to attach volume to instance", "volumeID", volumeID, "nodeID", nodeID) return nil, ErrAttachVolume(volumeID, nodeID) } devicePath, err := s.cloud.GetDeviceDiskID(volumeID) if err != nil { - llog.ErrorS(err, "ControllerPublishVolume; failed to get device path for volume", "volumeID", volumeID) + llog.ErrorS(err, "[ERROR] - ControllerPublishVolume; failed to get device path for volume", "volumeID", volumeID) return nil, ErrFailedToGetDevicePath(volumeID, nodeID) } - llog.V(5).InfoS("ControllerPublishVolume; volume attached to instance successfully", "volumeID", volumeID, "nodeID", nodeID) + llog.V(5).InfoS("[INFO] - ControllerPublishVolume; volume attached to instance successfully", "volumeID", volumeID, "nodeID", nodeID) return newControllerPublishVolumeResponse(devicePath), nil } func (s *controllerService) ControllerUnpublishVolume(_ lctx.Context, preq *lcsi.ControllerUnpublishVolumeRequest) (*lcsi.ControllerUnpublishVolumeResponse, error) { - llog.V(4).InfoS("ControllerUnpublishVolume: called", "preq", *preq) + llog.V(4).InfoS("[INFO] - ControllerUnpublishVolume: called", "preq", *preq) if err := validateControllerUnpublishVolumeRequest(preq); err != nil { - llog.ErrorS(err, "ControllerUnpublishVolume: invalid request") + llog.ErrorS(err, "[ERROR] - ControllerUnpublishVolume: invalid request") return nil, err } @@ -251,28 +269,33 @@ func (s *controllerService) ControllerUnpublishVolume(_ lctx.Context, preq *lcsi defer s.inFlight.Delete(volumeID + nodeID) if volumeID == "" { - llog.Errorf("ControllerUnpublishVolume: VolumeID is required") + llog.Errorf("[ERROR] - ControllerUnpublishVolume: VolumeID is required") return nil, ErrVolumeIDNotProvided } - _, getErr := s.cloud.GetVolume(volumeID) - if getErr != nil && getErr.Code == lsdkEH.ErrCodeVolumeNotFound { - llog.InfoS("ControllerUnpublishVolume: volume not found", "volumeID", volumeID) + vol, getErr := s.cloud.GetVolume(volumeID) + if getErr != nil && getErr.IsError(lsdkErrs.EcVServerVolumeNotFound) { + llog.InfoS("[INFO] - ControllerUnpublishVolume: volume not found", "volumeID", volumeID) + return &lcsi.ControllerUnpublishVolumeResponse{}, nil + } + + if !vol.AttachedTheInstance(nodeID) { + llog.InfoS("[INFO] - ControllerUnpublishVolume: server does not attach this volume", "volumeID", volumeID, "serverId", nodeID) return &lcsi.ControllerUnpublishVolumeResponse{}, nil } err := s.cloud.DetachVolume(nodeID, volumeID) if err != nil { - llog.ErrorS(err, "ControllerUnpublishVolume: failed to detach volume from instance", "volumeID", volumeID, "nodeID", nodeID) + llog.ErrorS(err, "[ERROR] - ControllerUnpublishVolume: failed to detach volume from instance", "volumeID", volumeID, "nodeID", nodeID) return nil, ErrDetachVolume(volumeID, nodeID) } - llog.V(4).InfoS("ControllerUnpublishVolume: volume detached from instance successfully", "volumeID", volumeID, "nodeID", nodeID) + llog.V(4).InfoS("[INFO] - ControllerUnpublishVolume: volume detached from instance successfully", "volumeID", volumeID, "nodeID", nodeID) return &lcsi.ControllerUnpublishVolumeResponse{}, nil } func (s *controllerService) CreateSnapshot(_ lctx.Context, preq *lcsi.CreateSnapshotRequest) (*lcsi.CreateSnapshotResponse, error) { - llog.V(4).InfoS("CreateSnapshot: called", "preq", *preq) + llog.V(4).InfoS("[INFO] - CreateSnapshot: called", "preq", *preq) if err := validateCreateSnapshotRequest(preq); err != nil { llog.ErrorS(err, "CreateSnapshot: invalid request") return nil, err @@ -299,14 +322,7 @@ func (s *controllerService) CreateSnapshot(_ lctx.Context, preq *lcsi.CreateSnap return newCreateSnapshotResponse(snapshot) } - snapshot, err = s.cloud.CreateSnapshotFromVolume(volumeID, - &lsdkSnapshotV2.CreateOpts{ - Name: snapshotName, - Description: lfmt.Sprintf(patternSnapshotDescription, volumeID, s.getClusterID()), - Permanently: true, - }, - ) - + snapshot, err = s.cloud.CreateSnapshotFromVolume(s.getClusterID(), volumeID, snapshotName) if err != nil { llog.ErrorS(err, "CreateSnapshot: Error creating snapshot", "snapshotName", snapshotName, "volumeID", volumeID) return nil, err @@ -353,9 +369,9 @@ func (s *controllerService) ListSnapshots(_ lctx.Context, preq *lcsi.ListSnapsho nextToken := parsePage(preq.GetStartingToken()) maxEntries := int(preq.GetMaxEntries()) - cloudSnapshots, err := s.cloud.ListSnapshots(volumeID, nextToken, maxEntries) - if err != nil { - llog.ErrorS(err, "ListSnapshots: Error listing snapshots", "volumeID", volumeID, "nextToken", nextToken, "maxEntries", maxEntries) + cloudSnapshots, ierr := s.cloud.ListSnapshots(volumeID, nextToken, maxEntries) + if ierr != nil { + llog.ErrorS(ierr.GetError(), "ListSnapshots: Error listing snapshots", "volumeID", volumeID, "nextToken", nextToken, "maxEntries", maxEntries) return nil, ErrFailedToListSnapshot(volumeID) } @@ -377,7 +393,7 @@ func (s *controllerService) ValidateVolumeCapabilities(pctx lctx.Context, preq * } if _, err := s.cloud.GetVolume(volumeID); err != nil { - if err.Code == lsdkEH.ErrCodeVolumeNotFound { + if err.IsError(lsdkErrs.EcVServerVolumeNotFound) { return nil, ErrVolumeNotFound(volumeID) } @@ -440,7 +456,7 @@ func (s *controllerService) ControllerExpandVolume(_ lctx.Context, preq *lcsi.Co volume, err := s.cloud.GetVolume(volumeID) if err != nil { - llog.ErrorS(err.Error, "ControllerExpandVolume: failed to get volume", "volumeID", volumeID) + llog.ErrorS(err.GetError(), "ControllerExpandVolume: failed to get volume", "volumeID", volumeID) return nil, ErrFailedToGetVolume(volumeID) } @@ -513,7 +529,7 @@ func (s *controllerService) ModifyVolumeProperties(pctx lctx.Context, preq *lvmr volume, errSdk := s.cloud.GetVolume(volumeID) if errSdk != nil { - llog.ErrorS(errSdk.Error, "ModifyVolumeProperties: failed to get volume", "volumeID", volumeID) + llog.ErrorS(errSdk.GetError(), "ModifyVolumeProperties: failed to get volume", "volumeID", volumeID) return nil, ErrFailedToGetVolume(volumeID) } @@ -553,7 +569,40 @@ func (s *controllerService) getClusterID() string { return s.driverOptions.clusterID } -func newCreateVolumeResponse(disk *lsdkObj.Volume, pcvr *CreateVolumeRequest, prespCtx map[string]string) *lcsi.CreateVolumeResponse { +func (s *controllerService) getVolSizeBytes(preq *lcsi.CreateVolumeRequest) (volSizeBytes int64, err error) { + // get the volume size that user provided + if preq.GetCapacityRange() != nil { + volSizeBytes = preq.GetCapacityRange().GetRequiredBytes() + } + + // Get the volume type that user specified in the StorageClass + volType, ok := preq.GetParameters()[VolumeTypeKey] + if !ok { + // If the user forget to specify the volume type, get the default volume type + tmpVolType, sdkErr := s.cloud.GetDefaultVolumeType() + if sdkErr != nil { + return 0, sdkErr.GetError() + } + + volType = tmpVolType.Id + } + + // Get the minimum volume size allowed by the volume type + volTypeEntity, sdkErr := s.cloud.GetVolumeTypeById(volType) + if sdkErr != nil { + return 0, sdkErr.GetError() + } + + // Calculate the bytes that cloud provider allowing to create the volume + cvs := lsutil.GiBToBytes(int64(volTypeEntity.MinSize)) + if volSizeBytes < cvs { + return 0, ErrVolumeSizeTooSmall(preq.GetName(), volSizeBytes) + } + + return volSizeBytes, nil +} + +func newCreateVolumeResponse(disk *lsentity.Volume, pcvr *CreateVolumeRequest, prespCtx map[string]string) *lcsi.CreateVolumeResponse { var vcs *lcsi.VolumeContentSource if pcvr.SnapshotID != "" { vcs = &lcsi.VolumeContentSource{ @@ -567,7 +616,7 @@ func newCreateVolumeResponse(disk *lsdkObj.Volume, pcvr *CreateVolumeRequest, pr return &lcsi.CreateVolumeResponse{ Volume: &lcsi.Volume{ - VolumeId: disk.VolumeId, + VolumeId: disk.Id, CapacityBytes: int64(disk.Size * 1024 * 1024 * 1024), VolumeContext: prespCtx, ContentSource: vcs, @@ -575,7 +624,7 @@ func newCreateVolumeResponse(disk *lsdkObj.Volume, pcvr *CreateVolumeRequest, pr } } -func newCreateSnapshotResponse(snapshot *lsdkObj.Snapshot) (*lcsi.CreateSnapshotResponse, error) { +func newCreateSnapshotResponse(snapshot *lsentity.Snapshot) (*lcsi.CreateSnapshotResponse, error) { creationTime, err := ltime.Parse("2006-01-02T15:04:05.000-07:00", snapshot.CreatedAt) if err != nil { creationTime = ltime.Now() @@ -583,8 +632,8 @@ func newCreateSnapshotResponse(snapshot *lsdkObj.Snapshot) (*lcsi.CreateSnapshot return &lcsi.CreateSnapshotResponse{ Snapshot: &lcsi.Snapshot{ - SnapshotId: snapshot.ID, - SourceVolumeId: snapshot.VolumeID, + SnapshotId: snapshot.Id, + SourceVolumeId: snapshot.VolumeId, SizeBytes: snapshot.VolumeSize * lsutil.GiB, CreationTime: lts.New(creationTime), ReadyToUse: true, @@ -592,10 +641,10 @@ func newCreateSnapshotResponse(snapshot *lsdkObj.Snapshot) (*lcsi.CreateSnapshot }, nil } -func newListSnapshotsResponse(psnapshotList *lsdkObj.SnapshotList) *lcsi.ListSnapshotsResponse { +func newListSnapshotsResponse(psnapshotList *lsentity.ListSnapshots) *lcsi.ListSnapshotsResponse { var entries []*lcsi.ListSnapshotsResponse_Entry for _, snapshot := range psnapshotList.Items { - snapshotResponseEntry := newListSnapshotsResponseEntry(&snapshot) + snapshotResponseEntry := newListSnapshotsResponseEntry(snapshot) entries = append(entries, snapshotResponseEntry) } @@ -627,7 +676,7 @@ func newGetSnapshotsResponse(psnapshotID string) *lcsi.ListSnapshotsResponse { } } -func newListSnapshotsResponseEntry(snapshot *lsdkObj.Snapshot) *lcsi.ListSnapshotsResponse_Entry { +func newListSnapshotsResponseEntry(snapshot *lsdkEntity.Snapshot) *lcsi.ListSnapshotsResponse_Entry { creationTime, err := ltime.Parse("2006-01-02T15:04:05.000-07:00", snapshot.CreatedAt) if err != nil { creationTime = ltime.Now() @@ -635,8 +684,8 @@ func newListSnapshotsResponseEntry(snapshot *lsdkObj.Snapshot) *lcsi.ListSnapsho return &lcsi.ListSnapshotsResponse_Entry{ Snapshot: &lcsi.Snapshot{ - SnapshotId: snapshot.ID, - SourceVolumeId: snapshot.VolumeID, + SnapshotId: snapshot.Id, + SourceVolumeId: snapshot.VolumeId, SizeBytes: snapshot.Size * lsutil.GiB, CreationTime: lts.New(creationTime), ReadyToUse: snapshot.Status == lscloud.SnapshotActiveStatus, diff --git a/pkg/driver/driver.go b/pkg/driver/driver.go index 6424aec..ecc30e8 100644 --- a/pkg/driver/driver.go +++ b/pkg/driver/driver.go @@ -4,13 +4,14 @@ import ( "context" "fmt" csi "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/vngcloud/vngcloud-blockstorage-csi-driver/pkg/util" "github.com/vngcloud/vngcloud-csi-volume-modifier/pkg/rpc" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" "k8s.io/klog/v2" "net" "time" + + "github.com/vngcloud/vngcloud-blockstorage-csi-driver/pkg/util" ) // Mode is the operating mode of the CSI driver. @@ -45,6 +46,9 @@ type DriverOptions struct { // nolint: maligned vServerURL string batching bool clusterID string + cacheUri string + alertChannel string + alertChannelSize int tagKeyLength int tagValueLength int } @@ -97,6 +101,24 @@ func WithMode(mode Mode) func(*DriverOptions) { } } +func WithAlertChannel(alertChannel string) func(*DriverOptions) { + return func(o *DriverOptions) { + o.alertChannel = alertChannel + } +} + +func WithAlertChannelSize(alertChannelSize int) func(*DriverOptions) { + return func(o *DriverOptions) { + o.alertChannelSize = alertChannelSize + } +} + +func WithCacheUri(cacheUri string) func(*DriverOptions) { + return func(o *DriverOptions) { + o.cacheUri = cacheUri + } +} + func WithOtelTracing(enableOtelTracing bool) func(*DriverOptions) { return func(o *DriverOptions) { o.otelTracing = enableOtelTracing diff --git a/pkg/driver/errors.go b/pkg/driver/errors.go index 72de27c..17b67c2 100644 --- a/pkg/driver/errors.go +++ b/pkg/driver/errors.go @@ -86,6 +86,10 @@ var ( return lstt.Errorf(lcodes.Internal, "CANNOT list snapshot for volume %s", pvolumeID) } + ErrDeleteVolumeHavingSnapshots = func(pvolId string) error { + return lstt.Errorf(lcodes.FailedPrecondition, "Volume %s can not deleted", pvolId) + } + ErrFailedToGetVolume = func(pvolumeID string) error { return lstt.Errorf(lcodes.Internal, "CANNOT get volume %s", pvolumeID) } @@ -106,6 +110,14 @@ var ( return lstt.Errorf(lcodes.Internal, "CANNOT list volume by name %s", pvolName) } + ErrFailedToValidateVolumeSize = func(pvolumeID string, err error) error { + return lstt.Errorf(lcodes.Internal, "CANNOT validate volume size for volume %s: %w", pvolumeID, err) + } + + ErrVolumeSizeTooSmall = func(pvolumeID string, psize int64) error { + return lstt.Errorf(lcodes.InvalidArgument, "Volume size %d is too small for volume %s", psize, pvolumeID) + } + ErrFailedToFindTargetPath = func(pdevicePath string, perr error) error { return lstt.Errorf(lcodes.Internal, "Failed to find device path %s. %v", pdevicePath, perr) } diff --git a/pkg/driver/internal/inflight.go b/pkg/driver/internal/inflight.go index f35d700..2286aa9 100644 --- a/pkg/driver/internal/inflight.go +++ b/pkg/driver/internal/inflight.go @@ -5,10 +5,6 @@ import ( "sync" ) -const ( - VolumeOperationAlreadyExistsErrorMsg = "An operation with the given Volume %s already exists" -) - // InFlight is a struct used to manage in flight requests for a unique identifier. type InFlight struct { mux *sync.Mutex diff --git a/pkg/driver/node.go b/pkg/driver/node.go index 4e550aa..02a03a0 100644 --- a/pkg/driver/node.go +++ b/pkg/driver/node.go @@ -4,13 +4,10 @@ import ( lctx "context" "encoding/json" "fmt" - "os" - "path/filepath" - "strconv" - "strings" - "time" - lcsi "github.com/container-storage-interface/spec/lib/go/csi" + lsdkClientV2 "github.com/vngcloud/vngcloud-go-sdk/v2/client" + lsdkPortalSvcV1 "github.com/vngcloud/vngcloud-go-sdk/v2/vngcloud/services/portal/v1" + lsdkPortalSvcV2 "github.com/vngcloud/vngcloud-go-sdk/v2/vngcloud/services/portal/v2" "golang.org/x/sys/unix" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -22,6 +19,11 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/volume" + "os" + "path/filepath" + "strconv" + "strings" + "time" lscloud "github.com/vngcloud/vngcloud-blockstorage-csi-driver/pkg/cloud" lsinternal "github.com/vngcloud/vngcloud-blockstorage-csi-driver/pkg/driver/internal" @@ -502,14 +504,49 @@ func (s *nodeService) NodeGetCapabilities(_ lctx.Context, req *lcsi.NodeGetCapab } func (s *nodeService) NodeGetInfo(_ lctx.Context, _ *lcsi.NodeGetInfoRequest) (*lcsi.NodeGetInfoResponse, error) { + klog.V(5).InfoS("[DEBUG] - NodeGetInfo: START to get the node info") nodeUUID := s.metadata.GetInstanceID() zone := s.metadata.GetAvailabilityZone() + projectID := s.metadata.GetProjectID() + + klog.InfoS("[INFO] - NodeGetInfo: the necessary information is retrieved successfully", "nodeId", nodeUUID, "zone", zone, "projectId", projectID) + if len(projectID) < 1 { + klog.ErrorS(nil, "[ERROR] - NodeGetInfo; projectID is empty") + return nil, fmt.Errorf("projectID is empty") + } + + clientCfg := lsdkClientV2.NewSdkConfigure(). + WithClientId(s.driverOptions.clientID). + WithClientSecret(s.driverOptions.clientSecret). + WithIamEndpoint(s.driverOptions.identityURL). + WithVServerEndpoint(s.driverOptions.vServerURL) + + cloudClient := lsdkClientV2.NewClient(lctx.TODO()).Configure(clientCfg) - klog.V(5).InfoS("NodeGetInfo; called to get node info", "nodeUUID", nodeUUID, "zone", zone) + klog.V(5).InfoS("[DEBUG] - NodeGetInfo: Get the portal info and quota") + portal, sdkErr := cloudClient.VServerGateway().V1().PortalService(). + GetPortalInfo(lsdkPortalSvcV1.NewGetPortalInfoRequest(projectID)) + if sdkErr != nil { + klog.ErrorS(sdkErr.GetError(), "[ERROR] - NodeGetInfo; failed to get portal info") + return nil, sdkErr.GetError() + } + + klog.InfoS("[INFO] - NodeGetInfo: Received the portal info successfully", "portal", portal) + cloudClient = cloudClient.WithProjectId(portal.ProjectID) + + quota, sdkErr := cloudClient.VServerGateway().V2().PortalService(). + GetQuotaByName(lsdkPortalSvcV2.NewGetQuotaByNameRequest(lsdkPortalSvcV2.QtVolumeAttachLimit)) + + if sdkErr != nil { + klog.ErrorS(sdkErr.GetError(), "[ERROR] - NodeGetInfo; failed to get quota") + return nil, sdkErr.GetError() + } + klog.InfoS("[INFO] - NodeGetInfo: Setup the VngCloud Manage CSI driver for this node successfully", + "quota", quota, "nodeId", nodeUUID, "zone", zone, "projectId", projectID) return &lcsi.NodeGetInfoResponse{ NodeId: nodeUUID, - MaxVolumesPerNode: 26, + MaxVolumesPerNode: int64(quota.Limit), AccessibleTopology: &lcsi.Topology{ Segments: map[string]string{ ZoneTopologyKey: zone, diff --git a/pkg/driver/validation.go b/pkg/driver/validation.go index 1328857..2a14829 100644 --- a/pkg/driver/validation.go +++ b/pkg/driver/validation.go @@ -3,18 +3,16 @@ package driver import ( "errors" "fmt" - "github.com/cuongpiger/joat/math" ljoat "github.com/cuongpiger/joat/parser" "strconv" "strings" lcsi "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/vngcloud/vngcloud-blockstorage-csi-driver/pkg/cloud" "github.com/vngcloud/vngcloud-csi-volume-modifier/pkg/rpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "k8s.io/klog/v2" - - "github.com/vngcloud/vngcloud-blockstorage-csi-driver/pkg/cloud" ) func ValidateDriverOptions(options *DriverOptions) error { @@ -219,12 +217,3 @@ func validateDeleteSnapshotRequest(req *lcsi.DeleteSnapshotRequest) error { } return nil } - -func getVolSizeBytes(preq *lcsi.CreateVolumeRequest) (volSizeBytes int64) { - // get the volume size that user provided - if preq.GetCapacityRange() != nil { - volSizeBytes = preq.GetCapacityRange().GetRequiredBytes() - } - - return math.MaxNumeric(volSizeBytes, cloud.DefaultVolumeSize) -} diff --git a/pkg/driver/volume_dto.go b/pkg/driver/volume_dto.go index ca6edaf..ff9cdd9 100644 --- a/pkg/driver/volume_dto.go +++ b/pkg/driver/volume_dto.go @@ -5,12 +5,13 @@ import ( lstr "strings" lcsi "github.com/container-storage-interface/spec/lib/go/csi" + ljset "github.com/cuongpiger/joat/data-structure/set" ljoat "github.com/cuongpiger/joat/string" + lsdkVolumeV2 "github.com/vngcloud/vngcloud-go-sdk/v2/vngcloud/services/volume/v2" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" lscloud "github.com/vngcloud/vngcloud-blockstorage-csi-driver/pkg/cloud" - lsdkVolV2 "github.com/vngcloud/vngcloud-go-sdk/vngcloud/services/blockstorage/v2/volume" ) type CreateVolumeRequest struct { @@ -25,7 +26,7 @@ type CreateVolumeRequest struct { PvNameTag string // the name of the PV on the PVC's Annotation IsPoc bool // whether the volume is a PoC volume SnapshotID string // the ID of the snapshot to create the volume from - CreateFrom lsdkVolV2.CreateOptsCreateFrom + CreateFrom lsdkVolumeV2.CreateVolumeFrom // The scope of mount commands BlockSize string @@ -34,18 +35,18 @@ type CreateVolumeRequest struct { NumberOfInodes string Ext4ClusterSize string Ext4BigAlloc bool -} + RetainPolicy string -type encryptedAlgorithm string + DriverOptions *DriverOptions +} -const ( - AesXtsPlain64_128 = encryptedAlgorithm("aes-xts-plain64_128") - AesXtsPlain64_256 = encryptedAlgorithm("aes-xts-plain64_256") +var ( + EncryptTypeSet = ljset.NewSet[lsdkVolumeV2.EncryptType](lsdkVolumeV2.AesXtsPlain64_128, lsdkVolumeV2.AesXtsPlain64_256) ) func NewCreateVolumeRequest() *CreateVolumeRequest { return &CreateVolumeRequest{ - CreateFrom: lsdkVolV2.CreateFromNew, // set default value for createFrom field + CreateFrom: lsdkVolumeV2.CreateFromNew, // set default value for createFrom field } } @@ -54,6 +55,15 @@ func (s *CreateVolumeRequest) WithClusterID(pclusterID string) *CreateVolumeRequ return s } +func (s *CreateVolumeRequest) WithDriverOptions(pdo *DriverOptions) *CreateVolumeRequest { + if pdo == nil { + return s + } + + s.DriverOptions = pdo + return s +} + func (s *CreateVolumeRequest) WithVolumeTypeID(pvolumeTypeID string) *CreateVolumeRequest { if pvolumeTypeID == "" { return s @@ -131,83 +141,57 @@ func (s *CreateVolumeRequest) WithExt4ClusterSize(pext4ClusterSize string) *Crea func (s *CreateVolumeRequest) WithSnapshotID(psnapshotID string) *CreateVolumeRequest { s.SnapshotID = psnapshotID if psnapshotID != "" { - s.CreateFrom = lsdkVolV2.CreateFromSnapshot + s.CreateFrom = lsdkVolumeV2.CreateFromSnapshot } return s } func (s *CreateVolumeRequest) WithEncrypted(pencrypted string) *CreateVolumeRequest { - pencrypted = lstr.ToLower(pencrypted) - switch pencrypted { - case string(AesXtsPlain64_128): - s.EncryptedAlgorithm = pencrypted - case string(AesXtsPlain64_256): + pencrypted = lstr.ToLower(lstr.TrimSpace(pencrypted)) + if EncryptTypeSet.ContainsOne(lsdkVolumeV2.EncryptType(pencrypted)) { s.EncryptedAlgorithm = pencrypted - default: - s.EncryptedAlgorithm = "" + return s } + s.EncryptedAlgorithm = "" return s } -func (s *CreateVolumeRequest) ToSdkCreateVolumeOpts(pdo *DriverOptions) *lsdkVolV2.CreateOpts { - opts := new(lsdkVolV2.CreateOpts) - - opts.IsPoc = s.IsPoc - opts.MultiAttach = s.IsMultiAttach - opts.Name = s.VolumeName - opts.Size = s.VolumeSize - opts.VolumeTypeId = s.VolumeTypeID - opts.CreatedFrom = s.CreateFrom - opts.EncryptionType = s.EncryptedAlgorithm - opts.Tags = s.prepareTag(pdo.GetTagKeyLength(), pdo.GetTagValueLength()) +func (s *CreateVolumeRequest) ToSdkCreateVolumeRequest() lsdkVolumeV2.ICreateBlockVolumeRequest { + opts := lsdkVolumeV2.NewCreateBlockVolumeRequest(s.VolumeName, s.VolumeTypeID, int64(s.VolumeSize)). + WithPoc(s.IsPoc). + WithMultiAttach(s.IsMultiAttach). + WithEncryptionType(lsdkVolumeV2.EncryptType(s.EncryptedAlgorithm)). + WithTags(s.prepareTag(s.DriverOptions.GetTagKeyLength(), s.DriverOptions.GetTagValueLength())...) if s.SnapshotID != "" { - opts.ConfigureVolumeRestore = &lsdkVolV2.ConfigureVolumeRestore{ - SnapshotVolumePointId: s.SnapshotID, - VolumeTypeId: s.VolumeTypeID, - } + opts = opts.WithVolumeRestoreFromSnapshot(s.SnapshotID, s.VolumeTypeID) } return opts } -func (s *CreateVolumeRequest) prepareTag(ptkl, ptvl int) []lsdkVolV2.CreateOptsTag { - var vts []lsdkVolV2.CreateOptsTag +func (s *CreateVolumeRequest) prepareTag(ptkl, ptvl int) []string { + var vts []string if s.ClusterID != "" { - vts = append(vts, lsdkVolV2.CreateOptsTag{ - Key: ljoat.Truncate(lscloud.VksClusterIdTagKey, ptkl), - Value: ljoat.Truncate(s.ClusterID, ptvl), - }) + vts = append(vts, ljoat.Truncate(lscloud.VksClusterIdTagKey, ptkl), ljoat.Truncate(s.ClusterID, ptvl)) } if s.PvcNameTag != "" { - vts = append(vts, lsdkVolV2.CreateOptsTag{ - Key: ljoat.Truncate(lscloud.VksPvcNameTagKey, ptkl), - Value: ljoat.Truncate(s.PvcNameTag, ptvl), - }) + vts = append(vts, ljoat.Truncate(lscloud.VksPvcNameTagKey, ptkl), ljoat.Truncate(s.PvcNameTag, ptvl)) } if s.PvcNamespaceTag != "" { - vts = append(vts, lsdkVolV2.CreateOptsTag{ - Key: ljoat.Truncate(lscloud.VksPvcNamespaceTagKey, ptkl), - Value: ljoat.Truncate(s.PvcNamespaceTag, ptvl), - }) + vts = append(vts, ljoat.Truncate(lscloud.VksPvcNamespaceTagKey, ptkl), ljoat.Truncate(s.PvcNamespaceTag, ptvl)) } if s.PvcNameTag != "" { - vts = append(vts, lsdkVolV2.CreateOptsTag{ - Key: ljoat.Truncate(lscloud.VksPvNameTagKey, ptkl), - Value: ljoat.Truncate(s.PvNameTag, ptvl), - }) + vts = append(vts, ljoat.Truncate(lscloud.VksPvNameTagKey, ptkl), ljoat.Truncate(s.PvNameTag, ptvl)) } if s.SnapshotID != "" { - vts = append(vts, lsdkVolV2.CreateOptsTag{ - Key: ljoat.Truncate(lscloud.VksSnapshotIdTagKey, ptkl), - Value: ljoat.Truncate(s.SnapshotID, ptvl), - }) + vts = append(vts, ljoat.Truncate(lscloud.VksSnapshotIdTagKey, ptkl), ljoat.Truncate(s.SnapshotID, ptvl)) } return vts