From da8203d5d0266d4bac01b25131201b1d30cb7e40 Mon Sep 17 00:00:00 2001 From: Jiawei Wang Date: Thu, 1 Oct 2020 16:23:20 -0700 Subject: [PATCH] Upgrade csi-lib-utils to 0.8.1 --- go.mod | 2 +- go.sum | 4 + .../csi-lib-utils/connection/connection.go | 25 +- .../leaderelection/leader_election.go | 13 +- .../csi-lib-utils/metrics/metrics.go | 243 ++++++++++++++++-- vendor/modules.txt | 2 +- 6 files changed, 257 insertions(+), 32 deletions(-) diff --git a/go.mod b/go.mod index 964a285a9b..1edf4724e9 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.15 require ( github.com/container-storage-interface/spec v1.2.0 github.com/golang/mock v1.4.3 - github.com/kubernetes-csi/csi-lib-utils v0.7.0 + github.com/kubernetes-csi/csi-lib-utils v0.8.1 github.com/kubernetes-csi/csi-test/v3 v3.1.1 github.com/kubernetes-csi/external-snapshotter/client/v2 v2.2.0-rc3 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index b77eebfce5..5d7a54abb8 100644 --- a/go.sum +++ b/go.sum @@ -364,6 +364,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kubernetes-csi/csi-lib-utils v0.7.0 h1:t1cS7HTD7z5D7h9iAdjWuHtMxJPb9s1fIv34rxytzqs= github.com/kubernetes-csi/csi-lib-utils v0.7.0/go.mod h1:bze+2G9+cmoHxN6+WyG1qT4MDxgZJMLGwc7V4acPNm0= +github.com/kubernetes-csi/csi-lib-utils v0.8.1 h1:DHFs4MgzjSGF/FH95TEdLVa7R1CCi9UJ76jTUPO8iF0= +github.com/kubernetes-csi/csi-lib-utils v0.8.1/go.mod h1:FZflf0cCYlCquPQxVHa6Tyy0i/so6VAZTiEVK1do7CU= github.com/kubernetes-csi/csi-test/v3 v3.1.1 h1:mFxPbUf7pti663WTCsfaT3YRPVIzy0yLx8HWbVKfN4I= github.com/kubernetes-csi/csi-test/v3 v3.1.1/go.mod h1:UWxYP5cDlD6iSNVKEiLFqfJnJinuhtI7MLt61rQQOfI= github.com/kubernetes-csi/external-snapshotter/client/v2 v2.2.0-rc3 h1://kqMNFTDX6gD13dbDfIKepSasiPbGMPw+kT90kxV2Y= @@ -542,6 +544,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/thecodeteam/goscaleio v0.1.0/go.mod h1:68sdkZAsK8bvEwBlbQnlLS+xU+hvLYM/iQ8KXej1AwM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= @@ -809,6 +812,7 @@ google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.29.0/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/vendor/github.com/kubernetes-csi/csi-lib-utils/connection/connection.go b/vendor/github.com/kubernetes-csi/csi-lib-utils/connection/connection.go index 1d8c6d1efb..c78f12bfc9 100644 --- a/vendor/github.com/kubernetes-csi/csi-lib-utils/connection/connection.go +++ b/vendor/github.com/kubernetes-csi/csi-lib-utils/connection/connection.go @@ -109,7 +109,7 @@ func connect( grpc.WithBlock(), // Block until connection succeeds. grpc.WithChainUnaryInterceptor( LogGRPC, // Log all messages. - extendedCSIMetricsManager{metricsManager}.recordMetricsInterceptor, // Record metrics for each gRPC call. + ExtendedCSIMetricsManager{metricsManager}.RecordMetricsClientInterceptor, // Record metrics for each gRPC call. ), ) unixPrefix := "unix://" @@ -140,7 +140,7 @@ func connect( } conn, err := net.DialTimeout("unix", address[len(unixPrefix):], timeout) if err == nil { - // Connection restablished. + // Connection reestablished. haveConnected = true lostConnection = false } @@ -187,12 +187,13 @@ func LogGRPC(ctx context.Context, method string, req, reply interface{}, cc *grp return err } -type extendedCSIMetricsManager struct { +type ExtendedCSIMetricsManager struct { metrics.CSIMetricsManager } -// recordMetricsInterceptor is a gPRC unary interceptor for recording metrics for CSI operations. -func (cmm extendedCSIMetricsManager) recordMetricsInterceptor( +// RecordMetricsClientInterceptor is a gPRC unary interceptor for recording metrics for CSI operations +// in a gRPC client. +func (cmm ExtendedCSIMetricsManager) RecordMetricsClientInterceptor( ctx context.Context, method string, req, reply interface{}, @@ -209,3 +210,17 @@ func (cmm extendedCSIMetricsManager) recordMetricsInterceptor( ) return err } + +// RecordMetricsServerInterceptor is a gPRC unary interceptor for recording metrics for CSI operations +// in a gRCP server. +func (cmm ExtendedCSIMetricsManager) RecordMetricsServerInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + start := time.Now() + resp, err := handler(ctx, req) + duration := time.Since(start) + cmm.RecordMetrics( + info.FullMethod, /* operationName */ + err, /* operationErr */ + duration, /* operationDuration */ + ) + return resp, err +} diff --git a/vendor/github.com/kubernetes-csi/csi-lib-utils/leaderelection/leader_election.go b/vendor/github.com/kubernetes-csi/csi-lib-utils/leaderelection/leader_election.go index efd58e8491..25fdf70a93 100644 --- a/vendor/github.com/kubernetes-csi/csi-lib-utils/leaderelection/leader_election.go +++ b/vendor/github.com/kubernetes-csi/csi-lib-utils/leaderelection/leader_election.go @@ -60,6 +60,8 @@ type leaderElection struct { renewDeadline time.Duration retryPeriod time.Duration + ctx context.Context + clientset kubernetes.Interface } @@ -127,6 +129,11 @@ func (l *leaderElection) WithRetryPeriod(retryPeriod time.Duration) { l.retryPeriod = retryPeriod } +// WithContext Add context +func (l *leaderElection) WithContext(ctx context.Context) { + l.ctx = ctx +} + func (l *leaderElection) Run() error { if l.identity == "" { id, err := defaultLeaderElectionIdentity() @@ -174,7 +181,11 @@ func (l *leaderElection) Run() error { }, } - leaderelection.RunOrDie(context.TODO(), leaderConfig) + ctx := l.ctx + if ctx == nil { + ctx = context.Background() + } + leaderelection.RunOrDie(ctx, leaderConfig) return nil // should never reach here } diff --git a/vendor/github.com/kubernetes-csi/csi-lib-utils/metrics/metrics.go b/vendor/github.com/kubernetes-csi/csi-lib-utils/metrics/metrics.go index 2e252d6541..c18ab160ae 100644 --- a/vendor/github.com/kubernetes-csi/csi-lib-utils/metrics/metrics.go +++ b/vendor/github.com/kubernetes-csi/csi-lib-utils/metrics/metrics.go @@ -20,6 +20,7 @@ import ( "bufio" "fmt" "net/http" + "sort" "strings" "time" @@ -30,8 +31,18 @@ import ( ) const ( + // SubsystemSidecar is the default subsystem name in a metrics + // (= the prefix in the final metrics name). It is to be used + // by CSI sidecars. Using the same subsystem in different CSI + // drivers makes it possible to reuse dashboards because + // the metrics names will be identical. Data from different + // drivers can be selected via the "driver_name" tag. + SubsystemSidecar = "csi_sidecar" + // SubsystemPlugin is what CSI driver's should use as + // subsystem name. + SubsystemPlugin = "csi_plugin" + // Common metric strings - subsystem = "csi_sidecar" labelCSIDriverName = "driver_name" labelCSIOperationName = "method_name" labelGrpcStatusCode = "grpc_status_code" @@ -56,11 +67,23 @@ type CSIMetricsManager interface { // operationName - Name of the CSI operation. // operationErr - Error, if any, that resulted from execution of operation. // operationDuration - time it took for the operation to complete + // + // If WithLabelNames was used to define additional labels when constructing + // the manager, then WithLabelValues should be used to create a wrapper which + // holds the corresponding values before calling RecordMetrics of the wrapper. + // Labels with missing values are recorded as empty. RecordMetrics( operationName string, operationErr error, operationDuration time.Duration) + // WithLabelValues must be used to add the additional label + // values defined via WithLabelNames. When calling RecordMetrics + // without it or with too few values, the missing values are + // recorded as empty. WithLabelValues can be called multiple times + // and then accumulates values. + WithLabelValues(labels map[string]string) (CSIMetricsManager, error) + // SetDriverName is called to update the CSI driver name. This should be done // as soon as possible, otherwise metrics recorded by this manager will be // recorded with an "unknown-driver" driver_name. @@ -73,25 +96,114 @@ type CSIMetricsManager interface { StartMetricsEndpoint(metricsAddress, metricsPath string) } -// NewCSIMetricsManager creates and registers metrics for for CSI Sidecars and -// returns an object that can be used to trigger the metrics. +// MetricsManagerOption is used to pass optional configuration to a +// new metrics manager. +type MetricsManagerOption func(*csiMetricsManager) + +// WithSubsystem overrides the default subsystem name. +func WithSubsystem(subsystem string) MetricsManagerOption { + return func(cmm *csiMetricsManager) { + cmm.subsystem = subsystem + } +} + +// WithStabilityLevel overrides the default stability level. The recommended +// usage is to keep metrics at a lower level when csi-lib-utils switches +// to beta or GA. Overriding the alpha default with beta or GA is risky +// because the metrics can still change in the library. +func WithStabilityLevel(stabilityLevel metrics.StabilityLevel) MetricsManagerOption { + return func(cmm *csiMetricsManager) { + cmm.stabilityLevel = stabilityLevel + } +} + +// WithLabelNames defines labels for each sample that get added to the +// default labels (driver, method call, and gRPC result). This makes +// it possible to partition the histograms along additional +// dimensions. +// +// To record a metrics with additional values, use +// CSIMetricManager.WithLabelValues().RecordMetrics(). +func WithLabelNames(labelNames ...string) MetricsManagerOption { + return func(cmm *csiMetricsManager) { + cmm.additionalLabelNames = labelNames + } +} + +// WithLabels defines some label name and value pairs that are added to all +// samples. They get recorded sorted by name. +func WithLabels(labels map[string]string) MetricsManagerOption { + return func(cmm *csiMetricsManager) { + var l []label + for name, value := range labels { + l = append(l, label{name, value}) + } + sort.Slice(l, func(i, j int) bool { + return l[i].name < l[j].name + }) + cmm.additionalLabels = l + } +} + +// NewCSIMetricsManagerForSidecar creates and registers metrics for CSI Sidecars and +// returns an object that can be used to trigger the metrics. It uses "csi_sidecar" +// as subsystem. +// +// driverName - Name of the CSI driver against which this operation was executed. +// If unknown, leave empty, and use SetDriverName method to update later. +func NewCSIMetricsManagerForSidecar(driverName string) CSIMetricsManager { + return NewCSIMetricsManagerWithOptions(driverName) +} + +// NewCSIMetricsManager is provided for backwards-compatibility. +var NewCSIMetricsManager = NewCSIMetricsManagerForSidecar + +// NewCSIMetricsManagerForPlugin creates and registers metrics for CSI drivers and +// returns an object that can be used to trigger the metrics. It uses "csi_plugin" +// as subsystem. +// +// driverName - Name of the CSI driver against which this operation was executed. +// If unknown, leave empty, and use SetDriverName method to update later. +func NewCSIMetricsManagerForPlugin(driverName string) CSIMetricsManager { + return NewCSIMetricsManagerWithOptions(driverName, + WithSubsystem(SubsystemPlugin), + ) +} + +// NewCSIMetricsManagerWithOptions is a customizable constructor, to be used only +// if there are special needs like changing the default subsystems. +// // driverName - Name of the CSI driver against which this operation was executed. // If unknown, leave empty, and use SetDriverName method to update later. -func NewCSIMetricsManager(driverName string) CSIMetricsManager { +func NewCSIMetricsManagerWithOptions(driverName string, options ...MetricsManagerOption) CSIMetricsManager { cmm := csiMetricsManager{ - registry: metrics.NewKubeRegistry(), - csiOperationsLatencyMetric: metrics.NewHistogramVec( - &metrics.HistogramOpts{ - Subsystem: subsystem, - Name: operationsLatencyMetricName, - Help: operationsLatencyHelp, - Buckets: operationsLatencyBuckets, - StabilityLevel: metrics.ALPHA, - }, - []string{labelCSIDriverName, labelCSIOperationName, labelGrpcStatusCode}, - ), + registry: metrics.NewKubeRegistry(), + subsystem: SubsystemSidecar, + stabilityLevel: metrics.ALPHA, } + // https://github.com/open-telemetry/opentelemetry-collector/issues/969 + // Add process_start_time_seconds into the metric to let the start time be parsed correctly + metrics.RegisterProcessStartTime(cmm.registry.Register) + + for _, option := range options { + option(&cmm) + } + labels := []string{labelCSIDriverName, labelCSIOperationName, labelGrpcStatusCode} + labels = append(labels, cmm.additionalLabelNames...) + for _, label := range cmm.additionalLabels { + labels = append(labels, label.name) + } + cmm.csiOperationsLatencyMetric = metrics.NewHistogramVec( + &metrics.HistogramOpts{ + Subsystem: cmm.subsystem, + Name: operationsLatencyMetricName, + Help: operationsLatencyHelp, + Buckets: operationsLatencyBuckets, + StabilityLevel: cmm.stabilityLevel, + }, + labels, + ) cmm.SetDriverName(driverName) cmm.registerMetrics() return &cmm @@ -101,26 +213,105 @@ var _ CSIMetricsManager = &csiMetricsManager{} type csiMetricsManager struct { registry metrics.KubeRegistry + subsystem string + stabilityLevel metrics.StabilityLevel driverName string - csiOperationsMetric *metrics.CounterVec + additionalLabelNames []string + additionalLabels []label csiOperationsLatencyMetric *metrics.HistogramVec } +type label struct { + name, value string +} + func (cmm *csiMetricsManager) GetRegistry() metrics.KubeRegistry { return cmm.registry } -// RecordMetrics must be called upon CSI Operation completion to record -// the operation's metric. -// operationName - Name of the CSI operation. -// operationErr - Error, if any, that resulted from execution of operation. -// operationDuration - time it took for the operation to complete +// RecordMetrics implements CSIMetricsManager.RecordMetrics. func (cmm *csiMetricsManager) RecordMetrics( operationName string, operationErr error, operationDuration time.Duration) { - cmm.csiOperationsLatencyMetric.WithLabelValues( - cmm.driverName, operationName, getErrorCode(operationErr)).Observe(operationDuration.Seconds()) + cmm.recordMetricsWithLabels(operationName, operationErr, operationDuration, nil) +} + +// recordMetricsWithLabels is the internal implementation of RecordMetrics. +func (cmm *csiMetricsManager) recordMetricsWithLabels( + operationName string, + operationErr error, + operationDuration time.Duration, + labelValues map[string]string) { + values := []string{cmm.driverName, operationName, getErrorCode(operationErr)} + for _, name := range cmm.additionalLabelNames { + values = append(values, labelValues[name]) + } + for _, label := range cmm.additionalLabels { + values = append(values, label.value) + } + cmm.csiOperationsLatencyMetric.WithLabelValues(values...).Observe(operationDuration.Seconds()) +} + +type csiMetricsManagerWithValues struct { + *csiMetricsManager + + // additionalValues holds the values passed via WithLabelValues. + additionalValues map[string]string +} + +// WithLabelValues in the base metrics manager creates a fresh wrapper with no labels and let's +// that deal with adding the label values. +func (cmm *csiMetricsManager) WithLabelValues(labels map[string]string) (CSIMetricsManager, error) { + cmmv := &csiMetricsManagerWithValues{ + csiMetricsManager: cmm, + additionalValues: map[string]string{}, + } + return cmmv.WithLabelValues(labels) +} + +// WithLabelValues in the wrapper creates a wrapper which has all existing labels and +// adds the new ones, with error checking. Can be called multiple times. Each call then +// can add some new value(s). It is an error to overwrite an already set value. +// If RecordMetrics is called before setting all additional values, the missing ones will +// be empty. +func (cmmv *csiMetricsManagerWithValues) WithLabelValues(labels map[string]string) (CSIMetricsManager, error) { + extended := &csiMetricsManagerWithValues{ + csiMetricsManager: cmmv.csiMetricsManager, + additionalValues: map[string]string{}, + } + // We need to copy the old values to avoid modifying the map in cmmv. + for name, value := range cmmv.additionalValues { + extended.additionalValues[name] = value + } + // Now add all new values. + for name, value := range labels { + if !extended.haveAdditionalLabel(name) { + return nil, fmt.Errorf("label %q was not defined via WithLabelNames", name) + } + if v, ok := extended.additionalValues[name]; ok { + return nil, fmt.Errorf("label %q already has value %q", name, v) + } + extended.additionalValues[name] = value + } + return extended, nil +} + +func (cmm *csiMetricsManager) haveAdditionalLabel(name string) bool { + for _, n := range cmm.additionalLabelNames { + if n == name { + return true + } + } + return false +} + +// RecordMetrics passes the stored values as to the implementation. +func (cmmv *csiMetricsManagerWithValues) RecordMetrics( + operationName string, + operationErr error, + operationDuration time.Duration) { + cmmv.recordMetricsWithLabels(operationName, operationErr, operationDuration, cmmv.additionalValues) } // SetDriverName is called to update the CSI driver name. This should be done @@ -171,7 +362,11 @@ func VerifyMetricsMatch(expectedMetrics, actualMetrics string, metricToIgnore st wantScanner.Scan() wantLine := strings.TrimSpace(wantScanner.Text()) gotLine := strings.TrimSpace(gotScanner.Text()) - if wantLine != gotLine && (metricToIgnore == "" || !strings.HasPrefix(gotLine, metricToIgnore)) { + if wantLine != gotLine && + (metricToIgnore == "" || !strings.HasPrefix(gotLine, metricToIgnore)) && + // We should ignore the comments from metricToIgnore, otherwise the verification will + // fail because of the comments. + !strings.HasPrefix(gotLine, "#") { return fmt.Errorf("\r\nMetric Want: %q\r\nMetric Got: %q\r\n", wantLine, gotLine) } } diff --git a/vendor/modules.txt b/vendor/modules.txt index c5e37937ef..a82bb9594e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -51,7 +51,7 @@ github.com/hashicorp/golang-lru/simplelru github.com/imdario/mergo # github.com/json-iterator/go v1.1.10 github.com/json-iterator/go -# github.com/kubernetes-csi/csi-lib-utils v0.7.0 +# github.com/kubernetes-csi/csi-lib-utils v0.8.1 ## explicit github.com/kubernetes-csi/csi-lib-utils/connection github.com/kubernetes-csi/csi-lib-utils/leaderelection