diff --git a/.golangci.yml b/.golangci.yml index ee3076a27..6479c7e47 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -61,3 +61,5 @@ linters-settings: gocritic: disabled-tags: - style + dogsled: + max-blank-identifiers: 3 diff --git a/cmd/collectors/power.go b/cmd/collectors/power.go index 38b22768e..da9afef5c 100644 --- a/cmd/collectors/power.go +++ b/cmd/collectors/power.go @@ -94,7 +94,7 @@ var eMetrics = []string{ "power", } -func calculateEnvironmentMetrics(data *matrix.Matrix, logger *logging.Logger, valueKey string, myData *matrix.Matrix, nodeToNumNode map[string]int) ([]*matrix.Matrix, error) { +func calculateEnvironmentMetrics(data *matrix.Matrix, logger *logging.Logger, valueKey string, myData *matrix.Matrix, nodeToNumNode map[string]int) []*matrix.Matrix { sensorEnvironmentMetricMap := make(map[string]*environmentMetric) excludedSensors := make(map[string][]sensorValue) @@ -375,7 +375,7 @@ func calculateEnvironmentMetrics(data *matrix.Matrix, logger *logging.Logger, va Msg("sensor with *hr units") } - return []*matrix.Matrix{myData}, nil + return []*matrix.Matrix{myData} } func NewSensor(p *plugin.AbstractPlugin) plugin.Plugin { @@ -422,11 +422,12 @@ func (my *Sensor) Init() error { return nil } -func (my *Sensor) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (my *Sensor) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[my.Object] // Purge and reset data my.data.PurgeInstances() my.data.Reset() + my.client.Metadata.Reset() // Set all global labels if they don't already exist my.data.SetGlobalLabels(data.GetGlobalLabels()) @@ -434,7 +435,7 @@ func (my *Sensor) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, erro // Collect chassis fru show, so we can determine if a controller's PSUs are shared or not nodeToNumNode, err := collectChassisFRU(my.client, my.Logger) if err != nil { - return nil, err + return nil, nil, err } if len(nodeToNumNode) == 0 { my.Logger.Debug().Msg("No chassis field replaceable units found") @@ -444,5 +445,7 @@ func (my *Sensor) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, erro if my.Parent == "Rest" { valueKey = restValueKey } - return calculateEnvironmentMetrics(data, my.Logger, valueKey, my.data, nodeToNumNode) + metrics := calculateEnvironmentMetrics(data, my.Logger, valueKey, my.data, nodeToNumNode) + + return metrics, my.client.Metadata, nil } diff --git a/cmd/collectors/rest/plugins/certificate/certificate.go b/cmd/collectors/rest/plugins/certificate/certificate.go index a8b7f4a55..c41a7c988 100644 --- a/cmd/collectors/rest/plugins/certificate/certificate.go +++ b/cmd/collectors/rest/plugins/certificate/certificate.go @@ -13,6 +13,7 @@ import ( "github.com/netapp/harvest/v2/pkg/conf" ontap "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" "github.com/tidwall/gjson" "time" ) @@ -51,7 +52,7 @@ func (my *Certificate) Init() error { return nil } -func (my *Certificate) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (my *Certificate) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { var ( adminVserver string @@ -59,6 +60,7 @@ func (my *Certificate) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, err error ) data := dataMap[my.Object] + my.client.Metadata.Reset() if my.currentVal >= my.PluginInvocationRate { my.currentVal = 0 @@ -70,7 +72,7 @@ func (my *Certificate) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, } else { my.Logger.Error().Err(err).Msg("Failed to collect admin SVM") } - return nil, nil + return nil, nil, nil } // invoke private ssl cli rest and get the admin SVM's serial number @@ -80,7 +82,7 @@ func (my *Certificate) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, } else { my.Logger.Error().Msg("Failed to collect admin SVM's serial number") } - return nil, nil + return nil, nil, nil } // update certificate instance based on admin vaserver serial @@ -102,7 +104,7 @@ func (my *Certificate) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, } my.currentVal++ - return nil, nil + return nil, my.client.Metadata, nil } func (my *Certificate) setCertificateIssuerType(instance *matrix.Instance) { diff --git a/cmd/collectors/rest/plugins/disk/disk.go b/cmd/collectors/rest/plugins/disk/disk.go index c2a94d63e..262557b51 100644 --- a/cmd/collectors/rest/plugins/disk/disk.go +++ b/cmd/collectors/rest/plugins/disk/disk.go @@ -5,6 +5,7 @@ package disk import ( "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" ) type Disk struct { @@ -15,7 +16,7 @@ func New(p *plugin.AbstractPlugin) plugin.Plugin { return &Disk{AbstractPlugin: p} } -func (d *Disk) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (d *Disk) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[d.Object] for _, instance := range data.GetInstances() { @@ -35,5 +36,5 @@ func (d *Disk) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) } - return nil, nil + return nil, nil, nil } diff --git a/cmd/collectors/rest/plugins/health/health.go b/cmd/collectors/rest/plugins/health/health.go index 7a2e3048d..a590376ad 100644 --- a/cmd/collectors/rest/plugins/health/health.go +++ b/cmd/collectors/rest/plugins/health/health.go @@ -9,6 +9,7 @@ import ( "github.com/netapp/harvest/v2/pkg/conf" "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" "github.com/tidwall/gjson" "strconv" "time" @@ -97,15 +98,16 @@ func (h *Health) initMatrix(name string) error { return nil } -func (h *Health) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (h *Health) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[h.Object] + h.client.Metadata.Reset() clusterVersion := h.client.Cluster().GetVersion() ontapVersion, err := goversion.NewVersion(clusterVersion) if err != nil { h.Logger.Error().Err(err). Str("version", clusterVersion). Msg("Failed to parse version") - return nil, nil + return nil, nil, nil } version96 := "9.6" version96After, err := goversion.NewVersion(version96) @@ -113,11 +115,11 @@ func (h *Health) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error h.Logger.Error().Err(err). Str("version", version96). Msg("Failed to parse version") - return nil, nil + return nil, nil, nil } if ontapVersion.LessThan(version96After) { - return nil, nil + return nil, nil, nil } // Purge and reset data @@ -125,7 +127,7 @@ func (h *Health) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error err = h.initAllMatrix() if err != nil { h.Logger.Warn().Err(err).Msg("error while init matrix") - return nil, err + return nil, nil, err } for k := range h.data { // Set all global labels if already not exist @@ -149,7 +151,7 @@ func (h *Health) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error for _, value := range h.data { result = append(result, value) } - return result, nil + return result, h.client.Metadata, nil } func (h *Health) collectLicenseAlerts() { diff --git a/cmd/collectors/rest/plugins/metroclustercheck/metroclustercheck.go b/cmd/collectors/rest/plugins/metroclustercheck/metroclustercheck.go index d81006f2f..7244140e6 100644 --- a/cmd/collectors/rest/plugins/metroclustercheck/metroclustercheck.go +++ b/cmd/collectors/rest/plugins/metroclustercheck/metroclustercheck.go @@ -4,6 +4,7 @@ import ( "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/tree/node" + "github.com/netapp/harvest/v2/pkg/util" "github.com/tidwall/gjson" ) @@ -42,7 +43,7 @@ func (m *MetroclusterCheck) Init() error { return nil } -func (m *MetroclusterCheck) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (m *MetroclusterCheck) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { // Purge and reset data m.data.PurgeInstances() m.data.Reset() @@ -59,7 +60,7 @@ func (m *MetroclusterCheck) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Ma m.update(instance.GetLabel("volume"), "volume") } - return []*matrix.Matrix{m.data}, nil + return []*matrix.Matrix{m.data}, nil, nil } func (m *MetroclusterCheck) update(objectInfo string, object string) { diff --git a/cmd/collectors/rest/plugins/netroute/netroute.go b/cmd/collectors/rest/plugins/netroute/netroute.go index ca825b5df..d41a176ea 100644 --- a/cmd/collectors/rest/plugins/netroute/netroute.go +++ b/cmd/collectors/rest/plugins/netroute/netroute.go @@ -8,6 +8,7 @@ import ( "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/tree/node" + "github.com/netapp/harvest/v2/pkg/util" "github.com/tidwall/gjson" "strconv" "strings" @@ -57,7 +58,7 @@ func (n *NetRoute) Init() error { return nil } -func (n *NetRoute) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (n *NetRoute) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[n.Object] // Purge and reset data @@ -82,7 +83,7 @@ func (n *NetRoute) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, err interfaceInstance, err := n.data.NewInstance(index) if err != nil { n.Logger.Error().Err(err).Str("add instance failed for instance key", key).Send() - return nil, err + return nil, nil, err } for _, l := range instanceLabels { @@ -97,5 +98,5 @@ func (n *NetRoute) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, err } } - return []*matrix.Matrix{n.data}, nil + return []*matrix.Matrix{n.data}, nil, nil } diff --git a/cmd/collectors/rest/plugins/ontaps3service/ontaps3service.go b/cmd/collectors/rest/plugins/ontaps3service/ontaps3service.go index 85c280e5e..80aaa4f97 100644 --- a/cmd/collectors/rest/plugins/ontaps3service/ontaps3service.go +++ b/cmd/collectors/rest/plugins/ontaps3service/ontaps3service.go @@ -11,6 +11,7 @@ import ( "github.com/netapp/harvest/v2/pkg/conf" "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" "github.com/tidwall/gjson" "strings" "time" @@ -54,7 +55,7 @@ func (o *OntapS3Service) Init() error { return nil } -func (o *OntapS3Service) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (o *OntapS3Service) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { var ( result []gjson.Result err error @@ -65,6 +66,7 @@ func (o *OntapS3Service) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matri // reset svmToS3serverMap map svmToURLMap = make(map[string][]string) data := dataMap[o.Object] + o.client.Metadata.Reset() fields := []string{"svm.name", "name", "is_http_enabled", "is_https_enabled", "secure_port", "port"} href := rest.NewHrefBuilder(). @@ -73,14 +75,14 @@ func (o *OntapS3Service) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matri Build() if result, err = collectors.InvokeRestCall(o.client, href, o.Logger); err != nil { - return nil, err + return nil, nil, err } // Iterate over services API response for _, ontaps3Service := range result { if !ontaps3Service.IsObject() { o.Logger.Error().Str("type", ontaps3Service.Type.String()).Msg("Ontap S3 Service is not an object, skipping") - return nil, errs.New(errs.ErrNoInstance, "Ontap S3 Service is not an object") + return nil, nil, errs.New(errs.ErrNoInstance, "Ontap S3 Service is not an object") } s3ServerName := ontaps3Service.Get("name").String() svm := ontaps3Service.Get("svm.name").String() @@ -122,5 +124,5 @@ func (o *OntapS3Service) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matri ontapS3.SetLabel("url", strings.Join(urlValue, ",")) } - return nil, nil + return nil, o.client.Metadata, nil } diff --git a/cmd/collectors/rest/plugins/qospolicyadaptive/qospolicyadaptive.go b/cmd/collectors/rest/plugins/qospolicyadaptive/qospolicyadaptive.go index 2333fa36f..6fc04babf 100644 --- a/cmd/collectors/rest/plugins/qospolicyadaptive/qospolicyadaptive.go +++ b/cmd/collectors/rest/plugins/qospolicyadaptive/qospolicyadaptive.go @@ -4,6 +4,7 @@ import ( "github.com/netapp/harvest/v2/cmd/collectors/zapi/plugins/qospolicyfixed" "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" ) var metrics = []string{ @@ -20,7 +21,7 @@ func New(p *plugin.AbstractPlugin) plugin.Plugin { return &QosPolicyAdaptive{AbstractPlugin: p} } -func (p *QosPolicyAdaptive) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (p *QosPolicyAdaptive) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[p.Object] // create metrics @@ -37,7 +38,7 @@ func (p *QosPolicyAdaptive) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Ma p.setIOPs(data, instance, "peak_iops") } - return nil, nil + return nil, nil, nil } func (p *QosPolicyAdaptive) setIOPs(data *matrix.Matrix, instance *matrix.Instance, labelName string) { diff --git a/cmd/collectors/rest/plugins/qospolicyfixed/qospolicyfixed.go b/cmd/collectors/rest/plugins/qospolicyfixed/qospolicyfixed.go index aceaf00d6..e4092a03e 100644 --- a/cmd/collectors/rest/plugins/qospolicyfixed/qospolicyfixed.go +++ b/cmd/collectors/rest/plugins/qospolicyfixed/qospolicyfixed.go @@ -4,6 +4,7 @@ import ( "github.com/netapp/harvest/v2/cmd/collectors/zapi/plugins/qospolicyfixed" "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" "strings" ) @@ -22,7 +23,7 @@ func New(p *plugin.AbstractPlugin) plugin.Plugin { return &QosPolicyFixed{AbstractPlugin: p} } -func (p *QosPolicyFixed) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (p *QosPolicyFixed) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[p.Object] // create metrics @@ -30,7 +31,7 @@ func (p *QosPolicyFixed) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matri err := matrix.CreateMetric(k, data) if err != nil { p.Logger.Error().Err(err).Str("key", k).Msg("error while creating metric") - return nil, err + return nil, nil, err } } @@ -38,7 +39,7 @@ func (p *QosPolicyFixed) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matri p.setFixed(data, instance) } - return nil, nil + return nil, nil, nil } func (p *QosPolicyFixed) setFixed(data *matrix.Matrix, instance *matrix.Instance) { diff --git a/cmd/collectors/rest/plugins/qtree/qtree.go b/cmd/collectors/rest/plugins/qtree/qtree.go index ca8eb4a2a..ede5cefed 100644 --- a/cmd/collectors/rest/plugins/qtree/qtree.go +++ b/cmd/collectors/rest/plugins/qtree/qtree.go @@ -134,13 +134,15 @@ func (q *Qtree) Init() error { return nil } -func (q *Qtree) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (q *Qtree) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { var ( result []gjson.Result err error numMetrics int ) data := dataMap[q.Object] + q.client.Metadata.Reset() + // Purge and reset data q.data.PurgeInstances() q.data.Reset() @@ -166,7 +168,7 @@ func (q *Qtree) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) Build() if result, err = collectors.InvokeRestCall(q.client, href, q.Logger); err != nil { - return nil, err + return nil, nil, err } quotaCount := 0 @@ -181,7 +183,7 @@ func (q *Qtree) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) } if err != nil { - return nil, err + return nil, nil, err } q.Logger.Info(). @@ -194,7 +196,7 @@ func (q *Qtree) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) qtreePluginData.UUID = q.Parent + ".Qtree" qtreePluginData.Object = "qtree" qtreePluginData.Identifier = "qtree" - return []*matrix.Matrix{qtreePluginData, q.data}, nil + return []*matrix.Matrix{qtreePluginData, q.data}, q.client.Metadata, nil } func (q *Qtree) handlingHistoricalMetrics(result []gjson.Result, data *matrix.Matrix, cluster string, quotaIndex *int, numMetrics *int) error { diff --git a/cmd/collectors/rest/plugins/securityaccount/securityaccount.go b/cmd/collectors/rest/plugins/securityaccount/securityaccount.go index 50d223034..7aed68732 100644 --- a/cmd/collectors/rest/plugins/securityaccount/securityaccount.go +++ b/cmd/collectors/rest/plugins/securityaccount/securityaccount.go @@ -5,12 +5,14 @@ package securityaccount import ( + "fmt" "github.com/netapp/harvest/v2/cmd/collectors" "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/cmd/tools/rest" "github.com/netapp/harvest/v2/pkg/conf" "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" "github.com/tidwall/gjson" "time" ) @@ -41,8 +43,7 @@ func (s *SecurityAccount) Init() error { s.Logger.Info().Str("timeout", timeout.String()).Msg("Using default timeout") } if s.client, err = rest.New(conf.ZapiPoller(s.ParentParams), timeout, s.Auth); err != nil { - s.Logger.Error().Stack().Err(err).Msg("connecting") - return err + return fmt.Errorf("failed to connect err=%w", err) } if err = s.client.Init(5); err != nil { @@ -53,7 +54,7 @@ func (s *SecurityAccount) Init() error { return nil } -func (s *SecurityAccount) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (s *SecurityAccount) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { var ( result []gjson.Result err error @@ -66,8 +67,9 @@ func (s *SecurityAccount) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matr Fields([]string{"applications"}). Build() + s.client.Metadata.Reset() if result, err = collectors.InvokeRestCall(s.client, href, s.Logger); err != nil { - return nil, err + return nil, nil, err } for _, securityAccount := range result { @@ -75,8 +77,7 @@ func (s *SecurityAccount) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matr svm := securityAccount.Get("owner.name").String() if !securityAccount.IsObject() { - s.Logger.Error().Str("type", securityAccount.Type.String()).Msg("Security Account is not an object, skipping") - return nil, errs.New(errs.ErrNoInstance, "security account is not an object") + return nil, nil, errs.New(errs.ErrNoInstance, "security account is not an object. type="+securityAccount.Type.String()) } // reset applicationToMethodMap map @@ -102,8 +103,7 @@ func (s *SecurityAccount) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matr var securityAccountNewInstance *matrix.Instance securityAccountNewKey := securityAccountKey + application + method if securityAccountNewInstance, err = data.NewInstance(securityAccountNewKey); err != nil { - s.Logger.Error().Err(err).Str("add instance failed for instance key", securityAccountNewKey).Send() - return nil, err + return nil, nil, fmt.Errorf("add instance failed for instance key %s err: %w", securityAccountNewKey, err) } for k, v := range securityAccountInstance.GetLabels() { @@ -116,5 +116,5 @@ func (s *SecurityAccount) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matr } } - return []*matrix.Matrix{data}, nil + return []*matrix.Matrix{data}, s.client.Metadata, nil } diff --git a/cmd/collectors/rest/plugins/shelf/shelf.go b/cmd/collectors/rest/plugins/shelf/shelf.go index 8179ecbc7..2d2e4940f 100644 --- a/cmd/collectors/rest/plugins/shelf/shelf.go +++ b/cmd/collectors/rest/plugins/shelf/shelf.go @@ -5,6 +5,7 @@ import ( "github.com/netapp/harvest/v2/cmd/collectors" "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" ) type Shelf struct { @@ -19,7 +20,7 @@ func (my *Shelf) Init() error { return my.InitAbc() } -func (my *Shelf) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (my *Shelf) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[my.Object] for _, instance := range data.GetInstances() { @@ -37,6 +38,6 @@ func (my *Shelf) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error instance.SetLabel("isEmbedded", "No") } } - return nil, nil + return nil, nil, nil } diff --git a/cmd/collectors/rest/plugins/snapmirror/snapmirror.go b/cmd/collectors/rest/plugins/snapmirror/snapmirror.go index 207e6b2a6..4d1c34a93 100644 --- a/cmd/collectors/rest/plugins/snapmirror/snapmirror.go +++ b/cmd/collectors/rest/plugins/snapmirror/snapmirror.go @@ -11,6 +11,7 @@ import ( "github.com/netapp/harvest/v2/pkg/conf" "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/tree/node" + "github.com/netapp/harvest/v2/pkg/util" "strings" "time" ) @@ -79,11 +80,12 @@ func (my *SnapMirror) Init() error { return nil } -func (my *SnapMirror) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (my *SnapMirror) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { // Purge and reset data data := dataMap[my.Object] my.data.PurgeInstances() my.data.Reset() + my.client.Metadata.Reset() // Set all global labels from Rest.go if already not exist my.data.SetGlobalLabels(data.GetGlobalLabels()) @@ -93,7 +95,7 @@ func (my *SnapMirror) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, if cluster, ok := data.GetGlobalLabels()["cluster"]; ok { if err := my.getSVMPeerData(cluster); err != nil { - return nil, err + return nil, nil, err } my.Logger.Debug().Msg("updated svm peer detail") } @@ -103,7 +105,7 @@ func (my *SnapMirror) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, my.updateSMLabels(data) my.currentVal++ - return []*matrix.Matrix{my.data}, nil + return []*matrix.Matrix{my.data}, my.client.Metadata, nil } func (my *SnapMirror) getSVMPeerData(cluster string) error { diff --git a/cmd/collectors/rest/plugins/svm/svm.go b/cmd/collectors/rest/plugins/svm/svm.go index df197568f..dadbd5e52 100644 --- a/cmd/collectors/rest/plugins/svm/svm.go +++ b/cmd/collectors/rest/plugins/svm/svm.go @@ -11,6 +11,7 @@ import ( "github.com/netapp/harvest/v2/pkg/conf" "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" "github.com/tidwall/gjson" "regexp" "sort" @@ -71,12 +72,14 @@ func (my *SVM) Init() error { return nil } -func (my *SVM) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (my *SVM) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { var ( err error ) data := dataMap[my.Object] + my.client.Metadata.Reset() + // update nsswitch info if my.nsswitchInfo, err = my.GetNSSwitchInfo(data); err != nil { if errs.IsRestErr(err, errs.APINotFound) { @@ -164,7 +167,7 @@ func (my *SVM) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) svmInstance.SetLabel("insecured", strconv.FormatBool(insecured)) } } - return nil, nil + return nil, my.client.Metadata, nil } func (my *SVM) GetNSSwitchInfo(data *matrix.Matrix) (map[string]Nsswitch, error) { diff --git a/cmd/collectors/rest/plugins/volume/volume.go b/cmd/collectors/rest/plugins/volume/volume.go index 35478a052..d6083e727 100644 --- a/cmd/collectors/rest/plugins/volume/volume.go +++ b/cmd/collectors/rest/plugins/volume/volume.go @@ -13,6 +13,7 @@ import ( "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/tree/node" + "github.com/netapp/harvest/v2/pkg/util" "github.com/tidwall/gjson" "strconv" "time" @@ -88,8 +89,10 @@ func (v *Volume) Init() error { return nil } -func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[v.Object] + v.client.Metadata.Reset() + if v.currentVal >= v.PluginInvocationRate { v.currentVal = 0 @@ -118,7 +121,7 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error v.handleARWProtection(data) v.currentVal++ - return []*matrix.Matrix{v.arw}, nil + return []*matrix.Matrix{v.arw}, v.client.Metadata, nil } func (v *Volume) updateVolumeLabels(data *matrix.Matrix, volumeMap map[string]volumeInfo) { diff --git a/cmd/collectors/rest/plugins/volumeanalytics/volumeanalytics.go b/cmd/collectors/rest/plugins/volumeanalytics/volumeanalytics.go index d57b095b4..77920b4b4 100644 --- a/cmd/collectors/rest/plugins/volumeanalytics/volumeanalytics.go +++ b/cmd/collectors/rest/plugins/volumeanalytics/volumeanalytics.go @@ -92,8 +92,10 @@ func (v *VolumeAnalytics) initMatrix() error { return nil } -func (v *VolumeAnalytics) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (v *VolumeAnalytics) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[v.Object] + v.client.Metadata.Reset() + cluster := data.GetGlobalLabels()["cluster"] clusterVersion := v.client.Cluster().GetVersion() ontapVersion, err := goversion.NewVersion(clusterVersion) @@ -101,7 +103,7 @@ func (v *VolumeAnalytics) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matr v.Logger.Error().Err(err). Str("version", clusterVersion). Msg("Failed to parse version") - return nil, nil + return nil, nil, nil } version98 := "9.8" version98After, err := goversion.NewVersion(version98) @@ -109,11 +111,11 @@ func (v *VolumeAnalytics) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matr v.Logger.Error().Err(err). Str("version", version98). Msg("Failed to parse version") - return nil, nil + return nil, nil, nil } if ontapVersion.LessThan(version98After) { - return nil, nil + return nil, nil, nil } // Purge and reset data @@ -121,7 +123,7 @@ func (v *VolumeAnalytics) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matr err = v.initMatrix() if err != nil { v.Logger.Warn().Err(err).Msg("error while init matrix") - return nil, err + return nil, nil, err } for k := range v.data { // Set all global labels if already not exist @@ -189,7 +191,7 @@ func (v *VolumeAnalytics) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matr m := explorerMatrix.GetMetric(key) if m == nil { if m, err = explorerMatrix.NewMetricFloat64(key, "bytes_used_by_modified_time"); err != nil { - return nil, err + return nil, nil, err } } m.SetLabel("time", mtBytesUsedLabels[i]) @@ -205,7 +207,7 @@ func (v *VolumeAnalytics) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matr m := explorerMatrix.GetMetric(key) if m == nil { if m, err = explorerMatrix.NewMetricFloat64(key, "bytes_used_percent_by_modified_time"); err != nil { - return nil, err + return nil, nil, err } } m.SetLabel("time", mtBytesUsedLabels[i]) @@ -225,7 +227,7 @@ func (v *VolumeAnalytics) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matr m := explorerMatrix.GetMetric(key) if m == nil { if m, err = explorerMatrix.NewMetricFloat64(key, "bytes_used_by_accessed_time"); err != nil { - return nil, err + return nil, nil, err } } m.SetLabel("time", atBytesUsedLabels[i]) @@ -241,7 +243,7 @@ func (v *VolumeAnalytics) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matr m := explorerMatrix.GetMetric(key) if m == nil { if m, err = explorerMatrix.NewMetricFloat64(key, "bytes_used_percent_by_accessed_time"); err != nil { - return nil, err + return nil, nil, err } } m.SetLabel("time", atBytesUsedLabels[i]) @@ -262,7 +264,7 @@ func (v *VolumeAnalytics) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matr for _, value := range v.data { result = append(result, value) } - return result, nil + return result, v.client.Metadata, nil } func (v *VolumeAnalytics) getLabelBucket(label string) string { diff --git a/cmd/collectors/rest/rest.go b/cmd/collectors/rest/rest.go index ced3747fe..adda3849b 100644 --- a/cmd/collectors/rest/rest.go +++ b/cmd/collectors/rest/rest.go @@ -28,6 +28,7 @@ import ( "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/set" "github.com/netapp/harvest/v2/pkg/tree/node" + "github.com/netapp/harvest/v2/pkg/util" "github.com/tidwall/gjson" "os" "sort" @@ -203,7 +204,7 @@ func (r *Rest) getClient(a *collector.AbstractCollector, c *auth.Credentials) (* } timeout, _ := time.ParseDuration(rest.DefaultTimeout) if a.Options.IsTest { - return &rest.Client{}, nil + return &rest.Client{Metadata: &util.Metadata{}}, nil } if client, err = rest.New(poller, timeout, c); err != nil { r.Logger.Error().Err(err).Str("poller", opt.Poller).Msg("error creating new client") @@ -296,6 +297,7 @@ func (r *Rest) PollData() (map[string]*matrix.Matrix, error) { r.Logger.Trace().Msg("starting data poll") r.Matrix[r.Object].Reset() + r.Client.Metadata.Reset() startTime = time.Now() @@ -346,6 +348,9 @@ func (r *Rest) pollData( _ = r.Metadata.LazySetValueInt64("parse_time", "data", parseD.Microseconds()) _ = r.Metadata.LazySetValueUint64("metrics", "data", count) _ = r.Metadata.LazySetValueUint64("instances", "data", uint64(numRecords)) + _ = r.Metadata.LazySetValueUint64("bytesRx", "data", r.Client.Metadata.BytesRx) + _ = r.Metadata.LazySetValueUint64("numCalls", "data", r.Client.Metadata.NumCalls) + r.AddCollectCount(count) return r.Matrix, nil diff --git a/cmd/collectors/restperf/plugins/disk/disk.go b/cmd/collectors/restperf/plugins/disk/disk.go index 30d0ab6fb..8c7f804a1 100644 --- a/cmd/collectors/restperf/plugins/disk/disk.go +++ b/cmd/collectors/restperf/plugins/disk/disk.go @@ -205,9 +205,11 @@ func (d *Disk) Init() error { return nil } -func (d *Disk) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (d *Disk) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[d.Object] + d.client.Metadata.Reset() + // Set all global labels from rest.go if already not exist for a := range d.instanceLabels { d.shelfData[a].SetGlobalLabels(data.GetGlobalLabels()) @@ -225,11 +227,11 @@ func (d *Disk) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) records, err := rest.Fetch(d.client, href) if err != nil { d.Logger.Error().Err(err).Str("href", href).Msg("Failed to fetch shelfData") - return nil, err + return nil, nil, err } if len(records) == 0 { - return nil, nil + return nil, nil, nil } d.initMaps() @@ -362,30 +364,30 @@ func (d *Disk) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) output, err = d.handleShelfPower(records, output) if err != nil { - return output, err + return output, nil, err } err = d.getAggregates() if err != nil { - return output, err + return output, nil, err } err = d.getDisks() if err != nil { - return output, err + return output, nil, err } err = d.populateShelfIOPS(data) if err != nil { - return output, err + return output, nil, err } output, err = d.calculateAggrPower(data, output) if err != nil { - return output, err + return output, nil, err } - return output, nil + return output, d.client.Metadata, nil } func (d *Disk) calculateAggrPower(data *matrix.Matrix, output []*matrix.Matrix) ([]*matrix.Matrix, error) { diff --git a/cmd/collectors/restperf/plugins/fabricpool/fabricpool.go b/cmd/collectors/restperf/plugins/fabricpool/fabricpool.go index 3c6be989c..e40884f8a 100644 --- a/cmd/collectors/restperf/plugins/fabricpool/fabricpool.go +++ b/cmd/collectors/restperf/plugins/fabricpool/fabricpool.go @@ -4,6 +4,7 @@ import ( "github.com/netapp/harvest/v2/cmd/collectors" "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" "strconv" "strings" ) @@ -31,7 +32,7 @@ func (f *FabricPool) Init() error { } // Run converts Rest lowercase metric names to uppercase to match ZapiPerf -func (f *FabricPool) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (f *FabricPool) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[f.Object] for _, metric := range data.GetMetrics() { if !metric.IsArray() { @@ -45,7 +46,7 @@ func (f *FabricPool) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, e cache, err := collectors.GetFlexGroupFabricPoolMetrics(dataMap, f.Object, "cloud_bin_op", f.includeConstituents, f.Logger) if err != nil { - return nil, err + return nil, nil, err } - return []*matrix.Matrix{cache}, nil + return []*matrix.Matrix{cache}, nil, nil } diff --git a/cmd/collectors/restperf/plugins/fcp/fcp.go b/cmd/collectors/restperf/plugins/fcp/fcp.go index 1c7d114e9..76ee2139e 100644 --- a/cmd/collectors/restperf/plugins/fcp/fcp.go +++ b/cmd/collectors/restperf/plugins/fcp/fcp.go @@ -4,6 +4,7 @@ import ( "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" "math" "strconv" "strings" @@ -17,25 +18,25 @@ func New(p *plugin.AbstractPlugin) plugin.Plugin { return &Fcp{AbstractPlugin: p} } -func (f *Fcp) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (f *Fcp) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { var rx, tx, util, read, write *matrix.Metric var err error data := dataMap[f.Object] if read = data.GetMetric("read_data"); read == nil { - return nil, errs.New(errs.ErrNoMetric, "read_data") + return nil, nil, errs.New(errs.ErrNoMetric, "read_data") } if write = data.GetMetric("write_data"); write == nil { - return nil, errs.New(errs.ErrNoMetric, "write_data") + return nil, nil, errs.New(errs.ErrNoMetric, "write_data") } if rx = data.GetMetric("read_percent"); rx == nil { if rx, err = data.NewMetricFloat64("read_percent"); err == nil { rx.SetProperty("raw") } else { - return nil, err + return nil, nil, err } } @@ -43,7 +44,7 @@ func (f *Fcp) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { if tx, err = data.NewMetricFloat64("write_percent"); err == nil { tx.SetProperty("raw") } else { - return nil, err + return nil, nil, err } } @@ -51,7 +52,7 @@ func (f *Fcp) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { if util, err = data.NewMetricFloat64("util_percent"); err == nil { util.SetProperty("raw") } else { - return nil, err + return nil, nil, err } } @@ -96,5 +97,5 @@ func (f *Fcp) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { } } } - return nil, nil + return nil, nil, nil } diff --git a/cmd/collectors/restperf/plugins/fcvi/fcvi.go b/cmd/collectors/restperf/plugins/fcvi/fcvi.go index adbc242ca..43f2c89ba 100644 --- a/cmd/collectors/restperf/plugins/fcvi/fcvi.go +++ b/cmd/collectors/restperf/plugins/fcvi/fcvi.go @@ -5,6 +5,7 @@ import ( "github.com/netapp/harvest/v2/cmd/tools/rest" "github.com/netapp/harvest/v2/pkg/conf" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" "time" ) @@ -32,8 +33,10 @@ func (f *FCVI) Init() error { return f.client.Init(5) } -func (f *FCVI) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (f *FCVI) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[f.Object] + f.client.Metadata.Reset() + query := "api/private/cli/metrocluster/interconnect/adapter" fields := []string{"node", "adapter", "port_name"} href := rest.NewHrefBuilder(). @@ -43,11 +46,11 @@ func (f *FCVI) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) records, err := rest.Fetch(f.client, href) if err != nil { f.Logger.Error().Err(err).Str("href", href).Msg("Failed to fetch data") - return nil, err + return nil, nil, err } if len(records) == 0 { - return nil, nil + return nil, nil, nil } for _, adapterData := range records { @@ -65,5 +68,5 @@ func (f *FCVI) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) } } - return nil, nil + return nil, f.client.Metadata, nil } diff --git a/cmd/collectors/restperf/plugins/headroom/headroom.go b/cmd/collectors/restperf/plugins/headroom/headroom.go index 9ca63bbf1..43994019f 100644 --- a/cmd/collectors/restperf/plugins/headroom/headroom.go +++ b/cmd/collectors/restperf/plugins/headroom/headroom.go @@ -3,6 +3,7 @@ package headroom import ( "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" "strings" ) @@ -14,7 +15,7 @@ func New(p *plugin.AbstractPlugin) plugin.Plugin { return &Headroom{AbstractPlugin: p} } -func (h *Headroom) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (h *Headroom) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[h.Object] for _, instance := range data.GetInstances() { @@ -36,5 +37,5 @@ func (h *Headroom) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, err } } - return nil, nil + return nil, nil, nil } diff --git a/cmd/collectors/restperf/plugins/nic/nic.go b/cmd/collectors/restperf/plugins/nic/nic.go index 4c045b191..8a76f3100 100644 --- a/cmd/collectors/restperf/plugins/nic/nic.go +++ b/cmd/collectors/restperf/plugins/nic/nic.go @@ -16,6 +16,7 @@ import ( "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" "math" "strconv" "strings" @@ -30,25 +31,25 @@ func New(p *plugin.AbstractPlugin) plugin.Plugin { } // Run speed label is reported in bits-per-second and rx/tx is reported as bytes-per-second -func (n *Nic) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (n *Nic) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { var read, write, rx, tx, util *matrix.Metric var err error data := dataMap[n.Object] if read = data.GetMetric("receive_bytes"); read == nil { - return nil, errs.New(errs.ErrNoMetric, "receive_bytes") + return nil, nil, errs.New(errs.ErrNoMetric, "receive_bytes") } if write = data.GetMetric("transmit_bytes"); write == nil { - return nil, errs.New(errs.ErrNoMetric, "transmit_bytes") + return nil, nil, errs.New(errs.ErrNoMetric, "transmit_bytes") } if rx = data.GetMetric("rx_percent"); rx == nil { if rx, err = data.NewMetricFloat64("rx_percent"); err == nil { rx.SetProperty("raw") } else { - return nil, err + return nil, nil, err } } @@ -56,7 +57,7 @@ func (n *Nic) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { if tx, err = data.NewMetricFloat64("tx_percent"); err == nil { tx.SetProperty("raw") } else { - return nil, err + return nil, nil, err } } @@ -64,7 +65,7 @@ func (n *Nic) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { if util, err = data.NewMetricFloat64("util_percent"); err == nil { util.SetProperty("raw") } else { - return nil, err + return nil, nil, err } } @@ -151,5 +152,5 @@ func (n *Nic) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { } - return nil, nil + return nil, nil, nil } diff --git a/cmd/collectors/restperf/plugins/volume/volume.go b/cmd/collectors/restperf/plugins/volume/volume.go index 2a37037bd..b19bb4448 100644 --- a/cmd/collectors/restperf/plugins/volume/volume.go +++ b/cmd/collectors/restperf/plugins/volume/volume.go @@ -5,6 +5,7 @@ import ( "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/set" + "github.com/netapp/harvest/v2/pkg/util" "maps" "regexp" "sort" @@ -39,7 +40,7 @@ func (v *Volume) Init() error { return nil } -func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { var ( err error @@ -60,7 +61,7 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error metric, err := volumeAggrmetric.NewMetricFloat64(metricName) if err != nil { v.Logger.Error().Err(err).Msg("add metric") - return nil, err + return nil, nil, err } v.Logger.Trace().Msgf("added metric: (%s) %v", metricName, metric) @@ -195,7 +196,7 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error if tempOps == nil { if tempOps, err = cache.NewMetricFloat64(tempOpsKey); err != nil { - return nil, err + return nil, nil, err } tempOps.SetExportable(false) } else { @@ -258,5 +259,5 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error } // volume_aggr_labels metric is deprecated now and will be removed later. - return []*matrix.Matrix{cache, volumeAggrmetric}, nil + return []*matrix.Matrix{cache, volumeAggrmetric}, nil, nil } diff --git a/cmd/collectors/restperf/plugins/volumetag/volumetag.go b/cmd/collectors/restperf/plugins/volumetag/volumetag.go index 643712736..42e7e46d7 100644 --- a/cmd/collectors/restperf/plugins/volumetag/volumetag.go +++ b/cmd/collectors/restperf/plugins/volumetag/volumetag.go @@ -5,6 +5,7 @@ import ( "github.com/netapp/harvest/v2/cmd/tools/rest" "github.com/netapp/harvest/v2/pkg/conf" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" "time" ) @@ -32,13 +33,15 @@ func (v *VolumeTag) Init() error { return v.client.Init(5) } -func (v *VolumeTag) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (v *VolumeTag) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { var ( err error ) data := dataMap[v.Object] + v.client.Metadata.Reset() + query := "api/storage/volumes" href := rest.NewHrefBuilder(). @@ -49,11 +52,11 @@ func (v *VolumeTag) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er records, err := rest.Fetch(v.client, href) if err != nil { v.Logger.Error().Err(err).Str("href", href).Msg("Failed to fetch data") - return nil, err + return nil, nil, err } if len(records) == 0 { - return nil, nil + return nil, nil, nil } for _, volume := range records { @@ -78,5 +81,5 @@ func (v *VolumeTag) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er } } - return nil, nil + return nil, v.client.Metadata, nil } diff --git a/cmd/collectors/restperf/plugins/vscan/vscan.go b/cmd/collectors/restperf/plugins/vscan/vscan.go index 8b5923b01..58accb415 100644 --- a/cmd/collectors/restperf/plugins/vscan/vscan.go +++ b/cmd/collectors/restperf/plugins/vscan/vscan.go @@ -3,6 +3,7 @@ package vscan import ( "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" "strconv" "strings" ) @@ -15,7 +16,7 @@ func New(p *plugin.AbstractPlugin) plugin.Plugin { return &Vscan{AbstractPlugin: p} } -func (v *Vscan) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (v *Vscan) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[v.Object] // defaults plugin options isPerScanner := true @@ -31,7 +32,7 @@ func (v *Vscan) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) v.addSvmAndScannerLabels(data) if !isPerScanner { - return nil, nil + return nil, nil, nil } return v.aggregatePerScanner(data) @@ -53,7 +54,7 @@ func (v *Vscan) addSvmAndScannerLabels(data *matrix.Matrix) { } } -func (v *Vscan) aggregatePerScanner(data *matrix.Matrix) ([]*matrix.Matrix, error) { +func (v *Vscan) aggregatePerScanner(data *matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { // When isPerScanner=true, Harvest 1.6 uses this form: // netapp.perf.dev.nltl-fas2520.vscan.scanner.10_64_30_62.scanner_stats_pct_mem_used 18 1501765640 @@ -152,5 +153,5 @@ func (v *Vscan) aggregatePerScanner(data *matrix.Matrix) ([]*matrix.Matrix, erro } } - return []*matrix.Matrix{cache}, nil + return []*matrix.Matrix{cache}, nil, nil } diff --git a/cmd/collectors/restperf/restperf.go b/cmd/collectors/restperf/restperf.go index 68de7b97d..0cb67e7bc 100644 --- a/cmd/collectors/restperf/restperf.go +++ b/cmd/collectors/restperf/restperf.go @@ -260,6 +260,8 @@ func (r *RestPerf) PollCounter() (map[string]*matrix.Matrix, error) { } apiT := time.Now() + r.Client.Metadata.Reset() + records, err = rest.Fetch(r.Client, href) if err != nil { return r.handleError(err, href) @@ -368,6 +370,8 @@ func (r *RestPerf) pollCounter(records []gjson.Result, apiD time.Duration) (map[ _ = r.Metadata.LazySetValueInt64("api_time", "counter", apiD.Microseconds()) _ = r.Metadata.LazySetValueInt64("parse_time", "counter", time.Since(parseT).Microseconds()) _ = r.Metadata.LazySetValueUint64("metrics", "counter", uint64(len(r.perfProp.counterInfo))) + _ = r.Metadata.LazySetValueUint64("bytesRx", "counter", r.Client.Metadata.BytesRx) + _ = r.Metadata.LazySetValueUint64("numCalls", "counter", r.Client.Metadata.NumCalls) return nil, nil } @@ -667,6 +671,7 @@ func (r *RestPerf) PollData() (map[string]*matrix.Matrix, error) { } startTime = time.Now() + r.Client.Metadata.Reset() dataQuery := path.Join(r.Prop.Query, "rows") @@ -1049,6 +1054,9 @@ func (r *RestPerf) pollData(startTime time.Time, perfRecords []rest.PerfRecord) _ = r.Metadata.LazySetValueInt64("parse_time", "data", parseD.Microseconds()) _ = r.Metadata.LazySetValueUint64("metrics", "data", count) _ = r.Metadata.LazySetValueUint64("instances", "data", uint64(len(curMat.GetInstances()))) + _ = r.Metadata.LazySetValueUint64("bytesRx", "data", r.Client.Metadata.BytesRx) + _ = r.Metadata.LazySetValueUint64("numCalls", "data", r.Client.Metadata.NumCalls) + r.AddCollectCount(count) // skip calculating from delta if no data from previous poll @@ -1396,6 +1404,7 @@ func (r *RestPerf) PollInstance() (map[string]*matrix.Matrix, error) { } apiT := time.Now() + r.Client.Metadata.Reset() records, err = rest.Fetch(r.Client, href) if err != nil { return r.handleError(err, href) @@ -1501,6 +1510,8 @@ func (r *RestPerf) pollInstance(records []gjson.Result, apiD time.Duration) (map _ = r.Metadata.LazySetValueInt64("api_time", "instance", apiD.Microseconds()) _ = r.Metadata.LazySetValueInt64("parse_time", "instance", time.Since(parseT).Microseconds()) _ = r.Metadata.LazySetValueUint64("instances", "instance", uint64(newSize)) + _ = r.Metadata.LazySetValueUint64("bytesRx", "instance", r.Client.Metadata.BytesRx) + _ = r.Metadata.LazySetValueUint64("numCalls", "instance", r.Client.Metadata.NumCalls) if newSize == 0 { return nil, errs.New(errs.ErrNoInstance, "") diff --git a/cmd/collectors/sensor_test.go b/cmd/collectors/sensor_test.go index 9b233f23d..56c1cef5f 100644 --- a/cmd/collectors/sensor_test.go +++ b/cmd/collectors/sensor_test.go @@ -135,11 +135,7 @@ func TestSensor_Run(t *testing.T) { "cdot-k3-07": 1, "cdot-k3-08": 1, } - omat, err := calculateEnvironmentMetrics(mat, logging.Get(), zapiValueKey, sensor.data, nodeToNumNode) - if err != nil { - t.Errorf("got err %v", err) - } - + omat := calculateEnvironmentMetrics(mat, logging.Get(), zapiValueKey, sensor.data, nodeToNumNode) expected := map[string]map[string]float64{ "average_ambient_temperature": {"cdot-k3-05": 22, "cdot-k3-06": 22.5, "cdot-k3-07": 22, "cdot-k3-08": 22.5}, "average_fan_speed": {"cdot-k3-05": 7030, "cdot-k3-06": 7050, "cdot-k3-07": 7040, "cdot-k3-08": 7050}, diff --git a/cmd/collectors/storagegrid/plugins/bucket/bucket.go b/cmd/collectors/storagegrid/plugins/bucket/bucket.go index 8ef4dc2ab..f993b27e6 100644 --- a/cmd/collectors/storagegrid/plugins/bucket/bucket.go +++ b/cmd/collectors/storagegrid/plugins/bucket/bucket.go @@ -4,6 +4,7 @@ import ( "github.com/netapp/harvest/v2/cmd/collectors/storagegrid/rest" "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" "github.com/tidwall/gjson" ) @@ -42,7 +43,7 @@ func (b *Bucket) Init() error { return nil } -func (b *Bucket) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (b *Bucket) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { var ( instanceKey string ) @@ -55,6 +56,7 @@ func (b *Bucket) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error // Purge and reset data b.data.PurgeInstances() b.data.Reset() + b.client.Metadata.Reset() // Set all global labels from Rest.go if already not exist b.data.SetGlobalLabels(data.GetGlobalLabels()) @@ -115,5 +117,5 @@ func (b *Bucket) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error } } - return []*matrix.Matrix{b.data}, nil + return []*matrix.Matrix{b.data}, b.client.Metadata, nil } diff --git a/cmd/collectors/storagegrid/plugins/joinrest/joinrest.go b/cmd/collectors/storagegrid/plugins/joinrest/joinrest.go index 1e8e23967..073c338e5 100644 --- a/cmd/collectors/storagegrid/plugins/joinrest/joinrest.go +++ b/cmd/collectors/storagegrid/plugins/joinrest/joinrest.go @@ -4,6 +4,7 @@ import ( "github.com/netapp/harvest/v2/cmd/collectors/storagegrid/rest" "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" "github.com/tidwall/gjson" "gopkg.in/yaml.v3" "strings" @@ -75,7 +76,9 @@ func (t *JoinRest) Init() error { return nil } -func (t *JoinRest) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (t *JoinRest) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { + t.client.Metadata.Reset() + if t.timesCalled >= t.PluginInvocationRate { // refresh cache t.timesCalled = 0 @@ -127,7 +130,7 @@ func (t *JoinRest) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, err } } t.timesCalled++ - return nil, nil + return nil, t.client.Metadata, nil } func (t *JoinRest) updateCache(model join, bytes *[]byte) { diff --git a/cmd/collectors/storagegrid/rest/client.go b/cmd/collectors/storagegrid/rest/client.go index 518917d3a..d72ab4ef3 100644 --- a/cmd/collectors/storagegrid/rest/client.go +++ b/cmd/collectors/storagegrid/rest/client.go @@ -29,17 +29,18 @@ const ( var NewClientFunc = NewClient type Client struct { - client *http.Client - request *http.Request - buffer *bytes.Buffer - Logger *logging.Logger - baseURL string - Cluster Cluster - token string - Timeout time.Duration - logRest bool // used to log Rest request/response - APIPath string - auth *auth.Credentials + client *http.Client + request *http.Request + buffer *bytes.Buffer + Logger *logging.Logger + baseURL string + Cluster Cluster + token string + Timeout time.Duration + logRest bool // used to log Rest request/response + APIPath string + auth *auth.Credentials + Metadata *util.Metadata } type Cluster struct { @@ -86,7 +87,8 @@ func New(poller *conf.Poller, timeout time.Duration, c *auth.Credentials) (*Clie ) client = Client{ - auth: c, + auth: c, + Metadata: &util.Metadata{}, } client.Logger = logging.Get().SubLogger("StorageGrid", "Client") @@ -263,6 +265,9 @@ func (c *Client) fetch() ([]byte, error) { } defer c.printRequestAndResponse(body) + c.Metadata.BytesRx += uint64(len(body)) + c.Metadata.NumCalls++ + return body, nil } diff --git a/cmd/collectors/storagegrid/rest/dummyclient.go b/cmd/collectors/storagegrid/rest/dummyclient.go index b9d4ecc8a..aff3371d2 100644 --- a/cmd/collectors/storagegrid/rest/dummyclient.go +++ b/cmd/collectors/storagegrid/rest/dummyclient.go @@ -2,6 +2,7 @@ package rest import ( "bytes" + "github.com/netapp/harvest/v2/pkg/util" "github.com/netapp/harvest/v2/pkg/auth" "github.com/netapp/harvest/v2/pkg/logging" @@ -29,17 +30,18 @@ func NewDummyClient() *Client { } client := &Client{ - client: httpClient, - request: httpRequest, - buffer: buffer, - Logger: logger, - baseURL: "http://example.com", - Cluster: cluster, - token: "TestToken", - Timeout: time.Second * 10, - logRest: true, - APIPath: "/api/v1", - auth: &auth.Credentials{}, + client: httpClient, + request: httpRequest, + buffer: buffer, + Logger: logger, + baseURL: "http://example.com", + Cluster: cluster, + token: "TestToken", + Timeout: time.Second * 10, + logRest: true, + APIPath: "/api/v1", + auth: &auth.Credentials{}, + Metadata: &util.Metadata{}, } return client diff --git a/cmd/collectors/storagegrid/storagegrid.go b/cmd/collectors/storagegrid/storagegrid.go index e32a4a0be..b6bd0cd33 100644 --- a/cmd/collectors/storagegrid/storagegrid.go +++ b/cmd/collectors/storagegrid/storagegrid.go @@ -134,6 +134,7 @@ func (s *StorageGrid) InitCache() error { } func (s *StorageGrid) PollData() (map[string]*matrix.Matrix, error) { + s.client.Metadata.Reset() if s.Props.Query == "prometheus" { return s.pollPrometheusMetrics() } @@ -175,6 +176,9 @@ func (s *StorageGrid) pollPrometheusMetrics() (map[string]*matrix.Matrix, error) _ = s.Metadata.LazySetValueInt64("parse_time", "data", 0) _ = s.Metadata.LazySetValueUint64("metrics", "data", count) _ = s.Metadata.LazySetValueInt64("instances", "data", int64(numRecords)) + _ = s.Metadata.LazySetValueUint64("bytesRx", "data", s.client.Metadata.BytesRx) + _ = s.Metadata.LazySetValueUint64("numCalls", "data", s.client.Metadata.NumCalls) + s.AddCollectCount(count) return metrics, nil @@ -279,6 +283,9 @@ func (s *StorageGrid) pollRest() (map[string]*matrix.Matrix, error) { _ = s.Metadata.LazySetValueInt64("parse_time", "data", parseD.Microseconds()) _ = s.Metadata.LazySetValueUint64("metrics", "data", count) _ = s.Metadata.LazySetValueInt64("instances", "data", int64(numRecords)) + _ = s.Metadata.LazySetValueUint64("bytesRx", "data", s.client.Metadata.BytesRx) + _ = s.Metadata.LazySetValueUint64("numCalls", "data", s.client.Metadata.NumCalls) + s.AddCollectCount(count) return s.Matrix, nil diff --git a/cmd/collectors/storagegrid/tenant.go b/cmd/collectors/storagegrid/tenant.go index 4c364dcd3..8023e4ea9 100644 --- a/cmd/collectors/storagegrid/tenant.go +++ b/cmd/collectors/storagegrid/tenant.go @@ -4,6 +4,7 @@ import ( "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" ) const ( @@ -19,7 +20,7 @@ func NewTenant(p *plugin.AbstractPlugin, s *StorageGrid) plugin.Plugin { return &Tenant{AbstractPlugin: p, sg: s} } -func (t *Tenant) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (t *Tenant) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { var ( used, quota, usedPercent *matrix.Metric @@ -27,19 +28,21 @@ func (t *Tenant) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error tenantNamesByID map[string]string ) data := dataMap[t.Object] + t.sg.client.Metadata.Reset() + if used = data.GetMetric("dataBytes"); used == nil { - return nil, errs.New(errs.ErrNoMetric, "logical_used") + return nil, nil, errs.New(errs.ErrNoMetric, "logical_used") } if quota = data.GetMetric("policy.quotaObjectBytes"); quota == nil { - return nil, errs.New(errs.ErrNoMetric, "logical_quota") + return nil, nil, errs.New(errs.ErrNoMetric, "logical_quota") } if usedPercent = data.GetMetric("used_percent"); usedPercent == nil { if usedPercent, err = data.NewMetricFloat64("used_percent"); err == nil { usedPercent.SetProperty("raw") } else { - return nil, err + return nil, nil, err } } @@ -72,7 +75,7 @@ func (t *Tenant) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error } promMetrics := t.collectPromMetrics(tenantNamesByID) - return promMetrics, nil + return promMetrics, t.sg.client.Metadata, nil } func (t *Tenant) collectPromMetrics(tenantNamesByID map[string]string) []*matrix.Matrix { diff --git a/cmd/collectors/zapi/collector/zapi.go b/cmd/collectors/zapi/collector/zapi.go index b42f04277..e04ed784b 100644 --- a/cmd/collectors/zapi/collector/zapi.go +++ b/cmd/collectors/zapi/collector/zapi.go @@ -263,6 +263,8 @@ func (z *Zapi) PollData() (map[string]*matrix.Matrix, error) { oldInstances := set.New() mat := z.Matrix[z.Object] + z.Client.Metadata.Reset() + // copy keys of current instances. This is used to remove deleted instances from matrix later for key := range mat.GetInstances() { oldInstances.Add(key) @@ -410,6 +412,9 @@ func (z *Zapi) PollData() (map[string]*matrix.Matrix, error) { _ = z.Metadata.LazySetValueInt64("parse_time", "data", parseD.Microseconds()) _ = z.Metadata.LazySetValueUint64("metrics", "data", count) _ = z.Metadata.LazySetValueUint64("instances", "data", uint64(numInstances)) + _ = z.Metadata.LazySetValueUint64("bytesRx", "data", z.Client.Metadata.BytesRx) + _ = z.Metadata.LazySetValueUint64("numCalls", "data", z.Client.Metadata.NumCalls) + z.AddCollectCount(count) if numInstances == 0 { diff --git a/cmd/collectors/zapi/plugins/aggregate/aggregate.go b/cmd/collectors/zapi/plugins/aggregate/aggregate.go index 938745cc0..529a762a5 100644 --- a/cmd/collectors/zapi/plugins/aggregate/aggregate.go +++ b/cmd/collectors/zapi/plugins/aggregate/aggregate.go @@ -10,6 +10,7 @@ import ( "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/tree/node" + "github.com/netapp/harvest/v2/pkg/util" "strconv" "strings" ) @@ -45,8 +46,9 @@ func (a *Aggregate) Init() error { return nil } -func (a *Aggregate) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (a *Aggregate) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[a.Object] + a.client.Metadata.Reset() // invoke aggr-object-store-get-iter zapi and populate cloud stores info if err := a.getCloudStores(); err != nil { @@ -95,7 +97,7 @@ func (a *Aggregate) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er } } } - return nil, nil + return nil, a.client.Metadata, nil } func (a *Aggregate) getCloudStores() error { diff --git a/cmd/collectors/zapi/plugins/certificate/certificate.go b/cmd/collectors/zapi/plugins/certificate/certificate.go index 6936765ed..7882d8fa1 100644 --- a/cmd/collectors/zapi/plugins/certificate/certificate.go +++ b/cmd/collectors/zapi/plugins/certificate/certificate.go @@ -14,6 +14,7 @@ import ( "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/tree/node" + "github.com/netapp/harvest/v2/pkg/util" "strconv" "time" ) @@ -61,7 +62,7 @@ func (my *Certificate) Init() error { return nil } -func (my *Certificate) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (my *Certificate) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { var ( adminVserver string @@ -70,6 +71,8 @@ func (my *Certificate) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, ) data := dataMap[my.Object] + my.client.Metadata.Reset() + if my.currentVal >= my.PluginInvocationRate { my.currentVal = 0 @@ -80,7 +83,7 @@ func (my *Certificate) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, } else { my.Logger.Error().Err(err).Msg("Failed to collect admin SVM") } - return nil, nil + return nil, nil, nil } // invoke security-ssl-get-iter zapi and get admin vserver's serial number @@ -90,7 +93,7 @@ func (my *Certificate) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, } else { my.Logger.Error().Err(err).Msg("Failed to collect admin SVM's serial number") } - return nil, nil + return nil, nil, nil } // update certificate instance based on admin vaserver serial @@ -110,7 +113,7 @@ func (my *Certificate) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, } my.currentVal++ - return nil, nil + return nil, my.client.Metadata, nil } func (my *Certificate) setCertificateIssuerType(instance *matrix.Instance, certificateInstanceKey string) { diff --git a/cmd/collectors/zapi/plugins/qospolicyadaptive/qospolicyadaptive.go b/cmd/collectors/zapi/plugins/qospolicyadaptive/qospolicyadaptive.go index 918539da7..ce4820067 100644 --- a/cmd/collectors/zapi/plugins/qospolicyadaptive/qospolicyadaptive.go +++ b/cmd/collectors/zapi/plugins/qospolicyadaptive/qospolicyadaptive.go @@ -4,6 +4,7 @@ import ( "github.com/netapp/harvest/v2/cmd/collectors/zapi/plugins/qospolicyfixed" "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" ) var metrics = []string{ @@ -20,14 +21,14 @@ func New(p *plugin.AbstractPlugin) plugin.Plugin { return &QosPolicyAdaptive{AbstractPlugin: p} } -func (p *QosPolicyAdaptive) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (p *QosPolicyAdaptive) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[p.Object] // create metrics for _, k := range metrics { err := matrix.CreateMetric(k, data) if err != nil { p.Logger.Error().Err(err).Str("key", k).Msg("error while creating metric") - return nil, err + return nil, nil, err } } @@ -40,7 +41,7 @@ func (p *QosPolicyAdaptive) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Ma p.setIOPs(data, instance, "peak_iops") } - return nil, nil + return nil, nil, nil } func (p *QosPolicyAdaptive) setIOPs(data *matrix.Matrix, instance *matrix.Instance, labelName string) { diff --git a/cmd/collectors/zapi/plugins/qospolicyfixed/qospolicyfixed.go b/cmd/collectors/zapi/plugins/qospolicyfixed/qospolicyfixed.go index d7098d777..c8967f8d6 100644 --- a/cmd/collectors/zapi/plugins/qospolicyfixed/qospolicyfixed.go +++ b/cmd/collectors/zapi/plugins/qospolicyfixed/qospolicyfixed.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" "regexp" "strconv" "strings" @@ -25,7 +26,7 @@ func New(p *plugin.AbstractPlugin) plugin.Plugin { return &QosPolicyFixed{AbstractPlugin: p} } -func (p *QosPolicyFixed) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (p *QosPolicyFixed) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { // Change ZAPI max-throughput/min-throughput to match what REST publishes data := dataMap[p.Object] @@ -35,7 +36,7 @@ func (p *QosPolicyFixed) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matri err := matrix.CreateMetric(k, data) if err != nil { p.Logger.Error().Err(err).Str("key", k).Msg("error while creating metric") - return nil, err + return nil, nil, err } } @@ -54,7 +55,7 @@ func (p *QosPolicyFixed) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matri p.setThroughput(data, instance, "min_xput", "min_throughput_iops", "min_throughput_mbps") } - return nil, nil + return nil, nil, nil } func (p *QosPolicyFixed) setThroughput(data *matrix.Matrix, instance *matrix.Instance, labelName string, iopLabel string, mbpsLabel string) { diff --git a/cmd/collectors/zapi/plugins/qtree/qtree.go b/cmd/collectors/zapi/plugins/qtree/qtree.go index 59921929c..831786419 100644 --- a/cmd/collectors/zapi/plugins/qtree/qtree.go +++ b/cmd/collectors/zapi/plugins/qtree/qtree.go @@ -135,7 +135,7 @@ func (q *Qtree) Init() error { return nil } -func (q *Qtree) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (q *Qtree) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { var ( request, response *node.Node quotas []*node.Node @@ -145,6 +145,8 @@ func (q *Qtree) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) ) data := dataMap[q.Object] + q.client.Metadata.Reset() + apiT := 0 * time.Second parseT := 0 * time.Second @@ -181,7 +183,7 @@ func (q *Qtree) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) response, tag, ad, pd, err = q.client.InvokeBatchWithTimers(request, tag) if err != nil { - return nil, err + return nil, nil, err } if response == nil { @@ -201,7 +203,7 @@ func (q *Qtree) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) if len(quotas) == 0 { q.Logger.Debug().Msg("no quota instances found") - return nil, nil + return nil, q.client.Metadata, nil } q.Logger.Debug().Int("quotas", len(quotas)).Msg("fetching quotas") @@ -215,7 +217,7 @@ func (q *Qtree) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) } if err != nil { - return nil, err + return nil, nil, err } } @@ -232,7 +234,7 @@ func (q *Qtree) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) qtreePluginData.UUID = q.Parent + ".Qtree" qtreePluginData.Object = "qtree" qtreePluginData.Identifier = "qtree" - return []*matrix.Matrix{qtreePluginData, q.data}, nil + return []*matrix.Matrix{qtreePluginData, q.data}, q.client.Metadata, nil } func (q *Qtree) handlingHistoricalMetrics(quotas []*node.Node, data *matrix.Matrix, cluster string, quotaIndex *int, numMetrics *int) error { diff --git a/cmd/collectors/zapi/plugins/security/security.go b/cmd/collectors/zapi/plugins/security/security.go index 0ea6f3a21..0d5363055 100644 --- a/cmd/collectors/zapi/plugins/security/security.go +++ b/cmd/collectors/zapi/plugins/security/security.go @@ -11,6 +11,7 @@ import ( "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/tree/node" + "github.com/netapp/harvest/v2/pkg/util" ) type Security struct { @@ -49,26 +50,28 @@ func (my *Security) Init() error { return nil } -func (my *Security) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (my *Security) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { var ( err error ) data := dataMap[my.Object] + my.client.Metadata.Reset() + if my.currentVal >= my.PluginInvocationRate { my.currentVal = 0 // invoke security-config-get zapi with 'ssl' interface and get fips status if my.fipsEnabled, err = my.getSecurityConfig(); err != nil { my.Logger.Warn().Err(err).Msg("Failed to collect fips enable status") - //return nil, nil + // return nil, nil } // invoke security-protocol-get zapi with 'telnet' and 'rsh' and get if my.telnetEnabled, my.rshEnabled, err = my.getSecurityProtocols(); err != nil { my.Logger.Warn().Err(err).Msg("Failed to collect telnet and rsh enable status") - //return nil, nil + // return nil, nil } // update instance based on the above zapi response @@ -86,7 +89,7 @@ func (my *Security) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er } my.currentVal++ - return nil, nil + return nil, my.client.Metadata, nil } func (my *Security) getSecurityConfig() (string, error) { diff --git a/cmd/collectors/zapi/plugins/shelf/shelf.go b/cmd/collectors/zapi/plugins/shelf/shelf.go index 96c90290b..1580ec460 100644 --- a/cmd/collectors/zapi/plugins/shelf/shelf.go +++ b/cmd/collectors/zapi/plugins/shelf/shelf.go @@ -139,7 +139,7 @@ func (my *Shelf) Init() error { return nil } -func (my *Shelf) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (my *Shelf) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { var ( err error @@ -147,6 +147,8 @@ func (my *Shelf) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error ) data := dataMap[my.Object] + my.client.Metadata.Reset() + if my.client.IsClustered() { for _, instance := range data.GetInstances() { if !instance.IsExportable() { @@ -163,7 +165,7 @@ func (my *Shelf) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error instance.SetLabel("isEmbedded", "No") } } - return nil, nil + return nil, nil, nil } // 7 mode handling @@ -183,18 +185,18 @@ func (my *Shelf) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error result, err := my.client.InvokeZapiCall(request) if err != nil { - return nil, err + return nil, nil, err } output, err = my.handle7Mode(data, result) if err != nil { - return output, err + return output, nil, err } my.Logger.Debug().Int("Shelves instance count", len(data.GetInstances())).Send() output = append(output, data) - return output, nil + return output, my.client.Metadata, nil } func (my *Shelf) handle7Mode(data *matrix.Matrix, result []*node.Node) ([]*matrix.Matrix, error) { diff --git a/cmd/collectors/zapi/plugins/snapmirror/snapmirror.go b/cmd/collectors/zapi/plugins/snapmirror/snapmirror.go index 5d6a86456..7ed86b83c 100644 --- a/cmd/collectors/zapi/plugins/snapmirror/snapmirror.go +++ b/cmd/collectors/zapi/plugins/snapmirror/snapmirror.go @@ -12,6 +12,7 @@ import ( "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/tree/node" + "github.com/netapp/harvest/v2/pkg/util" "regexp" "strings" ) @@ -51,14 +52,16 @@ func (my *SnapMirror) Init() error { my.Logger.Debug().Msg("plugin initialized") return nil } -func (my *SnapMirror) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (my *SnapMirror) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[my.Object] + my.client.Metadata.Reset() + destUpdCount := 0 srcUpdCount := 0 if cluster, ok := data.GetGlobalLabels()["cluster"]; ok { if err := my.getSVMPeerData(cluster); err != nil { - return nil, err + return nil, nil, err } my.Logger.Debug().Msg("updated svm peer detail") } @@ -66,10 +69,10 @@ func (my *SnapMirror) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, lastTransferSizeMetric := data.GetMetric("snapmirror-info.last-transfer-size") lagTimeMetric := data.GetMetric("snapmirror-info.lag-time") if lastTransferSizeMetric == nil { - return nil, errs.New(errs.ErrNoMetric, "last_transfer_size") + return nil, nil, errs.New(errs.ErrNoMetric, "last_transfer_size") } if lagTimeMetric == nil { - return nil, errs.New(errs.ErrNoMetric, "lag_time") + return nil, nil, errs.New(errs.ErrNoMetric, "lag_time") } for _, instance := range data.GetInstances() { @@ -130,8 +133,12 @@ func (my *SnapMirror) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, } } } - my.Logger.Debug().Msgf("updated %d destination and %d source nodes", destUpdCount, srcUpdCount) - return nil, nil + my.Logger.Debug(). + Int("destUpdCount", destUpdCount). + Int("srcUpdCount", srcUpdCount). + Msg("updated destination and source nodes") + + return nil, my.client.Metadata, nil } func (my *SnapMirror) getSVMPeerData(cluster string) error { diff --git a/cmd/collectors/zapi/plugins/svm/svm.go b/cmd/collectors/zapi/plugins/svm/svm.go index 0214dc8ed..cc44851bd 100644 --- a/cmd/collectors/zapi/plugins/svm/svm.go +++ b/cmd/collectors/zapi/plugins/svm/svm.go @@ -13,6 +13,7 @@ import ( "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/tree/node" + "github.com/netapp/harvest/v2/pkg/util" "regexp" "sort" "strconv" @@ -113,12 +114,14 @@ func (my *SVM) Init() error { return nil } -func (my *SVM) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (my *SVM) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { var ( err error ) data := dataMap[my.Object] + my.client.Metadata.Reset() + if my.currentVal >= my.PluginInvocationRate { my.currentVal = 0 @@ -307,7 +310,7 @@ func (my *SVM) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) } my.currentVal++ - return nil, nil + return nil, my.client.Metadata, nil } func (my *SVM) GetAuditProtocols() (map[string]string, error) { diff --git a/cmd/collectors/zapi/plugins/volume/volume.go b/cmd/collectors/zapi/plugins/volume/volume.go index 2a74144b2..e336f2d45 100644 --- a/cmd/collectors/zapi/plugins/volume/volume.go +++ b/cmd/collectors/zapi/plugins/volume/volume.go @@ -9,6 +9,7 @@ import ( "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/tree/node" + "github.com/netapp/harvest/v2/pkg/util" "strconv" ) @@ -60,9 +61,11 @@ func (v *Volume) Init() error { return nil } -func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[v.Object] + v.client.Metadata.Reset() + if v.currentVal >= v.PluginInvocationRate { v.currentVal = 0 @@ -105,7 +108,7 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error v.updateVolumeLabels(data, volumeCloneMap, volumeFootprintMap) v.currentVal++ - return nil, nil + return nil, v.client.Metadata, nil } func (v *Volume) updateVolumeLabels(data *matrix.Matrix, volumeCloneMap map[string]volumeClone, volumeFootprintMap map[string]map[string]string) { diff --git a/cmd/collectors/zapiperf/plugins/disk/disk.go b/cmd/collectors/zapiperf/plugins/disk/disk.go index 28703f23e..5c3858d04 100644 --- a/cmd/collectors/zapiperf/plugins/disk/disk.go +++ b/cmd/collectors/zapiperf/plugins/disk/disk.go @@ -259,13 +259,14 @@ func (d *Disk) initMaps() { d.aggrMap = make(map[string]*aggregate) } -func (d *Disk) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (d *Disk) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { var ( err error output []*matrix.Matrix ) data := dataMap[d.Object] + d.client.Metadata.Reset() // Set all global labels from zapi.go if already not exist for a := range d.instanceLabels { @@ -283,42 +284,42 @@ func (d *Disk) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) result, err := d.client.InvokeZapiCall(request) if err != nil { - return nil, err + return nil, nil, err } d.initMaps() output, err = d.handleCMode(result) if err != nil { - return output, err + return output, nil, err } output, err = d.handleShelfPower(result, output) if err != nil { - return output, err + return output, nil, err } err = d.getAggregates() if err != nil { - return output, err + return output, nil, err } err = d.getDisks() if err != nil { - return output, err + return output, nil, err } err = d.populateShelfIOPS(data) if err != nil { - return output, err + return output, nil, err } output, err = d.calculateAggrPower(data, output) if err != nil { - return output, err + return output, nil, err } - return output, nil + return output, d.client.Metadata, nil } func (d *Disk) calculateAggrPower(data *matrix.Matrix, output []*matrix.Matrix) ([]*matrix.Matrix, error) { diff --git a/cmd/collectors/zapiperf/plugins/externalserviceoperation/externalserviceoperation.go b/cmd/collectors/zapiperf/plugins/externalserviceoperation/externalserviceoperation.go index 9294b93b9..c787040f6 100644 --- a/cmd/collectors/zapiperf/plugins/externalserviceoperation/externalserviceoperation.go +++ b/cmd/collectors/zapiperf/plugins/externalserviceoperation/externalserviceoperation.go @@ -5,6 +5,7 @@ package externalserviceoperation import ( "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" ) const Hyphen = "-" @@ -17,7 +18,7 @@ func New(p *plugin.AbstractPlugin) plugin.Plugin { return &ExternalServiceOperation{AbstractPlugin: p} } -func (e *ExternalServiceOperation) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (e *ExternalServiceOperation) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[e.Object] datacenterClusterKey := data.GetGlobalLabels()["datacenter"] + Hyphen + data.GetGlobalLabels()["cluster"] + Hyphen for _, instance := range data.GetInstances() { @@ -28,5 +29,5 @@ func (e *ExternalServiceOperation) Run(dataMap map[string]*matrix.Matrix) ([]*ma key := datacenterClusterKey + instance.GetLabel("svm") + Hyphen + instance.GetLabel("service_name") + Hyphen + instance.GetLabel("operation") instance.SetLabel("key", key) } - return nil, nil + return nil, nil, nil } diff --git a/cmd/collectors/zapiperf/plugins/fabricpool/fabricpool.go b/cmd/collectors/zapiperf/plugins/fabricpool/fabricpool.go index e171c8cd8..38c77f0f3 100644 --- a/cmd/collectors/zapiperf/plugins/fabricpool/fabricpool.go +++ b/cmd/collectors/zapiperf/plugins/fabricpool/fabricpool.go @@ -4,6 +4,7 @@ import ( "github.com/netapp/harvest/v2/cmd/collectors" "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" "strconv" ) @@ -29,10 +30,10 @@ func (f *FabricPool) Init() error { return nil } -func (f *FabricPool) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (f *FabricPool) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { cache, err := collectors.GetFlexGroupFabricPoolMetrics(dataMap, f.Object, "cloud_bin_operation", f.includeConstituents, f.Logger) if err != nil { - return nil, err + return nil, nil, err } - return []*matrix.Matrix{cache}, nil + return []*matrix.Matrix{cache}, nil, nil } diff --git a/cmd/collectors/zapiperf/plugins/fcp/fcp.go b/cmd/collectors/zapiperf/plugins/fcp/fcp.go index 4c39ac3da..00610c516 100644 --- a/cmd/collectors/zapiperf/plugins/fcp/fcp.go +++ b/cmd/collectors/zapiperf/plugins/fcp/fcp.go @@ -8,6 +8,7 @@ import ( "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" "math" "strconv" "strings" @@ -21,7 +22,7 @@ func New(p *plugin.AbstractPlugin) plugin.Plugin { return &Fcp{AbstractPlugin: p} } -func (f *Fcp) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (f *Fcp) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { var rx, tx, util, read, write *matrix.Metric var err error @@ -30,14 +31,14 @@ func (f *Fcp) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { if read = data.GetMetric("read_data"); read == nil { // Check for 7 mode fcp counters, as they start with fcp_. if read = data.GetMetric("fcp_read_data"); read == nil { - return nil, errs.New(errs.ErrNoMetric, "read_data") + return nil, nil, errs.New(errs.ErrNoMetric, "read_data") } } if write = data.GetMetric("write_data"); write == nil { // Check for 7 mode fcp counters, as they start with fcp_. if write = data.GetMetric("fcp_write_data"); write == nil { - return nil, errs.New(errs.ErrNoMetric, "write_data") + return nil, nil, errs.New(errs.ErrNoMetric, "write_data") } } @@ -45,7 +46,7 @@ func (f *Fcp) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { if rx, err = data.NewMetricFloat64("read_percent"); err == nil { rx.SetProperty("raw") } else { - return nil, err + return nil, nil, err } } @@ -53,7 +54,7 @@ func (f *Fcp) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { if tx, err = data.NewMetricFloat64("write_percent"); err == nil { tx.SetProperty("raw") } else { - return nil, err + return nil, nil, err } } @@ -61,7 +62,7 @@ func (f *Fcp) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { if util, err = data.NewMetricFloat64("util_percent"); err == nil { util.SetProperty("raw") } else { - return nil, err + return nil, nil, err } } @@ -109,5 +110,5 @@ func (f *Fcp) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { } } } - return nil, nil + return nil, nil, nil } diff --git a/cmd/collectors/zapiperf/plugins/fcvi/fcvi.go b/cmd/collectors/zapiperf/plugins/fcvi/fcvi.go index 46bfd2fa4..df8d001b0 100644 --- a/cmd/collectors/zapiperf/plugins/fcvi/fcvi.go +++ b/cmd/collectors/zapiperf/plugins/fcvi/fcvi.go @@ -7,6 +7,7 @@ import ( "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/tree/node" + "github.com/netapp/harvest/v2/pkg/util" ) const batchSize = "500" @@ -33,14 +34,17 @@ func (f *FCVI) Init() error { return f.client.Init(5) } -func (f *FCVI) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (f *FCVI) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { var ( result []*node.Node err error ) adapterPortMap := make(map[string]string) + data := dataMap[f.Object] + f.client.Metadata.Reset() + query := "metrocluster-interconnect-adapter-get-iter" request := node.NewXMLS(query) request.NewChildS("max-records", batchSize) @@ -53,11 +57,11 @@ func (f *FCVI) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) request.AddChild(desired) if result, err = f.client.InvokeZapiCall(request); err != nil { - return nil, err + return nil, nil, err } if len(result) == 0 || result == nil { - return nil, errs.New(errs.ErrNoInstance, "no records found") + return nil, nil, errs.New(errs.ErrNoInstance, "no records found") } f.Logger.Info().Msgf("%d", len(result)) @@ -77,5 +81,5 @@ func (f *FCVI) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) instance.SetLabel("port", port) } } - return nil, nil + return nil, f.client.Metadata, nil } diff --git a/cmd/collectors/zapiperf/plugins/flexcache/flexcache.go b/cmd/collectors/zapiperf/plugins/flexcache/flexcache.go index 079530dd7..f2210cfe5 100644 --- a/cmd/collectors/zapiperf/plugins/flexcache/flexcache.go +++ b/cmd/collectors/zapiperf/plugins/flexcache/flexcache.go @@ -7,6 +7,7 @@ import ( "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/set" "github.com/netapp/harvest/v2/pkg/tree/node" + "github.com/netapp/harvest/v2/pkg/util" ) const ( @@ -42,11 +43,13 @@ func (f *FlexCache) Init() error { return nil } -func (f *FlexCache) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (f *FlexCache) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[f.Object] + f.client.Metadata.Reset() + flexCache, err := f.getFlexCaches() if err != nil { - return nil, err + return nil, nil, err } for _, instance := range data.GetInstances() { if !instance.IsExportable() { @@ -59,7 +62,7 @@ func (f *FlexCache) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er instance.SetExportable(false) } } - return nil, nil + return nil, f.client.Metadata, nil } func (f *FlexCache) getFlexCaches() (*set.Set, error) { diff --git a/cmd/collectors/zapiperf/plugins/headroom/headroom.go b/cmd/collectors/zapiperf/plugins/headroom/headroom.go index e4efee38b..5cbda91a3 100644 --- a/cmd/collectors/zapiperf/plugins/headroom/headroom.go +++ b/cmd/collectors/zapiperf/plugins/headroom/headroom.go @@ -5,6 +5,7 @@ package headroom import ( "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" "strings" ) @@ -16,7 +17,7 @@ func New(p *plugin.AbstractPlugin) plugin.Plugin { return &Headroom{AbstractPlugin: p} } -func (h *Headroom) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (h *Headroom) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[h.Object] for _, instance := range data.GetInstances() { @@ -41,5 +42,5 @@ func (h *Headroom) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, err } } - return nil, nil + return nil, nil, nil } diff --git a/cmd/collectors/zapiperf/plugins/nic/nic.go b/cmd/collectors/zapiperf/plugins/nic/nic.go index ac93f99fe..004683bda 100644 --- a/cmd/collectors/zapiperf/plugins/nic/nic.go +++ b/cmd/collectors/zapiperf/plugins/nic/nic.go @@ -18,6 +18,7 @@ import ( "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" "math" "strconv" "strings" @@ -32,25 +33,25 @@ func New(p *plugin.AbstractPlugin) plugin.Plugin { } // Run speed label is reported in bits-per-second and rx/tx is reported as bytes-per-second -func (n *Nic) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (n *Nic) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { var read, write, rx, tx, util *matrix.Metric var err error data := dataMap[n.Object] if read = data.GetMetric("rx_bytes"); read == nil { - return nil, errs.New(errs.ErrNoMetric, "rx_bytes") + return nil, nil, errs.New(errs.ErrNoMetric, "rx_bytes") } if write = data.GetMetric("tx_bytes"); write == nil { - return nil, errs.New(errs.ErrNoMetric, "tx_bytes") + return nil, nil, errs.New(errs.ErrNoMetric, "tx_bytes") } if rx = data.GetMetric("rx_percent"); rx == nil { if rx, err = data.NewMetricFloat64("rx_percent"); err == nil { rx.SetProperty("raw") } else { - return nil, err + return nil, nil, err } } @@ -58,7 +59,7 @@ func (n *Nic) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { if tx, err = data.NewMetricFloat64("tx_percent"); err == nil { tx.SetProperty("raw") } else { - return nil, err + return nil, nil, err } } @@ -66,7 +67,7 @@ func (n *Nic) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { if util, err = data.NewMetricFloat64("util_percent"); err == nil { util.SetProperty("raw") } else { - return nil, err + return nil, nil, err } } @@ -149,5 +150,5 @@ func (n *Nic) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { } - return nil, nil + return nil, nil, nil } diff --git a/cmd/collectors/zapiperf/plugins/volume/volume.go b/cmd/collectors/zapiperf/plugins/volume/volume.go index e95a951c8..ba7960914 100644 --- a/cmd/collectors/zapiperf/plugins/volume/volume.go +++ b/cmd/collectors/zapiperf/plugins/volume/volume.go @@ -9,6 +9,7 @@ import ( "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/set" + "github.com/netapp/harvest/v2/pkg/util" "maps" "regexp" "sort" @@ -47,7 +48,7 @@ func (v *Volume) Init() error { // @TODO rewrite using vector arithmetic // will simplify the code a whole!!! -func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { var ( err error @@ -68,7 +69,7 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error metric, err := volumeAggrmetric.NewMetricFloat64(metricName) if err != nil { v.Logger.Error().Err(err).Msg("add metric") - return nil, err + return nil, nil, err } v.Logger.Trace().Msgf("added metric: (%s) %v", metricName, metric) @@ -207,7 +208,7 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error if tempOps == nil { if tempOps, err = cache.NewMetricFloat64(tempOpsKey); err != nil { - return nil, err + return nil, nil, err } tempOps.SetExportable(false) } else { @@ -272,5 +273,5 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error } // volume_aggr_labels metric is deprecated now and will be removed later. - return []*matrix.Matrix{cache, volumeAggrmetric}, nil + return []*matrix.Matrix{cache, volumeAggrmetric}, nil, nil } diff --git a/cmd/collectors/zapiperf/plugins/volumetag/volumetag.go b/cmd/collectors/zapiperf/plugins/volumetag/volumetag.go index 9dca02d91..cedadbb28 100644 --- a/cmd/collectors/zapiperf/plugins/volumetag/volumetag.go +++ b/cmd/collectors/zapiperf/plugins/volumetag/volumetag.go @@ -6,6 +6,7 @@ import ( "github.com/netapp/harvest/v2/pkg/conf" "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/tree/node" + "github.com/netapp/harvest/v2/pkg/util" ) const batchSize = "500" @@ -32,7 +33,7 @@ func (v *VolumeTag) Init() error { return v.client.Init(5) } -func (v *VolumeTag) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (v *VolumeTag) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { var ( result *node.Node @@ -41,6 +42,8 @@ func (v *VolumeTag) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er ) data := dataMap[v.Object] + v.client.Metadata.Reset() + query := "volume-get-iter" tag := "initial" request := node.NewXMLS(query) @@ -56,7 +59,7 @@ func (v *VolumeTag) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er for { if result, tag, err = v.client.InvokeBatchRequest(request, tag, ""); err != nil { - return nil, err + return nil, nil, err } if result == nil { @@ -67,7 +70,7 @@ func (v *VolumeTag) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er volumes = x.GetChildren() } if len(volumes) == 0 { - return nil, nil + return nil, nil, nil } for _, volume := range volumes { @@ -88,5 +91,5 @@ func (v *VolumeTag) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er } } - return nil, nil + return nil, v.client.Metadata, nil } diff --git a/cmd/collectors/zapiperf/plugins/vscan/vscan.go b/cmd/collectors/zapiperf/plugins/vscan/vscan.go index 0256a0890..78d65595f 100644 --- a/cmd/collectors/zapiperf/plugins/vscan/vscan.go +++ b/cmd/collectors/zapiperf/plugins/vscan/vscan.go @@ -3,6 +3,7 @@ package vscan import ( "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" "strconv" "strings" ) @@ -15,7 +16,7 @@ func New(p *plugin.AbstractPlugin) plugin.Plugin { return &Vscan{AbstractPlugin: p} } -func (v *Vscan) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (v *Vscan) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[v.Object] // defaults plugin options isPerScanner := true @@ -31,7 +32,7 @@ func (v *Vscan) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) v.addSvmAndScannerLabels(data) if !isPerScanner { - return nil, nil + return nil, nil, nil } return v.aggregatePerScanner(data) @@ -56,7 +57,7 @@ func (v *Vscan) addSvmAndScannerLabels(data *matrix.Matrix) { } } -func (v *Vscan) aggregatePerScanner(data *matrix.Matrix) ([]*matrix.Matrix, error) { +func (v *Vscan) aggregatePerScanner(data *matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { // When isPerScanner=true, Harvest 1.6 uses this form: // netapp.perf.dev.nltl-fas2520.vscan.scanner.10_64_30_62.scanner_stats_pct_mem_used 18 1501765640 @@ -161,5 +162,5 @@ func (v *Vscan) aggregatePerScanner(data *matrix.Matrix) ([]*matrix.Matrix, erro } } - return []*matrix.Matrix{cache}, nil + return []*matrix.Matrix{cache}, nil, nil } diff --git a/cmd/collectors/zapiperf/zapiperf.go b/cmd/collectors/zapiperf/zapiperf.go index be9524cb4..e0e29fe08 100644 --- a/cmd/collectors/zapiperf/zapiperf.go +++ b/cmd/collectors/zapiperf/zapiperf.go @@ -359,6 +359,8 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) { z.Logger.Trace().Msg("updating data cache") prevMat := z.Matrix[z.Object] + z.Client.Metadata.Reset() + // clone matrix without numeric data and non-exportable all instances curMat := prevMat.Clone(matrix.With{Data: false, Metrics: true, Instances: true, ExportInstances: false}) curMat.Reset() @@ -751,6 +753,9 @@ func (z *ZapiPerf) PollData() (map[string]*matrix.Matrix, error) { _ = z.Metadata.LazySetValueInt64("parse_time", "data", parseT.Microseconds()) _ = z.Metadata.LazySetValueUint64("metrics", "data", count) _ = z.Metadata.LazySetValueUint64("instances", "data", uint64(len(instanceKeys))) + _ = z.Metadata.LazySetValueUint64("bytesRx", "data", z.Client.Metadata.BytesRx) + _ = z.Metadata.LazySetValueUint64("numCalls", "data", z.Client.Metadata.NumCalls) + z.AddCollectCount(count) // skip calculating from delta if no data from previous poll @@ -1054,6 +1059,8 @@ func (z *ZapiPerf) PollCounter() (map[string]*matrix.Matrix, error) { replaced = set.New() // deprecated and replaced counters mat := z.Matrix[z.Object] + z.Client.Metadata.Reset() + for key := range mat.GetMetrics() { oldMetrics.Add(key) } @@ -1327,6 +1334,8 @@ func (z *ZapiPerf) PollCounter() (map[string]*matrix.Matrix, error) { _ = z.Metadata.LazySetValueInt64("api_time", "counter", apiD.Microseconds()) _ = z.Metadata.LazySetValueInt64("parse_time", "counter", time.Since(parseT).Microseconds()) _ = z.Metadata.LazySetValueUint64("metrics", "counter", uint64(numMetrics)) + _ = z.Metadata.LazySetValueUint64("bytesRx", "counter", z.Client.Metadata.BytesRx) + _ = z.Metadata.LazySetValueUint64("numCalls", "counter", z.Client.Metadata.NumCalls) if numMetrics == 0 { return nil, errs.New(errs.ErrNoMetric, "") @@ -1543,6 +1552,8 @@ func (z *ZapiPerf) PollInstance() (map[string]*matrix.Matrix, error) { oldInstances = set.New() mat := z.Matrix[z.Object] + z.Client.Metadata.Reset() + for key := range mat.GetInstances() { oldInstances.Add(key) } @@ -1672,7 +1683,8 @@ func (z *ZapiPerf) PollInstance() (map[string]*matrix.Matrix, error) { _ = z.Metadata.LazySetValueInt64("api_time", "instance", apiD.Microseconds()) _ = z.Metadata.LazySetValueInt64("parse_time", "instance", parseD.Microseconds()) _ = z.Metadata.LazySetValueUint64("instances", "instance", uint64(newSize)) - + _ = z.Metadata.LazySetValueUint64("bytesRx", "instance", z.Client.Metadata.BytesRx) + _ = z.Metadata.LazySetValueUint64("numCalls", "instance", z.Client.Metadata.NumCalls) if newSize == 0 { return nil, errs.New(errs.ErrNoInstance, "") } diff --git a/cmd/poller/collector/collector.go b/cmd/poller/collector/collector.go index bf5f53f1e..1695d1861 100644 --- a/cmd/poller/collector/collector.go +++ b/cmd/poller/collector/collector.go @@ -254,6 +254,8 @@ func Init(c Collector) error { _, _ = md.NewMetricInt64("plugin_time") _, _ = md.NewMetricUint64("metrics") _, _ = md.NewMetricUint64("instances") + _, _ = md.NewMetricUint64("bytesRx") + _, _ = md.NewMetricUint64("numCalls") // Used by collector logging but not exported loggingOnly := []string{begin, "export_time"} @@ -440,18 +442,23 @@ func (c *AbstractCollector) Start(wg *sync.WaitGroup) { for _, v := range c.Plugins { for _, plg := range v { - if pluginData, err := plg.Run(data); err != nil { + pluginData, pluginMetadata, err := plg.Run(data) + if err != nil { c.Logger.Error().Err(err).Str("plugin", plg.GetName()).Send() - } else if pluginData != nil { + continue + } + if pluginData != nil { results = append(results, pluginData...) c.Logger.Debug(). Str("pluginName", plg.GetName()). Int("dataLength", len(pluginData)). Msg("plugin added data") } else { - c.Logger.Trace(). - Str("pluginName", plg.GetName()). - Msg("plugin completed") + c.Logger.Trace().Str("pluginName", plg.GetName()).Msg("plugin completed") + } + if pluginMetadata != nil { + _ = c.Metadata.LazyAddValueUint64("bytesRx", task.Name, pluginMetadata.BytesRx) + _ = c.Metadata.LazyAddValueUint64("numCalls", task.Name, pluginMetadata.NumCalls) } } } @@ -582,6 +589,12 @@ func (c *AbstractCollector) logMetadata(taskName string, stats exporter.Stats) { info.Str("task", taskName) } + bytesRx, _ := c.Metadata.GetMetric("bytesRx").GetValueUint64(inst) + info.Uint64("bytesRx", bytesRx) + + numCalls, _ := c.Metadata.GetMetric("numCalls").GetValueUint64(inst) + info.Uint64("numCalls", numCalls) + info.Msg("Collected") } diff --git a/cmd/poller/plugin/aggregator/aggregator.go b/cmd/poller/plugin/aggregator/aggregator.go index 82d0e92b3..bec012a9a 100644 --- a/cmd/poller/plugin/aggregator/aggregator.go +++ b/cmd/poller/plugin/aggregator/aggregator.go @@ -9,6 +9,7 @@ import ( "github.com/netapp/harvest/v2/pkg/dict" "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" "github.com/rs/zerolog" "golang.org/x/exp/maps" "regexp" @@ -115,7 +116,7 @@ func (a *Aggregator) parseRules() error { return nil } -func (a *Aggregator) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (a *Aggregator) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[a.Object] matrices := make([]*matrix.Matrix, len(a.rules)) @@ -191,7 +192,7 @@ func (a *Aggregator) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, e if objInstance = matrices[i].GetInstance(objKey); objInstance == nil { rule.counts[objKey] = make(map[string]float64) if objInstance, err = matrices[i].NewInstance(objKey); err != nil { - return nil, err + return nil, nil, err } if rule.allLabels { objInstance.SetLabels(instance.GetLabels()) @@ -294,7 +295,7 @@ func (a *Aggregator) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, e } } - return matrices, nil + return matrices, nil, nil } // NewLabels returns the new labels the receiver creates diff --git a/cmd/poller/plugin/aggregator/aggregator_test.go b/cmd/poller/plugin/aggregator/aggregator_test.go index c23e37a54..f8a6ed34e 100644 --- a/cmd/poller/plugin/aggregator/aggregator_test.go +++ b/cmd/poller/plugin/aggregator/aggregator_test.go @@ -37,7 +37,7 @@ func TestRuleSimpleAggregation(t *testing.T) { dataMap := map[string]*matrix.Matrix{ m.Object: m, } - results, err := p.Run(dataMap) + results, _, err := p.Run(dataMap) if err != nil { t.Fatal(err) } @@ -107,7 +107,7 @@ func TestRuleIncludeAllLabels(t *testing.T) { dataMap := map[string]*matrix.Matrix{ m.Object: m, } - results, err := p.Run(dataMap) + results, _, err := p.Run(dataMap) if err != nil { t.Fatal(err) } @@ -224,7 +224,7 @@ func TestComplexRuleRegex(t *testing.T) { dataMap := map[string]*matrix.Matrix{ m.Object: m, } - results, err := p.Run(dataMap) + results, _, err := p.Run(dataMap) if err != nil { t.Fatal(err) } @@ -367,7 +367,7 @@ func TestRuleSimpleLatencyAggregation(t *testing.T) { dataMap := map[string]*matrix.Matrix{ m.Object: m, } - results, err := p.Run(dataMap) + results, _, err := p.Run(dataMap) if err != nil { t.Fatal(err) } @@ -475,7 +475,7 @@ func TestRuleSimpleLatencyZeroAggregation(t *testing.T) { dataMap := map[string]*matrix.Matrix{ m.Object: m, } - results, err := p.Run(dataMap) + results, _, err := p.Run(dataMap) if err != nil { t.Fatal(err) } diff --git a/cmd/poller/plugin/changelog/change_log.go b/cmd/poller/plugin/changelog/change_log.go index 80e5e58e9..a3ed69ad1 100644 --- a/cmd/poller/plugin/changelog/change_log.go +++ b/cmd/poller/plugin/changelog/change_log.go @@ -5,6 +5,7 @@ import ( "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/set" "github.com/netapp/harvest/v2/pkg/tree/yaml" + "github.com/netapp/harvest/v2/pkg/util" "maps" "strconv" "time" @@ -108,13 +109,13 @@ func (c *ChangeLog) initMatrix() (map[string]*matrix.Matrix, error) { } // Run processes the data and generates ChangeLog instances -func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[c.Object] changeLogMap, err := c.initMatrix() if err != nil { c.Logger.Warn().Err(err).Msg("error while init matrix") - return nil, err + return nil, nil, err } // reset metric count @@ -123,7 +124,7 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er // if this is the first poll if c.previousData == nil { c.copyPreviousData(data) - return nil, nil + return nil, nil, nil } changeMat := changeLogMap[c.matrixName] @@ -132,7 +133,7 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er object := data.Object if c.changeLogConfig.Object == "" { c.Logger.Warn().Str("object", object).Msg("ChangeLog is not supported. Missing correct configuration") - return nil, nil + return nil, nil, nil } prevMat := c.previousData @@ -167,7 +168,7 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er prevInstance := c.previousData.GetInstance(prevKey) if prevInstance == nil { - //instance created + // instance created change := &Change{ key: uuid + "_" + object, object: object, @@ -239,7 +240,7 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er Msg("Collected") } - return matricesArray, nil + return matricesArray, nil, nil } // copyPreviousData creates a copy of the previous data for comparison diff --git a/cmd/poller/plugin/changelog/change_log_test.go b/cmd/poller/plugin/changelog/change_log_test.go index 5902d61ba..62fcbf145 100644 --- a/cmd/poller/plugin/changelog/change_log_test.go +++ b/cmd/poller/plugin/changelog/change_log_test.go @@ -78,7 +78,7 @@ func TestChangeLogModified(t *testing.T) { instance.SetLabel("svm", "s1") instance.SetLabel("type", "t1") - _, _ = p.Run(data) + _, _, _ = p.Run(data) m1 := matrix.New("TestChangeLog", "svm", "svm") data1 := map[string]*matrix.Matrix{ @@ -89,7 +89,7 @@ func TestChangeLogModified(t *testing.T) { instance1.SetLabel("svm", "s2") instance1.SetLabel("type", "t2") - o, _ := p.Run(data1) + o, _, _ := p.Run(data1) checkChangeLogInstances(t, o, 2, 9, update, opLabel) } @@ -105,7 +105,7 @@ func TestChangeLogCreated(t *testing.T) { instance.SetLabel("svm", "s1") instance.SetLabel("type", "t1") - _, _ = p.Run(data) + _, _, _ = p.Run(data) m1 := matrix.New("TestChangeLog", "svm", "svm") data1 := map[string]*matrix.Matrix{ @@ -121,7 +121,7 @@ func TestChangeLogCreated(t *testing.T) { instance2.SetLabel("svm", "s1") instance2.SetLabel("type", "t1") - o, _ := p.Run(data1) + o, _, _ := p.Run(data1) checkChangeLogInstances(t, o, 1, 4, create, opLabel) } @@ -137,14 +137,14 @@ func TestChangeLogDeleted(t *testing.T) { instance.SetLabel("svm", "s1") instance.SetLabel("type", "t1") - _, _ = p.Run(data) + _, _, _ = p.Run(data) m1 := matrix.New("TestChangeLog", "svm", "svm") data1 := map[string]*matrix.Matrix{ "svm": m1, } - o, _ := p.Run(data1) + o, _, _ := p.Run(data1) checkChangeLogInstances(t, o, 1, 4, del, opLabel) } @@ -159,7 +159,7 @@ func TestChangeLogUnsupported(t *testing.T) { instance.SetLabel("uuid", "u1") instance.SetLabel("lun", "l1") - _, _ = p.Run(data) + _, _, _ = p.Run(data) m1 := matrix.New("TestChangeLog", "lun", "lun") data1 := map[string]*matrix.Matrix{ @@ -173,7 +173,7 @@ func TestChangeLogUnsupported(t *testing.T) { instance2.SetLabel("uuid", "u1") instance2.SetLabel("lun", "l3") - o, _ := p.Run(data1) + o, _, _ := p.Run(data1) if len(o) != 0 { t.Errorf("ChangeLog mEtric size expected %d, actual %d", 0, len(o)) @@ -191,7 +191,7 @@ func TestChangeLogModifiedUnsupportedTrack(t *testing.T) { instance.SetLabel("uuid", "u1") instance.SetLabel("svm", "s1") - _, _ = p.Run(data) + _, _, _ = p.Run(data) m1 := matrix.New("TestChangeLog", "svm", "svm") data1 := map[string]*matrix.Matrix{ @@ -202,7 +202,7 @@ func TestChangeLogModifiedUnsupportedTrack(t *testing.T) { instance1.SetLabel("uuid", "u1") instance1.SetLabel("svm", "s2") - o, _ := p.Run(data1) + o, _, _ := p.Run(data1) checkChangeLogInstances(t, o, 0, 0, "", "") } diff --git a/cmd/poller/plugin/labelagent/label_agent.go b/cmd/poller/plugin/labelagent/label_agent.go index 394bd0891..3bfbf0774 100644 --- a/cmd/poller/plugin/labelagent/label_agent.go +++ b/cmd/poller/plugin/labelagent/label_agent.go @@ -10,6 +10,7 @@ import ( "github.com/netapp/harvest/v2/pkg/dict" "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" "strings" ) @@ -57,7 +58,7 @@ func (a *LabelAgent) Init() error { return err } -func (a *LabelAgent) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (a *LabelAgent) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { var err error @@ -66,7 +67,7 @@ func (a *LabelAgent) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, e _ = foo(data) } - return nil, err + return nil, nil, err } // splits one label value into multiple labels using separator symbol diff --git a/cmd/poller/plugin/max/max.go b/cmd/poller/plugin/max/max.go index 03f0b7069..7c7f6cc68 100644 --- a/cmd/poller/plugin/max/max.go +++ b/cmd/poller/plugin/max/max.go @@ -9,6 +9,7 @@ import ( "github.com/netapp/harvest/v2/pkg/dict" "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" "github.com/rs/zerolog" "regexp" "strconv" @@ -115,7 +116,7 @@ func (m *Max) parseRules() error { return nil } -func (m *Max) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (m *Max) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { data := dataMap[m.Object] matrices := make(map[string]*matrix.Matrix) @@ -194,7 +195,7 @@ func (m *Max) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { if objInstance = matrices[matrixKey].GetInstance(objKey); objInstance == nil { rule.counts[objKey] = make(map[string]int) if objInstance, err = matrices[matrixKey].NewInstance(objKey); err != nil { - return nil, err + return nil, nil, err } } @@ -237,7 +238,7 @@ func (m *Max) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { matricesArray = append(matricesArray, v) } - return matricesArray, nil + return matricesArray, nil, nil } // NewMetrics returns the new metrics the receiver creates diff --git a/cmd/poller/plugin/metricagent/metric_agent.go b/cmd/poller/plugin/metricagent/metric_agent.go index fb4bab920..5956a4c88 100644 --- a/cmd/poller/plugin/metricagent/metric_agent.go +++ b/cmd/poller/plugin/metricagent/metric_agent.go @@ -8,6 +8,7 @@ import ( "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/pkg/errs" "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/util" "strconv" "strings" ) @@ -42,7 +43,7 @@ func (a *MetricAgent) Init() error { return err } -func (a *MetricAgent) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (a *MetricAgent) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { var err error data := dataMap[a.Object] @@ -51,7 +52,7 @@ func (a *MetricAgent) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, _ = foo(data) } - return nil, err + return nil, nil, err } func (a *MetricAgent) computeMetrics(m *matrix.Matrix) error { diff --git a/cmd/poller/plugin/plugin.go b/cmd/poller/plugin/plugin.go index 30951376a..a6ed54ab4 100644 --- a/cmd/poller/plugin/plugin.go +++ b/cmd/poller/plugin/plugin.go @@ -33,6 +33,7 @@ import ( "github.com/netapp/harvest/v2/pkg/logging" "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/tree/node" + "github.com/netapp/harvest/v2/pkg/util" "sync" "time" ) @@ -44,7 +45,7 @@ const DefaultPollInterval = 3 * time.Minute type Plugin interface { GetName() string Init() error - Run(map[string]*matrix.Matrix) ([]*matrix.Matrix, error) + Run(map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) } var ( @@ -149,7 +150,7 @@ func (p *AbstractPlugin) InitAbc() error { // Run should run the plugin and return collected data as an array of matrices // (Since most plugins don't collect data, they will always return nil instead) -func (p *AbstractPlugin) Run(map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (p *AbstractPlugin) Run(map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) { panic(p.Name + " has not implemented Run()") } diff --git a/cmd/poller/plugin/test/plugin_test.go b/cmd/poller/plugin/test/plugin_test.go index 07b90a0c3..c17998fcd 100644 --- a/cmd/poller/plugin/test/plugin_test.go +++ b/cmd/poller/plugin/test/plugin_test.go @@ -96,7 +96,7 @@ func TestMultipleRule(t *testing.T) { m.Object: m, } for _, plg := range Plugins { - if pluginData, err := plg.Run(dataMap); err != nil { + if pluginData, _, err := plg.Run(dataMap); err != nil { panic(err) } else if pluginData != nil { results = append(results, pluginData...) diff --git a/cmd/tools/rest/client.go b/cmd/tools/rest/client.go index a982fcd0a..68803bdee 100644 --- a/cmd/tools/rest/client.go +++ b/cmd/tools/rest/client.go @@ -34,15 +34,16 @@ const ( ) type Client struct { - client *http.Client - request *http.Request - buffer *bytes.Buffer - Logger *logging.Logger - baseURL string - cluster Cluster - Timeout time.Duration - logRest bool // used to log Rest request/response - auth *auth.Credentials + client *http.Client + request *http.Request + buffer *bytes.Buffer + Logger *logging.Logger + baseURL string + cluster Cluster + Timeout time.Duration + logRest bool // used to log Rest request/response + auth *auth.Credentials + Metadata *util.Metadata } type Cluster struct { @@ -63,7 +64,8 @@ func New(poller *conf.Poller, timeout time.Duration, auth *auth.Credentials) (*C ) client = Client{ - auth: auth, + auth: auth, + Metadata: &util.Metadata{}, } client.Logger = logging.Get().SubLogger("REST", "Client") @@ -140,6 +142,9 @@ func (c *Client) GetRest(request string) ([]byte, error) { } result, err := c.invokeWithAuthRetry() + c.Metadata.BytesRx += uint64(len(result)) + c.Metadata.NumCalls++ + return result, err } diff --git a/integration/go.mod b/integration/go.mod index a1e2befc3..4b5dff18c 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -19,10 +19,10 @@ require ( dario.cat/mergo v1.0.0 // indirect github.com/bbrks/wrap/v2 v2.5.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect - github.com/go-openapi/jsonpointer v0.20.2 // indirect - github.com/go-openapi/jsonreference v0.20.4 // indirect - github.com/go-openapi/spec v0.20.14 // indirect - github.com/go-openapi/swag v0.22.9 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/spec v0.21.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/lufia/plan9stats v0.0.0-20240226150601-1dcf7310316a // indirect diff --git a/integration/go.sum b/integration/go.sum index bc92cb5f7..3e54b97ed 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -12,14 +12,14 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= -github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q= -github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs= -github.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdXSSgNeAhojU= -github.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHauHj0V9Lhc506VOpw4= -github.com/go-openapi/spec v0.20.14 h1:7CBlRnw+mtjFGlPDRZmAMnq35cRzI91xj03HVyUi/Do= -github.com/go-openapi/spec v0.20.14/go.mod h1:8EOhTpBoFiask8rrgwbLC3zmJfz4zsCUueRuPM6GNkw= -github.com/go-openapi/swag v0.22.9 h1:XX2DssF+mQKM2DHsbgZK74y/zj4mo9I99+89xUmuZCE= -github.com/go-openapi/swag v0.22.9/go.mod h1:3/OXnFfnMAwBD099SwYRk7GD3xOrr1iL7d/XNLXVVwE= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY= +github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -81,8 +81,9 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= diff --git a/pkg/api/ontapi/zapi/client.go b/pkg/api/ontapi/zapi/client.go index 9cb84a73a..57492486c 100644 --- a/pkg/api/ontapi/zapi/client.go +++ b/pkg/api/ontapi/zapi/client.go @@ -16,6 +16,7 @@ import ( "github.com/netapp/harvest/v2/pkg/requests" "github.com/netapp/harvest/v2/pkg/tree" "github.com/netapp/harvest/v2/pkg/tree/node" + "github.com/netapp/harvest/v2/pkg/util" "io" "net/http" "strconv" @@ -38,6 +39,7 @@ type Client struct { Logger *logging.Logger // logger used for logging logZapi bool // used to log ZAPI request/response auth *auth.Credentials + Metadata *util.Metadata } func New(poller *conf.Poller, c *auth.Credentials) (*Client, error) { @@ -52,7 +54,8 @@ func New(poller *conf.Poller, c *auth.Credentials) (*Client, error) { ) client = Client{ - auth: c, + auth: c, + Metadata: &util.Metadata{}, } client.Logger = logging.Get().SubLogger("Zapi", "Client") @@ -512,6 +515,9 @@ func (c *Client) invoke(withTimers bool) (*node.Node, time.Duration, time.Durati parseT = time.Since(start) } + c.Metadata.BytesRx += uint64(len(body)) + c.Metadata.NumCalls++ + // check if the request was successful if result = root.GetChildS("results"); result == nil { return result, responseT, parseT, errs.New(errs.ErrAPIResponse, "missing \"results\"") @@ -590,5 +596,9 @@ func (c *Client) tlsVersion(version string) uint16 { // NewTestClient It's used for unit test only func NewTestClient() *Client { - return &Client{system: &system{name: "testCluster", clustered: true}, request: &http.Request{}} + return &Client{ + system: &system{name: "testCluster", clustered: true}, + request: &http.Request{}, + Metadata: &util.Metadata{}, + } } diff --git a/pkg/matrix/lazy.go b/pkg/matrix/lazy.go index d0ccf6f4f..de2e3150b 100644 --- a/pkg/matrix/lazy.go +++ b/pkg/matrix/lazy.go @@ -64,6 +64,16 @@ func (m *Matrix) LazySetValueUint64(mkey, ikey string, v uint64) error { return errs.New(ErrInvalidInstanceKey, ikey) } +func (m *Matrix) LazyAddValueUint64(key, i string, v uint64) error { + if metric := m.GetMetric(key); metric != nil { + if instance := m.GetInstance(i); instance != nil { + return metric.AddValueUint64(instance, v) + } + return errs.New(ErrInvalidInstanceKey, i) + } + return errs.New(ErrInvalidMetricKey, key) +} + func (m *Matrix) LazySetValueFloat64(mkey, ikey string, v float64) error { if instance := m.GetInstance(ikey); instance != nil { if metric := m.GetMetric(mkey); metric != nil { diff --git a/pkg/util/metadata.go b/pkg/util/metadata.go new file mode 100644 index 000000000..ebab78e28 --- /dev/null +++ b/pkg/util/metadata.go @@ -0,0 +1,11 @@ +package util + +type Metadata struct { + BytesRx uint64 + NumCalls uint64 +} + +func (m *Metadata) Reset() { + m.BytesRx = 0 + m.NumCalls = 0 +}