feat: Harvest should track network bytes received and number of ONTAP…

… calls

.golangci.yml

@@ -61,3 +61,5 @@ linters-settings:
   gocritic:
     disabled-tags:
       - style
+  dogsled:
+    max-blank-identifiers: 3

cmd/collectors/power.go

@@ -94,7 +94,7 @@ var eMetrics = []string{
 	"power",
 }
 
-func calculateEnvironmentMetrics(data *matrix.Matrix, logger *logging.Logger, valueKey string, myData *matrix.Matrix, nodeToNumNode map[string]int) ([]*matrix.Matrix, error) {
+func calculateEnvironmentMetrics(data *matrix.Matrix, logger *logging.Logger, valueKey string, myData *matrix.Matrix, nodeToNumNode map[string]int) []*matrix.Matrix {
 	sensorEnvironmentMetricMap := make(map[string]*environmentMetric)
 	excludedSensors := make(map[string][]sensorValue)
 
@@ -375,7 +375,7 @@ func calculateEnvironmentMetrics(data *matrix.Matrix, logger *logging.Logger, va
 			Msg("sensor with *hr units")
 	}
 
-	return []*matrix.Matrix{myData}, nil
+	return []*matrix.Matrix{myData}
 }
 
 func NewSensor(p *plugin.AbstractPlugin) plugin.Plugin {
@@ -422,19 +422,20 @@ func (my *Sensor) Init() error {
 	return nil
 }
 
-func (my *Sensor) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) {
+func (my *Sensor) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) {
 	data := dataMap[my.Object]
 	// Purge and reset data
 	my.data.PurgeInstances()
 	my.data.Reset()
+	my.client.Metadata.Reset()
 
 	// Set all global labels if they don't already exist
 	my.data.SetGlobalLabels(data.GetGlobalLabels())
 
 	// Collect chassis fru show, so we can determine if a controller's PSUs are shared or not
 	nodeToNumNode, err := collectChassisFRU(my.client, my.Logger)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	if len(nodeToNumNode) == 0 {
 		my.Logger.Debug().Msg("No chassis field replaceable units found")
@@ -444,5 +445,7 @@ func (my *Sensor) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, erro
 	if my.Parent == "Rest" {
 		valueKey = restValueKey
 	}
-	return calculateEnvironmentMetrics(data, my.Logger, valueKey, my.data, nodeToNumNode)
+	metrics := calculateEnvironmentMetrics(data, my.Logger, valueKey, my.data, nodeToNumNode)
+
+	return metrics, my.client.Metadata, nil
 }

cmd/collectors/rest/plugins/certificate/certificate.go

@@ -13,6 +13,7 @@ import (
 	"github.com/netapp/harvest/v2/pkg/conf"
 	ontap "github.com/netapp/harvest/v2/pkg/errs"
 	"github.com/netapp/harvest/v2/pkg/matrix"
+	"github.com/netapp/harvest/v2/pkg/util"
 	"github.com/tidwall/gjson"
 	"time"
 )
@@ -51,14 +52,15 @@ func (my *Certificate) Init() error {
 	return nil
 }
 
-func (my *Certificate) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) {
+func (my *Certificate) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) {
 
 	var (
 		adminVserver       string
 		adminVserverSerial string
 		err                error
 	)
 	data := dataMap[my.Object]
+	my.client.Metadata.Reset()
 
 	if my.currentVal >= my.PluginInvocationRate {
 		my.currentVal = 0
@@ -70,7 +72,7 @@ func (my *Certificate) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix,
 			} else {
 				my.Logger.Error().Err(err).Msg("Failed to collect admin SVM")
 			}
-			return nil, nil
+			return nil, nil, nil
 		}
 
 		// invoke private ssl cli rest and get the admin SVM's serial number
@@ -80,7 +82,7 @@ func (my *Certificate) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix,
 			} else {
 				my.Logger.Error().Msg("Failed to collect admin SVM's serial number")
 			}
-			return nil, nil
+			return nil, nil, nil
 		}
 
 		// update certificate instance based on admin vaserver serial
@@ -102,7 +104,7 @@ func (my *Certificate) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix,
 	}
 
 	my.currentVal++
-	return nil, nil
+	return nil, my.client.Metadata, nil
 }
 
 func (my *Certificate) setCertificateIssuerType(instance *matrix.Instance) {

cmd/collectors/rest/plugins/disk/disk.go

@@ -5,6 +5,7 @@ package disk
 import (
 	"github.com/netapp/harvest/v2/cmd/poller/plugin"
 	"github.com/netapp/harvest/v2/pkg/matrix"
+	"github.com/netapp/harvest/v2/pkg/util"
 )
 
 type Disk struct {
@@ -15,7 +16,7 @@ func New(p *plugin.AbstractPlugin) plugin.Plugin {
 	return &Disk{AbstractPlugin: p}
 }
 
-func (d *Disk) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) {
+func (d *Disk) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) {
 	data := dataMap[d.Object]
 	for _, instance := range data.GetInstances() {
 
@@ -35,5 +36,5 @@ func (d *Disk) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error)
 
 	}
 
-	return nil, nil
+	return nil, nil, nil
 }

cmd/collectors/rest/plugins/health/health.go

@@ -9,6 +9,7 @@ import (
 	"github.com/netapp/harvest/v2/pkg/conf"
 	"github.com/netapp/harvest/v2/pkg/errs"
 	"github.com/netapp/harvest/v2/pkg/matrix"
+	"github.com/netapp/harvest/v2/pkg/util"
 	"github.com/tidwall/gjson"
 	"strconv"
 	"time"
@@ -97,35 +98,36 @@ func (h *Health) initMatrix(name string) error {
 	return nil
 }
 
-func (h *Health) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) {
+func (h *Health) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) {
 	data := dataMap[h.Object]
+	h.client.Metadata.Reset()
 	clusterVersion := h.client.Cluster().GetVersion()
 	ontapVersion, err := goversion.NewVersion(clusterVersion)
 	if err != nil {
 		h.Logger.Error().Err(err).
 			Str("version", clusterVersion).
 			Msg("Failed to parse version")
-		return nil, nil
+		return nil, nil, nil
 	}
 	version96 := "9.6"
 	version96After, err := goversion.NewVersion(version96)
 	if err != nil {
 		h.Logger.Error().Err(err).
 			Str("version", version96).
 			Msg("Failed to parse version")
-		return nil, nil
+		return nil, nil, nil
 	}
 
 	if ontapVersion.LessThan(version96After) {
-		return nil, nil
+		return nil, nil, nil
 	}
 
 	// Purge and reset data
 	// remove all metrics as analytics label may change over time
 	err = h.initAllMatrix()
 	if err != nil {
 		h.Logger.Warn().Err(err).Msg("error while init matrix")
-		return nil, err
+		return nil, nil, err
 	}
 	for k := range h.data {
 		// Set all global labels if already not exist
@@ -149,7 +151,7 @@ func (h *Health) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error
 	for _, value := range h.data {
 		result = append(result, value)
 	}
-	return result, nil
+	return result, h.client.Metadata, nil
 }
 
 func (h *Health) collectLicenseAlerts() {

cmd/collectors/rest/plugins/metroclustercheck/metroclustercheck.go

@@ -4,6 +4,7 @@ import (
 	"github.com/netapp/harvest/v2/cmd/poller/plugin"
 	"github.com/netapp/harvest/v2/pkg/matrix"
 	"github.com/netapp/harvest/v2/pkg/tree/node"
+	"github.com/netapp/harvest/v2/pkg/util"
 	"github.com/tidwall/gjson"
 )
 
@@ -42,7 +43,7 @@ func (m *MetroclusterCheck) Init() error {
 	return nil
 }
 
-func (m *MetroclusterCheck) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) {
+func (m *MetroclusterCheck) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) {
 	// Purge and reset data
 	m.data.PurgeInstances()
 	m.data.Reset()
@@ -59,7 +60,7 @@ func (m *MetroclusterCheck) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Ma
 		m.update(instance.GetLabel("volume"), "volume")
 	}
 
-	return []*matrix.Matrix{m.data}, nil
+	return []*matrix.Matrix{m.data}, nil, nil
 }
 
 func (m *MetroclusterCheck) update(objectInfo string, object string) {

cmd/collectors/rest/plugins/netroute/netroute.go

@@ -8,6 +8,7 @@ import (
 	"github.com/netapp/harvest/v2/cmd/poller/plugin"
 	"github.com/netapp/harvest/v2/pkg/matrix"
 	"github.com/netapp/harvest/v2/pkg/tree/node"
+	"github.com/netapp/harvest/v2/pkg/util"
 	"github.com/tidwall/gjson"
 	"strconv"
 	"strings"
@@ -57,7 +58,7 @@ func (n *NetRoute) Init() error {
 	return nil
 }
 
-func (n *NetRoute) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) {
+func (n *NetRoute) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) {
 
 	data := dataMap[n.Object]
 	// Purge and reset data
@@ -82,7 +83,7 @@ func (n *NetRoute) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, err
 				interfaceInstance, err := n.data.NewInstance(index)
 				if err != nil {
 					n.Logger.Error().Err(err).Str("add instance failed for instance key", key).Send()
-					return nil, err
+					return nil, nil, err
 				}
 
 				for _, l := range instanceLabels {
@@ -97,5 +98,5 @@ func (n *NetRoute) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, err
 		}
 	}
 
-	return []*matrix.Matrix{n.data}, nil
+	return []*matrix.Matrix{n.data}, nil, nil
 }

cmd/collectors/rest/plugins/ontaps3service/ontaps3service.go

@@ -11,6 +11,7 @@ import (
 	"github.com/netapp/harvest/v2/pkg/conf"
 	"github.com/netapp/harvest/v2/pkg/errs"
 	"github.com/netapp/harvest/v2/pkg/matrix"
+	"github.com/netapp/harvest/v2/pkg/util"
 	"github.com/tidwall/gjson"
 	"strings"
 	"time"
@@ -54,7 +55,7 @@ func (o *OntapS3Service) Init() error {
 	return nil
 }
 
-func (o *OntapS3Service) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) {
+func (o *OntapS3Service) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) {
 	var (
 		result      []gjson.Result
 		err         error
@@ -65,6 +66,7 @@ func (o *OntapS3Service) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matri
 	// reset svmToS3serverMap map
 	svmToURLMap = make(map[string][]string)
 	data := dataMap[o.Object]
+	o.client.Metadata.Reset()
 
 	fields := []string{"svm.name", "name", "is_http_enabled", "is_https_enabled", "secure_port", "port"}
 	href := rest.NewHrefBuilder().
@@ -73,14 +75,14 @@ func (o *OntapS3Service) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matri
 		Build()
 
 	if result, err = collectors.InvokeRestCall(o.client, href, o.Logger); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	// Iterate over services API response
 	for _, ontaps3Service := range result {
 		if !ontaps3Service.IsObject() {
 			o.Logger.Error().Str("type", ontaps3Service.Type.String()).Msg("Ontap S3 Service is not an object, skipping")
-			return nil, errs.New(errs.ErrNoInstance, "Ontap S3 Service is not an object")
+			return nil, nil, errs.New(errs.ErrNoInstance, "Ontap S3 Service is not an object")
 		}
 		s3ServerName := ontaps3Service.Get("name").String()
 		svm := ontaps3Service.Get("svm.name").String()
@@ -122,5 +124,5 @@ func (o *OntapS3Service) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matri
 		ontapS3.SetLabel("url", strings.Join(urlValue, ","))
 	}
 
-	return nil, nil
+	return nil, o.client.Metadata, nil
 }

cmd/collectors/rest/plugins/qospolicyadaptive/qospolicyadaptive.go

@@ -4,6 +4,7 @@ import (
 	"github.com/netapp/harvest/v2/cmd/collectors/zapi/plugins/qospolicyfixed"
 	"github.com/netapp/harvest/v2/cmd/poller/plugin"
 	"github.com/netapp/harvest/v2/pkg/matrix"
+	"github.com/netapp/harvest/v2/pkg/util"
 )
 
 var metrics = []string{
@@ -20,7 +21,7 @@ func New(p *plugin.AbstractPlugin) plugin.Plugin {
 	return &QosPolicyAdaptive{AbstractPlugin: p}
 }
 
-func (p *QosPolicyAdaptive) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) {
+func (p *QosPolicyAdaptive) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) {
 	data := dataMap[p.Object]
 
 	// create metrics
@@ -37,7 +38,7 @@ func (p *QosPolicyAdaptive) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Ma
 		p.setIOPs(data, instance, "peak_iops")
 	}
 
-	return nil, nil
+	return nil, nil, nil
 }
 
 func (p *QosPolicyAdaptive) setIOPs(data *matrix.Matrix, instance *matrix.Instance, labelName string) {

cmd/collectors/rest/plugins/qospolicyfixed/qospolicyfixed.go

@@ -4,6 +4,7 @@ import (
 	"github.com/netapp/harvest/v2/cmd/collectors/zapi/plugins/qospolicyfixed"
 	"github.com/netapp/harvest/v2/cmd/poller/plugin"
 	"github.com/netapp/harvest/v2/pkg/matrix"
+	"github.com/netapp/harvest/v2/pkg/util"
 	"strings"
 )
 
@@ -22,23 +23,23 @@ func New(p *plugin.AbstractPlugin) plugin.Plugin {
 	return &QosPolicyFixed{AbstractPlugin: p}
 }
 
-func (p *QosPolicyFixed) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) {
+func (p *QosPolicyFixed) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) {
 	data := dataMap[p.Object]
 
 	// create metrics
 	for _, k := range metrics {
 		err := matrix.CreateMetric(k, data)
 		if err != nil {
 			p.Logger.Error().Err(err).Str("key", k).Msg("error while creating metric")
-			return nil, err
+			return nil, nil, err
 		}
 	}
 
 	for _, instance := range data.GetInstances() {
 		p.setFixed(data, instance)
 	}
 
-	return nil, nil
+	return nil, nil, nil
 }
 
 func (p *QosPolicyFixed) setFixed(data *matrix.Matrix, instance *matrix.Instance) {