From 6b359313c3a7cf0963cdd04824050c9c4975e132 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Wed, 5 Jul 2023 14:30:39 +0530 Subject: [PATCH 01/20] feat: Change Log Detection in Harvest --- .../zapi/plugins/aggregate/aggregate.go | 4 +- cmd/poller/collector/helpers.go | 5 + cmd/poller/plugin/changelog/change_log.go | 287 ++++++++++++++++++ .../plugin/changelog/change_log_helper.go | 98 ++++++ .../plugin/changelog/change_log_test.go | 180 +++++++++++ cmd/tools/template/template_test.go | 1 + conf/rest/9.12.0/disk.yaml | 4 +- conf/rest/9.12.0/lif.yaml | 1 + conf/rest/9.12.0/node.yaml | 2 + conf/rest/9.12.0/shelf.yaml | 1 + conf/rest/9.12.0/svm.yaml | 2 + conf/rest/9.12.0/volume.yaml | 2 + conf/rest/9.12.0/volume_analytics.yaml | 2 +- conf/zapi/cdot/9.8.0/aggr.yaml | 13 +- conf/zapi/cdot/9.8.0/disk.yaml | 2 +- conf/zapi/cdot/9.8.0/lif.yaml | 1 + conf/zapi/cdot/9.8.0/node.yaml | 2 + conf/zapi/cdot/9.8.0/shelf.yaml | 4 +- conf/zapi/cdot/9.8.0/svm.yaml | 4 +- conf/zapi/cdot/9.8.0/volume.yaml | 3 +- pkg/dict/dict.go | 19 +- 21 files changed, 619 insertions(+), 18 deletions(-) create mode 100644 cmd/poller/plugin/changelog/change_log.go create mode 100644 cmd/poller/plugin/changelog/change_log_helper.go create mode 100644 cmd/poller/plugin/changelog/change_log_test.go diff --git a/cmd/collectors/zapi/plugins/aggregate/aggregate.go b/cmd/collectors/zapi/plugins/aggregate/aggregate.go index b93f55e27..5f9deaff5 100644 --- a/cmd/collectors/zapi/plugins/aggregate/aggregate.go +++ b/cmd/collectors/zapi/plugins/aggregate/aggregate.go @@ -57,11 +57,11 @@ func (a *Aggregate) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er // update aggregate instance label with cloud stores info if len(a.aggrCloudStoresMap) > 0 { - for aggrUUID, aggr := range data.GetInstances() { + for uuid, aggr := range data.GetInstances() { if !aggr.IsExportable() { continue } - aggr.SetLabel("cloud_stores", strings.Join(a.aggrCloudStoresMap[aggrUUID], ",")) + aggr.SetLabel("cloud_stores", strings.Join(a.aggrCloudStoresMap[uuid], ",")) } } return nil, nil diff --git a/cmd/poller/collector/helpers.go b/cmd/poller/collector/helpers.go index 9286d8240..ca7cdf5a0 100644 --- a/cmd/poller/collector/helpers.go +++ b/cmd/poller/collector/helpers.go @@ -19,6 +19,7 @@ import ( "github.com/hashicorp/go-version" "github.com/netapp/harvest/v2/cmd/poller/plugin" "github.com/netapp/harvest/v2/cmd/poller/plugin/aggregator" + "github.com/netapp/harvest/v2/cmd/poller/plugin/changelog" "github.com/netapp/harvest/v2/cmd/poller/plugin/labelagent" "github.com/netapp/harvest/v2/cmd/poller/plugin/max" "github.com/netapp/harvest/v2/cmd/poller/plugin/metricagent" @@ -211,5 +212,9 @@ func GetBuiltinPlugin(name string, abc *plugin.AbstractPlugin) plugin.Plugin { return metricagent.New(abc) } + if name == "ChangeLog" { + return changelog.New(abc) + } + return nil } diff --git a/cmd/poller/plugin/changelog/change_log.go b/cmd/poller/plugin/changelog/change_log.go new file mode 100644 index 000000000..f1108bb49 --- /dev/null +++ b/cmd/poller/plugin/changelog/change_log.go @@ -0,0 +1,287 @@ +package changelog + +import ( + "github.com/netapp/harvest/v2/cmd/poller/plugin" + "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/set" + "github.com/netapp/harvest/v2/pkg/tree/yaml" +) + +// Constants for ChangeLog metrics and labels +const ( + changeLog = "change" + objectTpeLabel = "object_type" + changeTypeLabel = "change_type" + create = "create" + modify = "modify" + del = "delete" + labelName = "label_name" + oldLabelValue = "old_label_value" + newLabelValue = "new_label_value" +) + +// Metrics to be used in ChangeLog +var metrics = []string{ + "log", +} + +// ChangeLog represents the main structure of the ChangeLog plugin +type ChangeLog struct { + *plugin.AbstractPlugin + matrixName string + previousData map[string]*matrix.Matrix + changeLogMap map[string]*matrix.Matrix + changeLogConfig map[string]Entry +} + +// Change represents a single change entry in the ChangeLog +type Change struct { + key string + object string + changeType string + labels map[string]string + labelName string + oldLabelValue string + newLabelValue string +} + +// New initializes a new instance of the ChangeLog plugin +func New(p *plugin.AbstractPlugin) plugin.Plugin { + return &ChangeLog{AbstractPlugin: p} +} + +// Init initializes the ChangeLog plugin +func (c *ChangeLog) Init() error { + + // Initialize the abstract plugin + if err := c.AbstractPlugin.Init(); err != nil { + return err + } + + // Initialize the changeLogMap + c.changeLogMap = make(map[string]*matrix.Matrix) + + object := c.ParentParams.GetChildS("object") + c.matrixName = object.GetContentS() + "_" + changeLog + + // Initialize the changeLogMatrix + if err := c.initMatrix(); err != nil { + return err + } + + // Populate the ChangeLog configuration + if err := c.populateChangeLogConfig(); err != nil { + return err + } + + return nil +} + +// populateChangeLogConfig populates the ChangeLog configuration from the plugin parameters +func (c *ChangeLog) populateChangeLogConfig() error { + var err error + dump, err := yaml.Dump(c.Params) + if err != nil { + return err + } + object := c.ParentParams.GetChildS("object") + + c.changeLogConfig, err = getChangeLogConfig(object.GetContentS(), string(dump), c.Logger) + if err != nil { + return err + } + return nil +} + +// initMatrix initializes a new matrix with the given name +func (c *ChangeLog) initMatrix() error { + c.changeLogMap[c.matrixName] = matrix.New(c.Parent+c.matrixName, changeLog, c.matrixName) + for _, changeLogMatrix := range c.changeLogMap { + changeLogMatrix.SetExportOptions(matrix.DefaultExportOptions()) + } + for _, k := range metrics { + err := matrix.CreateMetric(k, c.changeLogMap[c.matrixName]) + if err != nil { + c.Logger.Warn().Err(err).Str("key", k).Msg("error while creating metric") + return err + } + } + return nil +} + +// Run processes the data and generates ChangeLog instances +func (c *ChangeLog) Run(data map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { + // Purge and reset data + // remove all metrics as analytics label may change over time + err := c.initMatrix() + if err != nil { + c.Logger.Warn().Err(err).Msg("error while init matrix") + return nil, err + } + + // if this is first poll + if c.previousData == nil { + c.copyPreviousData(data) + return nil, nil + } + + changeMat := c.changeLogMap[c.matrixName] + + changeMat.SetGlobalLabels(data[c.Object].GetGlobalLabels()) + for objectKey, objectMatrix := range data { + object := objectMatrix.Object + if _, ok := c.changeLogConfig[object]; !ok { + c.Logger.Warn().Str("object", object).Msg("ChangeLog is not supported. Missing correct configuration") + return nil, nil + } + + prevMat := c.previousData[objectKey] + oldInstances := set.New() + prevInstancesUUIDKey := make(map[string]string) + for key, prevInstance := range prevMat.GetInstances() { + uuid := prevInstance.GetLabel("uuid") + if uuid == "" { + c.Logger.Warn().Str("object", object).Str("key", key).Msg("missing uuid") + continue + } + prevInstancesUUIDKey[uuid] = key + oldInstances.Add(key) + } + // check if object exists + if prev, ok := c.previousData[objectKey]; ok { + // loop over current instances + for key, instance := range objectMatrix.GetInstances() { + uuid := instance.GetLabel("uuid") + if uuid == "" { + c.Logger.Warn().Str("object", object).Str("key", key).Msg("missing uuid. ChangeLog is not supported") + continue + } + + prevKey := prevInstancesUUIDKey[uuid] + if prevKey != "" { + // instance already in cache + oldInstances.Remove(prevKey) + } + + prevInstance := prev.GetInstance(prevKey) + + if prevInstance == nil { + //instance created + change := &Change{ + key: uuid + "_" + object, + object: object, + changeType: create, + labels: make(map[string]string), + } + for _, l := range c.changeLogConfig[object].PublishLabels { + labelValue := instance.GetLabel(l) + if labelValue == "" { + c.Logger.Warn().Str("object", object).Str("key", key).Str("label", l).Msg("Missing label") + } else { + change.labels[l] = instance.GetLabel(l) + } + } + c.createChangeLogInstance(changeMat, change) + } else { + // check for any modification + cur, old := instance.GetLabels().CompareLabels(prevInstance.GetLabels(), c.changeLogConfig[object].TrackLabels) + if !cur.IsEmpty() { + for currentLabel, newLabel := range cur.Iter() { + change := &Change{ + key: uuid + "_" + object + "_" + currentLabel, + object: object, + changeType: modify, + labels: make(map[string]string), + labelName: currentLabel, + oldLabelValue: old.Get(currentLabel), + newLabelValue: newLabel, + } + for _, l := range c.changeLogConfig[object].PublishLabels { + labelValue := instance.GetLabel(l) + if labelValue == "" { + c.Logger.Warn().Str("object", object).Str("key", key).Str("label", l).Msg("Missing label") + } else { + change.labels[l] = instance.GetLabel(l) + } + } + // add changed labelname and its old, new value + change.labels[labelName] = currentLabel + change.labels[oldLabelValue] = old.Get(currentLabel) + change.labels[newLabelValue] = newLabelValue + c.createChangeLogInstance(changeMat, change) + } + } + } + } + // create deleted instances change_log + for key := range oldInstances.Iter() { + prevInstance := prevMat.GetInstance(key) + uuid := prevInstance.GetLabel("uuid") + if uuid == "" { + c.Logger.Warn().Str("object", object).Str("key", key).Msg("missing uuid. ChangeLog is not supported") + continue + } + if prevInstance != nil { + change := &Change{ + key: uuid + "_" + object, + object: object, + changeType: del, + labels: make(map[string]string), + } + for _, l := range c.changeLogConfig[object].PublishLabels { + labelValue := prevInstance.GetLabel(l) + if labelValue == "" { + c.Logger.Warn().Str("object", object).Str("key", key).Str("label", l).Msg("Missing label") + } else { + change.labels[l] = prevInstance.GetLabel(l) + } + } + c.createChangeLogInstance(changeMat, change) + } else { + c.Logger.Warn().Str("object", object).Str("key", key).Msg("missing instance") + } + } + } + } + + var matricesArray []*matrix.Matrix + matricesArray = append(matricesArray, changeMat) + + c.copyPreviousData(data) + + return matricesArray, nil +} + +// copyPreviousData creates a copy of the previous data for comparison +func (c *ChangeLog) copyPreviousData(data map[string]*matrix.Matrix) { + c.previousData = make(map[string]*matrix.Matrix) + for k, v := range data { + c.previousData[k] = v.Clone(matrix.With{Data: true, Metrics: false, Instances: true, ExportInstances: false}) + } +} + +// createChangeLogInstance creates a new ChangeLog instance with the given change data +func (c *ChangeLog) createChangeLogInstance(mat *matrix.Matrix, change *Change) { + cInstance, err := mat.NewInstance(change.key) + if err != nil { + c.Logger.Warn().Str("object", change.object).Str("key", change.key).Msg("error while creating instance") + return + } + // copy keys + cInstance.SetLabel(objectTpeLabel, change.object) + cInstance.SetLabel(changeTypeLabel, change.changeType) + for k, v := range change.labels { + cInstance.SetLabel(k, v) + } + m := mat.GetMetric("log") + if m == nil { + if m, err = mat.NewMetricFloat64("log"); err != nil { + c.Logger.Warn().Err(err).Str("key", "alerts").Msg("error while creating metric") + return + } + } + if err = m.SetValueFloat64(cInstance, 1); err != nil { + c.Logger.Error().Err(err).Str("metric", "alerts").Msg("Unable to set value on metric") + return + } +} diff --git a/cmd/poller/plugin/changelog/change_log_helper.go b/cmd/poller/plugin/changelog/change_log_helper.go new file mode 100644 index 000000000..5b11edfb3 --- /dev/null +++ b/cmd/poller/plugin/changelog/change_log_helper.go @@ -0,0 +1,98 @@ +package changelog + +import ( + "github.com/netapp/harvest/v2/pkg/logging" + "gopkg.in/yaml.v3" + "strings" +) + +// Entry represents a single ChangeLog entry +type Entry struct { + Object string `yaml:"object"` + TrackLabels []string `yaml:"track_labels"` + PublishLabels []string `yaml:"publish_labels"` +} + +// Config represents the structure of the ChangeLog configuration +type Config struct { + ChangeLogs []Entry `yaml:"ChangeLog"` +} + +// defaultChangeLogTemplate is the default ChangeLog configuration +const defaultChangeLogTemplate = ` +ChangeLog: + - object: svm + track_labels: + - svm + - state + - type + publish_labels: + - svm + - object: node + track_labels: + - node + - location + publish_labels: + - node + - object: volume + track_labels: + - node + - volume + - svm + - style + - type + publish_labels: + - volume + - svm +` + +// getChangeLogConfig returns a map of ChangeLog entries for the given object +func getChangeLogConfig(object string, s string, logger *logging.Logger) (map[string]Entry, error) { + var config Config + temp := defaultChangeLogTemplate + if s != "" { + temp = preprocessOverwrite(object, s) + err := yaml.Unmarshal([]byte(temp), &config) + if err != nil { + logger.Warn().Err(err).Str("template", s).Msg("failed to parse changelog dsl. Trying default") + temp = defaultChangeLogTemplate + } + } + err := yaml.Unmarshal([]byte(temp), &config) + if err != nil { + return nil, err + } + + objectMap := make(map[string]Entry) + for _, obj := range config.ChangeLogs { + if obj.Object == object { + objectMap[obj.Object] = obj + } + } + + return objectMap, nil +} + +// preprocessOverwrite updates the ChangeLog configuration by adding the given object and its properties +func preprocessOverwrite(object string, configStr string) string { + // Split the YAML content into lines + lines := strings.Split(configStr, "\n") + + // Add four spaces to indent each line, making them at the same level as object + indentedLines := make([]string, len(lines)) + for i, line := range lines { + indentedLines[i] = " " + line + } + + // Join the indented lines back into a single string + indentedYaml := strings.Join(indentedLines, "\n") + + // Add the ChangeLog prefix + prefix := ` +ChangeLog: + - object: ` + object + + newYaml := strings.Join([]string{prefix, indentedYaml}, "\n") + return newYaml + +} diff --git a/cmd/poller/plugin/changelog/change_log_test.go b/cmd/poller/plugin/changelog/change_log_test.go new file mode 100644 index 000000000..07c898e66 --- /dev/null +++ b/cmd/poller/plugin/changelog/change_log_test.go @@ -0,0 +1,180 @@ +package changelog + +import ( + "github.com/netapp/harvest/v2/cmd/poller/options" + "github.com/netapp/harvest/v2/cmd/poller/plugin" + "github.com/netapp/harvest/v2/pkg/matrix" + "github.com/netapp/harvest/v2/pkg/tree/node" + "testing" +) + +func newChangeLog() *ChangeLog { + + params := node.NewS("ChangeLog") + parentParams := node.NewS("parent") + parentParams.NewChildS("object", "svm") + + abc := plugin.New("Test", nil, params, parentParams, "", nil) + p := &ChangeLog{AbstractPlugin: abc} + p.Options = &options.Options{ + Poller: "Test", + } + p.Object = "svm" + + if err := p.Init(); err != nil { + panic(err) + } + return p +} + +func TestChangeLogModified(t *testing.T) { + p := newChangeLog() + m := matrix.New("TestChangeLog", "svm", "svm") + data := map[string]*matrix.Matrix{ + "svm": m, + } + instance, _ := m.NewInstance("0") + instance.SetLabel("uuid", "u1") + instance.SetLabel("svm", "s1") + instance.SetLabel("type", "t1") + + _, _ = p.Run(data) + + m1 := matrix.New("TestChangeLog", "svm", "svm") + data1 := map[string]*matrix.Matrix{ + "svm": m1, + } + instance1, _ := m1.NewInstance("0") + instance1.SetLabel("uuid", "u1") + instance1.SetLabel("svm", "s2") + instance1.SetLabel("type", "t2") + + o, _ := p.Run(data1) + + if len(o) == 1 { + cl := o[0] + if len(cl.GetInstances()) != 2 { + t.Errorf("ChangeLog instances size expected %d, actual %d", 2, len(cl.GetInstances())) + } else { + for _, i := range cl.GetInstances() { + if i.GetLabel(changeTypeLabel) != modify { + t.Errorf("ChangeLog %s label expected %s, actual %s", changeTypeLabel, modify, i.GetLabel(changeTypeLabel)) + } + } + } + } else { + t.Error("ChangeLog slice size is wrong") + } +} + +func TestChangeLogCreated(t *testing.T) { + p := newChangeLog() + m := matrix.New("TestChangeLog", "svm", "svm") + data := map[string]*matrix.Matrix{ + "svm": m, + } + instance, _ := m.NewInstance("0") + instance.SetLabel("uuid", "u1") + instance.SetLabel("svm", "s1") + instance.SetLabel("type", "t1") + + _, _ = p.Run(data) + + m1 := matrix.New("TestChangeLog", "svm", "svm") + data1 := map[string]*matrix.Matrix{ + "svm": m1, + } + instance1, _ := m1.NewInstance("1") + instance1.SetLabel("uuid", "u2") + instance1.SetLabel("svm", "s2") + instance1.SetLabel("type", "t2") + + instance2, _ := m1.NewInstance("0") + instance2.SetLabel("uuid", "u1") + instance2.SetLabel("svm", "s1") + instance2.SetLabel("type", "t1") + + o, _ := p.Run(data1) + + if len(o) == 1 { + cl := o[0] + if len(cl.GetInstances()) != 1 { + t.Errorf("ChangeLog instances size expected %d, actual %d", 1, len(cl.GetInstances())) + } else { + for _, i := range cl.GetInstances() { + if i.GetLabel(changeTypeLabel) != create { + t.Errorf("ChangeLog %s label expected %s, actual %s", changeTypeLabel, create, i.GetLabel(changeTypeLabel)) + } + } + } + } else { + t.Error("ChangeLog slice size is wrong") + } +} + +func TestChangeLogDeleted(t *testing.T) { + p := newChangeLog() + m := matrix.New("TestChangeLog", "svm", "svm") + data := map[string]*matrix.Matrix{ + "svm": m, + } + instance, _ := m.NewInstance("0") + instance.SetLabel("uuid", "u1") + instance.SetLabel("svm", "s1") + instance.SetLabel("type", "t1") + + _, _ = p.Run(data) + + m1 := matrix.New("TestChangeLog", "svm", "svm") + data1 := map[string]*matrix.Matrix{ + "svm": m1, + } + + o, _ := p.Run(data1) + + if len(o) == 1 { + cl := o[0] + if len(cl.GetInstances()) != 1 { + t.Errorf("ChangeLog instances size expected %d, actual %d", 1, len(cl.GetInstances())) + } else { + for _, i := range cl.GetInstances() { + if i.GetLabel(changeTypeLabel) != del { + t.Errorf("ChangeLog %s label expected %s, actual %s", changeTypeLabel, del, i.GetLabel(changeTypeLabel)) + } + } + } + } else { + t.Error("ChangeLog slice size is wrong") + } +} + +func TestChangeLogUnsupported(t *testing.T) { + p := newChangeLog() + m := matrix.New("TestChangeLog", "lun", "lun") + data := map[string]*matrix.Matrix{ + "svm": m, + } + instance, _ := m.NewInstance("0") + instance.SetLabel("uuid", "u1") + instance.SetLabel("lun", "l1") + + _, _ = p.Run(data) + + m1 := matrix.New("TestChangeLog", "lun", "lun") + data1 := map[string]*matrix.Matrix{ + "svm": m1, + } + instance1, _ := m1.NewInstance("1") + instance1.SetLabel("uuid", "u2") + instance1.SetLabel("lun", "l2") + + instance2, _ := m1.NewInstance("0") + instance2.SetLabel("uuid", "u1") + instance2.SetLabel("lun", "l3") + + o, _ := p.Run(data1) + + if len(o) != 0 { + t.Errorf("ChangeLog matric size expected %d, actual %d", 0, len(o)) + } +} diff --git a/cmd/tools/template/template_test.go b/cmd/tools/template/template_test.go index 2181fb698..9898cb2a3 100644 --- a/cmd/tools/template/template_test.go +++ b/cmd/tools/template/template_test.go @@ -595,6 +595,7 @@ func findCustomPlugins(path string, template *node.Node, model *TemplateModel) e "Aggregator": true, "Max": true, "Tenant": true, + "ChangeLog": true, } for _, child := range plug[0].Children { name := child.GetNameS() diff --git a/conf/rest/9.12.0/disk.yaml b/conf/rest/9.12.0/disk.yaml index 01f27b96b..792df660a 100644 --- a/conf/rest/9.12.0/disk.yaml +++ b/conf/rest/9.12.0/disk.yaml @@ -4,7 +4,7 @@ query: api/storage/disks object: disk counters: - - ^^uid + - ^^uid => uuid - ^bay => shelf_bay - ^container_type - ^home_node.name => owner_node @@ -25,7 +25,7 @@ counters: endpoints: - query: api/private/cli/disk counters: - - ^^uid => uid + - ^^uid => uuid - ^type - disk_io_kbps_total => stats_io_kbps - sectors_read => stats_sectors_read diff --git a/conf/rest/9.12.0/lif.yaml b/conf/rest/9.12.0/lif.yaml index badc19c87..a348ee2c4 100644 --- a/conf/rest/9.12.0/lif.yaml +++ b/conf/rest/9.12.0/lif.yaml @@ -15,6 +15,7 @@ counters: - ^services => services - ^state => status - ^subnet.name => subnet + - ^uuid => uuid endpoints: - query: api/private/cli/network/interface diff --git a/conf/rest/9.12.0/node.yaml b/conf/rest/9.12.0/node.yaml index 470882b63..5fdb59026 100644 --- a/conf/rest/9.12.0/node.yaml +++ b/conf/rest/9.12.0/node.yaml @@ -12,6 +12,7 @@ counters: - ^model - ^serial_number => serial - ^state + - ^uuid - ^version.full => version - controller.failed_fan.count => failed_fan - controller.failed_power_supply.count => failed_power @@ -34,6 +35,7 @@ plugins: - new_status healthy true up `0` join: - warnings `; ` failed_fan_message,failed_power_message,over_temperature + - ChangeLog export_options: instance_keys: diff --git a/conf/rest/9.12.0/shelf.yaml b/conf/rest/9.12.0/shelf.yaml index d54272827..7d9a569ac 100644 --- a/conf/rest/9.12.0/shelf.yaml +++ b/conf/rest/9.12.0/shelf.yaml @@ -10,6 +10,7 @@ counters: - ^module_type => module_type - ^name => shelf - ^state => state + - ^uid => uuid - disk_count => disk_count endpoints: diff --git a/conf/rest/9.12.0/svm.yaml b/conf/rest/9.12.0/svm.yaml index d164305d5..341e143d3 100644 --- a/conf/rest/9.12.0/svm.yaml +++ b/conf/rest/9.12.0/svm.yaml @@ -9,6 +9,7 @@ counters: - ^anti_ransomware_default_volume_state => anti_ransomware_state - ^operational_state => state - ^type => type + - ^uuid => uuid endpoints: - query: api/svm/svms @@ -58,6 +59,7 @@ plugins: - new_status state online online `0` - ldap_signed ldap_session_security sign sign `0` - ldap_encrypted ldap_session_security seal seal `0` + - ChangeLog export_options: instance_keys: diff --git a/conf/rest/9.12.0/volume.yaml b/conf/rest/9.12.0/volume.yaml index 1c2ea5fcd..66ce60410 100644 --- a/conf/rest/9.12.0/volume.yaml +++ b/conf/rest/9.12.0/volume.yaml @@ -17,6 +17,7 @@ counters: - ^state => state - ^style => style - ^type => type + - ^uuid => uuid - autosize.grow_threshold => autosize_grow_threshold_percent - autosize.maximum => autosize_maximum_size - snapshot_count @@ -119,6 +120,7 @@ plugins: # - volume `MDV_CRS_.+` # # Exclude Metadata volumes, Audit volumes have a “MDV_aud_” prefix # - volume `MDV_aud_.+` + - ChangeLog export_options: instance_keys: diff --git a/conf/rest/9.12.0/volume_analytics.yaml b/conf/rest/9.12.0/volume_analytics.yaml index 339b4059e..a3c006c9f 100644 --- a/conf/rest/9.12.0/volume_analytics.yaml +++ b/conf/rest/9.12.0/volume_analytics.yaml @@ -3,7 +3,7 @@ query: api/storage/volumes object: volume counters: - - ^^uuid => instance_uuid + - ^^uuid => uuid - ^name => volume - ^svm.name => svm - filter: diff --git a/conf/zapi/cdot/9.8.0/aggr.yaml b/conf/zapi/cdot/9.8.0/aggr.yaml index f8ae35c7b..0563f30df 100644 --- a/conf/zapi/cdot/9.8.0/aggr.yaml +++ b/conf/zapi/cdot/9.8.0/aggr.yaml @@ -5,7 +5,7 @@ object: aggr counters: aggr-attributes: - - ^^aggregate-uuid + - ^^aggregate-uuid => uuid - ^aggregate-name => aggr - aggr-inode-attributes: - files-private-used @@ -63,11 +63,12 @@ counters: plugins: - Aggregate - LabelAgent: - exclude_equals: - - root_aggr `true` - # metric label zapi_value rest_value `default_value` - value_to_num: - - new_status state online online `0` + exclude_equals: + - root_aggr `true` + # metric label zapi_value rest_value `default_value` + value_to_num: + - new_status state online online `0` + - ChangeLog export_options: instance_keys: diff --git a/conf/zapi/cdot/9.8.0/disk.yaml b/conf/zapi/cdot/9.8.0/disk.yaml index ca861e4c7..ef8dfe92b 100644 --- a/conf/zapi/cdot/9.8.0/disk.yaml +++ b/conf/zapi/cdot/9.8.0/disk.yaml @@ -6,7 +6,7 @@ object: disk counters: storage-disk-info: - ^^disk-name => disk - - ^^disk-uid + - ^^disk-uid => uuid - disk-inventory-info: - ^disk-type => type - ^is-shared => shared diff --git a/conf/zapi/cdot/9.8.0/lif.yaml b/conf/zapi/cdot/9.8.0/lif.yaml index 79ed64091..7b6e3c021 100644 --- a/conf/zapi/cdot/9.8.0/lif.yaml +++ b/conf/zapi/cdot/9.8.0/lif.yaml @@ -13,6 +13,7 @@ counters: - ^home-port => home_port - ^ipspace => ipspace - ^is-home => is_home + - ^lif-uuid => uuid - ^operational-status => status - ^subnet-name => subnet - data-protocols: diff --git a/conf/zapi/cdot/9.8.0/node.yaml b/conf/zapi/cdot/9.8.0/node.yaml index daac41d55..2543b8c3e 100644 --- a/conf/zapi/cdot/9.8.0/node.yaml +++ b/conf/zapi/cdot/9.8.0/node.yaml @@ -25,6 +25,7 @@ counters: - ^node-location => location - ^node-model => model - ^node-serial-number => serial + - ^node-uuid => uuid # - ^node-storage-configuration => storage_configuration - ^node-vendor => vendor - ^product-version => version @@ -46,6 +47,7 @@ plugins: - new_status healthy true up `0` join: - warnings `; ` failed_fan_message,failed_power_message,over_temperature + - ChangeLog export_options: instance_keys: diff --git a/conf/zapi/cdot/9.8.0/shelf.yaml b/conf/zapi/cdot/9.8.0/shelf.yaml index a3656ea43..0578ab2d5 100644 --- a/conf/zapi/cdot/9.8.0/shelf.yaml +++ b/conf/zapi/cdot/9.8.0/shelf.yaml @@ -6,11 +6,11 @@ object: shelf counters: storage-shelf-info: - - ^^shelf-uid + - ^^shelf-uid => uuid - ^module-type - ^op-status - ^serial-number - - ^shelf => shelf + - ^shelf => shelf - ^shelf-model - ^state - ^vendor-name diff --git a/conf/zapi/cdot/9.8.0/svm.yaml b/conf/zapi/cdot/9.8.0/svm.yaml index d402738c9..737dbc3e3 100644 --- a/conf/zapi/cdot/9.8.0/svm.yaml +++ b/conf/zapi/cdot/9.8.0/svm.yaml @@ -5,7 +5,7 @@ object: svm counters: vserver-info: - - ^^uuid => svm_uuid + - ^^uuid => uuid - ^anti-ransomware-default-volume-state => anti_ransomware_state - ^state => state - ^vserver-name => svm @@ -29,6 +29,7 @@ plugins: - new_status state online online `0` - ldap_signed ldap_session_security sign sign `0` - ldap_encrypted ldap_session_security seal seal `0` + - ChangeLog export_options: instance_keys: @@ -55,4 +56,3 @@ export_options: - smb_signing_required - state - type - diff --git a/conf/zapi/cdot/9.8.0/volume.yaml b/conf/zapi/cdot/9.8.0/volume.yaml index a33add834..90aa6de3e 100644 --- a/conf/zapi/cdot/9.8.0/volume.yaml +++ b/conf/zapi/cdot/9.8.0/volume.yaml @@ -12,7 +12,7 @@ counters: - grow-threshold-percent - maximum-size - volume-id-attributes: - - ^^instance-uuid => instance_uuid + - ^^instance-uuid => uuid - ^containing-aggregate-uuid => aggrUuid - ^name => volume - ^owning-vserver-name => svm @@ -100,6 +100,7 @@ plugins: - svm_root root_volume `false` `No` - node_root root_volume `true` `Yes` - svm_root root_volume `true` `Yes` + - ChangeLog export_options: instance_keys: diff --git a/pkg/dict/dict.go b/pkg/dict/dict.go index 12136f9d4..449fd0cf2 100644 --- a/pkg/dict/dict.go +++ b/pkg/dict/dict.go @@ -4,7 +4,11 @@ package dict -import "strings" +import ( + "github.com/netapp/harvest/v2/pkg/util" + "reflect" + "strings" +) type Dict struct { dict map[string]string @@ -108,3 +112,16 @@ func (d *Dict) IsEmpty() bool { func (d *Dict) Size() int { return len(d.dict) } + +func (d *Dict) CompareLabels(prev *Dict, labels []string) (*Dict, *Dict) { + cur := New() + old := New() + for key, val1 := range d.dict { + val2, ok := prev.dict[key] + if util.Contains(labels, key) && (!ok || !reflect.DeepEqual(val1, val2)) { + cur.dict[key] = val1 + old.dict[key] = val2 + } + } + return cur, old +} From dafc8999891aa3205fdff8f947c09db9b2c5a3d6 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Wed, 5 Jul 2023 20:43:26 +0530 Subject: [PATCH 02/20] feat: address review comments --- cmd/poller/plugin/changelog/change_log.go | 72 +++++++++---------- .../plugin/changelog/change_log_helper.go | 8 +-- .../plugin/changelog/change_log_test.go | 12 ++-- 3 files changed, 46 insertions(+), 46 deletions(-) diff --git a/cmd/poller/plugin/changelog/change_log.go b/cmd/poller/plugin/changelog/change_log.go index f1108bb49..7495953de 100644 --- a/cmd/poller/plugin/changelog/change_log.go +++ b/cmd/poller/plugin/changelog/change_log.go @@ -9,15 +9,15 @@ import ( // Constants for ChangeLog metrics and labels const ( - changeLog = "change" - objectTpeLabel = "object_type" - changeTypeLabel = "change_type" - create = "create" - modify = "modify" - del = "delete" - labelName = "label_name" - oldLabelValue = "old_label_value" - newLabelValue = "new_label_value" + changeLog = "change" + objectLabel = "object" + opLabel = "op" + create = "create" + update = "update" + del = "delete" + track = "track" + oldLabelValue = "old_value" + newLabelValue = "new_value" ) // Metrics to be used in ChangeLog @@ -36,13 +36,13 @@ type ChangeLog struct { // Change represents a single change entry in the ChangeLog type Change struct { - key string - object string - changeType string - labels map[string]string - labelName string - oldLabelValue string - newLabelValue string + key string + object string + op string + labels map[string]string + track string + oldValue string + newValue string } // New initializes a new instance of the ChangeLog plugin @@ -168,10 +168,10 @@ func (c *ChangeLog) Run(data map[string]*matrix.Matrix) ([]*matrix.Matrix, error if prevInstance == nil { //instance created change := &Change{ - key: uuid + "_" + object, - object: object, - changeType: create, - labels: make(map[string]string), + key: uuid + "_" + object, + object: object, + op: create, + labels: make(map[string]string), } for _, l := range c.changeLogConfig[object].PublishLabels { labelValue := instance.GetLabel(l) @@ -184,17 +184,17 @@ func (c *ChangeLog) Run(data map[string]*matrix.Matrix) ([]*matrix.Matrix, error c.createChangeLogInstance(changeMat, change) } else { // check for any modification - cur, old := instance.GetLabels().CompareLabels(prevInstance.GetLabels(), c.changeLogConfig[object].TrackLabels) + cur, old := instance.GetLabels().CompareLabels(prevInstance.GetLabels(), c.changeLogConfig[object].Track) if !cur.IsEmpty() { for currentLabel, newLabel := range cur.Iter() { change := &Change{ - key: uuid + "_" + object + "_" + currentLabel, - object: object, - changeType: modify, - labels: make(map[string]string), - labelName: currentLabel, - oldLabelValue: old.Get(currentLabel), - newLabelValue: newLabel, + key: uuid + "_" + object + "_" + currentLabel, + object: object, + op: update, + labels: make(map[string]string), + track: currentLabel, + oldValue: old.Get(currentLabel), + newValue: newLabel, } for _, l := range c.changeLogConfig[object].PublishLabels { labelValue := instance.GetLabel(l) @@ -205,9 +205,9 @@ func (c *ChangeLog) Run(data map[string]*matrix.Matrix) ([]*matrix.Matrix, error } } // add changed labelname and its old, new value - change.labels[labelName] = currentLabel + change.labels[track] = currentLabel change.labels[oldLabelValue] = old.Get(currentLabel) - change.labels[newLabelValue] = newLabelValue + change.labels[newLabelValue] = newLabel c.createChangeLogInstance(changeMat, change) } } @@ -223,10 +223,10 @@ func (c *ChangeLog) Run(data map[string]*matrix.Matrix) ([]*matrix.Matrix, error } if prevInstance != nil { change := &Change{ - key: uuid + "_" + object, - object: object, - changeType: del, - labels: make(map[string]string), + key: uuid + "_" + object, + object: object, + op: del, + labels: make(map[string]string), } for _, l := range c.changeLogConfig[object].PublishLabels { labelValue := prevInstance.GetLabel(l) @@ -268,8 +268,8 @@ func (c *ChangeLog) createChangeLogInstance(mat *matrix.Matrix, change *Change) return } // copy keys - cInstance.SetLabel(objectTpeLabel, change.object) - cInstance.SetLabel(changeTypeLabel, change.changeType) + cInstance.SetLabel(objectLabel, change.object) + cInstance.SetLabel(opLabel, change.op) for k, v := range change.labels { cInstance.SetLabel(k, v) } diff --git a/cmd/poller/plugin/changelog/change_log_helper.go b/cmd/poller/plugin/changelog/change_log_helper.go index 5b11edfb3..8485c05ef 100644 --- a/cmd/poller/plugin/changelog/change_log_helper.go +++ b/cmd/poller/plugin/changelog/change_log_helper.go @@ -9,7 +9,7 @@ import ( // Entry represents a single ChangeLog entry type Entry struct { Object string `yaml:"object"` - TrackLabels []string `yaml:"track_labels"` + Track []string `yaml:"track"` PublishLabels []string `yaml:"publish_labels"` } @@ -22,20 +22,20 @@ type Config struct { const defaultChangeLogTemplate = ` ChangeLog: - object: svm - track_labels: + track: - svm - state - type publish_labels: - svm - object: node - track_labels: + track: - node - location publish_labels: - node - object: volume - track_labels: + track: - node - volume - svm diff --git a/cmd/poller/plugin/changelog/change_log_test.go b/cmd/poller/plugin/changelog/change_log_test.go index 07c898e66..eba18dce4 100644 --- a/cmd/poller/plugin/changelog/change_log_test.go +++ b/cmd/poller/plugin/changelog/change_log_test.go @@ -57,8 +57,8 @@ func TestChangeLogModified(t *testing.T) { t.Errorf("ChangeLog instances size expected %d, actual %d", 2, len(cl.GetInstances())) } else { for _, i := range cl.GetInstances() { - if i.GetLabel(changeTypeLabel) != modify { - t.Errorf("ChangeLog %s label expected %s, actual %s", changeTypeLabel, modify, i.GetLabel(changeTypeLabel)) + if i.GetLabel(opLabel) != update { + t.Errorf("ChangeLog %s label expected %s, actual %s", opLabel, update, i.GetLabel(opLabel)) } } } @@ -102,8 +102,8 @@ func TestChangeLogCreated(t *testing.T) { t.Errorf("ChangeLog instances size expected %d, actual %d", 1, len(cl.GetInstances())) } else { for _, i := range cl.GetInstances() { - if i.GetLabel(changeTypeLabel) != create { - t.Errorf("ChangeLog %s label expected %s, actual %s", changeTypeLabel, create, i.GetLabel(changeTypeLabel)) + if i.GetLabel(opLabel) != create { + t.Errorf("ChangeLog %s label expected %s, actual %s", opLabel, create, i.GetLabel(opLabel)) } } } @@ -138,8 +138,8 @@ func TestChangeLogDeleted(t *testing.T) { t.Errorf("ChangeLog instances size expected %d, actual %d", 1, len(cl.GetInstances())) } else { for _, i := range cl.GetInstances() { - if i.GetLabel(changeTypeLabel) != del { - t.Errorf("ChangeLog %s label expected %s, actual %s", changeTypeLabel, del, i.GetLabel(changeTypeLabel)) + if i.GetLabel(opLabel) != del { + t.Errorf("ChangeLog %s label expected %s, actual %s", opLabel, del, i.GetLabel(opLabel)) } } } From 06201ac0e69f06afdc806a104984506f9edda441 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Thu, 6 Jul 2023 22:12:22 +0530 Subject: [PATCH 03/20] feat: change log changes --- cmd/poller/plugin/changelog/change_log.go | 227 +++++++++--------- .../plugin/changelog/change_log_helper.go | 37 ++- .../plugin/changelog/change_log_test.go | 18 +- pkg/dict/dict.go | 17 +- pkg/matrix/instance.go | 4 +- pkg/matrix/matrix.go | 5 +- 6 files changed, 169 insertions(+), 139 deletions(-) diff --git a/cmd/poller/plugin/changelog/change_log.go b/cmd/poller/plugin/changelog/change_log.go index 7495953de..e90b14d20 100644 --- a/cmd/poller/plugin/changelog/change_log.go +++ b/cmd/poller/plugin/changelog/change_log.go @@ -7,6 +7,11 @@ import ( "github.com/netapp/harvest/v2/pkg/tree/yaml" ) +/*The changelog feature is only applicable to labels and requires a UUID for the label name to exist. +A default configuration for volume, SVM, and node is available, but the DSL can be overwritten as needed. +The shape of the change_log is specific to each label change and is only applicable to matrix collected by the collector. +*/ + // Constants for ChangeLog metrics and labels const ( changeLog = "change" @@ -29,9 +34,9 @@ var metrics = []string{ type ChangeLog struct { *plugin.AbstractPlugin matrixName string - previousData map[string]*matrix.Matrix + previousData *matrix.Matrix changeLogMap map[string]*matrix.Matrix - changeLogConfig map[string]Entry + changeLogConfig Entry } // Change represents a single change entry in the ChangeLog @@ -80,13 +85,12 @@ func (c *ChangeLog) Init() error { // populateChangeLogConfig populates the ChangeLog configuration from the plugin parameters func (c *ChangeLog) populateChangeLogConfig() error { var err error - dump, err := yaml.Dump(c.Params) + changeLogYaml, err := yaml.Dump(c.Params) if err != nil { return err } - object := c.ParentParams.GetChildS("object") - c.changeLogConfig, err = getChangeLogConfig(object.GetContentS(), string(dump), c.Logger) + c.changeLogConfig, err = getChangeLogConfig(c.ParentParams, string(changeLogYaml), c.Logger) if err != nil { return err } @@ -110,7 +114,9 @@ func (c *ChangeLog) initMatrix() error { } // Run processes the data and generates ChangeLog instances -func (c *ChangeLog) Run(data map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { +func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { + + data := dataMap[c.Object] // Purge and reset data // remove all metrics as analytics label may change over time err := c.initMatrix() @@ -127,119 +133,96 @@ func (c *ChangeLog) Run(data map[string]*matrix.Matrix) ([]*matrix.Matrix, error changeMat := c.changeLogMap[c.matrixName] - changeMat.SetGlobalLabels(data[c.Object].GetGlobalLabels()) - for objectKey, objectMatrix := range data { - object := objectMatrix.Object - if _, ok := c.changeLogConfig[object]; !ok { - c.Logger.Warn().Str("object", object).Msg("ChangeLog is not supported. Missing correct configuration") - return nil, nil - } + changeMat.SetGlobalLabels(data.GetGlobalLabels()) + object := data.Object + if c.changeLogConfig.Object == "" { + c.Logger.Warn().Str("object", object).Msg("ChangeLog is not supported. Missing correct configuration") + return nil, nil + } - prevMat := c.previousData[objectKey] - oldInstances := set.New() - prevInstancesUUIDKey := make(map[string]string) - for key, prevInstance := range prevMat.GetInstances() { - uuid := prevInstance.GetLabel("uuid") + prevMat := c.previousData + oldInstances := set.New() + prevInstancesUUIDKey := make(map[string]string) + for key, prevInstance := range prevMat.GetInstances() { + uuid := prevInstance.GetLabel("uuid") + if uuid == "" { + c.Logger.Warn().Str("object", object).Str("key", key).Msg("missing uuid") + continue + } + prevInstancesUUIDKey[uuid] = key + oldInstances.Add(key) + } + // check if prev exists + if c.previousData != nil { + // loop over current instances + for key, instance := range data.GetInstances() { + uuid := instance.GetLabel("uuid") if uuid == "" { - c.Logger.Warn().Str("object", object).Str("key", key).Msg("missing uuid") + c.Logger.Warn().Str("object", object).Str("key", key).Msg("missing uuid. ChangeLog is not supported") continue } - prevInstancesUUIDKey[uuid] = key - oldInstances.Add(key) - } - // check if object exists - if prev, ok := c.previousData[objectKey]; ok { - // loop over current instances - for key, instance := range objectMatrix.GetInstances() { - uuid := instance.GetLabel("uuid") - if uuid == "" { - c.Logger.Warn().Str("object", object).Str("key", key).Msg("missing uuid. ChangeLog is not supported") - continue - } - prevKey := prevInstancesUUIDKey[uuid] - if prevKey != "" { - // instance already in cache - oldInstances.Remove(prevKey) - } + prevKey := prevInstancesUUIDKey[uuid] + if prevKey != "" { + // instance already in cache + oldInstances.Remove(prevKey) + } - prevInstance := prev.GetInstance(prevKey) + prevInstance := c.previousData.GetInstance(prevKey) - if prevInstance == nil { - //instance created - change := &Change{ - key: uuid + "_" + object, - object: object, - op: create, - labels: make(map[string]string), - } - for _, l := range c.changeLogConfig[object].PublishLabels { - labelValue := instance.GetLabel(l) - if labelValue == "" { - c.Logger.Warn().Str("object", object).Str("key", key).Str("label", l).Msg("Missing label") - } else { - change.labels[l] = instance.GetLabel(l) - } - } - c.createChangeLogInstance(changeMat, change) - } else { - // check for any modification - cur, old := instance.GetLabels().CompareLabels(prevInstance.GetLabels(), c.changeLogConfig[object].Track) - if !cur.IsEmpty() { - for currentLabel, newLabel := range cur.Iter() { - change := &Change{ - key: uuid + "_" + object + "_" + currentLabel, - object: object, - op: update, - labels: make(map[string]string), - track: currentLabel, - oldValue: old.Get(currentLabel), - newValue: newLabel, - } - for _, l := range c.changeLogConfig[object].PublishLabels { - labelValue := instance.GetLabel(l) - if labelValue == "" { - c.Logger.Warn().Str("object", object).Str("key", key).Str("label", l).Msg("Missing label") - } else { - change.labels[l] = instance.GetLabel(l) - } - } - // add changed labelname and its old, new value - change.labels[track] = currentLabel - change.labels[oldLabelValue] = old.Get(currentLabel) - change.labels[newLabelValue] = newLabel - c.createChangeLogInstance(changeMat, change) + if prevInstance == nil { + //instance created + change := &Change{ + key: uuid + "_" + object, + object: object, + op: create, + labels: make(map[string]string), + } + c.updateChangeLogLabels(object, instance, change) + c.createChangeLogInstance(changeMat, change) + } else { + // check for any modification + cur, old := instance.GetLabels().CompareLabels(prevInstance.GetLabels(), c.changeLogConfig.Track) + if !cur.IsEmpty() { + for currentLabel, newLabel := range cur.Iter() { + change := &Change{ + key: uuid + "_" + object + "_" + currentLabel, + object: object, + op: update, + labels: make(map[string]string), + track: currentLabel, + oldValue: old.Get(currentLabel), + newValue: newLabel, } + c.updateChangeLogLabels(object, instance, change) + // add changed track and its old, new value + change.labels[track] = currentLabel + change.labels[oldLabelValue] = old.Get(currentLabel) + change.labels[newLabelValue] = newLabel + c.createChangeLogInstance(changeMat, change) } } } - // create deleted instances change_log - for key := range oldInstances.Iter() { - prevInstance := prevMat.GetInstance(key) - uuid := prevInstance.GetLabel("uuid") - if uuid == "" { - c.Logger.Warn().Str("object", object).Str("key", key).Msg("missing uuid. ChangeLog is not supported") - continue - } - if prevInstance != nil { - change := &Change{ - key: uuid + "_" + object, - object: object, - op: del, - labels: make(map[string]string), - } - for _, l := range c.changeLogConfig[object].PublishLabels { - labelValue := prevInstance.GetLabel(l) - if labelValue == "" { - c.Logger.Warn().Str("object", object).Str("key", key).Str("label", l).Msg("Missing label") - } else { - change.labels[l] = prevInstance.GetLabel(l) - } - } - c.createChangeLogInstance(changeMat, change) - } else { - c.Logger.Warn().Str("object", object).Str("key", key).Msg("missing instance") + } + // create deleted instances change_log + for key := range oldInstances.Iter() { + prevInstance := prevMat.GetInstance(key) + uuid := prevInstance.GetLabel("uuid") + if uuid == "" { + c.Logger.Warn().Str("object", object).Str("key", key).Msg("missing uuid. ChangeLog is not supported") + continue + } + if prevInstance != nil { + change := &Change{ + key: uuid + "_" + object, + object: object, + op: del, + labels: make(map[string]string), } + c.updateChangeLogLabels(object, prevInstance, change) + c.createChangeLogInstance(changeMat, change) + } else { + c.Logger.Warn().Str("object", object).Str("key", key).Msg("missing instance") } } } @@ -253,11 +236,10 @@ func (c *ChangeLog) Run(data map[string]*matrix.Matrix) ([]*matrix.Matrix, error } // copyPreviousData creates a copy of the previous data for comparison -func (c *ChangeLog) copyPreviousData(data map[string]*matrix.Matrix) { - c.previousData = make(map[string]*matrix.Matrix) - for k, v := range data { - c.previousData[k] = v.Clone(matrix.With{Data: true, Metrics: false, Instances: true, ExportInstances: false}) - } +func (c *ChangeLog) copyPreviousData(cur *matrix.Matrix) { + labels := append(c.changeLogConfig.PublishLabels, c.changeLogConfig.Track...) + labels = append(labels, "uuid") + c.previousData = cur.Clone(matrix.With{Data: true, Metrics: false, Instances: true, ExportInstances: false, Labels: labels}) } // createChangeLogInstance creates a new ChangeLog instance with the given change data @@ -285,3 +267,24 @@ func (c *ChangeLog) createChangeLogInstance(mat *matrix.Matrix, change *Change) return } } + +// updateChangeLogLabels populates change log labels +func (c *ChangeLog) updateChangeLogLabels(object string, instance *matrix.Instance, change *Change) { + cl := c.changeLogConfig + if len(cl.PublishLabels) > 0 { + for _, l := range cl.PublishLabels { + labelValue := instance.GetLabel(l) + if labelValue == "" { + c.Logger.Warn().Str("object", object).Str("label", l).Msg("Missing label") + } else { + change.labels[l] = instance.GetLabel(l) + } + } + } else if cl.includeAll { + for k := range instance.GetLabels().Iter() { + change.labels[k] = instance.GetLabel(k) + } + } else { + c.Logger.Warn().Str("object", object).Msg("missing publish labels") + } +} diff --git a/cmd/poller/plugin/changelog/change_log_helper.go b/cmd/poller/plugin/changelog/change_log_helper.go index 8485c05ef..168cea7fe 100644 --- a/cmd/poller/plugin/changelog/change_log_helper.go +++ b/cmd/poller/plugin/changelog/change_log_helper.go @@ -2,7 +2,9 @@ package changelog import ( "github.com/netapp/harvest/v2/pkg/logging" + "github.com/netapp/harvest/v2/pkg/tree/node" "gopkg.in/yaml.v3" + "strconv" "strings" ) @@ -11,6 +13,7 @@ type Entry struct { Object string `yaml:"object"` Track []string `yaml:"track"` PublishLabels []string `yaml:"publish_labels"` + includeAll bool } // Config represents the structure of the ChangeLog configuration @@ -26,14 +29,10 @@ ChangeLog: - svm - state - type - publish_labels: - - svm - object: node track: - node - location - publish_labels: - - node - object: volume track: - node @@ -41,14 +40,13 @@ ChangeLog: - svm - style - type - publish_labels: - - volume - - svm ` // getChangeLogConfig returns a map of ChangeLog entries for the given object -func getChangeLogConfig(object string, s string, logger *logging.Logger) (map[string]Entry, error) { +func getChangeLogConfig(parentParams *node.Node, s string, logger *logging.Logger) (Entry, error) { var config Config + object := parentParams.GetChildS("object").GetContentS() + temp := defaultChangeLogTemplate if s != "" { temp = preprocessOverwrite(object, s) @@ -60,17 +58,32 @@ func getChangeLogConfig(object string, s string, logger *logging.Logger) (map[st } err := yaml.Unmarshal([]byte(temp), &config) if err != nil { - return nil, err + return Entry{}, err } - objectMap := make(map[string]Entry) for _, obj := range config.ChangeLogs { if obj.Object == object { - objectMap[obj.Object] = obj + // populate publish_labels if they are empty + if obj.PublishLabels == nil { + if exportOption := parentParams.GetChildS("export_options"); exportOption != nil { + if exportedKeys := exportOption.GetChildS("instance_keys"); exportedKeys != nil { + obj.PublishLabels = append(obj.PublishLabels, exportedKeys.GetAllChildContentS()...) + } else if x := exportOption.GetChildContentS("include_all_labels"); x != "" { + if includeAllLabels, err := strconv.ParseBool(x); err != nil { + logger.Logger.Error().Err(err).Msg("parameter: include_all_labels") + } else { + if includeAllLabels { + obj.includeAll = true + } + } + } + } + } + return obj, nil } } - return objectMap, nil + return Entry{}, nil } // preprocessOverwrite updates the ChangeLog configuration by adding the given object and its properties diff --git a/cmd/poller/plugin/changelog/change_log_test.go b/cmd/poller/plugin/changelog/change_log_test.go index eba18dce4..8be2fc4b0 100644 --- a/cmd/poller/plugin/changelog/change_log_test.go +++ b/cmd/poller/plugin/changelog/change_log_test.go @@ -8,11 +8,11 @@ import ( "testing" ) -func newChangeLog() *ChangeLog { +func newChangeLog(object string) *ChangeLog { params := node.NewS("ChangeLog") parentParams := node.NewS("parent") - parentParams.NewChildS("object", "svm") + parentParams.NewChildS("object", object) abc := plugin.New("Test", nil, params, parentParams, "", nil) p := &ChangeLog{AbstractPlugin: abc} @@ -28,7 +28,7 @@ func newChangeLog() *ChangeLog { } func TestChangeLogModified(t *testing.T) { - p := newChangeLog() + p := newChangeLog("svm") m := matrix.New("TestChangeLog", "svm", "svm") data := map[string]*matrix.Matrix{ "svm": m, @@ -65,10 +65,14 @@ func TestChangeLogModified(t *testing.T) { } else { t.Error("ChangeLog slice size is wrong") } + + //TODO add test case of published labels are correct + + // TODO test includealllabels } func TestChangeLogCreated(t *testing.T) { - p := newChangeLog() + p := newChangeLog("svm") m := matrix.New("TestChangeLog", "svm", "svm") data := map[string]*matrix.Matrix{ "svm": m, @@ -113,7 +117,7 @@ func TestChangeLogCreated(t *testing.T) { } func TestChangeLogDeleted(t *testing.T) { - p := newChangeLog() + p := newChangeLog("svm") m := matrix.New("TestChangeLog", "svm", "svm") data := map[string]*matrix.Matrix{ "svm": m, @@ -149,7 +153,7 @@ func TestChangeLogDeleted(t *testing.T) { } func TestChangeLogUnsupported(t *testing.T) { - p := newChangeLog() + p := newChangeLog("lun") m := matrix.New("TestChangeLog", "lun", "lun") data := map[string]*matrix.Matrix{ "svm": m, @@ -175,6 +179,6 @@ func TestChangeLogUnsupported(t *testing.T) { o, _ := p.Run(data1) if len(o) != 0 { - t.Errorf("ChangeLog matric size expected %d, actual %d", 0, len(o)) + t.Errorf("ChangeLog mEtric size expected %d, actual %d", 0, len(o)) } } diff --git a/pkg/dict/dict.go b/pkg/dict/dict.go index 449fd0cf2..f5308eebe 100644 --- a/pkg/dict/dict.go +++ b/pkg/dict/dict.go @@ -20,11 +20,20 @@ func New() *Dict { return &d } -func (d *Dict) Copy() *Dict { +func (d *Dict) Copy(labels ...string) *Dict { c := &Dict{} - c.dict = make(map[string]string, len(d.dict)) - for k, v := range d.dict { - c.dict[k] = v + if len(labels) == 0 { + c.dict = make(map[string]string, len(d.dict)) + for k, v := range d.dict { + c.dict[k] = v + } + } else { + c.dict = make(map[string]string, len(labels)) + for _, k := range labels { + if v, ok := d.dict[k]; ok { + c.dict[k] = v + } + } } return c } diff --git a/pkg/matrix/instance.go b/pkg/matrix/instance.go index 75357fdcf..d6a7bcb71 100644 --- a/pkg/matrix/instance.go +++ b/pkg/matrix/instance.go @@ -55,9 +55,9 @@ func (i *Instance) SetExportable(b bool) { i.exportable = b } -func (i *Instance) Clone(isExportable bool) *Instance { +func (i *Instance) Clone(isExportable bool, labels ...string) *Instance { clone := NewInstance(i.index) - clone.labels = i.labels.Copy() + clone.labels = i.labels.Copy(labels...) clone.exportable = isExportable return clone } diff --git a/pkg/matrix/matrix.go b/pkg/matrix/matrix.go index 59186bade..dd1f605d1 100644 --- a/pkg/matrix/matrix.go +++ b/pkg/matrix/matrix.go @@ -36,6 +36,7 @@ type With struct { Metrics bool Instances bool ExportInstances bool + Labels []string } func New(uuid, object string, identifier string) *Matrix { @@ -84,9 +85,9 @@ func (m *Matrix) Clone(with With) *Matrix { clone.instances = make(map[string]*Instance, len(m.GetInstances())) for key, instance := range m.GetInstances() { if with.ExportInstances { - clone.instances[key] = instance.Clone(instance.IsExportable()) + clone.instances[key] = instance.Clone(instance.IsExportable(), with.Labels...) } else { - clone.instances[key] = instance.Clone(false) + clone.instances[key] = instance.Clone(false, with.Labels...) } } } else { From 9290797712adc5be34a2beb2e2f72c4d62bbeb40 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Mon, 17 Jul 2023 15:36:09 +0530 Subject: [PATCH 04/20] feat: change log changes --- cmd/poller/plugin/changelog/change_log.go | 19 +- .../dashboards/cmode/changelogMonitor.json | 1257 +++++++++++++++++ 2 files changed, 1275 insertions(+), 1 deletion(-) create mode 100644 grafana/dashboards/cmode/changelogMonitor.json diff --git a/cmd/poller/plugin/changelog/change_log.go b/cmd/poller/plugin/changelog/change_log.go index e90b14d20..e9b14da3b 100644 --- a/cmd/poller/plugin/changelog/change_log.go +++ b/cmd/poller/plugin/changelog/change_log.go @@ -5,6 +5,8 @@ import ( "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/set" "github.com/netapp/harvest/v2/pkg/tree/yaml" + "strconv" + "time" ) /*The changelog feature is only applicable to labels and requires a UUID for the label name to exist. @@ -23,6 +25,7 @@ const ( track = "track" oldLabelValue = "old_value" newLabelValue = "new_value" + indexLabel = "index" ) // Metrics to be used in ChangeLog @@ -37,6 +40,7 @@ type ChangeLog struct { previousData *matrix.Matrix changeLogMap map[string]*matrix.Matrix changeLogConfig Entry + index int } // Change represents a single change entry in the ChangeLog @@ -48,6 +52,7 @@ type Change struct { track string oldValue string newValue string + time int64 } // New initializes a new instance of the ChangeLog plugin @@ -152,6 +157,9 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er prevInstancesUUIDKey[uuid] = key oldInstances.Add(key) } + + currentTime := time.Now().Unix() + // check if prev exists if c.previousData != nil { // loop over current instances @@ -177,6 +185,7 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er object: object, op: create, labels: make(map[string]string), + time: currentTime, } c.updateChangeLogLabels(object, instance, change) c.createChangeLogInstance(changeMat, change) @@ -193,6 +202,7 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er track: currentLabel, oldValue: old.Get(currentLabel), newValue: newLabel, + time: currentTime, } c.updateChangeLogLabels(object, instance, change) // add changed track and its old, new value @@ -218,6 +228,7 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er object: object, op: del, labels: make(map[string]string), + time: currentTime, } c.updateChangeLogLabels(object, prevInstance, change) c.createChangeLogInstance(changeMat, change) @@ -231,6 +242,11 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er matricesArray = append(matricesArray, changeMat) c.copyPreviousData(data) + if len(changeMat.GetInstances()) > 0 { + // The `index` variable is used to differentiate between changes to the same label in a Grafana dashboard. + // It has a value between 0 and 100 and is used in the `change_log` query as `last_over_time`. + c.index = (c.index + 1) % 100 + } return matricesArray, nil } @@ -252,6 +268,7 @@ func (c *ChangeLog) createChangeLogInstance(mat *matrix.Matrix, change *Change) // copy keys cInstance.SetLabel(objectLabel, change.object) cInstance.SetLabel(opLabel, change.op) + cInstance.SetLabel(indexLabel, strconv.Itoa(c.index)) for k, v := range change.labels { cInstance.SetLabel(k, v) } @@ -262,7 +279,7 @@ func (c *ChangeLog) createChangeLogInstance(mat *matrix.Matrix, change *Change) return } } - if err = m.SetValueFloat64(cInstance, 1); err != nil { + if err = m.SetValueInt64(cInstance, change.time); err != nil { c.Logger.Error().Err(err).Str("metric", "alerts").Msg("Unable to set value on metric") return } diff --git a/grafana/dashboards/cmode/changelogMonitor.json b/grafana/dashboards/cmode/changelogMonitor.json new file mode 100644 index 000000000..2155b704f --- /dev/null +++ b/grafana/dashboards/cmode/changelogMonitor.json @@ -0,0 +1,1257 @@ +{ + "__inputs": [ + { + "description": "", + "label": "Prometheus", + "name": "DS_PROMETHEUS", + "pluginId": "prometheus", + "pluginName": "Prometheus", + "type": "datasource" + } + ], + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "8.1.8" + }, + { + "id": "prometheus", + "name": "Prometheus", + "type": "datasource", + "version": "1.0.0" + }, + { + "id": "stat", + "name": "Stat", + "type": "panel", + "version": "" + }, + { + "id": "table", + "name": "Table", + "type": "panel", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "The ONTAP Changelog Monitor is a comprehensive dashboard that provides tracking of configuration changes for ONTAP objects such as volumes, SVMs, and nodes.", + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "id": null, + "iteration": 1689585041737, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "cdot" + ], + "targetBlank": false, + "title": "Related Dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "panels": [ + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 282, + "panels": [ + { + "datasource": "${DS_PROMETHEUS}", + "description": "This change type indicates that an existing ONTAP object has been updated.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 291, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"node\",op=\"update\"}[$__range]))", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Update", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "This change type indicates that a new ONTAP object has been created.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 285, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"node\",op=\"create\"}[$__range]))", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Create", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "This change type indicates that an existing ONTAP object has been deleted.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 7, + "x": 16, + "y": 1 + }, + "id": 286, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"node\",op=\"delete\"}[$__range]))", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Delete", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Time: The date and time when the change was made
\nObject: The name of the ONTAP object that was changed (e.g., volume, svm, node)
\nOP: The type of change that was made (e.g., create, update, delete)
\nTrack: Property of the object which was modified
\nNew Value: The new value of the object after the change was made
\nOld Value: The previous value of the object before the change was made
", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "filterable": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeAsIso" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 288, + "options": { + "showHeader": true + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"node\"}[$__range])", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Node Changes ", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "cluster", + "datacenter", + "new_value", + "node", + "object", + "old_value", + "op", + "track", + "Value" + ] + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Time", + "binary": { + "left": "Value", + "operator": "*", + "reducer": "sum", + "right": "1000" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": false + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Value": true + }, + "indexByName": { + "Time": 0, + "Value": 1, + "cluster": 3, + "datacenter": 2, + "new_value": 6, + "node": 9, + "object": 4, + "old_value": 7, + "op": 8, + "track": 5 + }, + "renameByName": { + "Time": "Time", + "Value": "", + "cluster": "Cluster", + "datacenter": "Datacenter", + "new_value": "New Value", + "node": "Node", + "object": "Object", + "old_value": "Old Value", + "op": "OP", + "track": "Track" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Time" + } + ] + } + } + ], + "type": "table" + } + ], + "title": "Node Changes", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 297, + "panels": [ + { + "datasource": "${DS_PROMETHEUS}", + "description": "This change type indicates that an existing ONTAP object has been updated.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 15 + }, + "id": 298, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"svm\",op=\"update\"}[$__range]))", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Update", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "This change type indicates that a new ONTAP object has been created.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 15 + }, + "id": 292, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"svm\",op=\"create\"}[$__range]))", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Create", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "This change type indicates that an existing ONTAP object has been deleted.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 15 + }, + "id": 300, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"svm\",op=\"delete\"}[$__range]))", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Delete", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Time: The date and time when the change was made
\nObject: The name of the ONTAP object that was changed (e.g., volume, svm, node)
\nOP: The type of change that was made (e.g., create, update, delete)
\nTrack: Property of the object which was modified
\nNew Value: The new value of the object after the change was made
\nOld Value: The previous value of the object before the change was made
", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "filterable": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeAsIso" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Datacenter" + }, + "properties": [ + { + "id": "custom.width", + "value": null + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cluster" + }, + "properties": [ + { + "id": "custom.width", + "value": null + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 301, + "options": { + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"svm\"}[$__range])", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "SVM Changes ", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "cluster", + "datacenter", + "new_value", + "node", + "object", + "old_value", + "op", + "track", + "Value", + "volume", + "style", + "svm" + ] + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Time", + "binary": { + "left": "Value", + "operator": "*", + "reducer": "sum", + "right": "1000" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": false + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Value": true + }, + "indexByName": { + "Time": 0, + "Value": 1, + "cluster": 3, + "datacenter": 2, + "new_value": 7, + "object": 4, + "old_value": 8, + "op": 9, + "svm": 5, + "track": 6 + }, + "renameByName": { + "Time": "Time", + "Value": "", + "cluster": "Cluster", + "datacenter": "Datacenter", + "new_value": "New Value", + "node": "Node", + "object": "Object", + "old_value": "Old Value", + "op": "OP", + "svm": "SVM", + "track": "Track" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Time" + } + ] + } + } + ], + "type": "table" + } + ], + "title": "SVM Changes", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 290, + "panels": [ + { + "datasource": "${DS_PROMETHEUS}", + "description": "This change type indicates that an existing ONTAP object has been updated.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 16 + }, + "id": 284, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"volume\",op=\"update\"}[$__range]))", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Update", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "This change type indicates that a new ONTAP object has been created.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 16 + }, + "id": 299, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"volume\",op=\"create\"}[$__range]))", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Create", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "This change type indicates that an existing ONTAP object has been deleted.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 16 + }, + "id": 293, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "count by (cluster, datacenter, object) (last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"volume\",op=\"delete\"}[$__range]))", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Delete", + "type": "stat" + }, + { + "datasource": "${DS_PROMETHEUS}", + "description": "Time: The date and time when the change was made
\nObject: The name of the ONTAP object that was changed (e.g., volume, svm, node)
\nOP: The type of change that was made (e.g., create, update, delete)
\nTrack: Property of the object which was modified
\nNew Value: The new value of the object after the change was made
\nOld Value: The previous value of the object before the change was made
", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "filterable": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeAsIso" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Datacenter" + }, + "properties": [ + { + "id": "custom.width", + "value": null + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cluster" + }, + "properties": [ + { + "id": "custom.width", + "value": null + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 295, + "options": { + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "8.1.8", + "targets": [ + { + "exemplar": false, + "expr": "last_over_time(change_log{datacenter=~\"$Datacenter\",cluster=~\"$Cluster\",object=~\"volume\"}[$__range])", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Volume Changes ", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "cluster", + "datacenter", + "new_value", + "node", + "object", + "old_value", + "op", + "track", + "Value", + "volume", + "style", + "svm" + ] + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Time", + "binary": { + "left": "Value", + "operator": "*", + "reducer": "sum", + "right": "1000" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": false + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Value": true + }, + "indexByName": { + "Time": 0, + "Value": 1, + "cluster": 3, + "datacenter": 2, + "new_value": 7, + "node": 11, + "object": 4, + "old_value": 8, + "op": 9, + "style": 12, + "svm": 10, + "track": 6, + "volume": 5 + }, + "renameByName": { + "Time": "Time", + "Value": "", + "cluster": "Cluster", + "datacenter": "Datacenter", + "new_value": "New Value", + "node": "Node", + "object": "Object", + "old_value": "Old Value", + "op": "OP", + "style": "Style", + "svm": "SVM", + "track": "Track", + "volume": "Volume" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": true, + "field": "Time" + } + ] + } + } + ], + "type": "table" + } + ], + "title": "Volume Changes", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 30, + "style": "dark", + "tags": [ + "harvest", + "ontap", + "cdot" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": null, + "error": null, + "hide": 2, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "label_values(node_labels{system_type!=\"7mode\"},datacenter)", + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": "", + "multi": true, + "name": "Datacenter", + "options": [], + "query": { + "query": "label_values(node_labels{system_type!=\"7mode\"},datacenter)", + "refId": "Prometheus-Datacenter-Variable-Query" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "label_values(node_labels{system_type!=\"7mode\",datacenter=~\"$Datacenter\"},cluster)", + "description": null, + "error": null, + "hide": 0, + "includeAll": true, + "label": "", + "multi": true, + "name": "Cluster", + "options": [], + "query": { + "query": "label_values(node_labels{system_type!=\"7mode\",datacenter=~\"$Datacenter\"},cluster)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "ONTAP: Changelog Monitor", + "uid": "", + "version": 1 +} From f3445f847af44a05b17eebca2df9653fff6bd2ed Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Mon, 17 Jul 2023 18:08:13 +0530 Subject: [PATCH 05/20] feat: change log changes --- .../dashboards/cmode/changelogMonitor.json | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/grafana/dashboards/cmode/changelogMonitor.json b/grafana/dashboards/cmode/changelogMonitor.json index 2155b704f..5fbf5d264 100644 --- a/grafana/dashboards/cmode/changelogMonitor.json +++ b/grafana/dashboards/cmode/changelogMonitor.json @@ -59,7 +59,7 @@ "gnetId": null, "graphTooltip": 1, "id": null, - "iteration": 1689585041737, + "iteration": 1689585041740, "links": [ { "asDropdown": true, @@ -229,7 +229,7 @@ }, "gridPos": { "h": 5, - "w": 7, + "w": 8, "x": 16, "y": 1 }, @@ -450,7 +450,7 @@ "h": 5, "w": 8, "x": 0, - "y": 15 + "y": 2 }, "id": 298, "options": { @@ -508,7 +508,7 @@ "h": 5, "w": 8, "x": 8, - "y": 15 + "y": 2 }, "id": 292, "options": { @@ -567,7 +567,7 @@ "h": 5, "w": 8, "x": 16, - "y": 15 + "y": 2 }, "id": 300, "options": { @@ -667,7 +667,7 @@ "h": 8, "w": 24, "x": 0, - "y": 20 + "y": 7 }, "id": 301, "options": { @@ -815,7 +815,7 @@ "h": 5, "w": 8, "x": 0, - "y": 16 + "y": 3 }, "id": 284, "options": { @@ -873,7 +873,7 @@ "h": 5, "w": 8, "x": 8, - "y": 16 + "y": 3 }, "id": 299, "options": { @@ -932,7 +932,7 @@ "h": 5, "w": 8, "x": 16, - "y": 16 + "y": 3 }, "id": 293, "options": { @@ -1032,7 +1032,7 @@ "h": 8, "w": 24, "x": 0, - "y": 21 + "y": 8 }, "id": 295, "options": { From e6404d79ccebbef29a646cc376f57cb12a30fdbf Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Tue, 18 Jul 2023 11:48:19 +0530 Subject: [PATCH 06/20] feat: change log changes --- .../plugin/changelog/change_log_helper.go | 5 + .../plugin/changelog/change_log_test.go | 128 +++++++++++------- pkg/dict/dict.go | 2 + 3 files changed, 83 insertions(+), 52 deletions(-) diff --git a/cmd/poller/plugin/changelog/change_log_helper.go b/cmd/poller/plugin/changelog/change_log_helper.go index 168cea7fe..f4192a952 100644 --- a/cmd/poller/plugin/changelog/change_log_helper.go +++ b/cmd/poller/plugin/changelog/change_log_helper.go @@ -29,10 +29,12 @@ ChangeLog: - svm - state - type + - anti_ransomware_state - object: node track: - node - location + - healthy - object: volume track: - node @@ -40,6 +42,9 @@ ChangeLog: - svm - style - type + - aggr + - state + - status ` // getChangeLogConfig returns a map of ChangeLog entries for the given object diff --git a/cmd/poller/plugin/changelog/change_log_test.go b/cmd/poller/plugin/changelog/change_log_test.go index 8be2fc4b0..0ce98598c 100644 --- a/cmd/poller/plugin/changelog/change_log_test.go +++ b/cmd/poller/plugin/changelog/change_log_test.go @@ -8,12 +8,22 @@ import ( "testing" ) -func newChangeLog(object string) *ChangeLog { - +func newChangeLog(object string, includeAll bool) *ChangeLog { params := node.NewS("ChangeLog") parentParams := node.NewS("parent") parentParams.NewChildS("object", object) + exportOptions := parentParams.NewChildS("export_options", "") + if includeAll { + exportOptions.NewChildS("include_all_labels", "true") + } else { + instanceKeys := exportOptions.NewChildS("instance_keys", "") + instanceKeys.NewChildS("", "svm") + } + + return createChangeLog(params, parentParams) +} +func createChangeLog(params, parentParams *node.Node) *ChangeLog { abc := plugin.New("Test", nil, params, parentParams, "", nil) p := &ChangeLog{AbstractPlugin: abc} p.Options = &options.Options{ @@ -27,8 +37,38 @@ func newChangeLog(object string) *ChangeLog { return p } +func newChangeLogUnsupportedTrack(object string) *ChangeLog { + params := node.NewS("ChangeLog") + t := params.NewChildS("Track", "") + t.NewChildS("", "abcd") + parentParams := node.NewS("parent") + parentParams.NewChildS("object", object) + + return createChangeLog(params, parentParams) +} + +func checkChangeLogInstances(t *testing.T, o []*matrix.Matrix, expectedInstances, expectedLabels int, expectedOpLabel, opLabel string) { + if len(o) == 1 { + cl := o[0] + if len(cl.GetInstances()) != expectedInstances { + t.Errorf("ChangeLog instances size expected %d, actual %d", expectedInstances, len(cl.GetInstances())) + } else { + for _, i := range cl.GetInstances() { + if i.GetLabel(opLabel) != expectedOpLabel { + t.Errorf("ChangeLog %s label expected %s, actual %s", opLabel, expectedOpLabel, i.GetLabel(opLabel)) + } + if i.GetLabels().Size() != expectedLabels { + t.Errorf("ChangeLog number of labels expected %d, actual %d", expectedLabels, i.GetLabels().Size()) + } + } + } + } else { + t.Error("ChangeLog slice size is wrong") + } +} + func TestChangeLogModified(t *testing.T) { - p := newChangeLog("svm") + p := newChangeLog("svm", true) m := matrix.New("TestChangeLog", "svm", "svm") data := map[string]*matrix.Matrix{ "svm": m, @@ -51,28 +91,11 @@ func TestChangeLogModified(t *testing.T) { o, _ := p.Run(data1) - if len(o) == 1 { - cl := o[0] - if len(cl.GetInstances()) != 2 { - t.Errorf("ChangeLog instances size expected %d, actual %d", 2, len(cl.GetInstances())) - } else { - for _, i := range cl.GetInstances() { - if i.GetLabel(opLabel) != update { - t.Errorf("ChangeLog %s label expected %s, actual %s", opLabel, update, i.GetLabel(opLabel)) - } - } - } - } else { - t.Error("ChangeLog slice size is wrong") - } - - //TODO add test case of published labels are correct - - // TODO test includealllabels + checkChangeLogInstances(t, o, 2, 9, update, opLabel) } func TestChangeLogCreated(t *testing.T) { - p := newChangeLog("svm") + p := newChangeLog("svm", false) m := matrix.New("TestChangeLog", "svm", "svm") data := map[string]*matrix.Matrix{ "svm": m, @@ -100,24 +123,11 @@ func TestChangeLogCreated(t *testing.T) { o, _ := p.Run(data1) - if len(o) == 1 { - cl := o[0] - if len(cl.GetInstances()) != 1 { - t.Errorf("ChangeLog instances size expected %d, actual %d", 1, len(cl.GetInstances())) - } else { - for _, i := range cl.GetInstances() { - if i.GetLabel(opLabel) != create { - t.Errorf("ChangeLog %s label expected %s, actual %s", opLabel, create, i.GetLabel(opLabel)) - } - } - } - } else { - t.Error("ChangeLog slice size is wrong") - } + checkChangeLogInstances(t, o, 1, 4, create, opLabel) } func TestChangeLogDeleted(t *testing.T) { - p := newChangeLog("svm") + p := newChangeLog("svm", false) m := matrix.New("TestChangeLog", "svm", "svm") data := map[string]*matrix.Matrix{ "svm": m, @@ -136,24 +146,11 @@ func TestChangeLogDeleted(t *testing.T) { o, _ := p.Run(data1) - if len(o) == 1 { - cl := o[0] - if len(cl.GetInstances()) != 1 { - t.Errorf("ChangeLog instances size expected %d, actual %d", 1, len(cl.GetInstances())) - } else { - for _, i := range cl.GetInstances() { - if i.GetLabel(opLabel) != del { - t.Errorf("ChangeLog %s label expected %s, actual %s", opLabel, del, i.GetLabel(opLabel)) - } - } - } - } else { - t.Error("ChangeLog slice size is wrong") - } + checkChangeLogInstances(t, o, 1, 4, del, opLabel) } func TestChangeLogUnsupported(t *testing.T) { - p := newChangeLog("lun") + p := newChangeLog("lun", false) m := matrix.New("TestChangeLog", "lun", "lun") data := map[string]*matrix.Matrix{ "svm": m, @@ -182,3 +179,30 @@ func TestChangeLogUnsupported(t *testing.T) { t.Errorf("ChangeLog mEtric size expected %d, actual %d", 0, len(o)) } } + +func TestChangeLogModifiedUnsupportedTrack(t *testing.T) { + p := newChangeLogUnsupportedTrack("svm") + + m := matrix.New("TestChangeLog", "svm", "svm") + data := map[string]*matrix.Matrix{ + "svm": m, + } + instance, _ := m.NewInstance("0") + instance.SetLabel("uuid", "u1") + instance.SetLabel("svm", "s1") + + _, _ = p.Run(data) + + m1 := matrix.New("TestChangeLog", "svm", "svm") + data1 := map[string]*matrix.Matrix{ + "svm": m1, + } + + instance1, _ := m1.NewInstance("0") + instance1.SetLabel("uuid", "u1") + instance1.SetLabel("svm", "s2") + + o, _ := p.Run(data1) + + checkChangeLogInstances(t, o, 0, 0, "", "") +} diff --git a/pkg/dict/dict.go b/pkg/dict/dict.go index f5308eebe..7511214d3 100644 --- a/pkg/dict/dict.go +++ b/pkg/dict/dict.go @@ -122,6 +122,8 @@ func (d *Dict) Size() int { return len(d.dict) } +// CompareLabels The function compares the labels in the current Dict with the previous Dict. +// returns current and previous value of a label if values are different. label should exist in []labels func (d *Dict) CompareLabels(prev *Dict, labels []string) (*Dict, *Dict) { cur := New() old := New() From 5861d2547f766153a4f8bb424639ba260b8f679c Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Tue, 18 Jul 2023 12:34:04 +0530 Subject: [PATCH 07/20] feat: change log changes --- integration/test/dashboard_json_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/integration/test/dashboard_json_test.go b/integration/test/dashboard_json_test.go index e8e26d8a3..20c62b777 100644 --- a/integration/test/dashboard_json_test.go +++ b/integration/test/dashboard_json_test.go @@ -88,6 +88,7 @@ var excludeCounters = []string{ "svm_read_total", "svm_vscan", "svm_write_total", + "change_log", } var flakyCounters = []string{ From e1255b3ae61bd3ef9fbd70e5c7253071d27cf8de Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Tue, 18 Jul 2023 17:41:24 +0530 Subject: [PATCH 08/20] feat: address review comments --- cmd/poller/plugin/changelog/change_log.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/poller/plugin/changelog/change_log.go b/cmd/poller/plugin/changelog/change_log.go index e9b14da3b..9e9c9e304 100644 --- a/cmd/poller/plugin/changelog/change_log.go +++ b/cmd/poller/plugin/changelog/change_log.go @@ -294,7 +294,7 @@ func (c *ChangeLog) updateChangeLogLabels(object string, instance *matrix.Instan if labelValue == "" { c.Logger.Warn().Str("object", object).Str("label", l).Msg("Missing label") } else { - change.labels[l] = instance.GetLabel(l) + change.labels[l] = labelValue } } } else if cl.includeAll { From 2f6f6f759366d7ab47477acae8d1f2c7b61ba9e3 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Tue, 18 Jul 2023 21:35:08 +0530 Subject: [PATCH 09/20] feat: address review comments --- .../dashboards/cmode/changelogMonitor.json | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/grafana/dashboards/cmode/changelogMonitor.json b/grafana/dashboards/cmode/changelogMonitor.json index 5fbf5d264..e58cd778c 100644 --- a/grafana/dashboards/cmode/changelogMonitor.json +++ b/grafana/dashboards/cmode/changelogMonitor.json @@ -59,7 +59,7 @@ "gnetId": null, "graphTooltip": 1, "id": null, - "iteration": 1689585041740, + "iteration": 1689695994842, "links": [ { "asDropdown": true, @@ -265,7 +265,7 @@ }, { "datasource": "${DS_PROMETHEUS}", - "description": "Time: The date and time when the change was made
\nObject: The name of the ONTAP object that was changed (e.g., volume, svm, node)
\nOP: The type of change that was made (e.g., create, update, delete)
\nTrack: Property of the object which was modified
\nNew Value: The new value of the object after the change was made
\nOld Value: The previous value of the object before the change was made
", + "description": "Poller Time: The timestamp when Harvest Poller captured the change
\nObject: The name of the ONTAP object that was changed (e.g., volume, svm, node)
\nOP: The type of change that was made (e.g., create, update, delete)
\nTrack: Property of the object which was modified
\nNew Value: The new value of the object after the change was made
\nOld Value: The previous value of the object before the change was made
", "fieldConfig": { "defaults": { "color": { @@ -373,15 +373,15 @@ "Value": 1, "cluster": 3, "datacenter": 2, - "new_value": 6, - "node": 9, + "new_value": 8, + "node": 5, "object": 4, - "old_value": 7, - "op": 8, - "track": 5 + "old_value": 9, + "op": 6, + "track": 7 }, "renameByName": { - "Time": "Time", + "Time": "Poller Time", "Value": "", "cluster": "Cluster", "datacenter": "Datacenter", @@ -601,7 +601,7 @@ }, { "datasource": "${DS_PROMETHEUS}", - "description": "Time: The date and time when the change was made
\nObject: The name of the ONTAP object that was changed (e.g., volume, svm, node)
\nOP: The type of change that was made (e.g., create, update, delete)
\nTrack: Property of the object which was modified
\nNew Value: The new value of the object after the change was made
\nOld Value: The previous value of the object before the change was made
", + "description": "Poller Time: The timestamp when Harvest Poller captured the change
\nObject: The name of the ONTAP object that was changed (e.g., volume, svm, node)
\nOP: The type of change that was made (e.g., create, update, delete)
\nTrack: Property of the object which was modified
\nNew Value: The new value of the object after the change was made
\nOld Value: The previous value of the object before the change was made
", "fieldConfig": { "defaults": { "color": { @@ -737,15 +737,15 @@ "Value": 1, "cluster": 3, "datacenter": 2, - "new_value": 7, + "new_value": 8, "object": 4, - "old_value": 8, - "op": 9, + "old_value": 9, + "op": 6, "svm": 5, - "track": 6 + "track": 7 }, "renameByName": { - "Time": "Time", + "Time": "Poller Time", "Value": "", "cluster": "Cluster", "datacenter": "Datacenter", @@ -966,7 +966,7 @@ }, { "datasource": "${DS_PROMETHEUS}", - "description": "Time: The date and time when the change was made
\nObject: The name of the ONTAP object that was changed (e.g., volume, svm, node)
\nOP: The type of change that was made (e.g., create, update, delete)
\nTrack: Property of the object which was modified
\nNew Value: The new value of the object after the change was made
\nOld Value: The previous value of the object before the change was made
", + "description": "Poller Time: The timestamp when Harvest Poller captured the change
\nObject: The name of the ONTAP object that was changed (e.g., volume, svm, node)
\nOP: The type of change that was made (e.g., create, update, delete)
\nTrack: Property of the object which was modified
\nNew Value: The new value of the object after the change was made
\nOld Value: The previous value of the object before the change was made
", "fieldConfig": { "defaults": { "color": { @@ -1102,18 +1102,18 @@ "Value": 1, "cluster": 3, "datacenter": 2, - "new_value": 7, + "new_value": 8, "node": 11, "object": 4, - "old_value": 8, - "op": 9, + "old_value": 9, + "op": 6, "style": 12, "svm": 10, - "track": 6, + "track": 7, "volume": 5 }, "renameByName": { - "Time": "Time", + "Time": "Poller Time", "Value": "", "cluster": "Cluster", "datacenter": "Datacenter", From e40f8b941e935232f204ffc4fc9d400cf6e2f07f Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Wed, 19 Jul 2023 12:58:32 +0530 Subject: [PATCH 10/20] feat: address review comments --- grafana/dashboards/cmode/changelogMonitor.json | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/grafana/dashboards/cmode/changelogMonitor.json b/grafana/dashboards/cmode/changelogMonitor.json index e58cd778c..132fa582e 100644 --- a/grafana/dashboards/cmode/changelogMonitor.json +++ b/grafana/dashboards/cmode/changelogMonitor.json @@ -59,7 +59,7 @@ "gnetId": null, "graphTooltip": 1, "id": null, - "iteration": 1689695994842, + "iteration": 1689751504619, "links": [ { "asDropdown": true, @@ -97,6 +97,7 @@ "mode": "thresholds" }, "mappings": [], + "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ @@ -155,6 +156,7 @@ "mode": "thresholds" }, "mappings": [], + "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ @@ -214,6 +216,7 @@ "mode": "thresholds" }, "mappings": [], + "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ @@ -433,6 +436,7 @@ "mode": "thresholds" }, "mappings": [], + "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ @@ -491,6 +495,7 @@ "mode": "thresholds" }, "mappings": [], + "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ @@ -550,6 +555,7 @@ "mode": "thresholds" }, "mappings": [], + "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ @@ -798,6 +804,7 @@ "mode": "thresholds" }, "mappings": [], + "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ @@ -856,6 +863,7 @@ "mode": "thresholds" }, "mappings": [], + "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ @@ -915,6 +923,7 @@ "mode": "thresholds" }, "mappings": [], + "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ From f14ef215850fe5add1c84a3831348d0c3b2de594 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Thu, 20 Jul 2023 15:51:29 +0530 Subject: [PATCH 11/20] feat: address review comments --- .../dashboards/cmode/changelogMonitor.json | 58 ++++++++----------- 1 file changed, 23 insertions(+), 35 deletions(-) diff --git a/grafana/dashboards/cmode/changelogMonitor.json b/grafana/dashboards/cmode/changelogMonitor.json index 132fa582e..009394e07 100644 --- a/grafana/dashboards/cmode/changelogMonitor.json +++ b/grafana/dashboards/cmode/changelogMonitor.json @@ -59,7 +59,7 @@ "gnetId": null, "graphTooltip": 1, "id": null, - "iteration": 1689751504619, + "iteration": 1689847178728, "links": [ { "asDropdown": true, @@ -268,7 +268,7 @@ }, { "datasource": "${DS_PROMETHEUS}", - "description": "Poller Time: The timestamp when Harvest Poller captured the change
\nObject: The name of the ONTAP object that was changed (e.g., volume, svm, node)
\nOP: The type of change that was made (e.g., create, update, delete)
\nTrack: Property of the object which was modified
\nNew Value: The new value of the object after the change was made
\nOld Value: The previous value of the object before the change was made
", + "description": "`Poller Time:` The timestamp when Harvest Poller captured the change \n\n`Object:` The name of the ONTAP object that was changed (e.g., volume, svm, node) \n\n`OP:` The type of change that was made (e.g., create, update, delete) \n\n`Track:` Property of the object which was modified \n\n`New Value:` The new value of the object after the change was made \n\n`Old Value:` The previous value of the object before the change was made", "fieldConfig": { "defaults": { "color": { @@ -303,6 +303,18 @@ "value": "dateTimeAsIso" } ] + }, + { + "matcher": { + "id": "byName", + "options": "Poller Time" + }, + "properties": [ + { + "id": "custom.align", + "value": "left" + } + ] } ] }, @@ -607,7 +619,7 @@ }, { "datasource": "${DS_PROMETHEUS}", - "description": "Poller Time: The timestamp when Harvest Poller captured the change
\nObject: The name of the ONTAP object that was changed (e.g., volume, svm, node)
\nOP: The type of change that was made (e.g., create, update, delete)
\nTrack: Property of the object which was modified
\nNew Value: The new value of the object after the change was made
\nOld Value: The previous value of the object before the change was made
", + "description": "`Poller Time:` The timestamp when Harvest Poller captured the change \n\n`Object:` The name of the ONTAP object that was changed (e.g., volume, svm, node) \n\n`OP:` The type of change that was made (e.g., create, update, delete) \n\n`Track:` Property of the object which was modified \n\n`New Value:` The new value of the object after the change was made \n\n`Old Value:` The previous value of the object before the change was made", "fieldConfig": { "defaults": { "color": { @@ -646,24 +658,12 @@ { "matcher": { "id": "byName", - "options": "Datacenter" + "options": "Poller Time" }, "properties": [ { - "id": "custom.width", - "value": null - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cluster" - }, - "properties": [ - { - "id": "custom.width", - "value": null + "id": "custom.align", + "value": "left" } ] } @@ -975,7 +975,7 @@ }, { "datasource": "${DS_PROMETHEUS}", - "description": "Poller Time: The timestamp when Harvest Poller captured the change
\nObject: The name of the ONTAP object that was changed (e.g., volume, svm, node)
\nOP: The type of change that was made (e.g., create, update, delete)
\nTrack: Property of the object which was modified
\nNew Value: The new value of the object after the change was made
\nOld Value: The previous value of the object before the change was made
", + "description": "`Poller Time:` The timestamp when Harvest Poller captured the change \n\n`Object:` The name of the ONTAP object that was changed (e.g., volume, svm, node) \n\n`OP:` The type of change that was made (e.g., create, update, delete) \n\n`Track:` Property of the object which was modified \n\n`New Value:` The new value of the object after the change was made \n\n`Old Value:` The previous value of the object before the change was made", "fieldConfig": { "defaults": { "color": { @@ -1014,24 +1014,12 @@ { "matcher": { "id": "byName", - "options": "Datacenter" + "options": "Poller Time" }, "properties": [ { - "id": "custom.width", - "value": null - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Cluster" - }, - "properties": [ - { - "id": "custom.width", - "value": null + "id": "custom.align", + "value": "left" } ] } @@ -1243,7 +1231,7 @@ ] }, "time": { - "from": "now-24h", + "from": "now-7d", "to": "now" }, "timepicker": { From 60d9b8fab99d9403554c2c7fd25b64f46969e28e Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Fri, 21 Jul 2023 13:46:09 +0530 Subject: [PATCH 12/20] feat: add metric count --- cmd/poller/plugin/changelog/change_log.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cmd/poller/plugin/changelog/change_log.go b/cmd/poller/plugin/changelog/change_log.go index 9e9c9e304..e9627f885 100644 --- a/cmd/poller/plugin/changelog/change_log.go +++ b/cmd/poller/plugin/changelog/change_log.go @@ -41,6 +41,7 @@ type ChangeLog struct { changeLogMap map[string]*matrix.Matrix changeLogConfig Entry index int + metricsCount int } // Change represents a single change entry in the ChangeLog @@ -246,8 +247,14 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er // The `index` variable is used to differentiate between changes to the same label in a Grafana dashboard. // It has a value between 0 and 100 and is used in the `change_log` query as `last_over_time`. c.index = (c.index + 1) % 100 + c.Logger.Info().Int("instances", len(changeMat.GetInstances())). + Int("metrics", c.metricsCount). + Msg("Collected") } + // reset metric count + c.metricsCount = 0 + return matricesArray, nil } @@ -272,6 +279,7 @@ func (c *ChangeLog) createChangeLogInstance(mat *matrix.Matrix, change *Change) for k, v := range change.labels { cInstance.SetLabel(k, v) } + c.metricsCount += cInstance.GetLabels().Size() m := mat.GetMetric("log") if m == nil { if m, err = mat.NewMetricFloat64("log"); err != nil { From 1372def84fb37d76b234392cceb594cf2bc47be5 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Fri, 21 Jul 2023 18:31:08 +0530 Subject: [PATCH 13/20] feat: add metric count --- cmd/poller/plugin/changelog/change_log.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cmd/poller/plugin/changelog/change_log.go b/cmd/poller/plugin/changelog/change_log.go index e9627f885..d681631fd 100644 --- a/cmd/poller/plugin/changelog/change_log.go +++ b/cmd/poller/plugin/changelog/change_log.go @@ -70,7 +70,7 @@ func (c *ChangeLog) Init() error { } // Initialize the changeLogMap - c.changeLogMap = make(map[string]*matrix.Matrix) + c.changeLogMap = make(map[string]*matrix.Matrix, 0) object := c.ParentParams.GetChildS("object") c.matrixName = object.GetContentS() + "_" + changeLog @@ -148,7 +148,7 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er prevMat := c.previousData oldInstances := set.New() - prevInstancesUUIDKey := make(map[string]string) + prevInstancesUUIDKey := make(map[string]string, 0) for key, prevInstance := range prevMat.GetInstances() { uuid := prevInstance.GetLabel("uuid") if uuid == "" { @@ -185,7 +185,7 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er key: uuid + "_" + object, object: object, op: create, - labels: make(map[string]string), + labels: make(map[string]string, 0), time: currentTime, } c.updateChangeLogLabels(object, instance, change) @@ -199,7 +199,7 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er key: uuid + "_" + object + "_" + currentLabel, object: object, op: update, - labels: make(map[string]string), + labels: make(map[string]string, 0), track: currentLabel, oldValue: old.Get(currentLabel), newValue: newLabel, @@ -228,7 +228,7 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er key: uuid + "_" + object, object: object, op: del, - labels: make(map[string]string), + labels: make(map[string]string, 0), time: currentTime, } c.updateChangeLogLabels(object, prevInstance, change) @@ -254,6 +254,7 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er // reset metric count c.metricsCount = 0 + c.metricsCount = 0 return matricesArray, nil } From 7e387012622b8e9018378e8f34b988f6f8f586ff Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Fri, 21 Jul 2023 22:52:02 +0530 Subject: [PATCH 14/20] feat: add metric count --- cmd/poller/plugin/changelog/change_log.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/poller/plugin/changelog/change_log.go b/cmd/poller/plugin/changelog/change_log.go index d681631fd..24becab24 100644 --- a/cmd/poller/plugin/changelog/change_log.go +++ b/cmd/poller/plugin/changelog/change_log.go @@ -254,7 +254,6 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er // reset metric count c.metricsCount = 0 - c.metricsCount = 0 return matricesArray, nil } From 400983edca82ab4810badbbc9121a02ff5cf4281 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Tue, 25 Jul 2023 16:25:37 +0530 Subject: [PATCH 15/20] feat: address review comments --- .../dashboards/cmode/changelogMonitor.json | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/grafana/dashboards/cmode/changelogMonitor.json b/grafana/dashboards/cmode/changelogMonitor.json index 009394e07..4ade9287f 100644 --- a/grafana/dashboards/cmode/changelogMonitor.json +++ b/grafana/dashboards/cmode/changelogMonitor.json @@ -59,7 +59,7 @@ "gnetId": null, "graphTooltip": 1, "id": null, - "iteration": 1689847178728, + "iteration": 1690282136692, "links": [ { "asDropdown": true, @@ -416,7 +416,7 @@ "sort": [ { "desc": true, - "field": "Time" + "field": "Poller Time" } ] } @@ -466,7 +466,7 @@ "h": 5, "w": 8, "x": 0, - "y": 2 + "y": 15 }, "id": 298, "options": { @@ -525,7 +525,7 @@ "h": 5, "w": 8, "x": 8, - "y": 2 + "y": 15 }, "id": 292, "options": { @@ -585,7 +585,7 @@ "h": 5, "w": 8, "x": 16, - "y": 2 + "y": 15 }, "id": 300, "options": { @@ -673,7 +673,7 @@ "h": 8, "w": 24, "x": 0, - "y": 7 + "y": 20 }, "id": 301, "options": { @@ -772,7 +772,7 @@ "sort": [ { "desc": true, - "field": "Time" + "field": "Poller Time" } ] } @@ -822,7 +822,7 @@ "h": 5, "w": 8, "x": 0, - "y": 3 + "y": 29 }, "id": 284, "options": { @@ -881,7 +881,7 @@ "h": 5, "w": 8, "x": 8, - "y": 3 + "y": 29 }, "id": 299, "options": { @@ -941,7 +941,7 @@ "h": 5, "w": 8, "x": 16, - "y": 3 + "y": 29 }, "id": 293, "options": { @@ -1029,7 +1029,7 @@ "h": 8, "w": 24, "x": 0, - "y": 8 + "y": 34 }, "id": 295, "options": { @@ -1133,7 +1133,7 @@ "sort": [ { "desc": true, - "field": "Time" + "field": "Poller Time" } ] } @@ -1231,7 +1231,7 @@ ] }, "time": { - "from": "now-7d", + "from": "now-24h", "to": "now" }, "timepicker": { From 7e447c8e7a03edd48f25bb4530bac444ba50e9e3 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Tue, 22 Aug 2023 12:02:18 +0530 Subject: [PATCH 16/20] feat: address lint errors --- cmd/poller/plugin/changelog/change_log.go | 7 +------ pkg/dict/dict.go | 4 ++-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/cmd/poller/plugin/changelog/change_log.go b/cmd/poller/plugin/changelog/change_log.go index 24becab24..476c037d5 100644 --- a/cmd/poller/plugin/changelog/change_log.go +++ b/cmd/poller/plugin/changelog/change_log.go @@ -80,12 +80,7 @@ func (c *ChangeLog) Init() error { return err } - // Populate the ChangeLog configuration - if err := c.populateChangeLogConfig(); err != nil { - return err - } - - return nil + return c.populateChangeLogConfig() } // populateChangeLogConfig populates the ChangeLog configuration from the plugin parameters diff --git a/pkg/dict/dict.go b/pkg/dict/dict.go index 7511214d3..5cc9aad3a 100644 --- a/pkg/dict/dict.go +++ b/pkg/dict/dict.go @@ -5,8 +5,8 @@ package dict import ( - "github.com/netapp/harvest/v2/pkg/util" "reflect" + "slices" "strings" ) @@ -129,7 +129,7 @@ func (d *Dict) CompareLabels(prev *Dict, labels []string) (*Dict, *Dict) { old := New() for key, val1 := range d.dict { val2, ok := prev.dict[key] - if util.Contains(labels, key) && (!ok || !reflect.DeepEqual(val1, val2)) { + if slices.Contains(labels, key) && (!ok || !reflect.DeepEqual(val1, val2)) { cur.dict[key] = val1 old.dict[key] = val2 } From f148cd3f4cb0bdbae39c505d0dda8260297486a0 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Tue, 22 Aug 2023 12:13:32 +0530 Subject: [PATCH 17/20] feat: fix testcase --- grafana/dashboards/cmode/changelogMonitor.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grafana/dashboards/cmode/changelogMonitor.json b/grafana/dashboards/cmode/changelogMonitor.json index 4ade9287f..bd2ea2f95 100644 --- a/grafana/dashboards/cmode/changelogMonitor.json +++ b/grafana/dashboards/cmode/changelogMonitor.json @@ -1231,7 +1231,7 @@ ] }, "time": { - "from": "now-24h", + "from": "now-3h", "to": "now" }, "timepicker": { From e9cfee93f0901adb2704968aaf36099b02640ad5 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Wed, 30 Aug 2023 16:23:20 +0530 Subject: [PATCH 18/20] feat: merge main and disable changelog by default --- cmd/poller/plugin/changelog/change_log.go | 13 +++++++------ conf/rest/9.10.0/svm.yaml | 2 +- conf/rest/9.10.0/volume.yaml | 1 + conf/rest/9.12.0/node.yaml | 2 +- conf/rest/9.9.0/svm.yaml | 1 + conf/rest/9.9.0/volume.yaml | 2 +- conf/zapi/cdot/9.8.0/aggr.yaml | 1 - conf/zapi/cdot/9.8.0/node.yaml | 2 +- conf/zapi/cdot/9.8.0/shelf.yaml | 2 +- conf/zapi/cdot/9.8.0/svm.yaml | 2 +- conf/zapi/cdot/9.8.0/volume.yaml | 2 +- grafana/dashboards/cmode/changelogMonitor.json | 2 +- 12 files changed, 17 insertions(+), 15 deletions(-) diff --git a/cmd/poller/plugin/changelog/change_log.go b/cmd/poller/plugin/changelog/change_log.go index 476c037d5..b2a031d69 100644 --- a/cmd/poller/plugin/changelog/change_log.go +++ b/cmd/poller/plugin/changelog/change_log.go @@ -70,7 +70,7 @@ func (c *ChangeLog) Init() error { } // Initialize the changeLogMap - c.changeLogMap = make(map[string]*matrix.Matrix, 0) + c.changeLogMap = make(map[string]*matrix.Matrix) object := c.ParentParams.GetChildS("object") c.matrixName = object.GetContentS() + "_" + changeLog @@ -143,7 +143,7 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er prevMat := c.previousData oldInstances := set.New() - prevInstancesUUIDKey := make(map[string]string, 0) + prevInstancesUUIDKey := make(map[string]string) for key, prevInstance := range prevMat.GetInstances() { uuid := prevInstance.GetLabel("uuid") if uuid == "" { @@ -180,7 +180,7 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er key: uuid + "_" + object, object: object, op: create, - labels: make(map[string]string, 0), + labels: make(map[string]string), time: currentTime, } c.updateChangeLogLabels(object, instance, change) @@ -194,7 +194,7 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er key: uuid + "_" + object + "_" + currentLabel, object: object, op: update, - labels: make(map[string]string, 0), + labels: make(map[string]string), track: currentLabel, oldValue: old.Get(currentLabel), newValue: newLabel, @@ -223,7 +223,7 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er key: uuid + "_" + object, object: object, op: del, - labels: make(map[string]string, 0), + labels: make(map[string]string), time: currentTime, } c.updateChangeLogLabels(object, prevInstance, change) @@ -255,7 +255,8 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er // copyPreviousData creates a copy of the previous data for comparison func (c *ChangeLog) copyPreviousData(cur *matrix.Matrix) { - labels := append(c.changeLogConfig.PublishLabels, c.changeLogConfig.Track...) + labels := c.changeLogConfig.PublishLabels + labels = append(labels, c.changeLogConfig.Track...) labels = append(labels, "uuid") c.previousData = cur.Clone(matrix.With{Data: true, Metrics: false, Instances: true, ExportInstances: false, Labels: labels}) } diff --git a/conf/rest/9.10.0/svm.yaml b/conf/rest/9.10.0/svm.yaml index e74d01eb6..5f58f2d03 100644 --- a/conf/rest/9.10.0/svm.yaml +++ b/conf/rest/9.10.0/svm.yaml @@ -59,7 +59,7 @@ plugins: - new_status state online online `0` - ldap_signed ldap_session_security sign sign `0` - ldap_encrypted ldap_session_security seal seal `0` - - ChangeLog +# - ChangeLog export_options: instance_keys: diff --git a/conf/rest/9.10.0/volume.yaml b/conf/rest/9.10.0/volume.yaml index 399be8bdb..64b750399 100644 --- a/conf/rest/9.10.0/volume.yaml +++ b/conf/rest/9.10.0/volume.yaml @@ -124,6 +124,7 @@ plugins: # - volume `MDV_CRS_.+` # # Exclude Metadata volumes, Audit volumes have a “MDV_aud_” prefix # - volume `MDV_aud_.+` +# - ChangeLog export_options: instance_keys: diff --git a/conf/rest/9.12.0/node.yaml b/conf/rest/9.12.0/node.yaml index d89b6e941..abebce45d 100644 --- a/conf/rest/9.12.0/node.yaml +++ b/conf/rest/9.12.0/node.yaml @@ -36,7 +36,7 @@ plugins: - new_status healthy true up `0` join: - warnings `; ` failed_fan_message,failed_power_message,over_temperature - - ChangeLog +# - ChangeLog export_options: instance_keys: diff --git a/conf/rest/9.9.0/svm.yaml b/conf/rest/9.9.0/svm.yaml index 2d2957d11..6e56853df 100644 --- a/conf/rest/9.9.0/svm.yaml +++ b/conf/rest/9.9.0/svm.yaml @@ -53,6 +53,7 @@ plugins: - new_status state online online `0` - ldap_signed ldap_session_security sign sign `0` - ldap_encrypted ldap_session_security seal seal `0` +# - ChangeLog export_options: instance_keys: diff --git a/conf/rest/9.9.0/volume.yaml b/conf/rest/9.9.0/volume.yaml index b6d61248e..01c062b08 100644 --- a/conf/rest/9.9.0/volume.yaml +++ b/conf/rest/9.9.0/volume.yaml @@ -119,7 +119,7 @@ plugins: # - volume `MDV_CRS_.+` # # Exclude Metadata volumes, Audit volumes have a “MDV_aud_” prefix # - volume `MDV_aud_.+` - - ChangeLog +# - ChangeLog export_options: instance_keys: diff --git a/conf/zapi/cdot/9.8.0/aggr.yaml b/conf/zapi/cdot/9.8.0/aggr.yaml index 0563f30df..3b7b3b0e5 100644 --- a/conf/zapi/cdot/9.8.0/aggr.yaml +++ b/conf/zapi/cdot/9.8.0/aggr.yaml @@ -68,7 +68,6 @@ plugins: # metric label zapi_value rest_value `default_value` value_to_num: - new_status state online online `0` - - ChangeLog export_options: instance_keys: diff --git a/conf/zapi/cdot/9.8.0/node.yaml b/conf/zapi/cdot/9.8.0/node.yaml index 34c9ec1ce..bee05a3a9 100644 --- a/conf/zapi/cdot/9.8.0/node.yaml +++ b/conf/zapi/cdot/9.8.0/node.yaml @@ -47,7 +47,7 @@ plugins: - new_status healthy true up `0` join: - warnings `; ` failed_fan_message,failed_power_message,over_temperature - - ChangeLog +# - ChangeLog export_options: instance_keys: diff --git a/conf/zapi/cdot/9.8.0/shelf.yaml b/conf/zapi/cdot/9.8.0/shelf.yaml index 77d441f63..f8b92c8d7 100644 --- a/conf/zapi/cdot/9.8.0/shelf.yaml +++ b/conf/zapi/cdot/9.8.0/shelf.yaml @@ -6,7 +6,7 @@ object: shelf counters: storage-shelf-info: - - ^^shelf-uid + - ^^shelf-uid => uuid - ^module-type => module_type - ^op-status => op_status - ^serial-number => serial_number diff --git a/conf/zapi/cdot/9.8.0/svm.yaml b/conf/zapi/cdot/9.8.0/svm.yaml index 69df7b62c..1f31da08b 100644 --- a/conf/zapi/cdot/9.8.0/svm.yaml +++ b/conf/zapi/cdot/9.8.0/svm.yaml @@ -29,7 +29,7 @@ plugins: - new_status state online online `0` - ldap_signed ldap_session_security sign sign `0` - ldap_encrypted ldap_session_security seal seal `0` - - ChangeLog +# - ChangeLog export_options: instance_keys: diff --git a/conf/zapi/cdot/9.8.0/volume.yaml b/conf/zapi/cdot/9.8.0/volume.yaml index f39e9793d..5072a1f8b 100644 --- a/conf/zapi/cdot/9.8.0/volume.yaml +++ b/conf/zapi/cdot/9.8.0/volume.yaml @@ -103,7 +103,7 @@ plugins: - svm_root root_volume `false` `No` - node_root root_volume `true` `Yes` - svm_root root_volume `true` `Yes` - - ChangeLog +# - ChangeLog export_options: instance_keys: diff --git a/grafana/dashboards/cmode/changelogMonitor.json b/grafana/dashboards/cmode/changelogMonitor.json index bd2ea2f95..3a285d7e8 100644 --- a/grafana/dashboards/cmode/changelogMonitor.json +++ b/grafana/dashboards/cmode/changelogMonitor.json @@ -54,7 +54,7 @@ } ] }, - "description": "The ONTAP Changelog Monitor is a comprehensive dashboard that provides tracking of configuration changes for ONTAP objects such as volumes, SVMs, and nodes.", + "description": "The ONTAP Changelog Monitor, tracks configuration modifications in volumes, SVMs, and nodes, is deactivated by default. To leverage this feature, one must enable the ChangeLog plugin within the Volume, SVM, and Node Templates.", "editable": true, "gnetId": null, "graphTooltip": 1, From a73e71f498e091e7a8746a007ba17fe4bb2dd0ee Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Wed, 4 Oct 2023 13:44:14 +0530 Subject: [PATCH 19/20] feat: address review comments --- .../zapi/plugins/aggregate/aggregate.go | 4 +- cmd/poller/plugin/changelog/change_log.go | 175 +++++++++--------- .../plugin/changelog/change_log_helper.go | 84 ++++----- .../plugin/changelog/change_log_test.go | 4 +- docs/plugins.md | 2 + .../dashboards/cmode/changelogMonitor.json | 60 ++++-- pkg/matrix/instance.go | 35 +++- 7 files changed, 213 insertions(+), 151 deletions(-) diff --git a/cmd/collectors/zapi/plugins/aggregate/aggregate.go b/cmd/collectors/zapi/plugins/aggregate/aggregate.go index cc4105957..938745cc0 100644 --- a/cmd/collectors/zapi/plugins/aggregate/aggregate.go +++ b/cmd/collectors/zapi/plugins/aggregate/aggregate.go @@ -63,11 +63,11 @@ func (a *Aggregate) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er } // update aggregate instance label with cloud stores info - for aggrUUID, aggr := range data.GetInstances() { + for uuid, aggr := range data.GetInstances() { if !aggr.IsExportable() { continue } - aggr.SetLabel("cloud_stores", strings.Join(a.aggrCloudStoresMap[aggrUUID], ",")) + aggr.SetLabel("cloud_stores", strings.Join(a.aggrCloudStoresMap[uuid], ",")) // Handling aggr footprint metrics aggrName := aggr.GetLabel("aggr") diff --git a/cmd/poller/plugin/changelog/change_log.go b/cmd/poller/plugin/changelog/change_log.go index b2a031d69..7315148be 100644 --- a/cmd/poller/plugin/changelog/change_log.go +++ b/cmd/poller/plugin/changelog/change_log.go @@ -5,6 +5,7 @@ import ( "github.com/netapp/harvest/v2/pkg/matrix" "github.com/netapp/harvest/v2/pkg/set" "github.com/netapp/harvest/v2/pkg/tree/yaml" + "maps" "strconv" "time" ) @@ -16,16 +17,16 @@ The shape of the change_log is specific to each label change and is only applica // Constants for ChangeLog metrics and labels const ( - changeLog = "change" - objectLabel = "object" - opLabel = "op" - create = "create" - update = "update" - del = "delete" - track = "track" - oldLabelValue = "old_value" - newLabelValue = "new_value" - indexLabel = "index" + changeLog = "change" + objectLabel = "object" + opLabel = "op" + create = "create" + update = "update" + del = "delete" + track = "track" + oldValue = "old_value" + newValue = "new_value" + indexLabel = "index" ) // Metrics to be used in ChangeLog @@ -91,7 +92,7 @@ func (c *ChangeLog) populateChangeLogConfig() error { return err } - c.changeLogConfig, err = getChangeLogConfig(c.ParentParams, string(changeLogYaml), c.Logger) + c.changeLogConfig, err = getChangeLogConfig(c.ParentParams, changeLogYaml, c.Logger) if err != nil { return err } @@ -119,14 +120,17 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er data := dataMap[c.Object] // Purge and reset data - // remove all metrics as analytics label may change over time + // remove all metrics err := c.initMatrix() if err != nil { c.Logger.Warn().Err(err).Msg("error while init matrix") return nil, err } - // if this is first poll + // reset metric count + c.metricsCount = 0 + + // if this is the first poll if c.previousData == nil { c.copyPreviousData(data) return nil, nil @@ -156,81 +160,78 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er currentTime := time.Now().Unix() - // check if prev exists - if c.previousData != nil { - // loop over current instances - for key, instance := range data.GetInstances() { - uuid := instance.GetLabel("uuid") - if uuid == "" { - c.Logger.Warn().Str("object", object).Str("key", key).Msg("missing uuid. ChangeLog is not supported") - continue - } + // loop over current instances + for key, instance := range data.GetInstances() { + uuid := instance.GetLabel("uuid") + if uuid == "" { + c.Logger.Warn().Str("object", object).Str("key", key).Msg("missing uuid. ChangeLog is not supported") + continue + } - prevKey := prevInstancesUUIDKey[uuid] - if prevKey != "" { - // instance already in cache - oldInstances.Remove(prevKey) - } + prevKey := prevInstancesUUIDKey[uuid] + if prevKey != "" { + // instance already in cache + oldInstances.Remove(prevKey) + } - prevInstance := c.previousData.GetInstance(prevKey) + prevInstance := c.previousData.GetInstance(prevKey) - if prevInstance == nil { - //instance created - change := &Change{ - key: uuid + "_" + object, - object: object, - op: create, - labels: make(map[string]string), - time: currentTime, - } - c.updateChangeLogLabels(object, instance, change) - c.createChangeLogInstance(changeMat, change) - } else { - // check for any modification - cur, old := instance.GetLabels().CompareLabels(prevInstance.GetLabels(), c.changeLogConfig.Track) - if !cur.IsEmpty() { - for currentLabel, newLabel := range cur.Iter() { - change := &Change{ - key: uuid + "_" + object + "_" + currentLabel, - object: object, - op: update, - labels: make(map[string]string), - track: currentLabel, - oldValue: old.Get(currentLabel), - newValue: newLabel, - time: currentTime, - } - c.updateChangeLogLabels(object, instance, change) - // add changed track and its old, new value - change.labels[track] = currentLabel - change.labels[oldLabelValue] = old.Get(currentLabel) - change.labels[newLabelValue] = newLabel - c.createChangeLogInstance(changeMat, change) + if prevInstance == nil { + //instance created + change := &Change{ + key: uuid + "_" + object, + object: object, + op: create, + labels: make(map[string]string), + time: currentTime, + } + c.updateChangeLogLabels(object, instance, change) + c.createChangeLogInstance(changeMat, change) + } else { + // check for any modification + cur, old := instance.CompareDiffs(prevInstance, c.changeLogConfig.Track) + if len(cur) > 0 { + for currentLabel, nVal := range cur { + change := &Change{ + key: uuid + "_" + object + "_" + currentLabel, + object: object, + op: update, + labels: make(map[string]string), + track: currentLabel, + oldValue: old[currentLabel], + newValue: nVal, + time: currentTime, } + c.updateChangeLogLabels(object, instance, change) + // add changed track and its old, new value + change.labels[track] = currentLabel + change.labels[oldValue] = change.oldValue + change.labels[newValue] = nVal + c.createChangeLogInstance(changeMat, change) } } } - // create deleted instances change_log - for key := range oldInstances.Iter() { - prevInstance := prevMat.GetInstance(key) - uuid := prevInstance.GetLabel("uuid") - if uuid == "" { - c.Logger.Warn().Str("object", object).Str("key", key).Msg("missing uuid. ChangeLog is not supported") - continue - } - if prevInstance != nil { - change := &Change{ - key: uuid + "_" + object, - object: object, - op: del, - labels: make(map[string]string), - time: currentTime, - } - c.updateChangeLogLabels(object, prevInstance, change) - c.createChangeLogInstance(changeMat, change) - } else { - c.Logger.Warn().Str("object", object).Str("key", key).Msg("missing instance") + } + // create deleted instances change_log + for key := range oldInstances.Iter() { + prevInstance := prevMat.GetInstance(key) + uuid := prevInstance.GetLabel("uuid") + if uuid == "" { + c.Logger.Warn().Str("object", object).Str("key", key).Msg("missing uuid. ChangeLog is not supported") + continue + } + if prevInstance != nil { + change := &Change{ + key: uuid + "_" + object, + object: object, + op: del, + labels: make(map[string]string), + time: currentTime, } + c.updateChangeLogLabels(object, prevInstance, change) + c.createChangeLogInstance(changeMat, change) + } else { + c.Logger.Warn().Str("object", object).Str("key", key).Msg("missing instance") } } @@ -244,12 +245,10 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er c.index = (c.index + 1) % 100 c.Logger.Info().Int("instances", len(changeMat.GetInstances())). Int("metrics", c.metricsCount). + Int("index", c.index). Msg("Collected") } - // reset metric count - c.metricsCount = 0 - return matricesArray, nil } @@ -275,16 +274,16 @@ func (c *ChangeLog) createChangeLogInstance(mat *matrix.Matrix, change *Change) for k, v := range change.labels { cInstance.SetLabel(k, v) } - c.metricsCount += cInstance.GetLabels().Size() + c.metricsCount += len(cInstance.GetLabels()) m := mat.GetMetric("log") if m == nil { if m, err = mat.NewMetricFloat64("log"); err != nil { - c.Logger.Warn().Err(err).Str("key", "alerts").Msg("error while creating metric") + c.Logger.Warn().Err(err).Str("key", "log").Msg("error while creating metric") return } } if err = m.SetValueInt64(cInstance, change.time); err != nil { - c.Logger.Error().Err(err).Str("metric", "alerts").Msg("Unable to set value on metric") + c.Logger.Error().Err(err).Int64("val", change.time).Msg("Unable to set value on metric") return } } @@ -302,9 +301,7 @@ func (c *ChangeLog) updateChangeLogLabels(object string, instance *matrix.Instan } } } else if cl.includeAll { - for k := range instance.GetLabels().Iter() { - change.labels[k] = instance.GetLabel(k) - } + maps.Copy(change.labels, instance.GetLabels()) } else { c.Logger.Warn().Str("object", object).Msg("missing publish labels") } diff --git a/cmd/poller/plugin/changelog/change_log_helper.go b/cmd/poller/plugin/changelog/change_log_helper.go index f4192a952..448310a99 100644 --- a/cmd/poller/plugin/changelog/change_log_helper.go +++ b/cmd/poller/plugin/changelog/change_log_helper.go @@ -4,8 +4,8 @@ import ( "github.com/netapp/harvest/v2/pkg/logging" "github.com/netapp/harvest/v2/pkg/tree/node" "gopkg.in/yaml.v3" + "slices" "strconv" - "strings" ) // Entry represents a single ChangeLog entry @@ -48,69 +48,65 @@ ChangeLog: ` // getChangeLogConfig returns a map of ChangeLog entries for the given object -func getChangeLogConfig(parentParams *node.Node, s string, logger *logging.Logger) (Entry, error) { - var config Config +func getChangeLogConfig(parentParams *node.Node, overwriteConfig []byte, logger *logging.Logger) (Entry, error) { + var ( + config Config + entry Entry + err error + ) object := parentParams.GetChildS("object").GetContentS() - temp := defaultChangeLogTemplate - if s != "" { - temp = preprocessOverwrite(object, s) - err := yaml.Unmarshal([]byte(temp), &config) + if len(overwriteConfig) > 0 { + entry, err = preprocessOverwrite(object, overwriteConfig) if err != nil { - logger.Warn().Err(err).Str("template", s).Msg("failed to parse changelog dsl. Trying default") - temp = defaultChangeLogTemplate + logger.Warn().Err(err).Str("template", string(overwriteConfig)).Msg("failed to parse changelog dsl. Trying default") + } else { + return entry, nil } } - err := yaml.Unmarshal([]byte(temp), &config) + + err = yaml.Unmarshal([]byte(defaultChangeLogTemplate), &config) if err != nil { return Entry{}, err } - for _, obj := range config.ChangeLogs { - if obj.Object == object { - // populate publish_labels if they are empty - if obj.PublishLabels == nil { - if exportOption := parentParams.GetChildS("export_options"); exportOption != nil { - if exportedKeys := exportOption.GetChildS("instance_keys"); exportedKeys != nil { - obj.PublishLabels = append(obj.PublishLabels, exportedKeys.GetAllChildContentS()...) - } else if x := exportOption.GetChildContentS("include_all_labels"); x != "" { - if includeAllLabels, err := strconv.ParseBool(x); err != nil { - logger.Logger.Error().Err(err).Msg("parameter: include_all_labels") - } else { - if includeAllLabels { - obj.includeAll = true - } - } + i := slices.IndexFunc(config.ChangeLogs, func(entry Entry) bool { + return entry.Object == object + }) + if i == -1 { + return Entry{}, nil + } + entry = config.ChangeLogs[i] + // populate publish_labels if they are empty + if entry.PublishLabels == nil { + if exportOption := parentParams.GetChildS("export_options"); exportOption != nil { + if exportedKeys := exportOption.GetChildS("instance_keys"); exportedKeys != nil { + entry.PublishLabels = append(entry.PublishLabels, exportedKeys.GetAllChildContentS()...) + } else if x := exportOption.GetChildContentS("include_all_labels"); x != "" { + if includeAllLabels, err := strconv.ParseBool(x); err != nil { + logger.Logger.Error().Err(err).Msg("parameter: include_all_labels") + } else { + if includeAllLabels { + entry.includeAll = true } } } - return obj, nil } } - return Entry{}, nil + return entry, nil } // preprocessOverwrite updates the ChangeLog configuration by adding the given object and its properties -func preprocessOverwrite(object string, configStr string) string { - // Split the YAML content into lines - lines := strings.Split(configStr, "\n") +func preprocessOverwrite(object string, configStr []byte) (Entry, error) { + var entry Entry - // Add four spaces to indent each line, making them at the same level as object - indentedLines := make([]string, len(lines)) - for i, line := range lines { - indentedLines[i] = " " + line + err := yaml.Unmarshal(configStr, &entry) + if err != nil { + return entry, err } - // Join the indented lines back into a single string - indentedYaml := strings.Join(indentedLines, "\n") - - // Add the ChangeLog prefix - prefix := ` -ChangeLog: - - object: ` + object - - newYaml := strings.Join([]string{prefix, indentedYaml}, "\n") - return newYaml + entry.Object = object + return entry, nil } diff --git a/cmd/poller/plugin/changelog/change_log_test.go b/cmd/poller/plugin/changelog/change_log_test.go index 0ce98598c..5902d61ba 100644 --- a/cmd/poller/plugin/changelog/change_log_test.go +++ b/cmd/poller/plugin/changelog/change_log_test.go @@ -57,8 +57,8 @@ func checkChangeLogInstances(t *testing.T, o []*matrix.Matrix, expectedInstances if i.GetLabel(opLabel) != expectedOpLabel { t.Errorf("ChangeLog %s label expected %s, actual %s", opLabel, expectedOpLabel, i.GetLabel(opLabel)) } - if i.GetLabels().Size() != expectedLabels { - t.Errorf("ChangeLog number of labels expected %d, actual %d", expectedLabels, i.GetLabels().Size()) + if len(i.GetLabels()) != expectedLabels { + t.Errorf("ChangeLog number of labels expected %d, actual %d", expectedLabels, len(i.GetLabels())) } } } diff --git a/docs/plugins.md b/docs/plugins.md index 0ae2719d8..71e2bfd75 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -630,3 +630,5 @@ compute_metric: # "inode_files_total" and multiplying the result by 100. # inode_used_percent = inode_files_used / inode_files_total * 100 ``` + +# ChangeLog \ No newline at end of file diff --git a/grafana/dashboards/cmode/changelogMonitor.json b/grafana/dashboards/cmode/changelogMonitor.json index 3a285d7e8..67e78e71e 100644 --- a/grafana/dashboards/cmode/changelogMonitor.json +++ b/grafana/dashboards/cmode/changelogMonitor.json @@ -33,6 +33,12 @@ "name": "Table", "type": "panel", "version": "" + }, + { + "id": "text", + "name": "Text", + "type": "panel", + "version": "" } ], "annotations": { @@ -59,7 +65,7 @@ "gnetId": null, "graphTooltip": 1, "id": null, - "iteration": 1690282136692, + "iteration": 1696405583422, "links": [ { "asDropdown": true, @@ -78,7 +84,7 @@ ], "panels": [ { - "collapsed": true, + "collapsed": false, "datasource": null, "gridPos": { "h": 1, @@ -86,6 +92,36 @@ "x": 0, "y": 0 }, + "id": 303, + "panels": [], + "title": "Important Information", + "type": "row" + }, + { + "datasource": "${DS_PROMETHEUS}", + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 305, + "options": { + "content": "To use this dashboard, enable the ChangeLog plugin for node, svm, and volume templates. For more details, visit the [ChangeLog documentation](https://netapp.github.io/harvest/latest/plugins/#changelog).", + "mode": "markdown" + }, + "pluginVersion": "8.1.8", + "type": "text" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, "id": 282, "panels": [ { @@ -435,7 +471,7 @@ "h": 1, "w": 24, "x": 0, - "y": 1 + "y": 4 }, "id": 297, "panels": [ @@ -466,7 +502,7 @@ "h": 5, "w": 8, "x": 0, - "y": 15 + "y": 2 }, "id": 298, "options": { @@ -525,7 +561,7 @@ "h": 5, "w": 8, "x": 8, - "y": 15 + "y": 2 }, "id": 292, "options": { @@ -585,7 +621,7 @@ "h": 5, "w": 8, "x": 16, - "y": 15 + "y": 2 }, "id": 300, "options": { @@ -673,7 +709,7 @@ "h": 8, "w": 24, "x": 0, - "y": 20 + "y": 7 }, "id": 301, "options": { @@ -791,7 +827,7 @@ "h": 1, "w": 24, "x": 0, - "y": 2 + "y": 5 }, "id": 290, "panels": [ @@ -822,7 +858,7 @@ "h": 5, "w": 8, "x": 0, - "y": 29 + "y": 3 }, "id": 284, "options": { @@ -881,7 +917,7 @@ "h": 5, "w": 8, "x": 8, - "y": 29 + "y": 3 }, "id": 299, "options": { @@ -941,7 +977,7 @@ "h": 5, "w": 8, "x": 16, - "y": 29 + "y": 3 }, "id": 293, "options": { @@ -1029,7 +1065,7 @@ "h": 8, "w": 24, "x": 0, - "y": 34 + "y": 8 }, "id": 295, "options": { diff --git a/pkg/matrix/instance.go b/pkg/matrix/instance.go index 78c57ba08..6464d7a4c 100644 --- a/pkg/matrix/instance.go +++ b/pkg/matrix/instance.go @@ -51,9 +51,40 @@ func (i *Instance) SetExportable(b bool) { i.exportable = b } -func (i *Instance) Clone(isExportable bool) *Instance { +func (i *Instance) Clone(isExportable bool, labels ...string) *Instance { clone := NewInstance(i.index) - clone.labels = maps.Clone(i.labels) + clone.labels = i.Copy(labels...) clone.exportable = isExportable return clone } + +func (i *Instance) Copy(labels ...string) map[string]string { + if len(labels) == 0 { + return maps.Clone(i.labels) + } + m := make(map[string]string, len(labels)) + for _, k := range labels { + m[k] = i.labels[k] + } + return m +} + +// CompareDiffs iterates through each key in compareKeys, checking if the receiver and prev have the same value for that key. +// When the values are different, return a new Map with the current and previous value +func (i *Instance) CompareDiffs(prev *Instance, compareKeys []string) (map[string]string, map[string]string) { + cur := make(map[string]string) + old := make(map[string]string) + + for _, compareKey := range compareKeys { + val1, ok1 := i.GetLabels()[compareKey] + if !ok1 { + continue + } + val2, ok2 := prev.GetLabels()[compareKey] + if !ok2 || val1 != val2 { + cur[compareKey] = val1 + old[compareKey] = val2 + } + } + return cur, old +} From 7f2cb5219e54156adcb608e4e5504f26667225b9 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Thu, 5 Oct 2023 13:26:25 +0530 Subject: [PATCH 20/20] feat: add uuid in rest volume template and move changelog matrix as local to run method --- cmd/poller/plugin/changelog/change_log.go | 28 ++++++++--------------- conf/rest/9.10.0/volume.yaml | 1 + 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/cmd/poller/plugin/changelog/change_log.go b/cmd/poller/plugin/changelog/change_log.go index 7315148be..80e5e58e9 100644 --- a/cmd/poller/plugin/changelog/change_log.go +++ b/cmd/poller/plugin/changelog/change_log.go @@ -39,7 +39,6 @@ type ChangeLog struct { *plugin.AbstractPlugin matrixName string previousData *matrix.Matrix - changeLogMap map[string]*matrix.Matrix changeLogConfig Entry index int metricsCount int @@ -70,17 +69,9 @@ func (c *ChangeLog) Init() error { return err } - // Initialize the changeLogMap - c.changeLogMap = make(map[string]*matrix.Matrix) - object := c.ParentParams.GetChildS("object") c.matrixName = object.GetContentS() + "_" + changeLog - // Initialize the changeLogMatrix - if err := c.initMatrix(); err != nil { - return err - } - return c.populateChangeLogConfig() } @@ -100,28 +91,27 @@ func (c *ChangeLog) populateChangeLogConfig() error { } // initMatrix initializes a new matrix with the given name -func (c *ChangeLog) initMatrix() error { - c.changeLogMap[c.matrixName] = matrix.New(c.Parent+c.matrixName, changeLog, c.matrixName) - for _, changeLogMatrix := range c.changeLogMap { +func (c *ChangeLog) initMatrix() (map[string]*matrix.Matrix, error) { + changeLogMap := make(map[string]*matrix.Matrix) + changeLogMap[c.matrixName] = matrix.New(c.Parent+c.matrixName, changeLog, c.matrixName) + for _, changeLogMatrix := range changeLogMap { changeLogMatrix.SetExportOptions(matrix.DefaultExportOptions()) } for _, k := range metrics { - err := matrix.CreateMetric(k, c.changeLogMap[c.matrixName]) + err := matrix.CreateMetric(k, changeLogMap[c.matrixName]) if err != nil { c.Logger.Warn().Err(err).Str("key", k).Msg("error while creating metric") - return err + return nil, err } } - return nil + return changeLogMap, nil } // Run processes the data and generates ChangeLog instances func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, error) { data := dataMap[c.Object] - // Purge and reset data - // remove all metrics - err := c.initMatrix() + changeLogMap, err := c.initMatrix() if err != nil { c.Logger.Warn().Err(err).Msg("error while init matrix") return nil, err @@ -136,7 +126,7 @@ func (c *ChangeLog) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, er return nil, nil } - changeMat := c.changeLogMap[c.matrixName] + changeMat := changeLogMap[c.matrixName] changeMat.SetGlobalLabels(data.GetGlobalLabels()) object := data.Object diff --git a/conf/rest/9.10.0/volume.yaml b/conf/rest/9.10.0/volume.yaml index e24702db1..e92909953 100644 --- a/conf/rest/9.10.0/volume.yaml +++ b/conf/rest/9.10.0/volume.yaml @@ -20,6 +20,7 @@ counters: - ^state => state - ^style => style - ^type => type + - ^uuid - autosize.grow_threshold => autosize_grow_threshold_percent - autosize.maximum => autosize_maximum_size - snapshot_count