From 0dfbe43b393c688d7c3c2d889f9ea3eb41998514 Mon Sep 17 00:00:00 2001 From: Andrew Fitzgerald Date: Tue, 25 Feb 2020 23:29:20 -0500 Subject: [PATCH 1/4] Ensure that metric ID is always set during kafka ingestion --- publish/kafka/publish.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/kafka/publish.go b/publish/kafka/publish.go index 1a4de6bff5..962df7827b 100644 --- a/publish/kafka/publish.go +++ b/publish/kafka/publish.go @@ -279,12 +279,12 @@ func (m *mtPublisher) Publish(metrics []*schema.MetricData) error { if m.autoInterval { _, s := m.schemas.Match(metric.Name, 0) metric.Interval = s.Retentions.Rets[0].SecondsPerPoint - metric.SetId() } else { log.Error("interval is 0 but can't deduce interval automatically. this should never happen") return errors.New("need to deduce interval but cannot") } } + metric.SetId() isMD := false isMP := false From 9cc9266d217799b71937e07cf54e3984ad33eff1 Mon Sep 17 00:00:00 2001 From: Andrew Fitzgerald Date: Wed, 26 Feb 2020 10:09:51 -0500 Subject: [PATCH 2/4] ensure that metric id is set by mt-gateway --- cmd/mt-gateway/ingest/metrics.go | 1 + publish/kafka/publish.go | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/mt-gateway/ingest/metrics.go b/cmd/mt-gateway/ingest/metrics.go index aaa6698730..7cbba16c24 100644 --- a/cmd/mt-gateway/ingest/metrics.go +++ b/cmd/mt-gateway/ingest/metrics.go @@ -89,6 +89,7 @@ func prepareIngest(in []*schema.MetricData, toPublish []*schema.MetricData) ([]* promDiscards.Add(m.OrgId, err.Error()) continue } + m.SetId() metricTimestamp = getMetricsTimestampStat(m.OrgId) metricTimestamp.ValueUint32(uint32(m.Time)) toPublish = append(toPublish, m) diff --git a/publish/kafka/publish.go b/publish/kafka/publish.go index 962df7827b..fe613220b8 100644 --- a/publish/kafka/publish.go +++ b/publish/kafka/publish.go @@ -284,7 +284,6 @@ func (m *mtPublisher) Publish(metrics []*schema.MetricData) error { return errors.New("need to deduce interval but cannot") } } - metric.SetId() isMD := false isMP := false From 5ac71f7242cf1ead442255f314d860e9279fd79a Mon Sep 17 00:00:00 2001 From: Andrew Fitzgerald Date: Thu, 27 Feb 2020 09:57:51 -0500 Subject: [PATCH 3/4] explicitly fail publishing if interval or ID are not set --- publish/kafka/publish.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/publish/kafka/publish.go b/publish/kafka/publish.go index fe613220b8..f16597c6c2 100644 --- a/publish/kafka/publish.go +++ b/publish/kafka/publish.go @@ -276,13 +276,12 @@ func (m *mtPublisher) Publish(metrics []*schema.MetricData) error { for _, metric := range metrics { if metric.Interval == 0 { - if m.autoInterval { - _, s := m.schemas.Match(metric.Name, 0) - metric.Interval = s.Retentions.Rets[0].SecondsPerPoint - } else { - log.Error("interval is 0 but can't deduce interval automatically. this should never happen") - return errors.New("need to deduce interval but cannot") - } + log.Error("interval must be set before publishing") + return errors.New("interval must be set before publishing") + } + if metric.Id == "" { + log.Error("metric ID must be set before publishing") + return errors.New("metric ID must be set before publishing") } isMD := false From 6c06422f28acce1ecbc64ec8a48345d60035e472 Mon Sep 17 00:00:00 2001 From: Andrew Fitzgerald Date: Thu, 27 Feb 2020 10:03:37 -0500 Subject: [PATCH 4/4] add docs to Publisher interface noting the expectations for published metrics --- publish/publish.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/publish/publish.go b/publish/publish.go index 27bea02491..a8e101ac20 100644 --- a/publish/publish.go +++ b/publish/publish.go @@ -18,7 +18,10 @@ var ( ) type Publisher interface { + //Publish the given metrics. + //Requires that the metric interval and ID have been set Publish(metrics []*schema.MetricData) error + //Type returns the type of system the Publisher publishes to Type() string }