From 0940b714bbdfa03e56407dd64a368f6d8188e0ab Mon Sep 17 00:00:00 2001 From: Owen Williams Date: Wed, 30 Aug 2023 13:44:59 -0400 Subject: [PATCH 1/4] UTF-8 support in metric and label names Signed-off-by: Owen Williams --- config/http_config.go | 2 +- expfmt/decode_test.go | 58 ++++++- expfmt/openmetrics_create.go | 79 +++++---- expfmt/openmetrics_create_test.go | 261 +++++++++++++++++++++--------- expfmt/text_create.go | 98 ++++++++--- expfmt/text_create_test.go | 175 ++++++++++++-------- expfmt/text_parse_test.go | 98 +++++------ model/labels.go | 20 ++- model/labels_test.go | 63 +++++--- model/labelset_test.go | 1 + model/metric.go | 47 +++++- model/metric_test.go | 70 +++++--- model/signature_test.go | 4 +- model/silence_test.go | 60 +++++-- 14 files changed, 719 insertions(+), 317 deletions(-) diff --git a/config/http_config.go b/config/http_config.go index 4a926e8d..7a67a0a6 100644 --- a/config/http_config.go +++ b/config/http_config.go @@ -30,7 +30,7 @@ import ( "sync" "time" - "github.com/mwitkow/go-conntrack" + conntrack "github.com/mwitkow/go-conntrack" "golang.org/x/net/http/httpproxy" "golang.org/x/net/http2" "golang.org/x/oauth2" diff --git a/expfmt/decode_test.go b/expfmt/decode_test.go index 7b7b41fd..e700946f 100644 --- a/expfmt/decode_test.go +++ b/expfmt/decode_test.go @@ -17,6 +17,7 @@ import ( "bufio" "errors" "io" + "math" "net/http" "reflect" "sort" @@ -105,9 +106,10 @@ func TestProtoDecoder(t *testing.T) { var testTime = model.Now() scenarios := []struct { - in string - expected model.Vector - fail bool + in string + expected model.Vector + legacyNameFail bool + fail bool }{ { in: "", @@ -333,6 +335,30 @@ func TestProtoDecoder(t *testing.T) { }, }, }, + { + in: "\xa8\x01\n\ngauge.name\x12\x11gauge\ndoc\nstr\"ing\x18\x01\"T\n\x1b\n\x06name.1\x12\x11val with\nnew line\n*\n\x06name*2\x12 val with \\backslash and \"quotes\"\x12\t\t\x00\x00\x00\x00\x00\x00\xf0\x7f\"/\n\x10\n\x06name.1\x12\x06Björn\n\x10\n\x06name*2\x12\x06佖佥\x12\t\t\xd1\xcfD\xb9\xd0\x05\xc2H", + legacyNameFail: true, + expected: model.Vector{ + &model.Sample{ + Metric: model.Metric{ + model.MetricNameLabel: "gauge.name", + "name.1": "val with\nnew line", + "name*2": "val with \\backslash and \"quotes\"", + }, + Value: model.SampleValue(math.Inf(+1)), + Timestamp: testTime, + }, + &model.Sample{ + Metric: model.Metric{ + model.MetricNameLabel: "gauge.name", + "name.1": "Björn", + "name*2": "佖佥", + }, + Value: 3.14e42, + Timestamp: testTime, + }, + }, + }, } for i, scenario := range scenarios { @@ -345,11 +371,31 @@ func TestProtoDecoder(t *testing.T) { var all model.Vector for { + model.NameValidationScheme = model.LegacyValidation var smpls model.Vector err := dec.Decode(&smpls) if err != nil && errors.Is(err, io.EOF) { break } + if scenario.legacyNameFail { + if err == nil { + t.Fatal("Expected error when decoding without UTF-8 support enabled but got none") + } + model.NameValidationScheme = model.UTF8Validation + dec = &SampleDecoder{ + Dec: &protoDecoder{r: strings.NewReader(scenario.in)}, + Opts: &DecodeOptions{ + Timestamp: testTime, + }, + } + err = dec.Decode(&smpls) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatalf("Unexpected error when decoding with UTF-8 support: %v", err) + } + } if scenario.fail { if err == nil { t.Fatal("Expected error but got none") @@ -436,7 +482,7 @@ func TestExtractSamples(t *testing.T) { Help: proto.String("Help for foo."), Type: dto.MetricType_COUNTER.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Counter: &dto.Counter{ Value: proto.Float64(4711), }, @@ -448,7 +494,7 @@ func TestExtractSamples(t *testing.T) { Help: proto.String("Help for bar."), Type: dto.MetricType_GAUGE.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Gauge: &dto.Gauge{ Value: proto.Float64(3.14), }, @@ -460,7 +506,7 @@ func TestExtractSamples(t *testing.T) { Help: proto.String("Help for bad."), Type: dto.MetricType(42).Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Gauge: &dto.Gauge{ Value: proto.Float64(2.7), }, diff --git a/expfmt/openmetrics_create.go b/expfmt/openmetrics_create.go index 21cdddcf..6bf5e5f3 100644 --- a/expfmt/openmetrics_create.go +++ b/expfmt/openmetrics_create.go @@ -35,6 +35,12 @@ import ( // sanity checks. If the input contains duplicate metrics or invalid metric or // label names, the conversion will result in invalid text format output. // +// If metric names conform to the legacy validation pattern, they will be placed +// outside the brackets in the traditional way, like `foo{}`. If the metric name +// fails the legacy validation check, it will be placed quoted inside the +// brackets: `{"foo"}`. As stated above, the input is assumed to be santized and +// no error will be thrown in this case. +// // This function fulfills the type 'expfmt.encoder'. // // Note that OpenMetrics requires a final `# EOF` line. Since this function acts @@ -98,7 +104,7 @@ func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily) (written int if err != nil { return } - n, err = w.WriteString(shortName) + n, err = writeName(w, shortName) written += n if err != nil { return @@ -124,7 +130,7 @@ func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily) (written int if err != nil { return } - n, err = w.WriteString(shortName) + n, err = writeName(w, shortName) written += n if err != nil { return @@ -303,21 +309,9 @@ func writeOpenMetricsSample( floatValue float64, intValue uint64, useIntValue bool, exemplar *dto.Exemplar, ) (int, error) { - var written int - n, err := w.WriteString(name) - written += n - if err != nil { - return written, err - } - if suffix != "" { - n, err = w.WriteString(suffix) - written += n - if err != nil { - return written, err - } - } - n, err = writeOpenMetricsLabelPairs( - w, metric.Label, additionalLabelName, additionalLabelValue, + written := 0 + n, err := writeOpenMetricsNameAndLabelPairs( + w, name+suffix, metric.Label, additionalLabelName, additionalLabelValue, ) written += n if err != nil { @@ -365,27 +359,58 @@ func writeOpenMetricsSample( return written, nil } -// writeOpenMetricsLabelPairs works like writeOpenMetrics but formats the float -// in OpenMetrics style. -func writeOpenMetricsLabelPairs( +// writeOpenMetricsNameAndLabelPairs works like writeOpenMetricsSample but +// formats the float in OpenMetrics style. +func writeOpenMetricsNameAndLabelPairs( w enhancedWriter, + name string, in []*dto.LabelPair, additionalLabelName string, additionalLabelValue float64, ) (int, error) { - if len(in) == 0 && additionalLabelName == "" { - return 0, nil - } var ( - written int - separator byte = '{' + written int + separator byte = '{' + metricInsideBraces = false ) + + if name != "" { + // If the name does not pass the legacy validity check, we must put the + // metric name inside the braces, quoted. + if !model.IsValidLegacyMetricName(model.LabelValue(name)) { + metricInsideBraces = true + err := w.WriteByte(separator) + written++ + if err != nil { + return written, err + } + separator = ',' + } + + n, err := writeName(w, name) + written += n + if err != nil { + return written, err + } + } + + if len(in) == 0 && additionalLabelName == "" { + if metricInsideBraces { + err := w.WriteByte('}') + written++ + if err != nil { + return written, err + } + } + return written, nil + } + for _, lp := range in { err := w.WriteByte(separator) written++ if err != nil { return written, err } - n, err := w.WriteString(lp.GetName()) + n, err := writeName(w, lp.GetName()) written += n if err != nil { return written, err @@ -451,7 +476,7 @@ func writeExemplar(w enhancedWriter, e *dto.Exemplar) (int, error) { if err != nil { return written, err } - n, err = writeOpenMetricsLabelPairs(w, e.Label, "", 0) + n, err = writeOpenMetricsNameAndLabelPairs(w, "", e.Label, "", 0) written += n if err != nil { return written, err diff --git a/expfmt/openmetrics_create_test.go b/expfmt/openmetrics_create_test.go index 455e5e5e..af3eea15 100644 --- a/expfmt/openmetrics_create_test.go +++ b/expfmt/openmetrics_create_test.go @@ -43,13 +43,13 @@ func TestCreateOpenMetrics(t *testing.T) { Help: proto.String("two-line\n doc str\\ing"), Type: dto.MetricType_COUNTER.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("labelname"), Value: proto.String("val1"), }, - &dto.LabelPair{ + { Name: proto.String("basename"), Value: proto.String("basevalue"), }, @@ -58,13 +58,13 @@ func TestCreateOpenMetrics(t *testing.T) { Value: proto.Float64(42), }, }, - &dto.Metric{ + { Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("labelname"), Value: proto.String("val2"), }, - &dto.LabelPair{ + { Name: proto.String("basename"), Value: proto.String("basevalue"), }, @@ -82,20 +82,92 @@ name{labelname="val1",basename="basevalue"} 42.0 name{labelname="val2",basename="basevalue"} 0.23 1.23456789e+06 `, }, - // 1: Gauge, some escaping required, +Inf as value, multi-byte characters in label values. + // 1: Dots in name + { + in: &dto.MetricFamily{ + Name: proto.String("name.with.dots"), + Help: proto.String("boring help"), + Type: dto.MetricType_COUNTER.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("labelname"), + Value: proto.String("val1"), + }, + { + Name: proto.String("basename"), + Value: proto.String("basevalue"), + }, + }, + Counter: &dto.Counter{ + Value: proto.Float64(42), + }, + }, + { + Label: []*dto.LabelPair{ + { + Name: proto.String("labelname"), + Value: proto.String("val2"), + }, + { + Name: proto.String("basename"), + Value: proto.String("basevalue"), + }, + }, + Counter: &dto.Counter{ + Value: proto.Float64(.23), + }, + TimestampMs: proto.Int64(1234567890), + }, + }, + }, + out: `# HELP "name.with.dots" boring help +# TYPE "name.with.dots" unknown +{"name.with.dots",labelname="val1",basename="basevalue"} 42.0 +{"name.with.dots",labelname="val2",basename="basevalue"} 0.23 1.23456789e+06 +`, + }, + // 2: Dots in name, no labels + { + in: &dto.MetricFamily{ + Name: proto.String("name.with.dots"), + Help: proto.String("boring help"), + Type: dto.MetricType_COUNTER.Enum(), + Metric: []*dto.Metric{ + { + Counter: &dto.Counter{ + Value: proto.Float64(42), + }, + }, + { + Counter: &dto.Counter{ + Value: proto.Float64(.23), + }, + TimestampMs: proto.Int64(1234567890), + }, + }, + }, + out: `# HELP "name.with.dots" boring help +# TYPE "name.with.dots" unknown +{"name.with.dots"} 42.0 +{"name.with.dots"} 0.23 1.23456789e+06 +`, + }, + // 3: Gauge, some escaping required, +Inf as value, multi-byte characters in label values. { in: &dto.MetricFamily{ Name: proto.String("gauge_name"), Help: proto.String("gauge\ndoc\nstr\"ing"), Type: dto.MetricType_GAUGE.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("name_1"), Value: proto.String("val with\nnew line"), }, - &dto.LabelPair{ + { Name: proto.String("name_2"), Value: proto.String("val with \\backslash and \"quotes\""), }, @@ -104,13 +176,13 @@ name{labelname="val2",basename="basevalue"} 0.23 1.23456789e+06 Value: proto.Float64(math.Inf(+1)), }, }, - &dto.Metric{ + { Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("name_1"), Value: proto.String("Björn"), }, - &dto.LabelPair{ + { Name: proto.String("name_2"), Value: proto.String("佖佥"), }, @@ -127,20 +199,65 @@ gauge_name{name_1="val with\nnew line",name_2="val with \\backslash and \"quotes gauge_name{name_1="Björn",name_2="佖佥"} 3.14e+42 `, }, - // 2: Unknown, no help, one sample with no labels and -Inf as value, another sample with one label. + // 4: Gauge, utf8, some escaping required, +Inf as value, multi-byte characters in label values. + { + in: &dto.MetricFamily{ + Name: proto.String("gauge.name\""), + Help: proto.String("gauge\ndoc\nstr\"ing"), + Type: dto.MetricType_GAUGE.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("name.1"), + Value: proto.String("val with\nnew line"), + }, + { + Name: proto.String("name*2"), + Value: proto.String("val with \\backslash and \"quotes\""), + }, + }, + Gauge: &dto.Gauge{ + Value: proto.Float64(math.Inf(+1)), + }, + }, + { + Label: []*dto.LabelPair{ + { + Name: proto.String("name.1"), + Value: proto.String("Björn"), + }, + { + Name: proto.String("name*2"), + Value: proto.String("佖佥"), + }, + }, + Gauge: &dto.Gauge{ + Value: proto.Float64(3.14e42), + }, + }, + }, + }, + out: `# HELP "gauge.name\"" gauge\ndoc\nstr\"ing +# TYPE "gauge.name\"" gauge +{"gauge.name\"","name.1"="val with\nnew line","name*2"="val with \\backslash and \"quotes\""} +Inf +{"gauge.name\"","name.1"="Björn","name*2"="佖佥"} 3.14e+42 +`, + }, + // 5: Unknown, no help, one sample with no labels and -Inf as value, another sample with one label. { in: &dto.MetricFamily{ Name: proto.String("unknown_name"), Type: dto.MetricType_UNTYPED.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Untyped: &dto.Untyped{ Value: proto.Float64(math.Inf(-1)), }, }, - &dto.Metric{ + { Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("name_1"), Value: proto.String("value 1"), }, @@ -156,40 +273,40 @@ unknown_name -Inf unknown_name{name_1="value 1"} -1.23e-45 `, }, - // 3: Summary. + // 6: Summary. { in: &dto.MetricFamily{ Name: proto.String("summary_name"), Help: proto.String("summary docstring"), Type: dto.MetricType_SUMMARY.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Summary: &dto.Summary{ SampleCount: proto.Uint64(42), SampleSum: proto.Float64(-3.4567), Quantile: []*dto.Quantile{ - &dto.Quantile{ + { Quantile: proto.Float64(0.5), Value: proto.Float64(-1.23), }, - &dto.Quantile{ + { Quantile: proto.Float64(0.9), Value: proto.Float64(.2342354), }, - &dto.Quantile{ + { Quantile: proto.Float64(0.99), Value: proto.Float64(0), }, }, }, }, - &dto.Metric{ + { Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("name_1"), Value: proto.String("value 1"), }, - &dto.LabelPair{ + { Name: proto.String("name_2"), Value: proto.String("value 2"), }, @@ -198,15 +315,15 @@ unknown_name{name_1="value 1"} -1.23e-45 SampleCount: proto.Uint64(4711), SampleSum: proto.Float64(2010.1971), Quantile: []*dto.Quantile{ - &dto.Quantile{ + { Quantile: proto.Float64(0.5), Value: proto.Float64(1), }, - &dto.Quantile{ + { Quantile: proto.Float64(0.9), Value: proto.Float64(2), }, - &dto.Quantile{ + { Quantile: proto.Float64(0.99), Value: proto.Float64(3), }, @@ -229,35 +346,35 @@ summary_name_sum{name_1="value 1",name_2="value 2"} 2010.1971 summary_name_count{name_1="value 1",name_2="value 2"} 4711 `, }, - // 4: Histogram + // 7: Histogram { in: &dto.MetricFamily{ Name: proto.String("request_duration_microseconds"), Help: proto.String("The response latency."), Type: dto.MetricType_HISTOGRAM.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Histogram: &dto.Histogram{ SampleCount: proto.Uint64(2693), SampleSum: proto.Float64(1756047.3), Bucket: []*dto.Bucket{ - &dto.Bucket{ + { UpperBound: proto.Float64(100), CumulativeCount: proto.Uint64(123), }, - &dto.Bucket{ + { UpperBound: proto.Float64(120), CumulativeCount: proto.Uint64(412), }, - &dto.Bucket{ + { UpperBound: proto.Float64(144), CumulativeCount: proto.Uint64(592), }, - &dto.Bucket{ + { UpperBound: proto.Float64(172.8), CumulativeCount: proto.Uint64(1524), }, - &dto.Bucket{ + { UpperBound: proto.Float64(math.Inf(+1)), CumulativeCount: proto.Uint64(2693), }, @@ -277,31 +394,31 @@ request_duration_microseconds_sum 1.7560473e+06 request_duration_microseconds_count 2693 `, }, - // 5: Histogram with missing +Inf bucket. + // 8: Histogram with missing +Inf bucket. { in: &dto.MetricFamily{ Name: proto.String("request_duration_microseconds"), Help: proto.String("The response latency."), Type: dto.MetricType_HISTOGRAM.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Histogram: &dto.Histogram{ SampleCount: proto.Uint64(2693), SampleSum: proto.Float64(1756047.3), Bucket: []*dto.Bucket{ - &dto.Bucket{ + { UpperBound: proto.Float64(100), CumulativeCount: proto.Uint64(123), }, - &dto.Bucket{ + { UpperBound: proto.Float64(120), CumulativeCount: proto.Uint64(412), }, - &dto.Bucket{ + { UpperBound: proto.Float64(144), CumulativeCount: proto.Uint64(592), }, - &dto.Bucket{ + { UpperBound: proto.Float64(172.8), CumulativeCount: proto.Uint64(1524), }, @@ -321,28 +438,28 @@ request_duration_microseconds_sum 1.7560473e+06 request_duration_microseconds_count 2693 `, }, - // 6: Histogram with missing +Inf bucket but with different exemplars. + // 9: Histogram with missing +Inf bucket but with different exemplars. { in: &dto.MetricFamily{ Name: proto.String("request_duration_microseconds"), Help: proto.String("The response latency."), Type: dto.MetricType_HISTOGRAM.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Histogram: &dto.Histogram{ SampleCount: proto.Uint64(2693), SampleSum: proto.Float64(1756047.3), Bucket: []*dto.Bucket{ - &dto.Bucket{ + { UpperBound: proto.Float64(100), CumulativeCount: proto.Uint64(123), }, - &dto.Bucket{ + { UpperBound: proto.Float64(120), CumulativeCount: proto.Uint64(412), Exemplar: &dto.Exemplar{ Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("foo"), Value: proto.String("bar"), }, @@ -351,16 +468,16 @@ request_duration_microseconds_count 2693 Timestamp: openMetricsTimestamp, }, }, - &dto.Bucket{ + { UpperBound: proto.Float64(144), CumulativeCount: proto.Uint64(592), Exemplar: &dto.Exemplar{ Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("foo"), Value: proto.String("baz"), }, - &dto.LabelPair{ + { Name: proto.String("dings"), Value: proto.String("bums"), }, @@ -368,7 +485,7 @@ request_duration_microseconds_count 2693 Value: proto.Float64(140.14), }, }, - &dto.Bucket{ + { UpperBound: proto.Float64(172.8), CumulativeCount: proto.Uint64(1524), }, @@ -388,14 +505,14 @@ request_duration_microseconds_sum 1.7560473e+06 request_duration_microseconds_count 2693 `, }, - // 7: Simple Counter. + // 10: Simple Counter. { in: &dto.MetricFamily{ Name: proto.String("foos_total"), Help: proto.String("Number of foos."), Type: dto.MetricType_COUNTER.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Counter: &dto.Counter{ Value: proto.Float64(42), }, @@ -407,7 +524,7 @@ request_duration_microseconds_count 2693 foos_total 42.0 `, }, - // 8: No metric. + // 11: No metric. { in: &dto.MetricFamily{ Name: proto.String("name_total"), @@ -450,17 +567,17 @@ func BenchmarkOpenMetricsCreate(b *testing.B) { Help: proto.String("The response latency."), Type: dto.MetricType_HISTOGRAM.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("name_1"), Value: proto.String("val with\nnew line"), }, - &dto.LabelPair{ + { Name: proto.String("name_2"), Value: proto.String("val with \\backslash and \"quotes\""), }, - &dto.LabelPair{ + { Name: proto.String("name_3"), Value: proto.String("Just a quite long label value to test performance."), }, @@ -469,40 +586,40 @@ func BenchmarkOpenMetricsCreate(b *testing.B) { SampleCount: proto.Uint64(2693), SampleSum: proto.Float64(1756047.3), Bucket: []*dto.Bucket{ - &dto.Bucket{ + { UpperBound: proto.Float64(100), CumulativeCount: proto.Uint64(123), }, - &dto.Bucket{ + { UpperBound: proto.Float64(120), CumulativeCount: proto.Uint64(412), }, - &dto.Bucket{ + { UpperBound: proto.Float64(144), CumulativeCount: proto.Uint64(592), }, - &dto.Bucket{ + { UpperBound: proto.Float64(172.8), CumulativeCount: proto.Uint64(1524), }, - &dto.Bucket{ + { UpperBound: proto.Float64(math.Inf(+1)), CumulativeCount: proto.Uint64(2693), }, }, }, }, - &dto.Metric{ + { Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("name_1"), Value: proto.String("Björn"), }, - &dto.LabelPair{ + { Name: proto.String("name_2"), Value: proto.String("佖佥"), }, - &dto.LabelPair{ + { Name: proto.String("name_3"), Value: proto.String("Just a quite long label value to test performance."), }, @@ -511,19 +628,19 @@ func BenchmarkOpenMetricsCreate(b *testing.B) { SampleCount: proto.Uint64(5699), SampleSum: proto.Float64(49484343543.4343), Bucket: []*dto.Bucket{ - &dto.Bucket{ + { UpperBound: proto.Float64(100), CumulativeCount: proto.Uint64(120), }, - &dto.Bucket{ + { UpperBound: proto.Float64(120), CumulativeCount: proto.Uint64(412), }, - &dto.Bucket{ + { UpperBound: proto.Float64(144), CumulativeCount: proto.Uint64(596), }, - &dto.Bucket{ + { UpperBound: proto.Float64(172.8), CumulativeCount: proto.Uint64(1535), }, @@ -555,7 +672,7 @@ func TestOpenMetricsCreateError(t *testing.T) { Help: proto.String("doc string"), Type: dto.MetricType_UNTYPED.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Untyped: &dto.Untyped{ Value: proto.Float64(math.Inf(-1)), }, @@ -571,7 +688,7 @@ func TestOpenMetricsCreateError(t *testing.T) { Help: proto.String("doc string"), Type: dto.MetricType_COUNTER.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Untyped: &dto.Untyped{ Value: proto.Float64(math.Inf(-1)), }, diff --git a/expfmt/text_create.go b/expfmt/text_create.go index 2946b8f1..f1b6a10b 100644 --- a/expfmt/text_create.go +++ b/expfmt/text_create.go @@ -62,6 +62,12 @@ var ( // contains duplicate metrics or invalid metric or label names, the conversion // will result in invalid text format output. // +// If metric names conform to the legacy validation pattern, they will be placed +// outside the brackets in the traditional way, like `foo{}`. If the metric name +// fails the legacy validation check, it will be placed quoted inside the +// brackets: `{"foo"}`. As stated above, the input is assumed to be santized and +// no error will be thrown in this case. +// // This method fulfills the type 'prometheus.encoder'. func MetricFamilyToText(out io.Writer, in *dto.MetricFamily) (written int, err error) { // Fail-fast checks. @@ -98,7 +104,7 @@ func MetricFamilyToText(out io.Writer, in *dto.MetricFamily) (written int, err e if err != nil { return } - n, err = w.WriteString(name) + n, err = writeName(w, name) written += n if err != nil { return @@ -124,7 +130,7 @@ func MetricFamilyToText(out io.Writer, in *dto.MetricFamily) (written int, err e if err != nil { return } - n, err = w.WriteString(name) + n, err = writeName(w, name) written += n if err != nil { return @@ -280,21 +286,9 @@ func writeSample( additionalLabelName string, additionalLabelValue float64, value float64, ) (int, error) { - var written int - n, err := w.WriteString(name) - written += n - if err != nil { - return written, err - } - if suffix != "" { - n, err = w.WriteString(suffix) - written += n - if err != nil { - return written, err - } - } - n, err = writeLabelPairs( - w, metric.Label, additionalLabelName, additionalLabelValue, + written := 0 + n, err := writeNameAndLabelPairs( + w, name+suffix, metric.Label, additionalLabelName, additionalLabelValue, ) written += n if err != nil { @@ -330,32 +324,62 @@ func writeSample( return written, nil } -// writeLabelPairs converts a slice of LabelPair proto messages plus the +// writeNameAndLabelPairs converts a slice of LabelPair proto messages plus the // explicitly given additional label pair into text formatted as required by the // text format and writes it to 'w'. An empty slice in combination with an empty // string 'additionalLabelName' results in nothing being written. Otherwise, the // label pairs are written, escaped as required by the text format, and enclosed // in '{...}'. The function returns the number of bytes written and any error // encountered. -func writeLabelPairs( +func writeNameAndLabelPairs( w enhancedWriter, + name string, in []*dto.LabelPair, additionalLabelName string, additionalLabelValue float64, ) (int, error) { - if len(in) == 0 && additionalLabelName == "" { - return 0, nil - } var ( - written int - separator byte = '{' + written int + separator byte = '{' + metricInsideBraces = false ) + + if name != "" { + // If the name does not pass the legacy validity check, we must put the + // metric name inside the braces. + if !model.IsValidLegacyMetricName(model.LabelValue(name)) { + metricInsideBraces = true + err := w.WriteByte(separator) + written++ + if err != nil { + return written, err + } + separator = ',' + } + n, err := writeName(w, name) + written += n + if err != nil { + return written, err + } + } + + if len(in) == 0 && additionalLabelName == "" { + if metricInsideBraces { + err := w.WriteByte('}') + written++ + if err != nil { + return written, err + } + } + return written, nil + } + for _, lp := range in { err := w.WriteByte(separator) written++ if err != nil { return written, err } - n, err := w.WriteString(lp.GetName()) + n, err := writeName(w, lp.GetName()) written += n if err != nil { return written, err @@ -462,3 +486,27 @@ func writeInt(w enhancedWriter, i int64) (int, error) { numBufPool.Put(bp) return written, err } + +// writeName writes a string as-is if it complies with the legacy naming +// scheme, or escapes it in double quotes if not. +func writeName(w enhancedWriter, name string) (int, error) { + if !model.IsValidLegacyMetricName(model.LabelValue(name)) { + var written int + var err error + err = w.WriteByte('"') + written++ + if err != nil { + return written, err + } + var n int + n, err = writeEscapedString(w, name, true) + written += n + if err != nil { + return written, err + } + err = w.WriteByte('"') + written++ + return written, err + } + return w.WriteString(name) +} diff --git a/expfmt/text_create_test.go b/expfmt/text_create_test.go index e6071666..fb20a863 100644 --- a/expfmt/text_create_test.go +++ b/expfmt/text_create_test.go @@ -36,13 +36,13 @@ func TestCreate(t *testing.T) { Help: proto.String("two-line\n doc str\\ing"), Type: dto.MetricType_COUNTER.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("labelname"), Value: proto.String("val1"), }, - &dto.LabelPair{ + { Name: proto.String("basename"), Value: proto.String("basevalue"), }, @@ -51,13 +51,13 @@ func TestCreate(t *testing.T) { Value: proto.Float64(math.NaN()), }, }, - &dto.Metric{ + { Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("labelname"), Value: proto.String("val2"), }, - &dto.LabelPair{ + { Name: proto.String("basename"), Value: proto.String("basevalue"), }, @@ -82,13 +82,13 @@ name{labelname="val2",basename="basevalue"} 0.23 1234567890 Help: proto.String("gauge\ndoc\nstr\"ing"), Type: dto.MetricType_GAUGE.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("name_1"), Value: proto.String("val with\nnew line"), }, - &dto.LabelPair{ + { Name: proto.String("name_2"), Value: proto.String("val with \\backslash and \"quotes\""), }, @@ -97,13 +97,13 @@ name{labelname="val2",basename="basevalue"} 0.23 1234567890 Value: proto.Float64(math.Inf(+1)), }, }, - &dto.Metric{ + { Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("name_1"), Value: proto.String("Björn"), }, - &dto.LabelPair{ + { Name: proto.String("name_2"), Value: proto.String("佖佥"), }, @@ -120,20 +120,65 @@ gauge_name{name_1="val with\nnew line",name_2="val with \\backslash and \"quotes gauge_name{name_1="Björn",name_2="佖佥"} 3.14e+42 `, }, - // 2: Untyped, no help, one sample with no labels and -Inf as value, another sample with one label. + // 2: Gauge, utf8, +Inf as value, multi-byte characters in label values. + { + in: &dto.MetricFamily{ + Name: proto.String("gauge.name"), + Help: proto.String("gauge\ndoc\nstr\"ing"), + Type: dto.MetricType_GAUGE.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("name.1"), + Value: proto.String("val with\nnew line"), + }, + { + Name: proto.String("name*2"), + Value: proto.String("val with \\backslash and \"quotes\""), + }, + }, + Gauge: &dto.Gauge{ + Value: proto.Float64(math.Inf(+1)), + }, + }, + { + Label: []*dto.LabelPair{ + { + Name: proto.String("name.1"), + Value: proto.String("Björn"), + }, + { + Name: proto.String("name*2"), + Value: proto.String("佖佥"), + }, + }, + Gauge: &dto.Gauge{ + Value: proto.Float64(3.14e42), + }, + }, + }, + }, + out: `# HELP "gauge.name" gauge\ndoc\nstr"ing +# TYPE "gauge.name" gauge +{"gauge.name","name.1"="val with\nnew line","name*2"="val with \\backslash and \"quotes\""} +Inf +{"gauge.name","name.1"="Björn","name*2"="佖佥"} 3.14e+42 +`, + }, + // 3: Untyped, no help, one sample with no labels and -Inf as value, another sample with one label. { in: &dto.MetricFamily{ Name: proto.String("untyped_name"), Type: dto.MetricType_UNTYPED.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Untyped: &dto.Untyped{ Value: proto.Float64(math.Inf(-1)), }, }, - &dto.Metric{ + { Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("name_1"), Value: proto.String("value 1"), }, @@ -149,40 +194,40 @@ untyped_name -Inf untyped_name{name_1="value 1"} -1.23e-45 `, }, - // 3: Summary. + // 4: Summary. { in: &dto.MetricFamily{ Name: proto.String("summary_name"), Help: proto.String("summary docstring"), Type: dto.MetricType_SUMMARY.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Summary: &dto.Summary{ SampleCount: proto.Uint64(42), SampleSum: proto.Float64(-3.4567), Quantile: []*dto.Quantile{ - &dto.Quantile{ + { Quantile: proto.Float64(0.5), Value: proto.Float64(-1.23), }, - &dto.Quantile{ + { Quantile: proto.Float64(0.9), Value: proto.Float64(.2342354), }, - &dto.Quantile{ + { Quantile: proto.Float64(0.99), Value: proto.Float64(0), }, }, }, }, - &dto.Metric{ + { Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("name_1"), Value: proto.String("value 1"), }, - &dto.LabelPair{ + { Name: proto.String("name_2"), Value: proto.String("value 2"), }, @@ -191,15 +236,15 @@ untyped_name{name_1="value 1"} -1.23e-45 SampleCount: proto.Uint64(4711), SampleSum: proto.Float64(2010.1971), Quantile: []*dto.Quantile{ - &dto.Quantile{ + { Quantile: proto.Float64(0.5), Value: proto.Float64(1), }, - &dto.Quantile{ + { Quantile: proto.Float64(0.9), Value: proto.Float64(2), }, - &dto.Quantile{ + { Quantile: proto.Float64(0.99), Value: proto.Float64(3), }, @@ -222,35 +267,35 @@ summary_name_sum{name_1="value 1",name_2="value 2"} 2010.1971 summary_name_count{name_1="value 1",name_2="value 2"} 4711 `, }, - // 4: Histogram + // 5: Histogram { in: &dto.MetricFamily{ Name: proto.String("request_duration_microseconds"), Help: proto.String("The response latency."), Type: dto.MetricType_HISTOGRAM.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Histogram: &dto.Histogram{ SampleCount: proto.Uint64(2693), SampleSum: proto.Float64(1756047.3), Bucket: []*dto.Bucket{ - &dto.Bucket{ + { UpperBound: proto.Float64(100), CumulativeCount: proto.Uint64(123), }, - &dto.Bucket{ + { UpperBound: proto.Float64(120), CumulativeCount: proto.Uint64(412), }, - &dto.Bucket{ + { UpperBound: proto.Float64(144), CumulativeCount: proto.Uint64(592), }, - &dto.Bucket{ + { UpperBound: proto.Float64(172.8), CumulativeCount: proto.Uint64(1524), }, - &dto.Bucket{ + { UpperBound: proto.Float64(math.Inf(+1)), CumulativeCount: proto.Uint64(2693), }, @@ -270,31 +315,31 @@ request_duration_microseconds_sum 1.7560473e+06 request_duration_microseconds_count 2693 `, }, - // 5: Histogram with missing +Inf bucket. + // 6: Histogram with missing +Inf bucket. { in: &dto.MetricFamily{ Name: proto.String("request_duration_microseconds"), Help: proto.String("The response latency."), Type: dto.MetricType_HISTOGRAM.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Histogram: &dto.Histogram{ SampleCount: proto.Uint64(2693), SampleSum: proto.Float64(1756047.3), Bucket: []*dto.Bucket{ - &dto.Bucket{ + { UpperBound: proto.Float64(100), CumulativeCount: proto.Uint64(123), }, - &dto.Bucket{ + { UpperBound: proto.Float64(120), CumulativeCount: proto.Uint64(412), }, - &dto.Bucket{ + { UpperBound: proto.Float64(144), CumulativeCount: proto.Uint64(592), }, - &dto.Bucket{ + { UpperBound: proto.Float64(172.8), CumulativeCount: proto.Uint64(1524), }, @@ -314,13 +359,13 @@ request_duration_microseconds_sum 1.7560473e+06 request_duration_microseconds_count 2693 `, }, - // 6: No metric type, should result in default type Counter. + // 7: No metric type, should result in default type Counter. { in: &dto.MetricFamily{ Name: proto.String("name"), Help: proto.String("doc string"), Metric: []*dto.Metric{ - &dto.Metric{ + { Counter: &dto.Counter{ Value: proto.Float64(math.Inf(-1)), }, @@ -363,17 +408,17 @@ func BenchmarkCreate(b *testing.B) { Help: proto.String("The response latency."), Type: dto.MetricType_HISTOGRAM.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("name_1"), Value: proto.String("val with\nnew line"), }, - &dto.LabelPair{ + { Name: proto.String("name_2"), Value: proto.String("val with \\backslash and \"quotes\""), }, - &dto.LabelPair{ + { Name: proto.String("name_3"), Value: proto.String("Just a quite long label value to test performance."), }, @@ -382,40 +427,40 @@ func BenchmarkCreate(b *testing.B) { SampleCount: proto.Uint64(2693), SampleSum: proto.Float64(1756047.3), Bucket: []*dto.Bucket{ - &dto.Bucket{ + { UpperBound: proto.Float64(100), CumulativeCount: proto.Uint64(123), }, - &dto.Bucket{ + { UpperBound: proto.Float64(120), CumulativeCount: proto.Uint64(412), }, - &dto.Bucket{ + { UpperBound: proto.Float64(144), CumulativeCount: proto.Uint64(592), }, - &dto.Bucket{ + { UpperBound: proto.Float64(172.8), CumulativeCount: proto.Uint64(1524), }, - &dto.Bucket{ + { UpperBound: proto.Float64(math.Inf(+1)), CumulativeCount: proto.Uint64(2693), }, }, }, }, - &dto.Metric{ + { Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("name_1"), Value: proto.String("Björn"), }, - &dto.LabelPair{ + { Name: proto.String("name_2"), Value: proto.String("佖佥"), }, - &dto.LabelPair{ + { Name: proto.String("name_3"), Value: proto.String("Just a quite long label value to test performance."), }, @@ -424,19 +469,19 @@ func BenchmarkCreate(b *testing.B) { SampleCount: proto.Uint64(5699), SampleSum: proto.Float64(49484343543.4343), Bucket: []*dto.Bucket{ - &dto.Bucket{ + { UpperBound: proto.Float64(100), CumulativeCount: proto.Uint64(120), }, - &dto.Bucket{ + { UpperBound: proto.Float64(120), CumulativeCount: proto.Uint64(412), }, - &dto.Bucket{ + { UpperBound: proto.Float64(144), CumulativeCount: proto.Uint64(596), }, - &dto.Bucket{ + { UpperBound: proto.Float64(172.8), CumulativeCount: proto.Uint64(1535), }, @@ -463,17 +508,17 @@ func BenchmarkCreateBuildInfo(b *testing.B) { Help: proto.String("Test the creation of constant 1-value build_info metric."), Type: dto.MetricType_GAUGE.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("version"), Value: proto.String("1.2.3"), }, - &dto.LabelPair{ + { Name: proto.String("revision"), Value: proto.String("2e84f5e4eacdffb574035810305191ff390360fe"), }, - &dto.LabelPair{ + { Name: proto.String("go_version"), Value: proto.String("1.11.1"), }, @@ -516,7 +561,7 @@ func TestCreateError(t *testing.T) { Help: proto.String("doc string"), Type: dto.MetricType_UNTYPED.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Untyped: &dto.Untyped{ Value: proto.Float64(math.Inf(-1)), }, @@ -532,7 +577,7 @@ func TestCreateError(t *testing.T) { Help: proto.String("doc string"), Type: dto.MetricType_COUNTER.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Untyped: &dto.Untyped{ Value: proto.Float64(math.Inf(-1)), }, diff --git a/expfmt/text_parse_test.go b/expfmt/text_parse_test.go index 204a88a3..f4a9c35f 100644 --- a/expfmt/text_parse_test.go +++ b/expfmt/text_parse_test.go @@ -45,22 +45,22 @@ no_labels{} 3 # HELP line for non-existing metric will be ignored. `, out: []*dto.MetricFamily{ - &dto.MetricFamily{ + { Name: proto.String("minimal_metric"), Type: dto.MetricType_UNTYPED.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Untyped: &dto.Untyped{ Value: proto.Float64(1.234), }, }, }, }, - &dto.MetricFamily{ + { Name: proto.String("another_metric"), Type: dto.MetricType_UNTYPED.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Untyped: &dto.Untyped{ Value: proto.Float64(-3e3), }, @@ -68,11 +68,11 @@ no_labels{} 3 }, }, }, - &dto.MetricFamily{ + { Name: proto.String("no_labels"), Type: dto.MetricType_UNTYPED.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Untyped: &dto.Untyped{ Value: proto.Float64(3), }, @@ -97,18 +97,18 @@ name2{labelname="val2" ,basename = "basevalue2" } +Inf 54321 name2{ labelname = "val1" , }-Inf `, out: []*dto.MetricFamily{ - &dto.MetricFamily{ + { Name: proto.String("name"), Help: proto.String("two-line\n doc str\\ing"), Type: dto.MetricType_COUNTER.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("labelname"), Value: proto.String("val1"), }, - &dto.LabelPair{ + { Name: proto.String("basename"), Value: proto.String("basevalue"), }, @@ -117,13 +117,13 @@ name2{ labelname = "val1" , }-Inf Value: proto.Float64(math.NaN()), }, }, - &dto.Metric{ + { Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("labelname"), Value: proto.String("val2"), }, - &dto.LabelPair{ + { Name: proto.String("basename"), Value: proto.String("base\"v\\al\nue"), }, @@ -135,18 +135,18 @@ name2{ labelname = "val1" , }-Inf }, }, }, - &dto.MetricFamily{ + { Name: proto.String("name2"), Help: proto.String("doc str\"ing 2"), Type: dto.MetricType_GAUGE.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("labelname"), Value: proto.String("val2"), }, - &dto.LabelPair{ + { Name: proto.String("basename"), Value: proto.String("basevalue2"), }, @@ -156,9 +156,9 @@ name2{ labelname = "val1" , }-Inf }, TimestampMs: proto.Int64(54321), }, - &dto.Metric{ + { Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("labelname"), Value: proto.String("val1"), }, @@ -197,13 +197,13 @@ my_summary{n1="val3", quantile="0.2"} 4711 # HELP my_summary `, out: []*dto.MetricFamily{ - &dto.MetricFamily{ + { Name: proto.String("fake_sum"), Type: dto.MetricType_UNTYPED.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("n1"), Value: proto.String("val1"), }, @@ -214,11 +214,11 @@ my_summary{n1="val3", quantile="0.2"} 4711 }, }, }, - &dto.MetricFamily{ + { Name: proto.String("decoy"), Type: dto.MetricType_UNTYPED.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Untyped: &dto.Untyped{ Value: proto.Float64(-1), }, @@ -226,13 +226,13 @@ my_summary{n1="val3", quantile="0.2"} 4711 }, }, }, - &dto.MetricFamily{ + { Name: proto.String("my_summary"), Type: dto.MetricType_SUMMARY.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("n1"), Value: proto.String("val1"), }, @@ -241,11 +241,11 @@ my_summary{n1="val3", quantile="0.2"} 4711 SampleCount: proto.Uint64(42), SampleSum: proto.Float64(4711), Quantile: []*dto.Quantile{ - &dto.Quantile{ + { Quantile: proto.Float64(0.5), Value: proto.Float64(110), }, - &dto.Quantile{ + { Quantile: proto.Float64(0.9), Value: proto.Float64(140), }, @@ -253,13 +253,13 @@ my_summary{n1="val3", quantile="0.2"} 4711 }, TimestampMs: proto.Int64(2), }, - &dto.Metric{ + { Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("n2"), Value: proto.String("val2"), }, - &dto.LabelPair{ + { Name: proto.String("n1"), Value: proto.String("val1"), }, @@ -267,7 +267,7 @@ my_summary{n1="val3", quantile="0.2"} 4711 Summary: &dto.Summary{ SampleCount: proto.Uint64(5), Quantile: []*dto.Quantile{ - &dto.Quantile{ + { Quantile: proto.Float64(-12.34), Value: proto.Float64(math.NaN()), }, @@ -275,9 +275,9 @@ my_summary{n1="val3", quantile="0.2"} 4711 }, TimestampMs: proto.Int64(5), }, - &dto.Metric{ + { Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("n1"), Value: proto.String("val2"), }, @@ -287,16 +287,16 @@ my_summary{n1="val3", quantile="0.2"} 4711 }, TimestampMs: proto.Int64(15), }, - &dto.Metric{ + { Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("n1"), Value: proto.String("val3"), }, }, Summary: &dto.Summary{ Quantile: []*dto.Quantile{ - &dto.Quantile{ + { Quantile: proto.Float64(0.2), Value: proto.Float64(4711), }, @@ -305,17 +305,17 @@ my_summary{n1="val3", quantile="0.2"} 4711 }, }, }, - &dto.MetricFamily{ + { Name: proto.String("another_summary"), Type: dto.MetricType_SUMMARY.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Label: []*dto.LabelPair{ - &dto.LabelPair{ + { Name: proto.String("n2"), Value: proto.String("val2"), }, - &dto.LabelPair{ + { Name: proto.String("n1"), Value: proto.String("val1"), }, @@ -323,7 +323,7 @@ my_summary{n1="val3", quantile="0.2"} 4711 Summary: &dto.Summary{ SampleCount: proto.Uint64(20), Quantile: []*dto.Quantile{ - &dto.Quantile{ + { Quantile: proto.Float64(0.3), Value: proto.Float64(-1.2), }, @@ -353,28 +353,28 @@ request_duration_microseconds_count 2693 Help: proto.String("The response latency."), Type: dto.MetricType_HISTOGRAM.Enum(), Metric: []*dto.Metric{ - &dto.Metric{ + { Histogram: &dto.Histogram{ SampleCount: proto.Uint64(2693), SampleSum: proto.Float64(1756047.3), Bucket: []*dto.Bucket{ - &dto.Bucket{ + { UpperBound: proto.Float64(100), CumulativeCount: proto.Uint64(123), }, - &dto.Bucket{ + { UpperBound: proto.Float64(120), CumulativeCount: proto.Uint64(412), }, - &dto.Bucket{ + { UpperBound: proto.Float64(144), CumulativeCount: proto.Uint64(592), }, - &dto.Bucket{ + { UpperBound: proto.Float64(172.8), CumulativeCount: proto.Uint64(1524), }, - &dto.Bucket{ + { UpperBound: proto.Float64(math.Inf(+1)), CumulativeCount: proto.Uint64(2693), }, diff --git a/model/labels.go b/model/labels.go index ef895633..73dafe43 100644 --- a/model/labels.go +++ b/model/labels.go @@ -97,17 +97,25 @@ var LabelNameRE = regexp.MustCompile("^[a-zA-Z_][a-zA-Z0-9_]*$") // therewith. type LabelName string -// IsValid is true iff the label name matches the pattern of LabelNameRE. This -// method, however, does not use LabelNameRE for the check but a much faster -// hardcoded implementation. +// IsValid returns true iff name matches the pattern of LabelNameRE for legacy +// names, and iff it's valid UTF-8 if NameValidationScheme is set to +// UTF8Validation. For the legacy matching, it does not use LabelNameRE for the +// check but a much faster hardcoded implementation. func (ln LabelName) IsValid() bool { if len(ln) == 0 { return false } - for i, b := range ln { - if !((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || b == '_' || (b >= '0' && b <= '9' && i > 0)) { - return false + switch NameValidationScheme { + case LegacyValidation: + for i, b := range ln { + if !((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || b == '_' || (b >= '0' && b <= '9' && i > 0)) { + return false + } } + case UTF8Validation: + return utf8.ValidString(string(ln)) + default: + panic(fmt.Sprintf("Invalid name validation scheme requested: %d", NameValidationScheme)) } return true } diff --git a/model/labels_test.go b/model/labels_test.go index 2ee5b31a..80bf9269 100644 --- a/model/labels_test.go +++ b/model/labels_test.go @@ -92,49 +92,68 @@ func BenchmarkLabelValues(b *testing.B) { func TestLabelNameIsValid(t *testing.T) { var scenarios = []struct { - ln LabelName - valid bool + ln LabelName + legacyValid bool + utf8Valid bool }{ { - ln: "Avalid_23name", - valid: true, + ln: "Avalid_23name", + legacyValid: true, + utf8Valid: true, }, { - ln: "_Avalid_23name", - valid: true, + ln: "_Avalid_23name", + legacyValid: true, + utf8Valid: true, }, { - ln: "1valid_23name", - valid: false, + ln: "1valid_23name", + legacyValid: false, + utf8Valid: true, }, { - ln: "avalid_23name", - valid: true, + ln: "avalid_23name", + legacyValid: true, + utf8Valid: true, }, { - ln: "Ava:lid_23name", - valid: false, + ln: "Ava:lid_23name", + legacyValid: false, + utf8Valid: true, }, { - ln: "a lid_23name", - valid: false, + ln: "a lid_23name", + legacyValid: false, + utf8Valid: true, }, { - ln: ":leading_colon", - valid: false, + ln: ":leading_colon", + legacyValid: false, + utf8Valid: true, }, { - ln: "colon:in:the:middle", - valid: false, + ln: "colon:in:the:middle", + legacyValid: false, + utf8Valid: true, + }, + { + ln: "a\xc5z", + legacyValid: false, + utf8Valid: false, }, } for _, s := range scenarios { - if s.ln.IsValid() != s.valid { - t.Errorf("Expected %v for %q using IsValid method", s.valid, s.ln) + NameValidationScheme = LegacyValidation + if s.ln.IsValid() != s.legacyValid { + t.Errorf("Expected %v for %q using legacy IsValid method", s.legacyValid, s.ln) + } + if LabelNameRE.MatchString(string(s.ln)) != s.legacyValid { + t.Errorf("Expected %v for %q using legacy regexp match", s.legacyValid, s.ln) } - if LabelNameRE.MatchString(string(s.ln)) != s.valid { - t.Errorf("Expected %v for %q using regexp match", s.valid, s.ln) + NameValidationScheme = UTF8Validation + if s.ln.IsValid() != s.utf8Valid { + t.Errorf("Expected %v for %q using UTF8 IsValid method", s.legacyValid, s.ln) } } } diff --git a/model/labelset_test.go b/model/labelset_test.go index dfdfc594..4d63f0b6 100644 --- a/model/labelset_test.go +++ b/model/labelset_test.go @@ -53,6 +53,7 @@ func TestUnmarshalJSONLabelSet(t *testing.T) { } }` + NameValidationScheme = LegacyValidation err = json.Unmarshal([]byte(invalidlabelSetJSON), &c) expectedErr := `"1nvalid_23name" is not a valid label name` if err == nil || err.Error() != expectedErr { diff --git a/model/metric.go b/model/metric.go index 00804b7f..9aa0b511 100644 --- a/model/metric.go +++ b/model/metric.go @@ -18,9 +18,34 @@ import ( "regexp" "sort" "strings" + "unicode/utf8" +) + +// ValidationScheme is a Go enum for determining how metric and label names will +// be validated by this library. +type ValidationScheme int + +const ( + // LegacyValidation is a setting that requirets that metric and label names + // conform to the original Prometheus character requirements described by + // MetricNameRE and LabelNameRE. + LegacyValidation ValidationScheme = iota + + // UTF8Validation only requires that metric and label names be valid UTF8 + // strings. + UTF8Validation ) var ( + // NameValidationScheme determines the method of name validation to be used by + // all calls to IsValidMetricName() and LabelName IsValid(). Setting UTF8 mode + // in isolation from other components that don't support UTF8 may result in + // bugs or other undefined behavior. This value is intended to be set by + // UTF8-aware binaries as part of their startup. To avoid need for locking, + // this value should be set once, ideally in an init(), before multiple + // goroutines are started. + NameValidationScheme = LegacyValidation + // MetricNameRE is a regular expression matching valid metric // names. Note that the IsValidMetricName function performs the same // check but faster than a match with this regular expression. @@ -86,10 +111,28 @@ func (m Metric) FastFingerprint() Fingerprint { return LabelSet(m).FastFingerprint() } -// IsValidMetricName returns true iff name matches the pattern of MetricNameRE. +// IsValidMetricName returns true iff name matches the pattern of MetricNameRE +// for legacy names, and iff it's valid UTF-8 if the UTF8Validation scheme is +// selected. +func IsValidMetricName(n LabelValue) bool { + switch NameValidationScheme { + case LegacyValidation: + return IsValidLegacyMetricName(n) + case UTF8Validation: + if len(n) == 0 { + return false + } + return utf8.ValidString(string(n)) + default: + panic(fmt.Sprintf("Invalid name validation scheme requested: %d", NameValidationScheme)) + } +} + +// IsValidLegacyMetricName is similar to IsValidMetricName but always uses the +// legacy validation scheme regardless of the value of NameValidationScheme. // This function, however, does not use MetricNameRE for the check but a much // faster hardcoded implementation. -func IsValidMetricName(n LabelValue) bool { +func IsValidLegacyMetricName(n LabelValue) bool { if len(n) == 0 { return false } diff --git a/model/metric_test.go b/model/metric_test.go index db447f6f..0beaeac1 100644 --- a/model/metric_test.go +++ b/model/metric_test.go @@ -82,55 +82,75 @@ func BenchmarkMetric(b *testing.B) { } } -func TestMetricNameIsValid(t *testing.T) { +func TestMetricNameIsLegacyValid(t *testing.T) { var scenarios = []struct { - mn LabelValue - valid bool + mn LabelValue + legacyValid bool + utf8Valid bool }{ { - mn: "Avalid_23name", - valid: true, + mn: "Avalid_23name", + legacyValid: true, + utf8Valid: true, }, { - mn: "_Avalid_23name", - valid: true, + mn: "_Avalid_23name", + legacyValid: true, + utf8Valid: true, }, { - mn: "1valid_23name", - valid: false, + mn: "1valid_23name", + legacyValid: false, + utf8Valid: true, }, { - mn: "avalid_23name", - valid: true, + mn: "avalid_23name", + legacyValid: true, + utf8Valid: true, }, { - mn: "Ava:lid_23name", - valid: true, + mn: "Ava:lid_23name", + legacyValid: true, + utf8Valid: true, }, { - mn: "a lid_23name", - valid: false, + mn: "a lid_23name", + legacyValid: false, + utf8Valid: true, }, { - mn: ":leading_colon", - valid: true, + mn: ":leading_colon", + legacyValid: true, + utf8Valid: true, }, { - mn: "colon:in:the:middle", - valid: true, + mn: "colon:in:the:middle", + legacyValid: true, + utf8Valid: true, }, { - mn: "", - valid: false, + mn: "", + legacyValid: false, + utf8Valid: false, + }, + { + mn: "a\xc5z", + legacyValid: false, + utf8Valid: false, }, } for _, s := range scenarios { - if IsValidMetricName(s.mn) != s.valid { - t.Errorf("Expected %v for %q using IsValidMetricName function", s.valid, s.mn) + NameValidationScheme = LegacyValidation + if IsValidMetricName(s.mn) != s.legacyValid { + t.Errorf("Expected %v for %q using legacy IsValidMetricName method", s.legacyValid, s.mn) + } + if MetricNameRE.MatchString(string(s.mn)) != s.legacyValid { + t.Errorf("Expected %v for %q using regexp matching", s.legacyValid, s.mn) } - if MetricNameRE.MatchString(string(s.mn)) != s.valid { - t.Errorf("Expected %v for %q using regexp matching", s.valid, s.mn) + NameValidationScheme = UTF8Validation + if IsValidMetricName(s.mn) != s.utf8Valid { + t.Errorf("Expected %v for %q using utf8 IsValidMetricName method", s.legacyValid, s.mn) } } } diff --git a/model/signature_test.go b/model/signature_test.go index 0c797058..9b04d533 100644 --- a/model/signature_test.go +++ b/model/signature_test.go @@ -157,12 +157,12 @@ func TestSignatureWithoutLabels(t *testing.T) { }, { in: Metric{"name": "garland, briggs", "fear": "love is not enough"}, - labels: map[LabelName]struct{}{"fear": struct{}{}, "name": struct{}{}}, + labels: map[LabelName]struct{}{"fear": {}, "name": {}}, out: 14695981039346656037, }, { in: Metric{"name": "garland, briggs", "fear": "love is not enough", "foo": "bar"}, - labels: map[LabelName]struct{}{"foo": struct{}{}}, + labels: map[LabelName]struct{}{"foo": {}}, out: 5799056148416392346, }, { diff --git a/model/silence_test.go b/model/silence_test.go index be50298b..d46afa47 100644 --- a/model/silence_test.go +++ b/model/silence_test.go @@ -21,8 +21,9 @@ import ( func TestMatcherValidate(t *testing.T) { var cases = []struct { - matcher *Matcher - err string + matcher *Matcher + legacyErr string + utf8Err string }{ { matcher: &Matcher{ @@ -42,46 +43,74 @@ func TestMatcherValidate(t *testing.T) { Name: "name!", Value: "value", }, - err: "invalid name", + legacyErr: "invalid name", }, { matcher: &Matcher{ Name: "", Value: "value", }, - err: "invalid name", + legacyErr: "invalid name", + utf8Err: "invalid name", }, { matcher: &Matcher{ Name: "name", Value: "value\xff", }, - err: "invalid value", + legacyErr: "invalid value", + utf8Err: "invalid value", }, { matcher: &Matcher{ Name: "name", Value: "", }, - err: "invalid value", + legacyErr: "invalid value", + utf8Err: "invalid value", + }, + { + matcher: &Matcher{ + Name: "a\xc5z", + Value: "", + }, + legacyErr: "invalid name", + utf8Err: "invalid name", }, } for i, c := range cases { - err := c.matcher.Validate() - if err == nil { - if c.err == "" { + NameValidationScheme = LegacyValidation + legacyErr := c.matcher.Validate() + NameValidationScheme = UTF8Validation + utf8Err := c.matcher.Validate() + if legacyErr == nil && utf8Err == nil { + if c.legacyErr == "" && c.utf8Err == "" { continue } - t.Errorf("%d. Expected error %q but got none", i, c.err) + if c.legacyErr != "" { + t.Errorf("%d. Expected error for legacy validation %q but got none", i, c.legacyErr) + } + if c.utf8Err != "" { + t.Errorf("%d. Expected error for utf8 validation %q but got none", i, c.utf8Err) + } continue } - if c.err == "" { - t.Errorf("%d. Expected no error but got %q", i, err) - continue + if legacyErr != nil { + if c.legacyErr == "" { + t.Errorf("%d. Expected no legacy validation error but got %q", i, legacyErr) + } else if !strings.Contains(legacyErr.Error(), c.legacyErr) { + t.Errorf("%d. Expected error to contain %q but got %q", i, c.legacyErr, legacyErr) + } } - if !strings.Contains(err.Error(), c.err) { - t.Errorf("%d. Expected error to contain %q but got %q", i, c.err, err) + if utf8Err != nil { + if c.utf8Err == "" { + t.Errorf("%d. Expected no utf8 validation error but got %q", i, utf8Err) + continue + } + if !strings.Contains(utf8Err.Error(), c.utf8Err) { + t.Errorf("%d. Expected error to contain %q but got %q", i, c.utf8Err, utf8Err) + } } } } @@ -219,6 +248,7 @@ func TestSilenceValidate(t *testing.T) { } for i, c := range cases { + NameValidationScheme = LegacyValidation err := c.sil.Validate() if err == nil { if c.err == "" { From 93343e3a60629724157057802f192e2903a3bd3c Mon Sep 17 00:00:00 2001 From: Owen Williams Date: Thu, 18 Jan 2024 09:44:41 -0500 Subject: [PATCH 2/4] Address notes Signed-off-by: Owen Williams --- expfmt/openmetrics_create.go | 6 ++++ expfmt/text_create.go | 55 +++++++++++++++++++++--------------- 2 files changed, 38 insertions(+), 23 deletions(-) diff --git a/expfmt/openmetrics_create.go b/expfmt/openmetrics_create.go index 6bf5e5f3..5622578e 100644 --- a/expfmt/openmetrics_create.go +++ b/expfmt/openmetrics_create.go @@ -41,6 +41,12 @@ import ( // brackets: `{"foo"}`. As stated above, the input is assumed to be santized and // no error will be thrown in this case. // +// Similar to metric names, if label names conform to the legacy validation +// pattern, they will be unquoted as normal, like `foo{bar="baz"}`. If the label +// name fails the legacy validation check, it will be quoted: +// `foo{"bar"="baz"}`. As stated above, the input is assumed to be santized and +// no error will be thrown in this case. +// // This function fulfills the type 'expfmt.encoder'. // // Note that OpenMetrics requires a final `# EOF` line. Since this function acts diff --git a/expfmt/text_create.go b/expfmt/text_create.go index f1b6a10b..415f3513 100644 --- a/expfmt/text_create.go +++ b/expfmt/text_create.go @@ -68,6 +68,13 @@ var ( // brackets: `{"foo"}`. As stated above, the input is assumed to be santized and // no error will be thrown in this case. // +// Similar to metric names, if label names conform to the legacy validation +// pattern, they will be unquoted as normal, like `foo{bar="baz"}`. If the label +// name fails the legacy validation check, it will be quoted: +// `foo{"bar"="baz"}`. As stated above, the input is assumed to be santized and +// no error will be thrown in this case. +// +// // This method fulfills the type 'prometheus.encoder'. func MetricFamilyToText(out io.Writer, in *dto.MetricFamily) (written int, err error) { // Fail-fast checks. @@ -325,12 +332,14 @@ func writeSample( } // writeNameAndLabelPairs converts a slice of LabelPair proto messages plus the -// explicitly given additional label pair into text formatted as required by the -// text format and writes it to 'w'. An empty slice in combination with an empty -// string 'additionalLabelName' results in nothing being written. Otherwise, the -// label pairs are written, escaped as required by the text format, and enclosed -// in '{...}'. The function returns the number of bytes written and any error -// encountered. +// explicitly given metric name and additional label pair into text formatted as +// required by the text format and writes it to 'w'. An empty slice in +// combination with an empty string 'additionalLabelName' results in nothing +// being written. Otherwise, the label pairs are written, escaped as required by +// the text format, and enclosed in '{...}'. The function returns the number of +// bytes written and any error encountered. If the metric name is not +// legacy-valid, it will be put inside the brackets as well. Legacy-invalid +// label names will also be quoted. func writeNameAndLabelPairs( w enhancedWriter, name string, @@ -490,23 +499,23 @@ func writeInt(w enhancedWriter, i int64) (int, error) { // writeName writes a string as-is if it complies with the legacy naming // scheme, or escapes it in double quotes if not. func writeName(w enhancedWriter, name string) (int, error) { - if !model.IsValidLegacyMetricName(model.LabelValue(name)) { - var written int - var err error - err = w.WriteByte('"') - written++ - if err != nil { - return written, err - } - var n int - n, err = writeEscapedString(w, name, true) - written += n - if err != nil { - return written, err - } - err = w.WriteByte('"') - written++ + if model.IsValidLegacyMetricName(model.LabelValue(name)) { + return w.WriteString(name) + } + var written int + var err error + err = w.WriteByte('"') + written++ + if err != nil { + return written, err + } + var n int + n, err = writeEscapedString(w, name, true) + written += n + if err != nil { return written, err } - return w.WriteString(name) + err = w.WriteByte('"') + written++ + return written, err } From ca4809721e9c855e40f5b7314d12e492d4b401a9 Mon Sep 17 00:00:00 2001 From: Owen Williams Date: Thu, 18 Jan 2024 10:48:09 -0500 Subject: [PATCH 3/4] lint Signed-off-by: Owen Williams --- expfmt/text_create.go | 1 - 1 file changed, 1 deletion(-) diff --git a/expfmt/text_create.go b/expfmt/text_create.go index 415f3513..f9b8265a 100644 --- a/expfmt/text_create.go +++ b/expfmt/text_create.go @@ -74,7 +74,6 @@ var ( // `foo{"bar"="baz"}`. As stated above, the input is assumed to be santized and // no error will be thrown in this case. // -// // This method fulfills the type 'prometheus.encoder'. func MetricFamilyToText(out io.Writer, in *dto.MetricFamily) (written int, err error) { // Fail-fast checks. From d6767c699aac6daaba595a20269c4b13b4b84da1 Mon Sep 17 00:00:00 2001 From: Owen Williams Date: Mon, 22 Jan 2024 09:15:55 -0500 Subject: [PATCH 4/4] Add comment about negotiation Signed-off-by: Owen Williams --- expfmt/expfmt.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/expfmt/expfmt.go b/expfmt/expfmt.go index c4cb20f0..d866b474 100644 --- a/expfmt/expfmt.go +++ b/expfmt/expfmt.go @@ -17,7 +17,13 @@ package expfmt // Format specifies the HTTP content type of the different wire protocols. type Format string -// Constants to assemble the Content-Type values for the different wire protocols. +// Constants to assemble the Content-Type values for the different wire +// protocols. The Content-Type strings here are all for the legacy exposition +// formats, where valid characters for metric names and label names are limited. +// Support for arbitrary UTF-8 characters in those names is already partially +// implemented in this module (see model.ValidationScheme), but to actually use +// it on the wire, new content-type strings will have to be agreed upon and +// added here. const ( TextVersion = "0.0.4" ProtoType = `application/vnd.google.protobuf`