From 8550c318ef49e46998dbc6fbfe36eca43f469b74 Mon Sep 17 00:00:00 2001 From: Saul Rennison Date: Sun, 27 Oct 2024 11:07:48 +0000 Subject: [PATCH 1/2] Add exemplar support to Prometheus exporter --- .../CHANGELOG.md | 1 + .../README.md | 8 - .../CHANGELOG.md | 1 + .../Internal/PrometheusExporterEventSource.cs | 6 + .../Internal/PrometheusSerializerExt.cs | 132 ++- .../PrometheusExporterMiddlewareTests.cs | 789 +++++++++++++++++- .../PrometheusHttpListenerTests.cs | 258 +++++- 7 files changed, 1127 insertions(+), 68 deletions(-) diff --git a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md index 33cf84f17dd..50ec5371cde 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/CHANGELOG.md @@ -9,6 +9,7 @@ Notes](../../RELEASENOTES.md). * Added meter-level tags to Prometheus exporter ([#5837](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5837)) +* Added exemplar support to Prometheus exporter ([#5929](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5929)) ## 1.9.0-beta.2 diff --git a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/README.md b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/README.md index 62cbca100f5..468ecef0e99 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/README.md +++ b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/README.md @@ -18,14 +18,6 @@ to scrape. Grafana](../../docs/metrics/getting-started-prometheus-grafana/README.md) tutorial for more information. - - -> [!NOTE] -> This exporter does not support Exemplars. For using Exemplars, use the [OTLP -Exporter](../OpenTelemetry.Exporter.OpenTelemetryProtocol/README.md) and use a -component like OTel Collector to expose metrics (with exemplars) to Prometheus. -This [tutorial](../../docs/metrics/exemplars/README.md) shows one way how to do that. - ## Prerequisite * [Get Prometheus](https://prometheus.io/docs/introduction/first_steps/) diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md index 5873577fb86..b91e40990f6 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/CHANGELOG.md @@ -9,6 +9,7 @@ Notes](../../RELEASENOTES.md). * Added meter-level tags to Prometheus exporter ([#5837](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5837)) +* Added exemplar support to Prometheus exporter ([#5929](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5929)) ## 1.9.0-beta.2 diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusExporterEventSource.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusExporterEventSource.cs index a55cc6ef553..de5bf61d93a 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusExporterEventSource.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusExporterEventSource.cs @@ -64,4 +64,10 @@ public void NoMetrics() { this.WriteEvent(4); } + + [Event(5, Message = "Ignoring exemplar tags that are too long for metric: '{0}'", Level = EventLevel.Warning)] + public void ExemplarTagsTooLong(string metricName) + { + this.WriteEvent(5); + } } diff --git a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusSerializerExt.cs b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusSerializerExt.cs index 70d67f468bf..23779842821 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusSerializerExt.cs +++ b/src/OpenTelemetry.Exporter.Prometheus.HttpListener/Internal/PrometheusSerializerExt.cs @@ -1,6 +1,7 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 +using System.Diagnostics; using OpenTelemetry.Metrics; namespace OpenTelemetry.Exporter.Prometheus; @@ -28,8 +29,11 @@ public static int WriteMetric(byte[] buffer, int cursor, Metric metric, Promethe cursor = WriteUnitMetadata(buffer, cursor, prometheusMetric, openMetricsRequested); cursor = WriteHelpMetadata(buffer, cursor, prometheusMetric, metric.Description, openMetricsRequested); + var isLong = metric.MetricType.IsLong(); if (!metric.MetricType.IsHistogram()) { + var isSum = metric.MetricType.IsSum(); + foreach (ref readonly var metricPoint in metric.GetMetricPoints()) { var timestamp = metricPoint.EndTime.ToUnixTimeMilliseconds(); @@ -40,12 +44,9 @@ public static int WriteMetric(byte[] buffer, int cursor, Metric metric, Promethe buffer[cursor++] = unchecked((byte)' '); - // TODO: MetricType is same for all MetricPoints - // within a given Metric, so this check can avoided - // for each MetricPoint - if (((int)metric.MetricType & 0b_0000_1111) == 0x0a /* I8 */) + if (isLong) { - if (metric.MetricType.IsSum()) + if (isSum) { cursor = WriteLong(buffer, cursor, metricPoint.GetSumLong()); } @@ -56,7 +57,7 @@ public static int WriteMetric(byte[] buffer, int cursor, Metric metric, Promethe } else { - if (metric.MetricType.IsSum()) + if (isSum) { cursor = WriteDouble(buffer, cursor, metricPoint.GetSumDouble()); } @@ -70,16 +71,27 @@ public static int WriteMetric(byte[] buffer, int cursor, Metric metric, Promethe cursor = WriteTimestamp(buffer, cursor, timestamp, openMetricsRequested); + if (isSum && openMetricsRequested && metricPoint.TryGetExemplars(out var exemplarCollection)) + { + cursor = WriteSumExemplar(buffer, cursor, metric, exemplarCollection); + } + buffer[cursor++] = ASCII_LINEFEED; } } else { + Debug.Assert(!isLong, "Expected histogram metric to be of type `double`"); + foreach (ref readonly var metricPoint in metric.GetMetricPoints()) { var tags = metricPoint.Tags; var timestamp = metricPoint.EndTime.ToUnixTimeMilliseconds(); + metricPoint.TryGetExemplars(out var exemplarCollection); + var exemplars = exemplarCollection.GetEnumerator(); + var hasExemplar = exemplars.MoveNext(); + long totalCount = 0; foreach (var histogramMeasurement in metricPoint.GetHistogramBuckets()) { @@ -107,6 +119,19 @@ public static int WriteMetric(byte[] buffer, int cursor, Metric metric, Promethe cursor = WriteTimestamp(buffer, cursor, timestamp, openMetricsRequested); + if (hasExemplar && openMetricsRequested) + { + if (exemplars.Current.DoubleValue <= histogramMeasurement.ExplicitBound) + { + cursor = WriteExemplar(buffer, cursor, exemplars.Current, metric.Name, isLong: false); + } + + while (hasExemplar && exemplars.Current.DoubleValue <= histogramMeasurement.ExplicitBound) + { + hasExemplar = exemplars.MoveNext(); + } + } + buffer[cursor++] = ASCII_LINEFEED; } @@ -142,4 +167,99 @@ public static int WriteMetric(byte[] buffer, int cursor, Metric metric, Promethe return cursor; } + + private static int WriteSumExemplar( + byte[] buffer, + int cursor, + in Metric metric, + in ReadOnlyExemplarCollection exemplarCollection) + { + var exemplars = exemplarCollection.GetEnumerator(); + if (!exemplars.MoveNext()) + { + return cursor; + } + + ref readonly Exemplar maxExemplar = ref exemplars.Current; + var isLong = metric.MetricType.IsLong(); + + while (exemplars.MoveNext()) + { + if (isLong) + { + if (exemplars.Current.LongValue >= maxExemplar.LongValue) + { + maxExemplar = ref exemplars.Current; + } + } + else + { + if (exemplars.Current.DoubleValue >= maxExemplar.DoubleValue) + { + maxExemplar = ref exemplars.Current; + } + } + } + + return WriteExemplar(buffer, cursor, maxExemplar, metric.Name, isLong); + } + + private static int WriteExemplar(byte[] buffer, int cursor, in Exemplar exemplar, string metricName, bool isLong) + { + buffer[cursor++] = unchecked((byte)' '); + buffer[cursor++] = unchecked((byte)'#'); + buffer[cursor++] = unchecked((byte)' '); + + buffer[cursor++] = unchecked((byte)'{'); + var labelSetCursorStart = cursor; + cursor = WriteAsciiStringNoEscape(buffer, cursor, "trace_id=\""); + cursor = WriteAsciiStringNoEscape(buffer, cursor, exemplar.TraceId.ToHexString()); + cursor = WriteAsciiStringNoEscape(buffer, cursor, "\",span_id=\""); + cursor = WriteAsciiStringNoEscape(buffer, cursor, exemplar.SpanId.ToHexString()); + buffer[cursor++] = unchecked((byte)'"'); + buffer[cursor++] = unchecked((byte)','); + + var labelSetWritten = cursor - labelSetCursorStart - 8; + + var tagResetCursor = cursor; + + foreach (var tag in exemplar.FilteredTags) + { + var prevCursor = cursor; + cursor = WriteLabel(buffer, cursor, tag.Key, tag.Value); + + // From the spec: + // Other characters in the text rendering of an exemplar such as ",= are not included in this limit + // for implementation simplicity and for consistency between the text and proto formats. + labelSetWritten += cursor - prevCursor - 3; // subtract 2 x " and 1 x = character + + buffer[cursor++] = unchecked((byte)','); + + // From the spec: + // The combined length of the label names and values of an Exemplar's LabelSet MUST NOT exceed 128 UTF-8 character code points. + if (labelSetWritten > 128) + { + cursor = tagResetCursor; + PrometheusExporterEventSource.Log.ExemplarTagsTooLong(metricName); + break; + } + } + + buffer[cursor - 1] = unchecked((byte)'}'); // Note: We write the '}' over the last written comma, which is extra. + buffer[cursor++] = unchecked((byte)' '); + + if (isLong) + { + cursor = WriteLong(buffer, cursor, exemplar.LongValue); + } + else + { + cursor = WriteDouble(buffer, cursor, exemplar.DoubleValue); + } + + buffer[cursor++] = unchecked((byte)' '); + cursor = WriteTimestamp(buffer, cursor, exemplar.Timestamp.ToUnixTimeMilliseconds(), useOpenMetrics: true); + + return cursor; + } } diff --git a/test/OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests/PrometheusExporterMiddlewareTests.cs b/test/OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests/PrometheusExporterMiddlewareTests.cs index c34dba29582..eae7d4790f8 100644 --- a/test/OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests/PrometheusExporterMiddlewareTests.cs +++ b/test/OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests/PrometheusExporterMiddlewareTests.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 #if !NETFRAMEWORK +using System.Diagnostics; using System.Diagnostics.Metrics; using System.Net; using System.Text.RegularExpressions; @@ -14,6 +15,7 @@ using OpenTelemetry.Metrics; using OpenTelemetry.Resources; using OpenTelemetry.Tests; +using OpenTelemetry.Trace; using Xunit; namespace OpenTelemetry.Exporter.Prometheus.AspNetCore.Tests; @@ -32,6 +34,758 @@ public Task PrometheusExporterMiddlewareIntegration() app => app.UseOpenTelemetryPrometheusScrapingEndpoint()); } + [Fact] + public async Task PrometheusExporterMiddlewareIntegration_Exemplars_DoubleHistogram() + { + using var activitySource = new ActivitySource(Utils.GetCurrentMethodName()); + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .AddSource(activitySource.Name) + .SetSampler(new AlwaysOnSampler()) + .Build(); + + using var host = await StartTestHostAsync( + app => app.UseOpenTelemetryPrometheusScrapingEndpoint(), + enableExemplars: true); + + using var meter = new Meter(MeterName, MeterVersion); + + // Due to the default histogram buckets, exemplars will only be recorded + // for some of the values (the last recorded value for each buckets wins): + // ✓ ✓ ✓ ✓ - - ✓ - ✓ ✓ + var values = new double[] { 10, 20, 50, 100, 150, 200, 250, 300, 350, 10001 }; + + var histogram = meter.CreateHistogram("dbl_histogram", unit: "s"); + foreach (var value in values) + { + using var activity = activitySource.StartActivity("testActivity"); + histogram.Record(value); + } + + using var client = host.GetTestClient(); + + using var response = await client.SendAsync(new HttpRequestMessage + { + RequestUri = new Uri("/metrics", UriKind.Relative), + Headers = + { + { "Accept", "application/openmetrics-text" }, + }, + }); + var text = await response.Content.ReadAsStringAsync(); + + var expectedPattern = + """ + \# TYPE target info + \# HELP target Target metadata + target_info\{service_name="my_service",service_instance_id="id1"} 1 + \# TYPE otel_scope_info info + \# HELP otel_scope_info Scope metadata + otel_scope_info\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor"} 1 + \# TYPE dbl_histogram_seconds histogram + \# UNIT dbl_histogram_seconds seconds + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="0"} 0 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="5"} 0 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="10"} 1 \d+\.\d{3} \# \{trace_id="([a-z0-9]{32})",span_id="([a-z0-9]{16})"} 10 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="25"} 2 \d+\.\d{3} \# \{trace_id="([a-z0-9]{32})",span_id="([a-z0-9]{16})"} 20 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="50"} 3 \d+\.\d{3} \# \{trace_id="([a-z0-9]{32})",span_id="([a-z0-9]{16})"} 50 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="75"} 3 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="100"} 4 \d+\.\d{3} \# \{trace_id="([a-z0-9]{32})",span_id="([a-z0-9]{16})"} 100 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="250"} 7 \d+\.\d{3} \# \{trace_id="([a-z0-9]{32})",span_id="([a-z0-9]{16})"} 250 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="500"} 9 \d+\.\d{3} \# \{trace_id="([a-z0-9]{32})",span_id="([a-z0-9]{16})"} 350 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="750"} 9 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="1000"} 9 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="2500"} 9 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="5000"} 9 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="7500"} 9 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="10000"} 9 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="\+Inf"} 10 \d+\.\d{3} \# \{trace_id="([a-z0-9]{32})",span_id="([a-z0-9]{16})"} 10001 \d+\.\d{3} + dbl_histogram_seconds_sum\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1"} 11431 \d+\.\d{3} + dbl_histogram_seconds_count\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1"} 10 \d+\.\d{3} + \# EOF + """.ReplaceLineEndings("\n"); + + Assert.Matches(expectedPattern, text); + + await host.StopAsync(); + } + + [Fact] + public async Task PrometheusExporterMiddlewareIntegration_Exemplars_DoubleHistogram_ValuesAreCorrect() + { + using var activitySource = new ActivitySource(Utils.GetCurrentMethodName()); + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .AddSource(activitySource.Name) + .SetSampler(new AlwaysOnSampler()) + .Build(); + + using var host = await StartTestHostAsync( + app => app.UseOpenTelemetryPrometheusScrapingEndpoint(), + enableExemplars: true); + + var beginTimestamp = DateTimeOffset.Now.ToUnixTimeMilliseconds(); + + using var meter = new Meter(MeterName, MeterVersion); + + var histogram = meter.CreateHistogram("dbl_histogram", unit: "s"); + + using (activitySource.StartActivity("testActivity")) + { + // Falls within <= 100 bucket + histogram.Record(90.0); + } + + using (activitySource.StartActivity("testActivity")) + { + // Falls within <= 100 bucket + // More than 90.0, so supersedes existing exemplar + + histogram.Record(95.1); + } + + string expectedTraceId; + string expectedSpanId; + using (var activity = activitySource.StartActivity("testActivity")) + { + Assert.NotNull(activity); + + // Falls within <= 100 bucket + // Less than 95.1, but still supersedes the existing exemplar + + histogram.Record(80.2); + + expectedTraceId = activity.TraceId.ToHexString(); + expectedSpanId = activity.SpanId.ToHexString(); + } + + using var client = host.GetTestClient(); + + using var response = await client.SendAsync(new HttpRequestMessage + { + RequestUri = new Uri("/metrics", UriKind.Relative), + Headers = + { + { "Accept", "application/openmetrics-text" }, + }, + }); + var text = await response.Content.ReadAsStringAsync(); + + var endTimestamp = DateTimeOffset.Now.ToUnixTimeMilliseconds(); + + var expectedPattern = + """ + \# TYPE target info + \# HELP target Target metadata + target_info\{service_name="my_service",service_instance_id="id1"} 1 + \# TYPE otel_scope_info info + \# HELP otel_scope_info Scope metadata + otel_scope_info\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor"} 1 + \# TYPE dbl_histogram_seconds histogram + \# UNIT dbl_histogram_seconds seconds + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="0"} 0 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="5"} 0 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="10"} 0 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="25"} 0 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="50"} 0 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="75"} 0 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="100"} 3 \d+\.\d{3} \# \{trace_id="([a-z0-9]{32})",span_id="([a-z0-9]{16})"} 80.2 (\d+\.\d{3}) + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="250"} 3 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="500"} 3 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="750"} 3 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="1000"} 3 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="2500"} 3 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="5000"} 3 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="7500"} 3 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="10000"} 3 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="\+Inf"} 3 \d+\.\d{3} + dbl_histogram_seconds_sum\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1"} 265.3 \d+\.\d{3} + dbl_histogram_seconds_count\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1"} 3 \d+\.\d{3} + \# EOF + """.ReplaceLineEndings("\n"); + + Assert.Matches(expectedPattern, text); + + var match = Regex.Match(text, expectedPattern); + Assert.True(match.Success); + + var traceId = match.Groups[1].Value; + Assert.Equal(expectedTraceId, traceId); + + var spanId = match.Groups[2].Value; + Assert.Equal(expectedSpanId, spanId); + + var timestamp = double.Parse(match.Groups[3].Value); + Assert.True(timestamp >= beginTimestamp / 1000.0 && timestamp <= endTimestamp / 1000.0); + + await host.StopAsync(); + } + + [Fact] + public async Task PrometheusExporterMiddlewareIntegration_Exemplars_PersistBetweenScrapes() + { + using var activitySource = new ActivitySource(Utils.GetCurrentMethodName()); + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .AddSource(activitySource.Name) + .SetSampler(new AlwaysOnSampler()) + .Build(); + + using var host = await StartTestHostAsync( + app => app.UseOpenTelemetryPrometheusScrapingEndpoint(), + configureOptions: options => + { + // Disable caching to test exemplars persisting between scrapes + options.ScrapeResponseCacheDurationMilliseconds = 0; + }, + enableExemplars: true); + + using var meter = new Meter(MeterName, MeterVersion); + + var histogram = meter.CreateHistogram("dbl_histogram", unit: "s"); + + using (activitySource.StartActivity("testActivity")) + { + // Falls within <= 100 bucket + histogram.Record(90.0); + } + + using var client = host.GetTestClient(); + + // First response is discarded + var text1 = await client.GetStringAsync("/metrics"); + + using var response2 = await client.SendAsync(new HttpRequestMessage + { + RequestUri = new Uri("/metrics", UriKind.Relative), + Headers = + { + { "Accept", "application/openmetrics-text" }, + }, + }); + var text2 = await response2.Content.ReadAsStringAsync(); + + var expectedPattern = + """ + \# TYPE target info + \# HELP target Target metadata + target_info\{service_name="my_service",service_instance_id="id1"} 1 + \# TYPE otel_scope_info info + \# HELP otel_scope_info Scope metadata + otel_scope_info\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor"} 1 + \# TYPE dbl_histogram_seconds histogram + \# UNIT dbl_histogram_seconds seconds + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="0"} 0 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="5"} 0 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="10"} 0 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="25"} 0 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="50"} 0 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="75"} 0 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="100"} 1 \d+\.\d{3} \# \{trace_id="([a-z0-9]{32})",span_id="([a-z0-9]{16})"} 90 (\d+\.\d{3}) + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="250"} 1 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="500"} 1 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="750"} 1 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="1000"} 1 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="2500"} 1 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="5000"} 1 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="7500"} 1 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="10000"} 1 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="\+Inf"} 1 \d+\.\d{3} + dbl_histogram_seconds_sum\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1"} 90 \d+\.\d{3} + dbl_histogram_seconds_count\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1"} 1 \d+\.\d{3} + \# EOF + """.ReplaceLineEndings("\n"); + + Assert.Matches(expectedPattern, text2); + + await host.StopAsync(); + } + + [Fact] + public async Task PrometheusExporterMiddlewareIntegration_Exemplars_LongHistogram_ValuesAreCorrect() + { + using var activitySource = new ActivitySource(Utils.GetCurrentMethodName()); + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .AddSource(activitySource.Name) + .SetSampler(new AlwaysOnSampler()) + .Build(); + + using var host = await StartTestHostAsync( + app => app.UseOpenTelemetryPrometheusScrapingEndpoint(), + enableExemplars: true); + + var beginTimestamp = DateTimeOffset.Now.ToUnixTimeMilliseconds(); + + using var meter = new Meter(MeterName, MeterVersion); + + var histogram = meter.CreateHistogram("histogram", unit: "s"); + + using (activitySource.StartActivity("testActivity")) + { + histogram.Record(90L); + } + + using (activitySource.StartActivity("testActivity")) + { + histogram.Record(95L); + } + + string expectedTraceId; + string expectedSpanId; + using (var activity = activitySource.StartActivity("testActivity")) + { + Assert.NotNull(activity); + histogram.Record(80L); + + expectedTraceId = activity.TraceId.ToHexString(); + expectedSpanId = activity.SpanId.ToHexString(); + } + + using var client = host.GetTestClient(); + + using var response = await client.SendAsync(new HttpRequestMessage + { + RequestUri = new Uri("/metrics", UriKind.Relative), + Headers = + { + { "Accept", "application/openmetrics-text" }, + }, + }); + var text = await response.Content.ReadAsStringAsync(); + + var endTimestamp = DateTimeOffset.Now.ToUnixTimeMilliseconds(); + + var expectedPattern = + """ + \# TYPE target info + \# HELP target Target metadata + target_info\{service_name="my_service",service_instance_id="id1"} 1 + \# TYPE otel_scope_info info + \# HELP otel_scope_info Scope metadata + otel_scope_info\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor"} 1 + \# TYPE histogram_seconds histogram + \# UNIT histogram_seconds seconds + histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="0"} 0 \d+\.\d{3} + histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="5"} 0 \d+\.\d{3} + histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="10"} 0 \d+\.\d{3} + histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="25"} 0 \d+\.\d{3} + histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="50"} 0 \d+\.\d{3} + histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="75"} 0 \d+\.\d{3} + histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="100"} 3 \d+\.\d{3} \# \{trace_id="([a-z0-9]{32})",span_id="([a-z0-9]{16})"} 80 (\d+\.\d{3}) + histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="250"} 3 \d+\.\d{3} + histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="500"} 3 \d+\.\d{3} + histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="750"} 3 \d+\.\d{3} + histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="1000"} 3 \d+\.\d{3} + histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="2500"} 3 \d+\.\d{3} + histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="5000"} 3 \d+\.\d{3} + histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="7500"} 3 \d+\.\d{3} + histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="10000"} 3 \d+\.\d{3} + histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="\+Inf"} 3 \d+\.\d{3} + histogram_seconds_sum\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1"} 265 \d+\.\d{3} + histogram_seconds_count\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1"} 3 \d+\.\d{3} + \# EOF + """.ReplaceLineEndings("\n"); + + Assert.Matches(expectedPattern, text); + + var match = Regex.Match(text, expectedPattern); + Assert.True(match.Success); + + var traceId = match.Groups[1].Value; + Assert.Equal(expectedTraceId, traceId); + + var spanId = match.Groups[2].Value; + Assert.Equal(expectedSpanId, spanId); + + var timestamp = double.Parse(match.Groups[3].Value); + Assert.True(timestamp >= beginTimestamp / 1000.0 && timestamp <= endTimestamp / 1000.0); + + await host.StopAsync(); + } + + [Fact] + public async Task PrometheusExporterMiddlewareIntegration_Exemplars_DoubleHistogram_Tags() + { + using var activitySource = new ActivitySource(Utils.GetCurrentMethodName()); + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .AddSource(activitySource.Name) + .SetSampler(new AlwaysOnSampler()) + .Build(); + + using var host = await StartTestHostAsync( + app => app.UseOpenTelemetryPrometheusScrapingEndpoint(), + enableExemplars: true, + enableTagFiltering: true); + + using var meter = new Meter(MeterName, MeterVersion); + + var histogram = meter.CreateHistogram("dbl_histogram", unit: "s"); + using (activitySource.StartActivity("testActivity")) + { + // This is precisely 128 chars of labels and values + histogram.Record(10, new KeyValuePair("ab", "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do")); + } + + using (activitySource.StartActivity("testActivity")) + { + // This is precisely 129 chars of labels and values + histogram.Record(20, new KeyValuePair("abc", "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do")); + } + + using (activitySource.StartActivity("testActivity")) + { + // This is precisely 127 chars of labels and values + histogram.Record(50, new KeyValuePair("a", "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do")); + } + + using (activitySource.StartActivity("testActivity")) + { + histogram.Record(80, new KeyValuePair("key1", "value1"), new KeyValuePair("key2", "value2")); + } + + using (activitySource.StartActivity("testActivity")) + { + // This is precisely 129 chars of labels and values + histogram.Record( + 200, + new KeyValuePair("key1", "value1"), + new KeyValuePair("a", "Lorem ipsum dolor sit amet, consectetur adipiscing elit")); + } + + using var client = host.GetTestClient(); + + using var response = await client.SendAsync(new HttpRequestMessage + { + RequestUri = new Uri("/metrics", UriKind.Relative), + Headers = + { + { "Accept", "application/openmetrics-text" }, + }, + }); + var text = await response.Content.ReadAsStringAsync(); + + var expectedPattern = + """ + \# TYPE target info + \# HELP target Target metadata + target_info\{service_name="my_service",service_instance_id="id1"} 1 + \# TYPE otel_scope_info info + \# HELP otel_scope_info Scope metadata + otel_scope_info\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor"} 1 + \# TYPE dbl_histogram_seconds histogram + \# UNIT dbl_histogram_seconds seconds + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="0"} 0 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="5"} 0 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="10"} 1 \d+\.\d{3} \# \{trace_id="[a-z0-9]{32}",span_id="[a-z0-9]{16}",ab="Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do"} 10 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="25"} 2 \d+\.\d{3} \# \{trace_id="[a-z0-9]{32}",span_id="[a-z0-9]{16}"} 20 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="50"} 3 \d+\.\d{3} \# \{trace_id="[a-z0-9]{32}",span_id="[a-z0-9]{16}",a="Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do"} 50 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="75"} 3 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="100"} 4 \d+\.\d{3} \# \{trace_id="[a-z0-9]{32}",span_id="[a-z0-9]{16}",key1="value1",key2="value2"} 80 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="250"} 5 \d+\.\d{3} \# \{trace_id="[a-z0-9]{32}",span_id="[a-z0-9]{16}"} 200 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="500"} 5 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="750"} 5 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="1000"} 5 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="2500"} 5 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="5000"} 5 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="7500"} 5 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="10000"} 5 \d+\.\d{3} + dbl_histogram_seconds_bucket\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",le="\+Inf"} 5 \d+\.\d{3} + dbl_histogram_seconds_sum\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1"} 360 \d+\.\d{3} + dbl_histogram_seconds_count\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1"} 5 \d+\.\d{3} + \# EOF + """.ReplaceLineEndings("\n"); + + Assert.Matches(expectedPattern, text); + + await host.StopAsync(); + } + + [Fact] + public async Task PrometheusExporterMiddlewareIntegration_Exemplars_DoubleCounter() + { + using var activitySource = new ActivitySource(Utils.GetCurrentMethodName()); + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .AddSource(activitySource.Name) + .SetSampler(new AlwaysOnSampler()) + .Build(); + + using var host = await StartTestHostAsync( + app => app.UseOpenTelemetryPrometheusScrapingEndpoint(), + enableExemplars: true); + + using var meter = new Meter(MeterName, MeterVersion); + + var beginTimestamp = DateTimeOffset.Now.ToUnixTimeMilliseconds(); + + var counter = meter.CreateCounter("counter", unit: "s"); + using (activitySource.StartActivity("testActivity")) + { + counter.Add(123.456); + } + + string expectedTraceId; + string expectedSpanId; + using (var activity = activitySource.StartActivity("testActivity")) + { + Assert.NotNull(activity); + + counter.Add(78.9); + expectedTraceId = activity.TraceId.ToHexString(); + expectedSpanId = activity.SpanId.ToHexString(); + } + + using var client = host.GetTestClient(); + + using var response = await client.SendAsync(new HttpRequestMessage + { + RequestUri = new Uri("/metrics", UriKind.Relative), + Headers = + { + { "Accept", "application/openmetrics-text" }, + }, + }); + + var endTimestamp = DateTimeOffset.Now.ToUnixTimeMilliseconds(); + + var text = await response.Content.ReadAsStringAsync(); + + var expectedPattern = + """ + \# TYPE target info + \# HELP target Target metadata + target_info\{service_name="my_service",service_instance_id="id1"} 1 + \# TYPE otel_scope_info info + \# HELP otel_scope_info Scope metadata + otel_scope_info\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor"} 1 + \# TYPE counter_seconds counter + \# UNIT counter_seconds seconds + counter_seconds_total\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1"} 202\.356 \d+\.\d{3} \# \{trace_id="([a-z0-9]{32})",span_id="([a-z0-9]{16})"} 78\.9 (\d+\.\d{3}) + \# EOF + """.ReplaceLineEndings("\n"); + + var match = Regex.Match(text, expectedPattern); + Assert.True(match.Success); + + Assert.Equal(expectedTraceId, match.Groups[1].Value); + Assert.Equal(expectedSpanId, match.Groups[2].Value); + + var timestamp = double.Parse(match.Groups[3].Value); + Assert.True(timestamp >= beginTimestamp / 1000.0 && timestamp <= endTimestamp / 1000.0); + + await host.StopAsync(); + } + + [Fact] + public async Task PrometheusExporterMiddlewareIntegration_Exemplars_LongCounter() + { + using var activitySource = new ActivitySource(Utils.GetCurrentMethodName()); + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .AddSource(activitySource.Name) + .SetSampler(new AlwaysOnSampler()) + .Build(); + + using var host = await StartTestHostAsync( + app => app.UseOpenTelemetryPrometheusScrapingEndpoint(), + enableExemplars: true); + + using var meter = new Meter(MeterName, MeterVersion); + + var beginTimestamp = DateTimeOffset.Now.ToUnixTimeMilliseconds(); + + var counter = meter.CreateCounter("counter", unit: "s"); + using (activitySource.StartActivity("testActivity")) + { + counter.Add(123L); + } + + string expectedTraceId; + string expectedSpanId; + using (var activity = activitySource.StartActivity("testActivity")) + { + Assert.NotNull(activity); + + counter.Add(78L); + expectedTraceId = activity.TraceId.ToHexString(); + expectedSpanId = activity.SpanId.ToHexString(); + } + + using var client = host.GetTestClient(); + + using var response = await client.SendAsync(new HttpRequestMessage + { + RequestUri = new Uri("/metrics", UriKind.Relative), + Headers = + { + { "Accept", "application/openmetrics-text" }, + }, + }); + + var endTimestamp = DateTimeOffset.Now.ToUnixTimeMilliseconds(); + + var text = await response.Content.ReadAsStringAsync(); + + var expectedPattern = + """ + \# TYPE target info + \# HELP target Target metadata + target_info\{service_name="my_service",service_instance_id="id1"} 1 + \# TYPE otel_scope_info info + \# HELP otel_scope_info Scope metadata + otel_scope_info\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor"} 1 + \# TYPE counter_seconds counter + \# UNIT counter_seconds seconds + counter_seconds_total\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1"} 201 \d+\.\d{3} \# \{trace_id="([a-z0-9]{32})",span_id="([a-z0-9]{16})"} 78 (\d+\.\d{3}) + \# EOF + """.ReplaceLineEndings("\n"); + + var match = Regex.Match(text, expectedPattern); + Assert.True(match.Success); + + Assert.Equal(expectedTraceId, match.Groups[1].Value); + Assert.Equal(expectedSpanId, match.Groups[2].Value); + + var timestamp = double.Parse(match.Groups[3].Value); + Assert.True(timestamp >= beginTimestamp / 1000.0 && timestamp <= endTimestamp / 1000.0); + + await host.StopAsync(); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task PrometheusExporterMiddlewareIntegration_Exemplars_LongCounter_Tags(bool enableTagFiltering) + { + using var activitySource = new ActivitySource(Utils.GetCurrentMethodName()); + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .AddSource(activitySource.Name) + .SetSampler(new AlwaysOnSampler()) + .Build(); + + using var host = await StartTestHostAsync( + app => app.UseOpenTelemetryPrometheusScrapingEndpoint(), + enableExemplars: true, + enableTagFiltering: enableTagFiltering); + + using var meter = new Meter(MeterName, MeterVersion); + + var counter = meter.CreateCounter("counter", unit: "s"); + using (activitySource.StartActivity("testActivity")) + { + counter.Add(123L, new KeyValuePair("key1", "value1")); + } + + using var client = host.GetTestClient(); + + using var response = await client.SendAsync(new HttpRequestMessage + { + RequestUri = new Uri("/metrics", UriKind.Relative), + Headers = + { + { "Accept", "application/openmetrics-text" }, + }, + }); + + var text = await response.Content.ReadAsStringAsync(); + + var expectedPattern = + enableTagFiltering + ? """ + \# TYPE target info + \# HELP target Target metadata + target_info\{service_name="my_service",service_instance_id="id1"} 1 + \# TYPE otel_scope_info info + \# HELP otel_scope_info Scope metadata + otel_scope_info\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor"} 1 + \# TYPE counter_seconds counter + \# UNIT counter_seconds seconds + counter_seconds_total\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1"} 123 \d+\.\d{3} \# \{trace_id="[a-z0-9]{32}",span_id="[a-z0-9]{16}",key1="value1"} 123 \d+\.\d{3} + \# EOF + """.ReplaceLineEndings("\n") + : """ + \# TYPE target info + \# HELP target Target metadata + target_info\{service_name="my_service",service_instance_id="id1"} 1 + \# TYPE otel_scope_info info + \# HELP otel_scope_info Scope metadata + otel_scope_info\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor"} 1 + \# TYPE counter_seconds counter + \# UNIT counter_seconds seconds + counter_seconds_total\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1",key1="value1"} 123 \d+\.\d{3} \# \{trace_id="[a-z0-9]{32}",span_id="[a-z0-9]{16}"} 123 \d+\.\d{3} + \# EOF + """.ReplaceLineEndings("\n"); + + Assert.Matches(expectedPattern, text); + + await host.StopAsync(); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task PrometheusExporterMiddlewareIntegration_Exemplars_OpenMetricsFormat(bool openMetricsFormat) + { + using var activitySource = new ActivitySource(Utils.GetCurrentMethodName()); + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .AddSource(activitySource.Name) + .SetSampler(new AlwaysOnSampler()) + .Build(); + + using var host = await StartTestHostAsync( + app => app.UseOpenTelemetryPrometheusScrapingEndpoint(), + enableExemplars: true); + + using var meter = new Meter(MeterName, MeterVersion); + + var counter = meter.CreateCounter("counter", unit: "s"); + using (activitySource.StartActivity("testActivity")) + { + counter.Add(123L); + } + + using (activitySource.StartActivity("testActivity")) + { + counter.Add(78L); + } + + using var client = host.GetTestClient(); + + var request = new HttpRequestMessage + { + RequestUri = new Uri("/metrics", UriKind.Relative), + }; + if (openMetricsFormat) + { + request.Headers.Add("Accept", "application/openmetrics-text"); + } + + using var response = await client.SendAsync(request); + + var text = await response.Content.ReadAsStringAsync(); + + var expectedPattern = + openMetricsFormat + ? """ + \# TYPE target info + \# HELP target Target metadata + target_info\{service_name="my_service",service_instance_id="id1"} 1 + \# TYPE otel_scope_info info + \# HELP otel_scope_info Scope metadata + otel_scope_info\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor"} 1 + \# TYPE counter_seconds counter + \# UNIT counter_seconds seconds + counter_seconds_total\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1"} 201 \d+\.\d{3} \# \{trace_id="([a-z0-9]{32})",span_id="([a-z0-9]{16})"} 78 (\d+\.\d{3}) + \# EOF + """.ReplaceLineEndings("\n") + : """ + \# TYPE counter_seconds_total counter + \# UNIT counter_seconds_total seconds + counter_seconds_total\{otel_scope_name="OpenTelemetry\.Exporter\.Prometheus\.AspNetCore\.Tests\.PrometheusExporterMiddlewareTests\.\.cctor",otel_scope_version="1\.0\.1"} 201 \d+ + \# EOF + """.ReplaceLineEndings("\n"); + + var match = Regex.Match(text, expectedPattern); + Assert.True(match.Success); + + await host.StopAsync(); + } + [Fact] public Task PrometheusExporterMiddlewareIntegration_Options() { @@ -369,12 +1123,13 @@ private static async Task RunPrometheusExporterMiddlewareIntegrationTest( bool registerMeterProvider = true, Action? configureOptions = null, bool skipMetrics = false, + bool enableExemplars = false, string acceptHeader = "application/openmetrics-text", KeyValuePair[]? meterTags = null) { var requestOpenMetrics = acceptHeader.StartsWith("application/openmetrics-text"); - using var host = await StartTestHostAsync(configure, configureServices, registerMeterProvider, configureOptions); + using var host = await StartTestHostAsync(configure, configureServices, registerMeterProvider, false, enableExemplars, configureOptions); var counterTags = new KeyValuePair[] { @@ -436,7 +1191,7 @@ private static async Task VerifyAsync(long beginTimestamp, long endTimestamp, Ht ? $"{string.Join(",", meterTags.Select(x => $"{x.Key}=\"{x.Value}\""))}," : string.Empty; - string content = (await response.Content.ReadAsStringAsync()).ReplaceLineEndings(); + string content = await response.Content.ReadAsStringAsync(); string expected = requestOpenMetrics ? $$""" @@ -451,14 +1206,14 @@ private static async Task VerifyAsync(long beginTimestamp, long endTimestamp, Ht counter_double_bytes_total{otel_scope_name="{{MeterName}}",otel_scope_version="{{MeterVersion}}",{{additionalTags}}key1="value1",key2="value2"} 101.17 (\d+\.\d{3}) # EOF - """.ReplaceLineEndings() + """.ReplaceLineEndings("\n") : $$""" # TYPE counter_double_bytes_total counter # UNIT counter_double_bytes_total bytes counter_double_bytes_total{otel_scope_name="{{MeterName}}",otel_scope_version="{{MeterVersion}}",{{additionalTags}}key1="value1",key2="value2"} 101.17 (\d+) # EOF - """.ReplaceLineEndings(); + """.ReplaceLineEndings("\n"); var matches = Regex.Matches(content, "^" + expected + "$"); @@ -473,6 +1228,8 @@ private static Task StartTestHostAsync( Action configure, Action? configureServices = null, bool registerMeterProvider = true, + bool enableExemplars = false, + bool enableTagFiltering = false, Action? configureOptions = null) { return new HostBuilder() @@ -482,13 +1239,25 @@ private static Task StartTestHostAsync( { if (registerMeterProvider) { - services.AddOpenTelemetry().WithMetrics(builder => builder - .ConfigureResource(x => x.Clear().AddService("my_service", serviceInstanceId: "id1")) - .AddMeter(MeterName) - .AddPrometheusExporter(o => + services.AddOpenTelemetry().WithMetrics(builder => + { + builder + .AddView(i => enableTagFiltering + ? new MetricStreamConfiguration { TagKeys = [], } + : null) + .ConfigureResource(x => x.Clear() + .AddService("my_service", serviceInstanceId: "id1")) + .AddMeter(MeterName) + .AddPrometheusExporter(o => + { + configureOptions?.Invoke(o); + }); + + if (enableExemplars) { - configureOptions?.Invoke(o); - })); + builder.SetExemplarFilter(ExemplarFilterType.AlwaysOn); + } + }); } configureServices?.Invoke(services); diff --git a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusHttpListenerTests.cs b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusHttpListenerTests.cs index b9b6201183e..2437f9fc14c 100644 --- a/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusHttpListenerTests.cs +++ b/test/OpenTelemetry.Exporter.Prometheus.HttpListener.Tests/PrometheusHttpListenerTests.cs @@ -1,14 +1,17 @@ // Copyright The OpenTelemetry Authors // SPDX-License-Identifier: Apache-2.0 +using System.Diagnostics; using System.Diagnostics.Metrics; using System.Net; +using System.Text.RegularExpressions; #if NETFRAMEWORK using System.Net.Http; #endif using OpenTelemetry.Metrics; using OpenTelemetry.Resources; using OpenTelemetry.Tests; +using OpenTelemetry.Trace; using Xunit; namespace OpenTelemetry.Exporter.Prometheus.Tests; @@ -182,7 +185,7 @@ public async Task PrometheusExporterHttpServerIntegration_TestBufferSizeIncrease attributes.Add(new KeyValuePair(x.ToString(), oneKb)); } - var provider = BuildMeterProvider(meter, attributes, out var address); + using var provider = BuildMeterProvider(meter, attributes, false, out var address); for (var x = 0; x < 1000; x++) { @@ -203,8 +206,165 @@ public async Task PrometheusExporterHttpServerIntegration_TestBufferSizeIncrease var content = await response.Content.ReadAsStringAsync(); Assert.Contains("counter_double_999", content); Assert.DoesNotContain('\0', content); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task PrometheusExporterHttpServerIntegration_Histogram_Exemplars(bool enableExemplars) + { + using var activitySource = new ActivitySource(Utils.GetCurrentMethodName()); + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .AddSource(activitySource.Name) + .SetSampler(new AlwaysOnSampler()) + .Build(); + + using var meter = new Meter(MeterName, MeterVersion); + + using var provider = BuildMeterProvider(meter, [], enableExemplars, out var address); + + var counterTags = new KeyValuePair[] + { + new("key1", "value1"), + new("key2", "value2"), + }; + + var counter = meter.CreateCounter("counter_double", unit: "By"); + counter.Add(100.18, counterTags); + + string expectedTraceId; + string expectedSpanId; + using (var activity = activitySource.StartActivity("testActivity")) + { + Assert.NotNull(activity); + + counter.Add(0.99, counterTags); + expectedTraceId = activity.TraceId.ToHexString(); + expectedSpanId = activity.SpanId.ToHexString(); + } + + using HttpClient client = new HttpClient(); + client.DefaultRequestHeaders.Add("Accept", "application/openmetrics-text"); + + using var response = await client.GetAsync($"{address}metrics"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + + var content = await response.Content.ReadAsStringAsync(); + + var expected = + enableExemplars + ? $$""" + \# TYPE target info + \# HELP target Target metadata + target_info\{service_name="my_service",service_instance_id="id1"} 1 + \# TYPE otel_scope_info info + \# HELP otel_scope_info Scope metadata + otel_scope_info\{otel_scope_name="{{MeterName}}"} 1 + \# TYPE counter_double_bytes counter + \# UNIT counter_double_bytes bytes + counter_double_bytes_total\{otel_scope_name="{{MeterName}}",otel_scope_version="{{MeterVersion}}",key1="value1",key2="value2"} 101\.17 \d+.\d{3} \# \{trace_id="([a-z0-9]{32})",span_id="([a-z0-9]{16})"} 0\.99 \d+.\d{3} + \# EOF + """.Replace("\r\n", "\n") + : $$""" + \# TYPE target info + \# HELP target Target metadata + target_info\{service_name="my_service",service_instance_id="id1"} 1 + \# TYPE otel_scope_info info + \# HELP otel_scope_info Scope metadata + otel_scope_info\{otel_scope_name="{{MeterName}}"} 1 + \# TYPE counter_double_bytes counter + \# UNIT counter_double_bytes bytes + counter_double_bytes_total\{otel_scope_name="{{MeterName}}",otel_scope_version="{{MeterVersion}}",key1="value1",key2="value2"} 101\.17 \d+.\d{3} + \# EOF + """.Replace("\r\n", "\n"); + + var match = Regex.Match(content, expected); + Assert.True(match.Success); + + if (enableExemplars) + { + Assert.Equal(expectedTraceId, match.Groups[1].Value); + Assert.Equal(expectedSpanId, match.Groups[2].Value); + } + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task PrometheusExporterHttpServerIntegration_Exemplars_OpenMetricsFormat(bool openMetricsFormat) + { + using var activitySource = new ActivitySource(Utils.GetCurrentMethodName()); + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .AddSource(activitySource.Name) + .SetSampler(new AlwaysOnSampler()) + .Build(); + + using var meter = new Meter(MeterName, MeterVersion); + + using var provider = BuildMeterProvider(meter, [], true, out var address); + + var counterTags = new KeyValuePair[] + { + new("key1", "value1"), + new("key2", "value2"), + }; + + var counter = meter.CreateCounter("counter_double", unit: "By"); + counter.Add(100.18, counterTags); + + string expectedTraceId; + string expectedSpanId; + using (var activity = activitySource.StartActivity("testActivity")) + { + Assert.NotNull(activity); + + counter.Add(0.99, counterTags); + expectedTraceId = activity.TraceId.ToHexString(); + expectedSpanId = activity.SpanId.ToHexString(); + } + + using HttpClient client = new HttpClient(); + if (openMetricsFormat) + { + client.DefaultRequestHeaders.Add("Accept", "application/openmetrics-text"); + } - provider.Dispose(); + using var response = await client.GetAsync($"{address}metrics"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + + var content = await response.Content.ReadAsStringAsync(); + + var expected = + openMetricsFormat + ? $$""" + \# TYPE target info + \# HELP target Target metadata + target_info\{service_name="my_service",service_instance_id="id1"} 1 + \# TYPE otel_scope_info info + \# HELP otel_scope_info Scope metadata + otel_scope_info\{otel_scope_name="{{MeterName}}"} 1 + \# TYPE counter_double_bytes counter + \# UNIT counter_double_bytes bytes + counter_double_bytes_total\{otel_scope_name="{{MeterName}}",otel_scope_version="{{MeterVersion}}",key1="value1",key2="value2"} 101\.17 \d+.\d{3} \# \{trace_id="([a-z0-9]{32})",span_id="([a-z0-9]{16})"} 0\.99 \d+.\d{3} + \# EOF + """.Replace("\r\n", "\n") + : $$""" + \# TYPE counter_double_bytes_total counter + \# UNIT counter_double_bytes_total bytes + counter_double_bytes_total\{otel_scope_name="{{MeterName}}",otel_scope_version="{{MeterVersion}}",key1="value1",key2="value2"} 101\.17 \d+ + \# EOF + """.Replace("\r\n", "\n"); + + var match = Regex.Match(content, expected); + Assert.True(match.Success); + + if (openMetricsFormat) + { + Assert.Equal(expectedTraceId, match.Groups[1].Value); + Assert.Equal(expectedSpanId, match.Groups[2].Value); + } } private static void TestPrometheusHttpListenerUriPrefixOptions(string[] uriPrefixes) @@ -218,7 +378,7 @@ private static void TestPrometheusHttpListenerUriPrefixOptions(string[] uriPrefi }); } - private static MeterProvider BuildMeterProvider(Meter meter, IEnumerable> attributes, out string address) + private static MeterProvider BuildMeterProvider(Meter meter, IEnumerable> attributes, bool enableExemplars, out string address) { Random random = new Random(); int retryAttempts = 5; @@ -233,14 +393,20 @@ private static MeterProvider BuildMeterProvider(Meter meter, IEnumerable x.Clear().AddService("my_service", serviceInstanceId: "id1").AddAttributes(attributes)) .AddPrometheusHttpListener(options => { options.UriPrefixes = new string[] { generatedAddress }; - }) - .Build(); + }); + + if (enableExemplars) + { + builder.SetExemplarFilter(ExemplarFilterType.AlwaysOn); + } + + provider = builder.Build(); break; } @@ -260,13 +426,17 @@ private static MeterProvider BuildMeterProvider(Meter meter, IEnumerable[]? meterTags = null) + private async Task RunPrometheusExporterHttpServerIntegrationTest( + bool skipMetrics = false, + string acceptHeader = "application/openmetrics-text", + KeyValuePair[]? meterTags = null, + bool enableExemplars = false) { var requestOpenMetrics = acceptHeader.StartsWith("application/openmetrics-text"); using var meter = new Meter(MeterName, MeterVersion, meterTags); - var provider = BuildMeterProvider(meter, [], out var address); + using var provider = BuildMeterProvider(meter, [], enableExemplars, out var address); var counterTags = new KeyValuePair[] { @@ -290,49 +460,49 @@ private async Task RunPrometheusExporterHttpServerIntegrationTest(bool skipMetri using var response = await client.GetAsync($"{address}metrics"); - if (!skipMetrics) + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.True(response.Content.Headers.Contains("Last-Modified")); + + if (requestOpenMetrics) { - Assert.Equal(HttpStatusCode.OK, response.StatusCode); - Assert.True(response.Content.Headers.Contains("Last-Modified")); + Assert.Equal("application/openmetrics-text; version=1.0.0; charset=utf-8", response.Content.Headers.ContentType!.ToString()); + } + else + { + Assert.Equal("text/plain; charset=utf-8; version=0.0.4", response.Content.Headers.ContentType!.ToString()); + } - if (requestOpenMetrics) - { - Assert.Equal("application/openmetrics-text; version=1.0.0; charset=utf-8", response.Content.Headers.ContentType!.ToString()); - } - else - { - Assert.Equal("text/plain; charset=utf-8; version=0.0.4", response.Content.Headers.ContentType!.ToString()); - } + var content = await response.Content.ReadAsStringAsync(); + if (!skipMetrics) + { var additionalTags = meterTags != null && meterTags.Any() - ? $"{string.Join(",", meterTags.Select(x => $"{x.Key}='{x.Value}'"))}," + ? $"{string.Join(",", meterTags.Select(x => $"{x.Key}=\"{x.Value}\""))}," : string.Empty; - var content = await response.Content.ReadAsStringAsync(); - var expected = requestOpenMetrics - ? "# TYPE target info\n" - + "# HELP target Target metadata\n" - + "target_info{service_name='my_service',service_instance_id='id1'} 1\n" - + "# TYPE otel_scope_info info\n" - + "# HELP otel_scope_info Scope metadata\n" - + $"otel_scope_info{{otel_scope_name='{MeterName}'}} 1\n" - + "# TYPE counter_double_bytes counter\n" - + "# UNIT counter_double_bytes bytes\n" - + $"counter_double_bytes_total{{otel_scope_name='{MeterName}',otel_scope_version='{MeterVersion}',{additionalTags}key1='value1',key2='value2'}} 101.17 (\\d+\\.\\d{{3}})\n" - + "# EOF\n" - : "# TYPE counter_double_bytes_total counter\n" - + "# UNIT counter_double_bytes_total bytes\n" - + $"counter_double_bytes_total{{otel_scope_name='{MeterName}',otel_scope_version='{MeterVersion}',{additionalTags}key1='value1',key2='value2'}} 101.17 (\\d+)\n" - + "# EOF\n"; - - Assert.Matches(("^" + expected + "$").Replace('\'', '"'), content); + ? $$""" + # TYPE target info + # HELP target Target metadata + target_info{service_name="my_service",service_instance_id="id1"} 1 + # TYPE otel_scope_info info + # HELP otel_scope_info Scope metadata + otel_scope_info{otel_scope_name="{{MeterName}}"} 1 + # TYPE counter_double_bytes counter + # UNIT counter_double_bytes bytes + counter_double_bytes_total{otel_scope_name="{{MeterName}}",otel_scope_version="{{MeterVersion}}",{{additionalTags}}key1="value1",key2="value2"} 101.17 (\d+\.\d{3}) + # EOF + + """.Replace("\r\n", "\n") + : $$""" + # TYPE counter_double_bytes_total counter + # UNIT counter_double_bytes_total bytes + counter_double_bytes_total{otel_scope_name="{{MeterName}}",otel_scope_version="{{MeterVersion}}",{{additionalTags}}key1="value1",key2="value2"} 101.17 (\d+) + # EOF + + """.Replace("\r\n", "\n"); + + Assert.Matches("^" + expected + "$", content); } - else - { - Assert.Equal(HttpStatusCode.OK, response.StatusCode); - } - - provider.Dispose(); } } From 606b69e255e0b3b5024fcb9558c53f31c63d2d4b Mon Sep 17 00:00:00 2001 From: Saul Rennison Date: Tue, 29 Oct 2024 19:49:29 +0000 Subject: [PATCH 2/2] Update README --- src/OpenTelemetry.Exporter.Prometheus.AspNetCore/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/README.md b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/README.md index 468ecef0e99..97d3cc18ecd 100644 --- a/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/README.md +++ b/src/OpenTelemetry.Exporter.Prometheus.AspNetCore/README.md @@ -18,6 +18,12 @@ to scrape. Grafana](../../docs/metrics/getting-started-prometheus-grafana/README.md) tutorial for more information. + + +> [!NOTE] +> Exemplars are only exported when the OpenMetrics format is requested (e.g. +> with a `Accept: application/openmetrics-text` header). + ## Prerequisite * [Get Prometheus](https://prometheus.io/docs/introduction/first_steps/)