diff --git a/docs/metrics/customizing-the-sdk/README.md b/docs/metrics/customizing-the-sdk/README.md index f4a9de3059d..7ed1a73855e 100644 --- a/docs/metrics/customizing-the-sdk/README.md +++ b/docs/metrics/customizing-the-sdk/README.md @@ -352,19 +352,30 @@ tutorial](../exemplars/README.md) demonstrates how to use exemplars to achieve correlation from metrics to traces, which is one of the primary use cases for exemplars. +#### Default behavior + +Exemplars in OpenTelemetry .NET are **off by default** +(`ExemplarFilterType.AlwaysOff`). The [OpenTelemetry +Specification](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/sdk.md#exemplarfilter) +recommends Exemplars collection should be on by default +(`ExemplarFilterType.TraceBased`) however there is a performance cost associated +with Exemplars so OpenTelemetry .NET has taken a more conservative stance for +its default behavior. + #### ExemplarFilter `ExemplarFilter` determines which measurements are offered to the configured `ExemplarReservoir`, which makes the final decision about whether or not the offered measurement gets recorded as an `Exemplar`. Generally `ExemplarFilter` -is a mechanism to control the overhead associated with `Exemplar` offering. +is a mechanism to control the overhead associated with the offering and +recording of `Exemplar`s. -OpenTelemetry SDK comes with the following `ExemplarFilters` (defined on +OpenTelemetry SDK comes with the following `ExemplarFilter`s (defined on `ExemplarFilterType`): -* `AlwaysOff`: Makes no measurements eligible for becoming an `Exemplar`. Using - this is as good as turning off the `Exemplar` feature and is the current - default. +* (Default behavior) `AlwaysOff`: Makes no measurements eligible for becoming an + `Exemplar`. Using this disables `Exemplar` collection and avoids all + performance costs associated with `Exemplar`s. * `AlwaysOn`: Makes all measurements eligible for becoming an `Exemplar`. * `TraceBased`: Makes those measurements eligible for becoming an `Exemplar` which are recorded in the context of a sampled `Activity` (span). @@ -372,6 +383,9 @@ OpenTelemetry SDK comes with the following `ExemplarFilters` (defined on The `SetExemplarFilter` extension method on `MeterProviderBuilder` can be used to set the desired `ExemplarFilterType` and enable `Exemplar` collection: +> [!NOTE] +> The `SetExemplarFilter` API was added in the `1.9.0` release. + ```csharp using OpenTelemetry; using OpenTelemetry.Metrics; @@ -382,6 +396,24 @@ using var meterProvider = Sdk.CreateMeterProviderBuilder() .Build(); ``` +It is also possible to configure the `ExemplarFilter` by using following +environmental variables: + +> [!NOTE] +> Programmatically calling `SetExemplarFilter` will override any defaults set + using environment variables or configuration. + +| Environment variable | Description | Notes | +| -------------------------- | -------------------------------------------------- |-------| +| `OTEL_METRICS_EXEMPLAR_FILTER` | Sets the default `ExemplarFilter` to use for all metrics. | Added in `1.9.0` | +| `OTEL_DOTNET_EXPERIMENTAL_METRICS_EXEMPLAR_FILTER_HISTOGRAMS` | Sets the default `ExemplarFilter` to use for histogram metrics. If set `OTEL_DOTNET_EXPERIMENTAL_METRICS_EXEMPLAR_FILTER_HISTOGRAMS` takes precedence over `OTEL_METRICS_EXEMPLAR_FILTER` for histogram metrics. | Experimental key (may be removed or changed in the future). Added in `1.9.0` | + +Allowed values: + +* `always_off`: Equivalent to `ExemplarFilterType.AlwaysOff` +* `always_on`: Equivalent to `ExemplarFilterType.AlwaysOn` +* `trace_based`: Equivalent to `ExemplarFilterType.TraceBased` + #### ExemplarReservoir `ExemplarReservoir` receives the measurements sampled by the `ExemplarFilter` @@ -398,7 +430,8 @@ metrics except Histograms with buckets. It has a fixed reservoir pool, and implements the equivalent of [naive reservoir](https://en.wikipedia.org/wiki/Reservoir_sampling). The reservoir pool size (currently defaulting to 1) determines the maximum number of exemplars -stored. +stored. Exponential histograms use a `SimpleFixedSizeExemplarReservoir` with a +pool size equal to the number of buckets up to a max of `20`. > [!NOTE] > Currently there is no ability to change or configure `ExemplarReservoir`. diff --git a/examples/AspNetCore/Program.cs b/examples/AspNetCore/Program.cs index 18279940976..89ac46932dc 100644 --- a/examples/AspNetCore/Program.cs +++ b/examples/AspNetCore/Program.cs @@ -84,9 +84,7 @@ // Ensure the MeterProvider subscribes to any custom Meters. builder .AddMeter(Instrumentation.MeterName) -#if EXPOSE_EXPERIMENTAL_FEATURES .SetExemplarFilter(ExemplarFilterType.TraceBased) -#endif .AddRuntimeInstrumentation() .AddHttpClientInstrumentation() .AddAspNetCoreInstrumentation(); diff --git a/src/OpenTelemetry/CHANGELOG.md b/src/OpenTelemetry/CHANGELOG.md index 1ba0afa19c7..ab9ab112b27 100644 --- a/src/OpenTelemetry/CHANGELOG.md +++ b/src/OpenTelemetry/CHANGELOG.md @@ -30,6 +30,13 @@ which has always been supported. ([#5614](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5614)) +* The `ExemplarFilter` used by SDK `MeterProvider`s for histogram metrics can + now be controlled via the experimental + `OTEL_DOTNET_EXPERIMENTAL_METRICS_EXEMPLAR_FILTER_HISTOGRAMS` environment + variable. The supported values are: `always_off`, `always_on`, and + `trace_based`. + ([#5611](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5611)) + ## 1.8.1 Released 2024-Apr-17 diff --git a/src/OpenTelemetry/Metrics/MeterProviderSdk.cs b/src/OpenTelemetry/Metrics/MeterProviderSdk.cs index 595fd54e77d..467da3f538b 100644 --- a/src/OpenTelemetry/Metrics/MeterProviderSdk.cs +++ b/src/OpenTelemetry/Metrics/MeterProviderSdk.cs @@ -16,6 +16,7 @@ internal sealed class MeterProviderSdk : MeterProvider internal const string EmitOverFlowAttributeConfigKey = "OTEL_DOTNET_EXPERIMENTAL_METRICS_EMIT_OVERFLOW_ATTRIBUTE"; internal const string ReclaimUnusedMetricPointsConfigKey = "OTEL_DOTNET_EXPERIMENTAL_METRICS_RECLAIM_UNUSED_METRIC_POINTS"; internal const string ExemplarFilterConfigKey = "OTEL_METRICS_EXEMPLAR_FILTER"; + internal const string ExemplarFilterHistogramsConfigKey = "OTEL_DOTNET_EXPERIMENTAL_METRICS_EXEMPLAR_FILTER_HISTOGRAMS"; internal readonly IServiceProvider ServiceProvider; internal readonly IDisposable? OwnedServiceProvider; @@ -24,6 +25,7 @@ internal sealed class MeterProviderSdk : MeterProvider internal bool EmitOverflowAttribute; internal bool ReclaimUnusedMetricPoints; internal ExemplarFilterType? ExemplarFilter; + internal ExemplarFilterType? ExemplarFilterForHistograms; internal Action? OnCollectObservableInstruments; private readonly List instrumentations = new(); @@ -72,6 +74,9 @@ internal MeterProviderSdk( this.viewConfigs = state.ViewConfigs; + OpenTelemetrySdkEventSource.Log.MeterProviderSdkEvent( + $"MeterProvider configuration: {{MetricLimit={state.MetricLimit}, CardinalityLimit={state.CardinalityLimit}, EmitOverflowAttribute={this.EmitOverflowAttribute}, ReclaimUnusedMetricPoints={this.ReclaimUnusedMetricPoints}, ExemplarFilter={this.ExemplarFilter}, ExemplarFilterForHistograms={this.ExemplarFilterForHistograms}}}."); + foreach (var reader in state.Readers) { Guard.ThrowIfNull(reader); @@ -83,7 +88,8 @@ internal MeterProviderSdk( state.CardinalityLimit, this.EmitOverflowAttribute, this.ReclaimUnusedMetricPoints, - this.ExemplarFilter); + this.ExemplarFilter, + this.ExemplarFilterForHistograms); if (this.reader == null) { @@ -490,37 +496,70 @@ private void ApplySpecificationConfigurationKeys(IConfiguration configuration) OpenTelemetrySdkEventSource.Log.MeterProviderSdkEvent("Reclaim unused metric point feature enabled via configuration."); } + var hasProgrammaticExemplarFilterValue = this.ExemplarFilter.HasValue; + if (configuration.TryGetStringValue(ExemplarFilterConfigKey, out var configValue)) { - if (this.ExemplarFilter.HasValue) + if (hasProgrammaticExemplarFilterValue) { OpenTelemetrySdkEventSource.Log.MeterProviderSdkEvent( $"Exemplar filter configuration value '{configValue}' has been ignored because a value '{this.ExemplarFilter}' was set programmatically."); return; } - ExemplarFilterType? exemplarFilter; + if (!TryParseExemplarFilterFromConfigurationValue(configValue, out var exemplarFilter)) + { + OpenTelemetrySdkEventSource.Log.MeterProviderSdkEvent($"Exemplar filter configuration was found but the value '{configValue}' is invalid and will be ignored."); + return; + } + + this.ExemplarFilter = exemplarFilter; + + OpenTelemetrySdkEventSource.Log.MeterProviderSdkEvent($"Exemplar filter set to '{exemplarFilter}' from configuration."); + } + + if (configuration.TryGetStringValue(ExemplarFilterHistogramsConfigKey, out configValue)) + { + if (hasProgrammaticExemplarFilterValue) + { + OpenTelemetrySdkEventSource.Log.MeterProviderSdkEvent( + $"Exemplar filter histogram configuration value '{configValue}' has been ignored because a value '{this.ExemplarFilter}' was set programmatically."); + return; + } + + if (!TryParseExemplarFilterFromConfigurationValue(configValue, out var exemplarFilter)) + { + OpenTelemetrySdkEventSource.Log.MeterProviderSdkEvent($"Exemplar filter histogram configuration was found but the value '{configValue}' is invalid and will be ignored."); + return; + } + + this.ExemplarFilterForHistograms = exemplarFilter; + + OpenTelemetrySdkEventSource.Log.MeterProviderSdkEvent($"Exemplar filter for histograms set to '{exemplarFilter}' from configuration."); + } + + static bool TryParseExemplarFilterFromConfigurationValue(string? configValue, out ExemplarFilterType? exemplarFilter) + { if (string.Equals("always_off", configValue, StringComparison.OrdinalIgnoreCase)) { exemplarFilter = ExemplarFilterType.AlwaysOff; + return true; } - else if (string.Equals("always_on", configValue, StringComparison.OrdinalIgnoreCase)) + + if (string.Equals("always_on", configValue, StringComparison.OrdinalIgnoreCase)) { exemplarFilter = ExemplarFilterType.AlwaysOn; + return true; } - else if (string.Equals("trace_based", configValue, StringComparison.OrdinalIgnoreCase)) + + if (string.Equals("trace_based", configValue, StringComparison.OrdinalIgnoreCase)) { exemplarFilter = ExemplarFilterType.TraceBased; + return true; } - else - { - OpenTelemetrySdkEventSource.Log.MeterProviderSdkEvent($"Exemplar filter configuration was found but the value '{configValue}' is invalid and will be ignored."); - return; - } - - this.ExemplarFilter = exemplarFilter; - OpenTelemetrySdkEventSource.Log.MeterProviderSdkEvent($"Exemplar filter set to '{exemplarFilter}' from configuration."); + exemplarFilter = null; + return false; } } } diff --git a/src/OpenTelemetry/Metrics/Metric.cs b/src/OpenTelemetry/Metrics/Metric.cs index 32b107c5b2f..7801c2dd452 100644 --- a/src/OpenTelemetry/Metrics/Metric.cs +++ b/src/OpenTelemetry/Metrics/Metric.cs @@ -125,12 +125,7 @@ internal Metric( aggType = AggregationType.LongGauge; this.MetricType = MetricType.LongGauge; } - else if (instrumentIdentity.InstrumentType == typeof(Histogram) - || instrumentIdentity.InstrumentType == typeof(Histogram) - || instrumentIdentity.InstrumentType == typeof(Histogram) - || instrumentIdentity.InstrumentType == typeof(Histogram) - || instrumentIdentity.InstrumentType == typeof(Histogram) - || instrumentIdentity.InstrumentType == typeof(Histogram)) + else if (instrumentIdentity.IsHistogram) { var explicitBucketBounds = instrumentIdentity.HistogramBucketBounds; var exponentialMaxSize = instrumentIdentity.ExponentialHistogramMaxSize; diff --git a/src/OpenTelemetry/Metrics/MetricReaderExt.cs b/src/OpenTelemetry/Metrics/MetricReaderExt.cs index dfb95fab2b9..a6a0a642d46 100644 --- a/src/OpenTelemetry/Metrics/MetricReaderExt.cs +++ b/src/OpenTelemetry/Metrics/MetricReaderExt.cs @@ -25,6 +25,7 @@ public abstract partial class MetricReader private bool emitOverflowAttribute; private bool reclaimUnusedMetricPoints; private ExemplarFilterType? exemplarFilter; + private ExemplarFilterType? exemplarFilterForHistograms; internal static void DeactivateMetric(Metric metric) { @@ -54,6 +55,11 @@ internal virtual List AddMetricWithNoViews(Instrument instrument) Debug.Assert(this.metrics != null, "this.metrics was null"); var metricStreamIdentity = new MetricStreamIdentity(instrument!, metricStreamConfiguration: null); + + var exemplarFilter = metricStreamIdentity.IsHistogram + ? this.exemplarFilterForHistograms ?? this.exemplarFilter + : this.exemplarFilter; + lock (this.instrumentCreationLock) { if (this.TryGetExistingMetric(in metricStreamIdentity, out var existingMetric)) @@ -72,7 +78,13 @@ internal virtual List AddMetricWithNoViews(Instrument instrument) Metric? metric = null; try { - metric = new Metric(metricStreamIdentity, this.GetAggregationTemporality(metricStreamIdentity.InstrumentType), this.cardinalityLimit, this.emitOverflowAttribute, this.reclaimUnusedMetricPoints, this.exemplarFilter); + metric = new Metric( + metricStreamIdentity, + this.GetAggregationTemporality(metricStreamIdentity.InstrumentType), + this.cardinalityLimit, + this.emitOverflowAttribute, + this.reclaimUnusedMetricPoints, + exemplarFilter); } catch (NotSupportedException nse) { @@ -114,6 +126,10 @@ internal virtual List AddMetricWithViews(Instrument instrument, List AddMetricWithViews(Instrument instrument, List + this.InstrumentType == typeof(Histogram) + || this.InstrumentType == typeof(Histogram) + || this.InstrumentType == typeof(Histogram) + || this.InstrumentType == typeof(Histogram) + || this.InstrumentType == typeof(Histogram) + || this.InstrumentType == typeof(Histogram); + public static bool operator ==(MetricStreamIdentity metricIdentity1, MetricStreamIdentity metricIdentity2) => metricIdentity1.Equals(metricIdentity2); public static bool operator !=(MetricStreamIdentity metricIdentity1, MetricStreamIdentity metricIdentity2) => !metricIdentity1.Equals(metricIdentity2); diff --git a/test/OpenTelemetry.Tests/Metrics/MetricExemplarTests.cs b/test/OpenTelemetry.Tests/Metrics/MetricExemplarTests.cs index 0e5058ba1bc..fe7fb5244d9 100644 --- a/test/OpenTelemetry.Tests/Metrics/MetricExemplarTests.cs +++ b/test/OpenTelemetry.Tests/Metrics/MetricExemplarTests.cs @@ -34,6 +34,7 @@ public void TestExemplarFilterSetFromConfiguration( configBuilder.AddInMemoryCollection(new Dictionary { [MeterProviderSdk.ExemplarFilterConfigKey] = configValue, + [MeterProviderSdk.ExemplarFilterHistogramsConfigKey] = configValue, }); } @@ -52,6 +53,14 @@ public void TestExemplarFilterSetFromConfiguration( Assert.NotNull(meterProviderSdk); Assert.Equal((ExemplarFilterType?)expectedValue, meterProviderSdk.ExemplarFilter); + if (programmaticValue.HasValue) + { + Assert.False(meterProviderSdk.ExemplarFilterForHistograms.HasValue); + } + else + { + Assert.Equal((ExemplarFilterType?)expectedValue, meterProviderSdk.ExemplarFilterForHistograms); + } } [Theory] @@ -260,9 +269,10 @@ static void ValidateSecondPhase( } [Theory] - [InlineData(MetricReaderTemporalityPreference.Cumulative)] - [InlineData(MetricReaderTemporalityPreference.Delta)] - public void TestExemplarsHistogramWithBuckets(MetricReaderTemporalityPreference temporality) + [InlineData(MetricReaderTemporalityPreference.Cumulative, null)] + [InlineData(MetricReaderTemporalityPreference.Delta, null)] + [InlineData(MetricReaderTemporalityPreference.Delta, "always_on")] + public void TestExemplarsHistogramWithBuckets(MetricReaderTemporalityPreference temporality, string? configValue) { DateTime testStartTime = DateTime.UtcNow; var exportedItems = new List(); @@ -275,31 +285,49 @@ public void TestExemplarsHistogramWithBuckets(MetricReaderTemporalityPreference var buckets = new double[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; - using var container = this.BuildMeterProvider(out var meterProvider, builder => builder - .AddMeter(meter.Name) - .SetExemplarFilter(ExemplarFilterType.AlwaysOn) - .AddView(i => + var configBuilder = new ConfigurationBuilder(); + if (!string.IsNullOrEmpty(configValue)) + { + configBuilder.AddInMemoryCollection(new Dictionary + { + [MeterProviderSdk.ExemplarFilterConfigKey] = "always_off", + [MeterProviderSdk.ExemplarFilterHistogramsConfigKey] = configValue, + }); + } + + using var container = this.BuildMeterProvider(out var meterProvider, builder => + { + if (string.IsNullOrEmpty(configValue)) { - if (i.Name.StartsWith("histogramWithBucketsAndMinMax")) + builder.SetExemplarFilter(ExemplarFilterType.AlwaysOn); + } + + builder + .ConfigureServices(s => s.AddSingleton(configBuilder.Build())) + .AddMeter(meter.Name) + .AddView(i => { - return new ExplicitBucketHistogramConfiguration + if (i.Name.StartsWith("histogramWithBucketsAndMinMax")) { - Boundaries = buckets, - }; - } - else - { - return new ExplicitBucketHistogramConfiguration + return new ExplicitBucketHistogramConfiguration + { + Boundaries = buckets, + }; + } + else { - Boundaries = buckets, - RecordMinMax = false, - }; - } - }) - .AddInMemoryExporter(exportedItems, metricReaderOptions => - { - metricReaderOptions.TemporalityPreference = temporality; - })); + return new ExplicitBucketHistogramConfiguration + { + Boundaries = buckets, + RecordMinMax = false, + }; + } + }) + .AddInMemoryExporter(exportedItems, metricReaderOptions => + { + metricReaderOptions.TemporalityPreference = temporality; + }); + }); var measurementValues = buckets /* 2000 is here to test overflow measurement */