diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/CHANGELOG.md b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/CHANGELOG.md index fa2eacc89fe..d8200afd2b1 100644 --- a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/CHANGELOG.md @@ -13,6 +13,17 @@ `parent_is_remote` information. ([#5563](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5563)) +* Introduced experimental support for automatically retrying export to the otlp + endpoint by storing the telemetry offline during transient network errors. + Users can enable this feature by setting the + `OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY` environment variable to `disk`. The + default path where the telemetry is stored is obtained by calling + [Path.GetTempPath()](https://learn.microsoft.com/dotnet/api/system.io.path.gettemppath) + or can be customized by setting + `OTEL_DOTNET_EXPERIMENTAL_OTLP_DISK_RETRY_DIRECTORY_PATH` environment + variable. + ([#5527](https://github.com/open-telemetry/opentelemetry-dotnet/pull/5527)) + ## 1.8.1 Released 2024-Apr-17 diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExperimentalOptions.cs b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExperimentalOptions.cs index da9c185a516..65d0bf57ee1 100644 --- a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExperimentalOptions.cs +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExperimentalOptions.cs @@ -17,6 +17,8 @@ internal sealed class ExperimentalOptions public const string OtlpRetryEnvVar = "OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY"; + public const string OtlpDiskRetryDirectoryPathEnvVar = "OTEL_DOTNET_EXPERIMENTAL_OTLP_DISK_RETRY_DIRECTORY_PATH"; + public ExperimentalOptions() : this(new ConfigurationBuilder().AddEnvironmentVariables().Build()) { @@ -29,9 +31,29 @@ public ExperimentalOptions(IConfiguration configuration) this.EmitLogEventAttributes = emitLogEventAttributes; } - if (configuration.TryGetStringValue(OtlpRetryEnvVar, out var retryPolicy) && retryPolicy != null && retryPolicy.Equals("in_memory", StringComparison.OrdinalIgnoreCase)) + if (configuration.TryGetStringValue(OtlpRetryEnvVar, out var retryPolicy) && retryPolicy != null) { - this.EnableInMemoryRetry = true; + if (retryPolicy.Equals("in_memory", StringComparison.OrdinalIgnoreCase)) + { + this.EnableInMemoryRetry = true; + } + else if (retryPolicy.Equals("disk", StringComparison.OrdinalIgnoreCase)) + { + this.EnableDiskRetry = true; + if (configuration.TryGetStringValue(OtlpDiskRetryDirectoryPathEnvVar, out var path) && path != null) + { + this.DiskRetryDirectoryPath = path; + } + else + { + // Fallback to temp location. + this.DiskRetryDirectoryPath = Path.GetTempPath(); + } + } + else + { + throw new NotSupportedException($"Retry Policy '{retryPolicy}' is not supported."); + } } } @@ -48,4 +70,14 @@ public ExperimentalOptions(IConfiguration configuration) /// href="https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md#retry"/>. /// public bool EnableInMemoryRetry { get; } + + /// + /// Gets a value indicating whether or not retry via disk should be enabled for transient errors. + /// + public bool EnableDiskRetry { get; } + + /// + /// Gets the path on disk where the telemetry will be stored for retries at a later point. + /// + public string? DiskRetryDirectoryPath { get; } } diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpExporterOptionsExtensions.cs b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpExporterOptionsExtensions.cs index 34e5a09d597..ba68de13261 100644 --- a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpExporterOptionsExtensions.cs +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpExporterOptionsExtensions.cs @@ -11,6 +11,8 @@ #if NETSTANDARD2_1 || NET6_0_OR_GREATER using Grpc.Net.Client; #endif +using System.Diagnostics; +using Google.Protobuf; using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.Transmission; using LogOtlpCollector = OpenTelemetry.Proto.Collector.Logs.V1; using MetricsOtlpCollector = OpenTelemetry.Proto.Collector.Metrics.V1; @@ -100,9 +102,29 @@ public static THeaders GetHeaders(this OtlpExporterOptions options, Ac ? httpTraceExportClient.HttpClient.Timeout.TotalMilliseconds : options.TimeoutMilliseconds; - return experimentalOptions.EnableInMemoryRetry - ? new OtlpExporterRetryTransmissionHandler(exportClient, timeoutMilliseconds) - : new OtlpExporterTransmissionHandler(exportClient, timeoutMilliseconds); + if (experimentalOptions.EnableInMemoryRetry) + { + return new OtlpExporterRetryTransmissionHandler(exportClient, timeoutMilliseconds); + } + else if (experimentalOptions.EnableDiskRetry) + { + Debug.Assert(!string.IsNullOrEmpty(experimentalOptions.DiskRetryDirectoryPath), $"{nameof(experimentalOptions.DiskRetryDirectoryPath)} is null or empty"); + + return new OtlpExporterPersistentStorageTransmissionHandler( + exportClient, + timeoutMilliseconds, + (byte[] data) => + { + var request = new TraceOtlpCollector.ExportTraceServiceRequest(); + request.MergeFrom(data); + return request; + }, + Path.Combine(experimentalOptions.DiskRetryDirectoryPath, "traces")); + } + else + { + return new OtlpExporterTransmissionHandler(exportClient, timeoutMilliseconds); + } } public static OtlpExporterTransmissionHandler GetMetricsExportTransmissionHandler(this OtlpExporterOptions options, ExperimentalOptions experimentalOptions) @@ -116,9 +138,29 @@ public static THeaders GetHeaders(this OtlpExporterOptions options, Ac ? httpMetricsExportClient.HttpClient.Timeout.TotalMilliseconds : options.TimeoutMilliseconds; - return experimentalOptions.EnableInMemoryRetry - ? new OtlpExporterRetryTransmissionHandler(exportClient, timeoutMilliseconds) - : new OtlpExporterTransmissionHandler(exportClient, timeoutMilliseconds); + if (experimentalOptions.EnableInMemoryRetry) + { + return new OtlpExporterRetryTransmissionHandler(exportClient, timeoutMilliseconds); + } + else if (experimentalOptions.EnableDiskRetry) + { + Debug.Assert(!string.IsNullOrEmpty(experimentalOptions.DiskRetryDirectoryPath), $"{nameof(experimentalOptions.DiskRetryDirectoryPath)} is null or empty"); + + return new OtlpExporterPersistentStorageTransmissionHandler( + exportClient, + timeoutMilliseconds, + (byte[] data) => + { + var request = new MetricsOtlpCollector.ExportMetricsServiceRequest(); + request.MergeFrom(data); + return request; + }, + Path.Combine(experimentalOptions.DiskRetryDirectoryPath, "metrics")); + } + else + { + return new OtlpExporterTransmissionHandler(exportClient, timeoutMilliseconds); + } } public static OtlpExporterTransmissionHandler GetLogsExportTransmissionHandler(this OtlpExporterOptions options, ExperimentalOptions experimentalOptions) @@ -128,9 +170,29 @@ public static THeaders GetHeaders(this OtlpExporterOptions options, Ac ? httpLogExportClient.HttpClient.Timeout.TotalMilliseconds : options.TimeoutMilliseconds; - return experimentalOptions.EnableInMemoryRetry - ? new OtlpExporterRetryTransmissionHandler(exportClient, timeoutMilliseconds) - : new OtlpExporterTransmissionHandler(exportClient, timeoutMilliseconds); + if (experimentalOptions.EnableInMemoryRetry) + { + return new OtlpExporterRetryTransmissionHandler(exportClient, timeoutMilliseconds); + } + else if (experimentalOptions.EnableDiskRetry) + { + Debug.Assert(!string.IsNullOrEmpty(experimentalOptions.DiskRetryDirectoryPath), $"{nameof(experimentalOptions.DiskRetryDirectoryPath)} is null or empty"); + + return new OtlpExporterPersistentStorageTransmissionHandler( + exportClient, + timeoutMilliseconds, + (byte[] data) => + { + var request = new LogOtlpCollector.ExportLogsServiceRequest(); + request.MergeFrom(data); + return request; + }, + Path.Combine(experimentalOptions.DiskRetryDirectoryPath, "logs")); + } + else + { + return new OtlpExporterTransmissionHandler(exportClient, timeoutMilliseconds); + } } public static IExportClient GetTraceExportClient(this OtlpExporterOptions options) => diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/README.md b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/README.md index dd14461e6a5..adff6d6c7f2 100644 --- a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/README.md +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/README.md @@ -627,10 +627,17 @@ want to solicit feedback from the community. * `OTEL_DOTNET_EXPERIMENTAL_OTLP_RETRY` - When set to `in_memory`, it enables in-memory retry for transient errors + * When set to `in_memory`, it enables in-memory retry for transient errors encountered while sending telemetry. - Added in `1.8.0`. + Added in `1.8.0`. + + * When set to `disk` along with setting + `OTEL_DOTNET_EXPERIMENTAL_OTLP_DISK_RETRY_DIRECTORY_PATH` to the path on + disk, it enables retries by storing telemetry on disk during transient + errors. + + Added in **TBD** (Unreleased). * Logs diff --git a/test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/OtlpExporterOptionsExtensionsTests.cs b/test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/OtlpExporterOptionsExtensionsTests.cs index b8bfb608288..2b7810dfadf 100644 --- a/test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/OtlpExporterOptionsExtensionsTests.cs +++ b/test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/OtlpExporterOptionsExtensionsTests.cs @@ -4,6 +4,7 @@ #if NETFRAMEWORK using System.Net.Http; #endif +using Microsoft.Extensions.Configuration; using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation; using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.ExportClient; using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.Transmission; @@ -130,16 +131,34 @@ public void AppendPathIfNotPresent_TracesPath_AppendsCorrectly(string inputUri, } [Theory] - [InlineData(OtlpExportProtocol.Grpc, typeof(OtlpGrpcTraceExportClient), false, 10000)] - [InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpTraceExportClient), false, 10000)] - [InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpTraceExportClient), true, 8000)] - [InlineData(OtlpExportProtocol.Grpc, typeof(OtlpGrpcMetricsExportClient), false, 10000)] - [InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpMetricsExportClient), false, 10000)] - [InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpMetricsExportClient), true, 8000)] - [InlineData(OtlpExportProtocol.Grpc, typeof(OtlpGrpcLogExportClient), false, 10000)] - [InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpLogExportClient), false, 10000)] - [InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpLogExportClient), true, 8000)] - public void GetTransmissionHandler_InitializesCorrectExportClientAndTimeoutValue(OtlpExportProtocol protocol, Type exportClientType, bool customHttpClient, int expectedTimeoutMilliseconds) + [InlineData(OtlpExportProtocol.Grpc, typeof(OtlpGrpcTraceExportClient), false, 10000, null)] + [InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpTraceExportClient), false, 10000, null)] + [InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpTraceExportClient), true, 8000, null)] + [InlineData(OtlpExportProtocol.Grpc, typeof(OtlpGrpcMetricsExportClient), false, 10000, null)] + [InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpMetricsExportClient), false, 10000, null)] + [InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpMetricsExportClient), true, 8000, null)] + [InlineData(OtlpExportProtocol.Grpc, typeof(OtlpGrpcLogExportClient), false, 10000, null)] + [InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpLogExportClient), false, 10000, null)] + [InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpLogExportClient), true, 8000, null)] + [InlineData(OtlpExportProtocol.Grpc, typeof(OtlpGrpcTraceExportClient), false, 10000, "in_memory")] + [InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpTraceExportClient), false, 10000, "in_memory")] + [InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpTraceExportClient), true, 8000, "in_memory")] + [InlineData(OtlpExportProtocol.Grpc, typeof(OtlpGrpcMetricsExportClient), false, 10000, "in_memory")] + [InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpMetricsExportClient), false, 10000, "in_memory")] + [InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpMetricsExportClient), true, 8000, "in_memory")] + [InlineData(OtlpExportProtocol.Grpc, typeof(OtlpGrpcLogExportClient), false, 10000, "in_memory")] + [InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpLogExportClient), false, 10000, "in_memory")] + [InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpLogExportClient), true, 8000, "in_memory")] + [InlineData(OtlpExportProtocol.Grpc, typeof(OtlpGrpcTraceExportClient), false, 10000, "disk")] + [InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpTraceExportClient), false, 10000, "disk")] + [InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpTraceExportClient), true, 8000, "disk")] + [InlineData(OtlpExportProtocol.Grpc, typeof(OtlpGrpcMetricsExportClient), false, 10000, "disk")] + [InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpMetricsExportClient), false, 10000, "disk")] + [InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpMetricsExportClient), true, 8000, "disk")] + [InlineData(OtlpExportProtocol.Grpc, typeof(OtlpGrpcLogExportClient), false, 10000, "disk")] + [InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpLogExportClient), false, 10000, "disk")] + [InlineData(OtlpExportProtocol.HttpProtobuf, typeof(OtlpHttpLogExportClient), true, 8000, "disk")] + public void GetTransmissionHandler_InitializesCorrectHandlerExportClientAndTimeoutValue(OtlpExportProtocol protocol, Type exportClientType, bool customHttpClient, int expectedTimeoutMilliseconds, string retryStrategy) { var exporterOptions = new OtlpExporterOptions() { Protocol = protocol }; if (customHttpClient) @@ -150,28 +169,45 @@ public void GetTransmissionHandler_InitializesCorrectExportClientAndTimeoutValue }; } + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { [ExperimentalOptions.OtlpRetryEnvVar] = retryStrategy }) + .Build(); + if (exportClientType == typeof(OtlpGrpcTraceExportClient) || exportClientType == typeof(OtlpHttpTraceExportClient)) { - var transmissionHandler = exporterOptions.GetTraceExportTransmissionHandler(new ExperimentalOptions()); + var transmissionHandler = exporterOptions.GetTraceExportTransmissionHandler(new ExperimentalOptions(configuration)); - AssertTransmissionHandlerProperties(transmissionHandler, exportClientType, expectedTimeoutMilliseconds); + AssertTransmissionHandler(transmissionHandler, exportClientType, expectedTimeoutMilliseconds, retryStrategy); } else if (exportClientType == typeof(OtlpGrpcMetricsExportClient) || exportClientType == typeof(OtlpHttpMetricsExportClient)) { - var transmissionHandler = exporterOptions.GetMetricsExportTransmissionHandler(new ExperimentalOptions()); + var transmissionHandler = exporterOptions.GetMetricsExportTransmissionHandler(new ExperimentalOptions(configuration)); - AssertTransmissionHandlerProperties(transmissionHandler, exportClientType, expectedTimeoutMilliseconds); + AssertTransmissionHandler(transmissionHandler, exportClientType, expectedTimeoutMilliseconds, retryStrategy); } else { - var transmissionHandler = exporterOptions.GetLogsExportTransmissionHandler(new ExperimentalOptions()); + var transmissionHandler = exporterOptions.GetLogsExportTransmissionHandler(new ExperimentalOptions(configuration)); - AssertTransmissionHandlerProperties(transmissionHandler, exportClientType, expectedTimeoutMilliseconds); + AssertTransmissionHandler(transmissionHandler, exportClientType, expectedTimeoutMilliseconds, retryStrategy); } } - private static void AssertTransmissionHandlerProperties(OtlpExporterTransmissionHandler transmissionHandler, Type exportClientType, int expectedTimeoutMilliseconds) + private static void AssertTransmissionHandler(OtlpExporterTransmissionHandler transmissionHandler, Type exportClientType, int expectedTimeoutMilliseconds, string retryStrategy) { + if (retryStrategy == "in_memory") + { + Assert.True(transmissionHandler is OtlpExporterRetryTransmissionHandler); + } + else if (retryStrategy == "disk") + { + Assert.True(transmissionHandler is OtlpExporterPersistentStorageTransmissionHandler); + } + else + { + Assert.True(transmissionHandler is OtlpExporterTransmissionHandler); + } + Assert.Equal(exportClientType, transmissionHandler.ExportClient.GetType()); Assert.Equal(expectedTimeoutMilliseconds, transmissionHandler.TimeoutMilliseconds);