From af31ca7a2ecdd3318f8e02c1ed80081a68fd1220 Mon Sep 17 00:00:00 2001 From: Michael Maxwell Date: Wed, 13 Apr 2022 10:48:59 -0700 Subject: [PATCH] Adding a Geneva Exporter - Exports to local ETW or UDS (#273) --- .github/workflows/dotnet-core-cov.yml | 2 +- opentelemetry-dotnet-contrib.sln | 27 + .../AssemblyInfo.cs | 16 + .../CHANGELOG.md | 3 + .../ConnectionStringBuilder.cs | 212 +++++ .../EtwDataTransport.cs | 77 ++ .../ExporterEventSource.cs | 74 ++ .../GenevaBaseExporter.cs | 77 ++ .../GenevaExporterHelperExtensions.cs | 41 + .../GenevaExporterOptions.cs | 63 ++ .../GenevaLogExporter.cs | 450 ++++++++++ .../GenevaLoggingExtensions.cs | 32 + .../GenevaMetricExporter.cs | 489 +++++++++++ .../GenevaMetricExporterExtensions.cs | 21 + .../GenevaMetricExporterOptions.cs | 66 ++ .../GenevaTraceExporter.cs | 435 ++++++++++ .../IDataTransport.cs | 9 + .../IMetricDataTransport.cs | 18 + .../MessagePackSerializer.cs | 559 +++++++++++++ .../MetricEtwDataTransport.cs | 46 ++ .../MetricSerializer.cs | 312 +++++++ .../MetricUnixDataTransport.cs | 37 + .../OpenTelemetry.Exporter.Geneva.csproj | 22 + src/OpenTelemetry.Exporter.Geneva/README.md | 3 + .../ReentrantExportProcessor.cs | 36 + src/OpenTelemetry.Exporter.Geneva/Schema.cs | 87 ++ .../ServiceProviderExtensions.cs | 31 + .../UnixDomainSocketDataTransport.cs | 93 +++ .../UnixDomainSocketEndPoint.cs | 79 ++ .../Exporter/LogExporterBenchmarks.cs | 160 ++++ .../Exporter/MetricExporterBenchmarks.cs | 624 ++++++++++++++ .../Exporter/TraceExporterBenchmarks.cs | 110 +++ ...Telemetry.Exporter.Geneva.Benchmark.csproj | 21 + .../Program.cs | 9 + .../DummyServer.cs | 86 ++ ...penTelemetry.Exporter.Geneva.Stress.csproj | 18 + .../Program.cs | 187 +++++ .../ConnectionStringBuilderTests.cs | 228 ++++++ .../GenevaLogExporterTests.cs | 768 ++++++++++++++++++ .../GenevaMetricExporterOptionsTests.cs | 57 ++ .../GenevaMetricExporterTests.cs | 726 +++++++++++++++++ .../GenevaTraceExporterTests.cs | 515 ++++++++++++ .../MessagePackSerializerTests.cs | 388 +++++++++ .../MetricsContract.cs | 665 +++++++++++++++ ...nTelemetry.Exporter.Geneva.UnitTest.csproj | 40 + .../UnixDomainSocketDataTransportTests.cs | 195 +++++ .../UnixDomainSocketEndPointTests.cs | 68 ++ 47 files changed, 8281 insertions(+), 1 deletion(-) create mode 100644 src/OpenTelemetry.Exporter.Geneva/AssemblyInfo.cs create mode 100644 src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md create mode 100644 src/OpenTelemetry.Exporter.Geneva/ConnectionStringBuilder.cs create mode 100644 src/OpenTelemetry.Exporter.Geneva/EtwDataTransport.cs create mode 100644 src/OpenTelemetry.Exporter.Geneva/ExporterEventSource.cs create mode 100644 src/OpenTelemetry.Exporter.Geneva/GenevaBaseExporter.cs create mode 100644 src/OpenTelemetry.Exporter.Geneva/GenevaExporterHelperExtensions.cs create mode 100644 src/OpenTelemetry.Exporter.Geneva/GenevaExporterOptions.cs create mode 100644 src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs create mode 100644 src/OpenTelemetry.Exporter.Geneva/GenevaLoggingExtensions.cs create mode 100644 src/OpenTelemetry.Exporter.Geneva/GenevaMetricExporter.cs create mode 100644 src/OpenTelemetry.Exporter.Geneva/GenevaMetricExporterExtensions.cs create mode 100644 src/OpenTelemetry.Exporter.Geneva/GenevaMetricExporterOptions.cs create mode 100644 src/OpenTelemetry.Exporter.Geneva/GenevaTraceExporter.cs create mode 100644 src/OpenTelemetry.Exporter.Geneva/IDataTransport.cs create mode 100644 src/OpenTelemetry.Exporter.Geneva/IMetricDataTransport.cs create mode 100644 src/OpenTelemetry.Exporter.Geneva/MessagePackSerializer.cs create mode 100644 src/OpenTelemetry.Exporter.Geneva/MetricEtwDataTransport.cs create mode 100644 src/OpenTelemetry.Exporter.Geneva/MetricSerializer.cs create mode 100644 src/OpenTelemetry.Exporter.Geneva/MetricUnixDataTransport.cs create mode 100644 src/OpenTelemetry.Exporter.Geneva/OpenTelemetry.Exporter.Geneva.csproj create mode 100644 src/OpenTelemetry.Exporter.Geneva/README.md create mode 100644 src/OpenTelemetry.Exporter.Geneva/ReentrantExportProcessor.cs create mode 100644 src/OpenTelemetry.Exporter.Geneva/Schema.cs create mode 100644 src/OpenTelemetry.Exporter.Geneva/ServiceProviderExtensions.cs create mode 100644 src/OpenTelemetry.Exporter.Geneva/UnixDomainSocketDataTransport.cs create mode 100644 src/OpenTelemetry.Exporter.Geneva/UnixDomainSocketEndPoint.cs create mode 100644 test/OpenTelemetry.Exporter.Geneva.Benchmark/Exporter/LogExporterBenchmarks.cs create mode 100644 test/OpenTelemetry.Exporter.Geneva.Benchmark/Exporter/MetricExporterBenchmarks.cs create mode 100644 test/OpenTelemetry.Exporter.Geneva.Benchmark/Exporter/TraceExporterBenchmarks.cs create mode 100644 test/OpenTelemetry.Exporter.Geneva.Benchmark/OpenTelemetry.Exporter.Geneva.Benchmark.csproj create mode 100644 test/OpenTelemetry.Exporter.Geneva.Benchmark/Program.cs create mode 100644 test/OpenTelemetry.Exporter.Geneva.Stress/DummyServer.cs create mode 100644 test/OpenTelemetry.Exporter.Geneva.Stress/OpenTelemetry.Exporter.Geneva.Stress.csproj create mode 100644 test/OpenTelemetry.Exporter.Geneva.Stress/Program.cs create mode 100644 test/OpenTelemetry.Exporter.Geneva.UnitTest/ConnectionStringBuilderTests.cs create mode 100644 test/OpenTelemetry.Exporter.Geneva.UnitTest/GenevaLogExporterTests.cs create mode 100644 test/OpenTelemetry.Exporter.Geneva.UnitTest/GenevaMetricExporterOptionsTests.cs create mode 100644 test/OpenTelemetry.Exporter.Geneva.UnitTest/GenevaMetricExporterTests.cs create mode 100644 test/OpenTelemetry.Exporter.Geneva.UnitTest/GenevaTraceExporterTests.cs create mode 100644 test/OpenTelemetry.Exporter.Geneva.UnitTest/MessagePackSerializerTests.cs create mode 100644 test/OpenTelemetry.Exporter.Geneva.UnitTest/MetricsContract.cs create mode 100644 test/OpenTelemetry.Exporter.Geneva.UnitTest/OpenTelemetry.Exporter.Geneva.UnitTest.csproj create mode 100644 test/OpenTelemetry.Exporter.Geneva.UnitTest/UnixDomainSocketDataTransportTests.cs create mode 100644 test/OpenTelemetry.Exporter.Geneva.UnitTest/UnixDomainSocketEndPointTests.cs diff --git a/.github/workflows/dotnet-core-cov.yml b/.github/workflows/dotnet-core-cov.yml index d97be6a35af..cd37a3197f3 100644 --- a/.github/workflows/dotnet-core-cov.yml +++ b/.github/workflows/dotnet-core-cov.yml @@ -31,7 +31,7 @@ jobs: run: dotnet build --configuration Release --no-restore - name: dotnet test - run: dotnet test --collect:"Code Coverage" --results-directory:"TestResults" --configuration Release --no-build -- RunConfiguration.DisableAppDomain=true + run: dotnet test --collect:"Code Coverage" --results-directory:"TestResults" --configuration Release --no-build --filter "Platform=Any|Platform=Windows" -- RunConfiguration.DisableAppDomain=true - name: Process code coverage run: .\build\process-codecoverage.ps1 diff --git a/opentelemetry-dotnet-contrib.sln b/opentelemetry-dotnet-contrib.sln index 440e9f1c952..069d9252dd2 100644 --- a/opentelemetry-dotnet-contrib.sln +++ b/opentelemetry-dotnet-contrib.sln @@ -169,6 +169,13 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenTelemetry.Extensions", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenTelemetry.Extensions.Tests", "test\OpenTelemetry.Extensions.Tests\OpenTelemetry.Extensions.Tests.csproj", "{2117F4E3-6612-4E4D-A757-27271EEB7783}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenTelemetry.Exporter.Geneva", "src\OpenTelemetry.Exporter.Geneva\OpenTelemetry.Exporter.Geneva.csproj", "{1105C814-31DA-4214-BEA8-6DB5FC12C808}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenTelemetry.Exporter.Geneva.Benchmark", "test\OpenTelemetry.Exporter.Geneva.Benchmark\OpenTelemetry.Exporter.Geneva.Benchmark.csproj", "{F53FD7F5-DBC0-4FA5-83BA-B4C07A5BD248}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenTelemetry.Exporter.Geneva.Stress", "test\OpenTelemetry.Exporter.Geneva.Stress\OpenTelemetry.Exporter.Geneva.Stress.csproj", "{F632DFB6-38AD-4356-8997-8CCC0492619C}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenTelemetry.Exporter.Geneva.UnitTest", "test\OpenTelemetry.Exporter.Geneva.UnitTest\OpenTelemetry.Exporter.Geneva.UnitTest.csproj", "{A3EB4E60-256C-45EC-92EE-68FD035CAD11}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenTelemetry.Instrumentation.Hangfire", "src\OpenTelemetry.Instrumentation.Hangfire\OpenTelemetry.Instrumentation.Hangfire.csproj", "{BE5FFBBB-D73F-4071-92F4-F1694881604F}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenTelemetry.Instrumentation.Hangfire.Tests", "test\OpenTelemetry.Instrumentation.Hangfire.Tests\OpenTelemetry.Instrumentation.Hangfire.Tests.csproj", "{ED774FC3-C1C0-44CD-BA41-686C04BEB3E5}" @@ -339,6 +346,22 @@ Global {2117F4E3-6612-4E4D-A757-27271EEB7783}.Debug|Any CPU.Build.0 = Debug|Any CPU {2117F4E3-6612-4E4D-A757-27271EEB7783}.Release|Any CPU.ActiveCfg = Release|Any CPU {2117F4E3-6612-4E4D-A757-27271EEB7783}.Release|Any CPU.Build.0 = Release|Any CPU + {1105C814-31DA-4214-BEA8-6DB5FC12C808}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1105C814-31DA-4214-BEA8-6DB5FC12C808}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1105C814-31DA-4214-BEA8-6DB5FC12C808}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1105C814-31DA-4214-BEA8-6DB5FC12C808}.Release|Any CPU.Build.0 = Release|Any CPU + {F53FD7F5-DBC0-4FA5-83BA-B4C07A5BD248}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F53FD7F5-DBC0-4FA5-83BA-B4C07A5BD248}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F53FD7F5-DBC0-4FA5-83BA-B4C07A5BD248}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F53FD7F5-DBC0-4FA5-83BA-B4C07A5BD248}.Release|Any CPU.Build.0 = Release|Any CPU + {F632DFB6-38AD-4356-8997-8CCC0492619C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F632DFB6-38AD-4356-8997-8CCC0492619C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F632DFB6-38AD-4356-8997-8CCC0492619C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F632DFB6-38AD-4356-8997-8CCC0492619C}.Release|Any CPU.Build.0 = Release|Any CPU + {A3EB4E60-256C-45EC-92EE-68FD035CAD11}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A3EB4E60-256C-45EC-92EE-68FD035CAD11}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A3EB4E60-256C-45EC-92EE-68FD035CAD11}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A3EB4E60-256C-45EC-92EE-68FD035CAD11}.Release|Any CPU.Build.0 = Release|Any CPU {BE5FFBBB-D73F-4071-92F4-F1694881604F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {BE5FFBBB-D73F-4071-92F4-F1694881604F}.Debug|Any CPU.Build.0 = Debug|Any CPU {BE5FFBBB-D73F-4071-92F4-F1694881604F}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -398,6 +421,10 @@ Global {6AE92AAD-CF08-4E60-98EF-A7F762DAAB4D} = {2097345F-4DD3-477D-BC54-A922F9B2B402} {42B3FB71-BB42-46E3-9CEC-56620CB76BD9} = {22DF5DC0-1290-4E83-A9D8-6BB7DE3B3E63} {2117F4E3-6612-4E4D-A757-27271EEB7783} = {2097345F-4DD3-477D-BC54-A922F9B2B402} + {1105C814-31DA-4214-BEA8-6DB5FC12C808} = {22DF5DC0-1290-4E83-A9D8-6BB7DE3B3E63} + {F53FD7F5-DBC0-4FA5-83BA-B4C07A5BD248} = {2097345F-4DD3-477D-BC54-A922F9B2B402} + {F632DFB6-38AD-4356-8997-8CCC0492619C} = {2097345F-4DD3-477D-BC54-A922F9B2B402} + {A3EB4E60-256C-45EC-92EE-68FD035CAD11} = {2097345F-4DD3-477D-BC54-A922F9B2B402} {BE5FFBBB-D73F-4071-92F4-F1694881604F} = {22DF5DC0-1290-4E83-A9D8-6BB7DE3B3E63} {ED774FC3-C1C0-44CD-BA41-686C04BEB3E5} = {2097345F-4DD3-477D-BC54-A922F9B2B402} EndGlobalSection diff --git a/src/OpenTelemetry.Exporter.Geneva/AssemblyInfo.cs b/src/OpenTelemetry.Exporter.Geneva/AssemblyInfo.cs new file mode 100644 index 00000000000..afc852f07fc --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/AssemblyInfo.cs @@ -0,0 +1,16 @@ +using System; +using System.Runtime.CompilerServices; + +[assembly: CLSCompliant(false)] +[assembly: InternalsVisibleTo("OpenTelemetry.Exporter.Geneva.Benchmark" + AssemblyInfo.PublicKey)] +[assembly: InternalsVisibleTo("OpenTelemetry.Exporter.Geneva.UnitTest" + AssemblyInfo.PublicKey)] +[assembly: InternalsVisibleTo("OpenTelemetry.Exporter.Geneva.Stress" + AssemblyInfo.PublicKey)] + +internal static class AssemblyInfo +{ +#if SIGNED + public const string PublicKey = ", PublicKey=002400000480000094000000060200000024000052534131000400000100010051C1562A090FB0C9F391012A32198B5E5D9A60E9B80FA2D7B434C9E5CCB7259BD606E66F9660676AFC6692B8CDC6793D190904551D2103B7B22FA636DCBB8208839785BA402EA08FC00C8F1500CCEF28BBF599AA64FFB1E1D5DC1BF3420A3777BADFE697856E9D52070A50C3EA5821C80BEF17CA3ACFFA28F89DD413F096F898"; +#else + public const string PublicKey = ""; +#endif +} diff --git a/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md b/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md new file mode 100644 index 00000000000..1512c421622 --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/CHANGELOG.md @@ -0,0 +1,3 @@ +# Changelog + +## Unreleased diff --git a/src/OpenTelemetry.Exporter.Geneva/ConnectionStringBuilder.cs b/src/OpenTelemetry.Exporter.Geneva/ConnectionStringBuilder.cs new file mode 100644 index 00000000000..116294e3135 --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/ConnectionStringBuilder.cs @@ -0,0 +1,212 @@ +using System; +using System.Collections.Generic; +using System.Globalization; + +namespace OpenTelemetry.Exporter.Geneva +{ + internal enum TransportProtocol + { + Etw, + Tcp, + Udp, + Unix, + Unspecified, + } + + internal class ConnectionStringBuilder + { + private readonly Dictionary _parts = new Dictionary(StringComparer.Ordinal); + + public ConnectionStringBuilder(string connectionString) + { + if (string.IsNullOrWhiteSpace(connectionString)) + { + throw new ArgumentNullException(nameof(connectionString), $"{nameof(connectionString)} is invalid."); + } + + const char Semicolon = ';'; + const char EqualSign = '='; + foreach (var token in connectionString.Split(Semicolon)) + { + if (string.IsNullOrWhiteSpace(token)) + { + continue; + } + + var index = token.IndexOf(EqualSign); + if (index == -1 || index != token.LastIndexOf(EqualSign)) + { + continue; + } + + var pair = token.Trim().Split(EqualSign); + + var key = pair[0].Trim(); + var value = pair[1].Trim(); + if (string.IsNullOrEmpty(key) || string.IsNullOrEmpty(value)) + { + throw new ArgumentException("Connection string cannot contain empty keys or values."); + } + + this._parts[key] = value; + } + + if (this._parts.Count == 0) + { + throw new ArgumentNullException(nameof(connectionString), $"{nameof(connectionString)} is invalid."); + } + } + + public string EtwSession + { + get => this.ThrowIfNotExists(nameof(this.EtwSession)); + set => this._parts[nameof(this.EtwSession)] = value; + } + + public string Endpoint + { + get => this.ThrowIfNotExists(nameof(this.Endpoint)); + set => this._parts[nameof(this.Endpoint)] = value; + } + + public TransportProtocol Protocol + { + get + { + try + { + // Checking Etw first, since it's preferred for Windows and enables fail fast on Linux + if (this._parts.ContainsKey(nameof(this.EtwSession))) + { + return TransportProtocol.Etw; + } + + if (!this._parts.ContainsKey(nameof(this.Endpoint))) + { + return TransportProtocol.Unspecified; + } + + var endpoint = new Uri(this.Endpoint); + if (Enum.TryParse(endpoint.Scheme, true, out TransportProtocol protocol)) + { + return protocol; + } + + throw new ArgumentException("Endpoint scheme is invalid."); + } + catch (UriFormatException ex) + { + throw new ArgumentException($"{nameof(this.Endpoint)} value is malformed.", ex); + } + } + } + + public string ParseUnixDomainSocketPath() + { + try + { + var endpoint = new Uri(this.Endpoint); + return endpoint.AbsolutePath; + } + catch (UriFormatException ex) + { + throw new ArgumentException($"{nameof(this.Endpoint)} value is malformed.", ex); + } + } + + public int TimeoutMilliseconds + { + get + { + if (!this._parts.TryGetValue(nameof(this.TimeoutMilliseconds), out string value)) + { + return UnixDomainSocketDataTransport.DefaultTimeoutMilliseconds; + } + + try + { + int timeout = int.Parse(value, CultureInfo.InvariantCulture); + if (timeout <= 0) + { + throw new ArgumentException( + $"{nameof(this.TimeoutMilliseconds)} should be greater than zero.", + nameof(this.TimeoutMilliseconds)); + } + + return timeout; + } + catch (ArgumentException) + { + throw; + } + catch (Exception ex) + { + throw new ArgumentException( + $"{nameof(this.TimeoutMilliseconds)} is malformed.", + nameof(this.TimeoutMilliseconds), + ex); + } + } + set => this._parts[nameof(this.TimeoutMilliseconds)] = value.ToString(CultureInfo.InvariantCulture); + } + + public string Host + { + get + { + try + { + var endpoint = new Uri(this.Endpoint); + return endpoint.Host; + } + catch (UriFormatException ex) + { + throw new ArgumentException($"{nameof(this.Endpoint)} value is malformed.", ex); + } + } + } + + public int Port + { + get + { + try + { + var endpoint = new Uri(this.Endpoint); + if (endpoint.IsDefaultPort) + { + throw new ArgumentException($"Port should be explicitly set in {nameof(this.Endpoint)} value."); + } + + return endpoint.Port; + } + catch (UriFormatException ex) + { + throw new ArgumentException($"{nameof(this.Endpoint)} value is malformed.", ex); + } + } + } + + public string Account + { + get => this.ThrowIfNotExists(nameof(this.Account)); + set => this._parts[nameof(this.Account)] = value; + } + + public string Namespace + { + get => this.ThrowIfNotExists(nameof(this.Namespace)); + set => this._parts[nameof(this.Namespace)] = value; + } + + private T ThrowIfNotExists(string name) + { + if (!this._parts.TryGetValue(name, out var value)) + { + throw new ArgumentException($"'{name}' value is missing in connection string."); + } + + return (T)Convert.ChangeType(value, typeof(T), CultureInfo.InvariantCulture); + } + } +} diff --git a/src/OpenTelemetry.Exporter.Geneva/EtwDataTransport.cs b/src/OpenTelemetry.Exporter.Geneva/EtwDataTransport.cs new file mode 100644 index 00000000000..f72c7f3aac9 --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/EtwDataTransport.cs @@ -0,0 +1,77 @@ +using System; +using System.Diagnostics.Tracing; + +namespace OpenTelemetry.Exporter.Geneva +{ + [EventSource(Name = "OpenTelemetry")] + internal class EtwEventSource : EventSource + { + public EtwEventSource(string providerName) + : base(providerName, EventSourceSettings.EtwManifestEventFormat) + { + } + + public enum EtwEventId + { + TraceEvent = 100, + } + + [Event((int)EtwEventId.TraceEvent, Version = 1, Level = EventLevel.Informational)] + public void InformationalEvent() + { + } + + [NonEvent] + public unsafe void SendEvent(int eventId, byte[] data, int size) + { + EventData* dataDesc = stackalloc EventData[1]; + fixed (byte* ptr = data) + { + dataDesc[0].DataPointer = (IntPtr)ptr; + dataDesc[0].Size = (int)size; + this.WriteEventCore(eventId, 1, dataDesc); + } + } + } + + internal class EtwDataTransport : IDataTransport, IDisposable + { + public EtwDataTransport(string providerName) + { + this.m_eventSource = new EtwEventSource(providerName); + } + + public void Send(byte[] data, int size) + { + this.m_eventSource.SendEvent((int)EtwEventSource.EtwEventId.TraceEvent, data, size); + } + + public bool IsEnabled() + { + return this.m_eventSource.IsEnabled(); + } + + private EtwEventSource m_eventSource; + private bool m_disposed; + + public void Dispose() + { + this.Dispose(true); + } + + protected virtual void Dispose(bool disposing) + { + if (this.m_disposed) + { + return; + } + + if (disposing) + { + this.m_eventSource.Dispose(); + } + + this.m_disposed = true; + } + } +} diff --git a/src/OpenTelemetry.Exporter.Geneva/ExporterEventSource.cs b/src/OpenTelemetry.Exporter.Geneva/ExporterEventSource.cs new file mode 100644 index 00000000000..ef7d349c4bf --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/ExporterEventSource.cs @@ -0,0 +1,74 @@ +using System; +using System.Diagnostics.Tracing; +using System.Globalization; +using System.Threading; + +namespace OpenTelemetry.Exporter.Geneva +{ + [EventSource(Name = "OpenTelemetry-Exporter-Geneva")] + internal class ExporterEventSource : EventSource + { + public static readonly ExporterEventSource Log = new ExporterEventSource(); + private const int EVENT_ID_TRACE = 1; + private const int EVENT_ID_METRICS = 2; + private const int EVENT_ID_ERROR = 3; + + [NonEvent] + public void ExporterException(Exception ex) + { + if (Log.IsEnabled(EventLevel.Error, EventKeywords.All)) + { + this.FailedToSendSpanData(ToInvariantString(ex)); + } + } + + [Event(EVENT_ID_TRACE, Message = "Exporter failed to send SpanData. Data will not be sent. Exception: {0}", Level = EventLevel.Error)] + public void FailedToSendSpanData(string ex) + { + // https://docs.microsoft.com/en-us/windows/win32/etw/about-event-tracing + // ETW has a size limit: The total event size is greater than 64K. This includes the ETW header plus the data or payload. + // TODO: Do not hit ETW size limit even for external library exception stack. But what is the ETW header size? + // Source code: https://referencesource.microsoft.com/#mscorlib/system/diagnostics/eventing/eventsource.cs,867 + // Why is size calculated like below in WriteEvent source code? + // descrs[0].Size = ((arg1.Length + 1) * 2); + // I'm assuming it calculates the size of string, then it should be: + // (count of chars) * sizeof(char) + sizeof(Length:int) = (str.Length * 2 + 4). + this.WriteEvent(EVENT_ID_TRACE, ex); + } + + [Event(EVENT_ID_METRICS, Message = "Exporter failed to send MetricData. Data will not be sent. MetricNamespace = {0}, MetricName = {1}, Message: {2}", Level = EventLevel.Error)] + public void FailedToSendMetricData(string metricNamespace, string metricName, string message) + { + this.WriteEvent(EVENT_ID_METRICS, metricNamespace, metricName, message); + } + + [Event(EVENT_ID_ERROR, Message = "Exporter failed.", Level = EventLevel.Error)] + public void ExporterError(string message) + { + if (Log.IsEnabled(EventLevel.Error, EventKeywords.All)) + { + // TODO: We should ensure that GenevaTraceExporter doesn't emit any message that could hit ETW size limit. + this.WriteEvent(EVENT_ID_ERROR, message); + } + } + + /// + /// Returns a culture-independent string representation of the given object, + /// appropriate for diagnostics tracing. + /// + private static string ToInvariantString(Exception exception) + { + var originalUICulture = Thread.CurrentThread.CurrentUICulture; + + try + { + Thread.CurrentThread.CurrentUICulture = CultureInfo.InvariantCulture; + return exception.ToString(); + } + finally + { + Thread.CurrentThread.CurrentUICulture = originalUICulture; + } + } + } +} diff --git a/src/OpenTelemetry.Exporter.Geneva/GenevaBaseExporter.cs b/src/OpenTelemetry.Exporter.Geneva/GenevaBaseExporter.cs new file mode 100644 index 00000000000..2e3033e634e --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/GenevaBaseExporter.cs @@ -0,0 +1,77 @@ +using System.Collections.Generic; + +namespace OpenTelemetry.Exporter.Geneva +{ + public abstract class GenevaBaseExporter : BaseExporter + where T : class + { + internal static readonly IReadOnlyDictionary V21_PART_A_MAPPING = new Dictionary + { + // Part A + [Schema.V21.PartA.IKey] = "env_iKey", + [Schema.V21.PartA.Name] = "env_name", + [Schema.V21.PartA.Ver] = "env_ver", + [Schema.V21.PartA.Time] = "env_time", + [Schema.V21.PartA.Cv] = "env_cv", + [Schema.V21.PartA.Epoch] = "env_epoch", + [Schema.V21.PartA.Flags] = "env_flags", + [Schema.V21.PartA.PopSample] = "env_popSample", + [Schema.V21.PartA.SeqNum] = "env_seqNum", + + // Part A Application extension + [Schema.V21.PartA.Extensions.App.Id] = "env_appId", + [Schema.V21.PartA.Extensions.App.Ver] = "env_appVer", + + // Part A Cloud extension + [Schema.V21.PartA.Extensions.Cloud.Environment] = "env_cloud_environment", + [Schema.V21.PartA.Extensions.Cloud.Location] = "env_cloud_location", + [Schema.V21.PartA.Extensions.Cloud.Name] = "env_cloud_name", + [Schema.V21.PartA.Extensions.Cloud.DeploymentUnit] = "env_cloud_deploymentUnit", + [Schema.V21.PartA.Extensions.Cloud.Role] = "env_cloud_role", + [Schema.V21.PartA.Extensions.Cloud.RoleInstance] = "env_cloud_roleInstance", + [Schema.V21.PartA.Extensions.Cloud.RoleVer] = "env_cloud_roleVer", + [Schema.V21.PartA.Extensions.Cloud.Ver] = "env_cloud_ver", + + // Part A Os extension + [Schema.V21.PartA.Extensions.Os.Name] = "env_os", + [Schema.V21.PartA.Extensions.Os.Ver] = "env_osVer", + }; + + internal static readonly IReadOnlyDictionary V40_PART_A_MAPPING = new Dictionary + { + // Part A + [Schema.V40.PartA.IKey] = "env_iKey", + [Schema.V40.PartA.Name] = "env_name", + [Schema.V40.PartA.Ver] = "env_ver", + [Schema.V40.PartA.Time] = "env_time", + + // Part A Application Extension + [Schema.V40.PartA.Extensions.App.Id] = "env_app_id", + [Schema.V40.PartA.Extensions.App.Ver] = "env_app_ver", + + // Part A Cloud Extension + [Schema.V40.PartA.Extensions.Cloud.Role] = "env_cloud_role", + [Schema.V40.PartA.Extensions.Cloud.RoleInstance] = "env_cloud_roleInstance", + [Schema.V40.PartA.Extensions.Cloud.RoleVer] = "env_cloud_roleVer", + + // Part A Os extension + [Schema.V40.PartA.Extensions.Os.Name] = "env_os_name", + [Schema.V40.PartA.Extensions.Os.Ver] = "env_os_ver", + }; + + internal static int AddPartAField(byte[] buffer, int cursor, string name, object value) + { + if (V40_PART_A_MAPPING.TryGetValue(name, out string replacementKey)) + { + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, replacementKey); + } + else + { + cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, name); + } + + cursor = MessagePackSerializer.Serialize(buffer, cursor, value); + return cursor; + } + } +} diff --git a/src/OpenTelemetry.Exporter.Geneva/GenevaExporterHelperExtensions.cs b/src/OpenTelemetry.Exporter.Geneva/GenevaExporterHelperExtensions.cs new file mode 100644 index 00000000000..88d21c564c8 --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/GenevaExporterHelperExtensions.cs @@ -0,0 +1,41 @@ +using System; +using System.Diagnostics; +using OpenTelemetry.Trace; + +namespace OpenTelemetry.Exporter.Geneva +{ + public static class GenevaExporterHelperExtensions + { + public static TracerProviderBuilder AddGenevaTraceExporter(this TracerProviderBuilder builder, Action configure) + { + if (builder == null) + { + throw new ArgumentNullException(nameof(builder)); + } + + if (builder is IDeferredTracerProviderBuilder deferredTracerProviderBuilder) + { + return deferredTracerProviderBuilder.Configure((sp, builder) => + { + AddGenevaTraceExporter(builder, sp.GetOptions(), configure); + }); + } + + return AddGenevaTraceExporter(builder, new GenevaExporterOptions(), configure); + } + + private static TracerProviderBuilder AddGenevaTraceExporter(this TracerProviderBuilder builder, GenevaExporterOptions options, Action configure) + { + configure?.Invoke(options); + var exporter = new GenevaTraceExporter(options); + if (exporter.IsUsingUnixDomainSocket) + { + return builder.AddProcessor(new BatchActivityExportProcessor(exporter)); + } + else + { + return builder.AddProcessor(new ReentrantExportProcessor(exporter)); + } + } + } +} diff --git a/src/OpenTelemetry.Exporter.Geneva/GenevaExporterOptions.cs b/src/OpenTelemetry.Exporter.Geneva/GenevaExporterOptions.cs new file mode 100644 index 00000000000..b3bdb807819 --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/GenevaExporterOptions.cs @@ -0,0 +1,63 @@ +using System; +using System.Collections.Generic; + +namespace OpenTelemetry.Exporter.Geneva +{ + public class GenevaExporterOptions + { + private IReadOnlyDictionary _fields = new Dictionary(1) + { + [Schema.V40.PartA.Ver] = "4.0", + }; + + public string ConnectionString { get; set; } + + public IEnumerable CustomFields { get; set; } + + public IReadOnlyDictionary TableNameMappings { get; set; } + + public IReadOnlyDictionary PrepopulatedFields + { + get => this._fields; + set + { + if (value == null) + { + throw new ArgumentNullException(nameof(value)); + } + + var schemaVersion = "4.0"; + + if (value.ContainsKey(Schema.V40.PartA.Ver)) + { + schemaVersion = value[Schema.V40.PartA.Ver] as string; + } + + if (schemaVersion != "2.1" && schemaVersion != "4.0") + { + throw new ArgumentException("Unsupported schema version, only 2.1 and 4.0 are supported."); + } + + if (value.ContainsKey(Schema.V40.PartA.Name)) + { + throw new ArgumentException("Event name cannot be pre-populated."); + } + + if (value.ContainsKey(Schema.V40.PartA.Time)) + { + throw new ArgumentException("Event timestamp cannot be pre-populated."); + } + + var copy = new Dictionary(value.Count + 1) { [Schema.V40.PartA.Ver] = schemaVersion }; + foreach (var entry in value) + { + copy[entry.Key] = entry.Value; // shallow copy + } + + this._fields = copy; + } + } + + internal Func ConvertToJson = obj => "ERROR: GenevaExporterOptions.ConvertToJson not configured."; + } +} diff --git a/src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs b/src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs new file mode 100644 index 00000000000..886a852438d --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/GenevaLogExporter.cs @@ -0,0 +1,450 @@ +#if NETSTANDARD2_0 || NET461 +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text; +using System.Threading; +using Microsoft.Extensions.Logging; +using OpenTelemetry.Logs; + +namespace OpenTelemetry.Exporter.Geneva +{ + public class GenevaLogExporter : GenevaBaseExporter + { + private const int BUFFER_SIZE = 65360; // the maximum ETW payload (inclusive) + + private readonly IReadOnlyDictionary m_customFields; + private readonly string m_defaultEventName = "Log"; + private readonly IReadOnlyDictionary m_prepopulatedFields; + private readonly List m_prepopulatedFieldKeys; + private static readonly ThreadLocal m_buffer = new ThreadLocal(() => null); + private readonly byte[] m_bufferEpilogue; + private static readonly string[] logLevels = new string[7] + { + "Trace", "Debug", "Information", "Warning", "Error", "Critical", "None", + }; + + private readonly IDataTransport m_dataTransport; + private bool isDisposed; + private Func convertToJson; + + public GenevaLogExporter(GenevaExporterOptions options) + { + if (options == null) + { + throw new ArgumentNullException(nameof(options)); + } + + if (string.IsNullOrWhiteSpace(options.ConnectionString)) + { + throw new ArgumentException($"{nameof(options.ConnectionString)} is invalid."); + } + + // TODO: Validate mappings for reserved tablenames etc. + if (options.TableNameMappings != null) + { + var tempTableMappings = new Dictionary(options.TableNameMappings.Count, StringComparer.Ordinal); + foreach (var kv in options.TableNameMappings) + { + if (Encoding.UTF8.GetByteCount(kv.Value) != kv.Value.Length) + { + throw new ArgumentException("The value: \"{tableName}\" provided for TableNameMappings option contains non-ASCII characters", kv.Value); + } + + if (kv.Key == "*") + { + this.m_defaultEventName = kv.Value; + } + else + { + tempTableMappings[kv.Key] = kv.Value; + } + } + + this.m_tableMappings = tempTableMappings; + } + + var connectionStringBuilder = new ConnectionStringBuilder(options.ConnectionString); + switch (connectionStringBuilder.Protocol) + { + case TransportProtocol.Etw: + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + throw new ArgumentException("ETW cannot be used on non-Windows operating systems."); + } + + this.m_dataTransport = new EtwDataTransport(connectionStringBuilder.EtwSession); + break; + case TransportProtocol.Unix: + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + throw new ArgumentException("Unix domain socket should not be used on Windows."); + } + + var unixDomainSocketPath = connectionStringBuilder.ParseUnixDomainSocketPath(); + this.m_dataTransport = new UnixDomainSocketDataTransport(unixDomainSocketPath); + break; + case TransportProtocol.Tcp: + throw new ArgumentException("TCP transport is not supported yet."); + case TransportProtocol.Udp: + throw new ArgumentException("UDP transport is not supported yet."); + default: + throw new ArgumentOutOfRangeException(nameof(connectionStringBuilder.Protocol)); + } + + this.convertToJson = options.ConvertToJson; + + if (options.PrepopulatedFields != null) + { + this.m_prepopulatedFieldKeys = new List(); + var tempPrepopulatedFields = new Dictionary(options.PrepopulatedFields.Count, StringComparer.Ordinal); + foreach (var kv in options.PrepopulatedFields) + { + tempPrepopulatedFields[kv.Key] = kv.Value; + this.m_prepopulatedFieldKeys.Add(kv.Key); + } + + this.m_prepopulatedFields = tempPrepopulatedFields; + } + + // TODO: Validate custom fields (reserved name? etc). + if (options.CustomFields != null) + { + var customFields = new Dictionary(StringComparer.Ordinal); + foreach (var name in options.CustomFields) + { + customFields[name] = true; + } + + this.m_customFields = customFields; + } + + var buffer = new byte[BUFFER_SIZE]; + var cursor = MessagePackSerializer.Serialize(buffer, 0, new Dictionary { { "TimeFormat", "DateTime" } }); + this.m_bufferEpilogue = new byte[cursor - 0]; + Buffer.BlockCopy(buffer, 0, this.m_bufferEpilogue, 0, cursor - 0); + } + + private readonly IReadOnlyDictionary m_tableMappings; + + public override ExportResult Export(in Batch batch) + { + var result = ExportResult.Success; + foreach (var logRecord in batch) + { + try + { + var cursor = this.SerializeLogRecord(logRecord); + this.m_dataTransport.Send(m_buffer.Value, cursor - 0); + } + catch (Exception ex) + { + ExporterEventSource.Log.ExporterException(ex); // TODO: preallocate exception or no exception + result = ExportResult.Failure; + } + } + + return result; + } + + protected override void Dispose(bool disposing) + { + if (this.isDisposed) + { + return; + } + + if (disposing) + { + // DO NOT Dispose m_buffer as it is a static type + try + { + (this.m_dataTransport as IDisposable)?.Dispose(); + this.m_prepopulatedFieldKeys.Clear(); + } + catch (Exception ex) + { + ExporterEventSource.Log.ExporterException(ex); + } + } + + this.isDisposed = true; + base.Dispose(disposing); + } + + internal bool IsUsingUnixDomainSocket + { + get => this.m_dataTransport is UnixDomainSocketDataTransport; + } + + internal int SerializeLogRecord(LogRecord logRecord) + { + bool isUnstructuredLog = true; + IReadOnlyList> listKvp; + if (logRecord.State == null) + { + listKvp = logRecord.StateValues; + } + else + { + listKvp = logRecord.State as IReadOnlyList>; + } + + if (listKvp != null) + { + isUnstructuredLog = listKvp.Count == 1; + } + + var name = logRecord.CategoryName; + + // If user configured explicit TableName, use it. + if (this.m_tableMappings == null || !this.m_tableMappings.TryGetValue(name, out var eventName)) + { + eventName = this.m_defaultEventName; + } + + var buffer = m_buffer.Value; + if (buffer == null) + { + buffer = new byte[BUFFER_SIZE]; // TODO: handle OOM + m_buffer.Value = buffer; + } + + /* Fluentd Forward Mode: + [ + "Log", + [ + [ , { "env_ver": "4.0", ... } ] + ], + { "TimeFormat": "DateTime" } + ] + */ + + // Structured log. + // 2 scenarios. + // 1. Structured logging with template + // eg: + // body + // "Hello from {food} {price}." + // part c + // food = onion + // price = 100 + // TODO: 2. Structured with strongly typed logging. + var timestamp = logRecord.Timestamp; + var cursor = 0; + cursor = MessagePackSerializer.WriteArrayHeader(buffer, cursor, 3); + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, eventName); + cursor = MessagePackSerializer.WriteArrayHeader(buffer, cursor, 1); + cursor = MessagePackSerializer.WriteArrayHeader(buffer, cursor, 2); + cursor = MessagePackSerializer.SerializeUtcDateTime(buffer, cursor, timestamp); + cursor = MessagePackSerializer.WriteMapHeader(buffer, cursor, ushort.MaxValue); // Note: always use Map16 for perf consideration + ushort cntFields = 0; + var idxMapSizePatch = cursor - 2; + + if (this.m_prepopulatedFieldKeys != null) + { + for (int i = 0; i < this.m_prepopulatedFieldKeys.Count; i++) + { + var key = this.m_prepopulatedFieldKeys[i]; + var value = this.m_prepopulatedFields[key]; + switch (value) + { + case bool vb: + case byte vui8: + case sbyte vi8: + case short vi16: + case ushort vui16: + case int vi32: + case uint vui32: + case long vi64: + case ulong vui64: + case float vf: + case double vd: + case string vs: + break; + default: + value = this.convertToJson(value); + break; + } + + cursor = AddPartAField(buffer, cursor, key, value); + cntFields += 1; + } + } + + // Part A - core envelope + cursor = AddPartAField(buffer, cursor, Schema.V40.PartA.Name, eventName); + cntFields += 1; + + cursor = AddPartAField(buffer, cursor, Schema.V40.PartA.Time, timestamp); + cntFields += 1; + + // Part A - dt extension + if (logRecord.TraceId != default) + { + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "env_dt_traceId"); + + // Note: ToHexString returns the pre-calculated hex representation without allocation + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, logRecord.TraceId.ToHexString()); + cntFields += 1; + } + + if (logRecord.SpanId != default) + { + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "env_dt_spanId"); + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, logRecord.SpanId.ToHexString()); + cntFields += 1; + } + + // Part A - ex extension + if (logRecord.Exception != null) + { + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "env_ex_type"); + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, logRecord.Exception.GetType().FullName); + cntFields += 1; + + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "env_ex_msg"); + cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, logRecord.Exception.Message); + cntFields += 1; + } + + // Part B + var logLevel = logRecord.LogLevel; + + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "severityText"); + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, logLevels[(int)logLevel]); + cntFields += 1; + + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "severityNumber"); + cursor = MessagePackSerializer.SerializeUInt8(buffer, cursor, GetSeverityNumber(logLevel)); + cntFields += 1; + + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "name"); + cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, name); + cntFields += 1; + + if (isUnstructuredLog) + { + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "body"); + cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, logRecord.FormattedMessage ?? (listKvp != null ? Convert.ToString(listKvp[0].Value, CultureInfo.InvariantCulture) : Convert.ToString(logRecord.State, CultureInfo.InvariantCulture))); + cntFields += 1; + } + else + { + bool hasEnvProperties = false; + bool bodyPopulated = false; + for (int i = 0; i < listKvp.Count; i++) + { + var entry = listKvp[i]; + + // Iteration #1 - Get those fields which become dedicated column + // i.e all PartB fields and opt-in part c fields. + if (entry.Key == "{OriginalFormat}") + { + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "body"); + cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, logRecord.FormattedMessage ?? Convert.ToString(entry.Value, CultureInfo.InvariantCulture)); + cntFields += 1; + bodyPopulated = true; + continue; + } + else if (this.m_customFields == null || this.m_customFields.ContainsKey(entry.Key)) + { + // TODO: the above null check can be optimized and avoided inside foreach. + if (entry.Value != null) + { + // Geneva doesn't support null. + cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, entry.Key); + cursor = MessagePackSerializer.Serialize(buffer, cursor, entry.Value); + cntFields += 1; + } + } + else + { + hasEnvProperties = true; + continue; + } + } + + if (!bodyPopulated && logRecord.FormattedMessage != null) + { + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "body"); + cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, logRecord.FormattedMessage); + cntFields += 1; + } + + if (hasEnvProperties) + { + // Iteration #2 - Get all "other" fields and collapse them into single field + // named "env_properties". + ushort envPropertiesCount = 0; + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "env_properties"); + cursor = MessagePackSerializer.WriteMapHeader(buffer, cursor, ushort.MaxValue); + int idxMapSizeEnvPropertiesPatch = cursor - 2; + for (int i = 0; i < listKvp.Count; i++) + { + var entry = listKvp[i]; + if (entry.Key == "{OriginalFormat}" || this.m_customFields.ContainsKey(entry.Key)) + { + continue; + } + else + { + cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, entry.Key); + cursor = MessagePackSerializer.Serialize(buffer, cursor, entry.Value); + envPropertiesCount += 1; + } + } + + cntFields += 1; + MessagePackSerializer.WriteUInt16(buffer, idxMapSizeEnvPropertiesPatch, envPropertiesCount); + } + } + + var eventId = logRecord.EventId; + if (eventId != default) + { + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "eventId"); + cursor = MessagePackSerializer.SerializeInt32(buffer, cursor, eventId.Id); + cntFields += 1; + } + + MessagePackSerializer.WriteUInt16(buffer, idxMapSizePatch, cntFields); + Buffer.BlockCopy(this.m_bufferEpilogue, 0, buffer, cursor, this.m_bufferEpilogue.Length); + cursor += this.m_bufferEpilogue.Length; + return cursor; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static byte GetSeverityNumber(LogLevel logLevel) + { + // Maps the Ilogger LogLevel to OpenTelemetry logging level. + // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#mapping-of-severitynumber + // TODO: for improving perf simply do ((int)loglevel * 4) + 1 + // or ((int)logLevel << 2) + 1 + switch (logLevel) + { + case LogLevel.Trace: + return 1; + case LogLevel.Debug: + return 5; + case LogLevel.Information: + return 9; + case LogLevel.Warning: + return 13; + case LogLevel.Error: + return 17; + case LogLevel.Critical: + return 21; + + // we reach default only for LogLevel.None + // but that is filtered out anyway. + // should we throw here then? + default: + return 1; + } + } + } +} +#endif diff --git a/src/OpenTelemetry.Exporter.Geneva/GenevaLoggingExtensions.cs b/src/OpenTelemetry.Exporter.Geneva/GenevaLoggingExtensions.cs new file mode 100644 index 00000000000..f0c7ee945b2 --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/GenevaLoggingExtensions.cs @@ -0,0 +1,32 @@ +#if NETSTANDARD2_0 || NET461 +using System; +using OpenTelemetry; +using OpenTelemetry.Exporter.Geneva; +using OpenTelemetry.Logs; + +namespace Microsoft.Extensions.Logging +{ + public static class GenevaLoggingExtensions + { + public static OpenTelemetryLoggerOptions AddGenevaLogExporter(this OpenTelemetryLoggerOptions options, Action configure) + { + if (options == null) + { + throw new ArgumentNullException(nameof(options)); + } + + var genevaOptions = new GenevaExporterOptions(); + configure?.Invoke(genevaOptions); + var exporter = new GenevaLogExporter(genevaOptions); + if (exporter.IsUsingUnixDomainSocket) + { + return options.AddProcessor(new BatchLogRecordExportProcessor(exporter)); + } + else + { + return options.AddProcessor(new ReentrantExportProcessor(exporter)); + } + } + } +} +#endif diff --git a/src/OpenTelemetry.Exporter.Geneva/GenevaMetricExporter.cs b/src/OpenTelemetry.Exporter.Geneva/GenevaMetricExporter.cs new file mode 100644 index 00000000000..09d8dc22434 --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/GenevaMetricExporter.cs @@ -0,0 +1,489 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Runtime.InteropServices; +using System.Text; +using OpenTelemetry.Metrics; + +namespace OpenTelemetry.Exporter.Geneva +{ + [AggregationTemporality(AggregationTemporality.Delta)] + public class GenevaMetricExporter : BaseExporter + { + private const int BufferSize = 65360; // the maximum ETW payload (inclusive) + + internal const int MaxDimensionNameSize = 256; + + internal const int MaxDimensionValueSize = 1024; + + private readonly ushort prepopulatedDimensionsCount; + + private readonly int fixedPayloadStartIndex; + + private readonly string monitoringAccount; + + private readonly string metricNamespace; + + private readonly IMetricDataTransport metricDataTransport; + + private readonly List serializedPrepopulatedDimensionsKeys; + + private readonly List serializedPrepopulatedDimensionsValues; + + private readonly byte[] bufferForNonHistogramMetrics = new byte[BufferSize]; + + private readonly byte[] bufferForHistogramMetrics = new byte[BufferSize]; + + private readonly int bufferIndexForNonHistogramMetrics; + + private readonly int bufferIndexForHistogramMetrics; + + private static readonly MetricData ulongZero = new MetricData { UInt64Value = 0 }; + + private bool isDisposed; + + public GenevaMetricExporter(GenevaMetricExporterOptions options) + { + if (options == null) + { + throw new ArgumentNullException(nameof(options)); + } + + if (string.IsNullOrWhiteSpace(options.ConnectionString)) + { + throw new ArgumentException($"{nameof(options.ConnectionString)} is invalid."); + } + + var connectionStringBuilder = new ConnectionStringBuilder(options.ConnectionString); + this.monitoringAccount = connectionStringBuilder.Account; + this.metricNamespace = connectionStringBuilder.Namespace; + + if (options.PrepopulatedMetricDimensions != null) + { + this.prepopulatedDimensionsCount = (ushort)options.PrepopulatedMetricDimensions.Count; + this.serializedPrepopulatedDimensionsKeys = this.SerializePrepopulatedDimensionsKeys(options.PrepopulatedMetricDimensions.Keys); + this.serializedPrepopulatedDimensionsValues = this.SerializePrepopulatedDimensionsValues(options.PrepopulatedMetricDimensions.Values); + } + + switch (connectionStringBuilder.Protocol) + { + case TransportProtocol.Unix: + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + throw new ArgumentException("Unix domain socket should not be used on Windows."); + } + + var unixDomainSocketPath = connectionStringBuilder.ParseUnixDomainSocketPath(); + this.metricDataTransport = new MetricUnixDataTransport(unixDomainSocketPath); + break; + case TransportProtocol.Tcp: + throw new ArgumentException("TCP transport is not supported yet."); + case TransportProtocol.Udp: + throw new ArgumentException("UDP transport is not supported yet."); + case TransportProtocol.Unspecified: + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + this.metricDataTransport = new MetricEtwDataTransport(); + break; + } + else + { + throw new ArgumentException("Endpoint not specified"); + } + + default: + throw new ArgumentOutOfRangeException(nameof(connectionStringBuilder.Protocol)); + } + + this.bufferIndexForNonHistogramMetrics = this.InitializeBufferForNonHistogramMetrics(); + this.bufferIndexForHistogramMetrics = this.InitializeBufferForHistogramMetrics(); + + unsafe + { + this.fixedPayloadStartIndex = sizeof(BinaryHeader); + } + } + + public override ExportResult Export(in Batch batch) + { + var result = ExportResult.Success; + foreach (var metric in batch) + { + foreach (ref readonly var metricPoint in metric.GetMetricPoints()) + { + try + { + switch (metric.MetricType) + { + case MetricType.LongSum: + { + var ulongSum = Convert.ToUInt64(metricPoint.GetSumLong()); + var metricData = new MetricData { UInt64Value = ulongSum }; + var bodyLength = this.SerializeMetric( + MetricEventType.ULongMetric, + metric.Name, + metricPoint.EndTime.ToFileTime(), // Using the endTime here as the timestamp as Geneva Metrics only allows for one field for timestamp + metricPoint.Tags, + metricData); + this.metricDataTransport.Send(MetricEventType.ULongMetric, this.bufferForNonHistogramMetrics, bodyLength); + break; + } + + case MetricType.LongGauge: + { + var ulongSum = Convert.ToUInt64(metricPoint.GetGaugeLastValueLong()); + var metricData = new MetricData { UInt64Value = ulongSum }; + var bodyLength = this.SerializeMetric( + MetricEventType.ULongMetric, + metric.Name, + metricPoint.EndTime.ToFileTime(), + metricPoint.Tags, + metricData); + this.metricDataTransport.Send(MetricEventType.ULongMetric, this.bufferForNonHistogramMetrics, bodyLength); + break; + } + + case MetricType.DoubleSum: + { + var doubleSum = metricPoint.GetSumDouble(); + var metricData = new MetricData { DoubleValue = doubleSum }; + var bodyLength = this.SerializeMetric( + MetricEventType.DoubleMetric, + metric.Name, + metricPoint.EndTime.ToFileTime(), + metricPoint.Tags, + metricData); + this.metricDataTransport.Send(MetricEventType.DoubleMetric, this.bufferForNonHistogramMetrics, bodyLength); + break; + } + + case MetricType.DoubleGauge: + { + var doubleSum = metricPoint.GetGaugeLastValueDouble(); + var metricData = new MetricData { DoubleValue = doubleSum }; + var bodyLength = this.SerializeMetric( + MetricEventType.DoubleMetric, + metric.Name, + metricPoint.EndTime.ToFileTime(), + metricPoint.Tags, + metricData); + this.metricDataTransport.Send(MetricEventType.DoubleMetric, this.bufferForNonHistogramMetrics, bodyLength); + break; + } + + case MetricType.Histogram: + { + var sum = new MetricData { UInt64Value = Convert.ToUInt64(metricPoint.GetHistogramSum()) }; + var count = Convert.ToUInt32(metricPoint.GetHistogramCount()); + var bodyLength = this.SerializeHistogramMetric( + metric.Name, + metricPoint.EndTime.ToFileTime(), + metricPoint.Tags, + metricPoint.GetHistogramBuckets(), + sum, + count); + this.metricDataTransport.Send(MetricEventType.ExternallyAggregatedULongDistributionMetric, this.bufferForHistogramMetrics, bodyLength); + break; + } + } + } + catch (Exception ex) + { + ExporterEventSource.Log.FailedToSendMetricData(this.metricNamespace, metric.Name, ex.Message); // TODO: preallocate exception or no exception + result = ExportResult.Failure; + } + } + } + + return result; + } + + protected override void Dispose(bool disposing) + { + if (this.isDisposed) + { + return; + } + + if (disposing) + { + try + { + this.metricDataTransport?.Dispose(); + } + catch (Exception ex) + { + ExporterEventSource.Log.ExporterException(ex); + } + } + + this.isDisposed = true; + base.Dispose(disposing); + } + + internal unsafe ushort SerializeMetric( + MetricEventType eventType, + string metricName, + long timestamp, + in ReadOnlyTagCollection tags, + MetricData value) + { + ushort bodyLength; + try + { + var bufferIndex = this.bufferIndexForNonHistogramMetrics; + MetricSerializer.SerializeString(this.bufferForNonHistogramMetrics, ref bufferIndex, metricName); + + ushort dimensionsWritten = 0; + + // Serialize PrepopulatedDimensions keys + for (ushort i = 0; i < this.prepopulatedDimensionsCount; i++) + { + MetricSerializer.SerializeEncodedString(this.bufferForNonHistogramMetrics, ref bufferIndex, this.serializedPrepopulatedDimensionsKeys[i]); + } + + if (this.prepopulatedDimensionsCount > 0) + { + dimensionsWritten += this.prepopulatedDimensionsCount; + } + + // Serialize MetricPoint Dimension keys + foreach (var tag in tags) + { + if (tag.Key.Length > MaxDimensionNameSize) + { + // TODO: Data Validation + } + + MetricSerializer.SerializeString(this.bufferForNonHistogramMetrics, ref bufferIndex, tag.Key); + } + + dimensionsWritten += (ushort)tags.Count; + + // Serialize PrepopulatedDimensions values + for (ushort i = 0; i < this.prepopulatedDimensionsCount; i++) + { + MetricSerializer.SerializeEncodedString(this.bufferForNonHistogramMetrics, ref bufferIndex, this.serializedPrepopulatedDimensionsValues[i]); + } + + // Serialize MetricPoint Dimension values + foreach (var tag in tags) + { + var dimensionValue = Convert.ToString(tag.Value, CultureInfo.InvariantCulture); + if (dimensionValue.Length > MaxDimensionValueSize) + { + // TODO: Data Validation + } + + MetricSerializer.SerializeString(this.bufferForNonHistogramMetrics, ref bufferIndex, dimensionValue); + } + + // The Autopilot container name is optional but still preserve the location with zero length if it is empty. + MetricSerializer.SerializeInt16(this.bufferForNonHistogramMetrics, ref bufferIndex, 0); + + // Write the final size of the payload + bodyLength = (ushort)(bufferIndex - this.fixedPayloadStartIndex); + + // Copy in the final structures to the front + fixed (byte* bufferBytes = this.bufferForNonHistogramMetrics) + { + var ptr = (BinaryHeader*)bufferBytes; + ptr->EventId = (ushort)eventType; + ptr->BodyLength = bodyLength; + + var payloadPtr = (MetricPayload*)&bufferBytes[this.fixedPayloadStartIndex]; + payloadPtr->CountDimension = dimensionsWritten; + payloadPtr->ReservedWord = 0; + payloadPtr->ReservedDword = 0; + payloadPtr->TimestampUtc = (ulong)timestamp; + payloadPtr->Data = value; + } + } + finally + { + } + + return bodyLength; + } + + internal unsafe ushort SerializeHistogramMetric( + string metricName, + long timestamp, + in ReadOnlyTagCollection tags, + HistogramBuckets buckets, + MetricData sum, + uint count) + { + ushort bodyLength; + try + { + var bufferIndex = this.bufferIndexForHistogramMetrics; + MetricSerializer.SerializeString(this.bufferForHistogramMetrics, ref bufferIndex, metricName); + + ushort dimensionsWritten = 0; + + // Serialize PrepopulatedDimensions keys + for (ushort i = 0; i < this.prepopulatedDimensionsCount; i++) + { + MetricSerializer.SerializeEncodedString(this.bufferForHistogramMetrics, ref bufferIndex, this.serializedPrepopulatedDimensionsKeys[i]); + } + + if (this.prepopulatedDimensionsCount > 0) + { + dimensionsWritten += this.prepopulatedDimensionsCount; + } + + // Serialize MetricPoint Dimension keys + foreach (var tag in tags) + { + if (tag.Key.Length > MaxDimensionNameSize) + { + // TODO: Data Validation + } + + MetricSerializer.SerializeString(this.bufferForHistogramMetrics, ref bufferIndex, tag.Key); + } + + dimensionsWritten += (ushort)tags.Count; + + // Serialize PrepopulatedDimensions values + for (ushort i = 0; i < this.prepopulatedDimensionsCount; i++) + { + MetricSerializer.SerializeEncodedString(this.bufferForHistogramMetrics, ref bufferIndex, this.serializedPrepopulatedDimensionsValues[i]); + } + + // Serialize MetricPoint Dimension values + foreach (var tag in tags) + { + var dimensionValue = Convert.ToString(tag.Value, CultureInfo.InvariantCulture); + if (dimensionValue.Length > MaxDimensionValueSize) + { + // TODO: Data Validation + } + + MetricSerializer.SerializeString(this.bufferForHistogramMetrics, ref bufferIndex, dimensionValue); + } + + // The Autopilot container name is optional but still preserve the location with zero length if it is empty. + MetricSerializer.SerializeInt16(this.bufferForHistogramMetrics, ref bufferIndex, 0); + + // Version + MetricSerializer.SerializeByte(this.bufferForHistogramMetrics, ref bufferIndex, 0); + + // Meta-data + // Value-count pairs is associated with the constant value of 2 in the distribution_type enum. + MetricSerializer.SerializeByte(this.bufferForHistogramMetrics, ref bufferIndex, 2); + + // Keep a position to record how many buckets are added + var itemsWrittenIndex = bufferIndex; + MetricSerializer.SerializeUInt16(this.bufferForHistogramMetrics, ref bufferIndex, 0); + + // Bucket values + ushort bucketCount = 0; + double lastExplicitBound = default; + foreach (var bucket in buckets) + { + if (bucket.BucketCount > 0) + { + if (bucket.ExplicitBound != double.PositiveInfinity) + { + MetricSerializer.SerializeUInt64(this.bufferForHistogramMetrics, ref bufferIndex, Convert.ToUInt64(bucket.ExplicitBound)); + lastExplicitBound = bucket.ExplicitBound; + } + else + { + // The bucket to catch the overflows is one greater than the last bound provided + MetricSerializer.SerializeUInt64(this.bufferForHistogramMetrics, ref bufferIndex, Convert.ToUInt64(lastExplicitBound + 1)); + } + + MetricSerializer.SerializeUInt32(this.bufferForHistogramMetrics, ref bufferIndex, Convert.ToUInt32(bucket.BucketCount)); + bucketCount++; + } + } + + // Write the number of items in distribution emitted and reset back to end. + MetricSerializer.SerializeUInt16(this.bufferForHistogramMetrics, ref itemsWrittenIndex, bucketCount); + + // Write the final size of the payload + bodyLength = (ushort)(bufferIndex - this.fixedPayloadStartIndex); + + // Copy in the final structures to the front + fixed (byte* bufferBytes = this.bufferForHistogramMetrics) + { + var ptr = (BinaryHeader*)bufferBytes; + ptr->EventId = (ushort)MetricEventType.ExternallyAggregatedULongDistributionMetric; + ptr->BodyLength = bodyLength; + + var payloadPtr = (ExternalPayload*)&bufferBytes[this.fixedPayloadStartIndex]; + payloadPtr[0].CountDimension = dimensionsWritten; + payloadPtr[0].ReservedWord = 0; + payloadPtr[0].Count = count; + payloadPtr[0].TimestampUtc = (ulong)timestamp; + payloadPtr[0].Sum = sum; + payloadPtr[0].Min = ulongZero; + payloadPtr[0].Max = ulongZero; + } + } + finally + { + } + + return bodyLength; + } + + private List SerializePrepopulatedDimensionsKeys(IEnumerable keys) + { + var serializedKeys = new List(this.prepopulatedDimensionsCount); + foreach (var key in keys) + { + serializedKeys.Add(Encoding.UTF8.GetBytes(key)); + } + + return serializedKeys; + } + + private List SerializePrepopulatedDimensionsValues(IEnumerable values) + { + var serializedValues = new List(this.prepopulatedDimensionsCount); + foreach (var value in values) + { + var valueAsString = Convert.ToString(value, CultureInfo.InvariantCulture); + serializedValues.Add(Encoding.UTF8.GetBytes(valueAsString)); + } + + return serializedValues; + } + + private unsafe int InitializeBufferForNonHistogramMetrics() + { + // The buffer format is as follows: + // -- BinaryHeader + // -- MetricPayload + // -- Variable length content + + // Leave enough space for the header and fixed payload + var bufferIndex = sizeof(BinaryHeader) + sizeof(MetricPayload); + + MetricSerializer.SerializeString(this.bufferForNonHistogramMetrics, ref bufferIndex, this.monitoringAccount); + MetricSerializer.SerializeString(this.bufferForNonHistogramMetrics, ref bufferIndex, this.metricNamespace); + + return bufferIndex; + } + + private unsafe int InitializeBufferForHistogramMetrics() + { + // The buffer format is as follows: + // -- BinaryHeader + // -- ExternalPayload + // -- Variable length content + + // Leave enough space for the header and fixed payload + var bufferIndex = sizeof(BinaryHeader) + sizeof(ExternalPayload); + + MetricSerializer.SerializeString(this.bufferForHistogramMetrics, ref bufferIndex, this.monitoringAccount); + MetricSerializer.SerializeString(this.bufferForHistogramMetrics, ref bufferIndex, this.metricNamespace); + + return bufferIndex; + } + } +} diff --git a/src/OpenTelemetry.Exporter.Geneva/GenevaMetricExporterExtensions.cs b/src/OpenTelemetry.Exporter.Geneva/GenevaMetricExporterExtensions.cs new file mode 100644 index 00000000000..dcf0df1bec5 --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/GenevaMetricExporterExtensions.cs @@ -0,0 +1,21 @@ +using System; +using OpenTelemetry.Metrics; + +namespace OpenTelemetry.Exporter.Geneva +{ + public static class GenevaMetricExporterExtensions + { + [System.Diagnostics.CodeAnalysis.SuppressMessage("Reliability", "CA2000:Dispose objects before losing scope", Justification = "The objects should not be disposed.")] + public static MeterProviderBuilder AddGenevaMetricExporter(this MeterProviderBuilder builder, Action configure = null) + { + if (builder == null) + { + throw new ArgumentNullException(nameof(builder)); + } + + var options = new GenevaMetricExporterOptions(); + configure?.Invoke(options); + return builder.AddReader(new PeriodicExportingMetricReader(new GenevaMetricExporter(options), options.MetricExportIntervalMilliseconds)); + } + } +} diff --git a/src/OpenTelemetry.Exporter.Geneva/GenevaMetricExporterOptions.cs b/src/OpenTelemetry.Exporter.Geneva/GenevaMetricExporterOptions.cs new file mode 100644 index 00000000000..81a629d3b03 --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/GenevaMetricExporterOptions.cs @@ -0,0 +1,66 @@ +using System; +using System.Collections.Generic; +using System.Globalization; + +namespace OpenTelemetry.Exporter.Geneva +{ + public class GenevaMetricExporterOptions + { + private IReadOnlyDictionary _prepopulatedMetricDimensions; + + /// + /// Gets or sets the ConnectionString which contains semicolon separated list of key-value pairs. + /// For e.g.: "Account=OTelMonitoringAccount;Namespace=OTelMetricNamespace". + /// + public string ConnectionString { get; set; } + + /// + /// Gets or sets the metric export interval in milliseconds. The default value is 20000. + /// + public int MetricExportIntervalMilliseconds { get; set; } = 20000; + + /// + /// Gets or sets the pre-populated dimensions for all the metrics exported by the exporter. + /// + public IReadOnlyDictionary PrepopulatedMetricDimensions + { + get + { + return this._prepopulatedMetricDimensions; + } + + set + { + if (value == null) + { + throw new ArgumentNullException(nameof(value)); + } + + var copy = new Dictionary(value.Count); + + foreach (var entry in value) + { + if (entry.Key.Length > GenevaMetricExporter.MaxDimensionNameSize) + { + throw new ArgumentException($"The dimension: {entry.Key} exceeds the maximum allowed limit of {GenevaMetricExporter.MaxDimensionNameSize} characters for a dimension name."); + } + + if (entry.Value == null) + { + throw new ArgumentNullException($"{nameof(this.PrepopulatedMetricDimensions)}[\"{entry.Key}\"]"); + } + + var dimensionValue = Convert.ToString(entry.Value, CultureInfo.InvariantCulture); + if (dimensionValue.Length > GenevaMetricExporter.MaxDimensionValueSize) + { + throw new ArgumentException($"Value provided for the dimension: {entry.Key} exceeds the maximum allowed limit of {GenevaMetricExporter.MaxDimensionValueSize} characters for dimension value."); + } + + copy[entry.Key] = entry.Value; // shallow copy + } + + this._prepopulatedMetricDimensions = copy; + } + } + } +} diff --git a/src/OpenTelemetry.Exporter.Geneva/GenevaTraceExporter.cs b/src/OpenTelemetry.Exporter.Geneva/GenevaTraceExporter.cs new file mode 100644 index 00000000000..1eff7967a53 --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/GenevaTraceExporter.cs @@ -0,0 +1,435 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Runtime.InteropServices; +using System.Text; +using System.Threading; + +namespace OpenTelemetry.Exporter.Geneva +{ + public class GenevaTraceExporter : GenevaBaseExporter + { + public GenevaTraceExporter(GenevaExporterOptions options) + { + if (options == null) + { + throw new ArgumentNullException(nameof(options)); + } + + if (string.IsNullOrWhiteSpace(options.ConnectionString)) + { + throw new ArgumentException($"{nameof(options.ConnectionString)} is invalid."); + } + + var partAName = "Span"; + if (options.TableNameMappings != null + && options.TableNameMappings.TryGetValue("Span", out var customTableName)) + { + if (string.IsNullOrWhiteSpace(customTableName)) + { + throw new ArgumentException("TableName mapping for Span is invalid."); + } + + if (Encoding.UTF8.GetByteCount(customTableName) != customTableName.Length) + { + throw new ArgumentException("The \"{customTableName}\" provided for TableNameMappings option contains non-ASCII characters", customTableName); + } + + partAName = customTableName; + } + + var connectionStringBuilder = new ConnectionStringBuilder(options.ConnectionString); + switch (connectionStringBuilder.Protocol) + { + case TransportProtocol.Etw: + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + throw new ArgumentException("ETW cannot be used on non-Windows operating systems."); + } + + this.m_dataTransport = new EtwDataTransport(connectionStringBuilder.EtwSession); + break; + case TransportProtocol.Unix: + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + throw new ArgumentException("Unix domain socket should not be used on Windows."); + } + + var unixDomainSocketPath = connectionStringBuilder.ParseUnixDomainSocketPath(); + this.m_dataTransport = new UnixDomainSocketDataTransport(unixDomainSocketPath); + break; + case TransportProtocol.Tcp: + throw new ArgumentException("TCP transport is not supported yet."); + case TransportProtocol.Udp: + throw new ArgumentException("UDP transport is not supported yet."); + default: + throw new ArgumentOutOfRangeException(nameof(connectionStringBuilder.Protocol)); + } + + // TODO: Validate custom fields (reserved name? etc). + if (options.CustomFields != null) + { + var customFields = new Dictionary(StringComparer.Ordinal); + var dedicatedFields = new Dictionary(StringComparer.Ordinal); + + // Seed customFields with Span PartB + customFields["azureResourceProvider"] = true; + dedicatedFields["azureResourceProvider"] = true; + foreach (var name in CS40_PART_B_MAPPING.Values) + { + customFields[name] = true; + dedicatedFields[name] = true; + } + + foreach (var name in options.CustomFields) + { + customFields[name] = true; + dedicatedFields[name] = true; + } + + this.m_customFields = customFields; + + foreach (var name in CS40_PART_B_MAPPING.Keys) + { + dedicatedFields[name] = true; + } + + dedicatedFields["otel.status_code"] = true; + this.m_dedicatedFields = dedicatedFields; + } + + var buffer = new byte[BUFFER_SIZE]; + + var cursor = 0; + + /* Fluentd Forward Mode: + [ + "Span", + [ + [ , { "env_ver": "4.0", ... } ] + ], + { "TimeFormat": "DateTime" } + ] + */ + cursor = MessagePackSerializer.WriteArrayHeader(buffer, cursor, 3); + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, partAName); + cursor = MessagePackSerializer.WriteArrayHeader(buffer, cursor, 1); + cursor = MessagePackSerializer.WriteArrayHeader(buffer, cursor, 2); + + // timestamp + cursor = MessagePackSerializer.WriteTimestamp96Header(buffer, cursor); + this.m_idxTimestampPatch = cursor; + cursor += 12; // reserve 12 bytes for the timestamp + + cursor = MessagePackSerializer.WriteMapHeader(buffer, cursor, ushort.MaxValue); // Note: always use Map16 for perf consideration + this.m_idxMapSizePatch = cursor - 2; + + this.m_cntPrepopulatedFields = 0; + + // TODO: Do we support PartB as well? + // Part A - core envelope + cursor = AddPartAField(buffer, cursor, Schema.V40.PartA.Name, partAName); + this.m_cntPrepopulatedFields += 1; + + foreach (var entry in options.PrepopulatedFields) + { + var value = entry.Value; + switch (value) + { + case bool vb: + case byte vui8: + case sbyte vi8: + case short vi16: + case ushort vui16: + case int vi32: + case uint vui32: + case long vi64: + case ulong vui64: + case float vf: + case double vd: + case string vs: + break; + default: + value = options.ConvertToJson(value); + break; + } + + cursor = AddPartAField(buffer, cursor, entry.Key, value); + this.m_cntPrepopulatedFields += 1; + } + + this.m_bufferPrologue = new byte[cursor - 0]; + Buffer.BlockCopy(buffer, 0, this.m_bufferPrologue, 0, cursor - 0); + + cursor = MessagePackSerializer.Serialize(buffer, 0, new Dictionary { { "TimeFormat", "DateTime" } }); + + this.m_bufferEpilogue = new byte[cursor - 0]; + Buffer.BlockCopy(buffer, 0, this.m_bufferEpilogue, 0, cursor - 0); + } + + public override ExportResult Export(in Batch batch) + { + // Note: The MessagePackSerializer takes way less time / memory than creating the activity itself. + // This makes the short-circuit check less useful. + // On the other side, running the serializer could help to catch error in the early phase of development lifecycle. + // + // if (!m_dataTransport.IsEnabled()) + // { + // return ExportResult.Success; + // } + + var result = ExportResult.Success; + foreach (var activity in batch) + { + try + { + var cursor = this.SerializeActivity(activity); + this.m_dataTransport.Send(this.m_buffer.Value, cursor - 0); + } + catch (Exception ex) + { + ExporterEventSource.Log.ExporterException(ex); // TODO: preallocate exception or no exception + result = ExportResult.Failure; + } + } + + return result; + } + + protected override void Dispose(bool disposing) + { + if (this.isDisposed) + { + return; + } + + if (disposing) + { + try + { + (this.m_dataTransport as IDisposable)?.Dispose(); + this.m_buffer.Dispose(); + } + catch (Exception ex) + { + ExporterEventSource.Log.ExporterException(ex); + } + } + + this.isDisposed = true; + base.Dispose(disposing); + } + + internal bool IsUsingUnixDomainSocket + { + get => this.m_dataTransport is UnixDomainSocketDataTransport; + } + + internal int SerializeActivity(Activity activity) + { + var buffer = this.m_buffer.Value; + if (buffer == null) + { + buffer = new byte[BUFFER_SIZE]; // TODO: handle OOM + Buffer.BlockCopy(this.m_bufferPrologue, 0, buffer, 0, this.m_bufferPrologue.Length); + this.m_buffer.Value = buffer; + } + + var cursor = this.m_bufferPrologue.Length; + var cntFields = this.m_cntPrepopulatedFields; + var dtBegin = activity.StartTimeUtc; + var tsBegin = dtBegin.Ticks; + var tsEnd = tsBegin + activity.Duration.Ticks; + var dtEnd = new DateTime(tsEnd); + + MessagePackSerializer.WriteTimestamp96(buffer, this.m_idxTimestampPatch, tsEnd); + + #region Part A - core envelope + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "env_time"); + cursor = MessagePackSerializer.SerializeUtcDateTime(buffer, cursor, dtEnd); + cntFields += 1; + #endregion + + #region Part A - dt extension + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "env_dt_traceId"); + + // Note: ToHexString returns the pre-calculated hex representation without allocation + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, activity.Context.TraceId.ToHexString()); + cntFields += 1; + + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "env_dt_spanId"); + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, activity.Context.SpanId.ToHexString()); + cntFields += 1; + #endregion + + #region Part B Span - required fields + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "name"); + cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, activity.DisplayName); + cntFields += 1; + + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "kind"); + cursor = MessagePackSerializer.SerializeInt32(buffer, cursor, (int)activity.Kind); + cntFields += 1; + + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "startTime"); + cursor = MessagePackSerializer.SerializeUtcDateTime(buffer, cursor, dtBegin); + cntFields += 1; + + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "success"); + cursor = MessagePackSerializer.SerializeBool(buffer, cursor, true); + var idxSuccessPatch = cursor - 1; + cntFields += 1; + #endregion + + #region Part B Span optional fields and Part C fields + var strParentId = activity.ParentSpanId.ToHexString(); + + // Note: this should be blazing fast since Object.ReferenceEquals(strParentId, INVALID_SPAN_ID) == true + if (!string.Equals(strParentId, INVALID_SPAN_ID, StringComparison.Ordinal)) + { + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "parentId"); + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, strParentId); + cntFields += 1; + } + + var links = activity.Links; + if (links.Any()) + { + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "links"); + cursor = MessagePackSerializer.WriteArrayHeader(buffer, cursor, ushort.MaxValue); // Note: always use Array16 for perf consideration + var idxLinkPatch = cursor - 2; + ushort cntLink = 0; + foreach (var link in links) + { + cursor = MessagePackSerializer.WriteMapHeader(buffer, cursor, 2); + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "toTraceId"); + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, link.Context.TraceId.ToHexString()); + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "toSpanId"); + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, link.Context.SpanId.ToHexString()); + cntLink += 1; + } + + MessagePackSerializer.WriteUInt16(buffer, idxLinkPatch, cntLink); + cntFields += 1; + } + + // TODO: The current approach is to iterate twice over TagObjects so that all + // env_properties can be added the very end. This avoids speculating the size + // and preallocating a separate buffer for it. + // Alternates include static allocation and iterate once. + // The TODO: here is to measure perf and change to alternate, if required. + + // Iteration #1 - Get those fields which become dedicated column + // i.e all PartB fields and opt-in part c fields. + bool hasEnvProperties = false; + foreach (var entry in activity.TagObjects) + { + // TODO: check name collision + if (CS40_PART_B_MAPPING.TryGetValue(entry.Key, out string replacementKey)) + { + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, replacementKey); + } + else if (string.Equals(entry.Key, "otel.status_code", StringComparison.Ordinal)) + { + if (string.Equals(entry.Value.ToString(), "ERROR", StringComparison.Ordinal)) + { + MessagePackSerializer.SerializeBool(buffer, idxSuccessPatch, false); + } + + continue; + } + else if (this.m_customFields == null || this.m_customFields.ContainsKey(entry.Key)) + { + // TODO: the above null check can be optimized and avoided inside foreach. + cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, entry.Key); + } + else + { + hasEnvProperties = true; + continue; + } + + cursor = MessagePackSerializer.Serialize(buffer, cursor, entry.Value); + cntFields += 1; + } + + if (hasEnvProperties) + { + // Iteration #2 - Get all "other" fields and collapse them into single field + // named "env_properties". + ushort envPropertiesCount = 0; + cursor = MessagePackSerializer.SerializeAsciiString(buffer, cursor, "env_properties"); + cursor = MessagePackSerializer.WriteMapHeader(buffer, cursor, ushort.MaxValue); + int idxMapSizeEnvPropertiesPatch = cursor - 2; + + foreach (var entry in activity.TagObjects) + { + // TODO: check name collision + if (this.m_dedicatedFields.ContainsKey(entry.Key)) + { + continue; + } + else + { + cursor = MessagePackSerializer.SerializeUnicodeString(buffer, cursor, entry.Key); + cursor = MessagePackSerializer.Serialize(buffer, cursor, entry.Value); + envPropertiesCount += 1; + } + } + + cntFields += 1; + MessagePackSerializer.WriteUInt16(buffer, idxMapSizeEnvPropertiesPatch, envPropertiesCount); + } + #endregion + + MessagePackSerializer.WriteUInt16(buffer, this.m_idxMapSizePatch, cntFields); + + Buffer.BlockCopy(this.m_bufferEpilogue, 0, buffer, cursor, this.m_bufferEpilogue.Length); + cursor += this.m_bufferEpilogue.Length; + + return cursor; + } + + private const int BUFFER_SIZE = 65360; // the maximum ETW payload (inclusive) + + private readonly ThreadLocal m_buffer = new ThreadLocal(() => null); + + private readonly byte[] m_bufferPrologue; + + private readonly byte[] m_bufferEpilogue; + + private readonly ushort m_cntPrepopulatedFields; + + private readonly int m_idxTimestampPatch; + + private readonly int m_idxMapSizePatch; + + private readonly IDataTransport m_dataTransport; + + private readonly IReadOnlyDictionary m_customFields; + + private readonly IReadOnlyDictionary m_dedicatedFields; + + private static readonly string INVALID_SPAN_ID = default(ActivitySpanId).ToHexString(); + + private static readonly IReadOnlyDictionary CS40_PART_B_MAPPING = new Dictionary + { + ["db.system"] = "dbSystem", + ["db.name"] = "dbName", + ["db.statement"] = "dbStatement", + + ["http.method"] = "httpMethod", + ["http.url"] = "httpUrl", + ["http.status_code"] = "httpStatusCode", + + ["messaging.system"] = "messagingSystem", + ["messaging.destination"] = "messagingDestination", + ["messaging.url"] = "messagingUrl", + + ["otel.status_description"] = "statusMessage", + }; + + private bool isDisposed; + } +} diff --git a/src/OpenTelemetry.Exporter.Geneva/IDataTransport.cs b/src/OpenTelemetry.Exporter.Geneva/IDataTransport.cs new file mode 100644 index 00000000000..2fb314b332e --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/IDataTransport.cs @@ -0,0 +1,9 @@ +namespace OpenTelemetry.Exporter.Geneva +{ + internal interface IDataTransport + { + bool IsEnabled(); + + void Send(byte[] data, int size); + } +} diff --git a/src/OpenTelemetry.Exporter.Geneva/IMetricDataTransport.cs b/src/OpenTelemetry.Exporter.Geneva/IMetricDataTransport.cs new file mode 100644 index 00000000000..09d84a21f81 --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/IMetricDataTransport.cs @@ -0,0 +1,18 @@ +using System; + +namespace OpenTelemetry.Exporter.Geneva +{ + internal interface IMetricDataTransport : IDisposable + { + /// + /// Writes a standard metric event containing only a single value. + /// + /// Type of the event. + /// The byte array containing the serialized data. + /// Length of the payload (fixed + variable). + void Send( + MetricEventType eventType, + byte[] body, + int size); + } +} diff --git a/src/OpenTelemetry.Exporter.Geneva/MessagePackSerializer.cs b/src/OpenTelemetry.Exporter.Geneva/MessagePackSerializer.cs new file mode 100644 index 00000000000..6ab2bb62582 --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/MessagePackSerializer.cs @@ -0,0 +1,559 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Runtime.CompilerServices; +using System.Text; + +namespace OpenTelemetry.Exporter.Geneva +{ + internal static class MessagePackSerializer + { + public const byte MIN_FIX_MAP = 0x80; + public const byte MIN_FIX_ARRAY = 0x90; + public const byte MIN_FIX_STR = 0xA0; + public const byte NIL = 0xC0; + public const byte FALSE = 0xC2; + public const byte TRUE = 0xC3; + public const byte BIN8 = 0xC4; + public const byte BIN16 = 0xC5; + public const byte BIN32 = 0xC6; + public const byte TIMESTAMP96 = 0xC7; + public const byte FLOAT32 = 0xCA; + public const byte FLOAT64 = 0xCB; + public const byte UINT8 = 0xCC; + public const byte UINT16 = 0xCD; + public const byte UINT32 = 0xCE; + public const byte UINT64 = 0xCF; + public const byte INT8 = 0xD0; + public const byte INT16 = 0xD1; + public const byte INT32 = 0xD2; + public const byte INT64 = 0xD3; + public const byte TIMESTAMP32 = 0xD6; + public const byte TIMESTAMP64 = 0xD7; + public const byte STR8 = 0xD9; + public const byte STR16 = 0xDA; + public const byte STR32 = 0xDB; + public const byte ARRAY16 = 0xDC; + public const byte ARRAY32 = 0xDD; + public const byte MAP16 = 0xDE; + public const byte MAP32 = 0xDF; + public const byte EXT_DATE_TIME = 0xFF; + + private const int LIMIT_MIN_FIX_NEGATIVE_INT = -32; + private const int LIMIT_MAX_FIX_STRING_LENGTH_IN_BYTES = 31; + private const int LIMIT_MAX_STR8_LENGTH_IN_BYTES = (1 << 8) - 1; // str8 stores 2^8 - 1 bytes + private const int LIMIT_MAX_FIX_MAP_COUNT = 15; + private const int LIMIT_MAX_FIX_ARRAY_LENGTH = 15; + private const int STRING_SIZE_LIMIT_CHAR_COUNT = (1 << 14) - 1; // 16 * 1024 - 1 = 16383 + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SerializeNull(byte[] buffer, int cursor) + { + buffer[cursor++] = NIL; + return cursor; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SerializeBool(byte[] buffer, int cursor, bool value) + { + buffer[cursor++] = value ? TRUE : FALSE; + return cursor; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SerializeInt8(byte[] buffer, int cursor, sbyte value) + { + if (value >= 0) + { + return SerializeUInt8(buffer, cursor, unchecked((byte)value)); + } + + if (value < LIMIT_MIN_FIX_NEGATIVE_INT) + { + buffer[cursor++] = INT8; + } + + buffer[cursor++] = unchecked((byte)value); + return cursor; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SerializeInt16(byte[] buffer, int cursor, short value) + { + if (value >= 0) + { + return SerializeUInt16(buffer, cursor, unchecked((ushort)value)); + } + + if (value >= sbyte.MinValue) + { + return SerializeInt8(buffer, cursor, unchecked((sbyte)value)); + } + + buffer[cursor++] = INT16; + return WriteInt16(buffer, cursor, value); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SerializeInt32(byte[] buffer, int cursor, int value) + { + if (value >= 0) + { + return SerializeUInt32(buffer, cursor, unchecked((uint)value)); + } + + if (value >= short.MinValue) + { + return SerializeInt16(buffer, cursor, unchecked((short)value)); + } + + buffer[cursor++] = INT32; + return WriteInt32(buffer, cursor, value); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SerializeInt64(byte[] buffer, int cursor, long value) + { + if (value >= 0) + { + return SerializeUInt64(buffer, cursor, unchecked((ulong)value)); + } + + if (value >= int.MinValue) + { + return SerializeInt32(buffer, cursor, unchecked((int)value)); + } + + buffer[cursor++] = INT64; + return WriteInt64(buffer, cursor, value); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SerializeUInt8(byte[] buffer, int cursor, byte value) + { + if (value > 127) + { + buffer[cursor++] = UINT8; + } + + buffer[cursor++] = value; + return cursor; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SerializeUInt16(byte[] buffer, int cursor, ushort value) + { + if (value <= byte.MaxValue) + { + return SerializeUInt8(buffer, cursor, unchecked((byte)value)); + } + + buffer[cursor++] = UINT16; + return WriteUInt16(buffer, cursor, value); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SerializeUInt32(byte[] buffer, int cursor, uint value) + { + if (value <= ushort.MaxValue) + { + return SerializeUInt16(buffer, cursor, unchecked((ushort)value)); + } + + buffer[cursor++] = UINT32; + return WriteUInt32(buffer, cursor, value); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SerializeUInt64(byte[] buffer, int cursor, ulong value) + { + if (value <= uint.MaxValue) + { + return SerializeUInt32(buffer, cursor, unchecked((uint)value)); + } + + buffer[cursor++] = UINT64; + return WriteUInt64(buffer, cursor, value); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int WriteInt16(byte[] buffer, int cursor, short value) + { + unchecked + { + buffer[cursor++] = (byte)(value >> 8); + buffer[cursor++] = (byte)value; + } + + return cursor; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int WriteInt32(byte[] buffer, int cursor, int value) + { + unchecked + { + buffer[cursor++] = (byte)(value >> 24); + buffer[cursor++] = (byte)(value >> 16); + buffer[cursor++] = (byte)(value >> 8); + buffer[cursor++] = (byte)value; + } + + return cursor; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int WriteInt64(byte[] buffer, int cursor, long value) + { + unchecked + { + buffer[cursor++] = (byte)(value >> 56); + buffer[cursor++] = (byte)(value >> 48); + buffer[cursor++] = (byte)(value >> 40); + buffer[cursor++] = (byte)(value >> 32); + buffer[cursor++] = (byte)(value >> 24); + buffer[cursor++] = (byte)(value >> 16); + buffer[cursor++] = (byte)(value >> 8); + buffer[cursor++] = (byte)value; + } + + return cursor; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int WriteUInt16(byte[] buffer, int cursor, ushort value) + { + unchecked + { + buffer[cursor++] = (byte)(value >> 8); + buffer[cursor++] = (byte)value; + } + + return cursor; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int WriteUInt32(byte[] buffer, int cursor, uint value) + { + unchecked + { + buffer[cursor++] = (byte)(value >> 24); + buffer[cursor++] = (byte)(value >> 16); + buffer[cursor++] = (byte)(value >> 8); + buffer[cursor++] = (byte)value; + } + + return cursor; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int WriteUInt64(byte[] buffer, int cursor, ulong value) + { + unchecked + { + buffer[cursor++] = (byte)(value >> 56); + buffer[cursor++] = (byte)(value >> 48); + buffer[cursor++] = (byte)(value >> 40); + buffer[cursor++] = (byte)(value >> 32); + buffer[cursor++] = (byte)(value >> 24); + buffer[cursor++] = (byte)(value >> 16); + buffer[cursor++] = (byte)(value >> 8); + buffer[cursor++] = (byte)value; + } + + return cursor; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SerializeFloat32(byte[] buffer, int cursor, float value) + { + buffer[cursor++] = FLOAT32; + return WriteInt32(buffer, cursor, Float32ToInt32(value)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe int Float32ToInt32(float value) + { + return *(int*)&value; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SerializeFloat64(byte[] buffer, int cursor, double value) + { + buffer[cursor++] = FLOAT64; + return WriteInt64(buffer, cursor, Float64ToInt64(value)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe long Float64ToInt64(double value) + { + return *(long*)&value; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SerializeAsciiString(byte[] buffer, int cursor, string value) + { + if (value == null) + { + return SerializeNull(buffer, cursor); + } + + int start = cursor; + var cch = value.Length; + int cb; + if (cch <= LIMIT_MAX_FIX_STRING_LENGTH_IN_BYTES) + { + cursor += 1; + cb = Encoding.ASCII.GetBytes(value, 0, cch, buffer, cursor); + if (cb <= LIMIT_MAX_FIX_STRING_LENGTH_IN_BYTES) + { + cursor += cb; + buffer[start] = unchecked((byte)(MIN_FIX_STR | cb)); + return cursor; + } + else + { + throw new ArgumentException("The input string: \"{inputString}\" has non-ASCII characters in it.", value); + } + } + + if (cch <= LIMIT_MAX_STR8_LENGTH_IN_BYTES) + { + cursor += 2; + cb = Encoding.ASCII.GetBytes(value, 0, cch, buffer, cursor); + cursor += cb; + if (cb <= LIMIT_MAX_STR8_LENGTH_IN_BYTES) + { + buffer[start] = STR8; + buffer[start + 1] = unchecked((byte)cb); + return cursor; + } + else + { + throw new ArgumentException("The input string: \"{inputString}\" has non-ASCII characters in it.", value); + } + } + + cursor += 3; + if (cch <= STRING_SIZE_LIMIT_CHAR_COUNT) + { + cb = Encoding.ASCII.GetBytes(value, 0, cch, buffer, cursor); + cursor += cb; + } + else + { + cb = Encoding.ASCII.GetBytes(value, 0, STRING_SIZE_LIMIT_CHAR_COUNT - 3, buffer, cursor); + cursor += cb; + cb += 3; + + // append "..." to indicate the string truncation + buffer[cursor++] = 0x2E; + buffer[cursor++] = 0x2E; + buffer[cursor++] = 0x2E; + } + + buffer[start] = STR16; + buffer[start + 1] = unchecked((byte)(cb >> 8)); + buffer[start + 2] = unchecked((byte)cb); + return cursor; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SerializeUnicodeString(byte[] buffer, int cursor, string value) + { + if (value == null) + { + return SerializeNull(buffer, cursor); + } + + int start = cursor; + var cch = value.Length; + int cb; + cursor += 3; + if (cch <= STRING_SIZE_LIMIT_CHAR_COUNT) + { + cb = Encoding.UTF8.GetBytes(value, 0, cch, buffer, cursor); + cursor += cb; + } + else + { + cb = Encoding.UTF8.GetBytes(value, 0, STRING_SIZE_LIMIT_CHAR_COUNT - 3, buffer, cursor); + cursor += cb; + cb += 3; + + // append "..." to indicate the string truncation + buffer[cursor++] = 0x2E; + buffer[cursor++] = 0x2E; + buffer[cursor++] = 0x2E; + } + + buffer[start] = STR16; + buffer[start + 1] = unchecked((byte)(cb >> 8)); + buffer[start + 2] = unchecked((byte)cb); + return cursor; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int WriteArrayHeader(byte[] buffer, int cursor, int length) + { + if (length <= LIMIT_MAX_FIX_ARRAY_LENGTH) + { + buffer[cursor++] = unchecked((byte)(MIN_FIX_ARRAY | length)); + } + else if (length <= ushort.MaxValue) + { + buffer[cursor++] = ARRAY16; + cursor = WriteUInt16(buffer, cursor, unchecked((ushort)length)); + } + else + { + buffer[cursor++] = ARRAY32; + cursor = WriteUInt32(buffer, cursor, unchecked((uint)length)); + } + + return cursor; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SerializeArray(byte[] buffer, int cursor, T[] array) + { + if (array == null) + { + return SerializeNull(buffer, cursor); + } + + cursor = WriteArrayHeader(buffer, cursor, array.Length); + for (int i = 0; i < array.Length; i++) + { + cursor = Serialize(buffer, cursor, array[i]); + } + + return cursor; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int WriteMapHeader(byte[] buffer, int cursor, int count) + { + if (count <= LIMIT_MAX_FIX_MAP_COUNT) + { + buffer[cursor++] = unchecked((byte)(MIN_FIX_MAP | count)); + } + else if (count <= ushort.MaxValue) + { + buffer[cursor++] = MAP16; + cursor = WriteUInt16(buffer, cursor, unchecked((ushort)count)); + } + else + { + buffer[cursor++] = MAP32; + cursor = WriteUInt32(buffer, cursor, unchecked((uint)count)); + } + + return cursor; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SerializeMap(byte[] buffer, int cursor, IDictionary map) + { + if (map == null) + { + return SerializeNull(buffer, cursor); + } + + cursor = WriteMapHeader(buffer, cursor, map.Count); + foreach (var entry in map) + { + cursor = SerializeUnicodeString(buffer, cursor, entry.Key); + cursor = Serialize(buffer, cursor, entry.Value); + } + + return cursor; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int WriteTimestamp96Header(byte[] buffer, int cursor) + { + buffer[cursor++] = TIMESTAMP96; + buffer[cursor++] = 12; + buffer[cursor++] = EXT_DATE_TIME; + return cursor; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int WriteTimestamp96(byte[] buffer, int cursor, long ticks) + { + cursor = WriteUInt32(buffer, cursor, unchecked((uint)((ticks % TimeSpan.TicksPerSecond) * 100))); + cursor = WriteInt64(buffer, cursor, (ticks / TimeSpan.TicksPerSecond) - 62135596800L); + return cursor; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SerializeTimestamp96(byte[] buffer, int cursor, long ticks) + { + cursor = WriteTimestamp96Header(buffer, cursor); + cursor = WriteTimestamp96(buffer, cursor, ticks); + return cursor; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SerializeUtcDateTime(byte[] buffer, int cursor, DateTime utc) + { + return SerializeTimestamp96(buffer, cursor, utc.Ticks); + } + + public static int Serialize(byte[] buffer, int cursor, object obj) + { + if (obj == null) + { + return SerializeNull(buffer, cursor); + } + + switch (obj) + { + case bool v: + return SerializeBool(buffer, cursor, v); + case byte v: + return SerializeUInt8(buffer, cursor, v); + case sbyte v: + return SerializeInt8(buffer, cursor, v); + case short v: + return SerializeInt16(buffer, cursor, v); + case ushort v: + return SerializeUInt16(buffer, cursor, v); + case int v: + return SerializeInt32(buffer, cursor, v); + case uint v: + return SerializeUInt32(buffer, cursor, v); + case long v: + return SerializeInt64(buffer, cursor, v); + case ulong v: + return SerializeUInt64(buffer, cursor, v); + case float v: + return SerializeFloat32(buffer, cursor, v); + case double v: + return SerializeFloat64(buffer, cursor, v); + case string v: + return SerializeUnicodeString(buffer, cursor, v); + case IDictionary v: + return SerializeMap(buffer, cursor, v); + case object[] v: + return SerializeArray(buffer, cursor, v); + case DateTime v: + return SerializeUtcDateTime(buffer, cursor, v.ToUniversalTime()); + default: + string repr = null; + + try + { + repr = Convert.ToString(obj, CultureInfo.InvariantCulture); + } + catch + { + repr = $"ERROR: type {obj.GetType().FullName} is not supported"; + } + + return SerializeUnicodeString(buffer, cursor, repr); + } + } + } +} diff --git a/src/OpenTelemetry.Exporter.Geneva/MetricEtwDataTransport.cs b/src/OpenTelemetry.Exporter.Geneva/MetricEtwDataTransport.cs new file mode 100644 index 00000000000..2ba66e8e39c --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/MetricEtwDataTransport.cs @@ -0,0 +1,46 @@ +using System; +using System.Diagnostics.Tracing; + +namespace OpenTelemetry.Exporter.Geneva +{ + [EventSource(Name = "OpenTelemetryGenevaMetricExporter", Guid = "{edc24920-e004-40f6-a8e1-0e6e48f39d84}")] + internal sealed class MetricEtwDataTransport : EventSource, IMetricDataTransport + { + private readonly int fixedPayloadEndIndex; + + public MetricEtwDataTransport() + { + unsafe + { + this.fixedPayloadEndIndex = sizeof(BinaryHeader); + } + } + + [NonEvent] + public unsafe void Send(MetricEventType eventType, byte[] data, int size) + { + var eventDataPtr = stackalloc EventData[1]; + fixed (byte* bufferPtr = data) + { + eventDataPtr[0].DataPointer = (IntPtr)bufferPtr + this.fixedPayloadEndIndex; + eventDataPtr[0].Size = size; + this.WriteEventCore((int)eventType, 1, eventDataPtr); + } + } + + [Event((int)MetricEventType.ULongMetric)] + private void ULongMetricEvent() + { + } + + [Event((int)MetricEventType.DoubleMetric)] + private void DoubleMetricEvent() + { + } + + [Event((int)MetricEventType.ExternallyAggregatedULongDistributionMetric)] + private void ExternallyAggregatedDoubleDistributionMetric() + { + } + } +} diff --git a/src/OpenTelemetry.Exporter.Geneva/MetricSerializer.cs b/src/OpenTelemetry.Exporter.Geneva/MetricSerializer.cs new file mode 100644 index 00000000000..8beab0ea72b --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/MetricSerializer.cs @@ -0,0 +1,312 @@ +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text; + +namespace OpenTelemetry.Exporter.Geneva +{ + internal static class MetricSerializer + { + /// + /// Writes the string to buffer. + /// + /// The buffer. + /// Index of the buffer. + /// The value. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SerializeString(byte[] buffer, ref int bufferIndex, string value) + { + if (!string.IsNullOrEmpty(value)) + { + if (bufferIndex + value.Length + sizeof(short) >= buffer.Length) + { + // TODO: What should we do when the data is invalid? + } + +#if NETSTANDARD2_1 + Span bufferSpan = new Span(buffer); + bufferSpan = bufferSpan.Slice(bufferIndex); + Span stringSpan = bufferSpan.Slice(2); + var lengthWritten = (short)Encoding.UTF8.GetBytes(value, stringSpan); + MemoryMarshal.Write(bufferSpan, ref lengthWritten); + bufferIndex += sizeof(short) + lengthWritten; +#else + // Advanced the buffer to account for the length, we will write it back after encoding. + var currentIndex = bufferIndex; + bufferIndex += sizeof(short); + var lengthWritten = Encoding.UTF8.GetBytes(value, 0, value.Length, buffer, bufferIndex); + bufferIndex += lengthWritten; + + // Write the length now that it is known + SerializeInt16(buffer, ref currentIndex, (short)lengthWritten); +#endif + } + else + { + SerializeInt16(buffer, ref bufferIndex, 0); + } + } + + /// + /// Writes the encoded string to buffer. + /// + /// The buffer. + /// Index of the buffer. + /// The encoded value. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SerializeEncodedString(byte[] buffer, ref int bufferIndex, byte[] encodedValue) + { + if (bufferIndex + encodedValue.Length + sizeof(short) >= buffer.Length) + { + // TODO: What should we do when the data is invalid? + } + +#if NETSTANDARD2_1 + Span sourceSpan = new Span(encodedValue); + Span bufferSpan = new Span(buffer); + bufferSpan = bufferSpan.Slice(bufferIndex); + sourceSpan.CopyTo(bufferSpan.Slice(2)); + short encodedLength = (short)encodedValue.Length; + MemoryMarshal.Write(bufferSpan, ref encodedLength); + bufferIndex += sizeof(short) + encodedLength; +#else + SerializeInt16(buffer, ref bufferIndex, (short)encodedValue.Length); + Array.Copy(encodedValue, 0, buffer, bufferIndex, encodedValue.Length); + bufferIndex += encodedValue.Length; +#endif + } + + /// + /// Writes the byte to buffer. + /// + /// The buffer. + /// Index of the buffer. + /// The value. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SerializeByte(byte[] buffer, ref int bufferIndex, byte value) + { + if (bufferIndex + sizeof(byte) >= buffer.Length) + { + // TODO: What should we do when the data is invalid? + } + + buffer[bufferIndex] = value; + bufferIndex += sizeof(byte); + } + + /// + /// Writes the unsigned short to buffer. + /// + /// The buffer. + /// Index of the buffer. + /// The value. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SerializeUInt16(byte[] buffer, ref int bufferIndex, ushort value) + { + if (bufferIndex + sizeof(ushort) >= buffer.Length) + { + // TODO: What should we do when the data is invalid? + } + + buffer[bufferIndex] = (byte)value; + buffer[bufferIndex + 1] = (byte)(value >> 8); + bufferIndex += sizeof(ushort); + } + + /// + /// Writes the short to buffer. + /// + /// The buffer. + /// Index of the buffer. + /// The value. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SerializeInt16(byte[] buffer, ref int bufferIndex, short value) + { + if (bufferIndex + sizeof(short) >= buffer.Length) + { + // TODO: What should we do when the data is invalid? + } + + buffer[bufferIndex] = (byte)value; + buffer[bufferIndex + 1] = (byte)(value >> 8); + bufferIndex += sizeof(short); + } + + /// + /// Writes the unsigned int to buffer. + /// + /// The buffer. + /// Index of the buffer. + /// The value. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SerializeUInt32(byte[] buffer, ref int bufferIndex, uint value) + { + if (bufferIndex + sizeof(uint) >= buffer.Length) + { + // TODO: What should we do when the data is invalid? + } + + buffer[bufferIndex] = (byte)value; + buffer[bufferIndex + 1] = (byte)(value >> 8); + buffer[bufferIndex + 2] = (byte)(value >> 0x10); + buffer[bufferIndex + 3] = (byte)(value >> 0x18); + bufferIndex += sizeof(uint); + } + + /// + /// Writes the ulong to buffer. + /// + /// The buffer. + /// Index of the buffer. + /// The value. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void SerializeUInt64(byte[] buffer, ref int bufferIndex, ulong value) + { + if (bufferIndex + sizeof(ulong) >= buffer.Length) + { + // TODO: What should we do when the data is invalid? + } + + buffer[bufferIndex] = (byte)value; + buffer[bufferIndex + 1] = (byte)(value >> 8); + buffer[bufferIndex + 2] = (byte)(value >> 0x10); + buffer[bufferIndex + 3] = (byte)(value >> 0x18); + buffer[bufferIndex + 4] = (byte)(value >> 0x20); + buffer[bufferIndex + 5] = (byte)(value >> 0x28); + buffer[bufferIndex + 6] = (byte)(value >> 0x30); + buffer[bufferIndex + 7] = (byte)(value >> 0x38); + bufferIndex += sizeof(ulong); + } + } + + internal enum MetricEventType + { + ULongMetric = 50, + DoubleMetric = 55, + ExternallyAggregatedULongDistributionMetric = 56, + } + + /// + /// Represents the binary header for non-ETW transmitted metrics. + /// + [StructLayout(LayoutKind.Explicit)] + internal struct BinaryHeader + { + /// + /// The event ID that represents how it will be processed. + /// + [FieldOffset(0)] + public ushort EventId; + + /// + /// The length of the body following the header. + /// + [FieldOffset(2)] + public ushort BodyLength; + } + + /// + /// Represents the fixed payload of a standard metric. + /// + [StructLayout(LayoutKind.Explicit)] + internal struct MetricPayload + { + /// + /// The dimension count. + /// + [FieldOffset(0)] + public ushort CountDimension; + + /// + /// Reserved for alignment. + /// + [FieldOffset(2)] + public ushort ReservedWord; // for 8-byte aligned + + /// + /// Reserved for alignment. + /// + [FieldOffset(4)] + public uint ReservedDword; + + /// + /// The UTC timestamp of the metric. + /// + [FieldOffset(8)] + public ulong TimestampUtc; + + /// + /// The value of the metric. + /// + [FieldOffset(16)] + public MetricData Data; + } + + /// + /// Represents the fixed payload of an externally aggregated metric. + /// + [StructLayout(LayoutKind.Explicit)] + internal struct ExternalPayload + { + /// + /// The dimension count. + /// + [FieldOffset(0)] + public ushort CountDimension; + + /// + /// Reserved for alignment. + /// + [FieldOffset(2)] + public ushort ReservedWord; // for alignment + + /// + /// The number of samples produced in the period. + /// + [FieldOffset(4)] + public uint Count; + + /// + /// The UTC timestamp of the metric. + /// + [FieldOffset(8)] + public ulong TimestampUtc; + + /// + /// The sum of the samples produced in the period. + /// + [FieldOffset(16)] + public MetricData Sum; + + /// + /// The minimum value of the samples produced in the period. + /// + [FieldOffset(24)] + public MetricData Min; + + /// + /// The maximum value of the samples produced in the period. + /// + [FieldOffset(32)] + public MetricData Max; + } + + /// + /// Represents the value of a metric. + /// + [StructLayout(LayoutKind.Explicit)] + internal struct MetricData + { + /// + /// The value represented as an integer. + /// + [FieldOffset(0)] + public ulong UInt64Value; + + /// + /// The value represented as a double. + /// + [FieldOffset(0)] + public double DoubleValue; + } +} diff --git a/src/OpenTelemetry.Exporter.Geneva/MetricUnixDataTransport.cs b/src/OpenTelemetry.Exporter.Geneva/MetricUnixDataTransport.cs new file mode 100644 index 00000000000..b4fc3b8ca12 --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/MetricUnixDataTransport.cs @@ -0,0 +1,37 @@ +namespace OpenTelemetry.Exporter.Geneva +{ + internal sealed class MetricUnixDataTransport : IMetricDataTransport + { + private readonly int fixedPayloadLength; + private readonly UnixDomainSocketDataTransport udsDataTransport; + private bool isDisposed; + + public MetricUnixDataTransport( + string unixDomainSocketPath, + int timeoutMilliseconds = UnixDomainSocketDataTransport.DefaultTimeoutMilliseconds) + { + unsafe + { + this.fixedPayloadLength = sizeof(BinaryHeader); + } + + this.udsDataTransport = new UnixDomainSocketDataTransport(unixDomainSocketPath, timeoutMilliseconds); + } + + public void Send(MetricEventType eventType, byte[] body, int size) + { + this.udsDataTransport.Send(body, size + this.fixedPayloadLength); + } + + public void Dispose() + { + if (this.isDisposed) + { + return; + } + + this.udsDataTransport?.Dispose(); + this.isDisposed = true; + } + } +} diff --git a/src/OpenTelemetry.Exporter.Geneva/OpenTelemetry.Exporter.Geneva.csproj b/src/OpenTelemetry.Exporter.Geneva/OpenTelemetry.Exporter.Geneva.csproj new file mode 100644 index 00000000000..48433825721 --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/OpenTelemetry.Exporter.Geneva.csproj @@ -0,0 +1,22 @@ + + + + true + An OpenTelemetry .NET exporter that exports to local ETW or UDS + OpenTelemetry Authors + $(NoWarn),NU5104,CS1591,SA1123,SA1633,SA1310,CA1031,CA1810,CA1822,CA2000,CA2208,SA1204,SA1201,SA1202,SA1308,SA1309,SA1311,SA1402,SA1602,SA1649 + netstandard2.0 + $(TargetFrameworks);net461 + + + + + + + + true + latest + AllEnabledByDefault + + + diff --git a/src/OpenTelemetry.Exporter.Geneva/README.md b/src/OpenTelemetry.Exporter.Geneva/README.md new file mode 100644 index 00000000000..eb7729a83ca --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/README.md @@ -0,0 +1,3 @@ +# Geneva Exporters for OpenTelemetry .NET + +TBD diff --git a/src/OpenTelemetry.Exporter.Geneva/ReentrantExportProcessor.cs b/src/OpenTelemetry.Exporter.Geneva/ReentrantExportProcessor.cs new file mode 100644 index 00000000000..d95406a1df2 --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/ReentrantExportProcessor.cs @@ -0,0 +1,36 @@ +using System; +using System.Linq.Expressions; +using System.Reflection; + +namespace OpenTelemetry.Exporter.Geneva +{ + // This export processor exports without synchronization. + // Once OpenTelemetry .NET officially support this, + // we can get rid of this class. + // This is currently only used in ETW export, where we know + // that the underlying system is safe under concurrent calls. + internal class ReentrantExportProcessor : BaseExportProcessor + where T : class + { + static ReentrantExportProcessor() + { + var flags = BindingFlags.Instance | BindingFlags.NonPublic; + var ctor = typeof(Batch).GetConstructor(flags, null, new Type[] { typeof(T) }, null); + var value = Expression.Parameter(typeof(T), null); + var lambda = Expression.Lambda>>(Expression.New(ctor, value), value); + CreateBatch = lambda.Compile(); + } + + public ReentrantExportProcessor(BaseExporter exporter) + : base(exporter) + { + } + + protected override void OnExport(T data) + { + this.exporter.Export(CreateBatch(data)); + } + + private static readonly Func> CreateBatch; + } +} diff --git a/src/OpenTelemetry.Exporter.Geneva/Schema.cs b/src/OpenTelemetry.Exporter.Geneva/Schema.cs new file mode 100644 index 00000000000..c641b8c32d7 --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/Schema.cs @@ -0,0 +1,87 @@ +namespace OpenTelemetry.Exporter.Geneva +{ + internal static class Schema + { + internal static class V21 + { + internal static class PartA + { + internal const string IKey = ".iKey"; + internal const string Name = ".name"; + internal const string Ver = ".ver"; + internal const string Time = ".time"; + internal const string Cv = ".cv"; + internal const string Epoch = ".epoch"; + internal const string Flags = ".flags"; + internal const string PopSample = ".popSample"; + internal const string SeqNum = ".seqNum"; + + internal static class Extensions + { + internal static class App + { + internal const string Id = "app.id"; + internal const string Ver = "app.ver"; + } + + internal static class Cloud + { + internal const string Environment = "cloud.environment"; + internal const string Location = "cloud.location"; + internal const string Name = "cloud.name"; + internal const string DeploymentUnit = "cloud.deploymentUnit"; + internal const string Role = "cloud.role"; + internal const string RoleInstance = "cloud.roleInstance"; + internal const string RoleVer = "cloud.roleVer"; + internal const string Ver = "cloud.ver"; + } + + internal static class Os + { + internal const string Name = "os.name"; + internal const string Ver = "os.ver"; + } + } + } + } + + internal static class V40 + { + internal static class PartA + { + internal const string IKey = ".iKey"; + internal const string Name = ".name"; + internal const string Ver = ".ver"; + internal const string Time = ".time"; + + internal static class Extensions + { + internal static class App + { + internal const string Id = "app.id"; + internal const string Ver = "app.ver"; + } + + internal static class Cloud + { + internal const string Role = "cloud.role"; + internal const string RoleInstance = "cloud.roleInstance"; + internal const string RoleVer = "cloud.roleVer"; + } + + internal static class Os + { + internal const string Name = "os.name"; + internal const string Ver = "os.ver"; + } + + internal static class Dt + { + internal const string TraceId = "dt.traceId"; + internal const string SpanId = "dt.spanId"; + } + } + } + } + } +} diff --git a/src/OpenTelemetry.Exporter.Geneva/ServiceProviderExtensions.cs b/src/OpenTelemetry.Exporter.Geneva/ServiceProviderExtensions.cs new file mode 100644 index 00000000000..47a2573c599 --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/ServiceProviderExtensions.cs @@ -0,0 +1,31 @@ +#if NET461 || NETSTANDARD2_0 || NETSTANDARD2_1 || NETCOREAPP3_1_OR_GREATER +using Microsoft.Extensions.Options; +#endif + +namespace System +{ + /// + /// Extension methods for OpenTelemetry dependency injection support. + /// + internal static class ServiceProviderExtensions + { + /// + /// Get options from the supplied . + /// + /// Options type. + /// . + /// Options instance. + public static T GetOptions(this IServiceProvider serviceProvider) + where T : class, new() + { +#if NET461 || NETSTANDARD2_0 || NETSTANDARD2_1 || NETCOREAPP3_1_OR_GREATER + IOptions options = (IOptions)serviceProvider.GetService(typeof(IOptions)); + + // Note: options could be null if user never invoked services.AddOptions(). + return options?.Value ?? new T(); +#else + return new T(); +#endif + } + } +} diff --git a/src/OpenTelemetry.Exporter.Geneva/UnixDomainSocketDataTransport.cs b/src/OpenTelemetry.Exporter.Geneva/UnixDomainSocketDataTransport.cs new file mode 100644 index 00000000000..280cd4e3bcf --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/UnixDomainSocketDataTransport.cs @@ -0,0 +1,93 @@ +using System; +using System.Net; +using System.Net.Sockets; + +namespace OpenTelemetry.Exporter.Geneva +{ + internal class UnixDomainSocketDataTransport : IDataTransport, IDisposable + { + public const int DefaultTimeoutMilliseconds = 15000; + private readonly EndPoint unixEndpoint; + private Socket socket; + private int timeoutMilliseconds; + + /// + /// Initializes a new instance of the class. + /// The class for transporting data over Unix domain socket. + /// + /// The path to connect a unix domain socket over. + /// + /// The time-out value, in milliseconds. + /// If you set the property with a value between 1 and 499, the value will be changed to 500. + /// The default value is 15,000 milliseconds. + /// + public UnixDomainSocketDataTransport( + string unixDomainSocketPath, + int timeoutMilliseconds = DefaultTimeoutMilliseconds) + { + this.unixEndpoint = new UnixDomainSocketEndPoint(unixDomainSocketPath); + this.timeoutMilliseconds = timeoutMilliseconds; + this.Connect(); + } + + public bool IsEnabled() + { + return true; + } + + public void Send(byte[] data, int size) + { + try + { + if (!this.socket.Connected) + { + // Socket connection is off! Server might have stopped. Trying to reconnect. + this.Reconnect(); + } + + this.socket.Send(data, size, SocketFlags.None); + } + catch (SocketException ex) + { + // SocketException from Socket.Send + ExporterEventSource.Log.ExporterException(ex); + } + catch (Exception ex) + { + ExporterEventSource.Log.ExporterException(ex); + } + } + + public void Dispose() + { + this.socket.Dispose(); + } + + private void Connect() + { + try + { + this.socket = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP) + { + SendTimeout = this.timeoutMilliseconds, + }; + this.socket.Connect(this.unixEndpoint); + } + catch (Exception ex) + { + ExporterEventSource.Log.ExporterException(ex); + + // Re-throw the exception to + // 1. fail fast in Geneva exporter contructor, or + // 2. fail in the Reconnect attempt. + throw; + } + } + + private void Reconnect() + { + this.socket.Close(); + this.Connect(); + } + } +} diff --git a/src/OpenTelemetry.Exporter.Geneva/UnixDomainSocketEndPoint.cs b/src/OpenTelemetry.Exporter.Geneva/UnixDomainSocketEndPoint.cs new file mode 100644 index 00000000000..7b317b72064 --- /dev/null +++ b/src/OpenTelemetry.Exporter.Geneva/UnixDomainSocketEndPoint.cs @@ -0,0 +1,79 @@ +using System; +using System.Net; +using System.Net.Sockets; +using System.Text; + +namespace OpenTelemetry.Exporter.Geneva +{ + internal class UnixDomainSocketEndPoint : EndPoint + { + // sockaddr_un.sun_path at http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/sys_un.h.html, -1 for terminator + private const int MaximumNativePathLength = 92 - 1; + + // The first 2 bytes of the underlying buffer are reserved for the AddressFamily enumerated value. + // https://docs.microsoft.com/dotnet/api/system.net.socketaddress + private const int NativePathOffset = 2; + private readonly string path; + private readonly byte[] nativePath; + + public UnixDomainSocketEndPoint(string path) + { + this.path = path ?? throw new ArgumentNullException(nameof(path), "Path cannot be null."); + this.nativePath = Encoding.UTF8.GetBytes(path); + if (this.nativePath.Length == 0 || this.nativePath.Length > MaximumNativePathLength) + { + throw new ArgumentOutOfRangeException(nameof(this.nativePath), "Path is of an invalid length for use with domain sockets."); + } + } + + public override AddressFamily AddressFamily => AddressFamily.Unix; + + public override EndPoint Create(SocketAddress socketAddress) => new UnixDomainSocketEndPoint(socketAddress); + + private UnixDomainSocketEndPoint(SocketAddress socketAddress) + { + if (socketAddress == null) + { + throw new ArgumentNullException(nameof(socketAddress), "SocketAddress cannot be null."); + } + + if (socketAddress.Family != this.AddressFamily || + socketAddress.Size > NativePathOffset + MaximumNativePathLength) + { + throw new ArgumentOutOfRangeException( + nameof(socketAddress), + "The path of SocketAddress is of an invalid length for use with domain sockets."); + } + + if (socketAddress.Size > NativePathOffset) + { + this.nativePath = new byte[socketAddress.Size - NativePathOffset]; + for (int i = 0; i < this.nativePath.Length; ++i) + { + this.nativePath[i] = socketAddress[NativePathOffset + i]; + } + + this.path = Encoding.UTF8.GetString(this.nativePath); + } + else + { + this.path = string.Empty; + this.nativePath = Array.Empty(); + } + } + + public override SocketAddress Serialize() + { + var socketAddress = new SocketAddress(AddressFamily.Unix, NativePathOffset + this.nativePath.Length + 1); + for (int i = 0; i < this.nativePath.Length; ++i) + { + socketAddress[NativePathOffset + i] = this.nativePath[i]; + } + + socketAddress[NativePathOffset + this.nativePath.Length] = 0; // SocketAddress should be NULL terminated + return socketAddress; + } + + public override string ToString() => this.path; + } +} diff --git a/test/OpenTelemetry.Exporter.Geneva.Benchmark/Exporter/LogExporterBenchmarks.cs b/test/OpenTelemetry.Exporter.Geneva.Benchmark/Exporter/LogExporterBenchmarks.cs new file mode 100644 index 00000000000..c1548a6db05 --- /dev/null +++ b/test/OpenTelemetry.Exporter.Geneva.Benchmark/Exporter/LogExporterBenchmarks.cs @@ -0,0 +1,160 @@ +using System; +using System.Collections.Generic; +using BenchmarkDotNet.Attributes; +using Microsoft.Extensions.Logging; +using OpenTelemetry.Logs; + +/* +BenchmarkDotNet=v0.13.1, OS=Windows 10.0.19044.1415 (21H2) +AMD Ryzen 9 3900X, 1 CPU, 24 logical and 12 physical cores +.NET SDK=6.0.101 + [Host] : .NET 5.0.12 (5.0.1221.52207), X64 RyuJIT + DefaultJob : .NET 5.0.12 (5.0.1221.52207), X64 RyuJIT + + +``` +| Method | Mean | Error | StdDev | Gen 0 | Allocated | +|------------------------- |------------:|---------:|---------:|-------:|----------:| +| NoListener | 58.41 ns | 0.360 ns | 0.337 ns | 0.0076 | 64 B | +| OneProcessor | 189.79 ns | 0.391 ns | 0.347 ns | 0.0277 | 232 B | +| TwoProcessors | 195.50 ns | 0.438 ns | 0.388 ns | 0.0277 | 232 B | +| ThreeProcessors | 198.98 ns | 0.500 ns | 0.468 ns | 0.0277 | 232 B | +| LoggerWithGenevaExporter | 1,101.36 ns | 4.134 ns | 3.452 ns | 0.0305 | 256 B | +| SerializeLogRecord | 743.53 ns | 2.233 ns | 2.088 ns | 0.0029 | 24 B | +*/ + +namespace OpenTelemetry.Exporter.Geneva.Benchmark +{ + [MemoryDiagnoser] + public class LogExporterBenchmarks + { + private readonly LogRecord logRecord; + private readonly GenevaLogExporter exporter; + private readonly ILogger loggerWithNoListener; + private readonly ILogger loggerWithGenevaExporter; + private readonly ILogger loggerWithOneProcessor; + private readonly ILogger loggerWithTwoProcessors; + private readonly ILogger loggerWithThreeProcessors; + + public LogExporterBenchmarks() + { + this.logRecord = this.GenerateTestLogRecord(); + + this.exporter = new GenevaLogExporter(new GenevaExporterOptions + { + ConnectionString = "EtwSession=OpenTelemetry", + PrepopulatedFields = new Dictionary + { + ["cloud.role"] = "BusyWorker", + ["cloud.roleInstance"] = "CY1SCH030021417", + ["cloud.roleVer"] = "9.0.15289.2", + }, + }); + + this.loggerWithNoListener = this.CreateLogger(); + + this.loggerWithOneProcessor = this.CreateLogger(options => options + .AddProcessor(new DummyLogProcessor())); + + this.loggerWithTwoProcessors = this.CreateLogger(options => options + .AddProcessor(new DummyLogProcessor()) + .AddProcessor(new DummyLogProcessor())); + + this.loggerWithThreeProcessors = this.CreateLogger(options => options + .AddProcessor(new DummyLogProcessor()) + .AddProcessor(new DummyLogProcessor()) + .AddProcessor(new DummyLogProcessor())); + + this.loggerWithGenevaExporter = this.CreateLogger(options => + { + options.AddGenevaLogExporter(genevaOptions => + { + genevaOptions.ConnectionString = "EtwSession=OpenTelemetry"; + genevaOptions.PrepopulatedFields = new Dictionary + { + ["cloud.role"] = "BusyWorker", + ["cloud.roleInstance"] = "CY1SCH030021417", + ["cloud.roleVer"] = "9.0.15289.2", + }; + }); + }); + } + + [Benchmark] + public void NoListener() + { + this.loggerWithNoListener.LogInformation("Hello from {food} {price}.", "artichoke", 3.99); + } + + [Benchmark] + public void OneProcessor() + { + this.loggerWithOneProcessor.LogInformation("Hello from {food} {price}.", "artichoke", 3.99); + } + + [Benchmark] + public void TwoProcessors() + { + this.loggerWithTwoProcessors.LogInformation("Hello from {food} {price}.", "artichoke", 3.99); + } + + [Benchmark] + public void ThreeProcessors() + { + this.loggerWithThreeProcessors.LogInformation("Hello from {food} {price}.", "artichoke", 3.99); + } + + [Benchmark] + public void LoggerWithGenevaExporter() + { + this.loggerWithGenevaExporter.LogInformation("Hello from {food} {price}.", "artichoke", 3.99); + } + + [Benchmark] + public void SerializeLogRecord() + { + this.exporter.SerializeLogRecord(this.logRecord); + } + + internal class DummyLogProcessor : BaseProcessor + { + } + + internal class DummyLogExporter : BaseExporter + { + public LogRecord LastRecord { get; set; } + + public override ExportResult Export(in Batch batch) + { + foreach (var record in batch) + { + this.LastRecord = record; + } + + return ExportResult.Success; + } + } + + internal ILogger CreateLogger(Action configure = null) + { + var loggerFactory = LoggerFactory.Create(builder => + { + if (configure != null) + { + builder.AddOpenTelemetry(configure); + } + }); + + return loggerFactory.CreateLogger(); + } + + internal LogRecord GenerateTestLogRecord() + { + var dummyLogExporter = new DummyLogExporter(); + var dummyLogger = this.CreateLogger(options => options + .AddProcessor(new SimpleLogRecordExportProcessor(dummyLogExporter))); + dummyLogger.LogInformation("Hello from {food} {price}.", "artichoke", 3.99); + return dummyLogExporter.LastRecord; + } + } +} diff --git a/test/OpenTelemetry.Exporter.Geneva.Benchmark/Exporter/MetricExporterBenchmarks.cs b/test/OpenTelemetry.Exporter.Geneva.Benchmark/Exporter/MetricExporterBenchmarks.cs new file mode 100644 index 00000000000..ee5395a1a65 --- /dev/null +++ b/test/OpenTelemetry.Exporter.Geneva.Benchmark/Exporter/MetricExporterBenchmarks.cs @@ -0,0 +1,624 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.Metrics; +using System.Threading; +using BenchmarkDotNet.Attributes; +using OpenTelemetry.Metrics; + +/* +BenchmarkDotNet=v0.13.1, OS=Windows 10.0.22000 +Intel Core i7-9700 CPU 3.00GHz, 1 CPU, 8 logical and 8 physical cores +.NET SDK=6.0.102 + [Host] : .NET 6.0.2 (6.0.222.6406), X64 RyuJIT + DefaultJob : .NET 6.0.2 (6.0.222.6406), X64 RyuJIT + +| Method | Mean | Error | StdDev | Allocated | +|--------------------------------------------------------- |----------:|---------:|---------:|----------:| +| InstrumentWithNoListener3Dimensions | 66.07 ns | 0.213 ns | 0.199 ns | - | +| InstrumentWithNoListener4Dimensions | 109.53 ns | 0.267 ns | 0.236 ns | - | +| InstrumentWithWithListener3Dimensions | 65.59 ns | 0.322 ns | 0.285 ns | - | +| InstrumentWithWithListener4Dimensions | 117.56 ns | 0.655 ns | 0.613 ns | - | +| InstrumentWithWithDummyReader3Dimensions | 182.75 ns | 0.787 ns | 0.698 ns | - | +| InstrumentWithWithDummyReader4Dimensions | 244.04 ns | 1.268 ns | 1.186 ns | - | +| InstrumentWithWithGenevaCounterMetricExporter3Dimensions | 181.74 ns | 0.595 ns | 0.527 ns | - | +| InstrumentWithWithGenevaCounterMetricExporter4Dimensions | 265.85 ns | 3.214 ns | 3.006 ns | - | +| SerializeCounterMetricItemWith3Dimensions | 166.33 ns | 0.470 ns | 0.439 ns | - | +| SerializeCounterMetricItemWith4Dimensions | 200.02 ns | 0.546 ns | 0.510 ns | - | +| ExportCounterMetricItemWith3Dimensions | 464.79 ns | 3.996 ns | 3.738 ns | - | +| ExportCounterMetricItemWith4Dimensions | 504.02 ns | 6.362 ns | 5.951 ns | - | +| SerializeHistogramMetricItemWith3Dimensions | 260.47 ns | 1.364 ns | 1.276 ns | - | +| SerializeHistogramMetricItemWith4Dimensions | 293.25 ns | 0.674 ns | 0.631 ns | - | +| ExportHistogramMetricItemWith3Dimensions | 585.69 ns | 5.137 ns | 4.805 ns | - | +| ExportHistogramMetricItemWith4Dimensions | 618.47 ns | 4.946 ns | 4.384 ns | - | +*/ + +namespace OpenTelemetry.Exporter.Geneva.Benchmark +{ + [MemoryDiagnoser] + public class MetricExporterBenchmarks + { + private Metric counterMetricWith3Dimensions; + private Metric counterMetricWith4Dimensions; + private MetricPoint counterMetricPointWith3Dimensions; + private MetricPoint counterMetricPointWith4Dimensions; + private MetricData counterMetricDataWith3Dimensions; + private MetricData counterMetricDataWith4Dimensions; + private Batch counterMetricBatchWith3Dimensions; + private Batch counterMetricBatchWith4Dimensions; + private Metric histogramMetricWith3Dimensions; + private Metric histogramMetricWith4Dimensions; + private MetricPoint histogramMetricPointWith3Dimensions; + private MetricPoint histogramMetricPointWith4Dimensions; + private MetricData histogramSumWith3Dimensions; + private MetricData histogramSumWith4Dimensions; + private uint histogramCountWith3Dimensions; + private uint histogramCountWith4Dimensions; + private Batch histogramMetricBatchWith3Dimensions; + private Batch histogramMetricBatchWith4Dimensions; + private Meter meterWithNoListener = new Meter("MeterWithNoListener", "0.0.1"); + private Meter meterWithListener = new Meter("MeterWithListener", "0.0.1"); + private Meter meterWithDummyReader = new Meter("MeterWithDummyReader", "0.0.1"); + private Meter meterWithGenevaMetricExporter = new Meter("MeterWithGenevaMetricExporter", "0.0.1"); + private Counter counterWithNoListener; + private Counter counterWithListener; + private Counter counterWithDummyReader; + private Counter counterWithGenevaMetricExporter; + private MeterListener listener; + private MeterProvider meterProviderWithDummyReader; + private MeterProvider meterProviderWithGenevaMetricExporter; + private MeterProvider meterProviderForCounterBatchWith3Dimensions; + private MeterProvider meterProviderForCounterBatchWith4Dimensions; + private MeterProvider meterProviderForHistogramBatchWith3Dimensions; + private MeterProvider meterProviderForHistogramBatchWith4Dimensions; + private GenevaMetricExporter exporter; + private ThreadLocal random = new ThreadLocal(() => new Random()); + + private static readonly Random randomForHistogram = new Random(); // Use the same seed for all the benchmarks to have the same data exported + private static readonly string[] dimensionValues = new string[] { "DimVal1", "DimVal2", "DimVal3", "DimVal4", "DimVal5", "DimVal6", "DimVal7", "DimVal8", "DimVal9", "DimVal10" }; + + [GlobalSetup] + public void Setup() + { + this.counterWithNoListener = this.meterWithNoListener.CreateCounter("counter"); + this.counterWithListener = this.meterWithListener.CreateCounter("counter"); + this.counterWithDummyReader = this.meterWithDummyReader.CreateCounter("counter"); + this.counterWithGenevaMetricExporter = this.meterWithGenevaMetricExporter.CreateCounter("counter"); + + var exporterOptions = new GenevaMetricExporterOptions() { ConnectionString = "Account=OTelMonitoringAccount;Namespace=OTelMetricNamespace" }; + this.exporter = new GenevaMetricExporter(exporterOptions); + + this.counterMetricPointWith3Dimensions = this.GenerateCounterMetricItemWith3Dimensions(out this.counterMetricDataWith3Dimensions); + this.counterMetricPointWith4Dimensions = this.GenerateCounterMetricItemWith4Dimensions(out this.counterMetricDataWith4Dimensions); + + this.counterMetricBatchWith3Dimensions = this.GenerateCounterBatchWith3Dimensions(); + this.counterMetricBatchWith4Dimensions = this.GenerateCounterBatchWith4Dimensions(); + + using var enumeratorForCounterBatchWith3Dimensions = this.counterMetricBatchWith3Dimensions.GetEnumerator(); + enumeratorForCounterBatchWith3Dimensions.MoveNext(); + this.counterMetricWith3Dimensions = enumeratorForCounterBatchWith3Dimensions.Current; + + using var enumeratorForCounterBatchWith4Dimensions = this.counterMetricBatchWith4Dimensions.GetEnumerator(); + enumeratorForCounterBatchWith4Dimensions.MoveNext(); + this.counterMetricWith4Dimensions = enumeratorForCounterBatchWith4Dimensions.Current; + + this.histogramMetricPointWith3Dimensions = this.GenerateHistogramMetricItemWith3Dimensions(out this.histogramSumWith3Dimensions, out this.histogramCountWith3Dimensions); + this.histogramMetricPointWith4Dimensions = this.GenerateHistogramMetricItemWith4Dimensions(out this.histogramSumWith4Dimensions, out this.histogramCountWith4Dimensions); + + this.histogramMetricBatchWith3Dimensions = this.GenerateHistogramBatchWith3Dimensions(); + this.histogramMetricBatchWith4Dimensions = this.GenerateHistogramBatchWith4Dimensions(); + + using var enumeratorForHistogramBatchWith3Dimensions = this.histogramMetricBatchWith3Dimensions.GetEnumerator(); + enumeratorForHistogramBatchWith3Dimensions.MoveNext(); + this.histogramMetricWith3Dimensions = enumeratorForHistogramBatchWith3Dimensions.Current; + + using var enumeratorForHistogramBatchWith4Dimensions = this.histogramMetricBatchWith4Dimensions.GetEnumerator(); + enumeratorForHistogramBatchWith4Dimensions.MoveNext(); + this.histogramMetricWith4Dimensions = enumeratorForHistogramBatchWith4Dimensions.Current; + + #region Setup MeterListener + this.listener = new MeterListener(); + this.listener.InstrumentPublished = (instrument, listener) => + { + if (instrument.Meter.Name == this.meterWithListener.Name) + { + listener.EnableMeasurementEvents(instrument); + } + }; + + this.listener.Start(); + #endregion + + this.meterProviderWithDummyReader = Sdk.CreateMeterProviderBuilder() + .AddMeter(this.meterWithDummyReader.Name) + .AddReader(new DummyReader(new DummyMetricExporter())) + .Build(); + + this.meterProviderWithGenevaMetricExporter = Sdk.CreateMeterProviderBuilder() + .AddMeter(this.meterWithGenevaMetricExporter.Name) + .AddGenevaMetricExporter(options => + { + options.ConnectionString = "Account=OTelMonitoringAccount;Namespace=OTelMetricNamespace"; + }) + .Build(); + } + + private MetricPoint GenerateCounterMetricItemWith3Dimensions(out MetricData metricData) + { + using var meterWithInMemoryExporter = new Meter("GenerateCounterMetricItemWith3Dimensions", "0.0.1"); + var counter = meterWithInMemoryExporter.CreateCounter("CounterWithThreeDimensions"); + + var exportedItems = new List(); + using var inMemoryReader = new BaseExportingMetricReader(new InMemoryExporter(exportedItems)) + { + Temporality = AggregationTemporality.Delta, + }; + + using var meterProvider = Sdk.CreateMeterProviderBuilder() + .AddMeter("GenerateCounterMetricItemWith3Dimensions") + .AddReader(inMemoryReader) + .Build(); + + counter.Add( + 100, + new("DimName1", dimensionValues[this.random.Value.Next(0, 10)]), + new("DimName2", dimensionValues[this.random.Value.Next(0, 10)]), + new("DimName3", dimensionValues[this.random.Value.Next(0, 10)])); + + inMemoryReader.Collect(); + + var metric = exportedItems[0]; + var metricPointsEnumerator = metric.GetMetricPoints().GetEnumerator(); + metricPointsEnumerator.MoveNext(); + var metricPoint = metricPointsEnumerator.Current; + var metricDataValue = Convert.ToUInt64(metricPoint.GetSumLong()); + metricData = new MetricData { UInt64Value = metricDataValue }; + + return metricPoint; + } + + private MetricPoint GenerateCounterMetricItemWith4Dimensions(out MetricData metricData) + { + using var meterWithInMemoryExporter = new Meter("GenerateCounterMetricItemWith4Dimensions", "0.0.1"); + var counter = meterWithInMemoryExporter.CreateCounter("CounterWith4Dimensions"); + + var exportedItems = new List(); + using var inMemoryReader = new BaseExportingMetricReader(new InMemoryExporter(exportedItems)) + { + Temporality = AggregationTemporality.Delta, + }; + + using var meterProvider = Sdk.CreateMeterProviderBuilder() + .AddMeter("GenerateCounterMetricItemWith4Dimensions") + .AddReader(inMemoryReader) + .Build(); + + var tags = new TagList + { + { "DimName1", dimensionValues[this.random.Value.Next(0, 2)] }, + { "DimName2", dimensionValues[this.random.Value.Next(0, 5)] }, + { "DimName3", dimensionValues[this.random.Value.Next(0, 10)] }, + { "DimName4", dimensionValues[this.random.Value.Next(0, 10)] }, + }; + + counter.Add(100, tags); + + inMemoryReader.Collect(); + + var metric = exportedItems[0]; + var metricPointsEnumerator = metric.GetMetricPoints().GetEnumerator(); + metricPointsEnumerator.MoveNext(); + var metricPoint = metricPointsEnumerator.Current; + var metricDataValue = Convert.ToUInt64(metricPoint.GetSumLong()); + metricData = new MetricData { UInt64Value = metricDataValue }; + + return metricPoint; + } + + private Batch GenerateCounterBatchWith3Dimensions() + { + using var meterWithInMemoryExporter = new Meter("GenerateCounterBatchWith3Dimensions", "0.0.1"); + var counter = meterWithInMemoryExporter.CreateCounter("CounterWithThreeDimensions"); + + var batchGeneratorExporter = new BatchGenerator(); + var batchGeneratorReader = new BaseExportingMetricReader(batchGeneratorExporter) + { + Temporality = AggregationTemporality.Delta, + }; + + this.meterProviderForCounterBatchWith3Dimensions = Sdk.CreateMeterProviderBuilder() + .AddMeter("GenerateCounterBatchWith3Dimensions") + .AddReader(batchGeneratorReader) + .Build(); + + counter.Add( + 100, + new("DimName1", dimensionValues[this.random.Value.Next(0, 10)]), + new("DimName2", dimensionValues[this.random.Value.Next(0, 10)]), + new("DimName3", dimensionValues[this.random.Value.Next(0, 10)])); + + this.meterProviderForCounterBatchWith3Dimensions.ForceFlush(); + return batchGeneratorExporter.Batch; + } + + private Batch GenerateCounterBatchWith4Dimensions() + { + using var meterWithInMemoryExporter = new Meter("GenerateCounterBatchWith4Dimensions", "0.0.1"); + var counter = meterWithInMemoryExporter.CreateCounter("CounterWith4Dimensions"); + + var batchGeneratorExporter = new BatchGenerator(); + var batchGeneratorReader = new BaseExportingMetricReader(batchGeneratorExporter) + { + Temporality = AggregationTemporality.Delta, + }; + + this.meterProviderForCounterBatchWith4Dimensions = Sdk.CreateMeterProviderBuilder() + .AddMeter("GenerateCounterBatchWith4Dimensions") + .AddReader(batchGeneratorReader) + .Build(); + + var tags = new TagList + { + { "DimName1", dimensionValues[this.random.Value.Next(0, 2)] }, + { "DimName2", dimensionValues[this.random.Value.Next(0, 5)] }, + { "DimName3", dimensionValues[this.random.Value.Next(0, 10)] }, + { "DimName4", dimensionValues[this.random.Value.Next(0, 10)] }, + }; + + counter.Add(100, tags); + + this.meterProviderForCounterBatchWith4Dimensions.ForceFlush(); + return batchGeneratorExporter.Batch; + } + + private MetricPoint GenerateHistogramMetricItemWith3Dimensions(out MetricData sum, out uint count) + { + using var meterWithInMemoryExporter = new Meter("GenerateHistogramMetricItemWith3Dimensions", "0.0.1"); + var histogram = meterWithInMemoryExporter.CreateHistogram("HistogramWith3Dimensions"); + + var exportedItems = new List(); + using var inMemoryReader = new BaseExportingMetricReader(new InMemoryExporter(exportedItems)) + { + Temporality = AggregationTemporality.Delta, + }; + + using var meterProvider = Sdk.CreateMeterProviderBuilder() + .AddMeter("GenerateHistogramMetricItemWith3Dimensions") + .AddReader(inMemoryReader) + .Build(); + + var tag1 = new KeyValuePair("DimName1", dimensionValues[this.random.Value.Next(0, 10)]); + var tag2 = new KeyValuePair("DimName2", dimensionValues[this.random.Value.Next(0, 10)]); + var tag3 = new KeyValuePair("DimName3", dimensionValues[this.random.Value.Next(0, 10)]); + + for (int i = 0; i < 1000; i++) + { + histogram.Record(randomForHistogram.Next(1, 1000), tag1, tag2, tag3); + } + + inMemoryReader.Collect(); + + var metric = exportedItems[0]; + var metricPointsEnumerator = metric.GetMetricPoints().GetEnumerator(); + metricPointsEnumerator.MoveNext(); + var metricPoint = metricPointsEnumerator.Current; + sum = new MetricData { UInt64Value = Convert.ToUInt64(metricPoint.GetHistogramSum()) }; + count = Convert.ToUInt32(metricPoint.GetHistogramCount()); + + return metricPoint; + } + + private MetricPoint GenerateHistogramMetricItemWith4Dimensions(out MetricData sum, out uint count) + { + using var meterWithInMemoryExporter = new Meter("GenerateHistogramMetricItemWith4Dimensions", "0.0.1"); + var histogram = meterWithInMemoryExporter.CreateHistogram("HistogramWith4Dimensions"); + + var exportedItems = new List(); + using var inMemoryReader = new BaseExportingMetricReader(new InMemoryExporter(exportedItems)) + { + Temporality = AggregationTemporality.Delta, + }; + + using var meterProvider = Sdk.CreateMeterProviderBuilder() + .AddMeter("GenerateHistogramMetricItemWith4Dimensions") + .AddReader(inMemoryReader) + .Build(); + + var tags = new TagList + { + { "DimName1", dimensionValues[this.random.Value.Next(0, 2)] }, + { "DimName2", dimensionValues[this.random.Value.Next(0, 5)] }, + { "DimName3", dimensionValues[this.random.Value.Next(0, 10)] }, + { "DimName4", dimensionValues[this.random.Value.Next(0, 10)] }, + }; + + for (int i = 0; i < 1000; i++) + { + histogram.Record(randomForHistogram.Next(1, 1000), tags); + } + + inMemoryReader.Collect(); + + var metric = exportedItems[0]; + var metricPointsEnumerator = metric.GetMetricPoints().GetEnumerator(); + metricPointsEnumerator.MoveNext(); + var metricPoint = metricPointsEnumerator.Current; + sum = new MetricData { UInt64Value = Convert.ToUInt64(metricPoint.GetHistogramSum()) }; + count = Convert.ToUInt32(metricPoint.GetHistogramCount()); + + return metricPoint; + } + + private Batch GenerateHistogramBatchWith3Dimensions() + { + using var meterWithInMemoryExporter = new Meter("GenerateHistogramBatchWith3Dimensions", "0.0.1"); + var histogram = meterWithInMemoryExporter.CreateHistogram("HistogramWith3Dimensions"); + + var batchGeneratorExporter = new BatchGenerator(); + var batchGeneratorReader = new BaseExportingMetricReader(batchGeneratorExporter) + { + Temporality = AggregationTemporality.Delta, + }; + + this.meterProviderForHistogramBatchWith3Dimensions = Sdk.CreateMeterProviderBuilder() + .AddMeter("GenerateHistogramBatchWith3Dimensions") + .AddReader(batchGeneratorReader) + .Build(); + + var tag1 = new KeyValuePair("DimName1", dimensionValues[this.random.Value.Next(0, 10)]); + var tag2 = new KeyValuePair("DimName2", dimensionValues[this.random.Value.Next(0, 10)]); + var tag3 = new KeyValuePair("DimName3", dimensionValues[this.random.Value.Next(0, 10)]); + + for (int i = 0; i < 1000; i++) + { + histogram.Record(randomForHistogram.Next(1, 1000), tag1, tag2, tag3); + } + + this.meterProviderForHistogramBatchWith3Dimensions.ForceFlush(); + return batchGeneratorExporter.Batch; + } + + private Batch GenerateHistogramBatchWith4Dimensions() + { + using var meterWithInMemoryExporter = new Meter("GenerateHistogramBatchWith4Dimensions", "0.0.1"); + var histogram = meterWithInMemoryExporter.CreateHistogram("HistogramWith4Dimensions"); + + var batchGeneratorExporter = new BatchGenerator(); + var batchGeneratorReader = new BaseExportingMetricReader(batchGeneratorExporter) + { + Temporality = AggregationTemporality.Delta, + }; + + this.meterProviderForHistogramBatchWith4Dimensions = Sdk.CreateMeterProviderBuilder() + .AddMeter("GenerateHistogramBatchWith4Dimensions") + .AddReader(batchGeneratorReader) + .Build(); + + var tags = new TagList + { + { "DimName1", dimensionValues[this.random.Value.Next(0, 2)] }, + { "DimName2", dimensionValues[this.random.Value.Next(0, 5)] }, + { "DimName3", dimensionValues[this.random.Value.Next(0, 10)] }, + { "DimName4", dimensionValues[this.random.Value.Next(0, 10)] }, + }; + + for (int i = 0; i < 1000; i++) + { + histogram.Record(randomForHistogram.Next(1, 1000), tags); + } + + this.meterProviderForHistogramBatchWith4Dimensions.ForceFlush(); + return batchGeneratorExporter.Batch; + } + + [GlobalCleanup] + public void Cleanup() + { + this.meterWithNoListener?.Dispose(); + this.meterWithListener?.Dispose(); + this.meterWithDummyReader?.Dispose(); + this.meterWithGenevaMetricExporter?.Dispose(); + this.listener?.Dispose(); + this.meterProviderWithDummyReader?.Dispose(); + this.meterProviderWithGenevaMetricExporter?.Dispose(); + this.meterProviderForCounterBatchWith3Dimensions?.Dispose(); + this.meterProviderForCounterBatchWith4Dimensions?.Dispose(); + this.meterProviderForHistogramBatchWith3Dimensions?.Dispose(); + this.meterProviderForHistogramBatchWith4Dimensions?.Dispose(); + this.exporter?.Dispose(); + } + + [Benchmark] + public void InstrumentWithNoListener3Dimensions() + { + var tag1 = new KeyValuePair("DimName1", dimensionValues[this.random.Value.Next(0, 10)]); + var tag2 = new KeyValuePair("DimName2", dimensionValues[this.random.Value.Next(0, 10)]); + var tag3 = new KeyValuePair("DimName3", dimensionValues[this.random.Value.Next(0, 10)]); + this.counterWithNoListener?.Add(100, tag1, tag2, tag3); + } + + [Benchmark] + public void InstrumentWithNoListener4Dimensions() + { + var tags = new TagList + { + { "DimName1", dimensionValues[this.random.Value.Next(0, 2)] }, + { "DimName2", dimensionValues[this.random.Value.Next(0, 5)] }, + { "DimName3", dimensionValues[this.random.Value.Next(0, 10)] }, + { "DimName4", dimensionValues[this.random.Value.Next(0, 10)] }, + }; + + // 2 * 5 * 10 * 10 = 1000 time series max. + this.counterWithNoListener?.Add(100, tags); + } + + [Benchmark] + public void InstrumentWithWithListener3Dimensions() + { + var tag1 = new KeyValuePair("DimName1", dimensionValues[this.random.Value.Next(0, 10)]); + var tag2 = new KeyValuePair("DimName2", dimensionValues[this.random.Value.Next(0, 10)]); + var tag3 = new KeyValuePair("DimName3", dimensionValues[this.random.Value.Next(0, 10)]); + this.counterWithListener?.Add(100, tag1, tag2, tag3); + } + + [Benchmark] + public void InstrumentWithWithListener4Dimensions() + { + var tags = new TagList + { + { "DimName1", dimensionValues[this.random.Value.Next(0, 2)] }, + { "DimName2", dimensionValues[this.random.Value.Next(0, 5)] }, + { "DimName3", dimensionValues[this.random.Value.Next(0, 10)] }, + { "DimName4", dimensionValues[this.random.Value.Next(0, 10)] }, + }; + + // 2 * 5 * 10 * 10 = 1000 time series max. + this.counterWithListener?.Add(100, tags); + } + + [Benchmark] + public void InstrumentWithWithDummyReader3Dimensions() + { + var tag1 = new KeyValuePair("DimName1", dimensionValues[this.random.Value.Next(0, 10)]); + var tag2 = new KeyValuePair("DimName2", dimensionValues[this.random.Value.Next(0, 10)]); + var tag3 = new KeyValuePair("DimName3", dimensionValues[this.random.Value.Next(0, 10)]); + this.counterWithDummyReader?.Add(100, tag1, tag2, tag3); + } + + [Benchmark] + public void InstrumentWithWithDummyReader4Dimensions() + { + var tags = new TagList + { + { "DimName1", dimensionValues[this.random.Value.Next(0, 2)] }, + { "DimName2", dimensionValues[this.random.Value.Next(0, 5)] }, + { "DimName3", dimensionValues[this.random.Value.Next(0, 10)] }, + { "DimName4", dimensionValues[this.random.Value.Next(0, 10)] }, + }; + + // 2 * 5 * 10 * 10 = 1000 time series max. + this.counterWithDummyReader?.Add(100, tags); + } + + [Benchmark] + public void InstrumentWithWithGenevaCounterMetricExporter3Dimensions() + { + var tag1 = new KeyValuePair("DimName1", dimensionValues[this.random.Value.Next(0, 10)]); + var tag2 = new KeyValuePair("DimName2", dimensionValues[this.random.Value.Next(0, 10)]); + var tag3 = new KeyValuePair("DimName3", dimensionValues[this.random.Value.Next(0, 10)]); + this.counterWithGenevaMetricExporter?.Add(100, tag1, tag2, tag3); + } + + [Benchmark] + public void InstrumentWithWithGenevaCounterMetricExporter4Dimensions() + { + var tags = new TagList + { + { "DimName1", dimensionValues[this.random.Value.Next(0, 2)] }, + { "DimName2", dimensionValues[this.random.Value.Next(0, 5)] }, + { "DimName3", dimensionValues[this.random.Value.Next(0, 10)] }, + { "DimName4", dimensionValues[this.random.Value.Next(0, 10)] }, + }; + + // 2 * 5 * 10 * 10 = 1000 time series max. + this.counterWithGenevaMetricExporter?.Add(100, tags); + } + + [Benchmark] + public void SerializeCounterMetricItemWith3Dimensions() + { + this.exporter.SerializeMetric( + MetricEventType.ULongMetric, + this.counterMetricWith3Dimensions.Name, + this.counterMetricPointWith3Dimensions.EndTime.ToFileTime(), + this.counterMetricPointWith3Dimensions.Tags, + this.counterMetricDataWith3Dimensions); + } + + [Benchmark] + public void SerializeCounterMetricItemWith4Dimensions() + { + this.exporter.SerializeMetric( + MetricEventType.ULongMetric, + this.counterMetricWith4Dimensions.Name, + this.counterMetricPointWith4Dimensions.EndTime.ToFileTime(), + this.counterMetricPointWith4Dimensions.Tags, + this.counterMetricDataWith4Dimensions); + } + + [Benchmark] + public void ExportCounterMetricItemWith3Dimensions() + { + this.exporter.Export(this.counterMetricBatchWith3Dimensions); + } + + [Benchmark] + public void ExportCounterMetricItemWith4Dimensions() + { + this.exporter.Export(this.counterMetricBatchWith4Dimensions); + } + + [Benchmark] + public void SerializeHistogramMetricItemWith3Dimensions() + { + this.exporter.SerializeHistogramMetric( + this.histogramMetricWith3Dimensions.Name, + this.histogramMetricPointWith3Dimensions.EndTime.ToFileTime(), + this.histogramMetricPointWith3Dimensions.Tags, + this.histogramMetricPointWith3Dimensions.GetHistogramBuckets(), + this.histogramSumWith3Dimensions, + this.histogramCountWith3Dimensions); + } + + [Benchmark] + public void SerializeHistogramMetricItemWith4Dimensions() + { + this.exporter.SerializeHistogramMetric( + this.histogramMetricWith4Dimensions.Name, + this.histogramMetricPointWith4Dimensions.EndTime.ToFileTime(), + this.histogramMetricPointWith4Dimensions.Tags, + this.histogramMetricPointWith4Dimensions.GetHistogramBuckets(), + this.histogramSumWith4Dimensions, + this.histogramCountWith4Dimensions); + } + + [Benchmark] + public void ExportHistogramMetricItemWith3Dimensions() + { + this.exporter.Export(this.histogramMetricBatchWith3Dimensions); + } + + [Benchmark] + public void ExportHistogramMetricItemWith4Dimensions() + { + this.exporter.Export(this.histogramMetricBatchWith4Dimensions); + } + + private class DummyReader : BaseExportingMetricReader + { + public DummyReader(BaseExporter exporter) + : base(exporter) + { + } + } + + private class DummyMetricExporter : BaseExporter + { + public override ExportResult Export(in Batch batch) + { + return ExportResult.Success; + } + } + + private class BatchGenerator : BaseExporter + { + public Batch Batch { get; set; } + + public override ExportResult Export(in Batch batch) + { + this.Batch = batch; + return ExportResult.Success; + } + } + } +} diff --git a/test/OpenTelemetry.Exporter.Geneva.Benchmark/Exporter/TraceExporterBenchmarks.cs b/test/OpenTelemetry.Exporter.Geneva.Benchmark/Exporter/TraceExporterBenchmarks.cs new file mode 100644 index 00000000000..3c702182f52 --- /dev/null +++ b/test/OpenTelemetry.Exporter.Geneva.Benchmark/Exporter/TraceExporterBenchmarks.cs @@ -0,0 +1,110 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using BenchmarkDotNet.Attributes; +using OpenTelemetry.Metrics; +using OpenTelemetry.Trace; + +/* +BenchmarkDotNet=v0.13.1, OS=Windows 10.0.19044.1415 (21H2) +AMD Ryzen 9 3900X, 1 CPU, 24 logical and 12 physical cores +.NET SDK=6.0.101 + [Host] : .NET 5.0.12 (5.0.1221.52207), X64 RyuJIT + DefaultJob : .NET 5.0.12 (5.0.1221.52207), X64 RyuJIT + +| Method | Mean | Error | StdDev | Gen 0 | Allocated | +|-------------------------- |----------:|---------:|---------:|-------:|----------:| +| CreateBoringActivity | 16.05 ns | 0.053 ns | 0.049 ns | - | - | +| CreateTediousActivity | 509.26 ns | 2.105 ns | 1.969 ns | 0.0486 | 408 B | +| CreateInterestingActivity | 959.87 ns | 6.014 ns | 5.625 ns | 0.0477 | 408 B | +| SerializeActivity | 506.55 ns | 3.862 ns | 3.612 ns | 0.0095 | 80 B | +*/ + +namespace OpenTelemetry.Exporter.Geneva.Benchmark +{ + [MemoryDiagnoser] + public class TraceExporterBenchmarks + { + private readonly Random r = new Random(); + private readonly Activity activity; + private readonly GenevaTraceExporter exporter; + private readonly ActivitySource sourceBoring = new ActivitySource("OpenTelemetry.Exporter.Geneva.Benchmark.Boring"); + private readonly ActivitySource sourceTedious = new ActivitySource("OpenTelemetry.Exporter.Geneva.Benchmark.Tedious"); + private readonly ActivitySource sourceInteresting = new ActivitySource("OpenTelemetry.Exporter.Geneva.Benchmark.Interesting"); + + public TraceExporterBenchmarks() + { + Activity.DefaultIdFormat = ActivityIdFormat.W3C; + + ActivitySource.AddActivityListener(new ActivityListener + { + ActivityStarted = null, + ActivityStopped = null, + ShouldListenTo = (activitySource) => activitySource.Name == this.sourceTedious.Name, + Sample = (ref ActivityCreationOptions options) => ActivitySamplingResult.AllDataAndRecorded, + }); + + using (var tedious = this.sourceTedious.StartActivity("Benchmark")) + { + this.activity = tedious; + this.activity?.SetTag("tagString", "value"); + this.activity?.SetTag("tagInt", 100); + this.activity?.SetStatus(Status.Error); + } + + this.exporter = new GenevaTraceExporter(new GenevaExporterOptions + { + ConnectionString = "EtwSession=OpenTelemetry", + CustomFields = new List { "azureResourceProvider", "clientRequestId" }, + PrepopulatedFields = new Dictionary + { + ["cloud.role"] = "BusyWorker", + ["cloud.roleInstance"] = "CY1SCH030021417", + ["cloud.roleVer"] = "9.0.15289.2", + }, + }); + + Sdk.CreateTracerProviderBuilder() + .SetSampler(new AlwaysOnSampler()) + .AddSource(this.sourceInteresting.Name) + .AddGenevaTraceExporter(options => + { + options.ConnectionString = "EtwSession=OpenTelemetry"; + options.PrepopulatedFields = new Dictionary + { + ["cloud.role"] = "BusyWorker", + ["cloud.roleInstance"] = "CY1SCH030021417", + ["cloud.roleVer"] = "9.0.15289.2", + }; + }) + .Build(); + } + + [Benchmark] + public void CreateBoringActivity() + { + // this activity won't be created as there is no listener + using var activity = this.sourceBoring.StartActivity("Benchmark"); + } + + [Benchmark] + public void CreateTediousActivity() + { + // this activity will be created and feed into an ActivityListener that simply drops everything on the floor + using var activity = this.sourceTedious.StartActivity("Benchmark"); + } + + [Benchmark] + public void CreateInterestingActivity() + { + // this activity will be created and feed into the actual Geneva exporter + using var activity = this.sourceInteresting.StartActivity("Benchmark"); + } + + [Benchmark] + public void SerializeActivity() + { + this.exporter.SerializeActivity(this.activity); + } + } +} diff --git a/test/OpenTelemetry.Exporter.Geneva.Benchmark/OpenTelemetry.Exporter.Geneva.Benchmark.csproj b/test/OpenTelemetry.Exporter.Geneva.Benchmark/OpenTelemetry.Exporter.Geneva.Benchmark.csproj new file mode 100644 index 00000000000..521c7b3e2f2 --- /dev/null +++ b/test/OpenTelemetry.Exporter.Geneva.Benchmark/OpenTelemetry.Exporter.Geneva.Benchmark.csproj @@ -0,0 +1,21 @@ + + + + Exe + netcoreapp3.1;net5.0;net6.0 + $(TargetFrameworks);net461;net462;net47;net471;net472;net48 + $(NoWarn),SA1633,SA1201,SA1202,SA1204,SA1311,SA1123 + + + + + + + + + + + + + + diff --git a/test/OpenTelemetry.Exporter.Geneva.Benchmark/Program.cs b/test/OpenTelemetry.Exporter.Geneva.Benchmark/Program.cs new file mode 100644 index 00000000000..5ec54db965e --- /dev/null +++ b/test/OpenTelemetry.Exporter.Geneva.Benchmark/Program.cs @@ -0,0 +1,9 @@ +using BenchmarkDotNet.Running; + +namespace OpenTelemetry.Exporter.Geneva.Benchmark +{ + internal class Program + { + private static void Main(string[] args) => BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args); + } +} diff --git a/test/OpenTelemetry.Exporter.Geneva.Stress/DummyServer.cs b/test/OpenTelemetry.Exporter.Geneva.Stress/DummyServer.cs new file mode 100644 index 00000000000..4af08fa4bee --- /dev/null +++ b/test/OpenTelemetry.Exporter.Geneva.Stress/DummyServer.cs @@ -0,0 +1,86 @@ +using System; +using System.IO; +using System.Net; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; + +namespace OpenTelemetry.Exporter.Geneva.Stress +{ + internal class DummyServer + { + private EndPoint endpoint; + private Socket serverSocket; + + public DummyServer(string path) + { + Console.WriteLine($"Server socket listening at path: {path}"); + + // Unix sockets must be unlink()ed before being reused again. + // Or there will be System.Net.Sockets.SocketException (98): SocketError.AddressAlreadyInUse + // https://github.com/dotnet/runtime/issues/23803 + // C# doesn't have the unlink() function in C + // Shutdown() and setting SocketOptions like ReuseAddress and Linger doesn't solve the problem as they do for TCP + // https://stackoverflow.com/questions/2821520/how-can-i-unbind-a-socket-in-c + File.Delete(path); + this.endpoint = new UnixDomainSocketEndPoint(path); + this.serverSocket = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); + } + + public void Start() + { + try + { + this.serverSocket.Bind(this.endpoint); + this.serverSocket.Listen(20); + + Console.CancelKeyPress += (object sender, ConsoleCancelEventArgs args) => + { + Console.WriteLine("Program is terminating."); + this.serverSocket.Close(); + }; + + while (true) + { + Socket acceptSocket = this.serverSocket.Accept(); + Task.Run(() => + { + int threadId = Thread.CurrentThread.ManagedThreadId; + Console.WriteLine($"ThreadID {threadId}: Start reading from socket."); + int totalBytes = 0; + try + { + while (acceptSocket.Connected) + { + var receivedData = new byte[1024]; + int receivedDataSize = acceptSocket.Receive(receivedData); + totalBytes += receivedDataSize; + } + + acceptSocket.Shutdown(SocketShutdown.Both); + } + catch (Exception e) + { + Console.WriteLine($"acceptSocket exception: {e}"); + } + finally + { + Console.WriteLine($"ThreadID {threadId}: Closing socket"); + acceptSocket.Close(); + } + + Console.WriteLine($"ThreadID {threadId}: Socket received {totalBytes} bytes in total."); + }); + } + } + catch (Exception e) + { + Console.WriteLine($"Server socket exception: {e}"); + } + finally + { + this.serverSocket.Close(); + } + } + } +} diff --git a/test/OpenTelemetry.Exporter.Geneva.Stress/OpenTelemetry.Exporter.Geneva.Stress.csproj b/test/OpenTelemetry.Exporter.Geneva.Stress/OpenTelemetry.Exporter.Geneva.Stress.csproj new file mode 100644 index 00000000000..41d0d5fb883 --- /dev/null +++ b/test/OpenTelemetry.Exporter.Geneva.Stress/OpenTelemetry.Exporter.Geneva.Stress.csproj @@ -0,0 +1,18 @@ + + + + Exe + netcoreapp3.1;net5.0;net6.0 + $(TargetFrameworks);net461;net462;net47;net471;net472;net48 + $(NoWarn),SA1633,SA1308,SA1201 + + + + + + + + + + + diff --git a/test/OpenTelemetry.Exporter.Geneva.Stress/Program.cs b/test/OpenTelemetry.Exporter.Geneva.Stress/Program.cs new file mode 100644 index 00000000000..ee3e337ed3d --- /dev/null +++ b/test/OpenTelemetry.Exporter.Geneva.Stress/Program.cs @@ -0,0 +1,187 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using CommandLine; +using OpenTelemetry.Trace; + +namespace OpenTelemetry.Exporter.Geneva.Stress +{ + internal class Program + { + private static volatile bool s_bContinue = true; + private static long s_nEvents = 0; + + private static ActivitySource source = new ActivitySource("OpenTelemetry.Exporter.Geneva.Stress"); + + private static int Main(string[] args) + { + return Parser.Default.ParseArguments(args) + .MapResult( + (WindowsOptions options) => EntryPoint(InitTraces, RunTraces), + (LinuxOptions options) => RunLinux(options), + (ServerOptions options) => RunServer(options), + (ExporterCreationOptions options) => RunExporterCreation(), + errs => 1); + + // return EntryPoint(InitMetrics, RunMetrics); + } + + [Verb("Windows", HelpText = "Run stress test on Windows.")] + private class WindowsOptions + { + } + + [Verb("Linux", HelpText = "Run stress test on Linux.")] + private class LinuxOptions + { + [Option('p', "path", Default = "/var/run/default_fluent.socket", HelpText = "Specify a path for Unix domain socket.")] + public string Path { get; set; } + } + + [Verb("server", HelpText = "Start a dummy server on Linux.")] + private class ServerOptions + { + [Option('p', "path", HelpText = "Specify a path for Unix domain socket.", Required = true)] + public string Path { get; set; } + } + + [Verb("ExporterCreation", HelpText = "Validate exporter dispose behavior")] + private class ExporterCreationOptions + { + } + + private static int RunExporterCreation() + { + var options = new GenevaExporterOptions() + { + ConnectionString = "EtwSession=OpenTelemetry", + PrepopulatedFields = new Dictionary + { + ["ver"] = "4.0", + ["cloud.role"] = "BusyWorker", + ["cloud.roleInstance"] = "CY1SCH030021417", + ["cloud.roleVer"] = "9.0.15289.2", + }, + }; + + for (var i = 0; i < 300000; ++i) + { + using var dataTransport = new EtwDataTransport("OpenTelemetry"); + } + + return 0; + } + + private static int RunLinux(LinuxOptions options) + { + return EntryPoint(() => InitTracesOnLinux(options.Path), RunTraces); + } + + private static int RunServer(ServerOptions options) + { + var server = new DummyServer(options.Path); + server.Start(); + return 0; + } + + private static int EntryPoint(Action init, Action run) + { + init(); + + var statistics = new long[Environment.ProcessorCount]; + Parallel.Invoke( + () => + { + Console.WriteLine("Running, press to stop..."); + var watch = new Stopwatch(); + while (true) + { + if (Console.KeyAvailable) + { + var key = Console.ReadKey(true).Key; + switch (key) + { + case ConsoleKey.Escape: + s_bContinue = false; + return; + } + + continue; + } + + s_nEvents = statistics.Sum(); + watch.Restart(); + Thread.Sleep(200); + watch.Stop(); + var nEvents = statistics.Sum(); + var nEventPerSecond = (int)((nEvents - s_nEvents) / (watch.ElapsedMilliseconds / 1000.0)); + Console.Title = string.Format("Loops: {0:n0}, Loops/Second: {1:n0}", nEvents, nEventPerSecond); + } + }, + () => + { + Parallel.For(0, statistics.Length, (i) => + { + statistics[i] = 0; + while (s_bContinue) + { + run(); + statistics[i]++; + } + }); + }); + return 0; + } + + private static void InitTraces() + { + Sdk.CreateTracerProviderBuilder() + .SetSampler(new AlwaysOnSampler()) + .AddSource("OpenTelemetry.Exporter.Geneva.Stress") + .AddGenevaTraceExporter(options => + { + options.ConnectionString = "EtwSession=OpenTelemetry"; + options.PrepopulatedFields = new Dictionary + { + ["cloud.role"] = "BusyWorker", + ["cloud.roleInstance"] = "CY1SCH030021417", + ["cloud.roleVer"] = "9.0.15289.2", + }; + }) + .Build(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void RunTraces() + { + using (var activity = source.StartActivity("Stress")) + { + activity?.SetTag("http.method", "GET"); + activity?.SetTag("http.url", "https://www.wikipedia.org/wiki/Rabbit"); + activity?.SetTag("http.status_code", 200); + } + } + + private static void InitTracesOnLinux(string path) + { + Sdk.CreateTracerProviderBuilder() + .SetSampler(new AlwaysOnSampler()) + .AddSource("OpenTelemetry.Exporter.Geneva.Stress") + .AddGenevaTraceExporter(options => + { + options.ConnectionString = "Endpoint=unix:" + path; + options.PrepopulatedFields = new Dictionary + { + ["cloud.role"] = "BusyWorker", + ["cloud.roleInstance"] = "CY1SCH030021417", + ["cloud.roleVer"] = "9.0.15289.2", + }; + }) + .Build(); + } + } +} diff --git a/test/OpenTelemetry.Exporter.Geneva.UnitTest/ConnectionStringBuilderTests.cs b/test/OpenTelemetry.Exporter.Geneva.UnitTest/ConnectionStringBuilderTests.cs new file mode 100644 index 00000000000..6e9f374a8c1 --- /dev/null +++ b/test/OpenTelemetry.Exporter.Geneva.UnitTest/ConnectionStringBuilderTests.cs @@ -0,0 +1,228 @@ +using System; +using Xunit; + +namespace OpenTelemetry.Exporter.Geneva.UnitTest +{ + public class ConnectionStringBuilderTests + { + [Fact] + [Trait("Platform", "Any")] + public void ConnectionStringBuilder_constructor_Invalid_Input() + { + // null connection string + Assert.Throws(() => _ = new ConnectionStringBuilder(null)); + + // empty connection string + Assert.Throws(() => _ = new ConnectionStringBuilder(string.Empty)); + Assert.Throws(() => _ = new ConnectionStringBuilder(" ")); + + // empty key + Assert.Throws(() => _ = new ConnectionStringBuilder("=value")); + Assert.Throws(() => _ = new ConnectionStringBuilder("=value1;key2=value2")); + Assert.Throws(() => _ = new ConnectionStringBuilder("key1=value1;=value2")); + + // empty value + Assert.Throws(() => _ = new ConnectionStringBuilder("key=")); + Assert.Throws(() => _ = new ConnectionStringBuilder("key1=;key2=value2")); + Assert.Throws(() => _ = new ConnectionStringBuilder("key1=value1;key2=")); + + // invalid format + Assert.Throws(() => _ = new ConnectionStringBuilder("key;value")); + Assert.Throws(() => _ = new ConnectionStringBuilder("key==value")); + } + + [Fact] + [Trait("Platform", "Any")] + public void ConnectionStringBuilder_constructor_Duplicated_Keys() + { + var builder = new ConnectionStringBuilder("Account=value1;Account=VALUE2"); + Assert.Equal("VALUE2", builder.Account); + } + + [Fact] + [Trait("Platform", "Any")] + public void ConnectionStringBuilder_Protocol_No_Default_Value() + { + var builder = new ConnectionStringBuilder("key1=value1"); + Assert.Equal(TransportProtocol.Unspecified, builder.Protocol); + + builder = new ConnectionStringBuilder("EtwSession=OpenTelemetry"); + Assert.Equal(TransportProtocol.Etw, builder.Protocol); + + builder = new ConnectionStringBuilder("Endpoint=udp://localhost:11013"); + Assert.Equal(TransportProtocol.Udp, builder.Protocol); + + builder = new ConnectionStringBuilder("Endpoint=tcp://localhost:11013"); + Assert.Equal(TransportProtocol.Tcp, builder.Protocol); + + builder = new ConnectionStringBuilder("Endpoint=foo://localhost:11013"); + Assert.Throws(() => _ = builder.Protocol); + } + + [Fact] + [Trait("Platform", "Any")] + public void ConnectionStringBuilder_EtwSession() + { + var builder = new ConnectionStringBuilder("EtwSession=OpenTelemetry"); + Assert.Equal(TransportProtocol.Etw, builder.Protocol); + Assert.Equal("OpenTelemetry", builder.EtwSession); + Assert.Throws(() => _ = builder.Host); + Assert.Throws(() => _ = builder.Port); + + builder = new ConnectionStringBuilder("Endpoint=udp://localhost:11013"); + Assert.Equal(TransportProtocol.Udp, builder.Protocol); + Assert.Throws(() => _ = builder.EtwSession); + } + + [Fact] + [Trait("Platform", "Any")] + public void ConnectionStringBuilder_Endpoint_UnixDomainSocketPath() + { + var builder = new ConnectionStringBuilder("Endpoint=unix:/var/run/default_fluent.socket"); + Assert.Equal("unix:/var/run/default_fluent.socket", builder.Endpoint); + Assert.Equal(TransportProtocol.Unix, builder.Protocol); + Assert.Equal("/var/run/default_fluent.socket", builder.ParseUnixDomainSocketPath()); + + builder = new ConnectionStringBuilder("Endpoint=unix:///var/run/default_fluent.socket"); + Assert.Equal("unix:///var/run/default_fluent.socket", builder.Endpoint); + Assert.Equal(TransportProtocol.Unix, builder.Protocol); + Assert.Equal("/var/run/default_fluent.socket", builder.ParseUnixDomainSocketPath()); + + builder = new ConnectionStringBuilder("Endpoint=unix://:11111"); + Assert.Throws(() => _ = builder.ParseUnixDomainSocketPath()); + + builder = new ConnectionStringBuilder("EtwSession=OpenTelemetry"); + Assert.Throws(() => _ = builder.ParseUnixDomainSocketPath()); + } + + [Fact] + [Trait("Platform", "Any")] + public void ConnectionStringBuilder_TimeoutMilliseconds() + { + var builder = new ConnectionStringBuilder("TimeoutMilliseconds=10000"); + Assert.Equal(10000, builder.TimeoutMilliseconds); + + builder.TimeoutMilliseconds = 6000; + Assert.Equal(6000, builder.TimeoutMilliseconds); + + builder = new ConnectionStringBuilder("Endpoint=unix:/var/run/default_fluent.socket"); + Assert.Equal(UnixDomainSocketDataTransport.DefaultTimeoutMilliseconds, builder.TimeoutMilliseconds); + + builder = new ConnectionStringBuilder("TimeoutMilliseconds=0"); + Assert.Throws(() => _ = builder.TimeoutMilliseconds); + + builder = new ConnectionStringBuilder("TimeoutMilliseconds=-1"); + Assert.Throws(() => _ = builder.TimeoutMilliseconds); + + builder = new ConnectionStringBuilder("TimeoutMilliseconds=-2"); + Assert.Throws(() => _ = builder.TimeoutMilliseconds); + + builder = new ConnectionStringBuilder("TimeoutMilliseconds=10.5"); + Assert.Throws(() => _ = builder.TimeoutMilliseconds); + + builder = new ConnectionStringBuilder("TimeoutMilliseconds=abc"); + Assert.Throws(() => _ = builder.TimeoutMilliseconds); + } + + [Fact] + [Trait("Platform", "Any")] + public void ConnectionStringBuilder_Endpoint_Udp() + { + var builder = new ConnectionStringBuilder("Endpoint=udp://localhost:11111"); + Assert.Equal("udp://localhost:11111", builder.Endpoint); + Assert.Equal(TransportProtocol.Udp, builder.Protocol); + Assert.Equal("localhost", builder.Host); + Assert.Equal(11111, builder.Port); + + builder = new ConnectionStringBuilder("Endpoint=Udp://localhost:11111"); + Assert.Equal(TransportProtocol.Udp, builder.Protocol); + + builder = new ConnectionStringBuilder("Endpoint=UDP://localhost:11111"); + Assert.Equal(TransportProtocol.Udp, builder.Protocol); + + builder = new ConnectionStringBuilder("Endpoint=udp://localhost"); + Assert.Equal(TransportProtocol.Udp, builder.Protocol); + Assert.Equal("localhost", builder.Host); + Assert.Throws(() => _ = builder.Port); + + builder = new ConnectionStringBuilder("Endpoint=udp://:11111"); + Assert.Throws(() => _ = builder.Protocol); + Assert.Throws(() => _ = builder.Host); + Assert.Throws(() => _ = builder.Port); + } + + [Fact] + [Trait("Platform", "Any")] + public void ConnectionStringBuilder_Endpoint_Tcp() + { + var builder = new ConnectionStringBuilder("Endpoint=tcp://localhost:33333"); + Assert.Equal("tcp://localhost:33333", builder.Endpoint); + Assert.Equal(TransportProtocol.Tcp, builder.Protocol); + Assert.Equal("localhost", builder.Host); + Assert.Equal(33333, builder.Port); + + builder = new ConnectionStringBuilder("Endpoint=Tcp://localhost:11111"); + Assert.Equal(TransportProtocol.Tcp, builder.Protocol); + + builder = new ConnectionStringBuilder("Endpoint=TCP://localhost:11111"); + Assert.Equal(TransportProtocol.Tcp, builder.Protocol); + + builder = new ConnectionStringBuilder("Endpoint=tcp://localhost"); + Assert.Equal(TransportProtocol.Tcp, builder.Protocol); + Assert.Equal("localhost", builder.Host); + Assert.Throws(() => _ = builder.Port); + + builder = new ConnectionStringBuilder("Endpoint=tpc://:11111"); + Assert.Throws(() => _ = builder.Protocol); + Assert.Throws(() => _ = builder.Host); + Assert.Throws(() => _ = builder.Port); + } + + [Fact] + [Trait("Platform", "Any")] + public void ConnectionStringBuilder_EtwSession_Endpoint_Both_Set() + { + var builder = new ConnectionStringBuilder("Endpoint=tcp://localhost:33333;EtwSession=OpenTelemetry"); + Assert.Equal(TransportProtocol.Etw, builder.Protocol); + + Assert.Equal("OpenTelemetry", builder.EtwSession); + + Assert.Equal("tcp://localhost:33333", builder.Endpoint); + Assert.Equal("localhost", builder.Host); + Assert.Equal(33333, builder.Port); + } + + [Fact] + [Trait("Platform", "Any")] + public void ConnectionStringBuilder_MonitoringAccount_No_Default_Value() + { + var builder = new ConnectionStringBuilder("key1=value1"); + Assert.Throws(() => _ = builder.Account); + + builder.Account = "TestAccount"; + Assert.Equal("TestAccount", builder.Account); + + builder = new ConnectionStringBuilder("Account=TestAccount"); + Assert.Equal("TestAccount", builder.Account); + } + + [Fact] + [Trait("Platform", "Any")] + public void ConnectionStringBuilder_Keywords_Are_Case_Sensitive() + { + var builder = new ConnectionStringBuilder("etwSession=OpenTelemetry"); + Assert.Throws(() => builder.EtwSession); + Assert.Equal(TransportProtocol.Unspecified, builder.Protocol); + + builder = new ConnectionStringBuilder("endpoint=tcp://localhost:33333"); + Assert.Throws(() => builder.Endpoint); + Assert.Equal(TransportProtocol.Unspecified, builder.Protocol); + Assert.Throws(() => builder.Host); + Assert.Throws(() => builder.Port); + + builder = new ConnectionStringBuilder("monitoringAccount=TestAccount"); + Assert.Throws(() => builder.Account); + Assert.Equal(TransportProtocol.Unspecified, builder.Protocol); + } + } +} diff --git a/test/OpenTelemetry.Exporter.Geneva.UnitTest/GenevaLogExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.UnitTest/GenevaLogExporterTests.cs new file mode 100644 index 00000000000..b6f59ca6fce --- /dev/null +++ b/test/OpenTelemetry.Exporter.Geneva.UnitTest/GenevaLogExporterTests.cs @@ -0,0 +1,768 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Net.Sockets; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading; +using Microsoft.Extensions.Logging; +using OpenTelemetry.Logs; +using Xunit; + +namespace OpenTelemetry.Exporter.Geneva.UnitTest +{ + public class GenevaLogExporterTests + { + [Fact] + [Trait("Platform", "Any")] + public void BadArgs() + { + GenevaExporterOptions exporterOptions = null; + Assert.Throws(() => + { + using var exporter = new GenevaLogExporter(exporterOptions); + }); + } + + [Fact] + [Trait("Platform", "Any")] + public void SpecialChractersInTableNameMappings() + { + Assert.Throws(() => + { + using var exporter = new GenevaLogExporter(new GenevaExporterOptions + { + TableNameMappings = new Dictionary { ["TestCategory"] = "\u0418" }, + }); + }); + + Assert.Throws(() => + { + using var exporter = new GenevaLogExporter(new GenevaExporterOptions + { + TableNameMappings = new Dictionary { ["*"] = "\u0418" }, + }); + }); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + [Trait("Platform", "Any")] + public void InvalidConnectionString(string connectionString) + { + var exporterOptions = new GenevaExporterOptions() { ConnectionString = connectionString }; + var exception = Assert.Throws(() => + { + using var exporter = new GenevaLogExporter(exporterOptions); + }); + Assert.Equal($"{nameof(exporterOptions.ConnectionString)} is invalid.", exception.Message); + } + + [Fact] + [Trait("Platform", "Windows")] + public void IncompatibleConnectionStringOnWindows() + { + var exporterOptions = new GenevaExporterOptions() { ConnectionString = "Endpoint=unix:" + @"C:\Users\user\AppData\Local\Temp\14tj4ac4.v2q" }; + var exception = Assert.Throws(() => + { + using var exporter = new GenevaLogExporter(exporterOptions); + }); + Assert.Equal("Unix domain socket should not be used on Windows.", exception.Message); + } + + [Fact] + [Trait("Platform", "Linux")] + public void IncompatibleConnectionStringOnLinux() + { + var exporterOptions = new GenevaExporterOptions() { ConnectionString = "EtwSession=OpenTelemetry" }; + var exception = Assert.Throws(() => + { + using var exporter = new GenevaLogExporter(exporterOptions); + }); + Assert.Equal("ETW cannot be used on non-Windows operating systems.", exception.Message); + } + + [Theory] + [InlineData("categoryA", "TableA")] + [InlineData("categoryB", "TableB")] + [InlineData("categoryA", "TableA", "categoryB", "TableB")] + [InlineData("categoryA", "TableA", "*", "CatchAll")] + [InlineData(null)] + [Trait("Platform", "Any")] + public void TableNameMappingTest(params string[] category) + { + // ARRANGE + string path = string.Empty; + Socket server = null; + var logRecordList = new List(); + Dictionary mappingsDict = null; + try + { + var exporterOptions = new GenevaExporterOptions(); + if (category?.Length > 0) + { + mappingsDict = new Dictionary(); + for (int i = 0; i < category.Length; i = i + 2) + { + mappingsDict.Add(category[i], category[i + 1]); + } + + exporterOptions.TableNameMappings = mappingsDict; + } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + exporterOptions.ConnectionString = "EtwSession=OpenTelemetry"; + } + else + { + path = GenerateTempFilePath(); + exporterOptions.ConnectionString = "Endpoint=unix:" + path; + var endpoint = new UnixDomainSocketEndPoint(path); + server = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); + server.Bind(endpoint); + server.Listen(1); + } + + using var loggerFactory = LoggerFactory.Create(builder => builder + .AddOpenTelemetry(options => + { + options.AddInMemoryExporter(logRecordList); + }) + .AddFilter("*", LogLevel.Trace)); // Enable all LogLevels + + // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. + using var exporter = new GenevaLogExporter(exporterOptions); + + ILogger logger; + ThreadLocal m_buffer; + object fluentdData; + string actualTableName; + string defaultLogTable = "Log"; + if (mappingsDict != null) + { + foreach (var mapping in mappingsDict) + { + if (!mapping.Key.Equals("*")) + { + logger = loggerFactory.CreateLogger(mapping.Key); + logger.LogError("this does not matter"); + + Assert.Single(logRecordList); + m_buffer = typeof(GenevaLogExporter).GetField("m_buffer", BindingFlags.NonPublic | BindingFlags.Static).GetValue(exporter) as ThreadLocal; + _ = exporter.SerializeLogRecord(logRecordList[0]); + fluentdData = MessagePack.MessagePackSerializer.Deserialize(m_buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); + actualTableName = (fluentdData as object[])[0] as string; + Assert.Equal(mapping.Value, actualTableName); + logRecordList.Clear(); + } + else + { + defaultLogTable = mapping.Value; + } + } + + // test default table + logger = loggerFactory.CreateLogger("random category"); + logger.LogError("this does not matter"); + + Assert.Single(logRecordList); + m_buffer = typeof(GenevaLogExporter).GetField("m_buffer", BindingFlags.NonPublic | BindingFlags.Static).GetValue(exporter) as ThreadLocal; + _ = exporter.SerializeLogRecord(logRecordList[0]); + fluentdData = MessagePack.MessagePackSerializer.Deserialize(m_buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); + actualTableName = (fluentdData as object[])[0] as string; + Assert.Equal(defaultLogTable, actualTableName); + logRecordList.Clear(); + } + } + finally + { + server?.Dispose(); + try + { + File.Delete(path); + } + catch + { + } + } + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + [Trait("Platform", "Any")] + public void SerializationTestWithILoggerLogMethod(bool includeFormattedMessage) + { + // Dedicated test for the raw ILogger.Log method + // https://docs.microsoft.com/dotnet/api/microsoft.extensions.logging.ilogger.log + + // ARRANGE + string path = string.Empty; + Socket server = null; + var logRecordList = new List(); + try + { + var exporterOptions = new GenevaExporterOptions(); + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + exporterOptions.ConnectionString = "EtwSession=OpenTelemetry"; + } + else + { + path = GenerateTempFilePath(); + exporterOptions.ConnectionString = "Endpoint=unix:" + path; + var endpoint = new UnixDomainSocketEndPoint(path); + server = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); + server.Bind(endpoint); + server.Listen(1); + } + + using var loggerFactory = LoggerFactory.Create(builder => builder + .AddOpenTelemetry(options => + { + options.AddGenevaLogExporter(options => + { + options.ConnectionString = exporterOptions.ConnectionString; + }); + options.AddInMemoryExporter(logRecordList); + options.IncludeFormattedMessage = includeFormattedMessage; + }) + .AddFilter(typeof(GenevaLogExporterTests).FullName, LogLevel.Trace)); // Enable all LogLevels + + // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. + using var exporter = new GenevaLogExporter(exporterOptions); + + // Emit a LogRecord and grab a copy of the LogRecord from the collection passed to InMemoryExporter + var logger = loggerFactory.CreateLogger(); + + // ACT + // This is treated as structured logging as the state can be converted to IReadOnlyList> + logger.Log( + LogLevel.Information, + default, + new List>() + { + new KeyValuePair("Key1", "Value1"), + new KeyValuePair("Key2", "Value2"), + }, + null, + (state, ex) => "Formatted Message"); + + // VALIDATE + Assert.Single(logRecordList); + var m_buffer = typeof(GenevaLogExporter).GetField("m_buffer", BindingFlags.NonPublic | BindingFlags.Static).GetValue(exporter) as ThreadLocal; + _ = exporter.SerializeLogRecord(logRecordList[0]); + object fluentdData = MessagePack.MessagePackSerializer.Deserialize(m_buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); + var body = GetField(fluentdData, "body"); + string expectedBody = includeFormattedMessage ? "Formatted Message" : null; + Assert.Equal(expectedBody, body); + Assert.Equal("Value1", GetField(fluentdData, "Key1")); + Assert.Equal("Value2", GetField(fluentdData, "Key2")); + + // ARRANGE + logRecordList.Clear(); + + // ACT + // This is treated as Un-structured logging as the state cannot be converted to IReadOnlyList> + logger.Log( + LogLevel.Information, + default, + state: "somestringasdata", + exception: null, + formatter: (state, ex) => "Formatted Message"); + + // VALIDATE + Assert.Single(logRecordList); + m_buffer = typeof(GenevaLogExporter).GetField("m_buffer", BindingFlags.NonPublic | BindingFlags.Static).GetValue(exporter) as ThreadLocal; + _ = exporter.SerializeLogRecord(logRecordList[0]); + fluentdData = MessagePack.MessagePackSerializer.Deserialize(m_buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); + body = GetField(fluentdData, "body"); + expectedBody = includeFormattedMessage ? "Formatted Message" : "somestringasdata"; + Assert.Equal(expectedBody, body); + + // ARRANGE + logRecordList.Clear(); + + // ACT + // This is treated as Un-structured logging as the state cannot be converted to IReadOnlyList> + logger.Log( + LogLevel.Information, + default, + state: "somestringasdata", + exception: null, + formatter: null); + + // VALIDATE + Assert.Single(logRecordList); + m_buffer = typeof(GenevaLogExporter).GetField("m_buffer", BindingFlags.NonPublic | BindingFlags.Static).GetValue(exporter) as ThreadLocal; + _ = exporter.SerializeLogRecord(logRecordList[0]); + fluentdData = MessagePack.MessagePackSerializer.Deserialize(m_buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); + body = GetField(fluentdData, "body"); + + // Formatter is null, hence body is always the ToString() of the data + expectedBody = "somestringasdata"; + Assert.Equal(expectedBody, body); + } + finally + { + server?.Dispose(); + try + { + File.Delete(path); + } + catch + { + } + } + } + + [Theory] + [InlineData(false, false, false)] + [InlineData(false, true, false)] + [InlineData(true, false, false)] + [InlineData(true, true, false)] + [InlineData(false, false, true)] + [InlineData(false, true, true)] + [InlineData(true, false, true)] + [InlineData(true, true, true)] + [Trait("Platform", "Any")] + public void SuccessfulSerialization(bool hasTableNameMapping, bool hasCustomFields, bool parseStateValues) + { + string path = string.Empty; + Socket server = null; + var logRecordList = new List(); + try + { + var exporterOptions = new GenevaExporterOptions + { + PrepopulatedFields = new Dictionary + { + ["cloud.role"] = "BusyWorker", + ["cloud.roleInstance"] = "CY1SCH030021417", + ["cloud.roleVer"] = "9.0.15289.2", + }, + }; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + exporterOptions.ConnectionString = "EtwSession=OpenTelemetry"; + } + else + { + path = GenerateTempFilePath(); + exporterOptions.ConnectionString = "Endpoint=unix:" + path; + var endpoint = new UnixDomainSocketEndPoint(path); + server = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); + server.Bind(endpoint); + server.Listen(1); + } + + if (hasTableNameMapping) + { + exporterOptions.TableNameMappings = new Dictionary + { + { typeof(GenevaLogExporterTests).FullName, "CustomLogRecord" }, + { "*", "DefaultLogRecord" }, + }; + } + + if (hasCustomFields) + { + // The field "customField" of LogRecord.State should be present in the mapping as a separate key. Other fields of LogRecord.State which are not present + // in CustomFields should be added in the mapping under "env_properties" + exporterOptions.CustomFields = new string[] { "customField" }; + } + + using var loggerFactory = LoggerFactory.Create(builder => builder + .AddOpenTelemetry(options => + { + options.AddGenevaLogExporter(options => + { + options.ConnectionString = exporterOptions.ConnectionString; + options.PrepopulatedFields = exporterOptions.PrepopulatedFields; + }); + options.AddInMemoryExporter(logRecordList); + options.ParseStateValues = parseStateValues; + }) + .AddFilter(typeof(GenevaLogExporterTests).FullName, LogLevel.Trace)); // Enable all LogLevels + + // Create a test exporter to get MessagePack byte data to validate if the data was serialized correctly. + using var exporter = new GenevaLogExporter(exporterOptions); + + // Emit a LogRecord and grab a copy of the LogRecord from the collection passed to InMemoryExporter + var logger = loggerFactory.CreateLogger(); + + // Set the ActivitySourceName to the unique value of the test method name to avoid interference with + // the ActivitySource used by other unit tests. + var sourceName = GetTestMethodName(); + + using var listener = new ActivityListener(); + listener.ShouldListenTo = (activitySource) => activitySource.Name == sourceName; + listener.Sample = (ref ActivityCreationOptions options) => ActivitySamplingResult.AllDataAndRecorded; + ActivitySource.AddActivityListener(listener); + + using var source = new ActivitySource(sourceName); + + using (var activity = source.StartActivity("Activity")) + { + // Log inside an activity to set LogRecord.TraceId and LogRecord.SpanId + logger.LogInformation("Hello from {food} {price}.", "artichoke", 3.99); // structured logging + } + + // When the exporter options are configured with TableMappings only "customField" will be logged as a separate key in the mapping + // "property" will be logged under "env_properties" in the mapping + logger.Log(LogLevel.Trace, 101, "Log a {customField} and {property}", "CustomFieldValue", "PropertyValue"); + logger.Log(LogLevel.Trace, 101, "Log a {customField} and {property}", "CustomFieldValue", null); + logger.Log(LogLevel.Trace, 101, "Log a {customField} and {property}", null, "PropertyValue"); + logger.Log(LogLevel.Debug, 101, "Log a {customField} and {property}", "CustomFieldValue", "PropertyValue"); + logger.Log(LogLevel.Information, 101, "Log a {customField} and {property}", "CustomFieldValue", "PropertyValue"); + logger.Log(LogLevel.Warning, 101, "Log a {customField} and {property}", "CustomFieldValue", "PropertyValue"); + logger.Log(LogLevel.Error, 101, "Log a {customField} and {property}", "CustomFieldValue", "PropertyValue"); + logger.Log(LogLevel.Critical, 101, "Log a {customField} and {property}", "CustomFieldValue", "PropertyValue"); + logger.LogInformation("Hello World!"); // unstructured logging + + logger.Log(LogLevel.Information, default, "Hello World!", null, null); // unstructured logging using a non-extension method call + + // logging custom state + // This is treated as structured logging as the state can be converted to IReadOnlyList> + logger.Log( + LogLevel.Information, + default, + new List>() + { + new KeyValuePair("Key1", "Value1"), + new KeyValuePair("Key2", "Value2"), + }, + null, + (state, ex) => "Formatted Exception!"); + + logger.LogError(new InvalidOperationException("Oops! Food is spoiled!"), "Hello from {food} {price}.", "artichoke", 3.99); + + var loggerWithDefaultCategory = loggerFactory.CreateLogger("DefaultCategory"); + loggerWithDefaultCategory.LogInformation("Basic test"); + + // logRecordList should have two logRecord entries after the logger.LogInformation calls + Assert.Equal(14, logRecordList.Count); + + var m_buffer = typeof(GenevaLogExporter).GetField("m_buffer", BindingFlags.NonPublic | BindingFlags.Static).GetValue(exporter) as ThreadLocal; + + foreach (var logRecord in logRecordList) + { + _ = exporter.SerializeLogRecord(logRecord); + object fluentdData = MessagePack.MessagePackSerializer.Deserialize(m_buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); + this.AssertFluentdForwardModeForLogRecord(exporterOptions, fluentdData, logRecord); + } + } + finally + { + server?.Dispose(); + try + { + File.Delete(path); + } + catch + { + } + } + } + + [Fact] + [Trait("Platform", "Windows")] + public void SuccessfulExportOnWindows() + { + var exporterOptions = new GenevaExporterOptions() + { + PrepopulatedFields = new Dictionary + { + ["cloud.role"] = "BusyWorker", + ["cloud.roleInstance"] = "CY1SCH030021417", + ["cloud.roleVer"] = "9.0.15289.2", + }, + }; + + using var loggerFactory = LoggerFactory.Create(builder => builder + .AddOpenTelemetry(options => + { + options.AddGenevaLogExporter(options => + { + options.ConnectionString = "EtwSession=OpenTelemetry"; + options.PrepopulatedFields = new Dictionary + { + ["cloud.role"] = "BusyWorker", + ["cloud.roleInstance"] = "CY1SCH030021417", + ["cloud.roleVer"] = "9.0.15289.2", + }; + }); + })); + + var logger = loggerFactory.CreateLogger(); + + logger.LogInformation("Hello from {food} {price}.", "artichoke", 3.99); + } + + [Fact] + [Trait("Platform", "Linux")] + public void SuccessfulExportOnLinux() + { + string path = GenerateTempFilePath(); + var logRecordList = new List(); + try + { + var endpoint = new UnixDomainSocketEndPoint(path); + using var server = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); + server.Bind(endpoint); + server.Listen(1); + + using var loggerFactory = LoggerFactory.Create(builder => builder + .AddOpenTelemetry(options => + { + options.AddGenevaLogExporter(options => + { + options.ConnectionString = "Endpoint=unix:" + path; + options.PrepopulatedFields = new Dictionary + { + ["cloud.role"] = "BusyWorker", + ["cloud.roleInstance"] = "CY1SCH030021417", + ["cloud.roleVer"] = "9.0.15289.2", + }; + }); + options.AddInMemoryExporter(logRecordList); + })); + using var serverSocket = server.Accept(); + serverSocket.ReceiveTimeout = 10000; + + // Create a test exporter to get MessagePack byte data for validation of the data received via Socket. + using var exporter = new GenevaLogExporter(new GenevaExporterOptions + { + ConnectionString = "Endpoint=unix:" + path, + PrepopulatedFields = new Dictionary + { + ["cloud.role"] = "BusyWorker", + ["cloud.roleInstance"] = "CY1SCH030021417", + ["cloud.roleVer"] = "9.0.15289.2", + }, + }); + + // Emit a LogRecord and grab a copy of internal buffer for validation. + var logger = loggerFactory.CreateLogger(); + + logger.LogInformation("Hello from {food} {price}.", "artichoke", 3.99); + + // logRecordList should have a singleLogRecord entry after the logger.LogInformation call + Assert.Single(logRecordList); + + int messagePackDataSize; + messagePackDataSize = exporter.SerializeLogRecord(logRecordList[0]); + + // Read the data sent via socket. + var receivedData = new byte[1024]; + int receivedDataSize = serverSocket.Receive(receivedData); + + // Validation + Assert.Equal(messagePackDataSize, receivedDataSize); + } + finally + { + try + { + File.Delete(path); + } + catch + { + } + } + } + + private static string GenerateTempFilePath() + { + while (true) + { + string path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + if (!File.Exists(path)) + { + return path; + } + } + } + + private static string GetTestMethodName([CallerMemberName] string callingMethodName = "") + { + return callingMethodName; + } + + private static object GetField(object fluentdData, string key) + { + /* Fluentd Forward Mode: + [ + "Log", + [ + [ , { "env_ver": "4.0", ... } ] + ], + { "TimeFormat": "DateTime" } + ] + */ + + var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; + var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; + + if (mapping.ContainsKey(key)) + { + return mapping[key]; + } + else + { + return null; + } + } + + private void AssertFluentdForwardModeForLogRecord(GenevaExporterOptions exporterOptions, object fluentdData, LogRecord logRecord) + { + /* Fluentd Forward Mode: + [ + "Log", + [ + [ , { "env_ver": "4.0", ... } ] + ], + { "TimeFormat": "DateTime" } + ] + */ + + var signal = (fluentdData as object[])[0] as string; + var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; + var timeStamp = (DateTime)(TimeStampAndMappings as object[])[0]; + var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; + var timeFormat = (fluentdData as object[])[2] as Dictionary; + + var partAName = "Log"; + if (exporterOptions.TableNameMappings != null) + { + if (exporterOptions.TableNameMappings.ContainsKey(logRecord.CategoryName)) + { + partAName = exporterOptions.TableNameMappings[logRecord.CategoryName]; + } + else if (exporterOptions.TableNameMappings.ContainsKey("*")) + { + partAName = exporterOptions.TableNameMappings["*"]; + } + } + + Assert.Equal(partAName, signal); + + // Timestamp check + Assert.Equal(logRecord.Timestamp.Ticks, timeStamp.Ticks); + + // Part A core envelope fields + + var nameKey = GenevaBaseExporter.V40_PART_A_MAPPING[Schema.V40.PartA.Name]; + + // Check if the user has configured a custom table mapping + Assert.Equal(partAName, mapping[nameKey]); + + // TODO: Update this when we support multiple Schema formats + var partAVer = "4.0"; + var verKey = GenevaBaseExporter.V40_PART_A_MAPPING[Schema.V40.PartA.Ver]; + Assert.Equal(partAVer, mapping[verKey]); + + foreach (var item in exporterOptions.PrepopulatedFields) + { + var partAValue = item.Value as string; + var partAKey = GenevaBaseExporter.V40_PART_A_MAPPING[item.Key]; + Assert.Equal(partAValue, mapping[partAKey]); + } + + var timeKey = GenevaBaseExporter.V40_PART_A_MAPPING[Schema.V40.PartA.Time]; + Assert.Equal(logRecord.Timestamp.Ticks, ((DateTime)mapping[timeKey]).Ticks); + + // Part A dt extensions + + if (logRecord.TraceId != default) + { + Assert.Equal(logRecord.TraceId.ToHexString(), mapping["env_dt_traceId"]); + } + + if (logRecord.SpanId != default) + { + Assert.Equal(logRecord.SpanId.ToHexString(), mapping["env_dt_spanId"]); + } + + if (logRecord.Exception != null) + { + Assert.Equal(logRecord.Exception.GetType().FullName, mapping["env_ex_type"]); + Assert.Equal(logRecord.Exception.Message, mapping["env_ex_msg"]); + } + + // Part B fields + Assert.Equal(logRecord.LogLevel.ToString(), mapping["severityText"]); + Assert.Equal((byte)(((int)logRecord.LogLevel * 4) + 1), mapping["severityNumber"]); + + Assert.Equal(logRecord.CategoryName, mapping["name"]); + + bool isUnstructuredLog = true; + IReadOnlyList> stateKeyValuePairList; + if (logRecord.State == null) + { + stateKeyValuePairList = logRecord.StateValues; + } + else + { + stateKeyValuePairList = logRecord.State as IReadOnlyList>; + } + + if (stateKeyValuePairList != null) + { + isUnstructuredLog = stateKeyValuePairList.Count == 1; + } + + if (isUnstructuredLog) + { + if (logRecord.State != null) + { + Assert.Equal(logRecord.State.ToString(), mapping["body"]); + } + else + { + Assert.Equal(stateKeyValuePairList[0].Value, mapping["body"]); + } + } + else + { + _ = mapping.TryGetValue("env_properties", out object envProprties); + var envPropertiesMapping = envProprties as IDictionary; + + foreach (var item in stateKeyValuePairList) + { + if (item.Key == "{OriginalFormat}") + { + Assert.Equal(item.Value.ToString(), mapping["body"]); + } + else if (exporterOptions.CustomFields == null || exporterOptions.CustomFields.Contains(item.Key)) + { + if (item.Value != null) + { + Assert.Equal(item.Value, mapping[item.Key]); + } + } + else + { + Assert.Equal(item.Value, envPropertiesMapping[item.Key]); + } + } + } + + if (logRecord.EventId != default) + { + Assert.Equal(logRecord.EventId.Id, int.Parse(mapping["eventId"].ToString())); + } + + // Epilouge + Assert.Equal("DateTime", timeFormat["TimeFormat"]); + } + } +} diff --git a/test/OpenTelemetry.Exporter.Geneva.UnitTest/GenevaMetricExporterOptionsTests.cs b/test/OpenTelemetry.Exporter.Geneva.UnitTest/GenevaMetricExporterOptionsTests.cs new file mode 100644 index 00000000000..e082213b6c4 --- /dev/null +++ b/test/OpenTelemetry.Exporter.Geneva.UnitTest/GenevaMetricExporterOptionsTests.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using Xunit; + +namespace OpenTelemetry.Exporter.Geneva.UnitTest +{ + public class GenevaMetricExporterOptionsTests + { + [Fact] + public void InvalidPrepopulatedDimensions() + { + var exception = Assert.Throws(() => + { + var exporterOptions = new GenevaMetricExporterOptions { PrepopulatedMetricDimensions = null }; + }); + + Assert.Throws(() => + { + var exporterOptions = new GenevaMetricExporterOptions + { + PrepopulatedMetricDimensions = new Dictionary + { + ["DimensionKey"] = null, + }, + }; + }); + + var invalidDimensionNameException = Assert.Throws(() => + { + var exporterOptions = new GenevaMetricExporterOptions + { + PrepopulatedMetricDimensions = new Dictionary + { + [new string('a', GenevaMetricExporter.MaxDimensionNameSize + 1)] = "DimensionValue", + }, + }; + }); + + var expectedErrorMessage = $"The dimension: {new string('a', GenevaMetricExporter.MaxDimensionNameSize + 1)} exceeds the maximum allowed limit of {GenevaMetricExporter.MaxDimensionNameSize} characters for a dimension name."; + Assert.Equal(expectedErrorMessage, invalidDimensionNameException.Message); + + var invalidDimensionValueException = Assert.Throws(() => + { + var exporterOptions = new GenevaMetricExporterOptions + { + PrepopulatedMetricDimensions = new Dictionary + { + ["DimensionKey"] = new string('a', GenevaMetricExporter.MaxDimensionValueSize + 1), + }, + }; + }); + + expectedErrorMessage = $"Value provided for the dimension: DimensionKey exceeds the maximum allowed limit of {GenevaMetricExporter.MaxDimensionValueSize} characters for dimension value."; + Assert.Equal(expectedErrorMessage, invalidDimensionValueException.Message); + } + } +} diff --git a/test/OpenTelemetry.Exporter.Geneva.UnitTest/GenevaMetricExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.UnitTest/GenevaMetricExporterTests.cs new file mode 100644 index 00000000000..5ce768c9303 --- /dev/null +++ b/test/OpenTelemetry.Exporter.Geneva.UnitTest/GenevaMetricExporterTests.cs @@ -0,0 +1,726 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net.Sockets; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Threading.Tasks; +using Kaitai; +using OpenTelemetry.Metrics; +using OpenTelemetry.Trace; +using Xunit; +using static OpenTelemetry.Exporter.Geneva.UnitTest.MetricsContract; + +namespace OpenTelemetry.Exporter.Geneva.UnitTest +{ + public class GenevaMetricExporterTests + { + [Fact] + [Trait("Platform", "Any")] + public void NullExporterOptions() + { + GenevaMetricExporterOptions exporterOptions = null; + Assert.Throws(() => new GenevaMetricExporter(exporterOptions)); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + [Trait("Platform", "Any")] + public void InvalidConnectionString(string connectionString) + { + var exporterOptions = new GenevaMetricExporterOptions() { ConnectionString = connectionString }; + var exception = Assert.Throws(() => + { + using var exporter = new GenevaMetricExporter(exporterOptions); + }); + Assert.Equal($"{nameof(exporterOptions.ConnectionString)} is invalid.", exception.Message); + } + + [Fact] + [Trait("Platform", "Any")] + public void ParseConnectionStringCorrectly() + { + string path = string.Empty; + Socket server = null; + try + { + var exporterOptions = new GenevaMetricExporterOptions(); + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + exporterOptions.ConnectionString = "Account=OTelMonitoringAccount;Namespace=OTelMetricNamespace"; + } + else + { + path = GenerateTempFilePath(); + exporterOptions.ConnectionString = $"Endpoint=unix:{path};Account=OTelMonitoringAccount;Namespace=OTelMetricNamespace"; + var endpoint = new UnixDomainSocketEndPoint(path); + server = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); + server.Bind(endpoint); + server.Listen(1); + } + + using var exporter = new GenevaMetricExporter(exporterOptions); + var monitoringAccount = typeof(GenevaMetricExporter).GetField("monitoringAccount", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(exporter) as string; + var metricNamespace = typeof(GenevaMetricExporter).GetField("metricNamespace", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(exporter) as string; + Assert.Equal("OTelMonitoringAccount", monitoringAccount); + Assert.Equal("OTelMetricNamespace", metricNamespace); + } + finally + { + server?.Dispose(); + try + { + File.Delete(path); + } + catch + { + } + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + [Trait("Platform", "Any")] + public void SuccessfulSerialization(bool testMaxLimits) + { + using var meter = new Meter("SuccessfulSerialization", "0.0.1"); + var longCounter = meter.CreateCounter("longCounter"); + var doubleCounter = meter.CreateCounter("doubleCounter"); + var histogram = meter.CreateHistogram("histogram"); + var exportedItems = new List(); + using var inMemoryReader = new BaseExportingMetricReader(new InMemoryExporter(exportedItems)) + { + Temporality = AggregationTemporality.Delta, + }; + + using var meterProvider = Sdk.CreateMeterProviderBuilder() + .AddMeter("SuccessfulSerialization") + .AddReader(inMemoryReader) + .Build(); + + long longValue = 123; + double doubleValue = 123.45; + + if (testMaxLimits) + { + longValue = long.MaxValue; + doubleValue = double.MaxValue; + } + + longCounter.Add( + longValue, new("tag1", "value1"), new("tag2", "value2")); + + doubleCounter.Add( + doubleValue, new("tag1", "value1"), new("tag2", "value2")); + + meter.CreateObservableCounter( + "observableLongCounter", + () => new List>() + { + new(longValue, new("tag1", "value1"), new("tag2", "value2")), + }); + + meter.CreateObservableCounter( + "observableDoubleCounter", + () => new List>() + { + new(doubleValue, new("tag1", "value1"), new("tag2", "value2")), + }); + + meter.CreateObservableGauge( + "observableLongGauge", + () => new List>() + { + new(longValue, new("tag1", "value1"), new("tag2", "value2")), + }); + + meter.CreateObservableGauge( + "observableDoubleGauge", + () => new List>() + { + new(doubleValue, new("tag1", "value1"), new("tag2", "value2")), + }); + + if (testMaxLimits) + { + // only testing the max value allowed for sum + // max value allowed for count is uint.MaxValue. It's not feasible to test that + histogram.Record(longValue, new("tag1", "value1"), new("tag2", "value2")); + } + else + { + // Record the following values from Histogram: + // (-inf - 0] : 1 + // (0 - 5] : 0 + // (5 - 10] : 0 + // (10 - 25] : 0 + // (25 - 50] : 0 + // (50 - 75] : 0 + // (75 - 100] : 0 + // (100 - 250] : 2 + // (250 - 500] : 0 + // (500 - 1000] : 1 + // (1000 - +inf) : 1 + // + // The corresponding value-count pairs to be sent for the given distribution: + // 0: 1 + // 250: 2 + // 1000: 1 + // 1001: 1 (We use one greater than the last bound provided (1000 + 1) as the value for the overflow bucket) + + histogram.Record(-1, new("tag1", "value1"), new("tag2", "value2")); + histogram.Record(150, new("tag1", "value1"), new("tag2", "value2")); + histogram.Record(150, new("tag1", "value1"), new("tag2", "value2")); + histogram.Record(750, new("tag1", "value1"), new("tag2", "value2")); + histogram.Record(2500, new("tag1", "value1"), new("tag2", "value2")); + } + + string path = string.Empty; + Socket server = null; + try + { + var exporterOptions = new GenevaMetricExporterOptions(); + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + exporterOptions.ConnectionString = "Account=OTelMonitoringAccount;Namespace=OTelMetricNamespace"; + } + else + { + path = GenerateTempFilePath(); + exporterOptions.ConnectionString = $"Endpoint=unix:{path};Account=OTelMonitoringAccount;Namespace=OTelMetricNamespace"; + var endpoint = new UnixDomainSocketEndPoint(path); + server = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); + server.Bind(endpoint); + server.Listen(1); + } + + exporterOptions.PrepopulatedMetricDimensions = new Dictionary + { + ["cloud.role"] = "BusyWorker", + ["cloud.roleInstance"] = "CY1SCH030021417", + ["cloud.roleVer"] = "9.0.15289.2", + }; + + using var exporter = new GenevaMetricExporter(exporterOptions); + + inMemoryReader.Collect(); + + Assert.Equal(7, exportedItems.Count); + + // check serialization for longCounter + this.CheckSerializationForSingleMetricPoint(exportedItems[0], exporter, exporterOptions); + + // check serialization for doubleCounter + this.CheckSerializationForSingleMetricPoint(exportedItems[1], exporter, exporterOptions); + + // check serialization for histogram + this.CheckSerializationForSingleMetricPoint(exportedItems[2], exporter, exporterOptions); + + // check serialization for observableLongCounter + this.CheckSerializationForSingleMetricPoint(exportedItems[3], exporter, exporterOptions); + + // check serialization for observableDoubleCounter + this.CheckSerializationForSingleMetricPoint(exportedItems[4], exporter, exporterOptions); + + // check serialization for observableLongGauge + this.CheckSerializationForSingleMetricPoint(exportedItems[5], exporter, exporterOptions); + + // check serialization for observableDoubleGauge + this.CheckSerializationForSingleMetricPoint(exportedItems[6], exporter, exporterOptions); + } + finally + { + server?.Dispose(); + try + { + File.Delete(path); + } + catch + { + } + } + } + + [Fact] + [Trait("Platform", "Any")] + public void SuccessfulSerializationWithViews() + { + using var meter = new Meter("SuccessfulSerializationWithViews", "0.0.1"); + var longCounter = meter.CreateCounter("longCounter"); + var doubleCounter = meter.CreateCounter("doubleCounter"); + var histogramWithCustomBounds = meter.CreateHistogram("histogramWithCustomBounds"); + var histogramWithNoBounds = meter.CreateHistogram("histogramWithNoBounds"); + var exportedItems = new List(); + using var inMemoryReader = new BaseExportingMetricReader(new InMemoryExporter(exportedItems)) + { + Temporality = AggregationTemporality.Delta, + }; + + using var meterProvider = Sdk.CreateMeterProviderBuilder() + .AddMeter("SuccessfulSerializationWithViews") + .AddView("longCounter", "renamedLongCounter") + .AddView("doubleCounter", new MetricStreamConfiguration { TagKeys = new string[] { "tag1" } }) + .AddView( + "histogramWithCustomBounds", + new ExplicitBucketHistogramConfiguration + { + Name = "renamedhistogramWithCustomBounds", + Description = "modifiedDescription", + Boundaries = new double[] { 500, 1000 }, + }) + .AddView(instrument => + { + if (instrument.Name == "histogramWithNoBounds") + { + return new ExplicitBucketHistogramConfiguration { Boundaries = new double[] { } }; + } + + return null; + }) + .AddView("observableLongCounter", MetricStreamConfiguration.Drop) + .AddView("observableDoubleCounter", new MetricStreamConfiguration { TagKeys = new string[] { } }) + .AddView(instrument => + { + if (instrument.Name == "observableLongGauge") + { + return new MetricStreamConfiguration + { + Name = "renamedobservableLongGauge", + Description = "modifiedDescription", + TagKeys = new string[] { "tag1" }, + }; + } + + return null; + }) + .AddView(instrument => + { + if (instrument.Name == "observableDoubleGauge") + { + return MetricStreamConfiguration.Drop; + } + + return null; + }) + .AddReader(inMemoryReader) + .Build(); + + longCounter.Add( + 123, new("tag1", "value1"), new("tag2", "value2")); + + doubleCounter.Add( + 123.45, new("tag1", "value1"), new("tag2", "value2")); + + meter.CreateObservableCounter( + "observableLongCounter", + () => new List>() + { + new(123, new("tag1", "value1"), new("tag2", "value2")), + }); + + meter.CreateObservableCounter( + "observableDoubleCounter", + () => new List>() + { + new(123.45, new("tag1", "value1"), new("tag2", "value2")), + }); + + meter.CreateObservableGauge( + "observableLongGauge", + () => new List>() + { + new(123, new("tag1", "value1"), new("tag2", "value2")), + }); + + meter.CreateObservableGauge( + "observableDoubleGauge", + () => new List>() + { + new(123.45, new("tag1", "value1"), new("tag2", "value2")), + }); + + // Record the following values for histogramWithCustomBounds: + // (-inf - 500] : 3 + // (500 - 1000] : 1 + // (1000 - +inf) : 1 + // + // The corresponding value-count pairs to be sent for histogramWithCustomBounds: + // 500: 3 + // 1000: 1 + // 1001: 1 (We use one greater than the last bound provided (1000 + 1) as the value for the overflow bucket) + + histogramWithCustomBounds.Record(-1, new("tag1", "value1"), new("tag2", "value2")); + histogramWithCustomBounds.Record(150, new("tag1", "value1"), new("tag2", "value2")); + histogramWithCustomBounds.Record(150, new("tag1", "value1"), new("tag2", "value2")); + histogramWithCustomBounds.Record(750, new("tag1", "value1"), new("tag2", "value2")); + histogramWithCustomBounds.Record(2500, new("tag1", "value1"), new("tag2", "value2")); + + // Record the following values for histogramWithNoBounds: + // (-inf - 500] : 3 + // (500 - 1000] : 1 + // (1000 - +inf) : 1 + // + // Only `sum` and `count` are sent for histogramWithNoBounds + // No value-count pairs are sent for histogramWithNoBounds + + histogramWithNoBounds.Record(-1, new("tag1", "value1"), new("tag2", "value2")); + histogramWithNoBounds.Record(150, new("tag1", "value1"), new("tag2", "value2")); + histogramWithNoBounds.Record(150, new("tag1", "value1"), new("tag2", "value2")); + histogramWithNoBounds.Record(750, new("tag1", "value1"), new("tag2", "value2")); + histogramWithNoBounds.Record(2500, new("tag1", "value1"), new("tag2", "value2")); + + string path = string.Empty; + Socket server = null; + try + { + var exporterOptions = new GenevaMetricExporterOptions(); + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + exporterOptions.ConnectionString = "Account=OTelMonitoringAccount;Namespace=OTelMetricNamespace"; + } + else + { + path = GenerateTempFilePath(); + exporterOptions.ConnectionString = $"Endpoint=unix:{path};Account=OTelMonitoringAccount;Namespace=OTelMetricNamespace"; + var endpoint = new UnixDomainSocketEndPoint(path); + server = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); + server.Bind(endpoint); + server.Listen(1); + } + + exporterOptions.PrepopulatedMetricDimensions = new Dictionary + { + ["cloud.role"] = "BusyWorker", + ["cloud.roleInstance"] = "CY1SCH030021417", + ["cloud.roleVer"] = "9.0.15289.2", + }; + + using var exporter = new GenevaMetricExporter(exporterOptions); + + inMemoryReader.Collect(); + + Assert.Equal(6, exportedItems.Count); + + // observableLongCounter and observableDoubleGauge are dropped + Assert.Empty(exportedItems.Where(item => item.Name == "observableLongCounter" || item.Name == "observableDoubleGauge")); + + // check serialization for longCounter + this.CheckSerializationForSingleMetricPoint(exportedItems[0], exporter, exporterOptions); + + // check serialization for doubleCounter + this.CheckSerializationForSingleMetricPoint(exportedItems[1], exporter, exporterOptions); + + // check serialization for histogramWithCustomBounds + this.CheckSerializationForSingleMetricPoint(exportedItems[2], exporter, exporterOptions); + + // check serialization for histogramWithNoBounds + this.CheckSerializationForSingleMetricPoint(exportedItems[3], exporter, exporterOptions); + + // check serialization for observableDoubleCounter + this.CheckSerializationForSingleMetricPoint(exportedItems[4], exporter, exporterOptions); + + // check serialization for observableLongGauge + this.CheckSerializationForSingleMetricPoint(exportedItems[5], exporter, exporterOptions); + } + finally + { + server?.Dispose(); + try + { + File.Delete(path); + } + catch + { + } + } + } + + [Fact] + [Trait("Platform", "Linux")] + public void SuccessfulExportOnLinux() + { + string path = GenerateTempFilePath(); + var exportedItems = new List(); + + using var meter = new Meter("SuccessfulExportOnLinux", "0.0.1"); + var counter = meter.CreateCounter("counter"); + + using var inMemoryMeter = new Meter("InMemoryExportOnLinux", "0.0.1"); + var inMemoryCounter = inMemoryMeter.CreateCounter("counter"); + + try + { + var endpoint = new UnixDomainSocketEndPoint(path); + using var server = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); + server.Bind(endpoint); + server.Listen(1); + + using var inMemoryReader = new BaseExportingMetricReader(new InMemoryExporter(exportedItems)) + { + Temporality = AggregationTemporality.Delta, + }; + + // Set up two different providers as only one Metric Processor is allowed. + // TODO: Simplify the setup when multiple Metric processors are allowed. + using var meterProvider = Sdk.CreateMeterProviderBuilder() + .AddMeter("SuccessfulExportOnLinux") + .AddGenevaMetricExporter(options => + { + options.ConnectionString = $"Endpoint=unix:{path};Account=OTelMonitoringAccount;Namespace=OTelMetricNamespace"; + options.MetricExportIntervalMilliseconds = 5000; + }) + .Build(); + + using var inMemoryMeterProvider = Sdk.CreateMeterProviderBuilder() + .AddMeter("InMemoryExportOnLinux") + .AddReader(inMemoryReader) + .Build(); + + using var serverSocket = server.Accept(); + serverSocket.ReceiveTimeout = 15000; + + // Create a test exporter to get byte data for validation of the data received via Socket. + var exporterOptions = new GenevaMetricExporterOptions() { ConnectionString = $"Endpoint=unix:{path};Account=OTelMonitoringAccount;Namespace=OTelMetricNamespace" }; + using var exporter = new GenevaMetricExporter(exporterOptions); + + // Emit a metric and grab a copy of internal buffer for validation. + counter.Add( + 123, + new KeyValuePair("tag1", "value1"), + new KeyValuePair("tag2", "value2")); + + inMemoryCounter.Add( + 123, + new KeyValuePair("tag1", "value1"), + new KeyValuePair("tag2", "value2")); + + // exportedItems list should have a single entry after the MetricReader.Collect call + inMemoryReader.Collect(); + + Assert.Single(exportedItems); + + var metric = exportedItems[0]; + var metricPointsEnumerator = metric.GetMetricPoints().GetEnumerator(); + metricPointsEnumerator.MoveNext(); + var metricPoint = metricPointsEnumerator.Current; + var metricDataValue = Convert.ToUInt64(metricPoint.GetSumLong()); + var metricData = new MetricData { UInt64Value = metricDataValue }; + var bodyLength = exporter.SerializeMetric( + MetricEventType.ULongMetric, + metric.Name, + metricPoint.EndTime.ToFileTime(), + metricPoint.Tags, + metricData); + + // Wait a little more than the ExportInterval for the exporter to export the data. + Task.Delay(5500).Wait(); + + // Read the data sent via socket. + var receivedData = new byte[1024]; + int receivedDataSize = serverSocket.Receive(receivedData); + + var fixedPayloadLength = (int)typeof(GenevaMetricExporter).GetField("fixedPayloadStartIndex", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(exporter); + + // The whole payload is sent to the Unix Domain Socket + // BinaryHeader (fixed payload) + variable payload which starts with MetricPayload + Assert.Equal(bodyLength + fixedPayloadLength, receivedDataSize); + + var stream = new KaitaiStream(receivedData); + var data = new MetricsContract(stream); + + Assert.Equal(metric.Name, data.Body.MetricName.Value); + Assert.Equal("OTelMonitoringAccount", data.Body.MetricAccount.Value); + Assert.Equal("OTelMetricNamespace", data.Body.MetricNamespace.Value); + + var valueSection = data.Body.ValueSection as SingleUint64Value; + Assert.Equal(metricDataValue, valueSection.Value); + + Assert.Equal(2, data.Body.NumDimensions); + + int i = 0; + foreach (var tag in metricPoint.Tags) + { + Assert.Equal(tag.Key, data.Body.DimensionsNames[i].Value); + Assert.Equal(tag.Value, data.Body.DimensionsValues[i].Value); + i++; + } + + Assert.Equal((ushort)MetricEventType.ULongMetric, data.EventId); + Assert.Equal(bodyLength, data.LenBody); + } + finally + { + try + { + File.Delete(path); + } + catch + { + } + } + } + + private static string GenerateTempFilePath() + { + while (true) + { + string path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + if (!File.Exists(path)) + { + return path; + } + } + } + + private void CheckSerializationForSingleMetricPoint(Metric metric, GenevaMetricExporter exporter, GenevaMetricExporterOptions exporterOptions) + { + var metricType = metric.MetricType; + var metricPointsEnumerator = metric.GetMetricPoints().GetEnumerator(); + metricPointsEnumerator.MoveNext(); + var metricPoint = metricPointsEnumerator.Current; + MetricsContract data = null; + + // Check metric value, timestamp, eventId, and length of payload + if (metricType == MetricType.LongSum || metricType == MetricType.LongGauge) + { + var metricDataValue = metricType == MetricType.LongSum ? + Convert.ToUInt64(metricPoint.GetSumLong()) : + Convert.ToUInt64(metricPoint.GetGaugeLastValueLong()); + var metricData = new MetricData { UInt64Value = metricDataValue }; + var bodyLength = exporter.SerializeMetric( + MetricEventType.ULongMetric, + metric.Name, + metricPoint.EndTime.ToFileTime(), + metricPoint.Tags, + metricData); + var buffer = typeof(GenevaMetricExporter).GetField("bufferForNonHistogramMetrics", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(exporter) as byte[]; + var stream = new KaitaiStream(buffer); + data = new MetricsContract(stream); + var valueSection = data.Body.ValueSection as SingleUint64Value; + Assert.Equal(metricDataValue, valueSection.Value); + Assert.Equal((ulong)metricPoint.EndTime.ToFileTime(), valueSection.Timestamp); + Assert.Equal((ushort)MetricEventType.ULongMetric, data.EventId); + Assert.Equal(bodyLength, data.LenBody); + } + else if (metricType == MetricType.DoubleSum || metricType == MetricType.DoubleGauge) + { + var metricDataValue = metricType == MetricType.DoubleSum ? + metricPoint.GetSumDouble() : + metricPoint.GetGaugeLastValueDouble(); + var metricData = new MetricData { DoubleValue = metricDataValue }; + var bodyLength = exporter.SerializeMetric( + MetricEventType.DoubleMetric, + metric.Name, + metricPoint.EndTime.ToFileTime(), + metricPoint.Tags, + metricData); + var buffer = typeof(GenevaMetricExporter).GetField("bufferForNonHistogramMetrics", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(exporter) as byte[]; + var stream = new KaitaiStream(buffer); + data = new MetricsContract(stream); + var valueSection = data.Body.ValueSection as SingleDoubleValue; + Assert.Equal(metricDataValue, valueSection.Value); + Assert.Equal((ulong)metricPoint.EndTime.ToFileTime(), valueSection.Timestamp); + Assert.Equal((ushort)MetricEventType.DoubleMetric, data.EventId); + Assert.Equal(bodyLength, data.LenBody); + } + else if (metricType == MetricType.Histogram) + { + var sum = new MetricData { UInt64Value = Convert.ToUInt64(metricPoint.GetHistogramSum()) }; + var count = Convert.ToUInt32(metricPoint.GetHistogramCount()); + var bodyLength = exporter.SerializeHistogramMetric( + metric.Name, + metricPoint.EndTime.ToFileTime(), + metricPoint.Tags, + metricPoint.GetHistogramBuckets(), + sum, + count); + var buffer = typeof(GenevaMetricExporter).GetField("bufferForHistogramMetrics", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(exporter) as byte[]; + var stream = new KaitaiStream(buffer); + data = new MetricsContract(stream); + var valueSection = data.Body.ValueSection as ExtAggregatedUint64Value; + var valueCountPairs = data.Body.Histogram.Body as HistogramValueCountPairs; + + Assert.Equal(0, data.Body.Histogram.Version); + Assert.Equal(2, (int)data.Body.Histogram.Type); + + int listIterator = 0; + int bucketsWithPositiveCount = 0; + double lastExplicitBound = default; + foreach (var bucket in metricPoint.GetHistogramBuckets()) + { + if (bucket.BucketCount > 0) + { + if (bucket.ExplicitBound != double.PositiveInfinity) + { + Assert.Equal(bucket.ExplicitBound, valueCountPairs.Columns[listIterator].Value); + lastExplicitBound = bucket.ExplicitBound; + } + else + { + Assert.Equal((ulong)lastExplicitBound + 1, valueCountPairs.Columns[listIterator].Value); + } + + Assert.Equal(bucket.BucketCount, valueCountPairs.Columns[listIterator].Count); + + listIterator++; + bucketsWithPositiveCount++; + } + } + + Assert.Equal(bucketsWithPositiveCount, valueCountPairs.DistributionSize); + + Assert.Equal(count, valueSection.Count); + Assert.Equal(Convert.ToUInt64(metricPoint.GetHistogramSum()), valueSection.Sum); + Assert.Equal(0UL, valueSection.Min); + Assert.Equal(0UL, valueSection.Max); + Assert.Equal((ulong)metricPoint.EndTime.ToFileTime(), valueSection.Timestamp); + Assert.Equal((ushort)MetricEventType.ExternallyAggregatedULongDistributionMetric, data.EventId); + Assert.Equal(bodyLength, data.LenBody); + } + + // Check metric name, account, and namespace + var connectionStringBuilder = new ConnectionStringBuilder(exporterOptions.ConnectionString); + Assert.Equal(metric.Name, data.Body.MetricName.Value); + Assert.Equal(connectionStringBuilder.Account, data.Body.MetricAccount.Value); + Assert.Equal(connectionStringBuilder.Namespace, data.Body.MetricNamespace.Value); + + var dimensionsCount = 0; + if (exporterOptions.PrepopulatedMetricDimensions != null) + { + foreach (var entry in exporterOptions.PrepopulatedMetricDimensions) + { + Assert.Contains(data.Body.DimensionsNames, dim => dim.Value == entry.Key); + Assert.Contains(data.Body.DimensionsValues, dim => dim.Value == Convert.ToString(entry.Value, CultureInfo.InvariantCulture)); + } + + dimensionsCount += exporterOptions.PrepopulatedMetricDimensions.Count; + } + + // Check metric dimensions + int i = 0; + foreach (var item in exporterOptions.PrepopulatedMetricDimensions) + { + Assert.Equal(item.Key, data.Body.DimensionsNames[i].Value); + Assert.Equal(item.Value, data.Body.DimensionsValues[i].Value); + i++; + } + + foreach (var tag in metricPoint.Tags) + { + Assert.Equal(tag.Key, data.Body.DimensionsNames[i].Value); + Assert.Equal(tag.Value, data.Body.DimensionsValues[i].Value); + i++; + } + + dimensionsCount += metricPoint.Tags.Count; + + Assert.Equal(dimensionsCount, data.Body.NumDimensions); + } + } +} diff --git a/test/OpenTelemetry.Exporter.Geneva.UnitTest/GenevaTraceExporterTests.cs b/test/OpenTelemetry.Exporter.Geneva.UnitTest/GenevaTraceExporterTests.cs new file mode 100644 index 00000000000..286b04323e5 --- /dev/null +++ b/test/OpenTelemetry.Exporter.Geneva.UnitTest/GenevaTraceExporterTests.cs @@ -0,0 +1,515 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Net.Sockets; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading; +using OpenTelemetry.Trace; +using Xunit; + +namespace OpenTelemetry.Exporter.Geneva.UnitTest +{ + public class GenevaTraceExporterTests + { + public GenevaTraceExporterTests() + { + Activity.DefaultIdFormat = ActivityIdFormat.W3C; + } + + [Fact] + [Trait("Platform", "Any")] + public void GenevaTraceExporter_constructor_Invalid_Input() + { + // no connection string + Assert.Throws(() => + { + using var exporter = new GenevaTraceExporter(new GenevaExporterOptions()); + }); + + // null connection string + Assert.Throws(() => + { + using var exporter = new GenevaTraceExporter(new GenevaExporterOptions + { + ConnectionString = null, + }); + }); + } + + [Fact] + [Trait("Platform", "Windows")] + public void GenevaTraceExporter_constructor_Invalid_Input_Windows() + { + // no ETW session name + Assert.Throws(() => + { + using var exporter = new GenevaTraceExporter(new GenevaExporterOptions + { + ConnectionString = "key=value", + }); + }); + + // empty ETW session name + Assert.Throws(() => + { + using var exporter = new GenevaTraceExporter(new GenevaExporterOptions + { + ConnectionString = "EtwSession=", + }); + }); + } + + [Fact] + [Trait("Platform", "Any")] + public void GenevaTraceExporter_TableNameMappings_SpecialCharacters() + { + Assert.Throws(() => + { + using var exporter = new GenevaTraceExporter(new GenevaExporterOptions + { + TableNameMappings = new Dictionary { ["Span"] = "\u0418" }, + }); + }); + } + + [Fact] + [Trait("Platform", "Windows")] + public void GenevaTraceExporter_Success() + { + // Set the ActivitySourceName to the unique value of the test method name to avoid interference with + // the ActivitySource used by other unit tests. + var sourceName = GetTestMethodName(); + + // TODO: Setup a mock or spy for eventLogger to assert that eventLogger.LogInformationalEvent is actually called. + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .SetSampler(new AlwaysOnSampler()) + .AddSource(sourceName) + .AddGenevaTraceExporter(options => + { + options.ConnectionString = "EtwSession=OpenTelemetry"; + options.PrepopulatedFields = new Dictionary + { + ["cloud.role"] = "BusyWorker", + ["cloud.roleInstance"] = "CY1SCH030021417", + ["cloud.roleVer"] = "9.0.15289.2", + }; + }) + .Build(); + + var source = new ActivitySource(sourceName); + using (var parent = source.StartActivity("HttpIn", ActivityKind.Server)) + { + parent?.SetTag("http.method", "GET"); + parent?.SetTag("http.url", "https://localhost/wiki/Rabbit"); + using (var child = source.StartActivity("HttpOut", ActivityKind.Client)) + { + child?.SetTag("http.method", "GET"); + child?.SetTag("http.url", "https://www.wikipedia.org/wiki/Rabbit"); + child?.SetTag("http.status_code", 404); + } + + parent?.SetTag("http.status_code", 200); + } + + var link = new ActivityLink(new ActivityContext(ActivityTraceId.CreateRandom(), ActivitySpanId.CreateRandom(), ActivityTraceFlags.Recorded)); + using (var activity = source.StartActivity("Foo", ActivityKind.Internal, null, null, new ActivityLink[] { link })) + { + } + + using (var activity = source.StartActivity("Bar")) + { + activity.SetStatus(Status.Error); + } + + using (var activity = source.StartActivity("Baz")) + { + activity.SetStatus(Status.Ok); + } + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + [Trait("Platform", "Any")] + public void GenevaTraceExporter_Serialization_Success(bool hasTableNameMapping, bool hasCustomFields) + { + string path = string.Empty; + Socket server = null; + try + { + int invocationCount = 0; + var exporterOptions = new GenevaExporterOptions + { + PrepopulatedFields = new Dictionary + { + ["cloud.role"] = "BusyWorker", + ["cloud.roleInstance"] = "CY1SCH030021417", + ["cloud.roleVer"] = "9.0.15289.2", + }, + }; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + exporterOptions.ConnectionString = "EtwSession=OpenTelemetry"; + } + else + { + path = GetRandomFilePath(); + exporterOptions.ConnectionString = "Endpoint=unix:" + path; + var endpoint = new UnixDomainSocketEndPoint(path); + server = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); + server.Bind(endpoint); + server.Listen(1); + } + + if (hasTableNameMapping) + { + exporterOptions.TableNameMappings = new Dictionary { { "Span", "CustomActivity" } }; + } + + if (hasCustomFields) + { + // The tag "clientRequestId" should be present in the mapping as a separate key. Other tags which are not present + // in the m_dedicatedFields should be added in the mapping under "env_properties" + exporterOptions.CustomFields = new string[] { "clientRequestId" }; + } + + using var exporter = new GenevaTraceExporter(exporterOptions); + var dedicatedFields = typeof(GenevaTraceExporter).GetField("m_dedicatedFields", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(exporter) as IReadOnlyDictionary; + var CS40_PART_B_MAPPING = typeof(GenevaTraceExporter).GetField("CS40_PART_B_MAPPING", BindingFlags.NonPublic | BindingFlags.Static).GetValue(exporter) as IReadOnlyDictionary; + var m_buffer = typeof(GenevaTraceExporter).GetField("m_buffer", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(exporter) as ThreadLocal; + + // Add an ActivityListener to serialize the activity and assert that it was valid on ActivityStopped event + + // Set the ActivitySourceName to the unique value of the test method name to avoid interference with + // the ActivitySource used by other unit tests. + var sourceName = GetTestMethodName(); + + using var listener = new ActivityListener(); + listener.ShouldListenTo = (activitySource) => activitySource.Name == sourceName; + listener.Sample = (ref ActivityCreationOptions options) => ActivitySamplingResult.AllDataAndRecorded; + listener.ActivityStopped = (activity) => + { + _ = exporter.SerializeActivity(activity); + object fluentdData = MessagePack.MessagePackSerializer.Deserialize(m_buffer.Value, MessagePack.Resolvers.ContractlessStandardResolver.Instance); + this.AssertFluentdForwardModeForActivity(exporterOptions, fluentdData, activity, CS40_PART_B_MAPPING, dedicatedFields); + invocationCount++; + }; + ActivitySource.AddActivityListener(listener); + + using var source = new ActivitySource(sourceName); + using (var parentActivity = source.StartActivity("ParentActivity")) + { + var linkedtraceId1 = ActivityTraceId.CreateFromString("e8ea7e9ac72de94e91fabc613f9686a1".AsSpan()); + var linkedSpanId1 = ActivitySpanId.CreateFromString("888915b6286b9c01".AsSpan()); + + var linkedtraceId2 = ActivityTraceId.CreateFromString("e8ea7e9ac72de94e91fabc613f9686a2".AsSpan()); + var linkedSpanId2 = ActivitySpanId.CreateFromString("888915b6286b9c02".AsSpan()); + + var links = new[] + { + new ActivityLink(new ActivityContext( + linkedtraceId1, + linkedSpanId1, + ActivityTraceFlags.Recorded)), + new ActivityLink(new ActivityContext( + linkedtraceId2, + linkedSpanId2, + ActivityTraceFlags.Recorded)), + }; + + using (var activity = source.StartActivity("SayHello", ActivityKind.Internal, parentActivity.Context, null, links)) + { + activity?.SetTag("http.status_code", 500); // This should be added as httpStatusCode in the mapping + activity?.SetTag("azureResourceProvider", "Microsoft.AAD"); + activity?.SetTag("clientRequestId", "58a37988-2c05-427a-891f-5e0e1266fcc5"); + activity?.SetTag("foo", 1); + activity?.SetTag("bar", 2); + activity?.SetStatus(Status.Error); + } + } + + Assert.Equal(2, invocationCount); + } + finally + { + server?.Dispose(); + try + { + File.Delete(path); + } + catch + { + } + } + } + + [Fact] + [Trait("Platform", "Linux")] + public void GenevaTraceExporter_Linux_constructor_Missing() + { + string path = GetRandomFilePath(); + + // System.Net.Internals.SocketExceptionFactory+ExtendedSocketException : Cannot assign requested address + try + { + // Set the ActivitySourceName to the unique value of the test method name to avoid interference with + // the ActivitySource used by other unit tests. + var sourceName = GetTestMethodName(); + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .SetSampler(new AlwaysOnSampler()) + .AddSource(sourceName) + .AddGenevaTraceExporter(options => + { + options.ConnectionString = "Endpoint=unix:" + path; + }) + .Build(); + Assert.True(false, "Should never reach here. GenevaTraceExporter should fail in constructor."); + } + catch (SocketException ex) + { + Assert.Contains("Cannot assign requested address", ex.Message); + } + + try + { + var exporter = new GenevaTraceExporter(new GenevaExporterOptions + { + ConnectionString = "Endpoint=unix:" + path, + }); + Assert.True(false, "Should never reach here. GenevaTraceExporter should fail in constructor."); + } + catch (SocketException ex) + { + Assert.Contains("Cannot assign requested address", ex.Message); + } + } + + [Fact] + [Trait("Platform", "Linux")] + public void GenevaTraceExporter_Linux_Success() + { + string path = GetRandomFilePath(); + try + { + var endpoint = new UnixDomainSocketEndPoint(path); + using var server = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); + server.Bind(endpoint); + server.Listen(1); + + // Set the ActivitySourceName to the unique value of the test method name to avoid interference with + // the ActivitySource used by other unit tests. + var sourceName = GetTestMethodName(); + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .SetSampler(new AlwaysOnSampler()) + .AddSource(sourceName) + .AddGenevaTraceExporter(options => + { + options.ConnectionString = "Endpoint=unix:" + path; + options.PrepopulatedFields = new Dictionary + { + ["cloud.role"] = "BusyWorker", + ["cloud.roleInstance"] = "CY1SCH030021417", + ["cloud.roleVer"] = "9.0.15289.2", + }; + }) + .Build(); + using Socket serverSocket = server.Accept(); + + // Create a test exporter to get MessagePack byte data for validation of the data received via Socket. + var exporter = new GenevaTraceExporter(new GenevaExporterOptions + { + ConnectionString = "Endpoint=unix:" + path, + PrepopulatedFields = new Dictionary + { + ["cloud.role"] = "BusyWorker", + ["cloud.roleInstance"] = "CY1SCH030021417", + ["cloud.roleVer"] = "9.0.15289.2", + }, + }); + + // Emit trace and grab a copy of internal buffer for validation. + var source = new ActivitySource(sourceName); + int messagePackDataSize; + using (var activity = source.StartActivity("Foo", ActivityKind.Internal)) + { + messagePackDataSize = exporter.SerializeActivity(activity); + } + + // Read the data sent via socket. + var receivedData = new byte[1024]; + int receivedDataSize = serverSocket.Receive(receivedData); + + // Validation + Assert.Equal(messagePackDataSize, receivedDataSize); + } + catch (Exception) + { + throw; + } + finally + { + try + { + File.Delete(path); + } + catch + { + } + } + } + + private static string GetRandomFilePath() + { + while (true) + { + string path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + if (!File.Exists(path)) + { + return path; + } + } + } + + private static string GetTestMethodName([CallerMemberName] string callingMethodName = "") + { + return callingMethodName; + } + + private void AssertFluentdForwardModeForActivity(GenevaExporterOptions exporterOptions, object fluentdData, Activity activity, IReadOnlyDictionary CS40_PART_B_MAPPING, IReadOnlyDictionary dedicatedFields) + { + /* Fluentd Forward Mode: + [ + "Span", + [ + [ , { "env_ver": "4.0", ... } ] + ], + { "TimeFormat": "DateTime" } + ] + */ + + var signal = (fluentdData as object[])[0] as string; + var TimeStampAndMappings = ((fluentdData as object[])[1] as object[])[0]; + var timeStamp = (DateTime)(TimeStampAndMappings as object[])[0]; + var mapping = (TimeStampAndMappings as object[])[1] as Dictionary; + var timeFormat = (fluentdData as object[])[2] as Dictionary; + + var partAName = "Span"; + if (exporterOptions.TableNameMappings != null) + { + partAName = exporterOptions.TableNameMappings["Span"]; + } + + Assert.Equal(partAName, signal); + + // Timestamp check + var dtBegin = activity.StartTimeUtc; + var tsBegin = dtBegin.Ticks; + var tsEnd = tsBegin + activity.Duration.Ticks; + Assert.Equal(tsEnd, timeStamp.Ticks); + + // Part A core envelope fields + + var nameKey = GenevaBaseExporter.V40_PART_A_MAPPING[Schema.V40.PartA.Name]; + + // Check if the user has configured a custom table mapping + Assert.Equal(partAName, mapping[nameKey]); + + // TODO: Update this when we support multiple Schema formats + var partAVer = "4.0"; + var verKey = GenevaBaseExporter.V40_PART_A_MAPPING[Schema.V40.PartA.Ver]; + Assert.Equal(partAVer, mapping[verKey]); + + foreach (var item in exporterOptions.PrepopulatedFields) + { + var partAValue = item.Value as string; + var partAKey = GenevaBaseExporter.V40_PART_A_MAPPING[item.Key]; + Assert.Equal(partAValue, mapping[partAKey]); + } + + var timeKey = GenevaBaseExporter.V40_PART_A_MAPPING[Schema.V40.PartA.Time]; + Assert.Equal(tsEnd, ((DateTime)mapping[timeKey]).Ticks); + + // Part A dt extensions + Assert.Equal(activity.TraceId.ToString(), mapping["env_dt_traceId"]); + Assert.Equal(activity.SpanId.ToString(), mapping["env_dt_spanId"]); + + // Part B Span - required fields + Assert.Equal(activity.DisplayName, mapping["name"]); + Assert.Equal((byte)activity.Kind, mapping["kind"]); + Assert.Equal(activity.StartTimeUtc, mapping["startTime"]); + + var activityStatusCode = activity.GetStatus().StatusCode; + Assert.Equal(activityStatusCode == StatusCode.Error ? false : true, mapping["success"]); + + // Part B Span optional fields and Part C fields + if (activity.ParentSpanId != default) + { + Assert.Equal(activity.ParentSpanId.ToHexString(), mapping["parentId"]); + } + + #region Assert Activity Links + if (activity.Links.Any()) + { + Assert.Contains(mapping, m => m.Key as string == "links"); + var mappingLinks = mapping["links"] as IEnumerable; + using IEnumerator activityLinksEnumerator = activity.Links.GetEnumerator(); + using IEnumerator mappingLinksEnumerator = mappingLinks.GetEnumerator(); + while (activityLinksEnumerator.MoveNext() && mappingLinksEnumerator.MoveNext()) + { + var activityLink = activityLinksEnumerator.Current; + var mappingLink = mappingLinksEnumerator.Current as Dictionary; + + Assert.Equal(activityLink.Context.TraceId.ToHexString(), mappingLink["toTraceId"]); + Assert.Equal(activityLink.Context.SpanId.ToHexString(), mappingLink["toSpanId"]); + } + + // Assert that mapping contains exactly the same number of ActivityLinks as present in the activity + // MoveNext() on both the enumerators should return false as we should have enumerated till the last element for both the Enumerables + Assert.Equal(activityLinksEnumerator.MoveNext(), mappingLinksEnumerator.MoveNext()); + } + else + { + Assert.DoesNotContain(mapping, m => m.Key as string == "links"); + } + #endregion + + #region Assert Activity Tags + _ = mapping.TryGetValue("env_properties", out object envProprties); + var envPropertiesMapping = envProprties as IDictionary; + foreach (var tag in activity.TagObjects) + { + if (CS40_PART_B_MAPPING.TryGetValue(tag.Key, out string replacementKey)) + { + Assert.Equal(tag.Value.ToString(), mapping[replacementKey].ToString()); + } + else if (string.Equals(tag.Key, "otel.status_code", StringComparison.Ordinal)) + { + // Status code check is already done when we check for "success" key in the mapping + continue; + } + else + { + // If CustomFields are proivded, dedicatedFields will be populated + if (exporterOptions.CustomFields == null || dedicatedFields.ContainsKey(tag.Key)) + { + Assert.Equal(tag.Value.ToString(), mapping[tag.Key].ToString()); + } + else + { + Assert.Equal(tag.Value.ToString(), envPropertiesMapping[tag.Key].ToString()); + } + } + } + #endregion + + // Epilouge + Assert.Equal("DateTime", timeFormat["TimeFormat"]); + } + } +} diff --git a/test/OpenTelemetry.Exporter.Geneva.UnitTest/MessagePackSerializerTests.cs b/test/OpenTelemetry.Exporter.Geneva.UnitTest/MessagePackSerializerTests.cs new file mode 100644 index 00000000000..7ee2f3f8a08 --- /dev/null +++ b/test/OpenTelemetry.Exporter.Geneva.UnitTest/MessagePackSerializerTests.cs @@ -0,0 +1,388 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Xunit; +using Xunit.Sdk; + +namespace OpenTelemetry.Exporter.Geneva.UnitTest +{ + public class MessagePackSerializerTests + { + private void AssertBytes(byte[] expected, byte[] actual, int length) + { + Assert.Equal(expected.Length, length); + for (int i = 0; i < length; i++) + { + byte expectedByte = expected[i]; + byte actualByte = actual[i]; + + Assert.True( + expectedByte == actualByte, + string.Format($"Expected: '{(byte)expectedByte}', Actual: '{(byte)actualByte}' at offset {i}.")); + } + } + + private void MessagePackSerializer_TestSerialization(object obj) + { + var buffer = new byte[64 * 1024]; + var length = MessagePackSerializer.Serialize(buffer, 0, obj); + this.AssertBytes(MessagePack.MessagePackSerializer.Serialize(obj), buffer, length); + } + + private void MessagePackSerializer_TestASCIIStringSerialization(string input) + { + var sizeLimit = (1 << 14) - 1; // // Max length of string allowed + var buffer = new byte[64 * 1024]; + var length = MessagePackSerializer.SerializeAsciiString(buffer, 0, input); + var deserializedString = MessagePack.MessagePackSerializer.Deserialize(buffer); + if (!string.IsNullOrEmpty(input) && input.Length > sizeLimit) + { + // We truncate the string using `.` in the last three characters which takes 3 bytes of memort + var byteCount = Encoding.ASCII.GetByteCount(input.Substring(0, sizeLimit - 3)) + 3; + Assert.Equal(0xDA, buffer[0]); + Assert.Equal(byteCount, (buffer[1] << 8) | buffer[2]); + Assert.Equal(byteCount, length - 3); // First three bytes are metadata + + Assert.NotEqual(input, deserializedString); + + int i; + for (i = 0; i < sizeLimit - 3; i++) + { + Assert.Equal(input[i], deserializedString[i]); + } + + Assert.Equal('.', deserializedString[i++]); + Assert.Equal('.', deserializedString[i++]); + Assert.Equal('.', deserializedString[i++]); + } + else + { + if (input != null) + { + var byteCount = Encoding.ASCII.GetByteCount(input); + if (input.Length <= 31) + { + Assert.Equal(0xA0 | byteCount, buffer[0]); + Assert.Equal(byteCount, length - 1); // First one byte is metadata + } + else if (input.Length <= 255) + { + Assert.Equal(0xD9, buffer[0]); + Assert.Equal(byteCount, buffer[1]); + Assert.Equal(byteCount, length - 2); // First two bytes are metadata + } + else if (input.Length <= sizeLimit) + { + Assert.Equal(0xDA, buffer[0]); + Assert.Equal(byteCount, (buffer[1] << 8) | buffer[2]); + Assert.Equal(byteCount, length - 3); // First three bytes are metadata + } + } + + Assert.Equal(input, deserializedString); + } + } + + private void MessagePackSerializer_TestUnicodeStringSerialization(string input) + { + var sizeLimit = (1 << 14) - 1; // // Max length of string allowed + var buffer = new byte[64 * 1024]; + var length = MessagePackSerializer.SerializeUnicodeString(buffer, 0, input); + + var deserializedString = MessagePack.MessagePackSerializer.Deserialize(buffer); + if (!string.IsNullOrEmpty(input) && input.Length > sizeLimit) + { + // We truncate the string using `.` in the last three characters which takes 3 bytes of memory + var byteCount = Encoding.UTF8.GetByteCount(input.Substring(0, sizeLimit - 3)) + 3; + Assert.Equal(0xDA, buffer[0]); + Assert.Equal(byteCount, (buffer[1] << 8) | buffer[2]); + Assert.Equal(byteCount, length - 3); // First three bytes are metadata + + Assert.NotEqual(input, deserializedString); + + int i; + for (i = 0; i < sizeLimit - 3; i++) + { + Assert.Equal(input[i], deserializedString[i]); + } + + Assert.Equal('.', deserializedString[i++]); + Assert.Equal('.', deserializedString[i++]); + Assert.Equal('.', deserializedString[i++]); + } + else + { + Assert.Equal(input, deserializedString); + + if (input != null) + { + var byteCount = Encoding.UTF8.GetByteCount(input); + Assert.Equal(0xDA, buffer[0]); + Assert.Equal(byteCount, (buffer[1] << 8) | buffer[2]); + Assert.Equal(byteCount, length - 3); // First three bytes are metadata + } + } + } + + [Fact] + [Trait("Platform", "Any")] + public void MessagePackSerializer_Null() + { + this.MessagePackSerializer_TestSerialization(null); + } + + [Fact] + [Trait("Platform", "Any")] + public void MessagePackSerializer_Boolean() + { + this.MessagePackSerializer_TestSerialization(true); + this.MessagePackSerializer_TestSerialization(false); + } + + [Fact] + [Trait("Platform", "Any")] + public void MessagePackSerializer_Int() + { + // 8 bits + for (sbyte value = sbyte.MinValue; value < sbyte.MaxValue; value++) + { + this.MessagePackSerializer_TestSerialization(value); + } + + this.MessagePackSerializer_TestSerialization(sbyte.MaxValue); + + // 16 bits + for (short value = short.MinValue; value < short.MaxValue; value++) + { + this.MessagePackSerializer_TestSerialization(value); + } + + this.MessagePackSerializer_TestSerialization(short.MaxValue); + + // 32 bits + this.MessagePackSerializer_TestSerialization(int.MinValue); + this.MessagePackSerializer_TestSerialization(int.MinValue + 1); + this.MessagePackSerializer_TestSerialization((int)short.MinValue - 1); + this.MessagePackSerializer_TestSerialization((int)short.MinValue); + this.MessagePackSerializer_TestSerialization((int)short.MinValue + 1); + this.MessagePackSerializer_TestSerialization((int)sbyte.MinValue - 1); + for (sbyte value = sbyte.MinValue; value < sbyte.MaxValue; value++) + { + this.MessagePackSerializer_TestSerialization((int)value); + } + + this.MessagePackSerializer_TestSerialization((int)sbyte.MaxValue); + this.MessagePackSerializer_TestSerialization((int)sbyte.MaxValue + 1); + this.MessagePackSerializer_TestSerialization((int)short.MaxValue - 1); + this.MessagePackSerializer_TestSerialization((int)short.MaxValue); + this.MessagePackSerializer_TestSerialization((int)short.MaxValue + 1); + this.MessagePackSerializer_TestSerialization(int.MaxValue - 1); + this.MessagePackSerializer_TestSerialization(int.MaxValue); + + // 64 bits + this.MessagePackSerializer_TestSerialization(long.MinValue); + this.MessagePackSerializer_TestSerialization(long.MinValue + 1); + this.MessagePackSerializer_TestSerialization((long)int.MinValue - 1); + this.MessagePackSerializer_TestSerialization((long)int.MinValue); + this.MessagePackSerializer_TestSerialization((long)int.MinValue + 1); + this.MessagePackSerializer_TestSerialization((long)short.MinValue - 1); + this.MessagePackSerializer_TestSerialization((long)short.MinValue); + this.MessagePackSerializer_TestSerialization((long)short.MinValue + 1); + this.MessagePackSerializer_TestSerialization((long)sbyte.MinValue - 1); + for (sbyte value = sbyte.MinValue; value < sbyte.MaxValue; value++) + { + this.MessagePackSerializer_TestSerialization((long)value); + } + + this.MessagePackSerializer_TestSerialization((long)sbyte.MaxValue); + this.MessagePackSerializer_TestSerialization((long)sbyte.MaxValue + 1); + this.MessagePackSerializer_TestSerialization((long)short.MaxValue - 1); + this.MessagePackSerializer_TestSerialization((long)short.MaxValue); + this.MessagePackSerializer_TestSerialization((long)short.MaxValue + 1); + this.MessagePackSerializer_TestSerialization((long)int.MaxValue - 1); + this.MessagePackSerializer_TestSerialization((long)int.MaxValue); + this.MessagePackSerializer_TestSerialization((long)int.MaxValue + 1); + this.MessagePackSerializer_TestSerialization(long.MaxValue - 1); + this.MessagePackSerializer_TestSerialization(long.MaxValue); + } + + [Fact] + [Trait("Platform", "Any")] + public void MessagePackSerializer_UInt() + { + // 8 bits + for (byte value = byte.MinValue; value < byte.MaxValue; value++) + { + this.MessagePackSerializer_TestSerialization(value); + } + + this.MessagePackSerializer_TestSerialization(byte.MaxValue); + + // 16 bits + for (ushort value = ushort.MinValue; value < ushort.MaxValue; value++) + { + this.MessagePackSerializer_TestSerialization(value); + } + + this.MessagePackSerializer_TestSerialization(ushort.MaxValue); + + // 32 bits + this.MessagePackSerializer_TestSerialization(uint.MinValue); + this.MessagePackSerializer_TestSerialization((uint)byte.MaxValue - 1); + this.MessagePackSerializer_TestSerialization((uint)byte.MaxValue); + this.MessagePackSerializer_TestSerialization((uint)byte.MaxValue + 1); + this.MessagePackSerializer_TestSerialization((uint)ushort.MaxValue - 1); + this.MessagePackSerializer_TestSerialization((uint)ushort.MaxValue); + this.MessagePackSerializer_TestSerialization((uint)ushort.MaxValue + 1); + this.MessagePackSerializer_TestSerialization(uint.MaxValue - 1); + this.MessagePackSerializer_TestSerialization(uint.MaxValue); + + // 64 bits + this.MessagePackSerializer_TestSerialization(ulong.MinValue); + this.MessagePackSerializer_TestSerialization((ulong)byte.MaxValue - 1); + this.MessagePackSerializer_TestSerialization((ulong)byte.MaxValue); + this.MessagePackSerializer_TestSerialization((ulong)byte.MaxValue + 1); + this.MessagePackSerializer_TestSerialization((ulong)ushort.MaxValue - 1); + this.MessagePackSerializer_TestSerialization((ulong)ushort.MaxValue); + this.MessagePackSerializer_TestSerialization((ulong)ushort.MaxValue + 1); + this.MessagePackSerializer_TestSerialization((ulong)uint.MaxValue - 1); + this.MessagePackSerializer_TestSerialization((ulong)uint.MaxValue); + this.MessagePackSerializer_TestSerialization((ulong)uint.MaxValue + 1); + this.MessagePackSerializer_TestSerialization(ulong.MaxValue - 1); + this.MessagePackSerializer_TestSerialization(ulong.MaxValue); + } + + [Fact] + [Trait("Platform", "Any")] + public void MessagePackSerializer_Float() + { + this.MessagePackSerializer_TestSerialization(0.0f); + this.MessagePackSerializer_TestSerialization(1.0f); + this.MessagePackSerializer_TestSerialization(-123.45f); + this.MessagePackSerializer_TestSerialization(float.MaxValue); + this.MessagePackSerializer_TestSerialization(float.MinValue); + this.MessagePackSerializer_TestSerialization(float.PositiveInfinity); + this.MessagePackSerializer_TestSerialization(float.NegativeInfinity); + + this.MessagePackSerializer_TestSerialization(0.0d); + this.MessagePackSerializer_TestSerialization(3.1415926d); + this.MessagePackSerializer_TestSerialization(-67.89f); + this.MessagePackSerializer_TestSerialization(double.MaxValue); + this.MessagePackSerializer_TestSerialization(double.MinValue); + this.MessagePackSerializer_TestSerialization(double.PositiveInfinity); + this.MessagePackSerializer_TestSerialization(double.NegativeInfinity); + } + + [Fact] + [Trait("Platform", "Any")] + public void MessagePackSerializer_SerializeAsciiString() + { + this.MessagePackSerializer_TestASCIIStringSerialization(null); + this.MessagePackSerializer_TestASCIIStringSerialization(string.Empty); + this.MessagePackSerializer_TestASCIIStringSerialization("Hello world!"); + + // fixstr stores a byte array whose length is upto 31 bytes + this.MessagePackSerializer_TestASCIIStringSerialization("1234567890123456789012345678901"); + + // str 8 stores a byte array whose length is upto (2^8)-1 bytes + this.MessagePackSerializer_TestASCIIStringSerialization("12345678901234567890123456789012"); + this.MessagePackSerializer_TestASCIIStringSerialization(new string('A', byte.MaxValue)); + this.MessagePackSerializer_TestASCIIStringSerialization(new string('B', byte.MaxValue + 1)); + this.MessagePackSerializer_TestASCIIStringSerialization(new string('Z', (1 << 14) - 1)); + this.MessagePackSerializer_TestASCIIStringSerialization(new string('Z', 1 << 14)); + + // Unicode special characters + // SerializeAsciiString will encode non-ASCII characters with '?' + Assert.Throws(() => this.MessagePackSerializer_TestASCIIStringSerialization("\u0418")); + } + + [Fact] + [Trait("Platform", "Any")] + public void MessagePackSerializer_SerializeUnicodeString() + { + this.MessagePackSerializer_TestUnicodeStringSerialization(null); + this.MessagePackSerializer_TestUnicodeStringSerialization(string.Empty); + this.MessagePackSerializer_TestUnicodeStringSerialization("Hello world!"); + + // fixstr stores a byte array whose length is upto 31 bytes + this.MessagePackSerializer_TestUnicodeStringSerialization("1234567890123456789012345678901"); + + // str 8 stores a byte array whose length is upto (2^8)-1 bytes + this.MessagePackSerializer_TestUnicodeStringSerialization("12345678901234567890123456789012"); + this.MessagePackSerializer_TestUnicodeStringSerialization(new string('A', byte.MaxValue)); + this.MessagePackSerializer_TestUnicodeStringSerialization(new string('B', byte.MaxValue + 1)); + this.MessagePackSerializer_TestUnicodeStringSerialization(new string('Z', (1 << 14) - 1)); + this.MessagePackSerializer_TestUnicodeStringSerialization(new string('Z', 1 << 14)); + + // ill-formed UTF-8 sequence + // This is replaced by `U+FFFD REPLACEMENT CHARACTER` in the returned string instance constructed from the byte array + // TODO: Update this test case once the serializer starts to throw exception for ill-formed UTF-8 sequence. + Assert.Throws(() => this.MessagePackSerializer_TestUnicodeStringSerialization("\uD801\uD802")); + + // Unicode special characters + this.MessagePackSerializer_TestUnicodeStringSerialization("\u0418"); + this.MessagePackSerializer_TestUnicodeStringSerialization(new string('\u0418', 31)); + this.MessagePackSerializer_TestUnicodeStringSerialization(new string('\u0418', 50)); + this.MessagePackSerializer_TestUnicodeStringSerialization(new string('\u0418', (1 << 8) - 1)); + this.MessagePackSerializer_TestUnicodeStringSerialization(new string('\u0418', 1 << 10)); + this.MessagePackSerializer_TestUnicodeStringSerialization(new string('\u0418', (1 << 14) - 1)); + this.MessagePackSerializer_TestUnicodeStringSerialization(new string('\u0418', 1 << 14)); + + // Unicode regular and special characters + this.MessagePackSerializer_TestUnicodeStringSerialization("\u0418TestString"); + this.MessagePackSerializer_TestUnicodeStringSerialization("TestString\u0418"); + this.MessagePackSerializer_TestUnicodeStringSerialization("Test\u0418String"); + } + + [Fact] + [Trait("Platform", "Any")] + public void MessagePackSerializer_Array() + { + this.MessagePackSerializer_TestSerialization((object[])null); + this.MessagePackSerializer_TestSerialization(new object[0]); + + // This object array has a custom string which will be serialized as STR16 + var objectArrayWithString = new object[] + { + "foo", + 1, + 0.6180340f, + 3.14159265358979323846264d, + }; + + var buffer = new byte[64 * 1024]; + _ = MessagePackSerializer.Serialize(buffer, 0, objectArrayWithString); + var objectArrayWithStringDeserialized = MessagePack.MessagePackSerializer.Deserialize(buffer); + Assert.Equal(objectArrayWithString.Length, objectArrayWithStringDeserialized.Length); + Assert.Equal(objectArrayWithString[0], objectArrayWithStringDeserialized[0]); + Assert.Equal(objectArrayWithString[1], Convert.ToInt32(objectArrayWithStringDeserialized[1])); + Assert.Equal(objectArrayWithString[2], objectArrayWithStringDeserialized[2]); + Assert.Equal(objectArrayWithString[3], objectArrayWithStringDeserialized[3]); + } + + [Fact] + [Trait("Platform", "Any")] + public void MessagePackSerializer_Map() + { + this.MessagePackSerializer_TestSerialization((Dictionary)null); + this.MessagePackSerializer_TestSerialization(new Dictionary()); + + // This dictionary has custom strings which will be serialized as STR16 + var dictionaryWithStrings = new Dictionary + { + ["foo"] = 1, + ["bar"] = "baz", + ["golden ratio"] = 0.6180340f, + ["pi"] = 3.14159265358979323846264d, + }; + var buffer = new byte[64 * 1024]; + _ = MessagePackSerializer.Serialize(buffer, 0, dictionaryWithStrings); + var dictionaryWithStringsDeserialized = MessagePack.MessagePackSerializer.Deserialize>(buffer); + Assert.Equal(dictionaryWithStrings.Count, dictionaryWithStringsDeserialized.Count); + Assert.Equal(dictionaryWithStrings["foo"], Convert.ToInt32(dictionaryWithStringsDeserialized["foo"])); + Assert.Equal(dictionaryWithStrings["bar"], dictionaryWithStringsDeserialized["bar"]); + Assert.Equal(dictionaryWithStrings["golden ratio"], dictionaryWithStringsDeserialized["golden ratio"]); + Assert.Equal(dictionaryWithStrings["pi"], dictionaryWithStringsDeserialized["pi"]); + } + } +} diff --git a/test/OpenTelemetry.Exporter.Geneva.UnitTest/MetricsContract.cs b/test/OpenTelemetry.Exporter.Geneva.UnitTest/MetricsContract.cs new file mode 100644 index 00000000000..eb813cdde1a --- /dev/null +++ b/test/OpenTelemetry.Exporter.Geneva.UnitTest/MetricsContract.cs @@ -0,0 +1,665 @@ +// + +using System.Collections.Generic; +using Kaitai; + +namespace OpenTelemetry.Exporter.Geneva.UnitTest +{ + public partial class MetricsContract : KaitaiStruct + { + public static MetricsContract FromFile(string fileName) + { + return new MetricsContract(new KaitaiStream(fileName)); + } + + + public enum MetricEventType + { + Old = 0, + Uint64Metric = 50, + DoubleScaledToLongMetric = 51, + BatchMetric = 52, + ExternallyAggregatedUlongMetric = 53, + ExternallyAggregatedDoubleMetric = 54, + DoubleMetric = 55, + ExternallyAggregatedUlongDistributionMetric = 56, + ExternallyAggregatedDoubleDistributionMetric = 57, + ExternallyAggregatedDoubleScaledToLongDistributionMetric = 58, + } + + public enum DistributionType + { + Bucketed = 0, + MonBucketed = 1, + ValueCountPairs = 2, + } + public MetricsContract(KaitaiStream p__io, KaitaiStruct p__parent = null, MetricsContract p__root = null) + : base(p__io) + { + this.m_parent = p__parent; + this.m_root = p__root ?? this; + this._read(); + } + private void _read() + { + this._eventId = this.m_io.ReadU2le(); + this._lenBody = this.m_io.ReadU2le(); + this.__raw_body = this.m_io.ReadBytes(this.LenBody); + var io___raw_body = new KaitaiStream(this.__raw_body); + this._body = new Userdata(this.EventId, io___raw_body, this, this.m_root); + } + + /// + /// This type represents "UserData" or "body" portion of Metrics message. + /// + public partial class Userdata : KaitaiStruct + { + public Userdata(ushort p_eventId, KaitaiStream p__io, MetricsContract p__parent = null, MetricsContract p__root = null) + : base(p__io) + { + this.m_parent = p__parent; + this.m_root = p__root; + this._eventId = p_eventId; + this.f_eventType = false; + this._read(); + } + private void _read() + { + this._numDimensions = this.m_io.ReadU2le(); + this._padding = this.m_io.ReadBytes(2); + switch (this.EventType) + { + case MetricsContract.MetricEventType.ExternallyAggregatedDoubleScaledToLongDistributionMetric: + { + this._valueSection = new ExtAggregatedDoubleValue(this.m_io, this, this.m_root); + break; + } + case MetricsContract.MetricEventType.DoubleMetric: + { + this._valueSection = new SingleDoubleValue(this.m_io, this, this.m_root); + break; + } + case MetricsContract.MetricEventType.ExternallyAggregatedUlongMetric: + { + this._valueSection = new ExtAggregatedUint64Value(this.m_io, this, this.m_root); + break; + } + case MetricsContract.MetricEventType.ExternallyAggregatedUlongDistributionMetric: + { + this._valueSection = new ExtAggregatedUint64Value(this.m_io, this, this.m_root); + break; + } + case MetricsContract.MetricEventType.ExternallyAggregatedDoubleDistributionMetric: + { + this._valueSection = new ExtAggregatedDoubleValue(this.m_io, this, this.m_root); + break; + } + case MetricsContract.MetricEventType.DoubleScaledToLongMetric: + { + this._valueSection = new SingleDoubleValue(this.m_io, this, this.m_root); + break; + } + case MetricsContract.MetricEventType.Uint64Metric: + { + this._valueSection = new SingleUint64Value(this.m_io, this, this.m_root); + break; + } + case MetricsContract.MetricEventType.ExternallyAggregatedDoubleMetric: + { + this._valueSection = new ExtAggregatedDoubleValue(this.m_io, this, this.m_root); + break; + } + case MetricsContract.MetricEventType.Old: + { + this._valueSection = new SingleUint64Value(this.m_io, this, this.m_root); + break; + } + } + this._metricAccount = new LenString(this.m_io, this, this.m_root); + this._metricNamespace = new LenString(this.m_io, this, this.m_root); + this._metricName = new LenString(this.m_io, this, this.m_root); + this._dimensionsNames = new List((int)this.NumDimensions); + for (var i = 0; i < this.NumDimensions; i++) + { + this._dimensionsNames.Add(new LenString(this.m_io, this, this.m_root)); + } + this._dimensionsValues = new List((int)this.NumDimensions); + for (var i = 0; i < this.NumDimensions; i++) + { + this._dimensionsValues.Add(new LenString(this.m_io, this, this.m_root)); + } + if (!this.M_Io.IsEof) + { + this._apContainer = new LenString(this.m_io, this, this.m_root); + } + if (((this.EventType == MetricsContract.MetricEventType.ExternallyAggregatedUlongDistributionMetric) || (this.EventType == MetricsContract.MetricEventType.ExternallyAggregatedDoubleDistributionMetric) || (this.EventType == MetricsContract.MetricEventType.ExternallyAggregatedDoubleScaledToLongDistributionMetric)) && (!this.M_Io.IsEof)) + { + this._histogram = new Histogram(this.m_io, this, this.m_root); + } + } + private bool f_eventType; + private MetricEventType _eventType; + public MetricEventType EventType + { + get + { + if (this.f_eventType) + return this._eventType; + this._eventType = (MetricEventType)(MetricsContract.MetricEventType)this.EventId; + this.f_eventType = true; + return this._eventType; + } + } + private ushort _numDimensions; + private byte[] _padding; + private KaitaiStruct _valueSection; + private LenString _metricAccount; + private LenString _metricNamespace; + private LenString _metricName; + private List _dimensionsNames; + private List _dimensionsValues; + private LenString _apContainer; + private Histogram _histogram; + private ushort _eventId; + private MetricsContract m_root; + private MetricsContract m_parent; + + /// + /// Number of dimensions specified in this event. + /// + public ushort NumDimensions { get { return this._numDimensions; } } + public byte[] Padding { get { return this._padding; } } + + /// + /// Value section of the body, stores fixed numeric metric value(s), as per event type. + /// + public KaitaiStruct ValueSection { get { return this._valueSection; } } + + /// + /// Geneva Metrics account name to be used for this metric. + /// + public LenString MetricAccount { get { return this._metricAccount; } } + + /// + /// Geneva Metrics namespace name to be used for this metric. + /// + public LenString MetricNamespace { get { return this._metricNamespace; } } + + /// + /// Geneva Metrics metric name to be used. + /// + public LenString MetricName { get { return this._metricName; } } + + /// + /// Dimension names strings ("key" parts of key-value pairs). Must be sorted, + /// unless MetricsExtenion's option `enableDimensionSortingOnIngestion` is + /// enabled. + /// + public List DimensionsNames { get { return this._dimensionsNames; } } + + /// + /// Dimension values strings ("value" parts of key-value pairs). + /// + public List DimensionsValues { get { return this._dimensionsValues; } } + + /// + /// AutoPilot container string, required for correct AP PKI certificate loading + /// in AutoPilot containers environment. + /// + public LenString ApContainer { get { return this._apContainer; } } + public Histogram Histogram { get { return this._histogram; } } + + /// + /// Type of message, affects format of the body. + /// + public ushort EventId { get { return this._eventId; } } + public MetricsContract M_Root { get { return this.m_root; } } + public MetricsContract M_Parent { get { return this.m_parent; } } + } + + /// + /// Bucket with an explicitly-defined value coordinate `value`, claiming to + /// hold `count` hits. Normally used to represent non-linear (e.g. exponential) + /// histograms payloads. + /// + public partial class PairValueCount : KaitaiStruct + { + public static PairValueCount FromFile(string fileName) + { + return new PairValueCount(new KaitaiStream(fileName)); + } + + public PairValueCount(KaitaiStream p__io, MetricsContract.HistogramValueCountPairs p__parent = null, MetricsContract p__root = null) + : base(p__io) + { + this.m_parent = p__parent; + this.m_root = p__root; + this._read(); + } + private void _read() + { + this._value = this.m_io.ReadU8le(); + this._count = this.m_io.ReadU4le(); + } + private ulong _value; + private uint _count; + private MetricsContract m_root; + private MetricsContract.HistogramValueCountPairs m_parent; + public ulong Value { get { return this._value; } } + public uint Count { get { return this._count; } } + public MetricsContract M_Root { get { return this.m_root; } } + public MetricsContract.HistogramValueCountPairs M_Parent { get { return this.m_parent; } } + } + + /// + /// Payload of a histogram with linear distribution of buckets. Such histogram + /// is defined by the parameters specified in `min`, `bucket_size` and + /// `bucket_count`. It is modelled as a series of buckets. First (index 0) and + /// last (indexed `bucket_count - 1`) buckets are special and are supposed to + /// catch all "underflow" and "overflow" values. Buckets with indexes 1 up to + /// `bucket_count - 2` are regular buckets of size `bucket_size`. + /// + public partial class HistogramUint16Bucketed : KaitaiStruct + { + public static HistogramUint16Bucketed FromFile(string fileName) + { + return new HistogramUint16Bucketed(new KaitaiStream(fileName)); + } + + public HistogramUint16Bucketed(KaitaiStream p__io, MetricsContract.Histogram p__parent = null, MetricsContract p__root = null) + : base(p__io) + { + this.m_parent = p__parent; + this.m_root = p__root; + this._read(); + } + private void _read() + { + this._min = this.m_io.ReadU8le(); + this._bucketSize = this.m_io.ReadU4le(); + this._bucketCount = this.m_io.ReadU4le(); + this._distributionSize = this.m_io.ReadU2le(); + this._columns = new List((int)this.DistributionSize); + for (var i = 0; i < this.DistributionSize; i++) + { + this._columns.Add(new PairUint16(this.m_io, this, this.m_root)); + } + } + private ulong _min; + private uint _bucketSize; + private uint _bucketCount; + private ushort _distributionSize; + private List _columns; + private MetricsContract m_root; + private MetricsContract.Histogram m_parent; + public ulong Min { get { return this._min; } } + public uint BucketSize { get { return this._bucketSize; } } + public uint BucketCount { get { return this._bucketCount; } } + public ushort DistributionSize { get { return this._distributionSize; } } + public List Columns { get { return this._columns; } } + public MetricsContract M_Root { get { return this.m_root; } } + public MetricsContract.Histogram M_Parent { get { return this.m_parent; } } + } + public partial class HistogramValueCountPairs : KaitaiStruct + { + public static HistogramValueCountPairs FromFile(string fileName) + { + return new HistogramValueCountPairs(new KaitaiStream(fileName)); + } + + public HistogramValueCountPairs(KaitaiStream p__io, MetricsContract.Histogram p__parent = null, MetricsContract p__root = null) + : base(p__io) + { + this.m_parent = p__parent; + this.m_root = p__root; + this._read(); + } + private void _read() + { + this._distributionSize = this.m_io.ReadU2le(); + this._columns = new List((int)this.DistributionSize); + for (var i = 0; i < this.DistributionSize; i++) + { + this._columns.Add(new PairValueCount(this.m_io, this, this.m_root)); + } + } + private ushort _distributionSize; + private List _columns; + private MetricsContract m_root; + private MetricsContract.Histogram m_parent; + public ushort DistributionSize { get { return this._distributionSize; } } + public List Columns { get { return this._columns; } } + public MetricsContract M_Root { get { return this.m_root; } } + public MetricsContract.Histogram M_Parent { get { return this.m_parent; } } + } + public partial class Histogram : KaitaiStruct + { + public static Histogram FromFile(string fileName) + { + return new Histogram(new KaitaiStream(fileName)); + } + + public Histogram(KaitaiStream p__io, MetricsContract.Userdata p__parent = null, MetricsContract p__root = null) + : base(p__io) + { + this.m_parent = p__parent; + this.m_root = p__root; + this._read(); + } + private void _read() + { + this._version = this.m_io.ReadU1(); + this._type = (MetricsContract.DistributionType)this.m_io.ReadU1(); + switch (this.Type) + { + case MetricsContract.DistributionType.Bucketed: + { + this._body = new HistogramUint16Bucketed(this.m_io, this, this.m_root); + break; + } + case MetricsContract.DistributionType.MonBucketed: + { + this._body = new HistogramUint16Bucketed(this.m_io, this, this.m_root); + break; + } + case MetricsContract.DistributionType.ValueCountPairs: + { + this._body = new HistogramValueCountPairs(this.m_io, this, this.m_root); + break; + } + } + } + private byte _version; + private DistributionType _type; + private KaitaiStruct _body; + private MetricsContract m_root; + private MetricsContract.Userdata m_parent; + public byte Version { get { return this._version; } } + public DistributionType Type { get { return this._type; } } + public KaitaiStruct Body { get { return this._body; } } + public MetricsContract M_Root { get { return this.m_root; } } + public MetricsContract.Userdata M_Parent { get { return this.m_parent; } } + } + + /// + /// Bucket #index, claiming to hold exactly `count` hits. See notes in + /// `histogram_uint16_bucketed` for interpreting index. + /// + public partial class PairUint16 : KaitaiStruct + { + public static PairUint16 FromFile(string fileName) + { + return new PairUint16(new KaitaiStream(fileName)); + } + + public PairUint16(KaitaiStream p__io, MetricsContract.HistogramUint16Bucketed p__parent = null, MetricsContract p__root = null) + : base(p__io) + { + this.m_parent = p__parent; + this.m_root = p__root; + this._read(); + } + private void _read() + { + this._index = this.m_io.ReadU2le(); + this._count = this.m_io.ReadU2le(); + } + private ushort _index; + private ushort _count; + private MetricsContract m_root; + private MetricsContract.HistogramUint16Bucketed m_parent; + public ushort Index { get { return this._index; } } + public ushort Count { get { return this._count; } } + public MetricsContract M_Root { get { return this.m_root; } } + public MetricsContract.HistogramUint16Bucketed M_Parent { get { return this.m_parent; } } + } + public partial class SingleUint64Value : KaitaiStruct + { + public static SingleUint64Value FromFile(string fileName) + { + return new SingleUint64Value(new KaitaiStream(fileName)); + } + + public SingleUint64Value(KaitaiStream p__io, MetricsContract.Userdata p__parent = null, MetricsContract p__root = null) + : base(p__io) + { + this.m_parent = p__parent; + this.m_root = p__root; + this._read(); + } + private void _read() + { + this._padding = this.m_io.ReadBytes(4); + this._timestamp = this.m_io.ReadU8le(); + this._value = this.m_io.ReadU8le(); + } + private byte[] _padding; + private ulong _timestamp; + private ulong _value; + private MetricsContract m_root; + private MetricsContract.Userdata m_parent; + public byte[] Padding { get { return this._padding; } } + + /// + /// Timestamp in Windows FILETIME format, i.e. number of 100 ns ticks passed since 1601-01-01 00:00:00 UTC. + /// + public ulong Timestamp { get { return this._timestamp; } } + + /// + /// Metric value as 64-bit unsigned integer. + /// + public ulong Value { get { return this._value; } } + public MetricsContract M_Root { get { return this.m_root; } } + public MetricsContract.Userdata M_Parent { get { return this.m_parent; } } + } + public partial class ExtAggregatedDoubleValue : KaitaiStruct + { + public static ExtAggregatedDoubleValue FromFile(string fileName) + { + return new ExtAggregatedDoubleValue(new KaitaiStream(fileName)); + } + + public ExtAggregatedDoubleValue(KaitaiStream p__io, MetricsContract.Userdata p__parent = null, MetricsContract p__root = null) + : base(p__io) + { + this.m_parent = p__parent; + this.m_root = p__root; + this._read(); + } + private void _read() + { + this._count = this.m_io.ReadU4le(); + this._timestamp = this.m_io.ReadU8le(); + this._sum = this.m_io.ReadF8le(); + this._min = this.m_io.ReadF8le(); + this._max = this.m_io.ReadF8le(); + } + private uint _count; + private ulong _timestamp; + private double _sum; + private double _min; + private double _max; + private MetricsContract m_root; + private MetricsContract.Userdata m_parent; + + /// + /// Count of events aggregated in this event. + /// + public uint Count { get { return this._count; } } + + /// + /// Timestamp in Windows FILETIME format, i.e. number of 100 ns ticks passed since 1601-01-01 00:00:00 UTC. + /// + public ulong Timestamp { get { return this._timestamp; } } + + /// + /// Sum of all metric values aggregated in this event. + /// + public double Sum { get { return this._sum; } } + + /// + /// Minimum of all metric values aggregated in this event. + /// + public double Min { get { return this._min; } } + + /// + /// Maximum of all metric values aggregated in this event. + /// + public double Max { get { return this._max; } } + public MetricsContract M_Root { get { return this.m_root; } } + public MetricsContract.Userdata M_Parent { get { return this.m_parent; } } + } + public partial class ExtAggregatedUint64Value : KaitaiStruct + { + public static ExtAggregatedUint64Value FromFile(string fileName) + { + return new ExtAggregatedUint64Value(new KaitaiStream(fileName)); + } + + public ExtAggregatedUint64Value(KaitaiStream p__io, MetricsContract.Userdata p__parent = null, MetricsContract p__root = null) + : base(p__io) + { + this.m_parent = p__parent; + this.m_root = p__root; + this._read(); + } + private void _read() + { + this._count = this.m_io.ReadU4le(); + this._timestamp = this.m_io.ReadU8le(); + this._sum = this.m_io.ReadU8le(); + this._min = this.m_io.ReadU8le(); + this._max = this.m_io.ReadU8le(); + } + private uint _count; + private ulong _timestamp; + private ulong _sum; + private ulong _min; + private ulong _max; + private MetricsContract m_root; + private MetricsContract.Userdata m_parent; + + /// + /// Count of events aggregated in this event. + /// + public uint Count { get { return this._count; } } + + /// + /// Timestamp in Windows FILETIME format, i.e. number of 100 ns ticks passed since 1601-01-01 00:00:00 UTC. + /// + public ulong Timestamp { get { return this._timestamp; } } + + /// + /// Sum of all metric values aggregated in this event. + /// + public ulong Sum { get { return this._sum; } } + + /// + /// Minimum of all metric values aggregated in this event. + /// + public ulong Min { get { return this._min; } } + + /// + /// Maximum of all metric values aggregated in this event. + /// + public ulong Max { get { return this._max; } } + public MetricsContract M_Root { get { return this.m_root; } } + public MetricsContract.Userdata M_Parent { get { return this.m_parent; } } + } + public partial class SingleDoubleValue : KaitaiStruct + { + public static SingleDoubleValue FromFile(string fileName) + { + return new SingleDoubleValue(new KaitaiStream(fileName)); + } + + public SingleDoubleValue(KaitaiStream p__io, MetricsContract.Userdata p__parent = null, MetricsContract p__root = null) + : base(p__io) + { + this.m_parent = p__parent; + this.m_root = p__root; + this._read(); + } + private void _read() + { + this._padding = this.m_io.ReadBytes(4); + this._timestamp = this.m_io.ReadU8le(); + this._value = this.m_io.ReadF8le(); + } + private byte[] _padding; + private ulong _timestamp; + private double _value; + private MetricsContract m_root; + private MetricsContract.Userdata m_parent; + public byte[] Padding { get { return this._padding; } } + + /// + /// Timestamp in Windows FILETIME format, i.e. number of 100 ns ticks passed since 1601-01-01 00:00:00 UTC. + /// + public ulong Timestamp { get { return this._timestamp; } } + + /// + /// Metric value as double. + /// + public double Value { get { return this._value; } } + public MetricsContract M_Root { get { return this.m_root; } } + public MetricsContract.Userdata M_Parent { get { return this.m_parent; } } + } + + /// + /// A simple string, length-prefixed with a 2-byte integer. + /// + public partial class LenString : KaitaiStruct + { + public static LenString FromFile(string fileName) + { + return new LenString(new KaitaiStream(fileName)); + } + + public LenString(KaitaiStream p__io, MetricsContract.Userdata p__parent = null, MetricsContract p__root = null) + : base(p__io) + { + this.m_parent = p__parent; + this.m_root = p__root; + this._read(); + } + private void _read() + { + this._lenValue = this.m_io.ReadU2le(); + this._value = System.Text.Encoding.GetEncoding("UTF-8").GetString(this.m_io.ReadBytes(this.LenValue)); + } + private ushort _lenValue; + private string _value; + private MetricsContract m_root; + private MetricsContract.Userdata m_parent; + public ushort LenValue { get { return this._lenValue; } } + public string Value { get { return this._value; } } + public MetricsContract M_Root { get { return this.m_root; } } + public MetricsContract.Userdata M_Parent { get { return this.m_parent; } } + } + private ushort _eventId; + private ushort _lenBody; + private Userdata _body; + private MetricsContract m_root; + private KaitaiStruct m_parent; + private byte[] __raw_body; + + /// + /// Type of message, affects format of the body. + /// + public ushort EventId { get { return this._eventId; } } + + /// + /// Size of body in bytes. + /// + public ushort LenBody { get { return this._lenBody; } } + + /// + /// Body of Metrics binary protocol message. + /// + public Userdata Body { get { return this._body; } } + public MetricsContract M_Root { get { return this.m_root; } } + public KaitaiStruct M_Parent { get { return this.m_parent; } } + public byte[] M_RawBody { get { return this.__raw_body; } } + } +} diff --git a/test/OpenTelemetry.Exporter.Geneva.UnitTest/OpenTelemetry.Exporter.Geneva.UnitTest.csproj b/test/OpenTelemetry.Exporter.Geneva.UnitTest/OpenTelemetry.Exporter.Geneva.UnitTest.csproj new file mode 100644 index 00000000000..372035fd34b --- /dev/null +++ b/test/OpenTelemetry.Exporter.Geneva.UnitTest/OpenTelemetry.Exporter.Geneva.UnitTest.csproj @@ -0,0 +1,40 @@ + + + + Unit test project for Geneva Exporters for OpenTelemetry + false + netcoreapp3.1;net5.0;net6.0 + $(TargetFrameworks);net461;net462;net47;net471;net472;net48 + $(NoWarn),SA1633,SA1311,SA1312,SA1313,SA1123,SA1202 + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers + + + + + + + + + + + + + + diff --git a/test/OpenTelemetry.Exporter.Geneva.UnitTest/UnixDomainSocketDataTransportTests.cs b/test/OpenTelemetry.Exporter.Geneva.UnitTest/UnixDomainSocketDataTransportTests.cs new file mode 100644 index 00000000000..6a7cb34567f --- /dev/null +++ b/test/OpenTelemetry.Exporter.Geneva.UnitTest/UnixDomainSocketDataTransportTests.cs @@ -0,0 +1,195 @@ +using System; +using System.IO; +using System.Net.Sockets; +using System.Reflection; +using Xunit; + +namespace OpenTelemetry.Exporter.Geneva.UnitTest +{ + public class UnixDomainSocketDataTransportTests + { + [Fact] + [Trait("Platform", "Linux")] + public void UnixDomainSocketDataTransport_Success() + { + string path = GetRandomFilePath(); + var endpoint = new UnixDomainSocketEndPoint(path); + try + { + using var server = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); + server.Bind(endpoint); + server.Listen(1); + + // Client + using var dataTransport = new UnixDomainSocketDataTransport(path); + using Socket serverSocket = server.Accept(); + var data = new byte[] { 12, 34, 56 }; + dataTransport.Send(data, data.Length); + var receivedData = new byte[5]; + serverSocket.Receive(receivedData); + Assert.Equal(data[0], receivedData[0]); + Assert.Equal(data[1], receivedData[1]); + Assert.Equal(data[2], receivedData[2]); + } + catch (Exception) + { + throw; + } + finally + { + try + { + File.Delete(path); + } + catch + { + } + } + } + + [Fact] + [Trait("Platform", "Linux")] + public void UnixDomainSocketDataTransport_SendTimesOutIfSocketBufferFull() + { + string path = GetRandomFilePath(); + var endpoint = new UnixDomainSocketEndPoint(path); + using var server = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); + server.Bind(endpoint); + server.Listen(1); + var data = new byte[1024]; + var i = 0; + using var dataTransport = new UnixDomainSocketDataTransport(path, 5000); // Set low timeout for faster tests + var socket = typeof(UnixDomainSocketDataTransport).GetField("socket", BindingFlags.NonPublic | BindingFlags.Instance).GetValue(dataTransport) as Socket; + try + { + // Client + using Socket serverSocket = server.Accept(); + while (true) + { + Console.WriteLine($"Sending request #{i++}."); + socket.Send(data, data.Length, SocketFlags.None); + } + + // The server is not processing sent data (because of heavy load, etc.) + } + catch (Exception) + { + // At this point, the outgoing buffer for the socket must be full, + // because the last Send failed. + // Send again and assert the exception to confirm: + Assert.Throws(() => + { + Console.WriteLine($"Sending request #{i}."); + socket.Send(data, data.Length, SocketFlags.None); + }); + } + finally + { + try + { + File.Delete(path); + } + catch + { + } + } + } + + [Fact] + [Trait("Platform", "Linux")] + public void UnixDomainSocketDataTransport_ServerRestart() + { + Console.WriteLine("Test starts."); + string path = GetRandomFilePath(); + var endpoint = new UnixDomainSocketEndPoint(path); + try + { + var server = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); + + // LingerOption lo = new LingerOption(false, 0); + // server.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.Linger, lo); + server.Bind(endpoint); + server.Listen(1); + + // Client + using var dataTransport = new UnixDomainSocketDataTransport(path); + Socket serverSocket = server.Accept(); + var data = new byte[] { 12, 34, 56 }; + dataTransport.Send(data, data.Length); + var receivedData = new byte[5]; + serverSocket.Receive(receivedData); + Assert.Equal(data[0], receivedData[0]); + Assert.Equal(data[1], receivedData[1]); + Assert.Equal(data[2], receivedData[2]); + + Console.WriteLine("Successfully sent a message."); + + // Emulate server stops + serverSocket.Shutdown(SocketShutdown.Both); + serverSocket.Disconnect(false); + serverSocket.Dispose(); + server.Shutdown(SocketShutdown.Both); + server.Disconnect(false); + server.Dispose(); + try + { + File.Delete(path); + } + catch + { + } + + Console.WriteLine("Destroyed server."); + + Console.WriteLine("Client will fail during Send, but shouldn't throw exception."); + dataTransport.Send(data, data.Length); + Console.WriteLine("Client will fail during reconnect, but shouldn't throw exception."); + dataTransport.Send(data, data.Length); + + using var server2 = new Socket(AddressFamily.Unix, SocketType.Stream, ProtocolType.IP); + server2.Bind(endpoint); + server2.Listen(1); + Console.WriteLine("Started a new server and listening."); + + var data2 = new byte[] { 34, 56, 78 }; + dataTransport.Send(data2, data2.Length); + Console.WriteLine("The same client sent a new message. Internally it should reconnect if server ever stopped and the socket is not connected anymore."); + + using Socket serverSocket2 = server2.Accept(); + Console.WriteLine("The new server is ready and accepting connections."); + var receivedData2 = new byte[5]; + serverSocket2.Receive(receivedData2); + Console.WriteLine("Server received a messge."); + Assert.Equal(data2[0], receivedData2[0]); + Assert.Equal(data2[1], receivedData2[1]); + Assert.Equal(data2[2], receivedData2[2]); + } + catch (Exception) + { + throw; + } + finally + { + try + { + File.Delete(path); + } + catch + { + } + } + } + + private static string GetRandomFilePath() + { + while (true) + { + string path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + if (!File.Exists(path)) + { + return path; + } + } + } + } +} diff --git a/test/OpenTelemetry.Exporter.Geneva.UnitTest/UnixDomainSocketEndPointTests.cs b/test/OpenTelemetry.Exporter.Geneva.UnitTest/UnixDomainSocketEndPointTests.cs new file mode 100644 index 00000000000..938eae9983e --- /dev/null +++ b/test/OpenTelemetry.Exporter.Geneva.UnitTest/UnixDomainSocketEndPointTests.cs @@ -0,0 +1,68 @@ +using System; +using System.Net; +using System.Net.Sockets; +using System.Text; +using Xunit; + +namespace OpenTelemetry.Exporter.Geneva.UnitTest +{ + public class UnixDomainSocketEndPointTests + { + [Fact] + public void UnixDomainSocketEndPoint_constructor_InvalidArgument() + { + Assert.Throws(() => _ = new UnixDomainSocketEndPoint(null)); + Assert.Throws(() => _ = new UnixDomainSocketEndPoint(string.Empty)); + Assert.Throws(() => _ = new UnixDomainSocketEndPoint(new string('a', 100))); + } + + [Fact] + public void UnixDomainSocketEndPoint_constructor_Success() + { + var endpoint = new UnixDomainSocketEndPoint("abc"); + Assert.Equal("abc", endpoint.ToString()); + } + + [Fact] + public void UnixDomainSocketEndPoint_Create_InvalidArgument() + { + var endpoint = new UnixDomainSocketEndPoint("abc"); + Assert.Throws(() => _ = endpoint.Create(null)); + Assert.Throws(() => _ = endpoint.Create(this.CreateSocketAddress(new string('a', 100)))); + } + + [Fact] + public void UnixDomainSocketEndPoint_Create_Success() + { + var endpoint = new UnixDomainSocketEndPoint("abc"); + + var sa = new SocketAddress(AddressFamily.Unix, 2); // SocketAddress size is 2 + Assert.Equal(string.Empty, endpoint.Create(sa).ToString()); + + Assert.Equal("\0", endpoint.Create(this.CreateSocketAddress(string.Empty)).ToString()); + Assert.Equal("test\0", endpoint.Create(this.CreateSocketAddress("test")).ToString()); + } + + [Fact] + public void UnixDomainSocketEndPoint_Serialize() + { + var path = "abc"; + var endpoint = new UnixDomainSocketEndPoint(path); + Assert.Equal(this.CreateSocketAddress(path), endpoint.Serialize()); + } + + private SocketAddress CreateSocketAddress(string path) + { + int NativePathOffset = 2; + var nativePath = Encoding.UTF8.GetBytes(path); + var sa = new SocketAddress(AddressFamily.Unix, NativePathOffset + nativePath.Length + 1); + for (int i = 0; i < nativePath.Length; ++i) + { + sa[NativePathOffset + i] = nativePath[i]; + } + + sa[NativePathOffset + nativePath.Length] = 0; + return sa; + } + } +}