[Flex] Adding support for OpenTelemetry (#10094)

* OpenTelemetry support (#9985) Adding support for OpenTelemetry --------- Co-authored-by: Jacob Viau <javia@microsoft.com> Co-authored-by: Shyju Krishnankutty <connectshyju@gmail.com> Co-authored-by: Daniel Castro <castro.daniel@microsoft.com> * Updating Otel nuget packages. (#10014) * Updating Otel nuget packages. * Update Azure.Identity to 1.11.2 (#10037) * Update Microsoft.Identity.Client to 4.60.3 * Update Azure.Identity to 1.11.2 * Remove Microsoft.Identity.Client * Update WebJobs package in test projects (#10061) * Update PowerShell workers to latest (#10016) * Update Python Worker Version to 4.28.0 (#10009) Co-authored-by: AzureFunctionsPython <funcdisc@microsoft.com> * Update Node.js Worker to 3.10.0 (#9999) * Update Node.js Worker to 3.10.0 * Add PR reference * Update Python Worker Version to 4.28.1 (#10028) Co-authored-by: AzureFunctionsPython <funcdisc@microsoft.com> --------- Co-authored-by: Jacob Viau <javia@microsoft.com> Co-authored-by: andystaples <77818326+andystaples@users.noreply.github.com> Co-authored-by: gavin-aguiar <80794152+gavin-aguiar@users.noreply.github.com> Co-authored-by: AzureFunctionsPython <funcdisc@microsoft.com> Co-authored-by: Daniel Castro <castro.daniel@microsoft.com>
Azure · May 2, 2024 · ee32ce6 · ee32ce6
1 parent 018e67c
commit ee32ce6
Show file tree

Hide file tree

Showing 44 changed files with 1,622 additions and 114 deletions.
diff --git a/build/python.props b/build/python.props
@@ -1,5 +1,5 @@
 <Project>
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.Functions.PythonWorker" Version="4.27.0" />
+    <PackageReference Include="Microsoft.Azure.Functions.PythonWorker" Version="4.28.1" />
   </ItemGroup>
 </Project>
diff --git a/release_notes.md b/release_notes.md
@@ -3,6 +3,11 @@
 <!-- Please add your release notes in the following format:
 - My change description (#PR)
 -->
+- Update Python Worker Version to [4.28.1](https://github.com/Azure/azure-functions-python-worker/releases/tag/4.28.1)
 - DotnetIsolated worker artifact clean up (#9976)
   - Move symbols from dotnet-isolated worker to symbols package
-  - Removed linux executables from dotnet-isolated worker.
+  - Removed linux executables from dotnet-isolated worker.
+- Update Azure.Identity to 1.11.0 (#10002)
+- Update PowerShell worker 7.2 to [4.0.3220](https://github.com/Azure/azure-functions-powershell-worker/releases/tag/v4.0.3220)
+- Update PowerShell worker 7.4 to [4.0.3219](https://github.com/Azure/azure-functions-powershell-worker/releases/tag/v4.0.3219)
+- Update Node.js Worker Version to [3.10.0](https://github.com/Azure/azure-functions-nodejs-worker/releases/tag/v3.10.0) (#9999)
diff --git a/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs b/src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs
@@ -20,6 +20,7 @@
 using Microsoft.Azure.WebJobs.Script.Config;
 using Microsoft.Azure.WebJobs.Script.Description;
 using Microsoft.Azure.WebJobs.Script.Diagnostics;
+using Microsoft.Azure.WebJobs.Script.Diagnostics.OpenTelemetry;
 using Microsoft.Azure.WebJobs.Script.Eventing;
 using Microsoft.Azure.WebJobs.Script.Extensions;
 using Microsoft.Azure.WebJobs.Script.Grpc.Eventing;
@@ -565,6 +566,12 @@ internal void ApplyCapabilities(IDictionary<string, string> capabilities, GrpcCa
                 _isWorkerApplicationInsightsLoggingEnabled = true;
             }
 
+            if (bool.TryParse(_workerCapabilities.GetCapabilityState(RpcWorkerConstants.WorkerOpenTelemetryEnabled), out bool otelEnabled) &&
+                otelEnabled)
+            {
+                ScriptHost.WorkerOpenTelemetryEnabled = true;
+            }
+
             // If http proxying is enabled, we need to get the proxying endpoint of this worker
             var httpUri = _workerCapabilities.GetCapabilityState(RpcWorkerConstants.HttpUri);
             if (!string.IsNullOrEmpty(httpUri))
@@ -1670,29 +1677,47 @@ private void AddSample<T>(List<T> samples, T sample)
 
         private void AddAdditionalTraceContext(MapField<string, string> attributes, ScriptInvocationContext context)
         {
-            // This is only applicable for AI agents running along side worker
-            if (_environment.IsApplicationInsightsAgentEnabled())
+            bool isOtelEnabled = _scriptHostOptions?.Value.TelemetryMode == TelemetryMode.OpenTelemetry;
+            bool isAIEnabled = _environment.IsApplicationInsightsAgentEnabled();
+
+            if (isOtelEnabled || isAIEnabled)
             {
-                attributes[ScriptConstants.LogPropertyProcessIdKey] = Convert.ToString(_rpcWorkerProcess.Id);
                 if (context.FunctionMetadata.Properties.TryGetValue(ScriptConstants.LogPropertyHostInstanceIdKey, out var hostInstanceIdValue))
                 {
-                    attributes[ScriptConstants.LogPropertyHostInstanceIdKey] = Convert.ToString(hostInstanceIdValue);
-                }
-                if (context.FunctionMetadata.Properties.TryGetValue(LogConstants.CategoryNameKey, out var categoryNameValue))
-                {
-                    attributes[LogConstants.CategoryNameKey] = Convert.ToString(categoryNameValue);
+                    string id = Convert.ToString(hostInstanceIdValue);
+
+                    if (isOtelEnabled)
+                    {
+                        Activity.Current?.AddTag(ResourceSemanticConventions.FaaSInstance, id);
+                    }
+                    if (isAIEnabled)
+                    {
+                        attributes[ScriptConstants.LogPropertyHostInstanceIdKey] = id;
+                    }
                 }
-                string sessionid = Activity.Current?.GetBaggageItem(ScriptConstants.LiveLogsSessionAIKey);
-                if (!string.IsNullOrEmpty(sessionid))
+            }
+
+            if (isAIEnabled)
+            {
+                attributes[ScriptConstants.LogPropertyProcessIdKey] = Convert.ToString(_rpcWorkerProcess.Id);
+                attributes[ScriptConstants.OperationNameKey] = context.FunctionMetadata.Name;
+                string sessionId = Activity.Current?.GetBaggageItem(ScriptConstants.LiveLogsSessionAIKey);
+                if (!string.IsNullOrEmpty(sessionId))
                 {
-                    attributes[ScriptConstants.LiveLogsSessionAIKey] = sessionid;
+                    attributes[ScriptConstants.LiveLogsSessionAIKey] = sessionId;
                 }
-                string operationName = context.FunctionMetadata.Name;
-                if (!string.IsNullOrEmpty(operationName))
+
+                if (context.FunctionMetadata.Properties.TryGetValue(LogConstants.CategoryNameKey, out var categoryNameValue))
                 {
-                    attributes[ScriptConstants.OperationNameKey] = operationName;
+                    attributes[LogConstants.CategoryNameKey] = Convert.ToString(categoryNameValue);
                 }
             }
+
+            if (isOtelEnabled)
+            {
+                Activity.Current?.AddTag(ResourceSemanticConventions.FaaSName, context.FunctionMetadata.Name);
+                Activity.Current?.AddTag(ResourceSemanticConventions.FaaSTrigger, OpenTelemetryConstants.ResolveTriggerType(context.FunctionMetadata?.Trigger?.Type));
+            }
         }
 
         private sealed class ExecutingInvocation : IDisposable

diff --git a/src/WebJobs.Script.WebHost/Features/FunctionExecutionFeature.cs b/src/WebJobs.Script.WebHost/Features/FunctionExecutionFeature.cs
@@ -3,11 +3,13 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Http;
 using Microsoft.Azure.WebJobs.Script.Description;
 using Microsoft.Azure.WebJobs.Script.Diagnostics;
+using Microsoft.Azure.WebJobs.Script.Diagnostics.OpenTelemetry;
 using Microsoft.Azure.WebJobs.Script.Extensions;
 using Microsoft.Azure.WebJobs.Script.Host;
 using Microsoft.Extensions.Logging;
@@ -58,6 +60,9 @@ public async Task ExecuteAsync(HttpRequest request, CancellationToken cancellati
                 {
                     coldStartData.Add("dispatchDuration", dispatchStopwatch.GetElapsedTime().TotalMilliseconds);
                 }
+
+                // Add tag for cold start
+                Activity.Current?.AddTag(ResourceSemanticConventions.FaaSColdStart, true);
             }
 
             var sw = ValueStopwatch.StartNew();

diff --git a/src/WebJobs.Script.WebHost/Metrics/HostMetricsProvider.cs b/src/WebJobs.Script.WebHost/Metrics/HostMetricsProvider.cs
@@ -8,6 +8,7 @@
 using System.Linq;
 using System.Threading;
 using Microsoft.Azure.WebJobs.Host.Executors;
+using Microsoft.Azure.WebJobs.Script.Diagnostics.OpenTelemetry;
 using Microsoft.Azure.WebJobs.Script.Metrics;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
@@ -60,7 +61,7 @@ public void Start()
                 {
                     listener.EnableMeasurementEvents(instrument);
 
-                    var instanceIdTag = instrument.Meter.Tags.FirstOrDefault(t => t.Key == TelemetryAttributes.ServiceInstanceId);
+                    var instanceIdTag = instrument.Meter.Tags.FirstOrDefault(t => t.Key == ResourceSemanticConventions.ServiceInstanceId);
                     InstanceId = instanceIdTag.Value?.ToString() ?? string.Empty;
                 }
             };

diff --git a/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj b/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj
@@ -62,7 +62,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Azure.Identity" Version="1.10.2" />
+    <PackageReference Include="Azure.Identity" Version="1.11.2" />
     <PackageReference Include="Azure.Security.KeyVault.Secrets" Version="4.2.0" />
     <PackageReference Include="Microsoft.ApplicationInsights" Version="2.22.0" />
     <PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.22.0" />

diff --git a/src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs b/src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs
@@ -28,6 +28,9 @@
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using OpenTelemetry.Logs;
+using OpenTelemetry.Metrics;
+using OpenTelemetry.Trace;
 using static Microsoft.Azure.WebJobs.Script.EnvironmentSettingNames;
 using IApplicationLifetime = Microsoft.AspNetCore.Hosting.IApplicationLifetime;
 
@@ -872,6 +875,7 @@ private async Task Orphan(IHost instance, CancellationToken cancellationToken =
                 else
                 {
                     DisposeDependencyTrackingModule(instance);
+                    FlushOpenTelemetry(instance);
                     instance.Dispose();
                     _logger.LogDebug("ScriptHost disposed");
                 }
@@ -958,6 +962,28 @@ public object GetService(Type serviceType)
             return Services?.GetService(serviceType);
         }
 
+        private void FlushOpenTelemetry(IHost host)
+        {
+            var logger = GetHostLogger(host);
+            foreach (var meterProvider in host.Services.GetServices<MeterProvider>())
+            {
+                logger.LogDebug(@"Flushing {providerName} ...", meterProvider);
+                meterProvider.ForceFlush();
+            }
+
+            foreach (var tracerProvider in host.Services.GetServices<TracerProvider>())
+            {
+                logger.LogDebug(@"Flushing {providerName} ...", tracerProvider);
+                tracerProvider.ForceFlush();
+            }
+
+            foreach (var logProvider in host.Services.GetServices<ILoggerProvider>().Where(i => i is OpenTelemetryLoggerProvider))
+            {
+                logger.LogDebug(@"Disposing {providerName} ...", logProvider);
+                logProvider.Dispose();
+            }
+        }
+
         protected virtual void Dispose(bool disposing)
         {
             if (!_disposed)

diff --git a/src/WebJobs.Script/Config/ConfigurationSectionNames.cs b/src/WebJobs.Script/Config/ConfigurationSectionNames.cs
@@ -24,5 +24,6 @@ public static class ConfigurationSectionNames
         public const string Retry = "retry";
         public const string SequentialJobHostRestart = JobHost + ":sequentialRestart";
         public const string SendCanceledInvocationsToWorker = "sendCanceledInvocationsToWorker";
+        public const string TelemetryMode = "telemetryMode";
     }
 }
diff --git a/src/WebJobs.Script/Config/ScriptJobHostOptions.cs b/src/WebJobs.Script/Config/ScriptJobHostOptions.cs
@@ -6,6 +6,7 @@
 using System.Collections.Immutable;
 using System.Collections.ObjectModel;
 using Microsoft.Azure.WebJobs.Script.Description;
+using Microsoft.Azure.WebJobs.Script.Diagnostics.OpenTelemetry;
 
 namespace Microsoft.Azure.WebJobs.Script
 {
@@ -42,7 +43,7 @@ public string RootScriptPath
         public string InstanceId { get; }
 
         /// <summary>
-        /// Gets or sets NugetFallBackPath
+        /// Gets or sets NugetFallBackPath.
         /// </summary>
         public string NugetFallBackPath { get; set; }
 
@@ -129,5 +130,10 @@ public string RootScriptPath
         /// invocation to the worker.
         /// </summary>
         public bool SendCanceledInvocationsToWorker { get; set; } = true;
+
+        /// <summary>
+        /// Gets or sets the telemetry mode.
+        /// </summary>
+        internal TelemetryMode TelemetryMode { get; set; } = TelemetryMode.ApplicationInsights;
     }
 }
diff --git a/src/WebJobs.Script/Config/ScriptJobHostOptionsSetup.cs b/src/WebJobs.Script/Config/ScriptJobHostOptionsSetup.cs
@@ -2,6 +2,7 @@
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System;
+using Microsoft.Azure.WebJobs.Script.Diagnostics.OpenTelemetry;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Options;
 
@@ -73,6 +74,16 @@ public void Configure(ScriptJobHostOptions options)
             options.TestDataPath = webHostOptions.TestDataPath;
             options.IsFileSystemReadOnly = webHostOptions.IsFileSystemReadOnly;
             options.IsStandbyConfiguration = webHostOptions.IsStandbyConfiguration;
+
+            var telemetryModeSection = jobHostSection.GetSection(ConfigurationSectionNames.TelemetryMode);
+            if (telemetryModeSection.Exists() && Enum.TryParse(telemetryModeSection.Value, true, out TelemetryMode telemetryMode))
+            {
+                options.TelemetryMode = telemetryMode;
+            }
+            else
+            {
+               options.TelemetryMode = TelemetryMode.ApplicationInsights;
+            }
         }
 
         private void ConfigureFunctionTimeout(ScriptJobHostOptions options)

diff --git a/src/WebJobs.Script/Config/ScriptSettingsManager.cs b/src/WebJobs.Script/Config/ScriptSettingsManager.cs
@@ -64,6 +64,12 @@ public virtual string ApplicationInsightsConnectionString
             set => SetSetting(EnvironmentSettingNames.AppInsightsConnectionString, value);
         }
 
+        public virtual string OtlpEndpoint
+        {
+            get => GetSetting(EnvironmentSettingNames.OtlpEndpoint);
+            set => SetSetting(EnvironmentSettingNames.OtlpEndpoint, value);
+        }
+
         public virtual string GetSetting(string settingKey)
         {
             if (string.IsNullOrEmpty(settingKey))

diff --git a/src/WebJobs.Script/Diagnostics/MetricEventNames.cs b/src/WebJobs.Script/Diagnostics/MetricEventNames.cs
@@ -9,6 +9,8 @@ public static class MetricEventNames
         public const string ApplicationStartLatency = "host.application.start";
         public const string ApplicationInsightsEnabled = "host.applicationinsights.enabled";
         public const string ApplicationInsightsDisabled = "host.applicationinsights.disabled";
+        public const string OpenTelemetryAzMonEnabled = "host.otel.azmon.enabled";
+        public const string OpenTelemetryOtlpEnabled = "host.otel.otlp.enabled";
         public const string HostStartupLatency = "host.startup.latency";
         public const string HostStartupReadFunctionMetadataLatency = "host.startup.readfunctionmetadata.latency";
         public const string HostStartupInitializeBindingProvidersLatency = "host.startup.initializebindingproviders.latency";

diff --git a/src/WebJobs.Script/Diagnostics/OpenTelemetry/ActivitySanitizingProcessor.cs b/src/WebJobs.Script/Diagnostics/OpenTelemetry/ActivitySanitizingProcessor.cs
@@ -0,0 +1,38 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using System.Diagnostics;
+using Microsoft.Azure.WebJobs.Logging;
+using OpenTelemetry;
+
+namespace Microsoft.Azure.WebJobs.Script.Diagnostics.OpenTelemetry
+{
+    internal class ActivitySanitizingProcessor : BaseProcessor<Activity>
+    {
+        private static readonly IReadOnlyCollection<string> TagsToSanitize = new HashSet<string> { ResourceSemanticConventions.QueryUrl, ResourceSemanticConventions.FullUrl };
+
+        private ActivitySanitizingProcessor() { }
+
+        public static ActivitySanitizingProcessor Instance { get; } = new ActivitySanitizingProcessor();
+
+        public override void OnEnd(Activity data)
+        {
+            Sanitize(data);
+
+            base.OnEnd(data);
+        }
+
+        private static void Sanitize(Activity data)
+        {
+            foreach (var t in TagsToSanitize)
+            {
+                if (data.GetTagItem(t) is string s and not null)
+                {
+                    var sanitizedValue = Sanitizer.Sanitize(s);
+                    data.SetTag(t, sanitizedValue);
+                }
+            }
+        }
+    }
+}
diff --git a/src/WebJobs.Script/Diagnostics/OpenTelemetry/FunctionsResourceDetector.cs b/src/WebJobs.Script/Diagnostics/OpenTelemetry/FunctionsResourceDetector.cs
@@ -0,0 +1,71 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Reflection;
+using OpenTelemetry.Resources;
+
+namespace Microsoft.Azure.WebJobs.Script.Diagnostics.OpenTelemetry
+{
+    internal sealed class FunctionsResourceDetector : IResourceDetector
+    {
+        public Resource Detect()
+        {
+            List<KeyValuePair<string, object>> attributeList = new(9);
+            try
+            {
+                string serviceName = Environment.GetEnvironmentVariable(OpenTelemetryConstants.SiteNameEnvVar);
+                string version = typeof(ScriptHost).Assembly.GetName().Version.ToString();
+
+                attributeList.Add(new KeyValuePair<string, object>(ResourceSemanticConventions.ServiceVersion, version));
+                attributeList.Add(new KeyValuePair<string, object>(ResourceSemanticConventions.AISDKPrefix, $@"{OpenTelemetryConstants.SDKPrefix}:{version}"));
+                attributeList.Add(new KeyValuePair<string, object>(ResourceSemanticConventions.ProcessId, Process.GetCurrentProcess().Id));
+                attributeList.Add(new KeyValuePair<string, object>(ResourceSemanticConventions.FaaSVersion, version));
+
+                // Add these attributes only if running in Azure.
+                if (!string.IsNullOrEmpty(serviceName))
+                {
+                    attributeList.Add(new KeyValuePair<string, object>(ResourceSemanticConventions.ServiceName, serviceName));
+                    attributeList.Add(new KeyValuePair<string, object>(ResourceSemanticConventions.CloudProvider, OpenTelemetryConstants.AzureCloudProviderValue));
+                    attributeList.Add(new KeyValuePair<string, object>(ResourceSemanticConventions.CloudPlatform, OpenTelemetryConstants.AzurePlatformValue));
+
+                    string region = Environment.GetEnvironmentVariable(OpenTelemetryConstants.RegionNameEnvVar);
+                    if (!string.IsNullOrEmpty(region))
+                    {
+                        attributeList.Add(new KeyValuePair<string, object>(ResourceSemanticConventions.CloudRegion, region));
+                    }
+
+                    var azureResourceUri = GetAzureResourceURI(serviceName);
+                    if (azureResourceUri != null)
+                    {
+                        attributeList.Add(new KeyValuePair<string, object>(ResourceSemanticConventions.CloudResourceId, azureResourceUri));
+                    }
+                }
+            }
+            catch
+            {
+                // return empty resource.
+                return Resource.Empty;
+            }
+
+            return new Resource(attributeList);
+        }
+
+        private static string GetAzureResourceURI(string websiteSiteName)
+        {
+            string websiteResourceGroup = Environment.GetEnvironmentVariable(OpenTelemetryConstants.ResourceGroupEnvVar);
+            string websiteOwnerName = Environment.GetEnvironmentVariable(OpenTelemetryConstants.OwnerNameEnvVar) ?? string.Empty;
+            int idx = websiteOwnerName.IndexOf('+', StringComparison.Ordinal);
+            string subscriptionId = idx > 0 ? websiteOwnerName.Substring(0, idx) : websiteOwnerName;
+
+            if (string.IsNullOrEmpty(websiteResourceGroup) || string.IsNullOrEmpty(subscriptionId))
+            {
+                return null;
+            }
+
+            return $"/subscriptions/{subscriptionId}/resourceGroups/{websiteResourceGroup}/providers/Microsoft.Web/sites/{websiteSiteName}";
+        }
+    }
+}