OpenTelemetry support

src/WebJobs.Script.Grpc/Channel/GrpcWorkerChannel.cs

@@ -20,6 +20,7 @@
 using Microsoft.Azure.WebJobs.Script.Config;
 using Microsoft.Azure.WebJobs.Script.Description;
 using Microsoft.Azure.WebJobs.Script.Diagnostics;
+using Microsoft.Azure.WebJobs.Script.Diagnostics.OpenTelemetry;
 using Microsoft.Azure.WebJobs.Script.Eventing;
 using Microsoft.Azure.WebJobs.Script.Extensions;
 using Microsoft.Azure.WebJobs.Script.Grpc.Eventing;
@@ -565,6 +566,12 @@ internal void ApplyCapabilities(IDictionary<string, string> capabilities, GrpcCa
                 _isWorkerApplicationInsightsLoggingEnabled = true;
             }
 
+            if (bool.TryParse(_workerCapabilities.GetCapabilityState(RpcWorkerConstants.WorkerOpenTelemetryEnabled), out bool otelEnabled) &&
+                otelEnabled)
+            {
+                ScriptHost.WorkerOpenTelemetryEnabled = true;
+            }
+
             // If http proxying is enabled, we need to get the proxying endpoint of this worker
             var httpUri = _workerCapabilities.GetCapabilityState(RpcWorkerConstants.HttpUri);
             if (!string.IsNullOrEmpty(httpUri))
@@ -1670,29 +1677,56 @@ private void AddSample<T>(List<T> samples, T sample)
 
         private void AddAdditionalTraceContext(MapField<string, string> attributes, ScriptInvocationContext context)
         {
-            // This is only applicable for AI agents running along side worker
+            bool isOtelEnabled = _scriptHostOptions?.Value.TelemetryMode == TelemetryMode.OpenTelemetry;
+            if (_scriptHostOptions?.Value.TelemetryMode == TelemetryMode.OpenTelemetry)
+            {
+                isOtelEnabled = true;
+            }
+
+            bool isAIEnabled = _environment.IsApplicationInsightsAgentEnabled();
             if (_environment.IsApplicationInsightsAgentEnabled())
             {
-                attributes[ScriptConstants.LogPropertyProcessIdKey] = Convert.ToString(_rpcWorkerProcess.Id);
+                isAIEnabled = true;
+            }
+
+            if (isOtelEnabled || isAIEnabled)
+            {
                 if (context.FunctionMetadata.Properties.TryGetValue(ScriptConstants.LogPropertyHostInstanceIdKey, out var hostInstanceIdValue))
                 {
-                    attributes[ScriptConstants.LogPropertyHostInstanceIdKey] = Convert.ToString(hostInstanceIdValue);
-                }
-                if (context.FunctionMetadata.Properties.TryGetValue(LogConstants.CategoryNameKey, out var categoryNameValue))
-                {
-                    attributes[LogConstants.CategoryNameKey] = Convert.ToString(categoryNameValue);
+                    string id = Convert.ToString(hostInstanceIdValue);
+
+                    if (isOtelEnabled)
+                    {
+                        Activity.Current?.AddTag(ResourceAttributeConstants.AttributeInstance, id);
+                    }
+                    if (isAIEnabled)
+                    {
+                        attributes[ScriptConstants.LogPropertyHostInstanceIdKey] = id;
+                    }
                 }
-                string sessionid = Activity.Current?.GetBaggageItem(ScriptConstants.LiveLogsSessionAIKey);
-                if (!string.IsNullOrEmpty(sessionid))
+            }
+
+            if (isAIEnabled)
+            {
+                attributes[ScriptConstants.LogPropertyProcessIdKey] = Convert.ToString(_rpcWorkerProcess.Id);
+                attributes[ScriptConstants.OperationNameKey] = context.FunctionMetadata.Name;
+                string sessionId = Activity.Current?.GetBaggageItem(ScriptConstants.LiveLogsSessionAIKey);
+                if (!string.IsNullOrEmpty(sessionId))
                 {
-                    attributes[ScriptConstants.LiveLogsSessionAIKey] = sessionid;
+                    attributes[ScriptConstants.LiveLogsSessionAIKey] = sessionId;
                 }
-                string operationName = context.FunctionMetadata.Name;
-                if (!string.IsNullOrEmpty(operationName))
+
+                if (context.FunctionMetadata.Properties.TryGetValue(LogConstants.CategoryNameKey, out var categoryNameValue))
                 {
-                    attributes[ScriptConstants.OperationNameKey] = operationName;
+                    attributes[LogConstants.CategoryNameKey] = Convert.ToString(categoryNameValue);
                 }
             }
+
+            if (isOtelEnabled)
+            {
+                Activity.Current?.AddTag(ResourceAttributeConstants.AttributeName, context.FunctionMetadata.Name);
+                Activity.Current?.AddTag(ResourceAttributeConstants.AttributeTrigger, ResourceAttributeConstants.ResolveTriggerType(context.FunctionMetadata?.Trigger?.Type));
+            }
         }
 
         private sealed class ExecutingInvocation : IDisposable

src/WebJobs.Script.WebHost/Features/FunctionExecutionFeature.cs

@@ -3,11 +3,13 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Http;
 using Microsoft.Azure.WebJobs.Script.Description;
 using Microsoft.Azure.WebJobs.Script.Diagnostics;
+using Microsoft.Azure.WebJobs.Script.Diagnostics.OpenTelemetry;
 using Microsoft.Azure.WebJobs.Script.Extensions;
 using Microsoft.Azure.WebJobs.Script.Host;
 using Microsoft.Extensions.Logging;
@@ -58,6 +60,9 @@ public async Task ExecuteAsync(HttpRequest request, CancellationToken cancellati
                 {
                     coldStartData.Add("dispatchDuration", dispatchStopwatch.GetElapsedTime().TotalMilliseconds);
                 }
+
+                // Add tag for cold start
+                Activity.Current?.AddTag(ResourceAttributeConstants.AttributeColdStart, true);
             }
 
             var sw = ValueStopwatch.StartNew();

src/WebJobs.Script.WebHost/Metrics/HostMetricsProvider.cs

@@ -8,6 +8,7 @@
 using System.Linq;
 using System.Threading;
 using Microsoft.Azure.WebJobs.Host.Executors;
+using Microsoft.Azure.WebJobs.Script.Diagnostics.OpenTelemetry;
 using Microsoft.Azure.WebJobs.Script.Metrics;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
@@ -60,7 +61,7 @@ public void Start()
                 {
                     listener.EnableMeasurementEvents(instrument);
 
-                    var instanceIdTag = instrument.Meter.Tags.FirstOrDefault(t => t.Key == TelemetryAttributes.ServiceInstanceId);
+                    var instanceIdTag = instrument.Meter.Tags.FirstOrDefault(t => t.Key == ResourceAttributeConstants.ServiceInstanceId);
                     InstanceId = instanceIdTag.Value?.ToString() ?? string.Empty;
                 }
             };

src/WebJobs.Script.WebHost/WebJobsScriptHostService.cs

@@ -28,6 +28,9 @@
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using OpenTelemetry.Logs;
+using OpenTelemetry.Metrics;
+using OpenTelemetry.Trace;
 using static Microsoft.Azure.WebJobs.Script.EnvironmentSettingNames;
 using IApplicationLifetime = Microsoft.AspNetCore.Hosting.IApplicationLifetime;
 
@@ -875,6 +878,7 @@ private async Task Orphan(IHost instance, CancellationToken cancellationToken =
                     instance.Dispose();
                     _logger.LogDebug("ScriptHost disposed");
                 }
+                FlushOpenTelemetry(instance);
             }
         }
 
@@ -958,6 +962,28 @@ public object GetService(Type serviceType)
             return Services?.GetService(serviceType);
         }
 
+        private void FlushOpenTelemetry(IHost host)
+        {
+            var logger = GetHostLogger(host);
+            foreach (var meterProvider in host.Services.GetServices<MeterProvider>())
+            {
+                logger.LogDebug(@"Flushing {providerName} ...", meterProvider);
+                meterProvider.ForceFlush();
+            }
+
+            foreach (var tracerProvider in host.Services.GetServices<TracerProvider>())
+            {
+                logger.LogDebug(@"Flushing {providerName} ...", tracerProvider);
+                tracerProvider.ForceFlush();
+            }
+
+            foreach (var logProvider in host.Services.GetServices<ILoggerProvider>().Where(i => i is OpenTelemetryLoggerProvider))
+            {
+                logger.LogDebug(@"Disposing {providerName} ...", logProvider);
+                logProvider.Dispose();
+            }
+        }
+
         protected virtual void Dispose(bool disposing)
         {
             if (!_disposed)

src/WebJobs.Script/Config/ConfigurationSectionNames.cs

@@ -24,5 +24,6 @@ public static class ConfigurationSectionNames
         public const string Retry = "retry";
         public const string SequentialJobHostRestart = JobHost + ":sequentialRestart";
         public const string SendCanceledInvocationsToWorker = "sendCanceledInvocationsToWorker";
+        public const string TelemetryMode = "telemetrymode";
     }
 }

src/WebJobs.Script/Config/ScriptJobHostOptions.cs

@@ -6,6 +6,7 @@
 using System.Collections.Immutable;
 using System.Collections.ObjectModel;
 using Microsoft.Azure.WebJobs.Script.Description;
+using Microsoft.Azure.WebJobs.Script.Diagnostics.OpenTelemetry;
 
 namespace Microsoft.Azure.WebJobs.Script
 {
@@ -42,7 +43,7 @@ public string RootScriptPath
         public string InstanceId { get; }
 
         /// <summary>
-        /// Gets or sets NugetFallBackPath
+        /// Gets or sets NugetFallBackPath.
         /// </summary>
         public string NugetFallBackPath { get; set; }
 
@@ -129,5 +130,10 @@ public string RootScriptPath
         /// invocation to the worker.
         /// </summary>
         public bool SendCanceledInvocationsToWorker { get; set; } = true;
+
+        /// <summary>
+        /// Gets or sets the telemetry mode.
+        /// </summary>
+        internal TelemetryMode TelemetryMode { get; set; } = TelemetryMode.ApplicationInsights;
     }
 }

src/WebJobs.Script/Config/ScriptJobHostOptionsSetup.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System;
+using Microsoft.Azure.WebJobs.Script.Diagnostics.OpenTelemetry;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Options;
 
@@ -73,6 +74,12 @@ public void Configure(ScriptJobHostOptions options)
             options.TestDataPath = webHostOptions.TestDataPath;
             options.IsFileSystemReadOnly = webHostOptions.IsFileSystemReadOnly;
             options.IsStandbyConfiguration = webHostOptions.IsStandbyConfiguration;
+
+            var telemetryModeSection = jobHostSection.GetSection(ConfigurationSectionNames.TelemetryMode);
+            if (telemetryModeSection.Exists() && Enum.TryParse(telemetryModeSection.Value, true, out TelemetryMode telemetryMode))
+            {
+                options.TelemetryMode = telemetryMode;
+            }
         }
 
         private void ConfigureFunctionTimeout(ScriptJobHostOptions options)

src/WebJobs.Script/Config/ScriptSettingsManager.cs

@@ -64,6 +64,18 @@ public virtual string ApplicationInsightsConnectionString
             set => SetSetting(EnvironmentSettingNames.AppInsightsConnectionString, value);
         }
 
+        public virtual string OtlpEndpoint
+        {
+            get => GetSetting(EnvironmentSettingNames.OtlpEndpoint);
+            set => SetSetting(EnvironmentSettingNames.OtlpEndpoint, value);
+        }
+
+        public virtual string TelemetryMode
+        {
+            get => GetSetting(EnvironmentSettingNames.TelemetryMode);
+            set => SetSetting(EnvironmentSettingNames.TelemetryMode, value);
+        }
+
         public virtual string GetSetting(string settingKey)
         {
             if (string.IsNullOrEmpty(settingKey))

src/WebJobs.Script/Diagnostics/MetricEventNames.cs

@@ -9,6 +9,8 @@ public static class MetricEventNames
         public const string ApplicationStartLatency = "host.application.start";
         public const string ApplicationInsightsEnabled = "host.applicationinsights.enabled";
         public const string ApplicationInsightsDisabled = "host.applicationinsights.disabled";
+        public const string OpenTelemetryAzMonEnabled = "host.otel.azmon.enabled";
+        public const string OpenTelemetryOtlpEnabled = "host.opentelemetry.otlp.enabled";
         public const string HostStartupLatency = "host.startup.latency";
         public const string HostStartupReadFunctionMetadataLatency = "host.startup.readfunctionmetadata.latency";
         public const string HostStartupInitializeBindingProvidersLatency = "host.startup.initializebindingproviders.latency";

src/WebJobs.Script/Diagnostics/OpenTelemetry/ActivitySanitizingProcessor.cs

@@ -0,0 +1,38 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System.Collections.Immutable;
+using System.Diagnostics;
+using Microsoft.Azure.WebJobs.Logging;
+using OpenTelemetry;
+
+namespace Microsoft.Azure.WebJobs.Script.Diagnostics.OpenTelemetry
+{
+    internal class ActivitySanitizingProcessor : BaseProcessor<Activity>
+    {
+        private static readonly ImmutableArray<string> TagsToSanitize = ImmutableArray.Create("url.query", "url.full");
+
+        private ActivitySanitizingProcessor() { }
+
+        public static ActivitySanitizingProcessor Instance { get; } = new ActivitySanitizingProcessor();
+
+        public override void OnEnd(Activity data)
+        {
+            Sanitize(data);
+
+            base.OnEnd(data);
+        }
+
+        private static void Sanitize(Activity data)
+        {
+            foreach (var t in TagsToSanitize)
+            {
+                if (data.GetTagItem(t) is string s and not null)
+                {
+                    var sanitizedValue = Sanitizer.Sanitize(s);
+                    data.SetTag(t, sanitizedValue);
+                }
+            }
+        }
+    }
+}

src/WebJobs.Script/Diagnostics/OpenTelemetry/FunctionsResourceDetector.cs

@@ -0,0 +1,76 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Reflection;
+using OpenTelemetry.Resources;
+
+namespace Microsoft.Azure.WebJobs.Script.Diagnostics.OpenTelemetry
+{
+    internal sealed class FunctionsResourceDetector : IResourceDetector
+    {
+        internal static readonly IReadOnlyDictionary<string, string> ResourceAttributes = new Dictionary<string, string>(1)
+        {
+            { ResourceAttributeConstants.CloudRegion, ResourceAttributeConstants.RegionNameEnvVar },
+        };
+
+        public Resource Detect()
+        {
+            List<KeyValuePair<string, object>> attributeList = new(5);
+            try
+            {
+                var siteName = Environment.GetEnvironmentVariable(ResourceAttributeConstants.SiteNameEnvVar);
+                attributeList.Add(new KeyValuePair<string, object>(ResourceAttributeConstants.CloudProvider, ResourceAttributeConstants.AzureCloudProviderValue));
+                attributeList.Add(new KeyValuePair<string, object>(ResourceAttributeConstants.CloudPlatform, ResourceAttributeConstants.AzurePlatformValue));
+
+                var version = Assembly.GetEntryAssembly()?.GetName().Version.ToString() ?? "0:0:0";
+                attributeList.Add(new KeyValuePair<string, object>(ResourceAttributeConstants.AttributeVersion, version));
+                if (!string.IsNullOrEmpty(siteName))
+                {
+                    var azureResourceUri = GetAzureResourceURI(siteName);
+                    if (azureResourceUri != null)
+                    {
+                        attributeList.Add(new KeyValuePair<string, object>(ResourceAttributeConstants.CloudResourceId, azureResourceUri));
+                    }
+
+                    foreach (var kvp in ResourceAttributes)
+                    {
+                        var attributeValue = Environment.GetEnvironmentVariable(kvp.Value);
+                        if (attributeValue != null)
+                        {
+                            attributeList.Add(new KeyValuePair<string, object>(kvp.Key, attributeValue));
+                        }
+                    }
+                }
+            }
+            catch
+            {
+                // return empty resource.
+                return Resource.Empty;
+            }
+
+            return new Resource(attributeList);
+        }
+
+        private static string GetAzureResourceURI(string websiteSiteName)
+        {
+            string websiteResourceGroup = Environment.GetEnvironmentVariable(ResourceAttributeConstants.ResourceGroupEnvVar);
+            string websiteOwnerName = Environment.GetEnvironmentVariable(ResourceAttributeConstants.OwnerNameEnvVar) ?? string.Empty;
+
+#if NET6_0_OR_GREATER
+            int idx = websiteOwnerName.IndexOf('+', StringComparison.Ordinal);
+#else
+            int idx = websiteOwnerName.IndexOf("+", StringComparison.Ordinal);
+#endif
+            string subscriptionId = idx > 0 ? websiteOwnerName.Substring(0, idx) : websiteOwnerName;
+
+            if (string.IsNullOrEmpty(websiteResourceGroup) || string.IsNullOrEmpty(subscriptionId))
+            {
+                return null;
+            }
+
+            return $"/subscriptions/{subscriptionId}/resourceGroups/{websiteResourceGroup}/providers/Microsoft.Web/sites/{websiteSiteName}";
+        }
+    }
+}