Introduce ResilienceEventSeverity (#1361)

bench/Polly.Core.Benchmarks/TelemetryBenchmark.cs

@@ -78,7 +78,7 @@ protected override ValueTask<Outcome<TResult>> ExecuteCoreAsync<TResult, TState>
             ResilienceContext context,
             TState state)
         {
-            _telemetry.Report("DummyEvent", context, "dummy-args");
+            _telemetry.Report(new ResilienceEvent(ResilienceEventSeverity.Warning, "DummyEvent"), context, "dummy-args");
             return callback(context, state);
         }
     }

src/Polly.Core/CircuitBreaker/Controller/CircuitStateController.cs

@@ -128,7 +128,7 @@ public ValueTask CloseCircuitAsync(ResilienceContext context)
             if (_circuitState == CircuitState.Open && PermitHalfOpenCircuitTest_NeedsLock())
             {
                 _circuitState = CircuitState.HalfOpen;
-                _telemetry.Report(CircuitBreakerConstants.OnHalfOpenEvent, context, new OnCircuitHalfOpenedArguments());
+                _telemetry.Report(new(ResilienceEventSeverity.Warning, CircuitBreakerConstants.OnHalfOpenEvent), context, new OnCircuitHalfOpenedArguments());
                 isHalfOpen = true;
             }
 
@@ -269,7 +269,7 @@ private void CloseCircuit_NeedsLock(Outcome<T> outcome, bool manual, ResilienceC
         if (priorState != CircuitState.Closed)
         {
             var args = new OutcomeArguments<T, OnCircuitClosedArguments>(context, outcome, new OnCircuitClosedArguments(manual));
-            _telemetry.Report(CircuitBreakerConstants.OnCircuitClosed, args);
+            _telemetry.Report(new(ResilienceEventSeverity.Information, CircuitBreakerConstants.OnCircuitClosed), args);
 
             if (_onClosed is not null)
             {
@@ -320,7 +320,7 @@ private void OpenCircuitFor_NeedsLock(Outcome<T> outcome, TimeSpan breakDuration
         _circuitState = CircuitState.Open;
 
         var args = new OutcomeArguments<T, OnCircuitOpenedArguments>(context, outcome, new OnCircuitOpenedArguments(breakDuration, manual));
-        _telemetry.Report(CircuitBreakerConstants.OnCircuitOpened, args);
+        _telemetry.Report(new(ResilienceEventSeverity.Error, CircuitBreakerConstants.OnCircuitOpened), args);
 
         if (_onOpened is not null)
         {

src/Polly.Core/Fallback/FallbackResilienceStrategy.cs

@@ -29,7 +29,7 @@ protected override async ValueTask<Outcome<T>> ExecuteCallbackAsync<TState>(Func
 
         var onFallbackArgs = new OutcomeArguments<T, OnFallbackArguments>(context, outcome, new OnFallbackArguments());
 
-        _telemetry.Report(FallbackConstants.OnFallback, onFallbackArgs);
+        _telemetry.Report(new(ResilienceEventSeverity.Warning, FallbackConstants.OnFallback), onFallbackArgs);
 
         if (_onFallback is not null)
         {

src/Polly.Core/Hedging/HedgingResilienceStrategy.cs

@@ -126,7 +126,7 @@ private async ValueTask HandleOnHedgingAsync(ResilienceContext context, Outcome<
             outcome,
             args);
 
-        _telemetry.Report(HedgingConstants.OnHedgingEventName, onHedgingArgs);
+        _telemetry.Report(new(ResilienceEventSeverity.Warning, HedgingConstants.OnHedgingEventName), onHedgingArgs);
 
         if (OnHedging is not null)
         {

src/Polly.Core/ResilienceContext.cs

@@ -64,7 +64,7 @@ private ResilienceContext()
     /// <remarks>
     /// If the number of resilience events is greater than zero it's an indication that the execution was unhealthy.
     /// </remarks>
-    public IReadOnlyCollection<ResilienceEvent> ResilienceEvents => _resilienceEvents;
+    public IReadOnlyList<ResilienceEvent> ResilienceEvents => _resilienceEvents;
 
     /// <summary>
     /// Gets a <see cref="ResilienceContext"/> instance from the pool.

src/Polly.Core/Retry/RetryResilienceStrategy.cs

@@ -74,7 +74,7 @@ protected override async ValueTask<Outcome<T>> ExecuteCallbackAsync<TState>(Func
             }
 
             var onRetryArgs = new OutcomeArguments<T, OnRetryArguments>(context, outcome, new OnRetryArguments(attempt, delay, executionTime));
-            _telemetry.Report(RetryConstants.OnRetryEvent, onRetryArgs);
+            _telemetry.Report(new(ResilienceEventSeverity.Warning, RetryConstants.OnRetryEvent), onRetryArgs);
 
             if (OnRetry is not null)
             {

src/Polly.Core/Telemetry/ResilienceEvent.cs

@@ -3,11 +3,12 @@ namespace Polly.Telemetry;
 /// <summary>
 /// Represents a resilience event that has been reported.
 /// </summary>
+/// <param name="Severity">The severity of the event.</param>
 /// <param name="EventName">The event name.</param>
 /// <remarks>
 /// Always use the constructor when creating this struct, otherwise we do not guarantee binary compatibility.
 /// </remarks>
-public readonly record struct ResilienceEvent(string EventName)
+public readonly record struct ResilienceEvent(ResilienceEventSeverity Severity, string EventName)
 {
     /// <summary>
     /// Returns an <see cref="EventName"/>.

src/Polly.Core/Telemetry/ResilienceEventSeverity.cs

@@ -0,0 +1,37 @@
+namespace Polly.Telemetry;
+
+/// <summary>
+/// The severity of reported resilience event.
+/// </summary>
+public enum ResilienceEventSeverity
+{
+    /// <summary>
+    /// The resilience event is not recorded.
+    /// </summary>
+    None = 0,
+
+    /// <summary>
+    /// The resilience event is used for debugging purposes only.
+    /// </summary>
+    Debug,
+
+    /// <summary>
+    /// The resilience event is informational.
+    /// </summary>
+    Information,
+
+    /// <summary>
+    /// The resilience event should be treated as a warning.
+    /// </summary>
+    Warning,
+
+    /// <summary>
+    /// The resilience event should be treated as an error.
+    /// </summary>
+    Error,
+
+    /// <summary>
+    /// The resilience event should be treated as a critical error.
+    /// </summary>
+    Critical,
+}

src/Polly.Core/Telemetry/ResilienceStrategyTelemetry.cs

@@ -27,25 +27,24 @@ internal ResilienceStrategyTelemetry(ResilienceTelemetrySource source, Diagnosti
     /// Reports an event that occurred in a resilience strategy.
     /// </summary>
     /// <typeparam name="TArgs">The arguments associated with this event.</typeparam>
-    /// <param name="eventName">The event name.</param>
+    /// <param name="resilienceEvent">The reported resilience event.</param>
     /// <param name="context">The resilience context associated with this event.</param>
     /// <param name="args">The event arguments.</param>
-    /// <exception cref="ArgumentNullException">Thrown when <paramref name="eventName"/> is <see langword="null"/>.</exception>
-    public void Report<TArgs>(string eventName, ResilienceContext context, TArgs args)
+    /// <exception cref="ArgumentNullException">Thrown when <paramref name="context"/> is <see langword="null"/>.</exception>
+    public void Report<TArgs>(ResilienceEvent resilienceEvent, ResilienceContext context, TArgs args)
     {
-        Guard.NotNull(eventName);
         Guard.NotNull(context);
 
-        AddResilienceEvent(eventName, context, args);
+        context.AddResilienceEvent(resilienceEvent);
 
-        if (DiagnosticSource is null || !DiagnosticSource.IsEnabled(eventName))
+        if (DiagnosticSource is null || !DiagnosticSource.IsEnabled(resilienceEvent.EventName) || resilienceEvent.Severity == ResilienceEventSeverity.None)
         {
             return;
         }
 
-        var telemetryArgs = TelemetryEventArguments.Get(TelemetrySource, eventName, context, null, args!);
+        var telemetryArgs = TelemetryEventArguments.Get(TelemetrySource, resilienceEvent, context, null, args!);
 
-        DiagnosticSource.Write(eventName, telemetryArgs);
+        DiagnosticSource.Write(resilienceEvent.EventName, telemetryArgs);
 
         TelemetryEventArguments.Return(telemetryArgs);
     }
@@ -55,37 +54,22 @@ public void Report<TArgs>(string eventName, ResilienceContext context, TArgs arg
     /// </summary>
     /// <typeparam name="TArgs">The arguments associated with this event.</typeparam>
     /// <typeparam name="TResult">The type of the result.</typeparam>
-    /// <param name="eventName">The event name.</param>
+    /// <param name="resilienceEvent">The reported resilience event.</param>
     /// <param name="args">The event arguments.</param>
-    /// <exception cref="ArgumentNullException">Thrown when <paramref name="eventName"/> is <see langword="null"/>.</exception>
-    public void Report<TArgs, TResult>(string eventName, OutcomeArguments<TResult, TArgs> args)
+    public void Report<TArgs, TResult>(ResilienceEvent resilienceEvent, OutcomeArguments<TResult, TArgs> args)
     {
-        Guard.NotNull(eventName);
+        args.Context.AddResilienceEvent(resilienceEvent);
 
-        AddResilienceEvent(eventName, args.Context, args.Arguments);
-
-        if (DiagnosticSource is null || !DiagnosticSource.IsEnabled(eventName))
+        if (DiagnosticSource is null || !DiagnosticSource.IsEnabled(resilienceEvent.EventName) || resilienceEvent.Severity == ResilienceEventSeverity.None)
         {
             return;
         }
 
-        var telemetryArgs = TelemetryEventArguments.Get(TelemetrySource, eventName, args.Context, args.Outcome.AsOutcome(), args.Arguments!);
+        var telemetryArgs = TelemetryEventArguments.Get(TelemetrySource, resilienceEvent, args.Context, args.Outcome.AsOutcome(), args.Arguments!);
 
-        DiagnosticSource.Write(eventName, telemetryArgs);
+        DiagnosticSource.Write(resilienceEvent.EventName, telemetryArgs);
 
         TelemetryEventArguments.Return(telemetryArgs);
     }
-
-    private static void AddResilienceEvent<TArgs>(string eventName, ResilienceContext context, TArgs args)
-    {
-        // ExecutionAttemptArguments is not reported as resilience event because that information is already contained
-        // in OnHedgingArguments and OnRetryArguments
-        if (args is ExecutionAttemptArguments attempt)
-        {
-            return;
-        }
-
-        context.AddResilienceEvent(new ResilienceEvent(eventName));
-    }
 }
 

src/Polly.Core/Telemetry/TelemetryEventArguments.Pool.cs

@@ -5,23 +5,23 @@ public sealed partial record class TelemetryEventArguments
     private static readonly ObjectPool<TelemetryEventArguments> Pool = new(() => new TelemetryEventArguments(), args =>
     {
         args.Source = null!;
-        args.EventName = null!;
+        args.Event = default;
         args.Context = null!;
         args.Outcome = default;
         args.Arguments = null!;
     });
 
     internal static TelemetryEventArguments Get(
         ResilienceTelemetrySource source,
-        string eventName,
+        ResilienceEvent resilienceEvent,
         ResilienceContext context,
         Outcome<object>? outcome,
         object arguments)
     {
         var args = Pool.Get();
 
         args.Source = source;
-        args.EventName = eventName;
+        args.Event = resilienceEvent;
         args.Context = context;
         args.Outcome = outcome;
         args.Arguments = arguments;

src/Polly.Core/Telemetry/TelemetryEventArguments.cs

@@ -18,9 +18,9 @@ private TelemetryEventArguments()
     public ResilienceTelemetrySource Source { get; private set; } = null!;
 
     /// <summary>
-    /// Gets the event name.
+    /// Gets the event.
     /// </summary>
-    public string EventName { get; private set; } = null!;
+    public ResilienceEvent Event { get; private set; }
 
     /// <summary>
     /// Gets the resilience context.

src/Polly.Core/Telemetry/TelemetryUtil.cs

@@ -34,7 +34,9 @@ public static void ReportExecutionAttempt<TResult>(
         }
 
         var attemptArgs = ExecutionAttemptArguments.Get(attempt, executionTime, handled);
-        telemetry.Report<ExecutionAttemptArguments, TResult>(ExecutionAttempt, new(context, outcome, attemptArgs));
+        telemetry.Report<ExecutionAttemptArguments, TResult>(
+            new(handled ? ResilienceEventSeverity.Warning : ResilienceEventSeverity.Information, ExecutionAttempt),
+            new(context, outcome, attemptArgs));
         ExecutionAttemptArguments.Return(attemptArgs);
     }
 }

src/Polly.Core/Timeout/TimeoutResilienceStrategy.cs

@@ -60,7 +60,7 @@ protected internal override async ValueTask<Outcome<TResult>> ExecuteCoreAsync<T
         if (isCancellationRequested && outcome.Exception is OperationCanceledException e && !previousToken.IsCancellationRequested)
         {
             var args = new OnTimeoutArguments(context, e, timeout);
-            _telemetry.Report(TimeoutConstants.OnTimeoutEvent, context, args);
+            _telemetry.Report(new(ResilienceEventSeverity.Error, TimeoutConstants.OnTimeoutEvent), context, args);
 
             if (OnTimeout is not null)
             {

src/Polly.Core/Utils/ReloadableResilienceStrategy.cs

@@ -10,6 +10,8 @@ internal sealed class ReloadableResilienceStrategy : ResilienceStrategy
 
     public const string ReloadFailedEvent = "ReloadFailed";
 
+    public const string OnReloadEvent = "OnReload";
+
     private readonly Func<CancellationToken> _onReload;
     private readonly Func<ResilienceStrategy> _resilienceStrategyFactory;
     private readonly ResilienceStrategyTelemetry _telemetry;
@@ -50,16 +52,18 @@ private void RegisterOnReload(CancellationToken previousToken)
 
         _registration = token.Register(() =>
         {
+            var context = ResilienceContext.Get().Initialize<VoidResult>(isSynchronous: true);
+
 #pragma warning disable CA1031 // Do not catch general exception types
             try
             {
+                _telemetry.Report(new(ResilienceEventSeverity.Information, OnReloadEvent), context, new OnReloadArguments());
                 Strategy = _resilienceStrategyFactory();
             }
             catch (Exception e)
             {
-                var context = ResilienceContext.Get().Initialize<VoidResult>(isSynchronous: true);
                 var args = new OutcomeArguments<VoidResult, ReloadFailedArguments>(context, Outcome.FromException(e), new ReloadFailedArguments(e));
-                _telemetry.Report(ReloadFailedEvent, args);
+                _telemetry.Report(new(ResilienceEventSeverity.Error, ReloadFailedEvent), args);
                 ResilienceContext.Return(context);
             }
 #pragma warning restore CA1031 // Do not catch general exception types
@@ -70,4 +74,6 @@ private void RegisterOnReload(CancellationToken previousToken)
     }
 
     internal readonly record struct ReloadFailedArguments(Exception Exception);
+
+    internal readonly record struct OnReloadArguments();
 }