Update AggregatorStore to reclaim unused MetricPoints for Delta aggre…

…gation temporality (#4486)

Co-authored-by: Mikel Blanchard <mblanchard@macrosssoftware.com>
Co-authored-by: Alan West <3676547+alanwest@users.noreply.github.com>
Co-authored-by: Vishwesh Bankwar <vishweshbankwar@users.noreply.github.com>

src/OpenTelemetry/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+* Update `AggregatorStore` to reclaim unused MetricPoints for Delta aggregation
+  temporality.
+  ([#4486](https://github.com/open-telemetry/opentelemetry-dotnet/issues/4486))
+
 ## 1.6.0
 
 Released 2023-Sep-05

src/OpenTelemetry/Metrics/LookupData.cs

@@ -0,0 +1,31 @@
+// <copyright file="LookupData.cs" company="OpenTelemetry Authors">
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+// </copyright>
+
+namespace OpenTelemetry.Metrics;
+
+internal sealed class LookupData
+{
+    public int Index;
+    public Tags SortedTags;
+    public Tags GivenTags;
+
+    public LookupData(int index, in Tags sortedTags, in Tags givenTags)
+    {
+        this.Index = index;
+        this.SortedTags = sortedTags;
+        this.GivenTags = givenTags;
+    }
+}

src/OpenTelemetry/Metrics/MetricPoint.cs

@@ -24,6 +24,18 @@ namespace OpenTelemetry.Metrics;
 /// </summary>
 public struct MetricPoint
 {
+    // Represents the number of update threads using this MetricPoint at any given point of time.
+    // If the value is equal to int.MinValue which is -2147483648, it means that this MetricPoint is available for reuse.
+    // We never increment the ReferenceCount for MetricPoint with no tags (index == 0) and the MetricPoint for overflow attribute,
+    // but we always decrement it (in the Update methods). This should be fine.
+    // ReferenceCount doesn't matter for MetricPoint with no tags and overflow attribute as they are never reclaimed.
+    internal int ReferenceCount;
+
+    // When the AggregatorStore is reclaiming MetricPoints, this serves the purpose of validating the a given thread is using the right
+    // MetricPoint for update by checking it against what as added in the Dictionary. Also, when a thread finds out that the MetricPoint
+    // that its using is already reclaimed, this helps avoid sorting of the tags for adding a new Dictionary entry.
+    internal LookupData? LookupData;
+
     private const int DefaultSimpleReservoirPoolSize = 1;
 
     private readonly AggregatorStore aggregatorStore;
@@ -46,17 +58,25 @@ internal MetricPoint(
         KeyValuePair<string, object?>[]? tagKeysAndValues,
         double[] histogramExplicitBounds,
         int exponentialHistogramMaxSize,
-        int exponentialHistogramMaxScale)
+        int exponentialHistogramMaxScale,
+        LookupData? lookupData = null)
     {
         Debug.Assert(aggregatorStore != null, "AggregatorStore was null.");
         Debug.Assert(histogramExplicitBounds != null, "Histogram explicit Bounds was null.");
 
+        if (aggregatorStore!.OutputDelta)
+        {
+            Debug.Assert(lookupData != null, "LookupData was null.");
+        }
+
         this.aggType = aggType;
         this.Tags = new ReadOnlyTagCollection(tagKeysAndValues);
         this.runningValue = default;
         this.snapshotValue = default;
         this.deltaLastValue = default;
         this.MetricPointStatus = MetricPointStatus.NoCollectPending;
+        this.ReferenceCount = 1;
+        this.LookupData = lookupData;
 
         ExemplarReservoir? reservoir = null;
         if (this.aggType == AggregationType.HistogramWithBuckets ||
@@ -437,6 +457,11 @@ internal void Update(long number)
         // TODO: For Delta, this can be mitigated
         // by ignoring Zero points
         this.MetricPointStatus = MetricPointStatus.CollectPending;
+
+        if (this.aggregatorStore.OutputDelta)
+        {
+            Interlocked.Decrement(ref this.ReferenceCount);
+        }
     }
 
     internal void UpdateWithExemplar(long number, ReadOnlySpan<KeyValuePair<string, object?>> tags, bool isSampled)
@@ -551,6 +576,11 @@ internal void UpdateWithExemplar(long number, ReadOnlySpan<KeyValuePair<string,
         // TODO: For Delta, this can be mitigated
         // by ignoring Zero points
         this.MetricPointStatus = MetricPointStatus.CollectPending;
+
+        if (this.aggregatorStore.OutputDelta)
+        {
+            Interlocked.Decrement(ref this.ReferenceCount);
+        }
     }
 
     internal void Update(double number)
@@ -642,6 +672,11 @@ internal void Update(double number)
         // TODO: For Delta, this can be mitigated
         // by ignoring Zero points
         this.MetricPointStatus = MetricPointStatus.CollectPending;
+
+        if (this.aggregatorStore.OutputDelta)
+        {
+            Interlocked.Decrement(ref this.ReferenceCount);
+        }
     }
 
     internal void UpdateWithExemplar(double number, ReadOnlySpan<KeyValuePair<string, object?>> tags, bool isSampled)
@@ -762,6 +797,11 @@ internal void UpdateWithExemplar(double number, ReadOnlySpan<KeyValuePair<string
         // TODO: For Delta, this can be mitigated
         // by ignoring Zero points
         this.MetricPointStatus = MetricPointStatus.CollectPending;
+
+        if (this.aggregatorStore.OutputDelta)
+        {
+            Interlocked.Decrement(ref this.ReferenceCount);
+        }
     }
 
     internal void TakeSnapshot(bool outputDelta)

src/OpenTelemetry/Metrics/Tags.cs

@@ -18,6 +18,8 @@ namespace OpenTelemetry.Metrics;
 
 internal readonly struct Tags : IEquatable<Tags>
 {
+    public static readonly Tags EmptyTags = new(Array.Empty<KeyValuePair<string, object?>>());
+
     private readonly int hashCode;
 
     public Tags(KeyValuePair<string, object?>[] keyValuePairs)

test/Benchmarks/Metrics/MetricsBenchmarks.cs

@@ -22,27 +22,27 @@
 using OpenTelemetry.Tests;
 
 /*
-BenchmarkDotNet=v0.13.5, OS=Windows 11 (10.0.23424.1000)
+BenchmarkDotNet v0.13.6, Windows 11 (10.0.23424.1000)
 Intel Core i7-9700 CPU 3.00GHz, 1 CPU, 8 logical and 8 physical cores
-.NET SDK=7.0.203
-  [Host]     : .NET 7.0.5 (7.0.523.17405), X64 RyuJIT AVX2
-  DefaultJob : .NET 7.0.5 (7.0.523.17405), X64 RyuJIT AVX2
+.NET SDK 7.0.400
+  [Host]     : .NET 7.0.10 (7.0.1023.36312), X64 RyuJIT AVX2
+  DefaultJob : .NET 7.0.10 (7.0.1023.36312), X64 RyuJIT AVX2
 
 
 |                    Method | AggregationTemporality |      Mean |    Error |   StdDev | Allocated |
 |-------------------------- |----------------------- |----------:|---------:|---------:|----------:|
-|            CounterHotPath |             Cumulative |  17.06 ns | 0.113 ns | 0.094 ns |         - |
-| CounterWith1LabelsHotPath |             Cumulative |  71.47 ns | 1.464 ns | 2.100 ns |         - |
-| CounterWith3LabelsHotPath |             Cumulative | 162.04 ns | 2.469 ns | 2.188 ns |         - |
-| CounterWith5LabelsHotPath |             Cumulative | 237.30 ns | 2.884 ns | 2.698 ns |         - |
-| CounterWith6LabelsHotPath |             Cumulative | 269.41 ns | 4.087 ns | 3.623 ns |         - |
-| CounterWith7LabelsHotPath |             Cumulative | 303.01 ns | 5.313 ns | 4.970 ns |         - |
-|            CounterHotPath |                  Delta |  17.30 ns | 0.350 ns | 0.310 ns |         - |
-| CounterWith1LabelsHotPath |                  Delta |  70.96 ns | 0.608 ns | 0.539 ns |         - |
-| CounterWith3LabelsHotPath |                  Delta | 156.55 ns | 3.139 ns | 3.358 ns |         - |
-| CounterWith5LabelsHotPath |                  Delta | 247.14 ns | 4.703 ns | 5.598 ns |         - |
-| CounterWith6LabelsHotPath |                  Delta | 271.30 ns | 5.310 ns | 5.215 ns |         - |
-| CounterWith7LabelsHotPath |                  Delta | 309.02 ns | 5.934 ns | 5.828 ns |         - |
+|            CounterHotPath |             Cumulative |  21.61 ns | 0.084 ns | 0.078 ns |         - |
+| CounterWith1LabelsHotPath |             Cumulative |  69.08 ns | 0.261 ns | 0.244 ns |         - |
+| CounterWith3LabelsHotPath |             Cumulative | 149.77 ns | 0.549 ns | 0.486 ns |         - |
+| CounterWith5LabelsHotPath |             Cumulative | 236.47 ns | 1.684 ns | 1.493 ns |         - |
+| CounterWith6LabelsHotPath |             Cumulative | 276.48 ns | 1.442 ns | 1.349 ns |         - |
+| CounterWith7LabelsHotPath |             Cumulative | 294.09 ns | 2.354 ns | 2.202 ns |         - |
+|            CounterHotPath |                  Delta |  27.32 ns | 0.380 ns | 0.355 ns |         - |
+| CounterWith1LabelsHotPath |                  Delta |  80.83 ns | 0.219 ns | 0.183 ns |         - |
+| CounterWith3LabelsHotPath |                  Delta | 162.48 ns | 1.053 ns | 0.985 ns |         - |
+| CounterWith5LabelsHotPath |                  Delta | 255.48 ns | 1.807 ns | 1.602 ns |         - |
+| CounterWith6LabelsHotPath |                  Delta | 281.75 ns | 2.761 ns | 2.583 ns |         - |
+| CounterWith7LabelsHotPath |                  Delta | 310.29 ns | 1.817 ns | 1.611 ns |         - |
 */
 
 namespace Benchmarks.Metrics;