Merge pull request #43502 from sharwell/segmented-array

Implement SegmentedArray<T>

src/Workspaces/CoreTest/Collections/SegmentedArrayHelperTests.cs

@@ -0,0 +1,98 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.CodeAnalysis.Shared.Collections;
+using Roslyn.Utilities;
+using Xunit;
+
+namespace Microsoft.CodeAnalysis.UnitTests.Collections
+{
+    public class SegmentedArrayHelperTests
+    {
+        [Theory]
+        [InlineData(1)]
+        [InlineData(2)]
+        [InlineData(3)]
+        [InlineData(4)]
+        [InlineData(5)]
+        [InlineData(6)]
+        [InlineData(7)]
+        [InlineData(8)]
+        [InlineData(9)]
+        [InlineData(10)]
+        [InlineData(16)]
+        [InlineData(32)]
+        public void CalculateSegmentSize(int elementSize)
+        {
+            var expected = elementSize switch
+            {
+                1 => 65536,
+                2 => 32768,
+                3 => 16384,
+                4 => 16384,
+                5 => 16384,
+                6 => 8192,
+                7 => 8192,
+                8 => 8192,
+                9 => 8192,
+                10 => 8192,
+                16 => 4096,
+                32 => 2048,
+                _ => throw ExceptionUtilities.Unreachable,
+            };
+
+            Assert.Equal(expected, SegmentedArrayHelper.CalculateSegmentSize(elementSize));
+        }
+
+        [Theory]
+        [InlineData(1024)]
+        [InlineData(2048)]
+        [InlineData(4096)]
+        [InlineData(8192)]
+        [InlineData(16384)]
+        [InlineData(32768)]
+        [InlineData(65536)]
+        public void CalculateSegmentShift(int segmentSize)
+        {
+            var expected = segmentSize switch
+            {
+                1024 => 10,
+                2048 => 11,
+                4096 => 12,
+                8192 => 13,
+                16384 => 14,
+                32768 => 15,
+                65536 => 16,
+                _ => throw ExceptionUtilities.Unreachable,
+            };
+
+            Assert.Equal(expected, SegmentedArrayHelper.CalculateSegmentShift(segmentSize));
+        }
+
+        [Theory]
+        [InlineData(1024)]
+        [InlineData(2048)]
+        [InlineData(4096)]
+        [InlineData(8192)]
+        [InlineData(16384)]
+        [InlineData(32768)]
+        [InlineData(65536)]
+        public void CalculateOffsetMask(int segmentSize)
+        {
+            var expected = segmentSize switch
+            {
+                1024 => 0x3FF,
+                2048 => 0x7FF,
+                4096 => 0xFFF,
+                8192 => 0x1FFF,
+                16384 => 0x3FFF,
+                32768 => 0x7FFF,
+                65536 => 0xFFFF,
+                _ => throw ExceptionUtilities.Unreachable,
+            };
+
+            Assert.Equal(expected, SegmentedArrayHelper.CalculateOffsetMask(segmentSize));
+        }
+    }
+}

src/Workspaces/CoreTest/Collections/SegmentedArrayTests.cs

@@ -0,0 +1,210 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.CodeAnalysis.Shared.Collections;
+using Xunit;
+
+namespace Microsoft.CodeAnalysis.UnitTests.Collections
+{
+    public class SegmentedArrayTests
+    {
+        public static IEnumerable<object[]> TestLengths
+        {
+            get
+            {
+                yield return new object[] { 1 };
+                yield return new object[] { 10 };
+                yield return new object[] { 100 };
+                yield return new object[] { SegmentedArray<IntPtr>.TestAccessor.SegmentSize / 2 };
+                yield return new object[] { SegmentedArray<IntPtr>.TestAccessor.SegmentSize };
+                yield return new object[] { SegmentedArray<IntPtr>.TestAccessor.SegmentSize * 2 };
+                yield return new object[] { 100000 };
+            }
+        }
+
+        private static void ResetToSequence(SegmentedArray<IntPtr> array)
+        {
+            for (var i = 0; i < array.Length; i++)
+            {
+                array[i] = (IntPtr)i;
+            }
+        }
+
+        [Fact]
+        public void TestDefaultInstance()
+        {
+            var data = default(SegmentedArray<IntPtr>);
+            Assert.Null(data.GetTestAccessor().Items);
+
+            Assert.True(data.IsFixedSize);
+            Assert.False(data.IsReadOnly);
+            Assert.False(data.IsSynchronized);
+            Assert.Equal(0, data.Length);
+            Assert.Null(data.SyncRoot);
+
+            Assert.Throws<NullReferenceException>(() => data[0]);
+            Assert.Throws<NullReferenceException>(() => ((IReadOnlyList<IntPtr>)data)[0]);
+            Assert.Throws<NullReferenceException>(() => ((IList<IntPtr>)data)[0]);
+            Assert.Throws<NullReferenceException>(() => ((IList<IntPtr>)data)[0] = IntPtr.Zero);
+            Assert.Throws<NullReferenceException>(() => ((IList)data)[0]);
+            Assert.Throws<NullReferenceException>(() => ((IList)data)[0] = IntPtr.Zero);
+
+            Assert.Equal(0, ((ICollection)data).Count);
+            Assert.Equal(0, ((ICollection<IntPtr>)data).Count);
+            Assert.Equal(0, ((IReadOnlyCollection<IntPtr>)data).Count);
+
+            Assert.Throws<NullReferenceException>(() => data.Clone());
+            Assert.Throws<NullReferenceException>(() => data.CopyTo(Array.Empty<IntPtr>(), 0));
+            Assert.Throws<NullReferenceException>(() => ((ICollection<IntPtr>)data).CopyTo(Array.Empty<IntPtr>(), 0));
+
+            var enumerator1 = data.GetEnumerator();
+            Assert.Throws<NullReferenceException>(() => enumerator1.MoveNext());
+
+            var enumerator2 = ((IEnumerable)data).GetEnumerator();
+            Assert.Throws<NullReferenceException>(() => enumerator1.MoveNext());
+
+            var enumerator3 = ((IEnumerable<IntPtr>)data).GetEnumerator();
+            Assert.Throws<NullReferenceException>(() => enumerator1.MoveNext());
+
+            Assert.Throws<NotSupportedException>(() => ((IList)data).Add(IntPtr.Zero));
+            Assert.Throws<NotSupportedException>(() => ((ICollection<IntPtr>)data).Add(IntPtr.Zero));
+            Assert.Throws<NotSupportedException>(() => ((ICollection<IntPtr>)data).Clear());
+            Assert.Throws<NotSupportedException>(() => ((IList)data).Insert(0, IntPtr.Zero));
+            Assert.Throws<NotSupportedException>(() => ((IList<IntPtr>)data).Insert(0, IntPtr.Zero));
+            Assert.Throws<NotSupportedException>(() => ((IList)data).Remove(IntPtr.Zero));
+            Assert.Throws<NotSupportedException>(() => ((ICollection<IntPtr>)data).Remove(IntPtr.Zero));
+            Assert.Throws<NotSupportedException>(() => ((IList)data).RemoveAt(0));
+            Assert.Throws<NotSupportedException>(() => ((IList<IntPtr>)data).RemoveAt(0));
+
+            Assert.Throws<NullReferenceException>(() => ((IList)data).Clear());
+            Assert.Throws<NullReferenceException>(() => ((IList)data).Contains(IntPtr.Zero));
+            Assert.Throws<NullReferenceException>(() => ((ICollection<IntPtr>)data).Contains(IntPtr.Zero));
+            Assert.Throws<NullReferenceException>(() => ((IList)data).IndexOf(IntPtr.Zero));
+            Assert.Throws<NullReferenceException>(() => ((IList<IntPtr>)data).IndexOf(IntPtr.Zero));
+        }
+
+        [Fact]
+        public void TestConstructor1()
+        {
+            Assert.Throws<ArgumentOutOfRangeException>("length", () => new SegmentedArray<byte>(-1));
+
+            Assert.Empty(new SegmentedArray<byte>(0));
+            Assert.Same(Array.Empty<byte[]>(), new SegmentedArray<byte>(0).GetTestAccessor().Items);
+        }
+
+        [Theory]
+        [MemberData(nameof(TestLengths))]
+        public void TestConstructor2(int length)
+        {
+            var data = new SegmentedArray<IntPtr>(length);
+            Assert.Equal(length, data.Length);
+
+            var items = data.GetTestAccessor().Items;
+            Assert.Equal(length, items.Sum(item => item.Length));
+
+            for (var i = 0; i < items.Length - 1; i++)
+            {
+                Assert.Equal(SegmentedArray<IntPtr>.TestAccessor.SegmentSize, items[i].Length);
+                Assert.True(items[i].Length <= SegmentedArray<IntPtr>.TestAccessor.SegmentSize);
+            }
+        }
+
+        [Theory]
+        [MemberData(nameof(TestLengths))]
+        public void TestBasicProperties(int length)
+        {
+            var data = new SegmentedArray<IntPtr>(length);
+
+            Assert.True(data.IsFixedSize);
+            Assert.False(data.IsReadOnly);
+            Assert.False(data.IsSynchronized);
+            Assert.Equal(length, data.Length);
+            Assert.Same(data.GetTestAccessor().Items, data.SyncRoot);
+
+            Assert.Equal(length, ((ICollection)data).Count);
+            Assert.Equal(length, ((ICollection<IntPtr>)data).Count);
+            Assert.Equal(length, ((IReadOnlyCollection<IntPtr>)data).Count);
+        }
+
+        [Theory]
+        [MemberData(nameof(TestLengths))]
+        public void TestIndexer(int length)
+        {
+            var data = new SegmentedArray<IntPtr>(length);
+            ResetToSequence(data);
+
+            for (var i = 0; i < length; i++)
+            {
+                data[i] = (IntPtr)i;
+            }
+
+            for (var i = 0; i < length; i++)
+            {
+                Assert.Equal((IntPtr)i, data[i]);
+            }
+
+            for (var i = 0; i < length; i++)
+            {
+                ref var value = ref data[i];
+                Assert.Equal((IntPtr)i, data[i]);
+                value = IntPtr.Add(value, 1);
+
+                Assert.Equal((IntPtr)(i + 1), value);
+                Assert.Equal((IntPtr)(i + 1), data[i]);
+            }
+
+            ResetToSequence(data);
+            for (var i = 0; i < length; i++)
+            {
+                Assert.Equal((IntPtr)i, ((IReadOnlyList<IntPtr>)data)[i]);
+                data[i] = IntPtr.Add(data[i], 1);
+                Assert.Equal((IntPtr)(i + 1), ((IReadOnlyList<IntPtr>)data)[i]);
+            }
+
+            ResetToSequence(data);
+            for (var i = 0; i < length; i++)
+            {
+                Assert.Equal((IntPtr)i, ((IList<IntPtr>)data)[i]);
+                ((IList<IntPtr>)data)[i] = IntPtr.Add(data[i], 1);
+                Assert.Equal((IntPtr)(i + 1), ((IList<IntPtr>)data)[i]);
+            }
+
+            ResetToSequence(data);
+            for (var i = 0; i < length; i++)
+            {
+                Assert.Equal((IntPtr)i, ((IList)data)[i]);
+                ((IList)data)[i] = IntPtr.Add(data[i], 1);
+                Assert.Equal((IntPtr)(i + 1), ((IList)data)[i]);
+            }
+        }
+
+        /// <summary>
+        /// Verify that indexing and iteration match for an array with many segments.
+        /// </summary>
+        [Fact]
+        public void TestIterateLargeArray()
+        {
+            var data = new SegmentedArray<Guid>(1000000);
+            Assert.True(data.GetTestAccessor().Items.Length > 10);
+
+            for (var i = 0; i < data.Length; i++)
+            {
+                data[i] = Guid.NewGuid();
+                Assert.NotEqual(Guid.Empty, data[i]);
+            }
+
+            var index = 0;
+            foreach (var guid in data)
+            {
+                Assert.Equal(guid, data[index++]);
+            }
+
+            Assert.Equal(data.Length, index);
+        }
+    }
+}

src/Workspaces/SharedUtilitiesAndExtensions/Compiler/Core/Collections/SegmentedArrayHelper.cs

@@ -0,0 +1,73 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+#nullable enable
+
+using System;
+using System.Diagnostics;
+
+namespace Microsoft.CodeAnalysis.Shared.Collections
+{
+    internal static class SegmentedArrayHelper
+    {
+        /// <summary>
+        /// Calculates the maximum number of elements of size <paramref name="elementSize"/> which can fit into an array
+        /// which has the following characteristics:
+        /// <list type="bullet">
+        /// <item><description>The array can be allocated in the small object heap.</description></item>
+        /// <item><description>The array length is a power of 2.</description></item>
+        /// </list>
+        /// </summary>
+        /// <param name="elementSize">The size of the elements in the array.</param>
+        /// <returns>The segment size to use for small object heap segmented arrays.</returns>
+        internal static int CalculateSegmentSize(int elementSize)
+        {
+            // Default Large Object Heap size threshold
+            // https://github.com/dotnet/runtime/blob/c9d69e38d0e54bea5d188593ef6c3b30139f3ab1/src/coreclr/src/gc/gc.h#L111
+            const int Threshold = 85000;
+
+            var segmentSize = 2;
+            while (ArraySize(elementSize, segmentSize << 1) < Threshold)
+            {
+                segmentSize <<= 1;
+            }
+
+            return segmentSize;
+
+            static int ArraySize(int elementSize, int segmentSize)
+            {
+                // Array object header, plus space for the elements
+                return (2 * IntPtr.Size) + (elementSize * segmentSize);
+            }
+        }
+
+        /// <summary>
+        /// Calculates a shift which can be applied to an absolute index to get the page index within a segmented array.
+        /// </summary>
+        /// <param name="segmentSize">The number of elements in each page of the segmented array. Must be a power of 2.</param>
+        /// <returns>The shift to apply to the absolute index to get the page index within a segmented array.</returns>
+        internal static int CalculateSegmentShift(int segmentSize)
+        {
+            var segmentShift = 0;
+            while (0 != (segmentSize >>= 1))
+            {
+                segmentShift++;
+            }
+
+            return segmentShift;
+        }
+
+        /// <summary>
+        /// Calculates a mask, which can be applied to an absolute index to get the index within a page of a segmented
+        /// array.
+        /// </summary>
+        /// <param name="segmentSize">The number of elements in each page of the segmented array. Must be a power of 2.</param>
+        /// <returns>The bit mask to obtain the index within a page from an absolute index within a segmented array.</returns>
+        internal static int CalculateOffsetMask(int segmentSize)
+        {
+            Debug.Assert(segmentSize == 1 || (segmentSize & (segmentSize - 1)) == 0, "Expected size of 1, or a power of 2");
+            return segmentSize - 1;
+        }
+    }
+}