diff --git a/src/Dependencies/Collections/ImmutableSegmentedDictionary`2+Builder+PrivateMarshal.cs b/src/Dependencies/Collections/ImmutableSegmentedDictionary`2+Builder+PrivateMarshal.cs
index f2d3c6cff5f6e..01f7a1c796de2 100644
--- a/src/Dependencies/Collections/ImmutableSegmentedDictionary`2+Builder+PrivateMarshal.cs
+++ b/src/Dependencies/Collections/ImmutableSegmentedDictionary`2+Builder+PrivateMarshal.cs
@@ -15,7 +15,7 @@ internal static class PrivateMarshal
         {
             /// <inheritdoc cref="SegmentedCollectionsMarshal.GetValueRefOrNullRef{TKey, TValue}(ImmutableSegmentedDictionary{TKey, TValue}.Builder, TKey)"/>
             public static ref TValue FindValue(Builder dictionary, TKey key)
-                => ref SegmentedCollectionsMarshal.GetValueRefOrNullRef(dictionary.GetOrCreateMutableDictionary(), key);
+                => ref SegmentedCollectionsMarshal.GetValueRefOrNullRef(dictionary._builder.GetOrCreateMutableDictionary(), key);
         }
     }
 }
diff --git a/src/Dependencies/Collections/ImmutableSegmentedDictionary`2+Builder.cs b/src/Dependencies/Collections/ImmutableSegmentedDictionary`2+Builder.cs
index c813f54169bb4..6e36d573d4a10 100644
--- a/src/Dependencies/Collections/ImmutableSegmentedDictionary`2+Builder.cs
+++ b/src/Dependencies/Collections/ImmutableSegmentedDictionary`2+Builder.cs
@@ -6,6 +6,7 @@
 using System.Collections;
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
+using Microsoft.CodeAnalysis.Collections.Internal;
 
 namespace Microsoft.CodeAnalysis.Collections
 {
@@ -14,50 +15,26 @@ internal readonly partial struct ImmutableSegmentedDictionary<TKey, TValue>
         public sealed partial class Builder : IDictionary<TKey, TValue>, IReadOnlyDictionary<TKey, TValue>, IDictionary
         {
             /// <summary>
-            /// The immutable collection this builder is based on.
+            /// The private builder implementation.
             /// </summary>
-            private ImmutableSegmentedDictionary<TKey, TValue> _dictionary;
-
-            /// <summary>
-            /// The current mutable collection this builder is operating on. This field is initialized to a copy of
-            /// <see cref="_dictionary"/> the first time a change is made.
-            /// </summary>
-            private SegmentedDictionary<TKey, TValue>? _mutableDictionary;
+            private ValueBuilder _builder;
 
             internal Builder(ImmutableSegmentedDictionary<TKey, TValue> dictionary)
-            {
-                _dictionary = dictionary;
-            }
+                => _builder = new ValueBuilder(dictionary);
 
             public IEqualityComparer<TKey> KeyComparer
             {
-                get
-                {
-                    return ReadOnlyDictionary.Comparer;
-                }
-
-                set
-                {
-                    if (value is null)
-                        throw new ArgumentNullException(nameof(value));
-
-                    if (value != KeyComparer)
-                    {
-                        // Rewrite the mutable dictionary using a new comparer
-                        var valuesToAdd = ReadOnlyDictionary;
-                        _mutableDictionary = new SegmentedDictionary<TKey, TValue>(value);
-                        AddRange(valuesToAdd);
-                    }
-                }
+                get => _builder.KeyComparer;
+                set => _builder.KeyComparer = value;
             }
 
-            public int Count => ReadOnlyDictionary.Count;
+            public int Count => _builder.Count;
 
             public KeyCollection Keys => new(this);
 
             public ValueCollection Values => new(this);
 
-            private SegmentedDictionary<TKey, TValue> ReadOnlyDictionary => _mutableDictionary ?? _dictionary._dictionary;
+            private SegmentedDictionary<TKey, TValue> ReadOnlyDictionary => _builder.ReadOnlyDictionary;
 
             IEnumerable<TKey> IReadOnlyDictionary<TKey, TValue>.Keys => Keys;
 
@@ -67,181 +44,105 @@ public IEqualityComparer<TKey> KeyComparer
 
             ICollection<TValue> IDictionary<TKey, TValue>.Values => Values;
 
-            bool ICollection<KeyValuePair<TKey, TValue>>.IsReadOnly => false;
+            bool ICollection<KeyValuePair<TKey, TValue>>.IsReadOnly => ICollectionCalls<KeyValuePair<TKey, TValue>>.IsReadOnly(ref _builder);
 
             ICollection IDictionary.Keys => Keys;
 
             ICollection IDictionary.Values => Values;
 
-            bool IDictionary.IsReadOnly => false;
+            bool IDictionary.IsReadOnly => IDictionaryCalls.IsReadOnly(ref _builder);
 
-            bool IDictionary.IsFixedSize => false;
+            bool IDictionary.IsFixedSize => IDictionaryCalls.IsFixedSize(ref _builder);
 
             object ICollection.SyncRoot => this;
 
-            bool ICollection.IsSynchronized => false;
+            bool ICollection.IsSynchronized => ICollectionCalls.IsSynchronized(ref _builder);
 
             public TValue this[TKey key]
             {
-                get => ReadOnlyDictionary[key];
-                set => GetOrCreateMutableDictionary()[key] = value;
+                get => _builder[key];
+                set => _builder[key] = value;
             }
 
             object? IDictionary.this[object key]
             {
-                get => ((IDictionary)ReadOnlyDictionary)[key];
-                set => ((IDictionary)GetOrCreateMutableDictionary())[key] = value;
-            }
-
-            private SegmentedDictionary<TKey, TValue> GetOrCreateMutableDictionary()
-            {
-                return _mutableDictionary ??= new SegmentedDictionary<TKey, TValue>(_dictionary._dictionary, _dictionary.KeyComparer);
+                get => IDictionaryCalls.GetItem(ref _builder, key);
+                set => IDictionaryCalls.SetItem(ref _builder, key, value);
             }
 
             public void Add(TKey key, TValue value)
-            {
-                if (Contains(new KeyValuePair<TKey, TValue>(key, value)))
-                    return;
-
-                GetOrCreateMutableDictionary().Add(key, value);
-            }
+                => _builder.Add(key, value);
 
             public void Add(KeyValuePair<TKey, TValue> item)
-                => Add(item.Key, item.Value);
+                => _builder.Add(item);
 
             public void AddRange(IEnumerable<KeyValuePair<TKey, TValue>> items)
-            {
-                if (items == null)
-                    throw new ArgumentNullException(nameof(items));
-
-                foreach (var pair in items)
-                    Add(pair.Key, pair.Value);
-            }
+                => _builder.AddRange(items);
 
             public void Clear()
-            {
-                if (ReadOnlyDictionary.Count != 0)
-                {
-                    if (_mutableDictionary is null)
-                        _mutableDictionary = new SegmentedDictionary<TKey, TValue>(KeyComparer);
-                    else
-                        _mutableDictionary.Clear();
-                }
-            }
+                => _builder.Clear();
 
             public bool Contains(KeyValuePair<TKey, TValue> item)
-            {
-                return TryGetValue(item.Key, out var value)
-                    && EqualityComparer<TValue>.Default.Equals(value, item.Value);
-            }
+                => _builder.Contains(item);
 
             public bool ContainsKey(TKey key)
-                => ReadOnlyDictionary.ContainsKey(key);
+                => _builder.ContainsKey(key);
 
             public bool ContainsValue(TValue value)
-            {
-                return _dictionary.ContainsValue(value);
-            }
+                => _builder.ContainsValue(value);
 
             public Enumerator GetEnumerator()
-                => new(GetOrCreateMutableDictionary(), Enumerator.ReturnType.KeyValuePair);
+                => _builder.GetEnumerator();
 
             public TValue? GetValueOrDefault(TKey key)
-            {
-                if (TryGetValue(key, out var value))
-                    return value;
-
-                return default;
-            }
+                => _builder.GetValueOrDefault(key);
 
             public TValue GetValueOrDefault(TKey key, TValue defaultValue)
-            {
-                if (TryGetValue(key, out var value))
-                    return value;
-
-                return defaultValue;
-            }
+                => _builder.GetValueOrDefault(key, defaultValue);
 
             public bool Remove(TKey key)
-            {
-                if (_mutableDictionary is null && !ContainsKey(key))
-                    return false;
-
-                return GetOrCreateMutableDictionary().Remove(key);
-            }
+                => _builder.Remove(key);
 
             public bool Remove(KeyValuePair<TKey, TValue> item)
-            {
-                if (!Contains(item))
-                {
-                    return false;
-                }
-
-                GetOrCreateMutableDictionary().Remove(item.Key);
-                return true;
-            }
+                => _builder.Remove(item);
 
             public void RemoveRange(IEnumerable<TKey> keys)
-            {
-                if (keys is null)
-                    throw new ArgumentNullException(nameof(keys));
-
-                foreach (var key in keys)
-                {
-                    Remove(key);
-                }
-            }
+                => _builder.RemoveRange(keys);
 
             public bool TryGetKey(TKey equalKey, out TKey actualKey)
-            {
-                foreach (var key in Keys)
-                {
-                    if (KeyComparer.Equals(key, equalKey))
-                    {
-                        actualKey = key;
-                        return true;
-                    }
-                }
-
-                actualKey = equalKey;
-                return false;
-            }
+                => _builder.TryGetKey(equalKey, out actualKey);
 
 #pragma warning disable CS8767 // Nullability of reference types in type of parameter doesn't match implicitly implemented member (possibly because of nullability attributes).
             public bool TryGetValue(TKey key, [MaybeNullWhen(false)] out TValue value)
 #pragma warning restore CS8767 // Nullability of reference types in type of parameter doesn't match implicitly implemented member (possibly because of nullability attributes).
-                => ReadOnlyDictionary.TryGetValue(key, out value);
+                => _builder.TryGetValue(key, out value);
 
             public ImmutableSegmentedDictionary<TKey, TValue> ToImmutable()
-            {
-                _dictionary = new ImmutableSegmentedDictionary<TKey, TValue>(ReadOnlyDictionary);
-                _mutableDictionary = null;
-                return _dictionary;
-            }
+                => _builder.ToImmutable();
 
             void ICollection<KeyValuePair<TKey, TValue>>.CopyTo(KeyValuePair<TKey, TValue>[] array, int arrayIndex)
-                => ((ICollection<KeyValuePair<TKey, TValue>>)ReadOnlyDictionary).CopyTo(array, arrayIndex);
+                => ICollectionCalls<KeyValuePair<TKey, TValue>>.CopyTo(ref _builder, array, arrayIndex);
 
             IEnumerator<KeyValuePair<TKey, TValue>> IEnumerable<KeyValuePair<TKey, TValue>>.GetEnumerator()
-                => new Enumerator(GetOrCreateMutableDictionary(), Enumerator.ReturnType.KeyValuePair);
+                => IEnumerableCalls<KeyValuePair<TKey, TValue>>.GetEnumerator(ref _builder);
 
             IEnumerator IEnumerable.GetEnumerator()
-                => new Enumerator(GetOrCreateMutableDictionary(), Enumerator.ReturnType.KeyValuePair);
+                => IEnumerableCalls.GetEnumerator(ref _builder);
 
             bool IDictionary.Contains(object key)
-                => ((IDictionary)ReadOnlyDictionary).Contains(key);
+                => IDictionaryCalls.Contains(ref _builder, key);
 
             void IDictionary.Add(object key, object? value)
-                => ((IDictionary)GetOrCreateMutableDictionary()).Add(key, value);
+                => IDictionaryCalls.Add(ref _builder, key, value);
 
             IDictionaryEnumerator IDictionary.GetEnumerator()
-                => new Enumerator(GetOrCreateMutableDictionary(), Enumerator.ReturnType.DictionaryEntry);
+                => IDictionaryCalls.GetEnumerator(ref _builder);
 
             void IDictionary.Remove(object key)
-                => ((IDictionary)GetOrCreateMutableDictionary()).Remove(key);
+                => IDictionaryCalls.Remove(ref _builder, key);
 
             void ICollection.CopyTo(Array array, int index)
-                => ((ICollection)ReadOnlyDictionary).CopyTo(array, index);
+                => ICollectionCalls.CopyTo(ref _builder, array, index);
 
             internal TestAccessor GetTestAccessor()
                 => new TestAccessor(this);
@@ -249,7 +150,7 @@ internal TestAccessor GetTestAccessor()
             internal readonly struct TestAccessor(Builder instance)
             {
                 internal SegmentedDictionary<TKey, TValue> GetOrCreateMutableDictionary()
-                    => instance.GetOrCreateMutableDictionary();
+                    => instance._builder.GetOrCreateMutableDictionary();
             }
         }
     }
diff --git a/src/Dependencies/Collections/ImmutableSegmentedDictionary`2+ValueBuilder.cs b/src/Dependencies/Collections/ImmutableSegmentedDictionary`2+ValueBuilder.cs
new file mode 100644
index 0000000000000..4c20bfdc65cbb
--- /dev/null
+++ b/src/Dependencies/Collections/ImmutableSegmentedDictionary`2+ValueBuilder.cs
@@ -0,0 +1,259 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+
+namespace Microsoft.CodeAnalysis.Collections
+{
+    internal readonly partial struct ImmutableSegmentedDictionary<TKey, TValue>
+    {
+        private struct ValueBuilder : IDictionary<TKey, TValue>, IReadOnlyDictionary<TKey, TValue>, IDictionary
+        {
+            /// <summary>
+            /// The immutable collection this builder is based on.
+            /// </summary>
+            private ImmutableSegmentedDictionary<TKey, TValue> _dictionary;
+
+            /// <summary>
+            /// The current mutable collection this builder is operating on. This field is initialized to a copy of
+            /// <see cref="_dictionary"/> the first time a change is made.
+            /// </summary>
+            private SegmentedDictionary<TKey, TValue>? _mutableDictionary;
+
+            internal ValueBuilder(ImmutableSegmentedDictionary<TKey, TValue> dictionary)
+            {
+                _dictionary = dictionary;
+                _mutableDictionary = null;
+            }
+
+            public IEqualityComparer<TKey> KeyComparer
+            {
+                readonly get
+                {
+                    return ReadOnlyDictionary.Comparer;
+                }
+
+                set
+                {
+                    if (value is null)
+                        throw new ArgumentNullException(nameof(value));
+
+                    if (value != KeyComparer)
+                    {
+                        // Rewrite the mutable dictionary using a new comparer
+                        var valuesToAdd = ReadOnlyDictionary;
+                        _mutableDictionary = new SegmentedDictionary<TKey, TValue>(value);
+                        _dictionary = default;
+                        AddRange(valuesToAdd);
+                    }
+                }
+            }
+
+            public readonly int Count => ReadOnlyDictionary.Count;
+
+            internal readonly SegmentedDictionary<TKey, TValue> ReadOnlyDictionary => _mutableDictionary ?? _dictionary._dictionary;
+
+            readonly IEnumerable<TKey> IReadOnlyDictionary<TKey, TValue>.Keys => throw new NotSupportedException();
+
+            readonly IEnumerable<TValue> IReadOnlyDictionary<TKey, TValue>.Values => throw new NotSupportedException();
+
+            readonly ICollection<TKey> IDictionary<TKey, TValue>.Keys => throw new NotSupportedException();
+
+            readonly ICollection<TValue> IDictionary<TKey, TValue>.Values => throw new NotSupportedException();
+
+            readonly bool ICollection<KeyValuePair<TKey, TValue>>.IsReadOnly => false;
+
+            readonly ICollection IDictionary.Keys => throw new NotSupportedException();
+
+            readonly ICollection IDictionary.Values => throw new NotSupportedException();
+
+            readonly bool IDictionary.IsReadOnly => false;
+
+            readonly bool IDictionary.IsFixedSize => false;
+
+            readonly object ICollection.SyncRoot => throw new NotSupportedException();
+
+            readonly bool ICollection.IsSynchronized => false;
+
+            public TValue this[TKey key]
+            {
+                readonly get => ReadOnlyDictionary[key];
+                set => GetOrCreateMutableDictionary()[key] = value;
+            }
+
+            object? IDictionary.this[object key]
+            {
+                readonly get => ((IDictionary)ReadOnlyDictionary)[key];
+                set => ((IDictionary)GetOrCreateMutableDictionary())[key] = value;
+            }
+
+            internal SegmentedDictionary<TKey, TValue> GetOrCreateMutableDictionary()
+            {
+                if (_mutableDictionary is null)
+                {
+                    var originalDictionary = RoslynImmutableInterlocked.InterlockedExchange(ref _dictionary, default);
+                    if (originalDictionary.IsDefault)
+                        throw new InvalidOperationException($"Unexpected concurrent access to {GetType()}");
+
+                    _mutableDictionary = new SegmentedDictionary<TKey, TValue>(originalDictionary._dictionary, originalDictionary.KeyComparer);
+                }
+
+                return _mutableDictionary;
+            }
+
+            public void Add(TKey key, TValue value)
+            {
+                if (Contains(new KeyValuePair<TKey, TValue>(key, value)))
+                    return;
+
+                GetOrCreateMutableDictionary().Add(key, value);
+            }
+
+            public void Add(KeyValuePair<TKey, TValue> item)
+                => Add(item.Key, item.Value);
+
+            public void AddRange(IEnumerable<KeyValuePair<TKey, TValue>> items)
+            {
+                if (items == null)
+                    throw new ArgumentNullException(nameof(items));
+
+                foreach (var pair in items)
+                    Add(pair.Key, pair.Value);
+            }
+
+            public void Clear()
+            {
+                if (ReadOnlyDictionary.Count != 0)
+                {
+                    if (_mutableDictionary is null)
+                    {
+                        _mutableDictionary = new SegmentedDictionary<TKey, TValue>(KeyComparer);
+                        _dictionary = default;
+                    }
+                    else
+                    {
+                        _mutableDictionary.Clear();
+                    }
+                }
+            }
+
+            public readonly bool Contains(KeyValuePair<TKey, TValue> item)
+            {
+                return TryGetValue(item.Key, out var value)
+                    && EqualityComparer<TValue>.Default.Equals(value, item.Value);
+            }
+
+            public readonly bool ContainsKey(TKey key)
+                => ReadOnlyDictionary.ContainsKey(key);
+
+            public readonly bool ContainsValue(TValue value)
+                => ReadOnlyDictionary.ContainsValue(value);
+
+            public Enumerator GetEnumerator()
+                => new(GetOrCreateMutableDictionary(), Enumerator.ReturnType.KeyValuePair);
+
+            public readonly TValue? GetValueOrDefault(TKey key)
+            {
+                if (TryGetValue(key, out var value))
+                    return value;
+
+                return default;
+            }
+
+            public readonly TValue GetValueOrDefault(TKey key, TValue defaultValue)
+            {
+                if (TryGetValue(key, out var value))
+                    return value;
+
+                return defaultValue;
+            }
+
+            public bool Remove(TKey key)
+            {
+                if (_mutableDictionary is null && !ContainsKey(key))
+                    return false;
+
+                return GetOrCreateMutableDictionary().Remove(key);
+            }
+
+            public bool Remove(KeyValuePair<TKey, TValue> item)
+            {
+                if (!Contains(item))
+                {
+                    return false;
+                }
+
+                GetOrCreateMutableDictionary().Remove(item.Key);
+                return true;
+            }
+
+            public void RemoveRange(IEnumerable<TKey> keys)
+            {
+                if (keys is null)
+                    throw new ArgumentNullException(nameof(keys));
+
+                foreach (var key in keys)
+                {
+                    Remove(key);
+                }
+            }
+
+#pragma warning disable IDE0251 // Make member 'readonly' (false positive: https://github.com/dotnet/roslyn/issues/72335)
+            public bool TryGetKey(TKey equalKey, out TKey actualKey)
+#pragma warning restore IDE0251 // Make member 'readonly'
+            {
+                foreach (var pair in this)
+                {
+                    if (KeyComparer.Equals(pair.Key, equalKey))
+                    {
+                        actualKey = pair.Key;
+                        return true;
+                    }
+                }
+
+                actualKey = equalKey;
+                return false;
+            }
+
+#pragma warning disable CS8767 // Nullability of reference types in type of parameter doesn't match implicitly implemented member (possibly because of nullability attributes).
+            public readonly bool TryGetValue(TKey key, [MaybeNullWhen(false)] out TValue value)
+#pragma warning restore CS8767 // Nullability of reference types in type of parameter doesn't match implicitly implemented member (possibly because of nullability attributes).
+                => ReadOnlyDictionary.TryGetValue(key, out value);
+
+            public ImmutableSegmentedDictionary<TKey, TValue> ToImmutable()
+            {
+                _dictionary = new ImmutableSegmentedDictionary<TKey, TValue>(ReadOnlyDictionary);
+                _mutableDictionary = null;
+                return _dictionary;
+            }
+
+            readonly void ICollection<KeyValuePair<TKey, TValue>>.CopyTo(KeyValuePair<TKey, TValue>[] array, int arrayIndex)
+                => ((ICollection<KeyValuePair<TKey, TValue>>)ReadOnlyDictionary).CopyTo(array, arrayIndex);
+
+            IEnumerator<KeyValuePair<TKey, TValue>> IEnumerable<KeyValuePair<TKey, TValue>>.GetEnumerator()
+                => new Enumerator(GetOrCreateMutableDictionary(), Enumerator.ReturnType.KeyValuePair);
+
+            IEnumerator IEnumerable.GetEnumerator()
+                => new Enumerator(GetOrCreateMutableDictionary(), Enumerator.ReturnType.KeyValuePair);
+
+            readonly bool IDictionary.Contains(object key)
+                => ((IDictionary)ReadOnlyDictionary).Contains(key);
+
+            void IDictionary.Add(object key, object? value)
+                => ((IDictionary)GetOrCreateMutableDictionary()).Add(key, value);
+
+            IDictionaryEnumerator IDictionary.GetEnumerator()
+                => new Enumerator(GetOrCreateMutableDictionary(), Enumerator.ReturnType.DictionaryEntry);
+
+            void IDictionary.Remove(object key)
+                => ((IDictionary)GetOrCreateMutableDictionary()).Remove(key);
+
+            readonly void ICollection.CopyTo(Array array, int index)
+                => ((ICollection)ReadOnlyDictionary).CopyTo(array, index);
+        }
+    }
+}
diff --git a/src/Dependencies/Collections/ImmutableSegmentedDictionary`2.cs b/src/Dependencies/Collections/ImmutableSegmentedDictionary`2.cs
index 99d8a6ce511d7..bdcdcdddcf96a 100644
--- a/src/Dependencies/Collections/ImmutableSegmentedDictionary`2.cs
+++ b/src/Dependencies/Collections/ImmutableSegmentedDictionary`2.cs
@@ -147,9 +147,9 @@ public ImmutableSegmentedDictionary<TKey, TValue> Add(TKey key, TValue value)
             if (self.Contains(new KeyValuePair<TKey, TValue>(key, value)))
                 return self;
 
-            var dictionary = new SegmentedDictionary<TKey, TValue>(self._dictionary, self._dictionary.Comparer);
-            dictionary.Add(key, value);
-            return new ImmutableSegmentedDictionary<TKey, TValue>(dictionary);
+            var builder = ToValueBuilder();
+            builder.Add(key, value);
+            return builder.ToImmutable();
         }
 
         public ImmutableSegmentedDictionary<TKey, TValue> AddRange(IEnumerable<KeyValuePair<TKey, TValue>> pairs)
@@ -162,21 +162,9 @@ public ImmutableSegmentedDictionary<TKey, TValue> AddRange(IEnumerable<KeyValueP
                 return other;
             }
 
-            SegmentedDictionary<TKey, TValue>? dictionary = null;
-            foreach (var pair in pairs)
-            {
-                ICollection<KeyValuePair<TKey, TValue>> collectionToCheck = dictionary ?? self._dictionary;
-                if (collectionToCheck.Contains(pair))
-                    continue;
-
-                dictionary ??= new SegmentedDictionary<TKey, TValue>(self._dictionary, self._dictionary.Comparer);
-                dictionary.Add(pair.Key, pair.Value);
-            }
-
-            if (dictionary is null)
-                return self;
-
-            return new ImmutableSegmentedDictionary<TKey, TValue>(dictionary);
+            var builder = ToValueBuilder();
+            builder.AddRange(pairs);
+            return builder.ToImmutable();
         }
 
         public ImmutableSegmentedDictionary<TKey, TValue> Clear()
@@ -211,9 +199,9 @@ public ImmutableSegmentedDictionary<TKey, TValue> Remove(TKey key)
             if (!self._dictionary.ContainsKey(key))
                 return self;
 
-            var dictionary = new SegmentedDictionary<TKey, TValue>(self._dictionary, self._dictionary.Comparer);
-            dictionary.Remove(key);
-            return new ImmutableSegmentedDictionary<TKey, TValue>(dictionary);
+            var builder = ToValueBuilder();
+            builder.Remove(key);
+            return builder.ToImmutable();
         }
 
         public ImmutableSegmentedDictionary<TKey, TValue> RemoveRange(IEnumerable<TKey> keys)
@@ -221,7 +209,7 @@ public ImmutableSegmentedDictionary<TKey, TValue> RemoveRange(IEnumerable<TKey>
             if (keys is null)
                 throw new ArgumentNullException(nameof(keys));
 
-            var result = ToBuilder();
+            var result = ToValueBuilder();
             result.RemoveRange(keys);
             return result.ToImmutable();
         }
@@ -234,9 +222,9 @@ public ImmutableSegmentedDictionary<TKey, TValue> SetItem(TKey key, TValue value
                 return self;
             }
 
-            var dictionary = new SegmentedDictionary<TKey, TValue>(self._dictionary, self._dictionary.Comparer);
-            dictionary[key] = value;
-            return new ImmutableSegmentedDictionary<TKey, TValue>(dictionary);
+            var builder = ToValueBuilder();
+            builder[key] = value;
+            return builder.ToImmutable();
         }
 
         public ImmutableSegmentedDictionary<TKey, TValue> SetItems(IEnumerable<KeyValuePair<TKey, TValue>> items)
@@ -244,7 +232,7 @@ public ImmutableSegmentedDictionary<TKey, TValue> SetItems(IEnumerable<KeyValueP
             if (items is null)
                 throw new ArgumentNullException(nameof(items));
 
-            var result = ToBuilder();
+            var result = ToValueBuilder();
             foreach (var item in items)
             {
                 result[item.Key] = item.Value;
@@ -304,6 +292,9 @@ public ImmutableSegmentedDictionary<TKey, TValue> WithComparer(IEqualityComparer
         public Builder ToBuilder()
             => new(this);
 
+        private ValueBuilder ToValueBuilder()
+            => new(this);
+
         public override int GetHashCode()
             => _dictionary?.GetHashCode() ?? 0;
 
diff --git a/src/Dependencies/Collections/ImmutableSegmentedHashSet`1+Builder.cs b/src/Dependencies/Collections/ImmutableSegmentedHashSet`1+Builder.cs
index 646c2a4bb93ca..f5140e38238f5 100644
--- a/src/Dependencies/Collections/ImmutableSegmentedHashSet`1+Builder.cs
+++ b/src/Dependencies/Collections/ImmutableSegmentedHashSet`1+Builder.cs
@@ -2,7 +2,6 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System;
 using System.Collections;
 using System.Collections.Generic;
 using System.Collections.Immutable;
@@ -15,263 +14,115 @@ internal readonly partial struct ImmutableSegmentedHashSet<T>
         public sealed class Builder : ISet<T>, IReadOnlyCollection<T>
         {
             /// <summary>
-            /// The immutable collection this builder is based on.
+            /// The private builder implementation.
             /// </summary>
-            private ImmutableSegmentedHashSet<T> _set;
-
-            /// <summary>
-            /// The current mutable collection this builder is operating on. This field is initialized to a copy of
-            /// <see cref="_set"/> the first time a change is made.
-            /// </summary>
-            private SegmentedHashSet<T>? _mutableSet;
+            private ValueBuilder _builder;
 
             internal Builder(ImmutableSegmentedHashSet<T> set)
-            {
-                _set = set;
-                _mutableSet = null;
-            }
+                => _builder = new ValueBuilder(set);
 
             /// <inheritdoc cref="ImmutableHashSet{T}.Builder.KeyComparer"/>
             public IEqualityComparer<T> KeyComparer
             {
-                get
-                {
-                    return ReadOnlySet.Comparer;
-                }
-
-                set
-                {
-                    if (Equals(KeyComparer, value ?? EqualityComparer<T>.Default))
-                        return;
-
-                    _mutableSet = new SegmentedHashSet<T>(ReadOnlySet, value ?? EqualityComparer<T>.Default);
-                    _set = default;
-                }
+                get => _builder.KeyComparer;
+                set => _builder.KeyComparer = value;
             }
 
             /// <inheritdoc cref="ImmutableHashSet{T}.Builder.Count"/>
-            public int Count => ReadOnlySet.Count;
-
-            private SegmentedHashSet<T> ReadOnlySet => _mutableSet ?? _set._set;
-
-            bool ICollection<T>.IsReadOnly => false;
-
-            private SegmentedHashSet<T> GetOrCreateMutableSet()
-            {
-                if (_mutableSet is null)
-                {
-                    var originalSet = RoslynImmutableInterlocked.InterlockedExchange(ref _set, default);
-                    if (originalSet.IsDefault)
-                        throw new InvalidOperationException($"Unexpected concurrent access to {GetType()}");
-
-                    _mutableSet = new SegmentedHashSet<T>(originalSet._set, originalSet.KeyComparer);
-                }
+            public int Count => _builder.Count;
 
-                return _mutableSet;
-            }
+            bool ICollection<T>.IsReadOnly => ICollectionCalls<T>.IsReadOnly(ref _builder);
 
             /// <inheritdoc cref="ImmutableHashSet{T}.Builder.Add(T)"/>
             public bool Add(T item)
-            {
-                if (_mutableSet is null && Contains(item))
-                    return false;
-
-                return GetOrCreateMutableSet().Add(item);
-            }
+                => _builder.Add(item);
 
             /// <inheritdoc cref="ImmutableHashSet{T}.Builder.Clear()"/>
             public void Clear()
-            {
-                if (ReadOnlySet.Count != 0)
-                {
-                    if (_mutableSet is null)
-                    {
-                        _mutableSet = new SegmentedHashSet<T>(KeyComparer);
-                        _set = default;
-                    }
-                    else
-                    {
-                        _mutableSet.Clear();
-                    }
-                }
-            }
+                => _builder.Clear();
 
             /// <inheritdoc cref="ImmutableHashSet{T}.Builder.Contains(T)"/>
             public bool Contains(T item)
-                => ReadOnlySet.Contains(item);
+                => _builder.Contains(item);
 
             /// <inheritdoc cref="ImmutableHashSet{T}.Builder.ExceptWith(IEnumerable{T})"/>
             public void ExceptWith(IEnumerable<T> other)
             {
-                if (other is null)
-                    ThrowHelper.ThrowArgumentNullException(ExceptionArgument.other);
-
-                if (_mutableSet is not null)
-                {
-                    _mutableSet.ExceptWith(other);
-                    return;
-                }
-
                 if (other == this)
                 {
-                    Clear();
-                    return;
-                }
-                else if (other is ImmutableSegmentedHashSet<T> otherSet)
-                {
-                    if (otherSet == _set)
-                    {
-                        Clear();
-                        return;
-                    }
-                    else if (otherSet.IsEmpty)
-                    {
-                        // No action required
-                        return;
-                    }
-                    else
-                    {
-                        GetOrCreateMutableSet().ExceptWith(otherSet._set);
-                        return;
-                    }
+                    // The ValueBuilder knows how to optimize for this case by calling Clear, provided it does not need
+                    // to access properties of the wrapping Builder instance.
+                    _builder.ExceptWith(_builder.ReadOnlySet);
                 }
                 else
                 {
-                    // Manually enumerate to avoid changes to the builder if 'other' is empty or does not contain any
-                    // items present in the current set.
-                    SegmentedHashSet<T>? mutableSet = null;
-                    foreach (var item in other)
-                    {
-                        if (mutableSet is null)
-                        {
-                            if (!ReadOnlySet.Contains(item))
-                                continue;
-
-                            mutableSet = GetOrCreateMutableSet();
-                        }
-
-                        mutableSet.Remove(item);
-                    }
-
-                    return;
+                    _builder.ExceptWith(other);
                 }
             }
 
             /// <inheritdoc cref="ImmutableHashSet{T}.Builder.GetEnumerator()"/>
             public Enumerator GetEnumerator()
-                => new Enumerator(GetOrCreateMutableSet());
+                => _builder.GetEnumerator();
 
             /// <inheritdoc cref="ImmutableHashSet{T}.Builder.IntersectWith(IEnumerable{T})"/>
             public void IntersectWith(IEnumerable<T> other)
-                => GetOrCreateMutableSet().IntersectWith(other);
+                => _builder.IntersectWith(other);
 
             /// <inheritdoc cref="ImmutableHashSet{T}.Builder.IsProperSubsetOf(IEnumerable{T})"/>
             public bool IsProperSubsetOf(IEnumerable<T> other)
-                => ReadOnlySet.IsProperSubsetOf(other);
+                => _builder.IsProperSubsetOf(other);
 
             /// <inheritdoc cref="ImmutableHashSet{T}.Builder.IsProperSupersetOf(IEnumerable{T})"/>
             public bool IsProperSupersetOf(IEnumerable<T> other)
-                => ReadOnlySet.IsProperSupersetOf(other);
+                => _builder.IsProperSupersetOf(other);
 
             /// <inheritdoc cref="ImmutableHashSet{T}.Builder.IsSubsetOf(IEnumerable{T})"/>
             public bool IsSubsetOf(IEnumerable<T> other)
-                => ReadOnlySet.IsSubsetOf(other);
+                => _builder.IsSubsetOf(other);
 
             /// <inheritdoc cref="ImmutableHashSet{T}.Builder.IsSupersetOf(IEnumerable{T})"/>
             public bool IsSupersetOf(IEnumerable<T> other)
-                => ReadOnlySet.IsSupersetOf(other);
+                => _builder.IsSupersetOf(other);
 
             /// <inheritdoc cref="ImmutableHashSet{T}.Builder.Overlaps(IEnumerable{T})"/>
             public bool Overlaps(IEnumerable<T> other)
-                => ReadOnlySet.Overlaps(other);
+                => _builder.Overlaps(other);
 
             /// <inheritdoc cref="ImmutableHashSet{T}.Builder.Remove(T)"/>
             public bool Remove(T item)
-            {
-                if (_mutableSet is null && !Contains(item))
-                    return false;
-
-                return GetOrCreateMutableSet().Remove(item);
-            }
+                => _builder.Remove(item);
 
             /// <inheritdoc cref="ImmutableHashSet{T}.Builder.SetEquals(IEnumerable{T})"/>
             public bool SetEquals(IEnumerable<T> other)
-                => ReadOnlySet.SetEquals(other);
+                => _builder.SetEquals(other);
 
             /// <inheritdoc cref="ImmutableHashSet{T}.Builder.SymmetricExceptWith(IEnumerable{T})"/>
             public void SymmetricExceptWith(IEnumerable<T> other)
-                => GetOrCreateMutableSet().SymmetricExceptWith(other);
+                => _builder.SymmetricExceptWith(other);
 
             /// <inheritdoc cref="ImmutableHashSet{T}.Builder.TryGetValue(T, out T)"/>
             public bool TryGetValue(T equalValue, out T actualValue)
-            {
-                if (ReadOnlySet.TryGetValue(equalValue, out var value))
-                {
-                    actualValue = value;
-                    return true;
-                }
-
-                actualValue = equalValue;
-                return false;
-            }
+                => _builder.TryGetValue(equalValue, out actualValue);
 
             /// <inheritdoc cref="ImmutableHashSet{T}.Builder.UnionWith(IEnumerable{T})"/>
             public void UnionWith(IEnumerable<T> other)
-            {
-                if (other is null)
-                    ThrowHelper.ThrowArgumentNullException(ExceptionArgument.other);
-
-                if (_mutableSet is not null)
-                {
-                    _mutableSet.UnionWith(other);
-                    return;
-                }
-
-                if (other is ImmutableSegmentedHashSet<T> { IsEmpty: true })
-                {
-                    return;
-                }
-                else
-                {
-                    // Manually enumerate to avoid changes to the builder if 'other' is empty or only contains items
-                    // already present in the current set.
-                    SegmentedHashSet<T>? mutableSet = null;
-                    foreach (var item in other)
-                    {
-                        if (mutableSet is null)
-                        {
-                            if (ReadOnlySet.Contains(item))
-                                continue;
-
-                            mutableSet = GetOrCreateMutableSet();
-                        }
-
-                        mutableSet.Add(item);
-                    }
-
-                    return;
-                }
-            }
+                => _builder.UnionWith(other);
 
             /// <inheritdoc cref="ImmutableHashSet{T}.Builder.ToImmutable()"/>
             public ImmutableSegmentedHashSet<T> ToImmutable()
-            {
-                _set = new ImmutableSegmentedHashSet<T>(ReadOnlySet);
-                _mutableSet = null;
-                return _set;
-            }
+                => _builder.ToImmutable();
 
             void ICollection<T>.Add(T item)
-                => ((ICollection<T>)GetOrCreateMutableSet()).Add(item);
+                => ICollectionCalls<T>.Add(ref _builder, item);
 
             void ICollection<T>.CopyTo(T[] array, int arrayIndex)
-                => ((ICollection<T>)ReadOnlySet).CopyTo(array, arrayIndex);
+                => ICollectionCalls<T>.CopyTo(ref _builder, array, arrayIndex);
 
             IEnumerator<T> IEnumerable<T>.GetEnumerator()
-                => GetEnumerator();
+                => IEnumerableCalls<T>.GetEnumerator(ref _builder);
 
             IEnumerator IEnumerable.GetEnumerator()
-                => GetEnumerator();
+                => IEnumerableCalls.GetEnumerator(ref _builder);
         }
     }
 }
diff --git a/src/Dependencies/Collections/ImmutableSegmentedHashSet`1+ValueBuilder.cs b/src/Dependencies/Collections/ImmutableSegmentedHashSet`1+ValueBuilder.cs
new file mode 100644
index 0000000000000..c672df7acd97f
--- /dev/null
+++ b/src/Dependencies/Collections/ImmutableSegmentedHashSet`1+ValueBuilder.cs
@@ -0,0 +1,282 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.Diagnostics;
+using Microsoft.CodeAnalysis.Collections.Internal;
+
+namespace Microsoft.CodeAnalysis.Collections
+{
+    internal readonly partial struct ImmutableSegmentedHashSet<T>
+    {
+        private struct ValueBuilder : ISet<T>, IReadOnlyCollection<T>
+        {
+            /// <summary>
+            /// The immutable collection this builder is based on.
+            /// </summary>
+            private ImmutableSegmentedHashSet<T> _set;
+
+            /// <summary>
+            /// The current mutable collection this builder is operating on. This field is initialized to a copy of
+            /// <see cref="_set"/> the first time a change is made.
+            /// </summary>
+            private SegmentedHashSet<T>? _mutableSet;
+
+            internal ValueBuilder(ImmutableSegmentedHashSet<T> set)
+            {
+                _set = set;
+                _mutableSet = null;
+            }
+
+            /// <inheritdoc cref="ImmutableHashSet{T}.Builder.KeyComparer"/>
+            public IEqualityComparer<T> KeyComparer
+            {
+                readonly get
+                {
+                    return ReadOnlySet.Comparer;
+                }
+
+                set
+                {
+                    if (Equals(KeyComparer, value ?? EqualityComparer<T>.Default))
+                        return;
+
+                    _mutableSet = new SegmentedHashSet<T>(ReadOnlySet, value ?? EqualityComparer<T>.Default);
+                    _set = default;
+                }
+            }
+
+            /// <inheritdoc cref="ImmutableHashSet{T}.Builder.Count"/>
+            public readonly int Count => ReadOnlySet.Count;
+
+            internal readonly SegmentedHashSet<T> ReadOnlySet => _mutableSet ?? _set._set;
+
+            readonly bool ICollection<T>.IsReadOnly => false;
+
+            private SegmentedHashSet<T> GetOrCreateMutableSet()
+            {
+                if (_mutableSet is null)
+                {
+                    var originalSet = RoslynImmutableInterlocked.InterlockedExchange(ref _set, default);
+                    if (originalSet.IsDefault)
+                        throw new InvalidOperationException($"Unexpected concurrent access to {GetType()}");
+
+                    _mutableSet = new SegmentedHashSet<T>(originalSet._set, originalSet.KeyComparer);
+                }
+
+                return _mutableSet;
+            }
+
+            /// <inheritdoc cref="ImmutableHashSet{T}.Builder.Add(T)"/>
+            public bool Add(T item)
+            {
+                if (_mutableSet is null && Contains(item))
+                    return false;
+
+                return GetOrCreateMutableSet().Add(item);
+            }
+
+            /// <inheritdoc cref="ImmutableHashSet{T}.Builder.Clear()"/>
+            public void Clear()
+            {
+                if (ReadOnlySet.Count != 0)
+                {
+                    if (_mutableSet is null)
+                    {
+                        _mutableSet = new SegmentedHashSet<T>(KeyComparer);
+                        _set = default;
+                    }
+                    else
+                    {
+                        _mutableSet.Clear();
+                    }
+                }
+            }
+
+            /// <inheritdoc cref="ImmutableHashSet{T}.Builder.Contains(T)"/>
+            public readonly bool Contains(T item)
+                => ReadOnlySet.Contains(item);
+
+            /// <inheritdoc cref="ImmutableHashSet{T}.Builder.ExceptWith(IEnumerable{T})"/>
+            public void ExceptWith(IEnumerable<T> other)
+            {
+                if (other is null)
+                    ThrowHelper.ThrowArgumentNullException(ExceptionArgument.other);
+
+                if (_mutableSet is not null)
+                {
+                    _mutableSet.ExceptWith(other);
+                    return;
+                }
+
+                // ValueBuilder is not a public API, so there shouldn't be any callers trying to pass a boxed instance
+                // to this method.
+                Debug.Assert(other is not ValueBuilder);
+
+                if (other == ReadOnlySet)
+                {
+                    Clear();
+                    return;
+                }
+                else if (other is ImmutableSegmentedHashSet<T> otherSet)
+                {
+                    if (otherSet == _set)
+                    {
+                        Clear();
+                        return;
+                    }
+                    else if (otherSet.IsEmpty)
+                    {
+                        // No action required
+                        return;
+                    }
+                    else
+                    {
+                        GetOrCreateMutableSet().ExceptWith(otherSet._set);
+                        return;
+                    }
+                }
+                else
+                {
+                    // Manually enumerate to avoid changes to the builder if 'other' is empty or does not contain any
+                    // items present in the current set.
+                    SegmentedHashSet<T>? mutableSet = null;
+                    foreach (var item in other)
+                    {
+                        if (mutableSet is null)
+                        {
+                            if (!ReadOnlySet.Contains(item))
+                                continue;
+
+                            mutableSet = GetOrCreateMutableSet();
+                        }
+
+                        mutableSet.Remove(item);
+                    }
+
+                    return;
+                }
+            }
+
+            /// <inheritdoc cref="ImmutableHashSet{T}.Builder.GetEnumerator()"/>
+            public Enumerator GetEnumerator()
+                => new(GetOrCreateMutableSet());
+
+            /// <inheritdoc cref="ImmutableHashSet{T}.Builder.IntersectWith(IEnumerable{T})"/>
+            public void IntersectWith(IEnumerable<T> other)
+                => GetOrCreateMutableSet().IntersectWith(other);
+
+            /// <inheritdoc cref="ImmutableHashSet{T}.Builder.IsProperSubsetOf(IEnumerable{T})"/>
+            public readonly bool IsProperSubsetOf(IEnumerable<T> other)
+                => ReadOnlySet.IsProperSubsetOf(other);
+
+            /// <inheritdoc cref="ImmutableHashSet{T}.Builder.IsProperSupersetOf(IEnumerable{T})"/>
+            public readonly bool IsProperSupersetOf(IEnumerable<T> other)
+                => ReadOnlySet.IsProperSupersetOf(other);
+
+            /// <inheritdoc cref="ImmutableHashSet{T}.Builder.IsSubsetOf(IEnumerable{T})"/>
+            public readonly bool IsSubsetOf(IEnumerable<T> other)
+                => ReadOnlySet.IsSubsetOf(other);
+
+            /// <inheritdoc cref="ImmutableHashSet{T}.Builder.IsSupersetOf(IEnumerable{T})"/>
+            public readonly bool IsSupersetOf(IEnumerable<T> other)
+                => ReadOnlySet.IsSupersetOf(other);
+
+            /// <inheritdoc cref="ImmutableHashSet{T}.Builder.Overlaps(IEnumerable{T})"/>
+            public readonly bool Overlaps(IEnumerable<T> other)
+                => ReadOnlySet.Overlaps(other);
+
+            /// <inheritdoc cref="ImmutableHashSet{T}.Builder.Remove(T)"/>
+            public bool Remove(T item)
+            {
+                if (_mutableSet is null && !Contains(item))
+                    return false;
+
+                return GetOrCreateMutableSet().Remove(item);
+            }
+
+            /// <inheritdoc cref="ImmutableHashSet{T}.Builder.SetEquals(IEnumerable{T})"/>
+            public readonly bool SetEquals(IEnumerable<T> other)
+                => ReadOnlySet.SetEquals(other);
+
+            /// <inheritdoc cref="ImmutableHashSet{T}.Builder.SymmetricExceptWith(IEnumerable{T})"/>
+            public void SymmetricExceptWith(IEnumerable<T> other)
+                => GetOrCreateMutableSet().SymmetricExceptWith(other);
+
+            /// <inheritdoc cref="ImmutableHashSet{T}.Builder.TryGetValue(T, out T)"/>
+            public readonly bool TryGetValue(T equalValue, out T actualValue)
+            {
+                if (ReadOnlySet.TryGetValue(equalValue, out var value))
+                {
+                    actualValue = value;
+                    return true;
+                }
+
+                actualValue = equalValue;
+                return false;
+            }
+
+            /// <inheritdoc cref="ImmutableHashSet{T}.Builder.UnionWith(IEnumerable{T})"/>
+            public void UnionWith(IEnumerable<T> other)
+            {
+                if (other is null)
+                    ThrowHelper.ThrowArgumentNullException(ExceptionArgument.other);
+
+                if (_mutableSet is not null)
+                {
+                    _mutableSet.UnionWith(other);
+                    return;
+                }
+
+                if (other is ImmutableSegmentedHashSet<T> { IsEmpty: true })
+                {
+                    return;
+                }
+                else
+                {
+                    // Manually enumerate to avoid changes to the builder if 'other' is empty or only contains items
+                    // already present in the current set.
+                    SegmentedHashSet<T>? mutableSet = null;
+                    foreach (var item in other)
+                    {
+                        if (mutableSet is null)
+                        {
+                            if (ReadOnlySet.Contains(item))
+                                continue;
+
+                            mutableSet = GetOrCreateMutableSet();
+                        }
+
+                        mutableSet.Add(item);
+                    }
+
+                    return;
+                }
+            }
+
+            /// <inheritdoc cref="ImmutableHashSet{T}.Builder.ToImmutable()"/>
+            public ImmutableSegmentedHashSet<T> ToImmutable()
+            {
+                _set = new ImmutableSegmentedHashSet<T>(ReadOnlySet);
+                _mutableSet = null;
+                return _set;
+            }
+
+            void ICollection<T>.Add(T item)
+                => ((ICollection<T>)GetOrCreateMutableSet()).Add(item);
+
+            readonly void ICollection<T>.CopyTo(T[] array, int arrayIndex)
+                => ((ICollection<T>)ReadOnlySet).CopyTo(array, arrayIndex);
+
+            IEnumerator<T> IEnumerable<T>.GetEnumerator()
+                => GetEnumerator();
+
+            IEnumerator IEnumerable.GetEnumerator()
+                => GetEnumerator();
+        }
+    }
+}
diff --git a/src/Dependencies/Collections/ImmutableSegmentedHashSet`1.cs b/src/Dependencies/Collections/ImmutableSegmentedHashSet`1.cs
index a00875ab0a459..b99e4e99373cc 100644
--- a/src/Dependencies/Collections/ImmutableSegmentedHashSet`1.cs
+++ b/src/Dependencies/Collections/ImmutableSegmentedHashSet`1.cs
@@ -10,6 +10,57 @@
 
 namespace Microsoft.CodeAnalysis.Collections
 {
+    /// <summary>
+    /// Represents a segmented hash set that is immutable; meaning it cannot be changed once it is created.
+    /// </summary>
+    /// <remarks>
+    /// <para>There are different scenarios best for <see cref="ImmutableSegmentedHashSet{T}"/> and others
+    /// best for <see cref="ImmutableHashSet{T}"/>.</para>
+    ///
+    /// <para>The following table summarizes the performance characteristics of
+    /// <see cref="ImmutableSegmentedHashSet{T}"/>:</para>
+    /// 
+    /// <list type="table">
+    ///   <item>
+    ///     <description>Operation</description>
+    ///     <description><see cref="ImmutableSegmentedHashSet{T}"/> Complexity</description>
+    ///     <description><see cref="ImmutableHashSet{T}"/> Complexity</description>
+    ///     <description>Comments</description>
+    ///   </item>
+    ///   <item>
+    ///     <description>Contains</description>
+    ///     <description>O(1)</description>
+    ///     <description>O(log n)</description>
+    ///     <description>Directly index into the underlying segmented list</description>
+    ///   </item>
+    ///   <item>
+    ///     <description>Add()</description>
+    ///     <description>O(n)</description>
+    ///     <description>O(log n)</description>
+    ///     <description>Requires creating a new segmented hash set and cloning all impacted segments</description>
+    ///   </item>
+    /// </list>
+    /// 
+    /// <para>This type is backed by segmented arrays to avoid using the Large Object Heap without impacting algorithmic
+    /// complexity.</para>
+    /// </remarks>
+    /// <typeparam name="T">The type of the value in the set.</typeparam>
+    /// <devremarks>
+    /// <para>This type has a documented contract of being exactly one reference-type field in size. Our own
+    /// <see cref="RoslynImmutableInterlocked"/> class depends on it, as well as others externally.</para>
+    ///
+    /// <para><strong>IMPORTANT NOTICE FOR MAINTAINERS AND REVIEWERS:</strong></para>
+    ///
+    /// <para>This type should be thread-safe. As a struct, it cannot protect its own fields from being changed from one
+    /// thread while its members are executing on other threads because structs can change <em>in place</em> simply by
+    /// reassigning the field containing this struct. Therefore it is extremely important that <strong>⚠⚠ Every member
+    /// should only dereference <c>this</c> ONCE ⚠⚠</strong>. If a member needs to reference the
+    /// <see cref="_set"/> field, that counts as a dereference of <c>this</c>. Calling other instance members
+    /// (properties or methods) also counts as dereferencing <c>this</c>. Any member that needs to use <c>this</c> more
+    /// than once must instead assign <c>this</c> to a local variable and use that for the rest of the code instead.
+    /// This effectively copies the one field in the struct to a local variable so that it is insulated from other
+    /// threads.</para>
+    /// </devremarks>
     internal readonly partial struct ImmutableSegmentedHashSet<T> : IImmutableSet<T>, ISet<T>, ICollection, IEquatable<ImmutableSegmentedHashSet<T>>
     {
         /// <inheritdoc cref="ImmutableHashSet{T}.Empty"/>
@@ -67,9 +118,8 @@ public ImmutableSegmentedHashSet<T> Add(T value)
             }
             else
             {
-                // TODO: Avoid the builder allocation
                 // TODO: Reuse all pages with no changes
-                var builder = self.ToBuilder();
+                var builder = self.ToValueBuilder();
                 builder.Add(value);
                 return builder.ToImmutable();
             }
@@ -113,9 +163,8 @@ public ImmutableSegmentedHashSet<T> Except(IEnumerable<T> other)
             }
             else
             {
-                // TODO: Avoid the builder allocation
                 // TODO: Reuse all pages with no changes
-                var builder = self.ToBuilder();
+                var builder = self.ToValueBuilder();
                 builder.ExceptWith(other);
                 return builder.ToImmutable();
             }
@@ -136,9 +185,8 @@ public ImmutableSegmentedHashSet<T> Intersect(IEnumerable<T> other)
             }
             else
             {
-                // TODO: Avoid the builder allocation
                 // TODO: Reuse all pages with no changes
-                var builder = self.ToBuilder();
+                var builder = self.ToValueBuilder();
                 builder.IntersectWith(other);
                 return builder.ToImmutable();
             }
@@ -175,9 +223,8 @@ public ImmutableSegmentedHashSet<T> Remove(T value)
             }
             else
             {
-                // TODO: Avoid the builder allocation
                 // TODO: Reuse all pages with no changes
-                var builder = self.ToBuilder();
+                var builder = self.ToValueBuilder();
                 builder.Remove(value);
                 return builder.ToImmutable();
             }
@@ -206,9 +253,8 @@ public ImmutableSegmentedHashSet<T> SymmetricExcept(IEnumerable<T> other)
             }
             else
             {
-                // TODO: Avoid the builder allocation
                 // TODO: Reuse all pages with no changes
-                var builder = self.ToBuilder();
+                var builder = self.ToValueBuilder();
                 builder.SymmetricExceptWith(other);
                 return builder.ToImmutable();
             }
@@ -240,9 +286,8 @@ public ImmutableSegmentedHashSet<T> Union(IEnumerable<T> other)
                     return otherSet.WithComparer(self.KeyComparer);
             }
 
-            // TODO: Avoid the builder allocation
             // TODO: Reuse all pages with no changes
-            var builder = self.ToBuilder();
+            var builder = self.ToValueBuilder();
             builder.UnionWith(other);
             return builder.ToImmutable();
         }
@@ -251,6 +296,9 @@ public ImmutableSegmentedHashSet<T> Union(IEnumerable<T> other)
         public Builder ToBuilder()
             => new(this);
 
+        private ValueBuilder ToValueBuilder()
+            => new ValueBuilder(this);
+
         /// <inheritdoc cref="ImmutableHashSet{T}.WithComparer(IEqualityComparer{T}?)"/>
         public ImmutableSegmentedHashSet<T> WithComparer(IEqualityComparer<T>? equalityComparer)
         {
diff --git a/src/Dependencies/Collections/ImmutableSegmentedList`1+Builder.cs b/src/Dependencies/Collections/ImmutableSegmentedList`1+Builder.cs
index 865107bbe052e..174d53ff8904b 100644
--- a/src/Dependencies/Collections/ImmutableSegmentedList`1+Builder.cs
+++ b/src/Dependencies/Collections/ImmutableSegmentedList`1+Builder.cs
@@ -14,7 +14,7 @@ internal partial struct ImmutableSegmentedList<T>
         public sealed class Builder : IList<T>, IReadOnlyList<T>, IList
         {
             /// <summary>
-            /// The immutable collection this builder is based on.
+            /// The private builder implementation.
             /// </summary>
             private ValueBuilder _builder;
 
diff --git a/src/Dependencies/Collections/Internal/ICollectionCalls`1.cs b/src/Dependencies/Collections/Internal/ICollectionCalls`1.cs
index 3eec844a90f5b..45fa0142859da 100644
--- a/src/Dependencies/Collections/Internal/ICollectionCalls`1.cs
+++ b/src/Dependencies/Collections/Internal/ICollectionCalls`1.cs
@@ -20,5 +20,13 @@ internal static class ICollectionCalls<T>
         public static bool IsReadOnly<TCollection>(ref TCollection collection)
             where TCollection : ICollection<T>
             => collection.IsReadOnly;
+
+        public static void Add<TCollection>(ref TCollection collection, T item)
+            where TCollection : ICollection<T>
+            => collection.Add(item);
+
+        public static void CopyTo<TCollection>(ref TCollection collection, T[] array, int arrayIndex)
+            where TCollection : ICollection<T>
+            => collection.CopyTo(array, arrayIndex);
     }
 }
diff --git a/src/Dependencies/Collections/Internal/IDictionaryCalls.cs b/src/Dependencies/Collections/Internal/IDictionaryCalls.cs
new file mode 100644
index 0000000000000..79d95fd22ef6e
--- /dev/null
+++ b/src/Dependencies/Collections/Internal/IDictionaryCalls.cs
@@ -0,0 +1,57 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections;
+
+namespace Microsoft.CodeAnalysis.Collections.Internal
+{
+    /// <summary>
+    /// Provides static methods to invoke <see cref="IDictionary"/> members on value types that explicitly implement the
+    /// member.
+    /// </summary>
+    /// <remarks>
+    /// Normally, invocation of explicit interface members requires boxing or copying the value type, which is
+    /// especially problematic for operations that mutate the value. Invocation through these helpers behaves like a
+    /// normal call to an implicitly implemented member.
+    /// </remarks>
+    internal static class IDictionaryCalls
+    {
+        public static bool IsFixedSize<TDictionary>(ref TDictionary dictionary)
+            where TDictionary : IDictionary
+            => dictionary.IsFixedSize;
+
+        public static bool IsReadOnly<TDictionary>(ref TDictionary dictionary)
+            where TDictionary : IDictionary
+            => dictionary.IsReadOnly;
+
+        public static object? GetItem<TDictionary>(ref TDictionary dictionary, object key)
+            where TDictionary : IDictionary
+            => dictionary[key];
+
+        public static void SetItem<TDictionary>(ref TDictionary dictionary, object key, object? value)
+            where TDictionary : IDictionary
+            => dictionary[key] = value;
+
+        public static void Add<TDictionary>(ref TDictionary dictionary, object key, object? value)
+            where TDictionary : IDictionary
+            => dictionary.Add(key, value);
+
+        public static bool Contains<TDictionary>(ref TDictionary dictionary, object key)
+            where TDictionary : IDictionary
+            => dictionary.Contains(key);
+
+        public static void CopyTo<TDictionary>(ref TDictionary dictionary, Array array, int index)
+            where TDictionary : IDictionary
+            => dictionary.CopyTo(array, index);
+
+        public static IDictionaryEnumerator GetEnumerator<TDictionary>(ref TDictionary dictionary)
+            where TDictionary : IDictionary
+            => dictionary.GetEnumerator();
+
+        public static void Remove<TDictionary>(ref TDictionary dictionary, object key)
+            where TDictionary : IDictionary
+            => dictionary.Remove(key);
+    }
+}
diff --git a/src/Dependencies/Collections/Microsoft.CodeAnalysis.Collections.projitems b/src/Dependencies/Collections/Microsoft.CodeAnalysis.Collections.projitems
index 793bff9426dc8..e17523e5a7ffd 100644
--- a/src/Dependencies/Collections/Microsoft.CodeAnalysis.Collections.projitems
+++ b/src/Dependencies/Collections/Microsoft.CodeAnalysis.Collections.projitems
@@ -19,6 +19,7 @@
     <Compile Include="$(MSBuildThisFileDirectory)ImmutableSegmentedDictionary`2+KeyCollection+Enumerator.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)ImmutableSegmentedDictionary`2+KeyCollection.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)ImmutableSegmentedDictionary`2+PrivateMarshal.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)ImmutableSegmentedDictionary`2+ValueBuilder.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)ImmutableSegmentedDictionary`2+ValueCollection+Enumerator.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)ImmutableSegmentedDictionary`2+ValueCollection.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)ImmutableSegmentedDictionary`2.cs" />
@@ -26,6 +27,7 @@
     <Compile Include="$(MSBuildThisFileDirectory)ImmutableSegmentedHashSet`1+Builder.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)ImmutableSegmentedHashSet`1+Enumerator.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)ImmutableSegmentedHashSet`1+PrivateMarshal.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)ImmutableSegmentedHashSet`1+ValueBuilder.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)ImmutableSegmentedHashSet`1.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)ImmutableSegmentedList.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)ImmutableSegmentedListExtensions.cs" />
@@ -40,6 +42,7 @@
     <Compile Include="$(MSBuildThisFileDirectory)Internal\ICollectionCalls.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Internal\ICollectionCalls`1.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Internal\ICollectionDebugView`1.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)Internal\IDictionaryCalls.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Internal\IDictionaryDebugView`2.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Internal\IEnumerableCalls.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)Internal\IEnumerableCalls`1.cs" />