Separate registration for global ComWrappers instance for tracker sup…

…port and marshalling (#35681)

src/coreclr/src/System.Private.CoreLib/src/System/Runtime/InteropServices/ComWrappers.cs

@@ -56,6 +56,16 @@ public enum CreateObjectFlags
         UniqueInstance = 2,
     }
 
+    /// <summary>
+    /// Internal enumeration used by the runtime to indicate the scenario for which ComWrappers is being used.
+    /// </summary>
+    internal enum ComWrappersScenario
+    {
+        Instance = 0,
+        TrackerSupportGlobalInstance = 1,
+        MarshallingGlobalInstance = 2,
+    }
+
     /// <summary>
     /// Class for managing wrappers of COM IUnknown types.
     /// </summary>
@@ -107,9 +117,14 @@ private struct ComInterfaceInstance
         }
 
         /// <summary>
-        /// Globally registered instance of the ComWrappers class.
+        /// Globally registered instance of the ComWrappers class for reference tracker support.
         /// </summary>
-        private static ComWrappers? s_globalInstance;
+        private static ComWrappers? s_globalInstanceForTrackerSupport;
+
+        /// <summary>
+        /// Globally registered instance of the ComWrappers class for marshalling.
+        /// </summary>
+        private static ComWrappers? s_globalInstanceForMarshalling;
 
         /// <summary>
         /// Create a COM representation of the supplied object that can be passed to a non-managed environment.
@@ -164,10 +179,23 @@ private static bool TryGetOrCreateComInterfaceForObjectInternal(ComWrappers? imp
         /// </remarks>
         protected unsafe abstract ComInterfaceEntry* ComputeVtables(object obj, CreateComInterfaceFlags flags, out int count);
 
-        // Call to execute the abstract instance function
-        internal static unsafe void* CallComputeVtables(ComWrappers? comWrappersImpl, object obj, CreateComInterfaceFlags flags, out int count)
+        // Called by the runtime to execute the abstract instance function
+        internal static unsafe void* CallComputeVtables(ComWrappersScenario scenario, ComWrappers? comWrappersImpl, object obj, CreateComInterfaceFlags flags, out int count)
         {
-            ComWrappers? impl = comWrappersImpl ?? s_globalInstance;
+            ComWrappers? impl = null;
+            switch (scenario)
+            {
+                case ComWrappersScenario.Instance:
+                    impl = comWrappersImpl;
+                    break;
+                case ComWrappersScenario.TrackerSupportGlobalInstance:
+                    impl = s_globalInstanceForTrackerSupport;
+                    break;
+                case ComWrappersScenario.MarshallingGlobalInstance:
+                    impl = s_globalInstanceForMarshalling;
+                    break;
+            }
+
             if (impl is null)
             {
                 count = -1;
@@ -203,10 +231,23 @@ public object GetOrCreateObjectForComInstance(IntPtr externalComObject, CreateOb
         /// </remarks>
         protected abstract object? CreateObject(IntPtr externalComObject, CreateObjectFlags flags);
 
-        // Call to execute the abstract instance function
-        internal static object? CallCreateObject(ComWrappers? comWrappersImpl, IntPtr externalComObject, CreateObjectFlags flags)
+        // Called by the runtime to execute the abstract instance function.
+        internal static object? CallCreateObject(ComWrappersScenario scenario, ComWrappers? comWrappersImpl, IntPtr externalComObject, CreateObjectFlags flags)
         {
-            ComWrappers? impl = comWrappersImpl ?? s_globalInstance;
+            ComWrappers? impl = null;
+            switch (scenario)
+            {
+                case ComWrappersScenario.Instance:
+                    impl = comWrappersImpl;
+                    break;
+                case ComWrappersScenario.TrackerSupportGlobalInstance:
+                    impl = s_globalInstanceForTrackerSupport;
+                    break;
+                case ComWrappersScenario.MarshallingGlobalInstance:
+                    impl = s_globalInstanceForMarshalling;
+                    break;
+            }
+
             if (impl == null)
                 return null;
 
@@ -268,32 +309,62 @@ private static bool TryGetOrCreateObjectForComInstanceInternal(ComWrappers? impl
 
         // Call to execute the virtual instance function
         internal static void CallReleaseObjects(ComWrappers? comWrappersImpl, IEnumerable objects)
-            => (comWrappersImpl ?? s_globalInstance!).ReleaseObjects(objects);
+            => (comWrappersImpl ?? s_globalInstanceForTrackerSupport!).ReleaseObjects(objects);
 
         /// <summary>
-        /// Register this class's implementation to be used as the single global instance.
+        /// Register a <see cref="ComWrappers" /> instance to be used as the global instance for reference tracker support.
         /// </summary>
+        /// <param name="instance">Instance to register</param>
         /// <remarks>
         /// This function can only be called a single time. Subsequent calls to this function will result
         /// in a <see cref="System.InvalidOperationException"/> being thrown.
         ///
-        /// Scenarios where the global instance may be used are:
+        /// Scenarios where this global instance may be used are:
         ///  * Object tracking via the <see cref="CreateComInterfaceFlags.TrackerSupport" /> and <see cref="CreateObjectFlags.TrackerObject" /> flags.
-        ///  * Usage of COM related Marshal APIs.
         /// </remarks>
-        public void RegisterAsGlobalInstance()
+        public static void RegisterForTrackerSupport(ComWrappers instance)
         {
-            if (null != Interlocked.CompareExchange(ref s_globalInstance, this, null))
+            if (instance == null)
+                throw new ArgumentNullException(nameof(instance));
+
+            if (null != Interlocked.CompareExchange(ref s_globalInstanceForTrackerSupport, instance, null))
+            {
+                throw new InvalidOperationException(SR.InvalidOperation_ResetGlobalComWrappersInstance);
+            }
+        }
+
+        /// <summary>
+        /// Register a <see cref="ComWrappers" /> instance to be used as the global instance for marshalling in the runtime.
+        /// </summary>
+        /// <param name="instance">Instance to register</param>
+        /// <remarks>
+        /// This function can only be called a single time. Subsequent calls to this function will result
+        /// in a <see cref="System.InvalidOperationException"/> being thrown.
+        ///
+        /// Scenarios where this global instance may be used are:
+        ///  * Usage of COM-related Marshal APIs
+        ///  * P/Invokes with COM-related types
+        ///  * COM activation
+        /// </remarks>
+        public static void RegisterForMarshalling(ComWrappers instance)
+        {
+            if (instance == null)
+                throw new ArgumentNullException(nameof(instance));
+
+            if (null != Interlocked.CompareExchange(ref s_globalInstanceForMarshalling, instance, null))
             {
                 throw new InvalidOperationException(SR.InvalidOperation_ResetGlobalComWrappersInstance);
             }
 
-            SetGlobalInstanceRegistered();
+            // Indicate to the runtime that a global instance has been registered for marshalling.
+            // This allows the native runtime know to call into the managed ComWrappers only if a
+            // global instance is registered for marshalling.
+            SetGlobalInstanceRegisteredForMarshalling();
         }
 
         [DllImport(RuntimeHelpers.QCall)]
         [SuppressGCTransition]
-        private static extern void SetGlobalInstanceRegistered();
+        private static extern void SetGlobalInstanceRegisteredForMarshalling();
 
         /// <summary>
         /// Get the runtime provided IUnknown implementation.

src/coreclr/src/vm/ecalllist.h

@@ -985,7 +985,7 @@ FCFuncStart(gComWrappersFuncs)
     QCFuncElement("GetIUnknownImplInternal", ComWrappersNative::GetIUnknownImpl)
     QCFuncElement("TryGetOrCreateComInterfaceForObjectInternal", ComWrappersNative::TryGetOrCreateComInterfaceForObject)
     QCFuncElement("TryGetOrCreateObjectForComInstanceInternal", ComWrappersNative::TryGetOrCreateObjectForComInstance)
-    QCFuncElement("SetGlobalInstanceRegistered", GlobalComWrappers::SetGlobalInstanceRegistered)
+    QCFuncElement("SetGlobalInstanceRegisteredForMarshalling", GlobalComWrappersForMarshalling::SetGlobalInstanceRegisteredForMarshalling)
 FCFuncEnd()
 #endif // FEATURE_COMWRAPPERS
 

src/coreclr/src/vm/interopconverter.cpp

@@ -28,7 +28,7 @@ namespace
         _Outptr_ IUnknown** wrapperRaw)
     {
 #ifdef FEATURE_COMWRAPPERS
-        return GlobalComWrappers::TryGetOrCreateComInterfaceForObject(instance, (void**)wrapperRaw);
+        return GlobalComWrappersForMarshalling::TryGetOrCreateComInterfaceForObject(instance, (void**)wrapperRaw);
 #else
         return false;
 #endif // FEATURE_COMWRAPPERS
@@ -40,7 +40,7 @@ namespace
         _Out_ OBJECTREF *pObjOut)
     {
 #ifdef FEATURE_COMWRAPPERS
-        return GlobalComWrappers::TryGetOrCreateObjectForComInstance(pUnknown, dwFlags, pObjOut);
+        return GlobalComWrappersForMarshalling::TryGetOrCreateObjectForComInstance(pUnknown, dwFlags, pObjOut);
 #else
         return false;
 #endif // FEATURE_COMWRAPPERS