Add simple marshaller which marshal only null COM interfaces (#8128)

src/Common/src/TypeSystem/Interop/IL/MarshalHelpers.cs

@@ -158,6 +158,9 @@ internal static TypeDesc GetNativeTypeFromMarshallerKind(TypeDesc type,
                 case MarshallerKind.AsAnyW:
                     return context.GetWellKnownType(WellKnownType.IntPtr);
 
+                case MarshallerKind.ComInterface:
+                    return context.GetWellKnownType(WellKnownType.IntPtr);
+
                 case MarshallerKind.Unknown:
                 default:
                     throw new NotSupportedException();
@@ -563,6 +566,10 @@ internal static MarshallerKind GetMarshallerKind(
                 else
                     return MarshallerKind.Invalid;
             }
+            else if (type.IsInterface)
+            {
+                return MarshallerKind.ComInterface;
+            }
             else
                 return MarshallerKind.Invalid;
         }

src/Common/src/TypeSystem/Interop/IL/Marshaller.Aot.cs

@@ -71,6 +71,8 @@ protected static Marshaller CreateMarshaller(MarshallerKind kind)
                     return new AsAnyMarshaller(isAnsi: true);
                 case MarshallerKind.AsAnyW:
                     return new AsAnyMarshaller(isAnsi: false);
+                case MarshallerKind.ComInterface:
+                    return new ComInterfaceMarshaller();
                 default:
                     // ensures we don't throw during create marshaller. We will throw NSE
                     // during EmitIL which will be handled and an Exception method body 
@@ -851,4 +853,31 @@ protected override void EmitCleanupManaged(ILCodeStream codeStream)
             codeStream.EmitLabel(lNull);
         }
     }
+
+    class ComInterfaceMarshaller : Marshaller
+    {
+        protected override void AllocAndTransformManagedToNative(ILCodeStream codeStream)
+        {
+            ILEmitter emitter = _ilCodeStreams.Emitter;
+
+            var helper = Context.GetHelperEntryPoint("InteropHelpers", "ConvertManagedComInterfaceToNative");
+            LoadManagedValue(codeStream);
+
+            codeStream.Emit(ILOpcode.call, emitter.NewToken(helper));
+
+            StoreNativeValue(codeStream);
+        }
+
+        protected override void AllocAndTransformNativeToManaged(ILCodeStream codeStream)
+        {
+            ILEmitter emitter = _ilCodeStreams.Emitter;
+
+            var helper = Context.GetHelperEntryPoint("InteropHelpers", "ConvertNativeComInterfaceToManaged");
+            LoadManagedValue(codeStream);
+
+            codeStream.Emit(ILOpcode.call, emitter.NewToken(helper));
+
+            StoreNativeValue(codeStream);
+        }
+    }
 }

src/Common/src/TypeSystem/Interop/IL/Marshaller.cs

@@ -50,6 +50,7 @@ enum MarshallerKind
         LayoutClassPtr,
         AsAnyA,
         AsAnyW,
+        ComInterface,
         Invalid
     }
     public enum MarshalDirection

src/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/InteropHelpers.cs

@@ -407,6 +407,26 @@ public static T GetCurrentCalleeDelegate<T>() where T : class // constraint can'
             return PInvokeMarshal.GetCurrentCalleeDelegate<T>();
         }
 
+        public static IntPtr ConvertManagedComInterfaceToNative(object pUnk)
+        {
+            if (pUnk == null)
+            {
+                return IntPtr.Zero;
+            }
+
+            throw new PlatformNotSupportedException(SR.PlatformNotSupported_ComInterop);
+        }
+
+        public static object ConvertNativeComInterfaceToManaged(IntPtr pUnk)
+        {
+            if (pUnk == IntPtr.Zero)
+            {
+                return null;
+            }
+
+            throw new PlatformNotSupportedException(SR.PlatformNotSupported_ComInterop);
+        }
+
         internal static int AsAnyGetNativeSize(object o)
         {
             // Array, string and StringBuilder are not implemented.

tests/src/Simple/PInvoke/PInvoke.cs

@@ -210,6 +210,9 @@ private static extern bool VerifySizeParamIndex(
         [DllImport("*", CallingConvention = CallingConvention.StdCall)]
         static extern bool IsNULL(SequentialStruct[] foo);
 
+        [DllImport("*", CallingConvention = CallingConvention.StdCall)]
+        static extern bool IsNULL(IComInterface foo);
+
         [StructLayout(LayoutKind.Sequential, CharSet= CharSet.Ansi, Pack = 4)]
         public unsafe struct InlineArrayStruct
         {
@@ -294,7 +297,10 @@ public static int Main(string[] args)
             TestLayoutClass();
             TestAsAny();
             TestMarshalStructAPIs();
-#endif            
+#if TARGET_WINDOWS
+            TestComInteropNullPointers();
+#endif
+#endif
             return 100;
         }
 
@@ -993,6 +999,14 @@ private static void TestMarshalStructAPIs()
             int cftf_size = Marshal.SizeOf(typeof(ClassForTestingFlowAnalysis));
             ThrowIfNotEquals(4, cftf_size, "ClassForTestingFlowAnalysis marshalling Marshal API failed");
         }
+
+        public static void TestComInteropNullPointers()
+        {
+            Console.WriteLine("Testing Marshal APIs for COM interfaces");
+            IComInterface comPointer = null;
+            var result = IsNULL(comPointer);
+            ThrowIfNotEquals(true, IsNULL(comPointer), "COM interface marshalling null check failed");
+        }
     }
 
     public class SafeMemoryHandle : SafeHandle //SafeHandle subclass
@@ -1021,6 +1035,14 @@ override protected bool ReleaseHandle()
         }
     } //end of SafeMemoryHandle class
 
+    [ComImport]
+    [ComVisible(true)]
+    [Guid("D6DD68D1-86FD-4332-8666-9ABEDEA2D24C")]
+    [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
+    public interface IComInterface
+    {
+    }
+
     public static class LowLevelExtensions
     {
         // Int32.ToString() calls into glob/loc garbage that hits CppCodegen limitations