Add test for boxing byreflike fail. (dotnet#101458)

* Check if type is compatible right before emitting box

* Add test for using byreflike type in isinst expressions.

src/tests/Loader/classloader/generics/ByRefLike/InvalidCSharp.il

@@ -149,6 +149,41 @@
         ret
     }
 
+    .method public hidebysig
+        instance bool BoxBranchToOther<byreflike U>(!T) cil managed
+    {
+        ldarg.1
+        // Begin sequence
+            box !!U
+            brfalse.s NEXT_1
+        // End sequence
+        NEXT_1:
+
+        ldarg.1
+        // Begin sequence
+            box !!U
+            brfalse NEXT_2
+        // End sequence
+        NEXT_2:
+
+        ldarg.1
+        // Begin sequence
+            box !!U
+            brtrue.s NEXT_3
+        // End sequence
+        NEXT_3:
+
+        ldarg.1
+        // Begin sequence
+            box !!U
+            brtrue NEXT_4
+        // End sequence
+        NEXT_4:
+
+        ldc.i4.1
+        ret
+    }
+
     .method public hidebysig
         instance bool BoxBranch_WithSideEffects(!T&) cil managed
     {
@@ -284,6 +319,26 @@
         ret
     }
 
+    .method public hidebysig
+        instance bool BoxIsinstBranch_UsingTypeConstraints<byreflike (InvalidCSharp.EmptyInterface) U>(class InvalidCSharp.EmptyInterface) cil managed
+    {
+        .locals init (
+            [0] !!U
+        )
+        ldarg.1
+        isinst !!U
+        brfalse.s NOT_U
+        ldarg.1
+        isinst !!U
+        unbox.any !!U
+        stloc.0
+        ldc.i4.0
+        ret
+        NOT_U:
+        ldc.i4.1
+        ret
+    }
+
     .method public hidebysig
         instance bool AllocArrayOfT() cil managed aggressiveinlining
     {
@@ -406,6 +461,19 @@
 {
 }
 
+.class public auto ansi beforefieldinit InvalidCSharp.ClassWithInterface
+    extends [System.Runtime]System.Object
+    implements InvalidCSharp.EmptyInterface
+{
+    .method public hidebysig specialname rtspecialname
+        instance void .ctor () cil managed
+    {
+        ldarg.0
+        call instance void [System.Runtime]System.Object::.ctor()
+        ret
+    }
+}
+
 // Generic substitution of allow-byreflike with allow-byreflike
 .class interface public auto ansi abstract InvalidCSharp.GenericDerivedInterface_OverByRef`1<byreflike T>
     implements class InvalidCSharp.GenericInterface_Over`1<!T>
@@ -651,6 +719,18 @@
         call instance bool valuetype InvalidCSharp.GenericByRefLike_Over`1<valuetype ByRefLikeType>::BoxBranch(!0)
         pop
 
+        ldloca.s 0
+        ldloc 0
+        ldfld !0 valuetype InvalidCSharp.GenericByRefLike_Over`1<valuetype ByRefLikeType>::Field
+        call instance bool valuetype InvalidCSharp.GenericByRefLike_Over`1<valuetype ByRefLikeType>::BoxBranchToOther<valuetype InvalidCSharp.ByRefLikeTypeWithInterface>(!0)
+        pop
+
+        ldloca.s 0
+        ldloc 0
+        ldfld !0 valuetype InvalidCSharp.GenericByRefLike_Over`1<valuetype ByRefLikeType>::Field
+        call instance bool valuetype InvalidCSharp.GenericByRefLike_Over`1<valuetype ByRefLikeType>::BoxBranchToOther<valuetype RegularValueType>(!0)
+        pop
+
         ldloca.s 0
         ldloca.s 0
         ldflda !0 valuetype InvalidCSharp.GenericByRefLike_Over`1<valuetype ByRefLikeType>::Field
@@ -695,6 +775,11 @@
         ldloca.s 0
         ldflda !0 valuetype InvalidCSharp.GenericByRefLike_Over`1<valuetype ByRefLikeType>::Field
         call instance bool valuetype InvalidCSharp.GenericByRefLike_Over`1<valuetype ByRefLikeType>::BoxIsinstBranch_WithSideEffects(!0&)
+        pop
+
+        ldloca.s 0
+        newobj instance void InvalidCSharp.ClassWithInterface::.ctor()
+        call instance bool valuetype InvalidCSharp.GenericByRefLike_Over`1<valuetype ByRefLikeType>::BoxIsinstBranch_UsingTypeConstraints<valuetype InvalidCSharp.ByRefLikeTypeWithInterface>(class InvalidCSharp.EmptyInterface)
 
         ret
     }