Fix ref struct scan for sagas (#7187)

* Prevent saga conventions from throwing on ref struct

* Better test

* Tweaks

---------

Co-authored-by: Mike Minutillo <mike.minutillo@particular.net>

src/NServiceBus.AcceptanceTests/Core/Conventions/When_scanning_an_assembly_containing_a_ref_struct_and_sagas_enabled.cs

@@ -0,0 +1,53 @@
+namespace NServiceBus.AcceptanceTests.Core.Conventions;
+
+using System;
+using System.Threading.Tasks;
+using AcceptanceTesting;
+using EndpointTemplates;
+using NServiceBus.AcceptanceTesting.Customization;
+using NUnit.Framework;
+
+public class When_scanning_an_assembly_containing_a_ref_struct_and_sagas_enabled : NServiceBusAcceptanceTest
+{
+    [Test]
+    public void It_should_not_throw_an_exception()
+        => Assert.DoesNotThrowAsync(
+            () => Scenario.Define<ScenarioContext>()
+                          .WithEndpoint<EndpointWithASaga>()
+                          .Run()
+    );
+
+    // HINT: This will get picked up by the AssemblyRouteSource created by the routing call below
+    // Even though it is not a message type, it is still checked by passing it to conventions.
+    // The conventions added by Sagas were throwing an exception when passed a ref struct.
+    // See https://github.com/Particular/NServiceBus/issues/7179 for details.
+    ref struct RefStruct { }
+
+    class EndpointWithASaga : EndpointConfigurationBuilder
+    {
+        public EndpointWithASaga() => EndpointSetup<DefaultServer>(cfg => cfg
+            .ConfigureRouting()
+            .RouteToEndpoint(
+                typeof(RefStruct).Assembly,
+                Conventions.EndpointNamingConvention(typeof(EndpointWithASaga))
+            )
+        );
+
+        class RealSagaToSetUpConventions : Saga<RealSagaToSetUpConventions.RealSagaToSetUpConventionsSagaData>, IAmStartedByMessages<SomeMessage>
+        {
+            public Task Handle(SomeMessage message, IMessageHandlerContext context) => Task.CompletedTask;
+            protected override void ConfigureHowToFindSaga(SagaPropertyMapper<RealSagaToSetUpConventionsSagaData> mapper)
+                => mapper.MapSaga(saga => saga.BusinessId).ToMessage<SomeMessage>(msg => msg.BusinessId);
+
+            public class RealSagaToSetUpConventionsSagaData : ContainSagaData
+            {
+                public virtual Guid BusinessId { get; set; }
+            }
+        }
+    }
+
+    public class SomeMessage : IMessage
+    {
+        public Guid BusinessId { get; set; }
+    }
+}

src/NServiceBus.Core/Sagas/Sagas.cs

@@ -98,6 +98,13 @@ static bool IsCompatible(Type t, Type source)
 
         static bool IsTypeATimeoutHandledByAnySaga(Type type, IEnumerable<Type> sagas)
         {
+            // MakeGenericType() throws an exception if passed a ref struct type
+            // Messages cannot be ref struct types
+            if (type.IsByRefLike)
+            {
+                return false;
+            }
+
             var timeoutHandler = typeof(IHandleTimeouts<>).MakeGenericType(type);
             var messageHandler = typeof(IHandleMessages<>).MakeGenericType(type);
 
@@ -106,4 +113,36 @@ static bool IsTypeATimeoutHandledByAnySaga(Type type, IEnumerable<Type> sagas)
 
         Conventions conventions;
     }
+
+    static bool IsSagaType(Type t)
+    {
+        return IsCompatible(t, typeof(Saga));
+    }
+
+    static bool IsSagaNotFoundHandler(Type t)
+    {
+        return IsCompatible(t, typeof(IHandleSagaNotFound));
+    }
+
+    static bool IsCompatible(Type t, Type source)
+    {
+        return source.IsAssignableFrom(t) && t != source && !t.IsAbstract && !t.IsInterface && !t.IsGenericType;
+    }
+
+    static bool IsTypeATimeoutHandledByAnySaga(Type type, IEnumerable<Type> sagas)
+    {
+        // MakeGenericType() throws an exception if passed a ref struct type
+        // Messages cannot be ref struct types
+        if (type.IsByRefLike)
+        {
+            return false;
+        }
+
+        var timeoutHandler = typeof(IHandleTimeouts<>).MakeGenericType(type);
+        var messageHandler = typeof(IHandleMessages<>).MakeGenericType(type);
+
+        return sagas.Any(t => timeoutHandler.IsAssignableFrom(t) && !messageHandler.IsAssignableFrom(t));
+    }
+
+    Conventions conventions;
 }