Merge pull request MessagePack-CSharp#1119 from neuecc/master

Merge master into develop

README.md

@@ -44,6 +44,7 @@ MessagePack has a compact binary size and a full set of general purpose expressi
     - [Be careful when copying buffers](#be-careful-when-copying-buffers)
     - [Choosing compression](#choosing-compression)
 - [Extensions](#extensions)
+- [Experimental Features](#experimental-features)
 - [High-Level API (`MessagePackSerializer`)](#high-level-api-messagepackserializer)
     - [Multiple MessagePack structures on a single `Stream`](#multiple-messagepack-structures-on-a-single-stream)
 - [Low-Level API (`IMessagePackFormatter<T>`)](#low-level-api-imessagepackformattert)
@@ -403,6 +404,16 @@ public struct Point
 }
 ```
 
+C# 9.0 record with primary constructor is similar immutable object, also supports serialize/deserialize.
+
+```csharp
+// use key as property name
+[MessagePackObject(true)]public record Point(int X, int Y);
+
+// use property: to set KeyAttribute
+[MessagePackObject] public record Point([property:Key(0)] int X, [property: Key(1)] int Y);
+```
+
 ## Serialization Callback
 
 Objects implementing the `IMessagePackSerializationCallbackReceiver` interface will received `OnBeforeSerialize` and `OnAfterDeserialize` calls during serialization/deserialization.
@@ -983,6 +994,16 @@ You can make your own extension serializers or integrate with frameworks. Let's
 * [WebApiContrib.Core.Formatter.MessagePack](https://github.com/WebApiContrib/WebAPIContrib.Core#formatters) - supports ASP.NET Core MVC ([details in blog post](https://www.strathweb.com/2017/06/using-messagepack-with-asp-net-core-mvc/))
 * [MessagePack.MediaTypeFormatter](https://github.com/sketch7/MessagePack.MediaTypeFormatter) - MessagePack MediaTypeFormatter
 
+## Experimental Features
+
+MessagePack for C# has experimental features which provides you with very performant formatters. There is an official package.
+
+```ps1
+Install-Package MessagePack.Experimental
+```
+
+For detailed information, see: [Experimental.md](src/MessagePack.Experimental/Experimental.md)
+
 # API
 
 ## High-Level API (`MessagePackSerializer`)

src/MessagePack.Experimental/Experimental.md

@@ -0,0 +1,11 @@
+# MessagePack.Experimental
+
+This C# project is the experimental project for the features which are very complex, unstable or unsafe.
+
+- [HardwareIntrinsics](HardwareIntrinsics/HardwareIntrinsics.md)
+- [UnsafeUnmanagedStructFormatter](UnsafeUnmanagedStructFormatter/UnsafeUnmanagedStructFormatter.md)
+
+**Caution!**
+
+`MessagePack.Experimental` only targets `.NET Core 3.1` and above.
+You can not use this in Unity and .NET Framework.

src/MessagePack.Experimental/HardwareIntrinsics/HardwareIntrinsics.md

@@ -0,0 +1,24 @@
+# Abstract
+
+`Hardware Intrinsics` is a feature in order to utilize maximum power of the cpu.
+You can serialize/deserialize primitive type array much faster than current implementation!
+
+Supported types
+
+- `sbyte[]`
+- `short[]`
+- `int[]`
+- `bool[]`
+- `float[]`
+- `double[]`
+
+# Usage
+
+```csharp
+var resolver = MessagePack.Resolvers.CompositeResolver.Create(new[] { PrimitiveArrayResolver.Instance, MessagePack.Resolvers.StandardResolver.Instance });
+```
+
+# When will this feature become official?
+
+- The MessagePack-CSharp's lowest target framework is greater or equals to `.NET Core 3.1`.
+- The current very complex and hard to understand implementation is rewritten.

src/MessagePack.Experimental/UnsafeUnmanagedStructFormatter/UnsafeUnmanagedStructFormatter.md

@@ -0,0 +1,29 @@
+# Abstract
+
+`UnsafeUnmanagedStructFormatter`s (de)serialize the blittable value(s) directly.
+They are very performant but unstable against the endian.
+
+You should be careful not to share the encoded byte[] among the different endian environments.
+
+Supported types (T where T : unamanaged)
+
+- `T` → `UnsafeUnmanagedStructFormatter<T>`
+- `T[]` → `UnsafeUnmanagedStructArrayFormatter<T>`
+- `Memory<T>` → `UnsafeUnmanagedStructMemoryFormatter<T>`
+- `ReadOnlyMemory<T>` → `UnsafeUnmanagedStructReadOnlyMemoryFormatter<T>`
+- `ReadOnlySequence<T>` → `UnsafeUnmanagedStructReadOnlySequenceFormatter<T>`
+
+# Usage
+
+```csharp
+var resolver = MessagePack.Resolvers.CompositeResolver.Create(
+    new[] { new UnsafeUnmanagedStructFormatter<Matrix4x4>(typeCode: 96) },
+    new[] { MessagePack.Resolvers.StandardResolver.Instance });
+```
+
+The constructor takes 1 sbyte value.
+The sbyte value is the extension type code embedded in serialized byte sequence.
+
+# When will this feature become official?
+
+- Requests are needed.

src/MessagePack.Generator/MessagepackCompiler.cs

@@ -39,7 +39,7 @@ await Host.CreateDefaultBuilder()
         }
 
         public async Task RunAsync(
-            [Option("i", "Input path of analyze MSBuild project file or directory, if input multiple project files split with ','.")] string input,
+            [Option("i", "Input path to MSBuild project file or the directory containing Unity source files.")] string input,
             [Option("o", "Output file path(.cs) or directory (multiple generate file).")] string output,
             [Option("c", "Conditional compiler symbols, split with ','. Ignored if a project file is specified for input.")] string? conditionalSymbol = null,
             [Option("r", "Set resolver name.")] string resolverName = "GeneratedResolver",

src/MessagePack.UnityClient/Assets/Scripts/MessagePack/Formatters/ExpandoObjectFormatter.cs

@@ -51,8 +51,16 @@ public ExpandoObject Deserialize(ref MessagePackReader reader, MessagePackSerial
 
         public void Serialize(ref MessagePackWriter writer, ExpandoObject value, MessagePackSerializerOptions options)
         {
-            var dictionaryFormatter = options.Resolver.GetFormatterWithVerify<IDictionary<string, object>>();
-            dictionaryFormatter.Serialize(ref writer, value, options);
+            var dict = (IDictionary<string, object>)value;
+            var keyFormatter = options.Resolver.GetFormatterWithVerify<string>();
+            var valueFormatter = options.Resolver.GetFormatterWithVerify<object>();
+
+            writer.WriteMapHeader(dict.Count);
+            foreach (var item in dict)
+            {
+                keyFormatter.Serialize(ref writer, item.Key, options);
+                valueFormatter.Serialize(ref writer, item.Value, options);
+            }
         }
     }
 }

src/MessagePack.UnityClient/Assets/Scripts/MessagePack/Formatters/StandardClassLibraryFormatter.cs

@@ -300,7 +300,7 @@ public void Serialize(ref MessagePackWriter writer, Uri value, MessagePackSerial
             }
             else
             {
-                writer.Write(value.ToString());
+                writer.Write(value.OriginalString);
             }
         }
 
@@ -476,7 +476,7 @@ private BigIntegerFormatter()
 
         public void Serialize(ref MessagePackWriter writer, System.Numerics.BigInteger value, MessagePackSerializerOptions options)
         {
-#if NETCOREAPP2_1
+#if NETCOREAPP
             if (!writer.OldSpec)
             {
                 // try to get bin8 buffer.
@@ -504,7 +504,7 @@ public void Serialize(ref MessagePackWriter writer, System.Numerics.BigInteger v
         public System.Numerics.BigInteger Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options)
         {
             ReadOnlySequence<byte> bytes = reader.ReadBytes().Value;
-#if NETCOREAPP2_1
+#if NETCOREAPP
             if (bytes.IsSingleSegment)
             {
                 return new System.Numerics.BigInteger(bytes.First.Span);

src/MessagePack.UnityClient/Assets/Scripts/MessagePack/Formatters/TypelessFormatter.cs

@@ -16,6 +16,26 @@
 
 namespace MessagePack.Formatters
 {
+#pragma warning disable SA1649 // File name should match first type name
+
+    /// <summary>
+    /// Force serialize object as typeless.
+    /// </summary>
+    public sealed class ForceTypelessFormatter<T> : IMessagePackFormatter<T>
+    {
+        public void Serialize(ref MessagePackWriter writer, T value, MessagePackSerializerOptions options)
+        {
+            TypelessFormatter.Instance.Serialize(ref writer, (object)value, options);
+        }
+
+        public T Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options)
+        {
+            return (T)TypelessFormatter.Instance.Deserialize(ref reader, options);
+        }
+    }
+
+#pragma warning restore SA1649 // File name should match first type name
+
     /// <summary>
     /// For `object` field that holds derived from `object` value, ex: var arr = new object[] { 1, "a", new Model() };.
     /// </summary>
@@ -30,11 +50,11 @@ public sealed class TypelessFormatter : IMessagePackFormatter<object>
         /// </summary>
         public static readonly IMessagePackFormatter<object> Instance = new TypelessFormatter();
 
-        private readonly ThreadsafeTypeKeyHashTable<SerializeMethod> serializers = new ThreadsafeTypeKeyHashTable<SerializeMethod>();
-        private readonly ThreadsafeTypeKeyHashTable<DeserializeMethod> deserializers = new ThreadsafeTypeKeyHashTable<DeserializeMethod>();
-        private readonly ThreadsafeTypeKeyHashTable<byte[]> fullTypeNameCache = new ThreadsafeTypeKeyHashTable<byte[]>();
-        private readonly ThreadsafeTypeKeyHashTable<byte[]> shortenedTypeNameCache = new ThreadsafeTypeKeyHashTable<byte[]>();
-        private readonly AsymmetricKeyHashTable<byte[], ArraySegment<byte>, Type> typeCache = new AsymmetricKeyHashTable<byte[], ArraySegment<byte>, Type>(new StringArraySegmentByteAscymmetricEqualityComparer());
+        private static readonly ThreadsafeTypeKeyHashTable<SerializeMethod> Serializers = new ThreadsafeTypeKeyHashTable<SerializeMethod>();
+        private static readonly ThreadsafeTypeKeyHashTable<DeserializeMethod> Deserializers = new ThreadsafeTypeKeyHashTable<DeserializeMethod>();
+        private static readonly ThreadsafeTypeKeyHashTable<byte[]> FullTypeNameCache = new ThreadsafeTypeKeyHashTable<byte[]>();
+        private static readonly ThreadsafeTypeKeyHashTable<byte[]> ShortenedTypeNameCache = new ThreadsafeTypeKeyHashTable<byte[]>();
+        private static readonly AsymmetricKeyHashTable<byte[], ArraySegment<byte>, Type> TypeCache = new AsymmetricKeyHashTable<byte[], ArraySegment<byte>, Type>(new StringArraySegmentByteAscymmetricEqualityComparer());
 
         private static readonly HashSet<Type> UseBuiltinTypes = new HashSet<Type>
         {
@@ -87,10 +107,10 @@ public sealed class TypelessFormatter : IMessagePackFormatter<object>
         // mscorlib or System.Private.CoreLib
         private static readonly bool IsMscorlib = typeof(int).AssemblyQualifiedName.Contains("mscorlib");
 
-        private TypelessFormatter()
+        static TypelessFormatter()
         {
-            this.serializers.TryAdd(typeof(object), _ => (object p1, ref MessagePackWriter p2, object p3, MessagePackSerializerOptions p4) => { });
-            this.deserializers.TryAdd(typeof(object), _ => (object p1, ref MessagePackReader p2, MessagePackSerializerOptions p3) => new object());
+            Serializers.TryAdd(typeof(object), _ => (object p1, ref MessagePackWriter p2, object p3, MessagePackSerializerOptions p4) => { });
+            Deserializers.TryAdd(typeof(object), _ => (object p1, ref MessagePackReader p2, MessagePackSerializerOptions p3) => new object());
         }
 
         private string BuildTypeName(Type type, MessagePackSerializerOptions options)
@@ -125,7 +145,7 @@ public void Serialize(ref MessagePackWriter writer, object value, MessagePackSer
             Type type = value.GetType();
 
             byte[] typeName;
-            var typeNameCache = options.OmitAssemblyVersion ? this.shortenedTypeNameCache : this.fullTypeNameCache;
+            var typeNameCache = options.OmitAssemblyVersion ? ShortenedTypeNameCache : FullTypeNameCache;
             if (!typeNameCache.TryGetValue(type, out typeName))
             {
                 TypeInfo ti = type.GetTypeInfo();
@@ -150,12 +170,12 @@ public void Serialize(ref MessagePackWriter writer, object value, MessagePackSer
             var formatter = options.Resolver.GetFormatterDynamicWithVerify(type);
 
             // don't use GetOrAdd for avoid closure capture.
-            if (!this.serializers.TryGetValue(type, out SerializeMethod serializeMethod))
+            if (!Serializers.TryGetValue(type, out SerializeMethod serializeMethod))
             {
                 // double check locking...
-                lock (this.serializers)
+                lock (Serializers)
                 {
-                    if (!this.serializers.TryGetValue(type, out serializeMethod))
+                    if (!Serializers.TryGetValue(type, out serializeMethod))
                     {
                         TypeInfo ti = type.GetTypeInfo();
 
@@ -176,7 +196,7 @@ public void Serialize(ref MessagePackWriter writer, object value, MessagePackSer
 
                         serializeMethod = Expression.Lambda<SerializeMethod>(body, param0, param1, param2, param3).Compile();
 
-                        this.serializers.TryAdd(type, serializeMethod);
+                        Serializers.TryAdd(type, serializeMethod);
                     }
                 }
             }
@@ -243,7 +263,7 @@ private object DeserializeByTypeName(ArraySegment<byte> typeName, ref MessagePac
         {
             // try get type with assembly name, throw if not found
             Type type;
-            if (!this.typeCache.TryGetValue(typeName, out type))
+            if (!TypeCache.TryGetValue(typeName, out type))
             {
                 var buffer = new byte[typeName.Count];
                 Buffer.BlockCopy(typeName.Array, typeName.Offset, buffer, 0, buffer.Length);
@@ -267,18 +287,18 @@ private object DeserializeByTypeName(ArraySegment<byte> typeName, ref MessagePac
                     }
                 }
 
-                this.typeCache.TryAdd(buffer, type);
+                TypeCache.TryAdd(buffer, type);
             }
 
             options.ThrowIfDeserializingTypeIsDisallowed(type);
 
             var formatter = options.Resolver.GetFormatterDynamicWithVerify(type);
 
-            if (!this.deserializers.TryGetValue(type, out DeserializeMethod deserializeMethod))
+            if (!Deserializers.TryGetValue(type, out DeserializeMethod deserializeMethod))
             {
-                lock (this.deserializers)
+                lock (Deserializers)
                 {
-                    if (!this.deserializers.TryGetValue(type, out deserializeMethod))
+                    if (!Deserializers.TryGetValue(type, out deserializeMethod))
                     {
                         TypeInfo ti = type.GetTypeInfo();
 
@@ -303,7 +323,7 @@ private object DeserializeByTypeName(ArraySegment<byte> typeName, ref MessagePac
 
                         deserializeMethod = Expression.Lambda<DeserializeMethod>(body, param0, param1, param2).Compile();
 
-                        this.deserializers.TryAdd(type, deserializeMethod);
+                        Deserializers.TryAdd(type, deserializeMethod);
                     }
                 }
             }

src/MessagePack.UnityClient/Assets/Scripts/MessagePack/Internal/CodeGenHelpers.cs

@@ -17,6 +17,11 @@ namespace MessagePack.Internal
     [EditorBrowsable(EditorBrowsableState.Never)]
     public static class CodeGenHelpers
     {
+        /// <summary>
+        /// Gets the messagepack encoding for a given string.
+        /// </summary>
+        /// <param name="value">The string to encode.</param>
+        /// <returns>The messagepack encoding for <paramref name="value"/>, including messagepack header and UTF-8 bytes.</returns>
         public static byte[] GetEncodedStringBytes(string value)
         {
             var byteCount = StringEncoding.UTF8.GetByteCount(value);
@@ -57,6 +62,12 @@ public static byte[] GetEncodedStringBytes(string value)
             }
         }
 
+        /// <summary>
+        /// Gets a single <see cref="ReadOnlySpan{T}"/> containing all bytes in a given <see cref="ReadOnlySequence{T}"/>.
+        /// An array may be allocated if the bytes are not already contiguous in memory.
+        /// </summary>
+        /// <param name="sequence">The sequence to get a span for.</param>
+        /// <returns>The span.</returns>
         public static ReadOnlySpan<byte> GetSpanFromSequence(in ReadOnlySequence<byte> sequence)
         {
             if (sequence.IsSingleSegment)
@@ -67,6 +78,12 @@ public static ReadOnlySpan<byte> GetSpanFromSequence(in ReadOnlySequence<byte> s
             return sequence.ToArray();
         }
 
+        /// <summary>
+        /// Reads a string as a contiguous span of UTF-8 encoded characters.
+        /// An array may be allocated if the string is not already contiguous in memory.
+        /// </summary>
+        /// <param name="reader">The reader to use.</param>
+        /// <returns>The span of UTF-8 encoded characters.</returns>
         public static ReadOnlySpan<byte> ReadStringSpan(ref MessagePackReader reader)
         {
             if (!reader.TryReadStringSpan(out ReadOnlySpan<byte> result))
@@ -77,6 +94,11 @@ public static ReadOnlySpan<byte> ReadStringSpan(ref MessagePackReader reader)
             return result;
         }
 
+        /// <summary>
+        /// Creates a <see cref="byte"/> array for a given sequence, or <see langword="null" /> if the optional sequence is itself <see langword="null" />.
+        /// </summary>
+        /// <param name="sequence">The sequence.</param>
+        /// <returns>The byte array or <see langword="null" /> .</returns>
         public static byte[] GetArrayFromNullableSequence(in ReadOnlySequence<byte>? sequence) => sequence?.ToArray();
 
         private static ReadOnlySpan<byte> GetSpanFromSequence(in ReadOnlySequence<byte>? sequence)

src/MessagePack.UnityClient/Assets/Scripts/MessagePack/MessagePackReader.cs

@@ -1109,7 +1109,7 @@ private string ReadStringSlow(int byteLength)
                 int bytesRead = Math.Min(remainingByteLength, this.reader.UnreadSpan.Length);
                 remainingByteLength -= bytesRead;
                 bool flush = remainingByteLength == 0;
-#if NETCOREAPP2_1
+#if NETCOREAPP
                 initializedChars += decoder.GetChars(this.reader.UnreadSpan.Slice(0, bytesRead), charArray.AsSpan(initializedChars), flush);
 #else
                 unsafe