diff --git a/src/HDInsight/Microsoft.Hadoop.Avro.Tests/AvroSerializerTests.cs b/src/HDInsight/Microsoft.Hadoop.Avro.Tests/AvroSerializerTests.cs index 4d10e9f53cb8e..c7b9a2a8dea44 100644 --- a/src/HDInsight/Microsoft.Hadoop.Avro.Tests/AvroSerializerTests.cs +++ b/src/HDInsight/Microsoft.Hadoop.Avro.Tests/AvroSerializerTests.cs @@ -12,11 +12,13 @@ // // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. + namespace Microsoft.Hadoop.Avro.Tests { using System; using System.Collections.Concurrent; using System.Collections.Generic; + using System.Collections.ObjectModel; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; @@ -240,10 +242,26 @@ public void Serializer_SerializeListClass() [TestCategory("CheckIn")] public void Serializer_SerializeIList() { - var knownTypes = new[] { typeof(List) }; + var knownTypes = new[] { typeof(List), typeof(List) }; RoundTripSerializationWithCheck(IListClass.Create(), new AvroSerializerSettings { KnownTypes = knownTypes }); } + [TestMethod] + [TestCategory("CheckIn")] + public void Serializer_SerializeIListWithArray() + { + var knownTypes = new[] { typeof(Guid[]), typeof(List) }; + RoundTripSerializationWithCheck(IListClass.CreateWithArray(), new AvroSerializerSettings { KnownTypes = knownTypes }); + } + + [TestMethod] + [TestCategory("CheckIn")] + public void Serializer_SerializeIListWithCollection() + { + var knownTypes = new[] { typeof(Collection), typeof(Collection) }; + RoundTripSerializationWithCheck(IListClass.CreateWithCollection(), new AvroSerializerSettings { KnownTypes = knownTypes }); + } + [TestMethod] [TestCategory("CheckIn")] public void Serializer_SerializeInheritedList() diff --git a/src/HDInsight/Microsoft.Hadoop.Avro.Tests/Microsoft.Hadoop.Avro.Tests.csproj b/src/HDInsight/Microsoft.Hadoop.Avro.Tests/Microsoft.Hadoop.Avro.Tests.csproj index 5fa5eece0c073..cc58ddf8751d9 100644 --- a/src/HDInsight/Microsoft.Hadoop.Avro.Tests/Microsoft.Hadoop.Avro.Tests.csproj +++ b/src/HDInsight/Microsoft.Hadoop.Avro.Tests/Microsoft.Hadoop.Avro.Tests.csproj @@ -124,6 +124,7 @@ + diff --git a/src/HDInsight/Microsoft.Hadoop.Avro.Tests/TestClasses/TestClassesCollections.cs b/src/HDInsight/Microsoft.Hadoop.Avro.Tests/TestClasses/TestClassesCollections.cs index 594564335ac2c..7233cda66bbfa 100644 --- a/src/HDInsight/Microsoft.Hadoop.Avro.Tests/TestClasses/TestClassesCollections.cs +++ b/src/HDInsight/Microsoft.Hadoop.Avro.Tests/TestClasses/TestClassesCollections.cs @@ -12,10 +12,12 @@ // // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. + namespace Microsoft.Hadoop.Avro.Tests { using System; using System.Collections.Generic; + using System.Collections.ObjectModel; using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Runtime.Serialization; @@ -58,13 +60,27 @@ internal class IListClass : IEquatable { public static IListClass Create() { - return new IListClass { Field1 = new List { Guid.NewGuid(), Guid.NewGuid(), Guid.NewGuid() } }; + return new IListClass { Field1 = new List { Guid.NewGuid() }, Field2 = new List { 117 } }; + } + + public static IListClass CreateWithArray() + { + return new IListClass { Field1 = new[] { Guid.NewGuid(), Guid.NewGuid() }, Field2 = new List { 12 } }; + } + + public static IListClass CreateWithCollection() + { + return new IListClass { Field1 = new Collection() { Guid.NewGuid(), Guid.NewGuid() }, Field2 = new Collection() { 12 } }; } [ProtoMember(1)] [DataMember] public IList Field1 { get; set; } + [ProtoMember(1)] + [DataMember] + public IList Field2 { get; set; } + public bool Equals(IListClass other) { if (other == null) diff --git a/src/HDInsight/Microsoft.Hadoop.Avro.Tests/TypeExtensionsTest.cs b/src/HDInsight/Microsoft.Hadoop.Avro.Tests/TypeExtensionsTest.cs new file mode 100644 index 0000000000000..7f5a399e2c3b9 --- /dev/null +++ b/src/HDInsight/Microsoft.Hadoop.Avro.Tests/TypeExtensionsTest.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy +// of the License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +namespace Microsoft.Hadoop.Avro.Tests +{ + using System; + using System.Collections.Generic; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class TypeExtensionsTest + { + [TestMethod] + public void CanBeKnownTypeOfTest() + { + Assert.IsFalse(typeof(IEnumerable).CanBeKnownTypeOf(typeof(IEnumerable))); + Assert.IsFalse(typeof(IEnumerable).CanBeKnownTypeOf(typeof(IEnumerable))); + Assert.IsFalse(typeof(IEnumerable).CanBeKnownTypeOf(typeof(IEnumerable))); + Assert.IsFalse(typeof(IList).CanBeKnownTypeOf(typeof(int[]))); + Assert.IsFalse(typeof(int).CanBeKnownTypeOf(typeof(Guid))); + Assert.IsFalse(typeof(IList).CanBeKnownTypeOf(typeof(IList))); + + Assert.IsFalse(typeof(int[]).CanBeKnownTypeOf(typeof(IList))); + Assert.IsFalse(typeof(int[]).CanBeKnownTypeOf(typeof(IList))); + } + + [TestMethod] + public void GenericIsAssignableTest() + { + Assert.IsFalse(typeof(int[]).GenericIsAssignable(typeof(IDictionary))); + Assert.IsFalse(typeof(IList).GenericIsAssignable(typeof(IDictionary))); + Assert.IsFalse(typeof(IDictionary).GenericIsAssignable(typeof(IList))); + } + } +} diff --git a/src/HDInsight/Microsoft.Hadoop.Avro/Resolvers/AvroDataContractResolver.cs b/src/HDInsight/Microsoft.Hadoop.Avro/Resolvers/AvroDataContractResolver.cs index 0d766acca5b45..67815a6a9b0c2 100644 --- a/src/HDInsight/Microsoft.Hadoop.Avro/Resolvers/AvroDataContractResolver.cs +++ b/src/HDInsight/Microsoft.Hadoop.Avro/Resolvers/AvroDataContractResolver.cs @@ -107,7 +107,7 @@ public override TypeSerializationInfo ResolveType(Type type) { return new TypeSerializationInfo { - Name = StripAvroNonCompatibleCharacters(type.Name), + Name = StripAvroNonCompatibleCharacters(type.AvroSchemaName()), Namespace = StripAvroNonCompatibleCharacters(type.Namespace), Nullable = canContainNull }; @@ -123,7 +123,7 @@ public override TypeSerializationInfo ResolveType(Type type) string.Format(CultureInfo.InvariantCulture, "Type '{0}' is not supported by the resolver.", type)); } - var name = StripAvroNonCompatibleCharacters(dataContract.Name ?? type.Name); + var name = StripAvroNonCompatibleCharacters(dataContract.Name ?? type.AvroSchemaName()); var nspace = StripAvroNonCompatibleCharacters(dataContract.Namespace ?? type.Namespace); return new TypeSerializationInfo { diff --git a/src/HDInsight/Microsoft.Hadoop.Avro/Serializers/ListSerializer.cs b/src/HDInsight/Microsoft.Hadoop.Avro/Serializers/ListSerializer.cs index ed36a647c0a36..16cc0862ec0e2 100644 --- a/src/HDInsight/Microsoft.Hadoop.Avro/Serializers/ListSerializer.cs +++ b/src/HDInsight/Microsoft.Hadoop.Avro/Serializers/ListSerializer.cs @@ -102,7 +102,7 @@ protected override Expression BuildDeserializerSafe(Expression decoder) Expression.Block( Expression.Assign(currentNumberOfElements, Expression.Call(decoder, "DecodeArrayChunk", new Type[] { })), Expression.IfThen(Expression.Equal(currentNumberOfElements, Expression.Constant(0)), Expression.Break(allRead)), - Expression.Assign(Expression.Property(result, "Capacity"), Expression.Add(index, currentNumberOfElements)), + Expression.Assign((type.GetProperty("Capacity") != null) ? Expression.Property(result, "Capacity") : (Expression)counter, Expression.Add(index, currentNumberOfElements)), Expression.Assign(counter, Expression.Constant(0)), Expression.Loop( Expression.Block( diff --git a/src/HDInsight/Microsoft.Hadoop.Avro/Serializers/UnionSerializer.cs b/src/HDInsight/Microsoft.Hadoop.Avro/Serializers/UnionSerializer.cs index b98a5dcd78b18..d8445cde5dbe1 100644 --- a/src/HDInsight/Microsoft.Hadoop.Avro/Serializers/UnionSerializer.cs +++ b/src/HDInsight/Microsoft.Hadoop.Avro/Serializers/UnionSerializer.cs @@ -134,7 +134,8 @@ private Expression BuildNullableSerializer(Expression encoder, Expression value, private Expression BuildUnionSerializer(Expression encoder, Expression value, IList schemas) { - Expression elseBranch = Expression.Throw(Expression.Constant(new SerializationException(string.Format(CultureInfo.InvariantCulture, "Object type does match any item schema of the union.")))); + Expression> messageFunc = objectType => new SerializationException(string.Format("Object type {0} does not match any item schema of the union: {1}", objectType == null ? null : objectType.GetType(), string.Join(",", schemas.Select(s => s.Schema.RuntimeType)))); + Expression elseBranch = Expression.Throw(Expression.Invoke(messageFunc, value)); ConditionalExpression conditions = null; for (int i = schemas.Count - 1; i >= 0; i--) { diff --git a/src/HDInsight/Microsoft.Hadoop.Avro/TypeExtensions.cs b/src/HDInsight/Microsoft.Hadoop.Avro/TypeExtensions.cs index 3a5bd6ba4d7b1..ae75bdbf018fe 100644 --- a/src/HDInsight/Microsoft.Hadoop.Avro/TypeExtensions.cs +++ b/src/HDInsight/Microsoft.Hadoop.Avro/TypeExtensions.cs @@ -245,14 +245,14 @@ public static bool CanBeKnownTypeOf(this Type type, Type baseType) && (type.IsSubclassOf(baseType) || type == baseType || (baseType.IsInterface && baseType.IsAssignableFrom(type)) - || (baseType.IsGenericType && baseType.IsInterface && baseType.GenericIsAssignable(baseType) + || (baseType.IsGenericType && baseType.IsInterface && baseType.GenericIsAssignable(type) && type.GetGenericArguments() .Zip(baseType.GetGenericArguments(), (type1, type2) => new Tuple(type1, type2)) .ToList() .TrueForAll(tuple => CanBeKnownTypeOf(tuple.Item1, tuple.Item2)))); } - private static bool GenericIsAssignable(this Type type, Type instanceType) + internal static bool GenericIsAssignable(this Type type, Type instanceType) { if (!type.IsGenericType || !instanceType.IsGenericType) { @@ -260,7 +260,9 @@ private static bool GenericIsAssignable(this Type type, Type instanceType) } var args = type.GetGenericArguments(); - return args.Any() && type.IsAssignableFrom(instanceType.GetGenericTypeDefinition().MakeGenericType(args)); + var typeDefinition = instanceType.GetGenericTypeDefinition(); + var args2 = typeDefinition.GetGenericArguments(); + return args.Any() && args.Length == args2.Length && type.IsAssignableFrom(typeDefinition.MakeGenericType(args)); } public static IEnumerable GetAllKnownTypes(this Type t) @@ -321,5 +323,34 @@ public static IList RemoveDuplicates(IEnumerable pro return result; } + + /// + /// According to Avro, name must: + /// start with [A-Za-z_] + /// subsequently contain only [A-Za-z0-9_] + /// http://avro.apache.org/docs/current/spec.html#schema_record. + /// + /// + /// The entity type. + /// + /// + /// The type name that comply with avro spec. + /// + internal static string AvroSchemaName(this Type type) + { + string result = type.Name; + if (type.IsGenericType) + { + result = type.Name + "_" + string.Join("_", type.GetGenericArguments().Select(AvroSchemaName)); + } + + if (type.IsArray) + { + Type elementType = type.GetElementType(); + result = elementType.AvroSchemaName() + "__"; + } + + return result.Replace("`1", string.Empty).Replace("`2", string.Empty).Replace("`3", string.Empty); + } } }