fix avro issue to support generic and collection/List/Array type bett…

…er and also improved error message for union serializer.

src/HDInsight/Microsoft.Hadoop.Avro.Tests/AvroSerializerTests.cs

@@ -12,11 +12,13 @@
 // 
 // See the Apache Version 2.0 License for specific language governing
 // permissions and limitations under the License.
+
 namespace Microsoft.Hadoop.Avro.Tests
 {
     using System;
     using System.Collections.Concurrent;
     using System.Collections.Generic;
+    using System.Collections.ObjectModel;
     using System.Diagnostics.CodeAnalysis;
     using System.IO;
     using System.Linq;
@@ -240,10 +242,26 @@ public void Serializer_SerializeListClass()
         [TestCategory("CheckIn")]
         public void Serializer_SerializeIList()
         {
-            var knownTypes = new[] { typeof(List<Guid>) };
+            var knownTypes = new[] { typeof(List<Guid>), typeof(List<int>) };
             RoundTripSerializationWithCheck(IListClass.Create(), new AvroSerializerSettings { KnownTypes = knownTypes });
         }
 
+        [TestMethod]
+        [TestCategory("CheckIn")]
+        public void Serializer_SerializeIListWithArray()
+        {
+            var knownTypes = new[] { typeof(Guid[]), typeof(List<int>) };
+            RoundTripSerializationWithCheck(IListClass.CreateWithArray(), new AvroSerializerSettings { KnownTypes = knownTypes });
+        }
+
+        [TestMethod]
+        [TestCategory("CheckIn")]
+        public void Serializer_SerializeIListWithCollection()
+        {
+            var knownTypes = new[] { typeof(Collection<Guid>), typeof(Collection<int>) };
+            RoundTripSerializationWithCheck(IListClass.CreateWithCollection(), new AvroSerializerSettings { KnownTypes = knownTypes });
+        }
+
         [TestMethod]
         [TestCategory("CheckIn")]
         public void Serializer_SerializeInheritedList()

src/HDInsight/Microsoft.Hadoop.Avro.Tests/Microsoft.Hadoop.Avro.Tests.csproj

@@ -124,6 +124,7 @@
     <Compile Include="TestClasses\TestClassesCollections.cs" />
     <Compile Include="TestClasses\TestClassesForKnownTypes.cs" />
     <Compile Include="TestClasses\TestClassesForSurrogates.cs" />
+    <Compile Include="TypeExtensionsTest.cs" />
     <Compile Include="Utilities.cs" />
     <Compile Include="PerformanceTests.cs" />
     <Compile Include="SequentialContainerTests.cs" />

src/HDInsight/Microsoft.Hadoop.Avro.Tests/TestClasses/TestClassesCollections.cs

@@ -12,10 +12,12 @@
 // 
 // See the Apache Version 2.0 License for specific language governing
 // permissions and limitations under the License.
+
 namespace Microsoft.Hadoop.Avro.Tests
 {
     using System;
     using System.Collections.Generic;
+    using System.Collections.ObjectModel;
     using System.Diagnostics.CodeAnalysis;
     using System.Linq;
     using System.Runtime.Serialization;
@@ -58,13 +60,27 @@ internal class IListClass : IEquatable<IListClass>
     {
         public static IListClass Create()
         {
-            return new IListClass { Field1 = new List<Guid> { Guid.NewGuid(), Guid.NewGuid(), Guid.NewGuid() } };
+            return new IListClass { Field1 = new List<Guid> { Guid.NewGuid() }, Field2 = new List<int> { 117 } };
+        }
+
+        public static IListClass CreateWithArray()
+        {
+            return new IListClass { Field1 = new[] { Guid.NewGuid(), Guid.NewGuid() }, Field2 = new List<int> { 12 } };
+        }
+
+        public static IListClass CreateWithCollection()
+        {
+            return new IListClass { Field1 = new Collection<Guid>() { Guid.NewGuid(), Guid.NewGuid() }, Field2 = new Collection<int>() { 12 } };
         }
 
         [ProtoMember(1)]
         [DataMember]
         public IList<Guid> Field1 { get; set; }
 
+        [ProtoMember(1)]
+        [DataMember]
+        public IList<int> Field2 { get; set; }
+
         public bool Equals(IListClass other)
         {
             if (other == null)

src/HDInsight/Microsoft.Hadoop.Avro.Tests/TypeExtensionsTest.cs

@@ -0,0 +1,47 @@
+// Copyright (c) Microsoft Corporation
+// All rights reserved.
+// 
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License.  You may obtain a copy
+// of the License at http://www.apache.org/licenses/LICENSE-2.0
+// 
+// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
+// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABLITY OR NON-INFRINGEMENT.
+// 
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+namespace Microsoft.Hadoop.Avro.Tests
+{
+    using System;
+    using System.Collections.Generic;
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+    [TestClass]
+    public class TypeExtensionsTest
+    {
+        [TestMethod]
+        public void CanBeKnownTypeOfTest()
+        {
+            Assert.IsFalse(typeof(IEnumerable<int>).CanBeKnownTypeOf(typeof(IEnumerable<string>)));
+            Assert.IsFalse(typeof(IEnumerable<int>).CanBeKnownTypeOf(typeof(IEnumerable<Guid>)));
+            Assert.IsFalse(typeof(IEnumerable<int>).CanBeKnownTypeOf(typeof(IEnumerable<IListClass>)));
+            Assert.IsFalse(typeof(IList<Guid>).CanBeKnownTypeOf(typeof(int[])));
+            Assert.IsFalse(typeof(int).CanBeKnownTypeOf(typeof(Guid)));
+            Assert.IsFalse(typeof(IList<int>).CanBeKnownTypeOf(typeof(IList<Guid>)));
+
+            Assert.IsFalse(typeof(int[]).CanBeKnownTypeOf(typeof(IList<IListClass>)));
+            Assert.IsFalse(typeof(int[]).CanBeKnownTypeOf(typeof(IList<Guid>)));
+        }
+
+        [TestMethod]
+        public void GenericIsAssignableTest()
+        {
+            Assert.IsFalse(typeof(int[]).GenericIsAssignable(typeof(IDictionary<int, Guid>)));
+            Assert.IsFalse(typeof(IList<int>).GenericIsAssignable(typeof(IDictionary<int, Guid>)));
+            Assert.IsFalse(typeof(IDictionary<int, Guid>).GenericIsAssignable(typeof(IList<int>)));
+        }
+    }
+}

src/HDInsight/Microsoft.Hadoop.Avro/Resolvers/AvroDataContractResolver.cs

@@ -107,7 +107,7 @@ public override TypeSerializationInfo ResolveType(Type type)
             {
                 return new TypeSerializationInfo
                 {
-                    Name = StripAvroNonCompatibleCharacters(type.Name),
+                    Name = StripAvroNonCompatibleCharacters(type.AvroSchemaName()),
                     Namespace = StripAvroNonCompatibleCharacters(type.Namespace),
                     Nullable = canContainNull
                 };
@@ -123,7 +123,7 @@ public override TypeSerializationInfo ResolveType(Type type)
                     string.Format(CultureInfo.InvariantCulture, "Type '{0}' is not supported by the resolver.", type));
             }
 
-            var name = StripAvroNonCompatibleCharacters(dataContract.Name ?? type.Name);
+            var name = StripAvroNonCompatibleCharacters(dataContract.Name ?? type.AvroSchemaName());
             var nspace = StripAvroNonCompatibleCharacters(dataContract.Namespace ?? type.Namespace);
             return new TypeSerializationInfo
             {

src/HDInsight/Microsoft.Hadoop.Avro/Serializers/ListSerializer.cs

@@ -102,7 +102,7 @@ protected override Expression BuildDeserializerSafe(Expression decoder)
                     Expression.Block(
                         Expression.Assign(currentNumberOfElements, Expression.Call(decoder, "DecodeArrayChunk", new Type[] { })),
                         Expression.IfThen(Expression.Equal(currentNumberOfElements, Expression.Constant(0)), Expression.Break(allRead)),
-                        Expression.Assign(Expression.Property(result, "Capacity"), Expression.Add(index, currentNumberOfElements)),
+                        Expression.Assign((type.GetProperty("Capacity") != null) ? Expression.Property(result, "Capacity") : (Expression)counter, Expression.Add(index, currentNumberOfElements)),
                         Expression.Assign(counter, Expression.Constant(0)),
                         Expression.Loop(
                             Expression.Block(

src/HDInsight/Microsoft.Hadoop.Avro/Serializers/UnionSerializer.cs

@@ -134,7 +134,8 @@ private Expression BuildNullableSerializer(Expression encoder, Expression value,
 
         private Expression BuildUnionSerializer(Expression encoder, Expression value, IList<IndexedSchema> schemas)
         {
-            Expression elseBranch = Expression.Throw(Expression.Constant(new SerializationException(string.Format(CultureInfo.InvariantCulture, "Object type does match any item schema of the union."))));
+            Expression<Func<object, Exception>> messageFunc = objectType => new SerializationException(string.Format("Object type {0} does not match any item schema of the union: {1}", objectType == null ? null : objectType.GetType(), string.Join(",", schemas.Select(s => s.Schema.RuntimeType))));
+            Expression elseBranch = Expression.Throw(Expression.Invoke(messageFunc, value));
             ConditionalExpression conditions = null;
             for (int i = schemas.Count - 1; i >= 0; i--)
             {

src/HDInsight/Microsoft.Hadoop.Avro/TypeExtensions.cs

@@ -245,22 +245,24 @@ public static bool CanBeKnownTypeOf(this Type type, Type baseType)
                    && (type.IsSubclassOf(baseType) 
                    || type == baseType 
                    || (baseType.IsInterface && baseType.IsAssignableFrom(type))
-                   || (baseType.IsGenericType && baseType.IsInterface && baseType.GenericIsAssignable(baseType)
+                   || (baseType.IsGenericType && baseType.IsInterface && baseType.GenericIsAssignable(type)
                            && type.GetGenericArguments()
                                   .Zip(baseType.GetGenericArguments(), (type1, type2) => new Tuple<Type, Type>(type1, type2))
                                   .ToList()
                                   .TrueForAll(tuple => CanBeKnownTypeOf(tuple.Item1, tuple.Item2))));
         }
 
-        private static bool GenericIsAssignable(this Type type, Type instanceType)
+        internal static bool GenericIsAssignable(this Type type, Type instanceType)
         {
             if (!type.IsGenericType || !instanceType.IsGenericType)
             {
                 return false;
             }
 
             var args = type.GetGenericArguments();
-            return args.Any() && type.IsAssignableFrom(instanceType.GetGenericTypeDefinition().MakeGenericType(args));
+            var typeDefinition = instanceType.GetGenericTypeDefinition();
+            var args2 = typeDefinition.GetGenericArguments();
+            return args.Any() && args.Length == args2.Length && type.IsAssignableFrom(typeDefinition.MakeGenericType(args));
         }
 
         public static IEnumerable<Type> GetAllKnownTypes(this Type t)
@@ -321,5 +323,34 @@ public static IList<PropertyInfo> RemoveDuplicates(IEnumerable<PropertyInfo> pro
 
             return result;
         }
+
+        /// <summary>
+        /// According to Avro, name must:
+        ///     start with [A-Za-z_] 
+        ///     subsequently contain only [A-Za-z0-9_] 
+        /// http://avro.apache.org/docs/current/spec.html#schema_record.
+        /// </summary>
+        /// <param name="type">
+        /// The entity type.
+        /// </param>
+        /// <returns>
+        /// The type name that comply with avro spec.
+        /// </returns>
+        internal static string AvroSchemaName(this Type type)
+        {
+            string result = type.Name;
+            if (type.IsGenericType)
+            {
+                result = type.Name + "_" + string.Join("_", type.GetGenericArguments().Select(AvroSchemaName));
+            }
+
+            if (type.IsArray)
+            {
+                Type elementType = type.GetElementType();
+                result = elementType.AvroSchemaName() + "__";
+            }
+
+            return result.Replace("`1", string.Empty).Replace("`2", string.Empty).Replace("`3", string.Empty);
+        }
     }
 }