diff --git a/src/csharp/Microsoft.Spark/Sql/PicklingUdfWrapper.cs b/src/csharp/Microsoft.Spark/Sql/PicklingUdfWrapper.cs index af985f263..5c889991b 100644 --- a/src/csharp/Microsoft.Spark/Sql/PicklingUdfWrapper.cs +++ b/src/csharp/Microsoft.Spark/Sql/PicklingUdfWrapper.cs @@ -3,7 +3,7 @@ // See the LICENSE file in the project root for more information. using System; -using Microsoft.Spark.Utils; +using static Microsoft.Spark.Utils.TypeConverter; namespace Microsoft.Spark.Sql { @@ -35,6 +35,9 @@ internal object Execute(int splitIndex, object[] input, int[] argOffsets) [UdfWrapper] internal class PicklingUdfWrapper { + [NonSerialized] + private bool? _sameT = null; + private readonly Func _func; internal PicklingUdfWrapper(Func func) @@ -44,7 +47,8 @@ internal PicklingUdfWrapper(Func func) internal object Execute(int splitIndex, object[] input, int[] argOffsets) { - return _func(TypeConverter.ConvertTo(input[argOffsets[0]])); + object param = input[argOffsets[0]]; + return _func((_sameT ??= param is T) ? (T)param : ConvertTo(param)); } } @@ -57,6 +61,9 @@ internal object Execute(int splitIndex, object[] input, int[] argOffsets) [UdfWrapper] internal class PicklingUdfWrapper { + [NonSerialized] + private readonly bool?[] _sameT = new bool?[2]; + private readonly Func _func; internal PicklingUdfWrapper(Func func) @@ -66,9 +73,11 @@ internal PicklingUdfWrapper(Func func) internal object Execute(int splitIndex, object[] input, int[] argOffsets) { + object param1 = input[argOffsets[0]]; + object param2 = input[argOffsets[1]]; return _func( - TypeConverter.ConvertTo(input[argOffsets[0]]), - TypeConverter.ConvertTo(input[argOffsets[1]])); + (_sameT[0] ??= param1 is T1) ? (T1)param1 : ConvertTo(param1), + (_sameT[1] ??= param2 is T2) ? (T2)param2 : ConvertTo(param2)); } } @@ -82,6 +91,9 @@ internal object Execute(int splitIndex, object[] input, int[] argOffsets) [UdfWrapper] internal class PicklingUdfWrapper { + [NonSerialized] + private readonly bool?[] _sameT = new bool?[3]; + private readonly Func _func; internal PicklingUdfWrapper(Func func) @@ -91,10 +103,13 @@ internal PicklingUdfWrapper(Func func) internal object Execute(int splitIndex, object[] input, int[] argOffsets) { + object param1 = input[argOffsets[0]]; + object param2 = input[argOffsets[1]]; + object param3 = input[argOffsets[2]]; return _func( - TypeConverter.ConvertTo(input[argOffsets[0]]), - TypeConverter.ConvertTo(input[argOffsets[1]]), - TypeConverter.ConvertTo(input[argOffsets[2]])); + (_sameT[0] ??= param1 is T1) ? (T1)param1 : ConvertTo(param1), + (_sameT[1] ??= param2 is T2) ? (T2)param2 : ConvertTo(param2), + (_sameT[2] ??= param3 is T3) ? (T3)param3 : ConvertTo(param3)); } } @@ -109,6 +124,9 @@ internal object Execute(int splitIndex, object[] input, int[] argOffsets) [UdfWrapper] internal class PicklingUdfWrapper { + [NonSerialized] + private readonly bool?[] _sameT = new bool?[4]; + private readonly Func _func; internal PicklingUdfWrapper(Func func) @@ -118,11 +136,15 @@ internal PicklingUdfWrapper(Func func) internal object Execute(int splitIndex, object[] input, int[] argOffsets) { + object param1 = input[argOffsets[0]]; + object param2 = input[argOffsets[1]]; + object param3 = input[argOffsets[2]]; + object param4 = input[argOffsets[3]]; return _func( - TypeConverter.ConvertTo(input[argOffsets[0]]), - TypeConverter.ConvertTo(input[argOffsets[1]]), - TypeConverter.ConvertTo(input[argOffsets[2]]), - TypeConverter.ConvertTo(input[argOffsets[3]])); + (_sameT[0] ??= param1 is T1) ? (T1)param1 : ConvertTo(param1), + (_sameT[1] ??= param2 is T2) ? (T2)param2 : ConvertTo(param2), + (_sameT[2] ??= param3 is T3) ? (T3)param3 : ConvertTo(param3), + (_sameT[3] ??= param4 is T4) ? (T4)param4 : ConvertTo(param4)); } } @@ -138,6 +160,9 @@ internal object Execute(int splitIndex, object[] input, int[] argOffsets) [UdfWrapper] internal class PicklingUdfWrapper { + [NonSerialized] + private readonly bool?[] _sameT = new bool?[5]; + private readonly Func _func; internal PicklingUdfWrapper(Func func) @@ -147,12 +172,17 @@ internal PicklingUdfWrapper(Func func) internal object Execute(int splitIndex, object[] input, int[] argOffsets) { + object param1 = input[argOffsets[0]]; + object param2 = input[argOffsets[1]]; + object param3 = input[argOffsets[2]]; + object param4 = input[argOffsets[3]]; + object param5 = input[argOffsets[4]]; return _func( - TypeConverter.ConvertTo(input[argOffsets[0]]), - TypeConverter.ConvertTo(input[argOffsets[1]]), - TypeConverter.ConvertTo(input[argOffsets[2]]), - TypeConverter.ConvertTo(input[argOffsets[3]]), - TypeConverter.ConvertTo(input[argOffsets[4]])); + (_sameT[0] ??= param1 is T1) ? (T1)param1 : ConvertTo(param1), + (_sameT[1] ??= param2 is T2) ? (T2)param2 : ConvertTo(param2), + (_sameT[2] ??= param3 is T3) ? (T3)param3 : ConvertTo(param3), + (_sameT[3] ??= param4 is T4) ? (T4)param4 : ConvertTo(param4), + (_sameT[4] ??= param5 is T5) ? (T5)param5 : ConvertTo(param5)); } } @@ -169,6 +199,9 @@ internal object Execute(int splitIndex, object[] input, int[] argOffsets) [UdfWrapper] internal class PicklingUdfWrapper { + [NonSerialized] + private readonly bool?[] _sameT = new bool?[6]; + private readonly Func _func; internal PicklingUdfWrapper(Func func) @@ -178,13 +211,19 @@ internal PicklingUdfWrapper(Func func) internal object Execute(int splitIndex, object[] input, int[] argOffsets) { + object param1 = input[argOffsets[0]]; + object param2 = input[argOffsets[1]]; + object param3 = input[argOffsets[2]]; + object param4 = input[argOffsets[3]]; + object param5 = input[argOffsets[4]]; + object param6 = input[argOffsets[5]]; return _func( - TypeConverter.ConvertTo(input[argOffsets[0]]), - TypeConverter.ConvertTo(input[argOffsets[1]]), - TypeConverter.ConvertTo(input[argOffsets[2]]), - TypeConverter.ConvertTo(input[argOffsets[3]]), - TypeConverter.ConvertTo(input[argOffsets[4]]), - TypeConverter.ConvertTo(input[argOffsets[5]])); + (_sameT[0] ??= param1 is T1) ? (T1)param1 : ConvertTo(param1), + (_sameT[1] ??= param2 is T2) ? (T2)param2 : ConvertTo(param2), + (_sameT[2] ??= param3 is T3) ? (T3)param3 : ConvertTo(param3), + (_sameT[3] ??= param4 is T4) ? (T4)param4 : ConvertTo(param4), + (_sameT[4] ??= param5 is T5) ? (T5)param5 : ConvertTo(param5), + (_sameT[5] ??= param6 is T6) ? (T6)param6 : ConvertTo(param6)); } } @@ -202,6 +241,9 @@ internal object Execute(int splitIndex, object[] input, int[] argOffsets) [UdfWrapper] internal class PicklingUdfWrapper { + [NonSerialized] + private readonly bool?[] _sameT = new bool?[7]; + private readonly Func _func; internal PicklingUdfWrapper(Func func) @@ -211,14 +253,21 @@ internal PicklingUdfWrapper(Func func) internal object Execute(int splitIndex, object[] input, int[] argOffsets) { + object param1 = input[argOffsets[0]]; + object param2 = input[argOffsets[1]]; + object param3 = input[argOffsets[2]]; + object param4 = input[argOffsets[3]]; + object param5 = input[argOffsets[4]]; + object param6 = input[argOffsets[5]]; + object param7 = input[argOffsets[6]]; return _func( - TypeConverter.ConvertTo(input[argOffsets[0]]), - TypeConverter.ConvertTo(input[argOffsets[1]]), - TypeConverter.ConvertTo(input[argOffsets[2]]), - TypeConverter.ConvertTo(input[argOffsets[3]]), - TypeConverter.ConvertTo(input[argOffsets[4]]), - TypeConverter.ConvertTo(input[argOffsets[5]]), - TypeConverter.ConvertTo(input[argOffsets[6]])); + (_sameT[0] ??= param1 is T1) ? (T1)param1 : ConvertTo(param1), + (_sameT[1] ??= param2 is T2) ? (T2)param2 : ConvertTo(param2), + (_sameT[2] ??= param3 is T3) ? (T3)param3 : ConvertTo(param3), + (_sameT[3] ??= param4 is T4) ? (T4)param4 : ConvertTo(param4), + (_sameT[4] ??= param5 is T5) ? (T5)param5 : ConvertTo(param5), + (_sameT[5] ??= param6 is T6) ? (T6)param6 : ConvertTo(param6), + (_sameT[6] ??= param7 is T7) ? (T7)param7 : ConvertTo(param7)); } } @@ -237,6 +286,9 @@ internal object Execute(int splitIndex, object[] input, int[] argOffsets) [UdfWrapper] internal class PicklingUdfWrapper { + [NonSerialized] + private readonly bool?[] _sameT = new bool?[8]; + private readonly Func _func; internal PicklingUdfWrapper(Func func) @@ -246,15 +298,23 @@ internal PicklingUdfWrapper(Func func) internal object Execute(int splitIndex, object[] input, int[] argOffsets) { + object param1 = input[argOffsets[0]]; + object param2 = input[argOffsets[1]]; + object param3 = input[argOffsets[2]]; + object param4 = input[argOffsets[3]]; + object param5 = input[argOffsets[4]]; + object param6 = input[argOffsets[5]]; + object param7 = input[argOffsets[6]]; + object param8 = input[argOffsets[7]]; return _func( - TypeConverter.ConvertTo(input[argOffsets[0]]), - TypeConverter.ConvertTo(input[argOffsets[1]]), - TypeConverter.ConvertTo(input[argOffsets[2]]), - TypeConverter.ConvertTo(input[argOffsets[3]]), - TypeConverter.ConvertTo(input[argOffsets[4]]), - TypeConverter.ConvertTo(input[argOffsets[5]]), - TypeConverter.ConvertTo(input[argOffsets[6]]), - TypeConverter.ConvertTo(input[argOffsets[7]])); + (_sameT[0] ??= param1 is T1) ? (T1)param1 : ConvertTo(param1), + (_sameT[1] ??= param2 is T2) ? (T2)param2 : ConvertTo(param2), + (_sameT[2] ??= param3 is T3) ? (T3)param3 : ConvertTo(param3), + (_sameT[3] ??= param4 is T4) ? (T4)param4 : ConvertTo(param4), + (_sameT[4] ??= param5 is T5) ? (T5)param5 : ConvertTo(param5), + (_sameT[5] ??= param6 is T6) ? (T6)param6 : ConvertTo(param6), + (_sameT[6] ??= param7 is T7) ? (T7)param7 : ConvertTo(param7), + (_sameT[7] ??= param8 is T8) ? (T8)param8 : ConvertTo(param8)); } } @@ -274,6 +334,9 @@ internal object Execute(int splitIndex, object[] input, int[] argOffsets) [UdfWrapper] internal class PicklingUdfWrapper { + [NonSerialized] + private readonly bool?[] _sameT = new bool?[9]; + private readonly Func _func; internal PicklingUdfWrapper(Func func) @@ -282,16 +345,25 @@ internal PicklingUdfWrapper(Func fu } internal object Execute(int splitIndex, object[] input, int[] argOffsets) { + object param1 = input[argOffsets[0]]; + object param2 = input[argOffsets[1]]; + object param3 = input[argOffsets[2]]; + object param4 = input[argOffsets[3]]; + object param5 = input[argOffsets[4]]; + object param6 = input[argOffsets[5]]; + object param7 = input[argOffsets[6]]; + object param8 = input[argOffsets[7]]; + object param9 = input[argOffsets[8]]; return _func( - TypeConverter.ConvertTo(input[argOffsets[0]]), - TypeConverter.ConvertTo(input[argOffsets[1]]), - TypeConverter.ConvertTo(input[argOffsets[2]]), - TypeConverter.ConvertTo(input[argOffsets[3]]), - TypeConverter.ConvertTo(input[argOffsets[4]]), - TypeConverter.ConvertTo(input[argOffsets[5]]), - TypeConverter.ConvertTo(input[argOffsets[6]]), - TypeConverter.ConvertTo(input[argOffsets[7]]), - TypeConverter.ConvertTo(input[argOffsets[8]])); + (_sameT[0] ??= param1 is T1) ? (T1)param1 : ConvertTo(param1), + (_sameT[1] ??= param2 is T2) ? (T2)param2 : ConvertTo(param2), + (_sameT[2] ??= param3 is T3) ? (T3)param3 : ConvertTo(param3), + (_sameT[3] ??= param4 is T4) ? (T4)param4 : ConvertTo(param4), + (_sameT[4] ??= param5 is T5) ? (T5)param5 : ConvertTo(param5), + (_sameT[5] ??= param6 is T6) ? (T6)param6 : ConvertTo(param6), + (_sameT[6] ??= param7 is T7) ? (T7)param7 : ConvertTo(param7), + (_sameT[7] ??= param8 is T8) ? (T8)param8 : ConvertTo(param8), + (_sameT[8] ??= param9 is T9) ? (T9)param9 : ConvertTo(param9)); } } @@ -312,6 +384,9 @@ internal object Execute(int splitIndex, object[] input, int[] argOffsets) [UdfWrapper] internal class PicklingUdfWrapper { + [NonSerialized] + private readonly bool?[] _sameT = new bool?[10]; + private readonly Func _func; internal PicklingUdfWrapper(Func func) @@ -321,17 +396,27 @@ internal PicklingUdfWrapper(Func(input[argOffsets[0]]), - TypeConverter.ConvertTo(input[argOffsets[1]]), - TypeConverter.ConvertTo(input[argOffsets[2]]), - TypeConverter.ConvertTo(input[argOffsets[3]]), - TypeConverter.ConvertTo(input[argOffsets[4]]), - TypeConverter.ConvertTo(input[argOffsets[5]]), - TypeConverter.ConvertTo(input[argOffsets[6]]), - TypeConverter.ConvertTo(input[argOffsets[7]]), - TypeConverter.ConvertTo(input[argOffsets[8]]), - TypeConverter.ConvertTo(input[argOffsets[9]])); + (_sameT[0] ??= param1 is T1) ? (T1)param1 : ConvertTo(param1), + (_sameT[1] ??= param2 is T2) ? (T2)param2 : ConvertTo(param2), + (_sameT[2] ??= param3 is T3) ? (T3)param3 : ConvertTo(param3), + (_sameT[3] ??= param4 is T4) ? (T4)param4 : ConvertTo(param4), + (_sameT[4] ??= param5 is T5) ? (T5)param5 : ConvertTo(param5), + (_sameT[5] ??= param6 is T6) ? (T6)param6 : ConvertTo(param6), + (_sameT[6] ??= param7 is T7) ? (T7)param7 : ConvertTo(param7), + (_sameT[7] ??= param8 is T8) ? (T8)param8 : ConvertTo(param8), + (_sameT[8] ??= param9 is T9) ? (T9)param9 : ConvertTo(param9), + (_sameT[9] ??= param10 is T10) ? (T10)param10 : ConvertTo(param10)); } } } diff --git a/src/csharp/Microsoft.Spark/Utils/TypeConverter.cs b/src/csharp/Microsoft.Spark/Utils/TypeConverter.cs index 6bb2ff77c..6c00c79dd 100644 --- a/src/csharp/Microsoft.Spark/Utils/TypeConverter.cs +++ b/src/csharp/Microsoft.Spark/Utils/TypeConverter.cs @@ -20,7 +20,7 @@ internal static class TypeConverter /// Type to convert to /// The object to convert /// Converted object. - internal static T ConvertTo(object obj) => obj is T t ? t : (T)Convert(obj, typeof(T)); + internal static T ConvertTo(object obj) => (T)Convert(obj, typeof(T)); private static object Convert(object obj, Type toType) { diff --git a/src/csharp/Microsoft.Spark/Utils/UdfSerDe.cs b/src/csharp/Microsoft.Spark/Utils/UdfSerDe.cs index d338ddbdb..69838b25f 100644 --- a/src/csharp/Microsoft.Spark/Utils/UdfSerDe.cs +++ b/src/csharp/Microsoft.Spark/Utils/UdfSerDe.cs @@ -201,12 +201,15 @@ private static TargetData SerializeTarget(object target) BindingFlags.Public | BindingFlags.NonPublic)) { - fields.Add(new FieldData() + if (!field.GetCustomAttributes(typeof(NonSerializedAttribute)).Any()) { - TypeData = SerializeType(field.FieldType), - Name = field.Name, - Value = field.GetValue(target) - }); + fields.Add(new FieldData() + { + TypeData = SerializeType(field.FieldType), + Name = field.Name, + Value = field.GetValue(target) + }); + } } // Even when an UDF does not have any closure, GetFields() returns some fields