Replace DV data type system with .NET standard type system.

build/Dependencies.props

@@ -13,7 +13,7 @@
     <BenchmarkDotNetVersion>0.11.1</BenchmarkDotNetVersion>
     <TensorFlowVersion>1.10.0</TensorFlowVersion>
     <SystemCollectionsImmutableVersion>1.5.0</SystemCollectionsImmutableVersion>
-
+    <SystemMemoryVersion>4.5.1</SystemMemoryVersion>
     <MicrosoftCodeAnalysisCSharpVersion>2.9.0</MicrosoftCodeAnalysisCSharpVersion>
     <MicrosoftCSharpVersion>4.5.0</MicrosoftCSharpVersion>
     <SystemCompositionVersion>1.2.0</SystemCompositionVersion>

pkg/Microsoft.ML/Microsoft.ML.nupkgproj

@@ -12,6 +12,7 @@
     <PackageReference Include="System.Reflection.Emit.Lightweight" Version="$(SystemReflectionEmitLightweightPackageVersion)" />
     <PackageReference Include="System.Threading.Tasks.Dataflow" Version="$(SystemThreadingTasksDataflowPackageVersion)" />
     <PackageReference Include="System.CodeDom" Version="$(SystemCodeDomPackageVersion)" />
+    <PackageReference Include="System.Memory" Version="$(SystemMemoryVersion)" />
   </ItemGroup>
 
   <ItemGroup>

src/Microsoft.ML.Api/ApiUtils.cs

@@ -19,11 +19,10 @@ private static OpCode GetAssignmentOpCode(Type t)
         {
             // REVIEW: This should be a Dictionary<Type, OpCode> based solution.
             // DvTypes, strings, arrays, all nullable types, VBuffers and UInt128.
-            if (t == typeof(DvInt8) || t == typeof(DvInt4) || t == typeof(DvInt2) || t == typeof(DvInt1) ||
-                t == typeof(DvBool) || t == typeof(DvText) || t == typeof(string) || t.IsArray ||
+            if (t == typeof(ReadOnlyMemory<char>) || t == typeof(string) || t.IsArray ||
                 (t.IsGenericType && t.GetGenericTypeDefinition() == typeof(VBuffer<>)) ||
                 (t.IsGenericType && t.GetGenericTypeDefinition() == typeof(Nullable<>)) ||
-                t == typeof(DvDateTime) || t == typeof(DvDateTimeZone) || t == typeof(DvTimeSpan) || t == typeof(UInt128))
+                t == typeof(DateTime) || t == typeof(DateTimeOffset) || t == typeof(TimeSpan) || t == typeof(UInt128))
             {
                 return OpCodes.Stobj;
             }

src/Microsoft.ML.Api/DataViewConstructionUtils.cs

@@ -125,61 +125,11 @@ private Delegate CreateGetter(int index)
                     if (outputType.IsArray)
                     {
                         Ch.Assert(colType.IsVector);
-                        // String[] -> VBuffer<DvText>
+                        // String[] -> ReadOnlyMemory<char>
                         if (outputType.GetElementType() == typeof(string))
                         {
                             Ch.Assert(colType.ItemType.IsText);
-                            return CreateConvertingArrayGetterDelegate<String, DvText>(index, x => x == null ? DvText.NA : new DvText(x));
-                        }
-                        else if (outputType.GetElementType() == typeof(int))
-                        {
-                            Ch.Assert(colType.ItemType == NumberType.I4);
-                            return CreateConvertingArrayGetterDelegate<int, DvInt4>(index, x => x);
-                        }
-                        else if (outputType.GetElementType() == typeof(int?))
-                        {
-                            Ch.Assert(colType.ItemType == NumberType.I4);
-                            return CreateConvertingArrayGetterDelegate<int?, DvInt4>(index, x => x ?? DvInt4.NA);
-                        }
-                        else if (outputType.GetElementType() == typeof(long))
-                        {
-                            Ch.Assert(colType.ItemType == NumberType.I8);
-                            return CreateConvertingArrayGetterDelegate<long, DvInt8>(index, x => x);
-                        }
-                        else if (outputType.GetElementType() == typeof(long?))
-                        {
-                            Ch.Assert(colType.ItemType == NumberType.I8);
-                            return CreateConvertingArrayGetterDelegate<long?, DvInt8>(index, x => x ?? DvInt8.NA);
-                        }
-                        else if (outputType.GetElementType() == typeof(short))
-                        {
-                            Ch.Assert(colType.ItemType == NumberType.I2);
-                            return CreateConvertingArrayGetterDelegate<short, DvInt2>(index, x => x);
-                        }
-                        else if (outputType.GetElementType() == typeof(short?))
-                        {
-                            Ch.Assert(colType.ItemType == NumberType.I2);
-                            return CreateConvertingArrayGetterDelegate<short?, DvInt2>(index, x => x ?? DvInt2.NA);
-                        }
-                        else if (outputType.GetElementType() == typeof(sbyte))
-                        {
-                            Ch.Assert(colType.ItemType == NumberType.I1);
-                            return CreateConvertingArrayGetterDelegate<sbyte, DvInt1>(index, x => x);
-                        }
-                        else if (outputType.GetElementType() == typeof(sbyte?))
-                        {
-                            Ch.Assert(colType.ItemType == NumberType.I1);
-                            return CreateConvertingArrayGetterDelegate<sbyte?, DvInt1>(index, x => x ?? DvInt1.NA);
-                        }
-                        else if (outputType.GetElementType() == typeof(bool))
-                        {
-                            Ch.Assert(colType.ItemType.IsBool);
-                            return CreateConvertingArrayGetterDelegate<bool, DvBool>(index, x => x);
-                        }
-                        else if (outputType.GetElementType() == typeof(bool?))
-                        {
-                            Ch.Assert(colType.ItemType.IsBool);
-                            return CreateConvertingArrayGetterDelegate<bool?, DvBool>(index, x => x ?? DvBool.NA);
+                            return CreateConvertingArrayGetterDelegate<string, ReadOnlyMemory<char>>(index, x =>  x != null ? x.AsMemory() : ReadOnlyMemory<char>.Empty);
                         }
 
                         // T[] -> VBuffer<T>
@@ -193,7 +143,7 @@ private Delegate CreateGetter(int index)
                     else if (colType.IsVector)
                     {
                         // VBuffer<T> -> VBuffer<T>
-                        // REVIEW: Do we care about accomodating VBuffer<string> -> VBuffer<DvText>?
+                        // REVIEW: Do we care about accomodating VBuffer<string> -> ReadOnlyMemory<char>?
                         Ch.Assert(outputType.IsGenericType);
                         Ch.Assert(outputType.GetGenericTypeDefinition() == typeof(VBuffer<>));
                         Ch.Assert(outputType.GetGenericArguments()[0] == colType.ItemType.RawType);
@@ -204,70 +154,11 @@ private Delegate CreateGetter(int index)
                     {
                         if (outputType == typeof(string))
                         {
-                            // String -> DvText
+                            // String -> ReadOnlyMemory<char>
                             Ch.Assert(colType.IsText);
-                            return CreateConvertingGetterDelegate<String, DvText>(index, x => x == null ? DvText.NA : new DvText(x));
-                        }
-                        else if (outputType == typeof(bool))
-                        {
-                            // Bool -> DvBool
-                            Ch.Assert(colType.IsBool);
-                            return CreateConvertingGetterDelegate<bool, DvBool>(index, x => x);
-                        }
-                        else if (outputType == typeof(bool?))
-                        {
-                            // Bool? -> DvBool
-                            Ch.Assert(colType.IsBool);
-                            return CreateConvertingGetterDelegate<bool?, DvBool>(index, x => x ?? DvBool.NA);
-                        }
-                        else if (outputType == typeof(int))
-                        {
-                            // int -> DvInt4
-                            Ch.Assert(colType == NumberType.I4);
-                            return CreateConvertingGetterDelegate<int, DvInt4>(index, x => x);
-                        }
-                        else if (outputType == typeof(int?))
-                        {
-                            // int? -> DvInt4
-                            Ch.Assert(colType == NumberType.I4);
-                            return CreateConvertingGetterDelegate<int?, DvInt4>(index, x => x ?? DvInt4.NA);
-                        }
-                        else if (outputType == typeof(short))
-                        {
-                            // short -> DvInt2
-                            Ch.Assert(colType == NumberType.I2);
-                            return CreateConvertingGetterDelegate<short, DvInt2>(index, x => x);
-                        }
-                        else if (outputType == typeof(short?))
-                        {
-                            // short? -> DvInt2
-                            Ch.Assert(colType == NumberType.I2);
-                            return CreateConvertingGetterDelegate<short?, DvInt2>(index, x => x ?? DvInt2.NA);
-                        }
-                        else if (outputType == typeof(long))
-                        {
-                            // long -> DvInt8
-                            Ch.Assert(colType == NumberType.I8);
-                            return CreateConvertingGetterDelegate<long, DvInt8>(index, x => x);
-                        }
-                        else if (outputType == typeof(long?))
-                        {
-                            // long? -> DvInt8
-                            Ch.Assert(colType == NumberType.I8);
-                            return CreateConvertingGetterDelegate<long?, DvInt8>(index, x => x ?? DvInt8.NA);
-                        }
-                        else if (outputType == typeof(sbyte))
-                        {
-                            // sbyte -> DvInt1
-                            Ch.Assert(colType == NumberType.I1);
-                            return CreateConvertingGetterDelegate<sbyte, DvInt1>(index, x => x);
-                        }
-                        else if (outputType == typeof(sbyte?))
-                        {
-                            // sbyte? -> DvInt1
-                            Ch.Assert(colType == NumberType.I1);
-                            return CreateConvertingGetterDelegate<sbyte?, DvInt1>(index, x => x ?? DvInt1.NA);
+                            return CreateConvertingGetterDelegate<String, ReadOnlyMemory<char>>(index, x => x != null ? x.AsMemory() : ReadOnlyMemory<char>.Empty);
                         }
+
                         // T -> T
                         if (outputType.IsGenericType && outputType.GetGenericTypeDefinition() == typeof(Nullable<>))
                             Ch.Assert(colType.RawType == Nullable.GetUnderlyingType(outputType));
@@ -805,12 +696,12 @@ public override ValueGetter<TDst> GetGetter<TDst>()
                 var itemType = typeT.GetElementType();
                 var dstItemType = typeof(TDst).GetGenericArguments()[0];
 
-                // String[] -> VBuffer<DvText>
+                // String[] -> VBuffer<ReadOnlyMemory<char>>
                 if (itemType == typeof(string))
                 {
-                    Contracts.Check(dstItemType == typeof(DvText));
+                    Contracts.Check(dstItemType == typeof(ReadOnlyMemory<char>));
 
-                    ValueGetter<VBuffer<DvText>> method = GetStringArray;
+                    ValueGetter<VBuffer<ReadOnlyMemory<char>>> method = GetStringArray;
                     return method as ValueGetter<TDst>;
                 }
 
@@ -825,7 +716,7 @@ public override ValueGetter<TDst> GetGetter<TDst>()
             if (MetadataType.IsVector)
             {
                 // VBuffer<T> -> VBuffer<T>
-                // REVIEW: Do we care about accomodating VBuffer<string> -> VBuffer<DvText>?
+                // REVIEW: Do we care about accomodating VBuffer<string> -> VBuffer<ReadOnlyMemory<char>>?
 
                 Contracts.Assert(typeT.IsGenericType);
                 Contracts.Check(typeof(TDst).IsGenericType);
@@ -845,9 +736,9 @@ public override ValueGetter<TDst> GetGetter<TDst>()
             {
                 if (typeT == typeof(string))
                 {
-                    // String -> DvText
+                    // String -> ReadOnlyMemory<char>
                     Contracts.Assert(MetadataType.IsText);
-                    ValueGetter<DvText> m = GetString;
+                    ValueGetter<ReadOnlyMemory<char>> m = GetString;
                     return m as ValueGetter<TDst>;
                 }
                 // T -> T
@@ -861,14 +752,14 @@ public class TElement
         {
         }
 
-        private void GetStringArray(ref VBuffer<DvText> dst)
+        private void GetStringArray(ref VBuffer<ReadOnlyMemory<char>> dst)
         {
             var value = (string[])(object)Value;
             var n = Utils.Size(value);
-            dst = new VBuffer<DvText>(n, Utils.Size(dst.Values) < n ? new DvText[n] : dst.Values, dst.Indices);
+            dst = new VBuffer<ReadOnlyMemory<char>>(n, Utils.Size(dst.Values) < n ? new ReadOnlyMemory<char>[n] : dst.Values, dst.Indices);
 
             for (int i = 0; i < n; i++)
-                dst.Values[i] = new DvText(value[i]);
+                dst.Values[i] = value[i].AsMemory();
 
         }
 
@@ -890,9 +781,9 @@ private ValueGetter<VBuffer<TDst>> GetVBufferGetter<TDst>()
             return (ref VBuffer<TDst> dst) => castValue.CopyTo(ref dst);
         }
 
-        private void GetString(ref DvText dst)
+        private void GetString(ref ReadOnlyMemory<char> dst)
         {
-            dst = new DvText((string)(object)Value);
+            dst = ((string)(object)Value).AsMemory();
         }
 
         private void GetDirectValue<TDst>(ref TDst dst)