From c073e6b3bcb271afe002cd1fb354bb2309551a28 Mon Sep 17 00:00:00 2001 From: LittleLittleCloud Date: Tue, 28 Apr 2020 12:25:38 -0700 Subject: [PATCH 01/18] upgrade to 3.1 --- .../Templates/Console/PredictProject.cs | 2 +- .../Templates/Console/PredictProject.tt | 2 +- ...s.AzureCodeGeneratorTest.test.ConsoleApp.csproj.approved.txt | 2 +- ...CodeGeneratorTest.CodeGenTest.ConsoleApp.csproj.approved.txt | 2 +- ...GeneratorTests.ConsoleAppProjectFileContentTest.approved.txt | 2 +- ...eConsoleAppProjectContents_VerifyPredictProject.approved.txt | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.ML.CodeGenerator/Templates/Console/PredictProject.cs b/src/Microsoft.ML.CodeGenerator/Templates/Console/PredictProject.cs index 96c4bbf124..29f288a076 100644 --- a/src/Microsoft.ML.CodeGenerator/Templates/Console/PredictProject.cs +++ b/src/Microsoft.ML.CodeGenerator/Templates/Console/PredictProject.cs @@ -27,7 +27,7 @@ internal partial class PredictProject : PredictProjectBase public virtual string TransformText() { this.Write("\r\n\r\n \r\n Exe\r\n netcoreapp2.1\r\n \r\n netcoreapp3.1\r\n \r\n \r\n \r\n"); diff --git a/src/Microsoft.ML.CodeGenerator/Templates/Console/PredictProject.tt b/src/Microsoft.ML.CodeGenerator/Templates/Console/PredictProject.tt index 1dd932085e..15e82d409c 100644 --- a/src/Microsoft.ML.CodeGenerator/Templates/Console/PredictProject.tt +++ b/src/Microsoft.ML.CodeGenerator/Templates/Console/PredictProject.tt @@ -8,7 +8,7 @@ Exe - netcoreapp2.1 + netcoreapp3.1 diff --git a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureCodeGeneratorTest.test.ConsoleApp.csproj.approved.txt b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureCodeGeneratorTest.test.ConsoleApp.csproj.approved.txt index 66e5ca3695..f94467d5dc 100644 --- a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureCodeGeneratorTest.test.ConsoleApp.csproj.approved.txt +++ b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureCodeGeneratorTest.test.ConsoleApp.csproj.approved.txt @@ -2,7 +2,7 @@ Exe - netcoreapp2.1 + netcoreapp3.1 diff --git a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureImageCodeGeneratorTest.CodeGenTest.ConsoleApp.csproj.approved.txt b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureImageCodeGeneratorTest.CodeGenTest.ConsoleApp.csproj.approved.txt index 23fa52e0c1..5a0f249e82 100644 --- a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureImageCodeGeneratorTest.CodeGenTest.ConsoleApp.csproj.approved.txt +++ b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.AzureImageCodeGeneratorTest.CodeGenTest.ConsoleApp.csproj.approved.txt @@ -2,7 +2,7 @@ Exe - netcoreapp2.1 + netcoreapp3.1 diff --git a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ConsoleAppProjectFileContentTest.approved.txt b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ConsoleAppProjectFileContentTest.approved.txt index 9ef4077dee..97df9ea672 100644 --- a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ConsoleAppProjectFileContentTest.approved.txt +++ b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.ConsoleAppProjectFileContentTest.approved.txt @@ -2,7 +2,7 @@ Exe - netcoreapp2.1 + netcoreapp3.1 diff --git a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.Recommendation_GenerateConsoleAppProjectContents_VerifyPredictProject.approved.txt b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.Recommendation_GenerateConsoleAppProjectContents_VerifyPredictProject.approved.txt index 14a45f8c0c..e261bed0fd 100644 --- a/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.Recommendation_GenerateConsoleAppProjectContents_VerifyPredictProject.approved.txt +++ b/test/Microsoft.ML.CodeGenerator.Tests/ApprovalTests/ConsoleCodeGeneratorTests.Recommendation_GenerateConsoleAppProjectContents_VerifyPredictProject.approved.txt @@ -2,7 +2,7 @@ Exe - netcoreapp2.1 + netcoreapp3.1 From 8f0fc1a59dd3aa6186ec4e94f9cccf93fab4dae4 Mon Sep 17 00:00:00 2001 From: LittleLittleCloud Date: Tue, 28 Apr 2020 12:31:31 -0700 Subject: [PATCH 02/18] write inline data using invariantCulture --- src/Microsoft.ML.CodeGenerator/Utils.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ML.CodeGenerator/Utils.cs b/src/Microsoft.ML.CodeGenerator/Utils.cs index 216097064c..075bc577ce 100644 --- a/src/Microsoft.ML.CodeGenerator/Utils.cs +++ b/src/Microsoft.ML.CodeGenerator/Utils.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; using System.Reflection; @@ -101,7 +102,7 @@ internal static string GetValueFromColumn(DataViewRowCursor rowCursor, DataVi return "Single.NegativeInfinity"; } - return f?.ToString() + "F"; + return f?.ToString(CultureInfo.InvariantCulture) + "F"; } if (val is bool) From 7cd6d8acc85bba0836e63e06a11dc46ec3ec38b9 Mon Sep 17 00:00:00 2001 From: BigBigMiao Date: Mon, 21 Sep 2020 12:53:14 -0700 Subject: [PATCH 03/18] upodate tensorflow --- build/Dependencies.props | 4 ++-- src/Microsoft.ML.TensorFlow/TensorTypeExtensions.cs | 1 + src/Microsoft.ML.TensorFlow/TensorflowTransform.cs | 9 +++++---- src/Microsoft.ML.TensorFlow/TensorflowUtils.cs | 3 ++- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/build/Dependencies.props b/build/Dependencies.props index e9aa31db0c..b0f3b7f078 100644 --- a/build/Dependencies.props +++ b/build/Dependencies.props @@ -22,9 +22,9 @@ 4.5.0 4.5.0 4.5.0 - 1.14.0 + 2.3 1 - 0.11.8.1 + 0.20.0 diff --git a/src/Microsoft.ML.TensorFlow/TensorTypeExtensions.cs b/src/Microsoft.ML.TensorFlow/TensorTypeExtensions.cs index 2004d631d8..63983d248b 100644 --- a/src/Microsoft.ML.TensorFlow/TensorTypeExtensions.cs +++ b/src/Microsoft.ML.TensorFlow/TensorTypeExtensions.cs @@ -5,6 +5,7 @@ using System; using Microsoft.ML.Internal.Utilities; using Tensorflow; +using Utils = Microsoft.ML.Internal.Utilities.Utils; namespace Microsoft.ML.TensorFlow { diff --git a/src/Microsoft.ML.TensorFlow/TensorflowTransform.cs b/src/Microsoft.ML.TensorFlow/TensorflowTransform.cs index 6c46f59b2d..97c1100b80 100644 --- a/src/Microsoft.ML.TensorFlow/TensorflowTransform.cs +++ b/src/Microsoft.ML.TensorFlow/TensorflowTransform.cs @@ -20,6 +20,7 @@ using Tensorflow; using static Microsoft.ML.TensorFlow.TensorFlowUtils; using static Tensorflow.Binding; +using Utils = Microsoft.ML.Internal.Utilities.Utils; [assembly: LoadableClass(TensorFlowTransformer.Summary, typeof(IDataTransform), typeof(TensorFlowTransformer), typeof(TensorFlowEstimator.Options), typeof(SignatureDataTransform), TensorFlowTransformer.UserName, TensorFlowTransformer.ShortName)] @@ -344,7 +345,7 @@ internal static TensorShape GetTensorShape(TF_Output output, Graph graph, Status new ObjectDisposedException(nameof(graph)); var cstatus = status == null ? new Status() : status; - var n = c_api.TF_GraphGetTensorNumDims(graph, output, cstatus); + var n = c_api.TF_GraphGetTensorNumDims(graph, output, cstatus.Handle); cstatus.Check(); @@ -352,7 +353,7 @@ internal static TensorShape GetTensorShape(TF_Output output, Graph graph, Status return new TensorShape(new int[0]); var dims = new long[n]; - c_api.TF_GraphGetTensorShape(graph, output, dims, dims.Length, cstatus); + c_api.TF_GraphGetTensorShape(graph, output, dims, dims.Length, cstatus.Handle); cstatus.Check(); return new TensorShape(dims.Select(x => (int)x).ToArray()); } @@ -430,7 +431,7 @@ private protected override void SaveModel(ModelSaveContext ctx) var buffer = Session.graph.ToGraphDef(status); ctx.SaveBinaryStream("TFModel", w => { - w.WriteByteArray(buffer.MemoryBlock.ToArray()); + w.WriteByteArray(buffer.DangerousMemoryBlock.ToArray()); }); } @@ -837,7 +838,7 @@ private Tensor CastDataAndReturnAsTensor(T[] data) bytes[i] = Encoding.UTF8.GetBytes(((ReadOnlyMemory)(object)data[i]).ToArray()); } - return new Tensor(bytes, _tfShape.dims.Select(x => (long)x).ToArray()); + return new Tensor(new NDArray(bytes, _tfShape)); } return new Tensor(new NDArray(data, _tfShape)); diff --git a/src/Microsoft.ML.TensorFlow/TensorflowUtils.cs b/src/Microsoft.ML.TensorFlow/TensorflowUtils.cs index b1c8a78a5b..95bee19629 100644 --- a/src/Microsoft.ML.TensorFlow/TensorflowUtils.cs +++ b/src/Microsoft.ML.TensorFlow/TensorflowUtils.cs @@ -14,6 +14,7 @@ using Microsoft.ML.Transforms; using Tensorflow; using static Tensorflow.Binding; +using Utils = Microsoft.ML.Internal.Utilities.Utils; namespace Microsoft.ML.TensorFlow { @@ -491,7 +492,7 @@ public Tensor[] Run() { c_api.TF_SessionRun(_session, null, _inputs, _inputValues, _inputs.Length, _outputs, _outputValues, _outputValues.Length, _operations, - _operations.Length, IntPtr.Zero, _status); + _operations.Length, IntPtr.Zero, _status.Handle); } catch (Exception ex) { From fdf7b737928fd859af36808d3bc4ea3560192ba9 Mon Sep 17 00:00:00 2001 From: BigBigMiao Date: Mon, 21 Sep 2020 14:09:29 -0700 Subject: [PATCH 04/18] update Microsoft.ML.Vision --- src/Microsoft.ML.Vision/DnnRetrainTransform.cs | 9 +++++---- .../ImageClassificationTrainer.cs | 16 ++++++++-------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/Microsoft.ML.Vision/DnnRetrainTransform.cs b/src/Microsoft.ML.Vision/DnnRetrainTransform.cs index 2388ba8e19..5ca2a72bb5 100644 --- a/src/Microsoft.ML.Vision/DnnRetrainTransform.cs +++ b/src/Microsoft.ML.Vision/DnnRetrainTransform.cs @@ -19,6 +19,7 @@ using Tensorflow; using static Microsoft.ML.TensorFlow.TensorFlowUtils; using static Tensorflow.Binding; +using Utils = Microsoft.ML.Internal.Utilities.Utils; [assembly: LoadableClass(DnnRetrainTransformer.Summary, typeof(IDataTransform), typeof(DnnRetrainTransformer), typeof(DnnRetrainEstimator.Options), typeof(SignatureDataTransform), DnnRetrainTransformer.UserName, DnnRetrainTransformer.ShortName)] @@ -607,7 +608,7 @@ internal static TensorShape GetTensorShape(TF_Output output, Graph graph, Status new ObjectDisposedException(nameof(graph)); var cstatus = status == null ? new Status() : status; - var n = c_api.TF_GraphGetTensorNumDims(graph, output, cstatus); + var n = c_api.TF_GraphGetTensorNumDims(graph, output, cstatus.Handle); cstatus.Check(); @@ -615,7 +616,7 @@ internal static TensorShape GetTensorShape(TF_Output output, Graph graph, Status return new TensorShape(new int[0]); var dims = new long[n]; - c_api.TF_GraphGetTensorShape(graph, output, dims, dims.Length, cstatus); + c_api.TF_GraphGetTensorShape(graph, output, dims, dims.Length, cstatus.Handle); cstatus.Check(); return new TensorShape(dims.Select(x => (int)x).ToArray()); } @@ -1078,7 +1079,7 @@ private Tensor CastDataAndReturnAsTensor(T[] data) bytes[i] = Encoding.UTF8.GetBytes(((ReadOnlyMemory)(object)data[i]).ToArray()); } - return new Tensor(bytes, _tfShape.dims.Select(x => (long)x).ToArray()); + return new Tensor(new NDArray(bytes, _tfShape)); } return new Tensor(new NDArray(data, _tfShape)); @@ -1161,7 +1162,7 @@ private Tensor CastDataAndReturnAsTensor(T[] data) bytes[i] = Encoding.UTF8.GetBytes(((ReadOnlyMemory)(object)data[i]).ToArray()); } - return new Tensor(bytes, _tfShape.dims.Select(x => (long)x).ToArray()); + return new Tensor(new NDArray(bytes, _tfShape)); } return new Tensor(new NDArray(data, _tfShape)); diff --git a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs index 5f94ff282b..020ac45724 100644 --- a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs +++ b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs @@ -752,15 +752,15 @@ private void CheckTrainingParameters(Options options) var decodedImage4d = tf.expand_dims(decodedImageAsFloat, 0); var resizeShape = tf.stack(new int[] { inputDim.Item1, inputDim.Item2 }); var resizeShapeAsInt = tf.cast(resizeShape, dtype: tf.int32); - var resizedImage = tf.image.resize_bilinear(decodedImage4d, resizeShapeAsInt, false, "ResizeTensor"); + var resizedImage = tf.image.resize_bilinear(decodedImage4d, resizeShapeAsInt, false, name: "ResizeTensor"); return (jpegData, resizedImage); } private static Tensor EncodeByteAsString(VBuffer buffer) { int length = buffer.Length; - var size = c_api.TF_StringEncodedSize((UIntPtr)length); - var handle = c_api.TF_AllocateTensor(TF_DataType.TF_STRING, IntPtr.Zero, 0, (UIntPtr)((ulong)size + 8)); + var size = c_api.TF_StringEncodedSize((ulong)length); + var handle = c_api.TF_AllocateTensor(TF_DataType.TF_STRING, new long[0], 0, ((ulong)size + 8)); IntPtr tensor = c_api.TF_TensorData(handle); Marshal.WriteInt64(tensor, 0); @@ -769,7 +769,7 @@ private static Tensor EncodeByteAsString(VBuffer buffer) unsafe { fixed (byte* src = buffer.GetValues()) - c_api.TF_StringEncode(src, (UIntPtr)length, (sbyte*)(tensor + sizeof(Int64)), size, status); + c_api.TF_StringEncode(src, (ulong)length, (byte*)(tensor + sizeof(Int64)), size, status.Handle); } status.Check(true); @@ -1213,7 +1213,7 @@ private void UpdateTransferLearningModelOnDisk(int classCount) sess.Dispose(); } - private void VariableSummaries(RefVariable var) + private void VariableSummaries(ResourceVariable var) { tf_with(tf.name_scope("summaries"), delegate { @@ -1256,7 +1256,7 @@ private void VariableSummaries(RefVariable var) Tensor logits = null; tf_with(tf.name_scope(layerName), scope => { - RefVariable layerWeights = null; + ResourceVariable layerWeights = null; tf_with(tf.name_scope("weights"), delegate { var initialValue = tf.truncated_normal(new int[] { bottleneck_tensor_size, classCount }, @@ -1266,7 +1266,7 @@ private void VariableSummaries(RefVariable var) VariableSummaries(layerWeights); }); - RefVariable layerBiases = null; + ResourceVariable layerBiases = null; tf_with(tf.name_scope("biases"), delegate { TensorShape shape = new TensorShape(classCount); @@ -1450,7 +1450,7 @@ private protected override void SaveCore(ModelSaveContext ctx) var buffer = _session.graph.ToGraphDef(status); ctx.SaveBinaryStream("TFModel", w => { - w.WriteByteArray(buffer.MemoryBlock.ToArray()); + w.WriteByteArray(buffer.DangerousMemoryBlock.ToArray()); }); status.Check(true); } From d0c261e657819f28936ea02a236897c6c80817c6 Mon Sep 17 00:00:00 2001 From: BigBigMiao Date: Mon, 28 Sep 2020 16:00:09 -0700 Subject: [PATCH 05/18] fix test && comment --- build/Dependencies.props | 2 +- .../TensorflowTransform.cs | 15 +++++++++------ src/Microsoft.ML.Vision/DnnRetrainTransform.cs | 4 ++-- .../ImageClassificationTrainer.cs | 17 +++++++++-------- 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/build/Dependencies.props b/build/Dependencies.props index b0f3b7f078..cf498d9da8 100644 --- a/build/Dependencies.props +++ b/build/Dependencies.props @@ -22,7 +22,7 @@ 4.5.0 4.5.0 4.5.0 - 2.3 + 2.3.0 1 0.20.0 diff --git a/src/Microsoft.ML.TensorFlow/TensorflowTransform.cs b/src/Microsoft.ML.TensorFlow/TensorflowTransform.cs index 97c1100b80..247878e81a 100644 --- a/src/Microsoft.ML.TensorFlow/TensorflowTransform.cs +++ b/src/Microsoft.ML.TensorFlow/TensorflowTransform.cs @@ -281,6 +281,7 @@ internal TensorFlowTransformer(IHostEnvironment env, Session session, string[] o _addBatchDimensionInput = addBatchDimensionInput; Inputs = inputColumnNames; Outputs = outputColumnNames; + tf.compat.v1.disable_eager_execution(); (TFOutputTypes, OutputTypes, TFOutputOperations) = GetOutputInfo(Host, Session, Outputs); (TFInputTypes, TFInputShapes, TFInputOperations) = GetInputInfo(Host, Session, Inputs, batchSize); @@ -427,12 +428,14 @@ private protected override void SaveModel(ModelSaveContext ctx) ctx.Writer.WriteBoolByte(_addBatchDimensionInput); if (isFrozen) { - Status status = new Status(); - var buffer = Session.graph.ToGraphDef(status); - ctx.SaveBinaryStream("TFModel", w => + using (var status = new Status()) + using (var buffer = Session.graph.ToGraphDef(status)) { - w.WriteByteArray(buffer.DangerousMemoryBlock.ToArray()); - }); + ctx.SaveBinaryStream("TFModel", w => + { + w.WriteByteArray(buffer.DangerousMemoryBlock.ToArray()); + }); + } } Host.AssertNonEmpty(Inputs); @@ -838,7 +841,7 @@ private Tensor CastDataAndReturnAsTensor(T[] data) bytes[i] = Encoding.UTF8.GetBytes(((ReadOnlyMemory)(object)data[i]).ToArray()); } - return new Tensor(new NDArray(bytes, _tfShape)); + return new Tensor(bytes); } return new Tensor(new NDArray(data, _tfShape)); diff --git a/src/Microsoft.ML.Vision/DnnRetrainTransform.cs b/src/Microsoft.ML.Vision/DnnRetrainTransform.cs index 5ca2a72bb5..d9fe3e6d85 100644 --- a/src/Microsoft.ML.Vision/DnnRetrainTransform.cs +++ b/src/Microsoft.ML.Vision/DnnRetrainTransform.cs @@ -1079,7 +1079,7 @@ private Tensor CastDataAndReturnAsTensor(T[] data) bytes[i] = Encoding.UTF8.GetBytes(((ReadOnlyMemory)(object)data[i]).ToArray()); } - return new Tensor(new NDArray(bytes, _tfShape)); + return new Tensor(bytes); } return new Tensor(new NDArray(data, _tfShape)); @@ -1162,7 +1162,7 @@ private Tensor CastDataAndReturnAsTensor(T[] data) bytes[i] = Encoding.UTF8.GetBytes(((ReadOnlyMemory)(object)data[i]).ToArray()); } - return new Tensor(new NDArray(bytes, _tfShape)); + return new Tensor(bytes); } return new Tensor(new NDArray(data, _tfShape)); diff --git a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs index 020ac45724..be9552ea7d 100644 --- a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs +++ b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs @@ -760,7 +760,7 @@ private static Tensor EncodeByteAsString(VBuffer buffer) { int length = buffer.Length; var size = c_api.TF_StringEncodedSize((ulong)length); - var handle = c_api.TF_AllocateTensor(TF_DataType.TF_STRING, new long[0], 0, ((ulong)size + 8)); + var handle = c_api.TF_AllocateTensor(TF_DataType.TF_STRING, Array.Empty(), 0, ((ulong)size + 8)); IntPtr tensor = c_api.TF_TensorData(handle); Marshal.WriteInt64(tensor, 0); @@ -1445,14 +1445,15 @@ private protected override void SaveCore(ModelSaveContext ctx) ctx.Writer.Write(_imagePreprocessorTensorOutput); ctx.Writer.Write(_graphInputTensor); ctx.Writer.Write(_graphOutputTensor); - - Status status = new Status(); - var buffer = _session.graph.ToGraphDef(status); - ctx.SaveBinaryStream("TFModel", w => + using(var status = new Status()) + using(var buffer = _session.graph.ToGraphDef(status)) { - w.WriteByteArray(buffer.DangerousMemoryBlock.ToArray()); - }); - status.Check(true); + ctx.SaveBinaryStream("TFModel", w => + { + w.WriteByteArray(buffer.DangerousMemoryBlock.ToArray()); + }); + status.Check(true); + } } private class Classifier From 48064a1359006d229006e04e0b90c9f6f2297a4d Mon Sep 17 00:00:00 2001 From: BigBigMiao Date: Tue, 29 Sep 2020 13:40:36 -0700 Subject: [PATCH 06/18] udpate tensorflow.net to 0.20.1 --- build/Dependencies.props | 2 +- src/Microsoft.ML.TensorFlow/TensorFlowModel.cs | 1 + .../TensorflowTransform.cs | 8 ++++---- src/Microsoft.ML.Vision/DnnRetrainTransform.cs | 16 ++++++++-------- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/build/Dependencies.props b/build/Dependencies.props index cf498d9da8..e8d1d39e39 100644 --- a/build/Dependencies.props +++ b/build/Dependencies.props @@ -24,7 +24,7 @@ 4.5.0 2.3.0 1 - 0.20.0 + 0.20.1 diff --git a/src/Microsoft.ML.TensorFlow/TensorFlowModel.cs b/src/Microsoft.ML.TensorFlow/TensorFlowModel.cs index f8fd1b476e..4f1d9d9ff8 100644 --- a/src/Microsoft.ML.TensorFlow/TensorFlowModel.cs +++ b/src/Microsoft.ML.TensorFlow/TensorFlowModel.cs @@ -6,6 +6,7 @@ using Microsoft.ML.Runtime; using Microsoft.ML.TensorFlow; using Tensorflow; +using static Tensorflow.Binding; namespace Microsoft.ML.Transforms { diff --git a/src/Microsoft.ML.TensorFlow/TensorflowTransform.cs b/src/Microsoft.ML.TensorFlow/TensorflowTransform.cs index 247878e81a..0786561d61 100644 --- a/src/Microsoft.ML.TensorFlow/TensorflowTransform.cs +++ b/src/Microsoft.ML.TensorFlow/TensorflowTransform.cs @@ -835,13 +835,13 @@ private Tensor CastDataAndReturnAsTensor(T[] data) return new Tensor((double[])(object)data, _dims, TF_DataType.TF_DOUBLE); else if (typeof(T) == typeof(ReadOnlyMemory)) { - byte[][] bytes = new byte[_vBuffer.Length][]; - for (int i = 0; i < bytes.Length; i++) + string[] strings = new string[_vBuffer.Length]; + for (int i = 0; i < strings.Length; i++) { - bytes[i] = Encoding.UTF8.GetBytes(((ReadOnlyMemory)(object)data[i]).ToArray()); + strings[i] = new string(((ReadOnlyMemory)(object)data[i]).ToArray()); } - return new Tensor(bytes); + return new Tensor(strings); } return new Tensor(new NDArray(data, _tfShape)); diff --git a/src/Microsoft.ML.Vision/DnnRetrainTransform.cs b/src/Microsoft.ML.Vision/DnnRetrainTransform.cs index d9fe3e6d85..a185d784dd 100644 --- a/src/Microsoft.ML.Vision/DnnRetrainTransform.cs +++ b/src/Microsoft.ML.Vision/DnnRetrainTransform.cs @@ -1073,13 +1073,13 @@ private Tensor CastDataAndReturnAsTensor(T[] data) return new Tensor((double[])(object)data, _dims, TF_DataType.TF_DOUBLE); else if (typeof(T) == typeof(ReadOnlyMemory)) { - byte[][] bytes = new byte[_bufferedData.Length][]; - for (int i = 0; i < bytes.Length; i++) + string[] strings = new string[data.Length]; + for (int i = 0; i < strings.Length; i++) { - bytes[i] = Encoding.UTF8.GetBytes(((ReadOnlyMemory)(object)data[i]).ToArray()); + strings[i] = new string(((ReadOnlyMemory)(object)data[i]).ToArray()); } - return new Tensor(bytes); + return new Tensor(strings); } return new Tensor(new NDArray(data, _tfShape)); @@ -1156,13 +1156,13 @@ private Tensor CastDataAndReturnAsTensor(T[] data) return new Tensor((double[])(object)data, _dims, TF_DataType.TF_DOUBLE); else if (typeof(T) == typeof(ReadOnlyMemory)) { - byte[][] bytes = new byte[_vBuffer.Length][]; - for (int i = 0; i < bytes.Length; i++) + string[] strings = new string[_vBuffer.Length]; + for (int i = 0; i < strings.Length; i++) { - bytes[i] = Encoding.UTF8.GetBytes(((ReadOnlyMemory)(object)data[i]).ToArray()); + strings[i] = new string(((ReadOnlyMemory)(object)data[i]).ToArray()); } - return new Tensor(bytes); + return new Tensor(strings); } return new Tensor(new NDArray(data, _tfShape)); From 35d4e2507577fdb8fb32d441a668701577d6cbba Mon Sep 17 00:00:00 2001 From: BigBigMiao Date: Tue, 29 Sep 2020 16:01:07 -0700 Subject: [PATCH 07/18] update tf major version --- build/Dependencies.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Dependencies.props b/build/Dependencies.props index e8d1d39e39..0c3651852c 100644 --- a/build/Dependencies.props +++ b/build/Dependencies.props @@ -23,7 +23,7 @@ 4.5.0 4.5.0 2.3.0 - 1 + 2 0.20.1 From 2aafe7bf1ea259239cd0e889ed276db28209aaca Mon Sep 17 00:00:00 2001 From: BigBigMiao Date: Tue, 29 Sep 2020 16:46:51 -0700 Subject: [PATCH 08/18] downgrade tf runtime to 1.14.1 --- build/Dependencies.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/Dependencies.props b/build/Dependencies.props index 0c3651852c..4787676d11 100644 --- a/build/Dependencies.props +++ b/build/Dependencies.props @@ -22,8 +22,8 @@ 4.5.0 4.5.0 4.5.0 - 2.3.0 - 2 + 1.14.1 + 1 0.20.1 From e7e657c772eb37aaa6b3f65ac43a47e050ab922c Mon Sep 17 00:00:00 2001 From: Xiaoyun Zhang Date: Wed, 30 Sep 2020 09:50:32 -0700 Subject: [PATCH 09/18] Update Dependencies.props --- build/Dependencies.props | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/Dependencies.props b/build/Dependencies.props index 4787676d11..0c3651852c 100644 --- a/build/Dependencies.props +++ b/build/Dependencies.props @@ -22,8 +22,8 @@ 4.5.0 4.5.0 4.5.0 - 1.14.1 - 1 + 2.3.0 + 2 0.20.1 From b92bc45d263f2bec65f70cda79b08deeeb956154 Mon Sep 17 00:00:00 2001 From: Xiaoyun Zhang Date: Wed, 7 Oct 2020 20:01:28 -0700 Subject: [PATCH 10/18] Update Dependencies.props --- build/Dependencies.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Dependencies.props b/build/Dependencies.props index 0c3651852c..31524dceec 100644 --- a/build/Dependencies.props +++ b/build/Dependencies.props @@ -22,7 +22,7 @@ 4.5.0 4.5.0 4.5.0 - 2.3.0 + 2.3.1 2 0.20.1 From b314ea5dd0ba160d07679fa0d5762aa1e3b02be2 Mon Sep 17 00:00:00 2001 From: BigBigMiao Date: Thu, 8 Oct 2020 11:26:42 -0700 Subject: [PATCH 11/18] update tffact to stop running test on linux with glibc < 2.3) --- .../Attributes/TensorflowFactAttribute.cs | 8 ++++++-- .../Attributes/TensorflowTheoryAttribute.cs | 9 +++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/test/Microsoft.ML.TestFramework/Attributes/TensorflowFactAttribute.cs b/test/Microsoft.ML.TestFramework/Attributes/TensorflowFactAttribute.cs index f7ae0ecf91..d2376ef3ac 100644 --- a/test/Microsoft.ML.TestFramework/Attributes/TensorflowFactAttribute.cs +++ b/test/Microsoft.ML.TestFramework/Attributes/TensorflowFactAttribute.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System; +using System.Runtime.InteropServices; using Microsoft.ML.TestFrameworkCommon.Attributes; namespace Microsoft.ML.TestFramework.Attributes @@ -12,14 +13,17 @@ namespace Microsoft.ML.TestFramework.Attributes /// public sealed class TensorFlowFactAttribute : EnvironmentSpecificFactAttribute { - public TensorFlowFactAttribute() : base("TensorFlow is 64-bit only") + public TensorFlowFactAttribute() : base("TensorFlow is 64-bit only and is not supported on Linux with libc < v2.23") { } /// protected override bool IsEnvironmentSupported() { - return Environment.Is64BitProcess; + return Environment.Is64BitProcess && + ( RuntimeInformation.IsOSPlatform(OSPlatform.Windows) || + AttributeHelpers.CheckLibcVersionGreaterThanMinimum(new Version(2, 23))); + } } } diff --git a/test/Microsoft.ML.TestFramework/Attributes/TensorflowTheoryAttribute.cs b/test/Microsoft.ML.TestFramework/Attributes/TensorflowTheoryAttribute.cs index 24b73a56f5..feed8047d7 100644 --- a/test/Microsoft.ML.TestFramework/Attributes/TensorflowTheoryAttribute.cs +++ b/test/Microsoft.ML.TestFramework/Attributes/TensorflowTheoryAttribute.cs @@ -3,6 +3,9 @@ // See the LICENSE file in the project root for more information. using System; +using System.Runtime.InteropServices; +using Microsoft.ML.TestFrameworkCommon.Attributes; + namespace Microsoft.ML.TestFramework.Attributes { /// @@ -10,14 +13,16 @@ namespace Microsoft.ML.TestFramework.Attributes /// public sealed class TensorFlowTheoryAttribute : EnvironmentSpecificTheoryAttribute { - public TensorFlowTheoryAttribute() : base("TensorFlow is 64-bit only") + public TensorFlowTheoryAttribute() : base("TensorFlow is 64-bit only and is not supported on Linux with libc < v2.23") { } /// protected override bool IsEnvironmentSupported() { - return Environment.Is64BitProcess; + return Environment.Is64BitProcess && + (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) || + AttributeHelpers.CheckLibcVersionGreaterThanMinimum(new Version(2, 23))); } } } \ No newline at end of file From 80f3c16bd86fff3bd46ba4951b07852117fef17c Mon Sep 17 00:00:00 2001 From: BigBigMiao Date: Thu, 8 Oct 2020 12:19:54 -0700 Subject: [PATCH 12/18] fix TensorFlowTransformInputShapeTest --- .../ScenariosWithDirectInstantiation/TensorflowTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs index 60517fcd97..c5b9e839fc 100644 --- a/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs +++ b/test/Microsoft.ML.Tests/ScenariosWithDirectInstantiation/TensorflowTests.cs @@ -278,7 +278,7 @@ private List GetShapeData() }); } - [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] // TensorFlow is 64-bit only + [TensorFlowFact] // TensorFlow is 64-bit only public void TensorFlowTransformInputShapeTest() { var modelLocation = "model_shape_test"; From 65271c305094b0e94b9f23932a6e097a66d9eff3 Mon Sep 17 00:00:00 2001 From: BigBigMiao Date: Thu, 8 Oct 2020 15:14:04 -0700 Subject: [PATCH 13/18] use tf.v1 api --- src/Microsoft.ML.Vision/ImageClassificationTrainer.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs index be9552ea7d..e5fe486ae5 100644 --- a/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs +++ b/src/Microsoft.ML.Vision/ImageClassificationTrainer.cs @@ -530,6 +530,7 @@ internal ImageClassificationTrainer(IHostEnvironment env, Options options) Host.CheckNonEmpty(options.LabelColumnName, nameof(options.LabelColumnName)); Host.CheckNonEmpty(options.ScoreColumnName, nameof(options.ScoreColumnName)); Host.CheckNonEmpty(options.PredictedLabelColumnName, nameof(options.PredictedLabelColumnName)); + tf.compat.v1.disable_eager_execution(); if (string.IsNullOrEmpty(options.WorkspacePath)) { From ec4a5eebfa53bf03df0b86647c4d7e1912870666 Mon Sep 17 00:00:00 2001 From: LittleLittleCloud Date: Fri, 9 Oct 2020 13:45:34 -0700 Subject: [PATCH 14/18] fix comment: --- .../TensorFlowModel.cs | 1 - .../TensorflowTransform.cs | 42 +--------- .../TensorflowUtils.cs | 41 +++++++++ .../DnnRetrainTransform.cs | 83 +------------------ 4 files changed, 46 insertions(+), 121 deletions(-) diff --git a/src/Microsoft.ML.TensorFlow/TensorFlowModel.cs b/src/Microsoft.ML.TensorFlow/TensorFlowModel.cs index 4f1d9d9ff8..f8fd1b476e 100644 --- a/src/Microsoft.ML.TensorFlow/TensorFlowModel.cs +++ b/src/Microsoft.ML.TensorFlow/TensorFlowModel.cs @@ -6,7 +6,6 @@ using Microsoft.ML.Runtime; using Microsoft.ML.TensorFlow; using Tensorflow; -using static Tensorflow.Binding; namespace Microsoft.ML.Transforms { diff --git a/src/Microsoft.ML.TensorFlow/TensorflowTransform.cs b/src/Microsoft.ML.TensorFlow/TensorflowTransform.cs index 0786561d61..998ad30418 100644 --- a/src/Microsoft.ML.TensorFlow/TensorflowTransform.cs +++ b/src/Microsoft.ML.TensorFlow/TensorflowTransform.cs @@ -805,48 +805,10 @@ public Tensor GetTensor() // This is done to reduce memory allocation every time tensor is created. _denseData = new T[_vBuffer.Length]; _vBuffer.CopyTo(_denseData); - var tensor = CastDataAndReturnAsTensor(_denseData); + var tensor = TensorFlowUtils.CastDataAndReturnAsTensor(_denseData, _tfShape); return tensor; } - private Tensor CastDataAndReturnAsTensor(T[] data) - { - if (typeof(T) == typeof(sbyte)) - return new Tensor((sbyte[])(object)data, _dims, TF_DataType.TF_INT8); - else if (typeof(T) == typeof(long)) - return new Tensor((long[])(object)data, _dims, TF_DataType.TF_INT64); - else if (typeof(T) == typeof(Int32)) - return new Tensor((Int32[])(object)data, _dims, TF_DataType.TF_INT32); - else if (typeof(T) == typeof(Int16)) - return new Tensor((Int16[])(object)data, _dims, TF_DataType.TF_INT16); - else if (typeof(T) == typeof(byte)) - return new Tensor((byte[])(object)data, _dims, TF_DataType.TF_UINT8); - else if (typeof(T) == typeof(ulong)) - return new Tensor((ulong[])(object)data, _dims, TF_DataType.TF_UINT64); - else if (typeof(T) == typeof(UInt32)) - return new Tensor((UInt32[])(object)data, _dims, TF_DataType.TF_UINT32); - else if (typeof(T) == typeof(UInt16)) - return new Tensor((UInt16[])(object)data, _dims, TF_DataType.TF_UINT16); - else if (typeof(T) == typeof(bool)) - return new Tensor((bool[])(object)data, _dims, TF_DataType.TF_BOOL); - else if (typeof(T) == typeof(float)) - return new Tensor((float[])(object)data, _dims, TF_DataType.TF_FLOAT); - else if (typeof(T) == typeof(double)) - return new Tensor((double[])(object)data, _dims, TF_DataType.TF_DOUBLE); - else if (typeof(T) == typeof(ReadOnlyMemory)) - { - string[] strings = new string[_vBuffer.Length]; - for (int i = 0; i < strings.Length; i++) - { - strings[i] = new string(((ReadOnlyMemory)(object)data[i]).ToArray()); - } - - return new Tensor(strings); - } - - return new Tensor(new NDArray(data, _tfShape)); - } - public void BufferTrainingData() { _srcgetter(ref _vBuffer); @@ -857,7 +819,7 @@ public void BufferTrainingData() public Tensor GetBufferedBatchTensor() { _position = 0; - var tensor = CastDataAndReturnAsTensor(_bufferedData); + var tensor = TensorFlowUtils.CastDataAndReturnAsTensor(_denseData, _tfShape); _bufferedData = new T[_bufferedDataSize]; return tensor; diff --git a/src/Microsoft.ML.TensorFlow/TensorflowUtils.cs b/src/Microsoft.ML.TensorFlow/TensorflowUtils.cs index 95bee19629..0bf95517ae 100644 --- a/src/Microsoft.ML.TensorFlow/TensorflowUtils.cs +++ b/src/Microsoft.ML.TensorFlow/TensorflowUtils.cs @@ -12,6 +12,7 @@ using Microsoft.ML.Runtime; using Microsoft.ML.TensorFlow; using Microsoft.ML.Transforms; +using NumSharp; using Tensorflow; using static Tensorflow.Binding; using Utils = Microsoft.ML.Internal.Utilities.Utils; @@ -411,6 +412,46 @@ internal static bool IsTypeSupported(TF_DataType tfoutput) } } + internal static Tensor CastDataAndReturnAsTensor(T[] data, TensorShape tfShape) + { + var dims = tfShape.dims.Select(x => (long)x).ToArray(); + + if (typeof(T) == typeof(sbyte)) + return new Tensor((sbyte[])(object)data, dims, TF_DataType.TF_INT8); + else if (typeof(T) == typeof(long)) + return new Tensor((long[])(object)data, dims, TF_DataType.TF_INT64); + else if (typeof(T) == typeof(Int32)) + return new Tensor((Int32[])(object)data, dims, TF_DataType.TF_INT32); + else if (typeof(T) == typeof(Int16)) + return new Tensor((Int16[])(object)data, dims, TF_DataType.TF_INT16); + else if (typeof(T) == typeof(byte)) + return new Tensor((byte[])(object)data, dims, TF_DataType.TF_UINT8); + else if (typeof(T) == typeof(ulong)) + return new Tensor((ulong[])(object)data, dims, TF_DataType.TF_UINT64); + else if (typeof(T) == typeof(UInt32)) + return new Tensor((UInt32[])(object)data, dims, TF_DataType.TF_UINT32); + else if (typeof(T) == typeof(UInt16)) + return new Tensor((UInt16[])(object)data, dims, TF_DataType.TF_UINT16); + else if (typeof(T) == typeof(bool)) + return new Tensor((bool[])(object)data, dims, TF_DataType.TF_BOOL); + else if (typeof(T) == typeof(float)) + return new Tensor((float[])(object)data, dims, TF_DataType.TF_FLOAT); + else if (typeof(T) == typeof(double)) + return new Tensor((double[])(object)data, dims, TF_DataType.TF_DOUBLE); + else if (typeof(T) == typeof(ReadOnlyMemory)) + { + string[] strings = new string[data.Length]; + for (int i = 0; i < strings.Length; i++) + { + strings[i] = ((ReadOnlyMemory)(object)data[i]).ToString(); + } + + return new Tensor(strings); + } + + return new Tensor(new NDArray(data, tfShape)); + } + /// /// Use the runner class to easily configure inputs, outputs and targets to be passed to the session runner. /// diff --git a/src/Microsoft.ML.Vision/DnnRetrainTransform.cs b/src/Microsoft.ML.Vision/DnnRetrainTransform.cs index a185d784dd..e13da33173 100644 --- a/src/Microsoft.ML.Vision/DnnRetrainTransform.cs +++ b/src/Microsoft.ML.Vision/DnnRetrainTransform.cs @@ -1041,51 +1041,12 @@ public Tensor GetBufferedBatchTensor() } else { - var tensor = CastDataAndReturnAsTensor(_bufferedData); + var tensor = TensorFlowUtils.CastDataAndReturnAsTensor(_denseData, _tfShape) _position = 0; return tensor; } } - private Tensor CastDataAndReturnAsTensor(T[] data) - { - if (typeof(T) == typeof(sbyte)) - return new Tensor((sbyte[])(object)data, _dims, TF_DataType.TF_INT8); - else if (typeof(T) == typeof(long)) - return new Tensor((long[])(object)data, _dims, TF_DataType.TF_INT64); - else if (typeof(T) == typeof(Int32)) - return new Tensor((Int32[])(object)data, _dims, TF_DataType.TF_INT32); - else if (typeof(T) == typeof(Int16)) - return new Tensor((Int16[])(object)data, _dims, TF_DataType.TF_INT16); - else if (typeof(T) == typeof(byte)) - return new Tensor((byte[])(object)data, _dims, TF_DataType.TF_UINT8); - else if (typeof(T) == typeof(ulong)) - return new Tensor((ulong[])(object)data, _dims, TF_DataType.TF_UINT64); - else if (typeof(T) == typeof(UInt32)) - return new Tensor((UInt32[])(object)data, _dims, TF_DataType.TF_UINT32); - else if (typeof(T) == typeof(UInt16)) - return new Tensor((UInt16[])(object)data, _dims, TF_DataType.TF_UINT16); - else if (typeof(T) == typeof(bool)) - return new Tensor((bool[])(object)data, _dims, TF_DataType.TF_BOOL); - else if (typeof(T) == typeof(float)) - return new Tensor((float[])(object)data, _dims, TF_DataType.TF_FLOAT); - else if (typeof(T) == typeof(float)) - return new Tensor((double[])(object)data, _dims, TF_DataType.TF_DOUBLE); - else if (typeof(T) == typeof(ReadOnlyMemory)) - { - string[] strings = new string[data.Length]; - for (int i = 0; i < strings.Length; i++) - { - strings[i] = new string(((ReadOnlyMemory)(object)data[i]).ToArray()); - } - - return new Tensor(strings); - } - - return new Tensor(new NDArray(data, _tfShape)); - } - } - private class TensorValueGetterVec : ITensorValueGetter { private readonly ValueGetter> _srcgetter; @@ -1127,45 +1088,7 @@ public Tensor GetTensor() // This is done to reduce memory allocation every time tensor is created. _denseData = new T[_vBuffer.Length]; _vBuffer.CopyTo(_denseData); - return CastDataAndReturnAsTensor(_denseData); - } - - private Tensor CastDataAndReturnAsTensor(T[] data) - { - if (typeof(T) == typeof(sbyte)) - return new Tensor((sbyte[])(object)data, _dims, TF_DataType.TF_INT8); - else if (typeof(T) == typeof(long)) - return new Tensor((long[])(object)data, _dims, TF_DataType.TF_INT64); - else if (typeof(T) == typeof(Int32)) - return new Tensor((Int32[])(object)data, _dims, TF_DataType.TF_INT32); - else if (typeof(T) == typeof(Int16)) - return new Tensor((Int16[])(object)data, _dims, TF_DataType.TF_INT16); - else if (typeof(T) == typeof(byte)) - return new Tensor((byte[])(object)data, _dims, TF_DataType.TF_UINT8); - else if (typeof(T) == typeof(ulong)) - return new Tensor((ulong[])(object)data, _dims, TF_DataType.TF_UINT64); - else if (typeof(T) == typeof(UInt32)) - return new Tensor((UInt32[])(object)data, _dims, TF_DataType.TF_UINT32); - else if (typeof(T) == typeof(UInt16)) - return new Tensor((UInt16[])(object)data, _dims, TF_DataType.TF_UINT16); - else if (typeof(T) == typeof(bool)) - return new Tensor((bool[])(object)data, _dims, TF_DataType.TF_BOOL); - else if (typeof(T) == typeof(float)) - return new Tensor((float[])(object)data, _dims, TF_DataType.TF_FLOAT); - else if (typeof(T) == typeof(double)) - return new Tensor((double[])(object)data, _dims, TF_DataType.TF_DOUBLE); - else if (typeof(T) == typeof(ReadOnlyMemory)) - { - string[] strings = new string[_vBuffer.Length]; - for (int i = 0; i < strings.Length; i++) - { - strings[i] = new string(((ReadOnlyMemory)(object)data[i]).ToArray()); - } - - return new Tensor(strings); - } - - return new Tensor(new NDArray(data, _tfShape)); + return TensorFlowUtils.CastDataAndReturnAsTensor(_denseData, _tfShape); } public void BufferTrainingData() @@ -1178,7 +1101,7 @@ public void BufferTrainingData() public Tensor GetBufferedBatchTensor() { _position = 0; - var tensor = CastDataAndReturnAsTensor(_bufferedData); + var tensor = TensorFlowUtils.CastDataAndReturnAsTensor(_denseData, _tfShape); _bufferedData = new T[_bufferedDataSize]; return tensor; } From df6cbcbd4b25affbd90e4152e7a11e90718eba02 Mon Sep 17 00:00:00 2001 From: LittleLittleCloud Date: Fri, 9 Oct 2020 15:13:54 -0700 Subject: [PATCH 15/18] fix building error --- src/Microsoft.ML.Vision/DnnRetrainTransform.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ML.Vision/DnnRetrainTransform.cs b/src/Microsoft.ML.Vision/DnnRetrainTransform.cs index e13da33173..f53122bde5 100644 --- a/src/Microsoft.ML.Vision/DnnRetrainTransform.cs +++ b/src/Microsoft.ML.Vision/DnnRetrainTransform.cs @@ -1041,11 +1041,12 @@ public Tensor GetBufferedBatchTensor() } else { - var tensor = TensorFlowUtils.CastDataAndReturnAsTensor(_denseData, _tfShape) + var tensor = TensorFlowUtils.CastDataAndReturnAsTensor(_bufferedData, _tfShape); _position = 0; return tensor; } } + } private class TensorValueGetterVec : ITensorValueGetter { From 3d00c41a0d98fbf2e324c03fbd975c9f82360a93 Mon Sep 17 00:00:00 2001 From: LittleLittleCloud Date: Mon, 12 Oct 2020 12:29:33 -0700 Subject: [PATCH 16/18] fix test --- src/Microsoft.ML.Vision/DnnRetrainTransform.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.Vision/DnnRetrainTransform.cs b/src/Microsoft.ML.Vision/DnnRetrainTransform.cs index f53122bde5..240b2773f7 100644 --- a/src/Microsoft.ML.Vision/DnnRetrainTransform.cs +++ b/src/Microsoft.ML.Vision/DnnRetrainTransform.cs @@ -1102,7 +1102,7 @@ public void BufferTrainingData() public Tensor GetBufferedBatchTensor() { _position = 0; - var tensor = TensorFlowUtils.CastDataAndReturnAsTensor(_denseData, _tfShape); + var tensor = TensorFlowUtils.CastDataAndReturnAsTensor(_bufferedData, _tfShape); _bufferedData = new T[_bufferedDataSize]; return tensor; } From fae686483f8f87fb4bbacbdd6c0019bf7a080e86 Mon Sep 17 00:00:00 2001 From: LittleLittleCloud Date: Mon, 12 Oct 2020 15:26:17 -0700 Subject: [PATCH 17/18] fix nit --- src/Microsoft.ML.TensorFlow/TensorflowUtils.cs | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/Microsoft.ML.TensorFlow/TensorflowUtils.cs b/src/Microsoft.ML.TensorFlow/TensorflowUtils.cs index 0bf95517ae..a7c90bc4c8 100644 --- a/src/Microsoft.ML.TensorFlow/TensorflowUtils.cs +++ b/src/Microsoft.ML.TensorFlow/TensorflowUtils.cs @@ -439,15 +439,7 @@ internal static Tensor CastDataAndReturnAsTensor(T[] data, TensorShape tfShap else if (typeof(T) == typeof(double)) return new Tensor((double[])(object)data, dims, TF_DataType.TF_DOUBLE); else if (typeof(T) == typeof(ReadOnlyMemory)) - { - string[] strings = new string[data.Length]; - for (int i = 0; i < strings.Length; i++) - { - strings[i] = ((ReadOnlyMemory)(object)data[i]).ToString(); - } - - return new Tensor(strings); - } + return new Tensor(data.Select(x => x.ToString()).ToArray()); return new Tensor(new NDArray(data, tfShape)); } From 5f3e34bb690add65becd5bb503b42d8c76f7297b Mon Sep 17 00:00:00 2001 From: LittleLittleCloud Date: Mon, 12 Oct 2020 19:40:53 -0700 Subject: [PATCH 18/18] remove linq --- src/Microsoft.ML.TensorFlow/TensorflowUtils.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ML.TensorFlow/TensorflowUtils.cs b/src/Microsoft.ML.TensorFlow/TensorflowUtils.cs index a7c90bc4c8..dd25bb104b 100644 --- a/src/Microsoft.ML.TensorFlow/TensorflowUtils.cs +++ b/src/Microsoft.ML.TensorFlow/TensorflowUtils.cs @@ -439,7 +439,15 @@ internal static Tensor CastDataAndReturnAsTensor(T[] data, TensorShape tfShap else if (typeof(T) == typeof(double)) return new Tensor((double[])(object)data, dims, TF_DataType.TF_DOUBLE); else if (typeof(T) == typeof(ReadOnlyMemory)) - return new Tensor(data.Select(x => x.ToString()).ToArray()); + { + string[] strings = new string[data.Length]; + for (int i = 0; i < strings.Length; i++) + { + strings[i] = data[i].ToString(); + } + + return new Tensor(strings); + } return new Tensor(new NDArray(data, tfShape)); }