Merge PR# 207.

dotnet · May 23, 2018 · 0ef9ce1 · 0ef9ce1
1 parent 363cad8
commit 0ef9ce1
Show file tree

Hide file tree

Showing 20 changed files with 223 additions and 118 deletions.
diff --git a/ZBaselines/Common/EntryPoints/core_ep-list.tsv b/ZBaselines/Common/EntryPoints/core_ep-list.tsv
@@ -3,6 +3,7 @@ Data.DataViewReference	Pass dataview from memory to experiment	Microsoft.ML.Runt
 Data.IDataViewArrayConverter	Create and array variable	Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro	MakeArray	Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIDataViewInput	Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIDataViewOutput
 Data.PredictorModelArrayConverter	Create and array variable	Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro	MakeArray	Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIPredictorModelInput	Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayIPredictorModelOutput
 Data.TextLoader	Import a dataset from a text file	Microsoft.ML.Runtime.EntryPoints.ImportTextData	TextLoader	Microsoft.ML.Runtime.EntryPoints.ImportTextData+LoaderInput	Microsoft.ML.Runtime.EntryPoints.ImportTextData+Output
+Data.TransformModelArrayConverter	Create and array variable	Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro	MakeArray	Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayITransformModelInput	Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+ArrayITransformModelOutput
 Models.AnomalyDetectionEvaluator	Evaluates an anomaly detection scored dataset.	Microsoft.ML.Runtime.Data.Evaluate	AnomalyDetection	Microsoft.ML.Runtime.Data.AnomalyDetectionMamlEvaluator+Arguments	Microsoft.ML.Runtime.EntryPoints.CommonOutputs+CommonEvaluateOutput
 Models.BinaryClassificationEvaluator	Evaluates a binary classification scored dataset.	Microsoft.ML.Runtime.Data.Evaluate	Binary	Microsoft.ML.Runtime.Data.BinaryClassifierMamlEvaluator+Arguments	Microsoft.ML.Runtime.EntryPoints.CommonOutputs+ClassificationEvaluateOutput
 Models.BinaryCrossValidator	Cross validation for binary classification	Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro	CrossValidateBinary	Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+Arguments	Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MacroOutput`1[Microsoft.ML.Runtime.EntryPoints.CrossValidationBinaryMacro+Output]

diff --git a/ZBaselines/Common/EntryPoints/core_manifest.json b/ZBaselines/Common/EntryPoints/core_manifest.json
@@ -469,6 +469,35 @@
         "ILearningPipelineLoader"
       ]
     },
+    {
+      "Name": "Data.TransformModelArrayConverter",
+      "Desc": "Create and array variable",
+      "FriendlyName": null,
+      "ShortName": null,
+      "Inputs": [
+        {
+          "Name": "TransformModel",
+          "Type": {
+            "Kind": "Array",
+            "ItemType": "TransformModel"
+          },
+          "Desc": "The models",
+          "Required": true,
+          "SortOrder": 1.0,
+          "IsNullable": false
+        }
+      ],
+      "Outputs": [
+        {
+          "Name": "OutputModel",
+          "Type": {
+            "Kind": "Array",
+            "ItemType": "TransformModel"
+          },
+          "Desc": "The model array"
+        }
+      ]
+    },
     {
       "Name": "Models.AnomalyDetectionEvaluator",
       "Desc": "Evaluates an anomaly detection scored dataset.",
@@ -1411,9 +1440,28 @@
                 "Name": "Model",
                 "Type": "PredictorModel",
                 "Desc": "The model",
-                "Required": true,
+                "Required": false,
                 "SortOrder": 1.0,
-                "IsNullable": false
+                "IsNullable": false,
+                "Default": null
+              },
+              {
+                "Name": "TransformModel",
+                "Type": "TransformModel",
+                "Desc": "The transform model",
+                "Required": false,
+                "SortOrder": 2.0,
+                "IsNullable": false,
+                "Default": null
+              },
+              {
+                "Name": "UseTransformModel",
+                "Type": "Bool",
+                "Desc": "Indicates to use transform model instead of predictor model.",
+                "Required": false,
+                "SortOrder": 3.0,
+                "IsNullable": false,
+                "Default": false
               }
             ]
           },
@@ -1476,6 +1524,14 @@
           },
           "Desc": "The final model including the trained predictor model and the model from the transforms, provided as the Input.TransformModel."
         },
+        {
+          "Name": "TransformModel",
+          "Type": {
+            "Kind": "Array",
+            "ItemType": "TransformModel"
+          },
+          "Desc": "The final model including the trained predictor model and the model from the transforms, provided as the Input.TransformModel."
+        },
         {
           "Name": "Warnings",
           "Type": "DataView",
@@ -3002,9 +3058,28 @@
                 "Name": "Model",
                 "Type": "PredictorModel",
                 "Desc": "The model",
-                "Required": true,
+                "Required": false,
                 "SortOrder": 1.0,
-                "IsNullable": false
+                "IsNullable": false,
+                "Default": null
+              },
+              {
+                "Name": "TransformModel",
+                "Type": "TransformModel",
+                "Desc": "Transform model",
+                "Required": false,
+                "SortOrder": 2.0,
+                "IsNullable": false,
+                "Default": null
+              },
+              {
+                "Name": "UseTransformModel",
+                "Type": "Bool",
+                "Desc": "Indicates to use transform model instead of predictor model.",
+                "Required": false,
+                "SortOrder": 3.0,
+                "IsNullable": false,
+                "Default": false
               }
             ]
           },
@@ -3058,6 +3133,11 @@
           "Type": "PredictorModel",
           "Desc": "The final model including the trained predictor model and the model from the transforms, provided as the Input.TransformModel."
         },
+        {
+          "Name": "TransformModel",
+          "Type": "TransformModel",
+          "Desc": "The final model including the trained predictor model and the model from the transforms, provided as the Input.TransformModel."
+        },
         {
           "Name": "Warnings",
           "Type": "DataView",

diff --git a/src/Microsoft.ML.Core/Data/ITransformModel.cs b/src/Microsoft.ML.Core/Data/ITransformModel.cs
@@ -25,8 +25,6 @@ public interface ITransformModel
         /// </summary>
         ISchema InputSchema { get; }
 
-        IDataView Data { get; }
-
         /// <summary>
         /// Apply the transform(s) in the model to the given input data.
         /// </summary>

diff --git a/src/Microsoft.ML.Data/EntryPoints/TransformModel.cs b/src/Microsoft.ML.Data/EntryPoints/TransformModel.cs
@@ -44,11 +44,6 @@ public ISchema InputSchema
             get { return _schemaRoot; }
         }
 
-        public IDataView Data
-        {
-            get { return _chain;  }
-        }
-
         /// <summary>
         /// Create a TransformModel containing the transforms from "result" back to "input".
         /// </summary>

diff --git a/src/Microsoft.ML/CSharpApi.cs b/src/Microsoft.ML/CSharpApi.cs
@@ -11582,11 +11582,6 @@ public sealed class Output
                 /// </summary>
                 public Var<Microsoft.ML.Runtime.EntryPoints.ITransformModel> OutputModel { get; set; } = new Var<Microsoft.ML.Runtime.EntryPoints.ITransformModel>();
 
-                /// <summary>
-                /// Data
-                /// </summary>
-                public Var<Microsoft.ML.Runtime.Data.IDataView> Data { get; set; } = new Var<Microsoft.ML.Runtime.Data.IDataView>();
-
             }
         }
     }

diff --git a/src/Microsoft.ML/Models/BinaryClassificationEvaluator.cs b/src/Microsoft.ML/Models/BinaryClassificationEvaluator.cs
@@ -6,6 +6,7 @@
 using Microsoft.ML.Runtime.Data;
 using Microsoft.ML.Runtime.EntryPoints;
 using Microsoft.ML.Transforms;
+using System.Collections.Generic;
 
 namespace Microsoft.ML.Models
 {
@@ -23,7 +24,7 @@ public sealed partial class BinaryClassificationEvaluator
         /// <returns>
         /// A BinaryClassificationMetrics instance that describes how well the model performed against the test data.
         /// </returns>
-        public BinaryClassificationMetrics Evaluate(PredictionModel model, ILearningPipelineLoader testData)
+        public List<BinaryClassificationMetrics> Evaluate(PredictionModel model, ILearningPipelineLoader testData)
         {
             using (var environment = new TlcEnvironment())
             {

diff --git a/src/Microsoft.ML/Models/BinaryClassificationMetrics.cs b/src/Microsoft.ML/Models/BinaryClassificationMetrics.cs
@@ -6,6 +6,7 @@
 using Microsoft.ML.Runtime.Api;
 using Microsoft.ML.Runtime.Data;
 using System;
+using System.Collections.Generic;
 
 namespace Microsoft.ML.Models
 {
@@ -18,7 +19,7 @@ private BinaryClassificationMetrics()
         {
         }
 
-        internal static BinaryClassificationMetrics FromMetrics(IHostEnvironment env, IDataView overallMetrics, IDataView confusionMatrix)
+        internal static List<BinaryClassificationMetrics> FromMetrics(IHostEnvironment env, IDataView overallMetrics, IDataView confusionMatrix)
         {
             Contracts.AssertValue(env);
             env.AssertValue(overallMetrics);
@@ -31,28 +32,37 @@ internal static BinaryClassificationMetrics FromMetrics(IHostEnvironment env, ID
                 throw env.Except("The overall RegressionMetrics didn't have any rows.");
             }
 
-            SerializationClass metrics = enumerator.Current;
-
-            if (enumerator.MoveNext())
-            {
-                throw env.Except("The overall RegressionMetrics contained more than 1 row.");
-            }
-
-            return new BinaryClassificationMetrics()
+            List<BinaryClassificationMetrics> metrics = new List<BinaryClassificationMetrics>();
+            var confusionMatrices = ConfusionMatrix.Create(env, confusionMatrix).GetEnumerator();
+            do
             {
-                Auc = metrics.Auc,
-                Accuracy = metrics.Accuracy,
-                PositivePrecision = metrics.PositivePrecision,
-                PositiveRecall = metrics.PositiveRecall,
-                NegativePrecision = metrics.NegativePrecision,
-                NegativeRecall = metrics.NegativeRecall,
-                LogLoss = metrics.LogLoss,
-                LogLossReduction = metrics.LogLossReduction,
-                Entropy = metrics.Entropy,
-                F1Score = metrics.F1Score,
-                Auprc = metrics.Auprc,
-                ConfusionMatrix = ConfusionMatrix.Create(env, confusionMatrix),
-            };
+                SerializationClass metric = enumerator.Current;
+
+                if (!confusionMatrices.MoveNext())
+                {
+                    throw env.Except("Confusion matrices didn't have enough matrices.");
+                }
+
+                metrics.Add( 
+                    new BinaryClassificationMetrics()
+                {
+                    Auc = metric.Auc,
+                    Accuracy = metric.Accuracy,
+                    PositivePrecision = metric.PositivePrecision,
+                    PositiveRecall = metric.PositiveRecall,
+                    NegativePrecision = metric.NegativePrecision,
+                    NegativeRecall = metric.NegativeRecall,
+                    LogLoss = metric.LogLoss,
+                    LogLossReduction = metric.LogLossReduction,
+                    Entropy = metric.Entropy,
+                    F1Score = metric.F1Score,
+                    Auprc = metric.Auprc,
+                    ConfusionMatrix = confusionMatrices.Current,
+                });
+
+            } while (enumerator.MoveNext());
+
+            return metrics;
         }
 
         /// <summary>

diff --git a/src/Microsoft.ML/Models/ClassificationEvaluator.cs b/src/Microsoft.ML/Models/ClassificationEvaluator.cs
@@ -5,6 +5,7 @@
 using Microsoft.ML.Runtime;
 using Microsoft.ML.Runtime.Data;
 using Microsoft.ML.Transforms;
+using System.Collections.Generic;
 
 namespace Microsoft.ML.Models
 {
@@ -23,7 +24,7 @@ public sealed partial class ClassificationEvaluator
         /// <returns>
         /// A ClassificationMetrics instance that describes how well the model performed against the test data.
         /// </returns>
-        public ClassificationMetrics Evaluate(PredictionModel model, ILearningPipelineLoader testData)
+        public List<ClassificationMetrics> Evaluate(PredictionModel model, ILearningPipelineLoader testData)
         {
             using (var environment = new TlcEnvironment())
             {

diff --git a/src/Microsoft.ML/Models/ClassificationMetrics.cs b/src/Microsoft.ML/Models/ClassificationMetrics.cs
@@ -5,6 +5,7 @@
 using Microsoft.ML.Runtime;
 using Microsoft.ML.Runtime.Api;
 using Microsoft.ML.Runtime.Data;
+using System.Collections.Generic;
 
 namespace Microsoft.ML.Models
 {
@@ -17,7 +18,7 @@ private ClassificationMetrics()
         {
         }
 
-        internal static ClassificationMetrics FromMetrics(IHostEnvironment env, IDataView overallMetrics, IDataView confusionMatrix)
+        internal static List<ClassificationMetrics> FromMetrics(IHostEnvironment env, IDataView overallMetrics, IDataView confusionMatrix)
         {
             Contracts.AssertValue(env);
             env.AssertValue(overallMetrics);
@@ -29,24 +30,32 @@ internal static ClassificationMetrics FromMetrics(IHostEnvironment env, IDataVie
             {
                 throw env.Except("The overall RegressionMetrics didn't have any rows.");
             }
-
-            SerializationClass metrics = enumerator.Current;
-
-            if (enumerator.MoveNext())
-            {
-                throw env.Except("The overall RegressionMetrics contained more than 1 row.");
-            }
-
-            return new ClassificationMetrics()
+
+            List<ClassificationMetrics> metrics = new List<ClassificationMetrics>();
+            var confusionMatrices = ConfusionMatrix.Create(env, confusionMatrix).GetEnumerator();
+            do
             {
-                AccuracyMicro = metrics.AccuracyMicro,
-                AccuracyMacro = metrics.AccuracyMacro,
-                LogLoss = metrics.LogLoss,
-                LogLossReduction = metrics.LogLossReduction,
-                TopKAccuracy = metrics.TopKAccuracy,
-                PerClassLogLoss = metrics.PerClassLogLoss,
-                ConfusionMatrix = ConfusionMatrix.Create(env, confusionMatrix)
-            };
+                if (!confusionMatrices.MoveNext())
+                {
+                    throw env.Except("Confusion matrices didn't have enough matrices.");
+                }
+
+                SerializationClass metric = enumerator.Current;
+                metrics.Add(
+                    new ClassificationMetrics()
+                    {
+                        AccuracyMicro = metric.AccuracyMicro,
+                        AccuracyMacro = metric.AccuracyMacro,
+                        LogLoss = metric.LogLoss,
+                        LogLossReduction = metric.LogLossReduction,
+                        TopKAccuracy = metric.TopKAccuracy,
+                        PerClassLogLoss = metric.PerClassLogLoss,
+                        ConfusionMatrix = confusionMatrices.Current
+                    });
+
+            } while (enumerator.MoveNext());
+
+            return metrics;
         }
 
         /// <summary>