Skip to content

Commit

Permalink
Suggested sample nits (feel free to ask for any of these to be revert…
Browse files Browse the repository at this point in the history 
…ed) (dotnet#219)
  • Loading branch information
@daholste @Dmitry-A
daholste authored and Dmitry-A committed Aug 22, 2019
1 parent e7a57a1 commit bf4ece8
Show file tree
Hide file tree
Showing 12 changed files with 71 additions and 148 deletions.
4 changes: 2 additions & 2 deletions src/Microsoft.ML.Auto/API/BinaryClassificationExperiment.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ namespace Microsoft.ML.Auto
{
public sealed class BinaryExperimentSettings : ExperimentSettings
{
public IProgress<RunResult<BinaryClassificationMetrics>> ProgressCallback;
public IProgress<RunResult<BinaryClassificationMetrics>> ProgressHandler;
public BinaryClassificationMetric OptimizingMetric = BinaryClassificationMetric.Accuracy;
public BinaryClassificationTrainer[] WhitelistedTrainers;
}
Expand Down Expand Up @@ -85,7 +85,7 @@ internal IEnumerable<RunResult<BinaryClassificationMetrics>> Execute(MLContext c

// run autofit & get all pipelines run in that process
var experiment = new Experiment<BinaryClassificationMetrics>(context, TaskKind.BinaryClassification, trainData, columnInfo,
validationData, preFeaturizers, new OptimizingMetricInfo(_settings.OptimizingMetric), _settings.ProgressCallback,
validationData, preFeaturizers, new OptimizingMetricInfo(_settings.OptimizingMetric), _settings.ProgressHandler,
_settings, new BinaryMetricsAgent(_settings.OptimizingMetric),
TrainerExtensionUtil.GetTrainerNames(_settings.WhitelistedTrainers));

Expand Down
4 changes: 2 additions & 2 deletions src/Microsoft.ML.Auto/API/MulticlassClassificationExperiment.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ namespace Microsoft.ML.Auto
{
public sealed class MulticlassExperimentSettings : ExperimentSettings
{
public IProgress<RunResult<MultiClassClassifierMetrics>> ProgressCallback;
public IProgress<RunResult<MultiClassClassifierMetrics>> ProgressHandler;
public MulticlassClassificationMetric OptimizingMetric = MulticlassClassificationMetric.AccuracyMicro;
public MulticlassClassificationTrainer[] WhitelistedTrainers;
}
Expand Down Expand Up @@ -84,7 +84,7 @@ internal IEnumerable<RunResult<MultiClassClassifierMetrics>> Execute(MLContext c
// run autofit & get all pipelines run in that process
var experiment = new Experiment<MultiClassClassifierMetrics>(context, TaskKind.MulticlassClassification, trainData,
columnInfo, validationData, preFeaturizers, new OptimizingMetricInfo(_settings.OptimizingMetric),
_settings.ProgressCallback, _settings, new MultiMetricsAgent(_settings.OptimizingMetric),
_settings.ProgressHandler, _settings, new MultiMetricsAgent(_settings.OptimizingMetric),
TrainerExtensionUtil.GetTrainerNames(_settings.WhitelistedTrainers));

return experiment.Execute();
Expand Down
4 changes: 2 additions & 2 deletions src/Microsoft.ML.Auto/API/RegressionExperiment.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ namespace Microsoft.ML.Auto
{
public sealed class RegressionExperimentSettings : ExperimentSettings
{
public IProgress<RunResult<RegressionMetrics>> ProgressCallback;
public IProgress<RunResult<RegressionMetrics>> ProgressHandler;
public RegressionMetric OptimizingMetric = RegressionMetric.RSquared;
public RegressionTrainer[] WhitelistedTrainers;
}
Expand Down Expand Up @@ -81,7 +81,7 @@ internal IEnumerable<RunResult<RegressionMetrics>> Execute(MLContext context,
// run autofit & get all pipelines run in that process
var experiment = new Experiment<RegressionMetrics>(context, TaskKind.Regression, trainData, columnInfo,
validationData, preFeaturizers, new OptimizingMetricInfo(_settings.OptimizingMetric),
_settings.ProgressCallback, _settings, new RegressionMetricsAgent(_settings.OptimizingMetric),
_settings.ProgressHandler, _settings, new RegressionMetricsAgent(_settings.OptimizingMetric),
TrainerExtensionUtil.GetTrainerNames(_settings.WhitelistedTrainers));

return experiment.Execute();
Expand Down
22 changes: 10 additions & 12 deletions src/Samples/AutoTrainBinaryClassification.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,7 @@
// See the LICENSE file in the project root for more information.

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Microsoft.Data.DataView;
using Microsoft.ML;
using Microsoft.ML.Auto;
Expand All @@ -19,40 +17,40 @@ public class AutoTrainBinaryClassification
private static string TrainDataPath = $"{BaseDatasetsLocation}/wikipedia-detox-250-line-data.tsv";
private static string TestDataPath = $"{BaseDatasetsLocation}/wikipedia-detox-250-line-test.tsv";
private static string ModelPath = $"{BaseDatasetsLocation}/SentimentModel.zip";
private static string LabelColumnName = "Sentiment";
private static string LabelColumn = "Sentiment";

public static void Run()
{
MLContext mlContext = new MLContext();

// STEP 1: Infer columns
var columnInference = mlContext.AutoInference().InferColumns(TrainDataPath, LabelColumnName);
var columnInference = mlContext.AutoInference().InferColumns(TrainDataPath, LabelColumn);

// STEP 2: Load data
TextLoader textLoader = mlContext.Data.CreateTextLoader(columnInference.TextLoaderArgs);
IDataView trainDataView = textLoader.Read(TrainDataPath);
IDataView testDataView = textLoader.Read(TestDataPath);

// STEP 3: Auto featurize, auto train and auto hyperparameter tuning
Console.WriteLine($"Invoking BinaryClassification.AutoFit");
var autoFitResults = mlContext.AutoInference()
// STEP 3: Auto featurize, auto train and auto hyperparameter tune
Console.WriteLine($"Invoking new AutoML binary classification experiment...");
var runResults = mlContext.AutoInference()
.CreateBinaryClassificationExperiment(60)
.Execute(trainDataView, LabelColumnName);
.Execute(trainDataView, LabelColumn);

// STEP 4: Print metric from the best model
var best = autoFitResults.Best();
var best = runResults.Best();
Console.WriteLine($"Accuracy of best model from validation data: {best.Metrics.Accuracy}");

// STEP 5: Evaluate test data
IDataView testDataViewWithBestScore = best.Model.Transform(testDataView);
var testMetrics = mlContext.BinaryClassification.EvaluateNonCalibrated(testDataViewWithBestScore, label: LabelColumnName, DefaultColumnNames.Score);
Console.WriteLine($"Accuracy of best model from test data: {best.Metrics.Accuracy}");
var testMetrics = mlContext.BinaryClassification.EvaluateNonCalibrated(testDataViewWithBestScore, label: LabelColumn);
Console.WriteLine($"Accuracy of best model on test data: {best.Metrics.Accuracy}");

// STEP 6: Save the best model for later deployment and inferencing
using (var fs = File.Create(ModelPath))
best.Model.SaveTo(mlContext, fs);

Console.WriteLine("Press any key to continue..");
Console.WriteLine("Press any key to continue...");
Console.ReadLine();
}
}
Expand Down
19 changes: 8 additions & 11 deletions src/Samples/AutoTrainMulticlassClassification.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,7 @@
// See the LICENSE file in the project root for more information.

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Microsoft.Data.DataView;
using Microsoft.ML;
using Microsoft.ML.Auto;
Expand All @@ -19,40 +17,39 @@ public class AutoTrainMulticlassClassification
private static string TrainDataPath = $"{BaseDatasetsLocation}/iris-train.txt";
private static string TestDataPath = $"{BaseDatasetsLocation}/iris-test.txt";
private static string ModelPath = $"{BaseDatasetsLocation}/IrisClassificationModel.zip";
private static string LabelColumnName = "Label";

public static void Run()
{
MLContext mlContext = new MLContext();

// STEP 1: Infer columns
var columnInference = mlContext.AutoInference().InferColumns(TrainDataPath, LabelColumnName);
var columnInference = mlContext.AutoInference().InferColumns(TrainDataPath);

// STEP 2: Load data
TextLoader textLoader = mlContext.Data.CreateTextLoader(columnInference.TextLoaderArgs);
IDataView trainDataView = textLoader.Read(TrainDataPath);
IDataView testDataView = textLoader.Read(TestDataPath);

// STEP 3: Auto featurize, auto train and auto hyperparameter tuning
Console.WriteLine($"Invoking MulticlassClassification.AutoFit");
var autoFitResults = mlContext.AutoInference()
// STEP 3: Auto featurize, auto train and auto hyperparameter tune
Console.WriteLine($"Invoking new AutoML multiclass classification experiment...");
var runResults = mlContext.AutoInference()
.CreateMulticlassClassificationExperiment(60)
.Execute(trainDataView);

// STEP 4: Print metric from the best model
var best = autoFitResults.Best();
var best = runResults.Best();
Console.WriteLine($"AccuracyMacro of best model from validation data: {best.Metrics.AccuracyMacro}");

// STEP 5: Evaluate test data
IDataView testDataViewWithBestScore = best.Model.Transform(testDataView);
var testMetrics = mlContext.MulticlassClassification.Evaluate(testDataViewWithBestScore, label: LabelColumnName, DefaultColumnNames.Score);
Console.WriteLine($"AccuracyMacro of best model from test data: {best.Metrics.AccuracyMacro}");
var testMetrics = mlContext.MulticlassClassification.Evaluate(testDataViewWithBestScore);
Console.WriteLine($"AccuracyMacro of best model on test data: {best.Metrics.AccuracyMacro}");

// STEP 6: Save the best model for later deployment and inferencing
using (var fs = File.Create(ModelPath))
best.Model.SaveTo(mlContext, fs);

Console.WriteLine("Press any key to continue..");
Console.WriteLine("Press any key to continue...");
Console.ReadLine();
}
}
Expand Down
26 changes: 12 additions & 14 deletions src/Samples/AutoTrainRegression.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,7 @@
// See the LICENSE file in the project root for more information.

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Microsoft.Data.DataView;
using Microsoft.ML;
using Microsoft.ML.Auto;
Expand All @@ -19,40 +17,40 @@ static class AutoTrainRegression
private static string TrainDataPath = $"{BaseDatasetsLocation}/taxi-fare-train.csv";
private static string TestDataPath = $"{BaseDatasetsLocation}/taxi-fare-test.csv";
private static string ModelPath = $"{BaseDatasetsLocation}/TaxiFareModel.zip";
private static string LabelColumnName = "fare_amount";
private static string LabelColumn = "fare_amount";

public static void Run()
{
MLContext mlContext = new MLContext();

// STEP 1: Common data loading configuration
var columnInference = mlContext.AutoInference().InferColumns(TrainDataPath, LabelColumnName);
// STEP 1: Infer columns
var columnInference = mlContext.AutoInference().InferColumns(TrainDataPath, LabelColumn);

// STEP 2: Load data
TextLoader textLoader = mlContext.Data.CreateTextLoader(columnInference.TextLoaderArgs);
IDataView trainDataView = textLoader.Read(TrainDataPath);
IDataView testDataView = textLoader.Read(TestDataPath);

// STEP 3: Auto featurize, auto train and auto hyperparameter tuning
Console.WriteLine($"Invoking Regression.AutoFit");
var autoFitResults = mlContext.AutoInference()
// STEP 3: Auto featurize, auto train and auto hyperparameter tune
Console.WriteLine($"Invoking new AutoML regression experiment...");
var runResults = mlContext.AutoInference()
.CreateRegressionExperiment(0)
.Execute(trainDataView, LabelColumnName);
.Execute(trainDataView, LabelColumn);

// STEP 4: Compare and print actual value vs predicted value for top 5 rows from validation data
var best = autoFitResults.Best();
// STEP 4: Print metric from best model
var best = runResults.Best();
Console.WriteLine($"RSquared of best model from validation data: {best.Metrics.RSquared}");

// STEP 5: Evaluate test data
IDataView testDataViewWithBestScore = best.Model.Transform(testDataView);
var testMetrics = mlContext.Regression.Evaluate(testDataViewWithBestScore, label: LabelColumnName, DefaultColumnNames.Score);
Console.WriteLine($"RSquared of best model from test data: {best.Metrics.RSquared}");
var testMetrics = mlContext.Regression.Evaluate(testDataViewWithBestScore, label: LabelColumn);
Console.WriteLine($"RSquared of best model on test data: {best.Metrics.RSquared}");

// STEP 6: Save the best model for later deployment and inferencing
using (var fs = File.Create(ModelPath))
best.Model.SaveTo(mlContext, fs);

Console.WriteLine("Press any key to continue..");
Console.WriteLine("Press any key to continue...");
Console.ReadLine();
}
}
Expand Down
16 changes: 8 additions & 8 deletions src/Samples/Cancellation.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,14 @@ static class Cancellation
private static string TrainDataPath = $"{BaseDatasetsLocation}/taxi-fare-train.csv";
private static string TestDataPath = $"{BaseDatasetsLocation}/taxi-fare-test.csv";
private static string ModelPath = $"{BaseDatasetsLocation}/TaxiFareModel.zip";
private static string LabelColumnName = "fare_amount";
private static string LabelColumn = "fare_amount";

public static void Run()
{
MLContext mlContext = new MLContext();

// STEP 1: Infer columns
var columnInference = mlContext.AutoInference().InferColumns(TrainDataPath, LabelColumnName, ',');
var columnInference = mlContext.AutoInference().InferColumns(TrainDataPath, LabelColumn, ',');

// STEP 2: Load data
TextLoader textLoader = mlContext.Data.CreateTextLoader(columnInference.TextLoaderArgs);
Expand All @@ -39,19 +39,19 @@ public static void Run()

Stopwatch watch = Stopwatch.StartNew();

// STEP 3: Autofit with a cancellation token
Console.WriteLine($"Invoking Regression.AutoFit");
var autoFitResults = mlContext.AutoInference()
// STEP 3: Auto inference with a cancellation token
Console.WriteLine($"Invoking new AutoML regression experiment...");
var runResults = mlContext.AutoInference()
.CreateRegressionExperiment(new RegressionExperimentSettings()
{
MaxInferenceTimeInSeconds = 60,
CancellationToken = cts.Token
})
.Execute(trainDataView, LabelColumnName);
.Execute(trainDataView, LabelColumn);

Console.WriteLine($"{autoFitResults.Count()} models were returned after {cancelAfterInSeconds} seconds");
Console.WriteLine($"{runResults.Count()} models were returned after {cancelAfterInSeconds} seconds");

Console.WriteLine("Press any key to continue..");
Console.WriteLine("Press any key to continue...");
Console.ReadLine();
}
}
Expand Down
14 changes: 7 additions & 7 deletions src/Samples/CustomizeTraining.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,31 +16,31 @@ static class CustomizeTraining
private static string TrainDataPath = $"{BaseDatasetsLocation}/taxi-fare-train.csv";
private static string TestDataPath = $"{BaseDatasetsLocation}/taxi-fare-test.csv";
private static string ModelPath = $"{BaseDatasetsLocation}/TaxiFareModel.zip";
private static string LabelColumnName = "fare_amount";
private static string LabelColumn = "fare_amount";

public static void Run()
{
MLContext mlContext = new MLContext();

// STEP 1: Infer columns
var columnInference = mlContext.AutoInference().InferColumns(TrainDataPath, LabelColumnName, ',');
var columnInference = mlContext.AutoInference().InferColumns(TrainDataPath, LabelColumn, ',');

// STEP 2: Load data
TextLoader textLoader = mlContext.Data.CreateTextLoader(columnInference.TextLoaderArgs);
IDataView trainDataView = textLoader.Read(TrainDataPath);
IDataView testDataView = textLoader.Read(TestDataPath);

// STEP 3: Autofit with a callback configured
var autoFitExperiment = mlContext.AutoInference().CreateRegressionExperiment(new RegressionExperimentSettings()
// STEP 3: Auto inference with a callback configured
var autoExperiment = mlContext.AutoInference().CreateRegressionExperiment(new RegressionExperimentSettings()
{
MaxInferenceTimeInSeconds = 20,
OptimizingMetric = RegressionMetric.L2,
WhitelistedTrainers = new[] { RegressionTrainer.LightGbm },
ProgressCallback = new Progress()
ProgressHandler = new ProgressHandler()
});
autoFitExperiment.Execute(trainDataView, LabelColumnName);
autoExperiment.Execute(trainDataView, LabelColumn);

Console.WriteLine("Press any key to continue..");
Console.WriteLine("Press any key to continue...");
Console.ReadLine();
}
}
Expand Down
Loading

0 comments on commit bf4ece8

Please sign in to comment.