FeatureHasher

src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/BucketizerTests.cs

@@ -13,19 +13,24 @@
 namespace Microsoft.Spark.E2ETest.IpcTests.ML.Feature
 {
     [Collection("Spark E2E Tests")]
-    public class BucketizerTests
+    public class BucketizerTests : FeatureBaseTests<Bucketizer>
     {
         private readonly SparkSession _spark;
 
-        public BucketizerTests(SparkFixture fixture)
+        public BucketizerTests(SparkFixture fixture) : base(fixture)
         {
             _spark = fixture.Spark;
         }
 
+        /// <summary>
+        /// Create a <see cref="DataFrame"/>, create a <see cref="Bucketizer"/> and test the
+        /// available methods. Test the FeatureBase methods using <see cref="FeatureBaseTests"/>.
+        /// </summary>
         [Fact]
         public void TestBucketizer()
         {
-            var expectedSplits = new double[] { double.MinValue, 0.0, 10.0, 50.0, double.MaxValue };
+            var expectedSplits =
+                new double[] { double.MinValue, 0.0, 10.0, 50.0, double.MaxValue };
 
             string expectedHandle = "skip";
             string expectedUid = "uid";
@@ -60,18 +65,7 @@ public void TestBucketizer()
                 Assert.Equal(bucketizer.Uid(), loadedBucketizer.Uid());
             }
 
-            Assert.NotEmpty(bucketizer.ExplainParams());
-
-            Param handleInvalidParam = bucketizer.GetParam("handleInvalid");
-            Assert.NotEmpty(handleInvalidParam.Doc);
-            Assert.NotEmpty(handleInvalidParam.Name);
-            Assert.Equal(handleInvalidParam.Parent, bucketizer.Uid());
-
-            Assert.NotEmpty(bucketizer.ExplainParam(handleInvalidParam));
-            bucketizer.Set(handleInvalidParam, "keep");
-            Assert.Equal("keep", bucketizer.GetHandleInvalid());
-
-            Assert.Equal("error", bucketizer.Clear(handleInvalidParam).GetHandleInvalid());
+            TestFeatureBase(bucketizer, "handleInvalid", "keep");
         }
 
         [Fact]

src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/CountVectorizerModelTests.cs

@@ -12,17 +12,17 @@
 namespace Microsoft.Spark.E2ETest.IpcTests.ML.Feature
 {
     [Collection("Spark E2E Tests")]
-    public class CountVectorizerModelTests
+    public class CountVectorizerModelTests : FeatureBaseTests<CountVectorizerModel>
     {
         private readonly SparkSession _spark;
 
-        public CountVectorizerModelTests(SparkFixture fixture)
+        public CountVectorizerModelTests(SparkFixture fixture) : base(fixture)
         {
             _spark = fixture.Spark;
         }
 
         /// <summary>
-        /// Test that we can create a CountVectorizerModel, pass in a specifc vocabulary to use
+        /// Test that we can create a CountVectorizerModel, pass in a specific vocabulary to use
         /// when creating the model. Verify the standard features methods as well as load/save.
         /// </summary>
         [Fact]
@@ -68,6 +68,8 @@ public void TestCountVectorizerModel()
             Assert.IsType<int>(countVectorizerModel.GetVocabSize());
             Assert.NotEmpty(countVectorizerModel.ExplainParams());
             Assert.NotEmpty(countVectorizerModel.ToString());
+
+            TestFeatureBase(countVectorizerModel, "minDF", 100);
         } 
     }
 }

src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/CountVectorizerTests.cs

@@ -13,11 +13,11 @@
 namespace Microsoft.Spark.E2ETest.IpcTests.ML.Feature
 {
     [Collection("Spark E2E Tests")]
-    public class CountVectorizerTests
+    public class CountVectorizerTests : FeatureBaseTests<CountVectorizer>
     {
         private readonly SparkSession _spark;
 
-        public CountVectorizerTests(SparkFixture fixture)
+        public CountVectorizerTests(SparkFixture fixture) : base(fixture)
         {
             _spark = fixture.Spark;
         }
@@ -67,6 +67,8 @@ public void TestCountVectorizer()
 
             Assert.NotEmpty(countVectorizer.ExplainParams());
             Assert.NotEmpty(countVectorizer.ToString());
+
+            TestFeatureBase(countVectorizer, "minDF", 0.4);
         }
 
         /// <summary>

src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/FeatureBaseTests.cs

@@ -0,0 +1,46 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using Microsoft.Spark.ML.Feature;
+using Microsoft.Spark.ML.Feature.Param;
+using Microsoft.Spark.Sql;
+using Xunit;
+
+namespace Microsoft.Spark.E2ETest.IpcTests.ML.Feature
+{
+    public class FeatureBaseTests<T>
+    {
+        private readonly SparkSession _spark;
+
+        protected FeatureBaseTests(SparkFixture fixture)
+        {
+            _spark = fixture.Spark;
+        }
+
+        /// <summary>
+        /// Tests the common functionality across all ML.Feature classes.
+        /// </summary>
+        /// <param name="testObject">The object that implemented FeatureBase</param>
+        /// <param name="paramName">The name of a parameter that can be set on this object</param>
+        /// <param name="paramValue">A parameter value that can be set on this object</param>
+        public void TestFeatureBase(
+            FeatureBase<T> testObject, 
+            string paramName, 
+            object paramValue)
+        {
+            Assert.NotEmpty(testObject.ExplainParams());
+
+            Param param = testObject.GetParam(paramName);
+            Assert.NotEmpty(param.Doc);
+            Assert.NotEmpty(param.Name);
+            Assert.Equal(param.Parent, testObject.Uid());
+
+            Assert.NotEmpty(testObject.ExplainParam(param));
+            testObject.Set(param, paramValue);
+            Assert.IsAssignableFrom<Identifiable>(testObject.Clear(param));
+
+            Assert.IsType<string>(testObject.Uid());
+        }
+    }
+}

src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/FeatureHasherTests.cs

@@ -0,0 +1,61 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using Microsoft.Spark.ML.Feature;
+using Microsoft.Spark.Sql;
+using Microsoft.Spark.Sql.Types;
+using Xunit;
+
+namespace Microsoft.Spark.E2ETest.IpcTests.ML.Feature
+{
+    [Collection("Spark E2E Tests")]
+    public class FeatureHasherTests : FeatureBaseTests<FeatureHasher>
+    {
+        private readonly SparkSession _spark;
+
+        public FeatureHasherTests(SparkFixture fixture) : base(fixture)
+        {
+            _spark = fixture.Spark;
+        }
+
+        /// <summary>
+        /// Create a <see cref="DataFrame"/>, create a <see cref="FeatureHasher"/> and test the
+        /// available methods. Test the FeatureBase methods using <see cref="FeatureBaseTests"/>.
+        /// </summary>
+        [Fact]
+        public void TestFeatureHasher()
+        {
+            DataFrame dataFrame = _spark.CreateDataFrame(
+                new List<GenericRow>
+                {
+                    new GenericRow(new object[] { 2.0D, true, "1", "foo" }),
+                    new GenericRow(new object[] { 3.0D, false, "2", "bar" })
+                },
+                new StructType(new List<StructField>
+                {
+                    new StructField("real", new DoubleType()),
+                    new StructField("bool", new BooleanType()),
+                    new StructField("stringNum", new StringType()),
+                    new StructField("string", new StringType())
+                }));
+
+            FeatureHasher hasher = new FeatureHasher()
+                .SetInputCols(new List<string>() { "real", "bool", "stringNum", "string" })
+                .SetOutputCol("features")
+                .SetCategoricalCols(new List<string>() { "real", "string" })
+                .SetNumFeatures(10);
+
+            Assert.IsType<string>(hasher.GetOutputCol());
+            Assert.IsType<string[]>(hasher.GetInputCols());
+            Assert.IsType<string[]>(hasher.GetCategoricalCols());
+            Assert.IsType<int>(hasher.GetNumFeatures());
+            Assert.IsType<StructType>(hasher.TransformSchema(dataFrame.Schema()));
+            Assert.IsType<DataFrame>(hasher.Transform(dataFrame));
+
+            TestFeatureBase(hasher, "numFeatures", 1000);
+        }
+    }
+}

src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/HashingTFTests.cs

@@ -11,11 +11,11 @@
 namespace Microsoft.Spark.E2ETest.IpcTests.ML.Feature
 {
     [Collection("Spark E2E Tests")]
-    public class HashingTFTests
+    public class HashingTFTests : FeatureBaseTests<HashingTF>
     {
         private readonly SparkSession _spark;
 
-        public HashingTFTests(SparkFixture fixture)
+        public HashingTFTests(SparkFixture fixture) : base(fixture)
         {
             _spark = fixture.Spark;
         }
@@ -57,6 +57,8 @@ public void TestHashingTF()
 
             hashingTf.SetBinary(true);
             Assert.True(hashingTf.GetBinary());
+
+            TestFeatureBase(hashingTf, "numFeatures", 1000);
         }
     }
 }

src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/IDFModelTests.cs

@@ -11,11 +11,11 @@
 namespace Microsoft.Spark.E2ETest.IpcTests.ML.Feature
 {
     [Collection("Spark E2E Tests")]
-    public class IDFModelTests
+    public class IDFModelTests : FeatureBaseTests<IDFModel>
     {
         private readonly SparkSession _spark;
 
-        public IDFModelTests(SparkFixture fixture)
+        public IDFModelTests(SparkFixture fixture) : base(fixture)
         {
             _spark = fixture.Spark;
         }
@@ -65,6 +65,8 @@ public void TestIDFModel()
                 IDFModel loadedModel = IDFModel.Load(modelPath);
                 Assert.Equal(idfModel.Uid(), loadedModel.Uid());
             }
+
+            TestFeatureBase(idfModel, "minDocFreq", 1000);
         }
     }
 }

src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/IDFTests.cs

@@ -11,11 +11,11 @@
 namespace Microsoft.Spark.E2ETest.IpcTests.ML.Feature
 {
     [Collection("Spark E2E Tests")]
-    public class IDFTests
+    public class IDFTests : FeatureBaseTests<IDF>
     {
         private readonly SparkSession _spark;
 
-        public IDFTests(SparkFixture fixture)
+        public IDFTests(SparkFixture fixture) : base(fixture)
         {
             _spark = fixture.Spark;
         }
@@ -44,6 +44,8 @@ public void TestIDFModel()
                 IDF loadedIdf = IDF.Load(savePath);
                 Assert.Equal(idf.Uid(), loadedIdf.Uid());
             }
+
+            TestFeatureBase(idf, "minDocFreq", 1000);
         }
     }
 }

src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/TokenizerTests.cs

@@ -11,11 +11,11 @@
 namespace Microsoft.Spark.E2ETest.IpcTests.ML.Feature
 {
     [Collection("Spark E2E Tests")]
-    public class TokenizerTests
+    public class TokenizerTests : FeatureBaseTests<Tokenizer>
     {
         private readonly SparkSession _spark;
 
-        public TokenizerTests(SparkFixture fixture)
+        public TokenizerTests(SparkFixture fixture) : base(fixture)
         {
             _spark = fixture.Spark;
         }
@@ -50,6 +50,8 @@ public void TestTokenizer()
             }
 
             Assert.Equal(expectedUid, tokenizer.Uid());
+
+            TestFeatureBase(tokenizer, "inputCol", "input_col");
         }
     }
 }

src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/Word2VecModelTests.cs

@@ -11,11 +11,11 @@
 namespace Microsoft.Spark.E2ETest.IpcTests.ML.Feature
 {
     [Collection("Spark E2E Tests")]
-    public class Word2VecModelTests
+    public class Word2VecModelTests : FeatureBaseTests<Word2VecModel>
     {
         private readonly SparkSession _spark;
 
-        public Word2VecModelTests(SparkFixture fixture)
+        public Word2VecModelTests(SparkFixture fixture) : base(fixture)
         {
             _spark = fixture.Spark;
         }
@@ -47,6 +47,8 @@ public void TestWord2VecModel()
                 Word2VecModel loadedModel = Word2VecModel.Load(savePath);
                 Assert.Equal(model.Uid(), loadedModel.Uid());
             }
+
+            TestFeatureBase(model, "maxIter", 2);
         }
     }
 }

src/csharp/Microsoft.Spark.E2ETest/IpcTests/ML/Feature/Word2VecTests.cs

@@ -11,11 +11,11 @@
 namespace Microsoft.Spark.E2ETest.IpcTests.ML.Feature
 {
     [Collection("Spark E2E Tests")]
-    public class Word2VecTests
+    public class Word2VecTests : FeatureBaseTests<Word2Vec>
     {
         private readonly SparkSession _spark;
 
-        public Word2VecTests(SparkFixture fixture)
+        public Word2VecTests(SparkFixture fixture) : base(fixture)
         {
             _spark = fixture.Spark;
         }
@@ -67,6 +67,8 @@ public void TestWord2Vec()
                 Word2Vec loadedWord2Vec = Word2Vec.Load(savePath);
                 Assert.Equal(word2vec.Uid(), loadedWord2Vec.Uid());
             }
+
+            TestFeatureBase(word2vec, "maxIter", 2);
         }
     }
 }