Merge pull request #1 from yinxusen/SomilBranch1.33

review result

examples/src/main/java/org/apache/spark/examples/ml/JavaBinarizer.java → examples/src/main/java/org/apache/spark/examples/ml/JavaBinarizerExample.java

@@ -17,43 +17,39 @@
 
 package org.apache.spark.examples.ml;
 
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.sql.SQLContext;
+
+// $example on$
 import java.util.Arrays;
 
-import org.apache.spark.SparkConf;
 import org.apache.spark.api.java.JavaRDD;
-import org.apache.spark.api.java.JavaSparkContext;
 import org.apache.spark.ml.feature.Binarizer;
 import org.apache.spark.sql.DataFrame;
 import org.apache.spark.sql.Row;
 import org.apache.spark.sql.RowFactory;
-import org.apache.spark.sql.SQLContext;
 import org.apache.spark.sql.types.DataTypes;
 import org.apache.spark.sql.types.Metadata;
 import org.apache.spark.sql.types.StructField;
 import org.apache.spark.sql.types.StructType;
+// $example off$
 
-/**
- * An example demonstrating a Binarizer.
- * Run with
- * <pre>
- * bin/run-example ml.JavaBinarizer <file> <k>
- * </pre>
- */
-public class JavaBinarizer {
-
+public class JavaBinarizerExample {
   public static void main(String[] args) {
-    SparkConf conf = new SparkConf().setAppName("JavaBinarizer");
+    SparkConf conf = new SparkConf().setAppName("JavaBinarizerExample");
     JavaSparkContext jsc = new JavaSparkContext(conf);
     SQLContext jsql = new SQLContext(jsc);
 
+    // $example on$
     JavaRDD<Row> jrdd = jsc.parallelize(Arrays.asList(
-        RowFactory.create(0, 0.1),
-        RowFactory.create(1, 0.8),
-        RowFactory.create(2, 0.2)
+      RowFactory.create(0, 0.1),
+      RowFactory.create(1, 0.8),
+      RowFactory.create(2, 0.2)
     ));
     StructType schema = new StructType(new StructField[]{
-        new StructField("label", DataTypes.DoubleType, false, Metadata.empty()),
-        new StructField("feature", DataTypes.DoubleType, false, Metadata.empty())
+      new StructField("label", DataTypes.DoubleType, false, Metadata.empty()),
+      new StructField("feature", DataTypes.DoubleType, false, Metadata.empty())
     });
     DataFrame continuousDataFrame = jsql.createDataFrame(jrdd, schema);
     Binarizer binarizer = new Binarizer()
@@ -66,5 +62,7 @@ public static void main(String[] args) {
     Double binarized_value = r.getDouble(0);
       System.out.println(binarized_value);
     }
+    // $example off$
+    jsc.stop();
   }
 }

examples/src/main/java/org/apache/spark/examples/ml/JavaBucketizer.java → examples/src/main/java/org/apache/spark/examples/ml/JavaBucketizerExample.java

@@ -17,57 +17,54 @@
 
 package org.apache.spark.examples.ml;
 
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.sql.SQLContext;
+
+// $example on$
 import java.util.Arrays;
 
-import org.apache.spark.SparkConf;
 import org.apache.spark.api.java.JavaRDD;
-import org.apache.spark.api.java.JavaSparkContext;
 import org.apache.spark.ml.feature.Bucketizer;
 import org.apache.spark.sql.DataFrame;
 import org.apache.spark.sql.Row;
 import org.apache.spark.sql.RowFactory;
-import org.apache.spark.sql.SQLContext;
 import org.apache.spark.sql.types.DataTypes;
 import org.apache.spark.sql.types.Metadata;
 import org.apache.spark.sql.types.StructField;
 import org.apache.spark.sql.types.StructType;
+// $example off$
 
-/**
- * An example demonstrating a Bucketizer.
- * Run with
- * <pre>
- * bin/run-example ml.JavaBucketizer <file> <k>
- * </pre>
- */
-public class JavaBucketizer {
-
+public class JavaBucketizerExample {
   public static void main(String[] args) {
-    SparkConf conf = new SparkConf().setAppName("JavaBucketizer");
+    SparkConf conf = new SparkConf().setAppName("JavaBucketizerExample");
     JavaSparkContext jsc = new JavaSparkContext(conf);
     SQLContext jsql = new SQLContext(jsc);
 
+    // $example on$
     double[] splits = {Double.NEGATIVE_INFINITY, -0.5, 0.0, 0.5, Double.POSITIVE_INFINITY};
 
     JavaRDD<Row> data = jsc.parallelize(Arrays.asList(
-        RowFactory.create(-0.5),
-        RowFactory.create(-0.3),
-        RowFactory.create(0.0),
-        RowFactory.create(0.2)
+      RowFactory.create(-0.5),
+      RowFactory.create(-0.3),
+      RowFactory.create(0.0),
+      RowFactory.create(0.2)
     ));
     StructType schema = new StructType(new StructField[]{
-        new StructField("features", DataTypes.DoubleType, false, Metadata.empty())
+      new StructField("features", DataTypes.DoubleType, false, Metadata.empty())
     });
     DataFrame dataFrame = jsql.createDataFrame(data, schema);
 
     Bucketizer bucketizer = new Bucketizer()
-        .setInputCol("features")
-        .setOutputCol("bucketedFeatures")
-        .setSplits(splits);
+      .setInputCol("features")
+      .setOutputCol("bucketedFeatures")
+      .setSplits(splits);
 
     // Transform original data into its bucket index.
     DataFrame bucketedData = bucketizer.transform(dataFrame);
-
-    }
+    // $example off$
+    jsc.stop();
+  }
 }
 
 

examples/src/main/java/org/apache/spark/examples/ml/JavaDCT.java → examples/src/main/java/org/apache/spark/examples/ml/JavaDCTExample.java

@@ -17,51 +17,49 @@
 
 package org.apache.spark.examples.ml;
 
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.sql.SQLContext;
+
+// $example on$
 import java.util.Arrays;
 
-import org.apache.spark.SparkConf;
 import org.apache.spark.api.java.JavaRDD;
-import org.apache.spark.api.java.JavaSparkContext;
 import org.apache.spark.ml.feature.DCT;
 import org.apache.spark.mllib.linalg.VectorUDT;
 import org.apache.spark.mllib.linalg.Vectors;
 import org.apache.spark.sql.DataFrame;
 import org.apache.spark.sql.Row;
 import org.apache.spark.sql.RowFactory;
-import org.apache.spark.sql.SQLContext;
 import org.apache.spark.sql.types.Metadata;
 import org.apache.spark.sql.types.StructField;
 import org.apache.spark.sql.types.StructType;
+// $example off$
 
-/**
- * An example demonstrating a discrete cosine transform.
- * Run with
- * <pre>
- * bin/run-example ml.JavaDCT <file> <k>
- * </pre>
- */
-public class JavaDCT {
-
+public class JavaDCTExample {
   public static void main(String[] args) {
-    SparkConf conf = new SparkConf().setAppName("JavaDCT");
+    SparkConf conf = new SparkConf().setAppName("JavaDCTExample");
     JavaSparkContext jsc = new JavaSparkContext(conf);
     SQLContext jsql = new SQLContext(jsc);
 
+    // $example on$
     JavaRDD<Row> data = jsc.parallelize(Arrays.asList(
-        RowFactory.create(Vectors.dense(0.0, 1.0, -2.0, 3.0)),
-        RowFactory.create(Vectors.dense(-1.0, 2.0, 4.0, -7.0)),
-        RowFactory.create(Vectors.dense(14.0, -2.0, -5.0, 1.0))
+      RowFactory.create(Vectors.dense(0.0, 1.0, -2.0, 3.0)),
+      RowFactory.create(Vectors.dense(-1.0, 2.0, 4.0, -7.0)),
+      RowFactory.create(Vectors.dense(14.0, -2.0, -5.0, 1.0))
     ));
     StructType schema = new StructType(new StructField[]{
-        new StructField("features", new VectorUDT(), false, Metadata.empty()),
+      new StructField("features", new VectorUDT(), false, Metadata.empty()),
     });
     DataFrame df = jsql.createDataFrame(data, schema);
     DCT dct = new DCT()
-        .setInputCol("features")
-        .setOutputCol("featuresDCT")
-        .setInverse(false);
+      .setInputCol("features")
+      .setOutputCol("featuresDCT")
+      .setInverse(false);
     DataFrame dctDf = dct.transform(df);
-        dctDf.select("featuresDCT").show(3);
+    dctDf.select("featuresDCT").show(3);
+    // $example off$
+    jsc.stop();
   }
 }
 

examples/src/main/java/org/apache/spark/examples/ml/JavaElementwiseProduct.java → examples/src/main/java/org/apache/spark/examples/ml/JavaElementwiseProductExample.java

@@ -17,54 +17,58 @@
 
 package org.apache.spark.examples.ml;
 
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.sql.SQLContext;
+
+// $example on$
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
-import org.apache.spark.SparkConf;
 import org.apache.spark.api.java.JavaRDD;
-import org.apache.spark.api.java.JavaSparkContext;
 import org.apache.spark.ml.feature.ElementwiseProduct;
 import org.apache.spark.mllib.linalg.Vector;
 import org.apache.spark.mllib.linalg.Vectors;
 import org.apache.spark.sql.DataFrame;
 import org.apache.spark.sql.Row;
 import org.apache.spark.sql.RowFactory;
-import org.apache.spark.sql.SQLContext;
 import org.apache.spark.sql.types.DataTypes;
 import org.apache.spark.sql.types.StructField;
 import org.apache.spark.sql.types.StructType;
+// $example off$
 
-/**
- * An example demonstrating a Element wise Product.
- * Run with
- * <pre>
- * bin/run-example ml.JavaElementwiseProduct <file> <k>
- * </pre>
- */
-public class JavaElementwiseProduct {
-
+public class JavaElementwiseProductExample {
   public static void main(String[] args) {
-    SparkConf conf = new SparkConf().setAppName("JavaElementwiseProduct");
+    SparkConf conf = new SparkConf().setAppName("JavaElementwiseProductExample");
     JavaSparkContext jsc = new JavaSparkContext(conf);
     SQLContext sqlContext = new SQLContext(jsc);
 
+    // $example on$
     // Create some vector data; also works for sparse vectors
     JavaRDD<Row> jrdd = jsc.parallelize(Arrays.asList(
-        RowFactory.create("a", Vectors.dense(1.0, 2.0, 3.0)),
-        RowFactory.create("b", Vectors.dense(4.0, 5.0, 6.0))
+      RowFactory.create("a", Vectors.dense(1.0, 2.0, 3.0)),
+      RowFactory.create("b", Vectors.dense(4.0, 5.0, 6.0))
     ));
+
     List<StructField> fields = new ArrayList<StructField>(2);
     fields.add(DataTypes.createStructField("id", DataTypes.StringType, false));
     fields.add(DataTypes.createStructField("vector", DataTypes.StringType, false));
+
     StructType schema = DataTypes.createStructType(fields);
+
     DataFrame dataFrame = sqlContext.createDataFrame(jrdd, schema);
+
     Vector transformingVector = Vectors.dense(0.0, 1.0, 2.0);
+
     ElementwiseProduct transformer = new ElementwiseProduct()
-        .setScalingVec(transformingVector)
-        .setInputCol("vector")
-        .setOutputCol("transformedVector");
+      .setScalingVec(transformingVector)
+      .setInputCol("vector")
+      .setOutputCol("transformedVector");
+
     // Batch transform the vectors to create new column:
     transformer.transform(dataFrame).show();
+    // $example off$
+    jsc.stop();
   }
 }

examples/src/main/java/org/apache/spark/examples/ml/JavaMinMaxScalerExample.java

@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.examples.ml;
+
+import org.apache.spark.SparkConf;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.sql.SQLContext;
+
+// $example on$
+import org.apache.spark.ml.feature.MinMaxScaler;
+import org.apache.spark.ml.feature.MinMaxScalerModel;
+import org.apache.spark.sql.DataFrame;
+// $example off$
+
+public class JavaMinMaxScalerExample {
+  public static void main(String[] args) {
+    SparkConf conf = new SparkConf().setAppName("JaveMinMaxScalerExample");
+    JavaSparkContext jsc = new JavaSparkContext(conf);
+    SQLContext jsql = new SQLContext(jsc);
+
+    // $example on$
+    DataFrame dataFrame = jsql.read().format("libsvm").load("data/mllib/sample_libsvm_data.txt");
+    MinMaxScaler scaler = new MinMaxScaler()
+      .setInputCol("features")
+      .setOutputCol("scaledFeatures");
+
+    // Compute summary statistics and generate MinMaxScalerModel
+    MinMaxScalerModel scalerModel = scaler.fit(dataFrame);
+
+    // rescale each feature to range [min, max].
+    DataFrame scaledData = scalerModel.transform(dataFrame);
+    // $example off$
+    jsc.stop();
+  }
+}