From 1a3b63dff2d234946419f8ed09afe9150b3904c0 Mon Sep 17 00:00:00 2001 From: cafreeman Date: Wed, 8 Apr 2015 14:12:50 -0500 Subject: [PATCH] new version of `CreateDF` New version uses takes a `StructType` from R and creates a DataFrame. Commented out the `tojson` version since we don't currently use it. --- .../main/scala/org/apache/spark/sql/api/r/SQLUtils.scala | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala index bd5080d5de283..e09bd25788310 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/api/r/SQLUtils.scala @@ -66,6 +66,15 @@ private[r] object SQLUtils { StructField(name, dtObj, nullable) } + // cfreeman: I don't think we need this anymore since we can pass structType from R + // def createDF(rdd: RDD[Array[Byte]], schemaString: String, sqlContext: SQLContext): DataFrame = { + // val schema = DataType.fromJson(schemaString).asInstanceOf[StructType] + // val num = schema.fields.size + // val rowRDD = rdd.map(bytesToRow) + // sqlContext.createDataFrame(rowRDD, schema) + // } + + def createDF(rdd: RDD[Array[Byte]], schema: StructType, sqlContext: SQLContext): DataFrame = { val num = schema.fields.size val rowRDD = rdd.map(bytesToRow) sqlContext.createDataFrame(rowRDD, schema)