diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala index cd2acf50ab468..08ef02e88ba8f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala @@ -2631,8 +2631,9 @@ class Dataset[T] private[sql]( * @since 2.0.0 */ def toJSON: Dataset[String] = { - val rowSchema = this.schema - val rdd: RDD[String] = queryExecution.toRdd.mapPartitions { iter => + val rowSchema = schema + import sparkSession.implicits.newStringEncoder + mapPartitions { iter => val writer = new CharArrayWriter() // create the Generator without separator inserted between 2 records val gen = new JacksonGenerator(rowSchema, writer) @@ -2640,7 +2641,7 @@ class Dataset[T] private[sql]( new Iterator[String] { override def hasNext: Boolean = iter.hasNext override def next(): String = { - gen.write(iter.next()) + gen.write(exprEnc.toRow(iter.next())) gen.flush() val json = writer.toString @@ -2654,8 +2655,6 @@ class Dataset[T] private[sql]( } } } - import sparkSession.implicits.newStringEncoder - sparkSession.createDataset(rdd) } /** diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonFileFormat.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonFileFormat.scala index 5a409c04c929d..2e37952fc86ef 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonFileFormat.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JsonFileFormat.scala @@ -156,6 +156,16 @@ class JsonFileFormat extends TextBasedFileFormat with DataSourceRegister { override def equals(other: Any): Boolean = other.isInstanceOf[JsonFileFormat] } +/** + * Implementation of OutputWriter for Json file format. Uses [[JacksonGenerator]] + * to do json conversion. + * + * @param path output location for the job + * @param options json generator options + * @param bucketId bucket for this partition + * @param dataSchema JacksonGenerator factory + * @param context hadoop task context + */ private[json] class JsonOutputWriter( path: String, options: JSONOptions,