format code

maropu · Jul 17, 2020 · d37ef86 · d37ef86
1 parent ce8a0a5
commit d37ef86
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 37 deletions.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
@@ -91,17 +91,11 @@ trait BaseScriptTransformationExec extends UnaryExecNode {
       hadoopConf: Configuration): Iterator[InternalRow]
 
   protected def processOutputWithoutSerde(prevLine: String, reader: BufferedReader): InternalRow = {
-    if (!ioschema.schemaLess) {
-      new GenericInternalRow(
-        prevLine.split(ioschema.outputRowFormatMap("TOK_TABLEROWFORMATFIELD"))
-          .zip(fieldWriters)
-          .map { case (data, writer) => writer(data) })
-    } else {
-      new GenericInternalRow(
-        prevLine.split(ioschema.outputRowFormatMap("TOK_TABLEROWFORMATFIELD"), 2)
-          .zip(fieldWriters)
-          .map { case (data, writer) => writer(data) })
-    }
+    val limit = if (ioschema.schemaLess) 2 else 0
+    new GenericInternalRow(
+      prevLine.split(ioschema.outputRowFormatMap("TOK_TABLEROWFORMATFIELD"), limit)
+        .zip(fieldWriters)
+        .map { case (data, writer) => writer(data) })
   }
 
   protected def checkFailureAndPropagate(
@@ -273,6 +267,18 @@ object ScriptTransformationIOSchema {
     ("TOK_TABLEROWFORMATLINES", "\n")
   )
 
+  val defaultIOSchema = ScriptTransformationIOSchema(
+    inputRowFormat = Seq.empty,
+    outputRowFormat = Seq.empty,
+    inputSerdeClass = None,
+    outputSerdeClass = None,
+    inputSerdeProps = Seq.empty,
+    outputSerdeProps = Seq.empty,
+    recordReaderClass = None,
+    recordWriterClass = None,
+    schemaLess = false
+  )
+
   def apply(input: ScriptInputOutputSchema): ScriptTransformationIOSchema = {
     ScriptTransformationIOSchema(
       input.inputRowFormat,

diff --git a/...ve/src/test/scala/org/apache/spark/sql/hive/execution/BaseScriptTransformationSuite.scala b/...ve/src/test/scala/org/apache/spark/sql/hive/execution/BaseScriptTransformationSuite.scala
@@ -44,7 +44,7 @@ abstract class BaseScriptTransformationSuite extends SparkPlanTest with SQLTestU
 
   import spark.implicits._
 
-  var noSerdeIOSchema: ScriptTransformationIOSchema = _
+  var noSerdeIOSchema: ScriptTransformationIOSchema = ScriptTransformationIOSchema.defaultIOSchema
 
   private var defaultUncaughtExceptionHandler: Thread.UncaughtExceptionHandler = _
 

diff --git a/...ve/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala b/...ve/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala
@@ -33,18 +33,6 @@ class HiveScriptTransformationSuite extends BaseScriptTransformationSuite {
 
   import spark.implicits._
 
-  noSerdeIOSchema = ScriptTransformationIOSchema(
-    inputRowFormat = Seq.empty,
-    outputRowFormat = Seq.empty,
-    inputSerdeClass = None,
-    outputSerdeClass = None,
-    inputSerdeProps = Seq.empty,
-    outputSerdeProps = Seq.empty,
-    recordReaderClass = None,
-    recordWriterClass = None,
-    schemaLess = false
-  )
-
   private val serdeIOSchema: ScriptTransformationIOSchema = {
     noSerdeIOSchema.copy(
       inputSerdeClass = Some(classOf[LazySimpleSerDe].getCanonicalName),

diff --git a/...e/src/test/scala/org/apache/spark/sql/hive/execution/SparkScriptTransformationSuite.scala b/...e/src/test/scala/org/apache/spark/sql/hive/execution/SparkScriptTransformationSuite.scala
@@ -21,7 +21,7 @@ import java.sql.{Date, Timestamp}
 
 import org.apache.spark.TestUtils
 import org.apache.spark.sql.catalyst.expressions.AttributeReference
-import org.apache.spark.sql.execution.{ScriptTransformationIOSchema, SparkPlan}
+import org.apache.spark.sql.execution.SparkPlan
 import org.apache.spark.sql.functions.struct
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.CalendarInterval
@@ -32,18 +32,6 @@ class SparkScriptTransformationSuite extends BaseScriptTransformationSuite {
 
   override def scriptType: String = "SPARK"
 
-  noSerdeIOSchema = ScriptTransformationIOSchema(
-    inputRowFormat = Seq.empty,
-    outputRowFormat = Seq.empty,
-    inputSerdeClass = None,
-    outputSerdeClass = None,
-    inputSerdeProps = Seq.empty,
-    outputSerdeProps = Seq.empty,
-    recordReaderClass = None,
-    recordWriterClass = None,
-    schemaLess = false
-  )
-
   test("SPARK-32106: SparkScriptTransformExec should handle different data types correctly") {
     assume(TestUtils.testCommandAvailable("python"))
     case class Struct(d: Int, str: String)