From 08d97c879412a68b199473034e11eb026cdf690c Mon Sep 17 00:00:00 2001
From: angerszhu <angers.zhu@gmail.com>
Date: Wed, 22 Jul 2020 18:02:24 +0800
Subject: [PATCH] throw exception when complex data type

---
 .../BaseScriptTransformationExec.scala        |  3 ++
 .../SparkScriptTransformationSuite.scala      | 41 ++++++++++++++++++-
 2 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
index ba5bc83af512f..3d44e2596fbae 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala
@@ -216,6 +216,9 @@ trait BaseScriptTransformationExec extends UnaryExecNode {
         converter)
       case udt: UserDefinedType[_] =>
         wrapperConvertException(data => udt.deserialize(data), converter)
+      case ArrayType(_, _) | MapType(_, _, _) | StructType(_) =>
+        throw new SparkException("TRANSFORM without serde don't support" +
+          " ArrayType/MapType/StructType as output data type")
       case _ => wrapperConvertException(data => data, converter)
     }
   }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkScriptTransformationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkScriptTransformationSuite.scala
index 183390a846056..0826ecceec931 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkScriptTransformationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkScriptTransformationSuite.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.execution
 
-import org.apache.spark.TestUtils
+import org.apache.spark.{SparkException, TestUtils}
 import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression}
 import org.apache.spark.sql.catalyst.parser.ParseException
 import org.apache.spark.sql.test.SharedSparkSession
@@ -61,4 +61,43 @@ class SparkScriptTransformationSuite extends BaseScriptTransformationSuite with
       assert(e.contains("TRANSFORM with serde is only supported in hive mode"))
     }
   }
+
+  test("TRANSFORM don't support ArrayType/MapType/StructType as output data type (no serde)") {
+    assume(TestUtils.testCommandAvailable("/bin/bash"))
+    // check for ArrayType
+    val e1 = intercept[SparkException] {
+      sql(
+        """
+          |SELECT TRANSFORM(a)
+          |USING 'cat' AS (a array<int>)
+          |FROM VALUES (array(1, 1), map('1', 1), struct(1, 'a')) t(a, b, c)
+        """.stripMargin).collect()
+    }.getMessage
+    assert(e1.contains("TRANSFORM without serde don't support" +
+      " ArrayType/MapType/StructType as output data type"))
+
+    // check for MapType
+    val e2 = intercept[SparkException] {
+      sql(
+        """
+          |SELECT TRANSFORM(b)
+          |USING 'cat' AS (b map<int, string>)
+          |FROM VALUES (array(1, 1), map('1', 1), struct(1, 'a')) t(a, b, c)
+        """.stripMargin).collect()
+    }.getMessage
+    assert(e2.contains("TRANSFORM without serde don't support" +
+      " ArrayType/MapType/StructType as output data type"))
+
+    // check for StructType
+    val e3 = intercept[SparkException] {
+      sql(
+        """
+          |SELECT TRANSFORM(c)
+          |USING 'cat' AS (c struct<col1:int, col2:string>)
+          |FROM VALUES (array(1, 1), map('1', 1), struct(1, 'a')) t(a, b, c)
+        """.stripMargin).collect()
+    }.getMessage
+    assert(e3.contains("TRANSFORM without serde don't support" +
+      " ArrayType/MapType/StructType as output data type"))
+  }
 }