Skip to content

Commit

Permalink
fix failed UT
Browse files Browse the repository at this point in the history
  • Loading branch information
AngersZhuuuu committed Dec 29, 2020
1 parent 6a7438b commit d3b9cec
Show file tree
Hide file tree
Showing 2 changed files with 1 addition and 41 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -304,7 +304,7 @@ abstract class BaseScriptTransformationSuite extends SparkPlanTest with SQLTestU
child = child,
ioschema = defaultIOSchema
),
df.select('a, 'b.cast("string"), 'c.cast("string"), 'd.cast("string"), 'e).collect())
df.select('a, 'b, 'c, 'd, 'e).collect())
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,44 +59,4 @@ class SparkScriptTransformationSuite extends BaseScriptTransformationSuite with
assert(e.contains("TRANSFORM with serde is only supported in hive mode"))
}
}

test("SPARK-32106: TRANSFORM doesn't support ArrayType/MapType/StructType " +
"as output data type (no serde)") {
assume(TestUtils.testCommandAvailable("/bin/bash"))
// check for ArrayType
val e1 = intercept[SparkException] {
sql(
"""
|SELECT TRANSFORM(a)
|USING 'cat' AS (a array<int>)
|FROM VALUES (array(1, 1), map('1', 1), struct(1, 'a')) t(a, b, c)
""".stripMargin).collect()
}.getMessage
assert(e1.contains("SparkScriptTransformation without serde does not support" +
" ArrayType as output data type"))

// check for MapType
val e2 = intercept[SparkException] {
sql(
"""
|SELECT TRANSFORM(b)
|USING 'cat' AS (b map<int, string>)
|FROM VALUES (array(1, 1), map('1', 1), struct(1, 'a')) t(a, b, c)
""".stripMargin).collect()
}.getMessage
assert(e2.contains("SparkScriptTransformation without serde does not support" +
" MapType as output data type"))

// check for StructType
val e3 = intercept[SparkException] {
sql(
"""
|SELECT TRANSFORM(c)
|USING 'cat' AS (c struct<col1:int, col2:string>)
|FROM VALUES (array(1, 1), map('1', 1), struct(1, 'a')) t(a, b, c)
""".stripMargin).collect()
}.getMessage
assert(e3.contains("SparkScriptTransformation without serde does not support" +
" StructType as output data type"))
}
}

0 comments on commit d3b9cec

Please sign in to comment.