[GLUTEN-5936][VL] Add more types in function type validation and docu…

…ment Cast function (#6963)
apache · Aug 22, 2024 · ed3d333 · ed3d333
1 parent fbd066a
commit ed3d333
Show file tree

Hide file tree

Showing 2 changed files with 33 additions and 1 deletion.
diff --git a/docs/velox-backend-support-progress.md b/docs/velox-backend-support-progress.md
@@ -92,6 +92,34 @@ Gluten supports 28 operators (Drag to right to see all data types)
 
 Gluten supports 199 functions. (Drag to right to see all data types)
 
+#### Cast function's support status
+
+ * S: supported.
+ * NS: not supported.
+ * -: not accepted by Spark.
+ * N/A: not applicable case, e.g., from type is as same as to type, where cast will not actually happen.
+
+| From \ To | BOOLEAN | BYTE | SHORT | INT | LONG | FLOAT | DOUBLE | DECIMAL | DATE | TIMESTAMP | STRING | BINARY | ARRAY | MAP | STRUCT | NULL |
+|-----------|---------|------|-------|-----|------|-------|--------|---------|------|-----------|--------|--------|-------|-----|--------|------|
+| BOOLEAN   | N/A     | S    | S     | S   | S    | S     | S      | S       | -    | NS        | S      | -      | -     | -   | -      | -    |
+| BYTE      | S       | N/A  | S     | S   | S    | S     | S      | S       | -    | NS        | S      | S      | -     | -   | -      | -    |
+| SHORT     | S       | S    | N/A   | S   | S    | S     | S      | S       | -    | NS        | S      | S      | -     | -   | -      | -    |
+| INT       | S       | S    | S     | N/A | S    | S     | S      | S       | -    | NS        | S      | S      | -     | -   | -      | -    |
+| LONG      | S       | S    | S     | S   | N/A  | S     | S      | S       | -    | NS        | S      | S      | -     | -   | -      | -    |
+| FLOAT     | S       | S    | S     | S   | S    | N/A   | S      | S       | -    | NS        | S      | -      | -     | -   | -      | -    |
+| DOUBLE    | S       | S    | S     | S   | S    | S     | N/A    | S       | -    | NS        | S      | -      | -     | -   | -      | -    |
+| DECIMAL   | S       | S    | S     | S   | S    | S     | S      | N/A     | -    | NS        | S      | -      | -     | -   | -      | -    |
+| DATE      | NS      | NS   | NS    | NS  | NS   | NS    | NS     | NS      | N/A  | NS        | NS     | -      | -     | -   | -      | -    |
+| TIMESTAMP | NS      | NS   | NS    | NS  | NS   | NS    | NS     | NS      | NS   | N/A       | NS     | -      | -     | -   | -      | -    |
+| STRING    | S       | S    | S     | S   | S    | S     | S      | S       | NS   | NS        | N/A    | -      | -     | -   | -      | -    |
+| BINARY    | S       | S    | S     | S   | S    | S     | S      | S       | NS   | NS        | S      | N/A    | -     | -   | -      | -    |
+| ARRAY     | -       | -    | -     | -   | -    | -     | -      | -       | -    | -         | NS     | -      | N/A   | -   | -      | -    |
+| Map       | -       | -    | -     | -   | -    | -     | -      | -       | -    | -         | NS     | -      | -     | N/A | -      | -    |
+| STRUCT    | -       | -    | -     | -   | -    | -     | -      | -       | -    | -         | NS     | -      | -     | -   | N/A    | -    |
+| NULL      | S       | S    | S     | S   | S    | S     | S      | S       | S    | NS        | S      | S      | S     | S   | S      | N/A  |
+
+#### Other functions' support status 
+
 | Spark Functions              | Velox/Presto Functions | Velox/Spark functions | Gluten | Restrictions             | BOOLEAN | BYTE | SHORT | INT | LONG | FLOAT | DOUBLE | DATE | TIMESTAMP | STRING | DECIMAL | NULL | BINARY | CALENDAR | ARRAY | MAP | STRUCT | UDT |
 |------------------------------|------------------------|-----------------------|--------|--------------------------|---------|------|-------|-----|------|-------|--------|------|-----------|--------|---------|------|--------|----------|-------|-----|--------|-----|
 | !                            |                        | not                   | S      |                          | S       | S    | S     | S   | S    | S     | S      |      |           | S      |         |      |        |          |       |     |        |     |

diff --git a/gluten-ut/test/src/test/scala/org/apache/spark/sql/GlutenExpressionDataTypesValidation.scala b/gluten-ut/test/src/test/scala/org/apache/spark/sql/GlutenExpressionDataTypesValidation.scala
@@ -63,6 +63,7 @@ class GlutenExpressionDataTypesValidation extends WholeStageTransformerSuite {
 
   private val allPrimitiveDataTypes: Seq[DataType] =
     Seq(
+      BooleanType,
       ByteType,
       ShortType,
       IntegerType,
@@ -73,7 +74,8 @@ class GlutenExpressionDataTypesValidation extends WholeStageTransformerSuite {
       StringType,
       BinaryType,
       DateType,
-      TimestampType)
+      TimestampType,
+      NullType)
 
   private val allComplexDataTypes: Seq[DataType] = Seq(
     // Currently, only check certain inner types, assuming they are representative
@@ -85,6 +87,7 @@ class GlutenExpressionDataTypesValidation extends WholeStageTransformerSuite {
 
   def generateChildExpression(t: DataType): Expression = {
     t match {
+      case _: BooleanType => Literal(true, t)
       case _: IntegralType => Literal(null, t)
       case _: FractionalType => Literal(null, t)
       case StringType | BinaryType => Literal("123")
@@ -93,6 +96,7 @@ class GlutenExpressionDataTypesValidation extends WholeStageTransformerSuite {
       case ArrayType(_, _) => Literal(null, t)
       case MapType(_, _, _) => Literal(null, t)
       case StructType(_) => Literal(null, t)
+      case NullType => Literal(null, t)
       case _ => throw new UnsupportedOperationException("Not supported type: " + t)
     }
   }