diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala index 5f415d77ed268..c5347218c4b40 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/DataSourceUtils.scala @@ -61,7 +61,8 @@ object DataSourceUtils { case BooleanType | ByteType | ShortType | IntegerType | LongType | FloatType | DoubleType | StringType | BinaryType | DateType | TimestampType | _: DecimalType => - case _: CalendarIntervalType | _: StructType | _: ArrayType | _: MapType + // All the unsupported types for CSV + case _: NullType | _: CalendarIntervalType | _: StructType | _: ArrayType | _: MapType if format.isInstanceOf[CSVFileFormat] => throwUnsupportedException(dataType) @@ -73,23 +74,27 @@ object DataSourceUtils { verifyType(keyType) verifyType(valueType) + case udt: UserDefinedType[_] => verifyType(udt.sqlType) + + // Interval type not supported in all the write path + case _: CalendarIntervalType if !isReadPath => + throwUnsupportedException(dataType) + // JSON and ORC don't support an Interval type, but we pass it in read pass // for back-compatibility. - case _: CalendarIntervalType if isReadPath && - (format.isInstanceOf[JsonFileFormat] | format.isInstanceOf[OrcFileFormat]) => + case _: CalendarIntervalType if format.isInstanceOf[JsonFileFormat] || + format.isInstanceOf[OrcFileFormat] => - case udt: UserDefinedType[_] => verifyType(udt.sqlType) + // Interval type not supported in the other read path + case _: CalendarIntervalType => + throwUnsupportedException(dataType) - // For JSON backward-compatibility - case NullType if format.isInstanceOf[JsonFileFormat] || + // For JSON & ORC backward-compatibility + case _: NullType if format.isInstanceOf[JsonFileFormat] || (isReadPath && format.isInstanceOf[OrcFileFormat]) => - // Actually we won't pass in unsupported data types below, this is a safety check - case _: CalendarIntervalType if format.isInstanceOf[JsonFileFormat] => - throwUnsupportedException(dataType) - - case _: CalendarIntervalType | _: NullType - if format.isInstanceOf[ParquetFileFormat] || format.isInstanceOf[OrcFileFormat] => + // Null type not supported in the other path + case _: NullType => throwUnsupportedException(dataType) // We keep this default case for safeguards