From 7bfc77d49f22c42bebedfe1adeea47d370262b03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Isma=C3=ABl=20Mej=C3=ADa?= Date: Mon, 3 Jul 2023 14:58:10 +0200 Subject: [PATCH] [SPARK-44277][BUILD] Upgrade to Avro 1.11.2 --- .../main/scala/org/apache/spark/sql/avro/AvroOptions.scala | 4 ++-- dev/deps/spark-deps-hadoop-3-hive-2.3 | 6 +++--- docs/sql-data-sources-avro.md | 4 ++-- pom.xml | 2 +- project/SparkBuild.scala | 2 +- .../org/apache/spark/sql/hive/client/HiveClientSuite.scala | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/connector/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala b/connector/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala index c8057ca58798e..edaaa8835cc01 100644 --- a/connector/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala +++ b/connector/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala @@ -81,14 +81,14 @@ private[sql] class AvroOptions( /** * Top level record name in write result, which is required in Avro spec. - * See https://avro.apache.org/docs/1.11.1/specification/#schema-record . + * See https://avro.apache.org/docs/1.11.2/specification/#schema-record . * Default value is "topLevelRecord" */ val recordName: String = parameters.getOrElse(RECORD_NAME, "topLevelRecord") /** * Record namespace in write result. Default value is "". - * See Avro spec for details: https://avro.apache.org/docs/1.11.1/specification/#schema-record . + * See Avro spec for details: https://avro.apache.org/docs/1.11.2/specification/#schema-record . */ val recordNamespace: String = parameters.getOrElse(RECORD_NAMESPACE, "") diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index 3ac03fa64727f..ddd367696bb5e 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -21,9 +21,9 @@ arrow-memory-core/12.0.1//arrow-memory-core-12.0.1.jar arrow-memory-netty/12.0.1//arrow-memory-netty-12.0.1.jar arrow-vector/12.0.1//arrow-vector-12.0.1.jar audience-annotations/0.5.0//audience-annotations-0.5.0.jar -avro-ipc/1.11.1//avro-ipc-1.11.1.jar -avro-mapred/1.11.1//avro-mapred-1.11.1.jar -avro/1.11.1//avro-1.11.1.jar +avro-ipc/1.11.2//avro-ipc-1.11.2.jar +avro-mapred/1.11.2//avro-mapred-1.11.2.jar +avro/1.11.2//avro-1.11.2.jar aws-java-sdk-bundle/1.12.367//aws-java-sdk-bundle-1.12.367.jar azure-data-lake-store-sdk/2.3.9//azure-data-lake-store-sdk-2.3.9.jar azure-keyvault-core/1.0.0//azure-keyvault-core-1.0.0.jar diff --git a/docs/sql-data-sources-avro.md b/docs/sql-data-sources-avro.md index 977886a6f34e2..b01174b918245 100644 --- a/docs/sql-data-sources-avro.md +++ b/docs/sql-data-sources-avro.md @@ -417,7 +417,7 @@ applications. Read the [Advanced Dependency Management](https://spark.apache Submission Guide for more details. ## Supported types for Avro -> Spark SQL conversion -Currently Spark supports reading all [primitive types](https://avro.apache.org/docs/1.11.1/specification/#primitive-types) and [complex types](https://avro.apache.org/docs/1.11.1/specification/#complex-types) under records of Avro. +Currently Spark supports reading all [primitive types](https://avro.apache.org/docs/1.11.2/specification/#primitive-types) and [complex types](https://avro.apache.org/docs/1.11.2/specification/#complex-types) under records of Avro. @@ -481,7 +481,7 @@ In addition to the types listed above, it supports reading `union` types. The fo 3. `union(something, null)`, where something is any supported Avro type. This will be mapped to the same Spark SQL type as that of something, with nullable set to true. All other union types are considered complex. They will be mapped to StructType where field names are member0, member1, etc., in accordance with members of the union. This is consistent with the behavior when converting between Avro and Parquet. -It also supports reading the following Avro [logical types](https://avro.apache.org/docs/1.11.1/specification/#logical-types): +It also supports reading the following Avro [logical types](https://avro.apache.org/docs/1.11.2/specification/#logical-types):
Avro typeSpark SQL type
diff --git a/pom.xml b/pom.xml index deccc904dd910..a7bd218f441af 100644 --- a/pom.xml +++ b/pom.xml @@ -154,7 +154,7 @@ --> 4.2.18 - 1.11.1 + 1.11.21.12.01.11.655 diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 7f9da32224f70..8f2f5d7878794 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -1110,7 +1110,7 @@ object DependencyOverrides { dependencyOverrides += "com.google.guava" % "guava" % guavaVersion, dependencyOverrides += "xerces" % "xercesImpl" % "2.12.2", dependencyOverrides += "jline" % "jline" % "2.14.6", - dependencyOverrides += "org.apache.avro" % "avro" % "1.11.1") + dependencyOverrides += "org.apache.avro" % "avro" % "1.11.2") } /** diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala index 6f799bbe7d3f5..bd1b5b557896b 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala @@ -900,7 +900,7 @@ class HiveClientSuite(version: String, allVersions: Seq[String]) test("Decimal support of Avro Hive serde") { val tableName = "tab1" // TODO: add the other logical types. For details, see the link: - // https://avro.apache.org/docs/1.11.1/specification/#logical-types + // https://avro.apache.org/docs/1.11.2/specification/#logical-types val avroSchema = """{ | "name": "test_record",
Avro logical typeAvro typeSpark SQL type