diff --git a/.asf.yaml b/.asf.yaml index ae5e99cf230d8..22042b355b2fa 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -36,3 +36,4 @@ notifications: pullrequests: reviews@spark.apache.org issues: reviews@spark.apache.org commits: commits@spark.apache.org + jira_options: link label diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index c0c21c7cb6eb0..f0bd65bcf415c 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -283,7 +283,7 @@ jobs: name: test-results-${{ matrix.modules }}-${{ matrix.comment }}-${{ matrix.java }}-${{ matrix.hadoop }}-${{ matrix.hive }} path: "**/target/test-reports/*.xml" - name: Upload unit tests log files - if: failure() + if: ${{ !success() }} uses: actions/upload-artifact@v3 with: name: unit-tests-log-${{ matrix.modules }}-${{ matrix.comment }}-${{ matrix.java }}-${{ matrix.hadoop }}-${{ matrix.hive }} @@ -425,7 +425,7 @@ jobs: run: | if [[ "$MODULES_TO_TEST" != *"pyspark-ml"* ]] && [[ "$BRANCH" != "branch-3.5" ]]; then # uninstall libraries dedicated for ML testing - python3.9 -m pip uninstall -y torch torchvision torcheval torchtnt tensorboard mlflow + python3.9 -m pip uninstall -y torch torchvision torcheval torchtnt tensorboard mlflow deepspeed fi if [ -f ./dev/free_disk_space_container ]; then ./dev/free_disk_space_container @@ -470,7 +470,7 @@ jobs: name: test-results-${{ matrix.modules }}--8-${{ inputs.hadoop }}-hive2.3 path: "**/target/test-reports/*.xml" - name: Upload unit tests log files - if: failure() + if: ${{ !success() }} uses: actions/upload-artifact@v3 with: name: unit-tests-log-${{ matrix.modules }}--8-${{ inputs.hadoop }}-hive2.3 @@ -961,7 +961,7 @@ jobs: name: test-results-tpcds--8-${{ inputs.hadoop }}-hive2.3 path: "**/target/test-reports/*.xml" - name: Upload unit tests log files - if: failure() + if: ${{ !success() }} uses: actions/upload-artifact@v3 with: name: unit-tests-log-tpcds--8-${{ inputs.hadoop }}-hive2.3 @@ -1028,7 +1028,7 @@ jobs: name: test-results-docker-integration--8-${{ inputs.hadoop }}-hive2.3 path: "**/target/test-reports/*.xml" - name: Upload unit tests log files - if: failure() + if: ${{ !success() }} uses: actions/upload-artifact@v3 with: name: unit-tests-log-docker-integration--8-${{ inputs.hadoop }}-hive2.3 @@ -1103,7 +1103,7 @@ jobs: eval $(minikube docker-env) build/sbt -Psparkr -Pkubernetes -Pvolcano -Pkubernetes-integration-tests -Dspark.kubernetes.test.driverRequestCores=0.5 -Dspark.kubernetes.test.executorRequestCores=0.2 -Dspark.kubernetes.test.volcanoMaxConcurrencyJobNum=1 -Dtest.exclude.tags=local "kubernetes-integration-tests/test" - name: Upload Spark on K8S integration tests log files - if: failure() + if: ${{ !success() }} uses: actions/upload-artifact@v3 with: name: spark-on-kubernetes-it-log diff --git a/common/sketch/src/main/java/org/apache/spark/util/sketch/BloomFilter.java b/common/sketch/src/main/java/org/apache/spark/util/sketch/BloomFilter.java index f3c2b05e7af9d..172b394689ca9 100644 --- a/common/sketch/src/main/java/org/apache/spark/util/sketch/BloomFilter.java +++ b/common/sketch/src/main/java/org/apache/spark/util/sketch/BloomFilter.java @@ -178,6 +178,13 @@ public static BloomFilter readFrom(InputStream in) throws IOException { return BloomFilterImpl.readFrom(in); } + /** + * Reads in a {@link BloomFilter} from a byte array. + */ + public static BloomFilter readFrom(byte[] bytes) throws IOException { + return BloomFilterImpl.readFrom(bytes); + } + /** * Computes the optimal k (number of hashes per item inserted in Bloom filter), given the * expected insertions and total number of bits in the Bloom filter. diff --git a/common/sketch/src/main/java/org/apache/spark/util/sketch/BloomFilterImpl.java b/common/sketch/src/main/java/org/apache/spark/util/sketch/BloomFilterImpl.java index ccf1833af9945..3fba5e3325223 100644 --- a/common/sketch/src/main/java/org/apache/spark/util/sketch/BloomFilterImpl.java +++ b/common/sketch/src/main/java/org/apache/spark/util/sketch/BloomFilterImpl.java @@ -266,6 +266,12 @@ public static BloomFilterImpl readFrom(InputStream in) throws IOException { return filter; } + public static BloomFilterImpl readFrom(byte[] bytes) throws IOException { + try (ByteArrayInputStream bis = new ByteArrayInputStream(bytes)) { + return readFrom(bis); + } + } + private void writeObject(ObjectOutputStream out) throws IOException { writeTo(out); } diff --git a/common/sketch/src/test/scala/org/apache/spark/util/sketch/BloomFilterSuite.scala b/common/sketch/src/test/scala/org/apache/spark/util/sketch/BloomFilterSuite.scala index cfdc9954772c5..4d0ba66637b46 100644 --- a/common/sketch/src/test/scala/org/apache/spark/util/sketch/BloomFilterSuite.scala +++ b/common/sketch/src/test/scala/org/apache/spark/util/sketch/BloomFilterSuite.scala @@ -17,7 +17,7 @@ package org.apache.spark.util.sketch -import java.io.{ByteArrayInputStream, ByteArrayOutputStream} +import java.io.ByteArrayOutputStream import scala.reflect.ClassTag import scala.util.Random @@ -34,9 +34,7 @@ class BloomFilterSuite extends AnyFunSuite { // scalastyle:ignore funsuite filter.writeTo(out) out.close() - val in = new ByteArrayInputStream(out.toByteArray) - val deserialized = BloomFilter.readFrom(in) - in.close() + val deserialized = BloomFilter.readFrom(out.toByteArray) assert(filter == deserialized) } diff --git a/common/utils/src/main/resources/error/error-classes.json b/common/utils/src/main/resources/error/error-classes.json index 3b537cc3d9fc6..c5a63dd68b9e0 100644 --- a/common/utils/src/main/resources/error/error-classes.json +++ b/common/utils/src/main/resources/error/error-classes.json @@ -1035,6 +1035,11 @@ "Cannot safely cast to ." ] }, + "EXTRA_COLUMNS" : { + "message" : [ + "Cannot write extra columns ." + ] + }, "EXTRA_STRUCT_FIELDS" : { "message" : [ "Cannot write extra fields to the struct ." @@ -2215,6 +2220,12 @@ ], "sqlState" : "42607" }, + "NON_FOLDABLE_ARGUMENT" : { + "message" : [ + "The function requires the parameter to be a foldable expression of the type , but the actual argument is a non-foldable." + ], + "sqlState" : "22024" + }, "NON_LAST_MATCHED_CLAUSE_OMIT_CONDITION" : { "message" : [ "When there are more than one MATCHED clauses in a MERGE statement, only the last MATCHED clause can omit the condition." @@ -2687,6 +2698,18 @@ "Failed to analyze the Python user defined table function: " ] }, + "TABLE_VALUED_FUNCTION_REQUIRED_METADATA_INCOMPATIBLE_WITH_CALL" : { + "message" : [ + "Failed to evaluate the table function because its table metadata , but the function call ." + ], + "sqlState" : "22023" + }, + "TABLE_VALUED_FUNCTION_REQUIRED_METADATA_INVALID" : { + "message" : [ + "Failed to evaluate the table function because its table metadata was invalid; ." + ], + "sqlState" : "22023" + }, "TABLE_VALUED_FUNCTION_TOO_MANY_TABLE_ARGUMENTS" : { "message" : [ "There are too many table arguments for table-valued function. It allows one table argument, but got: . If you want to allow it, please set \"spark.sql.allowMultipleTableArguments.enabled\" to \"true\"" @@ -4029,11 +4052,6 @@ "() doesn't support the mode. Acceptable modes are and ." ] }, - "_LEGACY_ERROR_TEMP_1100" : { - "message" : [ - "The '' parameter of function '' needs to be a literal." - ] - }, "_LEGACY_ERROR_TEMP_1103" : { "message" : [ "Unsupported component type in arrays." diff --git a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/functions.scala b/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/functions.scala index 8ea5f07c528f7..baafdd4e17222 100644 --- a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/functions.scala +++ b/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/functions.scala @@ -987,7 +987,7 @@ object functions { * @group agg_funcs * @since 3.5.0 */ - def std(e: Column): Column = stddev(e) + def std(e: Column): Column = Column.fn("std", e) /** * Aggregate function: alias for `stddev_samp`. @@ -2337,7 +2337,7 @@ object functions { * @group math_funcs * @since 3.5.0 */ - def ceiling(e: Column, scale: Column): Column = ceil(e, scale) + def ceiling(e: Column, scale: Column): Column = Column.fn("ceiling", e, scale) /** * Computes the ceiling of the given value of `e` to 0 decimal places. @@ -2345,7 +2345,7 @@ object functions { * @group math_funcs * @since 3.5.0 */ - def ceiling(e: Column): Column = ceil(e) + def ceiling(e: Column): Column = Column.fn("ceiling", e) /** * Convert a number in a string column from one base to another. @@ -2800,7 +2800,7 @@ object functions { * @group math_funcs * @since 3.5.0 */ - def power(l: Column, r: Column): Column = pow(l, r) + def power(l: Column, r: Column): Column = Column.fn("power", l, r) /** * Returns the positive value of dividend mod divisor. @@ -2937,7 +2937,7 @@ object functions { * @group math_funcs * @since 3.5.0 */ - def sign(e: Column): Column = signum(e) + def sign(e: Column): Column = Column.fn("sign", e) /** * Computes the signum of the given value. @@ -4428,7 +4428,7 @@ object functions { * @since 3.5.0 */ def printf(format: Column, arguments: Column*): Column = - Column.fn("format_string", lit(format) +: arguments: _*) + Column.fn("printf", (format +: arguments): _*) /** * Decodes a `str` in 'application/x-www-form-urlencoded' format using a specific encoding diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala index df416ef93d83d..c457f26921358 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/PlanGenerationTestSuite.scala @@ -3235,16 +3235,19 @@ class PlanGenerationTestSuite private val testDescFilePath: String = s"${IntegrationTestUtils.sparkHome}/connector/" + "connect/common/src/test/resources/protobuf-tests/common.desc" - test("from_protobuf messageClassName") { - binary.select( - pbFn.from_protobuf(fn.col("bytes"), "org.apache.spark.sql.protobuf.protos.TestProtoObj")) + // TODO(SPARK-45030): Re-enable this test when all Maven test scenarios succeed and there + // are no other negative impacts. For the problem description, please refer to SPARK-45029 + ignore("from_protobuf messageClassName") { + binary.select(pbFn.from_protobuf(fn.col("bytes"), classOf[StorageLevel].getName)) } - test("from_protobuf messageClassName options") { + // TODO(SPARK-45030): Re-enable this test when all Maven test scenarios succeed and there + // are no other negative impacts. For the problem description, please refer to SPARK-45029 + ignore("from_protobuf messageClassName options") { binary.select( pbFn.from_protobuf( fn.col("bytes"), - "org.apache.spark.sql.protobuf.protos.TestProtoObj", + classOf[StorageLevel].getName, Map("recursive.fields.max.depth" -> "2").asJava)) } diff --git a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CheckConnectJvmClientCompatibility.scala b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CheckConnectJvmClientCompatibility.scala index 1e536cd37fec1..bf512ed71fd3f 100644 --- a/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CheckConnectJvmClientCompatibility.scala +++ b/connector/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/client/CheckConnectJvmClientCompatibility.scala @@ -208,6 +208,8 @@ object CheckConnectJvmClientCompatibility { // functions ProblemFilters.exclude[Problem]("org.apache.spark.sql.functions.unwrap_udt"), ProblemFilters.exclude[Problem]("org.apache.spark.sql.functions.udaf"), + ProblemFilters.exclude[DirectMissingMethodProblem]( + "org.apache.spark.sql.functions.try_reflect"), // KeyValueGroupedDataset ProblemFilters.exclude[Problem]( diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/describe.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/describe.explain index f205f7ef7a140..b203f715c71a6 100644 --- a/connector/connect/common/src/test/resources/query-tests/explain-results/describe.explain +++ b/connector/connect/common/src/test/resources/query-tests/explain-results/describe.explain @@ -1,6 +1,6 @@ Project [summary#0, element_at(id#0, summary#0, None, false) AS id#0, element_at(b#0, summary#0, None, false) AS b#0] +- Project [id#0, b#0, summary#0] +- Generate explode([count,mean,stddev,min,max]), false, [summary#0] - +- Aggregate [map(cast(count as string), cast(count(id#0L) as string), cast(mean as string), cast(avg(id#0L) as string), cast(stddev as string), cast(stddev_samp(cast(id#0L as double)) as string), cast(min as string), cast(min(id#0L) as string), cast(max as string), cast(max(id#0L) as string)) AS id#0, map(cast(count as string), cast(count(b#0) as string), cast(mean as string), cast(avg(b#0) as string), cast(stddev as string), cast(stddev_samp(b#0) as string), cast(min as string), cast(min(b#0) as string), cast(max as string), cast(max(b#0) as string)) AS b#0] + +- Aggregate [map(cast(count as string), cast(count(id#0L) as string), cast(mean as string), cast(avg(id#0L) as string), cast(stddev as string), cast(stddev(cast(id#0L as double)) as string), cast(min as string), cast(min(id#0L) as string), cast(max as string), cast(max(id#0L) as string)) AS id#0, map(cast(count as string), cast(count(b#0) as string), cast(mean as string), cast(avg(b#0) as string), cast(stddev as string), cast(stddev(b#0) as string), cast(min as string), cast(min(b#0) as string), cast(max as string), cast(max(b#0) as string)) AS b#0] +- Project [id#0L, b#0] +- LocalRelation , [id#0L, a#0, b#0] diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/from_protobuf_messageClassName.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/from_protobuf_messageClassName.explain deleted file mode 100644 index 6f48cb090cde5..0000000000000 --- a/connector/connect/common/src/test/resources/query-tests/explain-results/from_protobuf_messageClassName.explain +++ /dev/null @@ -1,2 +0,0 @@ -Project [from_protobuf(bytes#0, org.apache.spark.sql.protobuf.protos.TestProtoObj, None) AS from_protobuf(bytes)#0] -+- LocalRelation , [id#0L, bytes#0] diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/from_protobuf_messageClassName_options.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/from_protobuf_messageClassName_options.explain deleted file mode 100644 index ba87e4774f1af..0000000000000 --- a/connector/connect/common/src/test/resources/query-tests/explain-results/from_protobuf_messageClassName_options.explain +++ /dev/null @@ -1,2 +0,0 @@ -Project [from_protobuf(bytes#0, org.apache.spark.sql.protobuf.protos.TestProtoObj, None, (recursive.fields.max.depth,2)) AS from_protobuf(bytes)#0] -+- LocalRelation , [id#0L, bytes#0] diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/function_ceiling.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/function_ceiling.explain index 9cf776a8dbaa7..217d7434b8020 100644 --- a/connector/connect/common/src/test/resources/query-tests/explain-results/function_ceiling.explain +++ b/connector/connect/common/src/test/resources/query-tests/explain-results/function_ceiling.explain @@ -1,2 +1,2 @@ -Project [CEIL(b#0) AS CEIL(b)#0L] +Project [ceiling(b#0) AS ceiling(b)#0L] +- LocalRelation , [id#0L, a#0, b#0, d#0, e#0, f#0, g#0] diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/function_ceiling_scale.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/function_ceiling_scale.explain index cdf8d356e47dd..2c41c12278bad 100644 --- a/connector/connect/common/src/test/resources/query-tests/explain-results/function_ceiling_scale.explain +++ b/connector/connect/common/src/test/resources/query-tests/explain-results/function_ceiling_scale.explain @@ -1,2 +1,2 @@ -Project [ceil(cast(b#0 as decimal(30,15)), 2) AS ceil(b, 2)#0] +Project [ceiling(cast(b#0 as decimal(30,15)), 2) AS ceiling(b, 2)#0] +- LocalRelation , [id#0L, a#0, b#0, d#0, e#0, f#0, g#0] diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/function_java_method.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/function_java_method.explain index 0d467be225f98..d2d5730eedf9e 100644 --- a/connector/connect/common/src/test/resources/query-tests/explain-results/function_java_method.explain +++ b/connector/connect/common/src/test/resources/query-tests/explain-results/function_java_method.explain @@ -1,2 +1,2 @@ -Project [java_method(java.util.UUID, fromString, g#0) AS java_method(java.util.UUID, fromString, g)#0] +Project [java_method(java.util.UUID, fromString, g#0, true) AS java_method(java.util.UUID, fromString, g)#0] +- LocalRelation , [id#0L, a#0, b#0, d#0, e#0, f#0, g#0] diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/function_printf.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/function_printf.explain index 10409df007070..8d55d77340002 100644 --- a/connector/connect/common/src/test/resources/query-tests/explain-results/function_printf.explain +++ b/connector/connect/common/src/test/resources/query-tests/explain-results/function_printf.explain @@ -1,2 +1,2 @@ -Project [format_string(g#0, a#0, g#0) AS format_string(g, a, g)#0] +Project [printf(g#0, a#0, g#0) AS printf(g, a, g)#0] +- LocalRelation , [id#0L, a#0, b#0, d#0, e#0, f#0, g#0] diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/function_reflect.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/function_reflect.explain index f52d3e1b0ff42..df790f0878062 100644 --- a/connector/connect/common/src/test/resources/query-tests/explain-results/function_reflect.explain +++ b/connector/connect/common/src/test/resources/query-tests/explain-results/function_reflect.explain @@ -1,2 +1,2 @@ -Project [reflect(java.util.UUID, fromString, g#0) AS reflect(java.util.UUID, fromString, g)#0] +Project [reflect(java.util.UUID, fromString, g#0, true) AS reflect(java.util.UUID, fromString, g)#0] +- LocalRelation , [id#0L, a#0, b#0, d#0, e#0, f#0, g#0] diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/function_sign.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/function_sign.explain index 807fa3300836c..5d41e16b6cef4 100644 --- a/connector/connect/common/src/test/resources/query-tests/explain-results/function_sign.explain +++ b/connector/connect/common/src/test/resources/query-tests/explain-results/function_sign.explain @@ -1,2 +1,2 @@ -Project [SIGNUM(b#0) AS SIGNUM(b)#0] +Project [sign(b#0) AS sign(b)#0] +- LocalRelation , [id#0L, a#0, b#0, d#0, e#0, f#0, g#0] diff --git a/connector/connect/common/src/test/resources/query-tests/explain-results/function_std.explain b/connector/connect/common/src/test/resources/query-tests/explain-results/function_std.explain index 106191e5a32ec..cf5b86ae3a571 100644 --- a/connector/connect/common/src/test/resources/query-tests/explain-results/function_std.explain +++ b/connector/connect/common/src/test/resources/query-tests/explain-results/function_std.explain @@ -1,2 +1,2 @@ -Aggregate [stddev(cast(a#0 as double)) AS stddev(a)#0] +Aggregate [std(cast(a#0 as double)) AS std(a)#0] +- LocalRelation , [id#0L, a#0, b#0, d#0, e#0, f#0, g#0] diff --git a/connector/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName.json b/connector/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName.json deleted file mode 100644 index 6c5891e701654..0000000000000 --- a/connector/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "common": { - "planId": "1" - }, - "project": { - "input": { - "common": { - "planId": "0" - }, - "localRelation": { - "schema": "struct\u003cid:bigint,bytes:binary\u003e" - } - }, - "expressions": [{ - "unresolvedFunction": { - "functionName": "from_protobuf", - "arguments": [{ - "unresolvedAttribute": { - "unparsedIdentifier": "bytes" - } - }, { - "literal": { - "string": "org.apache.spark.sql.protobuf.protos.TestProtoObj" - } - }] - } - }] - } -} \ No newline at end of file diff --git a/connector/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName.proto.bin deleted file mode 100644 index 9d7aeaf308969..0000000000000 Binary files a/connector/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName.proto.bin and /dev/null differ diff --git a/connector/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_options.json b/connector/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_options.json deleted file mode 100644 index 691e144eabd07..0000000000000 --- a/connector/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_options.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "common": { - "planId": "1" - }, - "project": { - "input": { - "common": { - "planId": "0" - }, - "localRelation": { - "schema": "struct\u003cid:bigint,bytes:binary\u003e" - } - }, - "expressions": [{ - "unresolvedFunction": { - "functionName": "from_protobuf", - "arguments": [{ - "unresolvedAttribute": { - "unparsedIdentifier": "bytes" - } - }, { - "literal": { - "string": "org.apache.spark.sql.protobuf.protos.TestProtoObj" - } - }, { - "unresolvedFunction": { - "functionName": "map", - "arguments": [{ - "literal": { - "string": "recursive.fields.max.depth" - } - }, { - "literal": { - "string": "2" - } - }] - } - }] - } - }] - } -} \ No newline at end of file diff --git a/connector/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_options.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_options.proto.bin deleted file mode 100644 index 89000e473fac9..0000000000000 Binary files a/connector/connect/common/src/test/resources/query-tests/queries/from_protobuf_messageClassName_options.proto.bin and /dev/null differ diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_ceiling.json b/connector/connect/common/src/test/resources/query-tests/queries/function_ceiling.json index 5a9961ab47f55..99726305e8524 100644 --- a/connector/connect/common/src/test/resources/query-tests/queries/function_ceiling.json +++ b/connector/connect/common/src/test/resources/query-tests/queries/function_ceiling.json @@ -13,7 +13,7 @@ }, "expressions": [{ "unresolvedFunction": { - "functionName": "ceil", + "functionName": "ceiling", "arguments": [{ "unresolvedAttribute": { "unparsedIdentifier": "b" diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_ceiling.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_ceiling.proto.bin index 3761deb1663a2..cc91ac246a57c 100644 Binary files a/connector/connect/common/src/test/resources/query-tests/queries/function_ceiling.proto.bin and b/connector/connect/common/src/test/resources/query-tests/queries/function_ceiling.proto.bin differ diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_ceiling_scale.json b/connector/connect/common/src/test/resources/query-tests/queries/function_ceiling_scale.json index bda5e85924c30..c0b0742b12157 100644 --- a/connector/connect/common/src/test/resources/query-tests/queries/function_ceiling_scale.json +++ b/connector/connect/common/src/test/resources/query-tests/queries/function_ceiling_scale.json @@ -13,7 +13,7 @@ }, "expressions": [{ "unresolvedFunction": { - "functionName": "ceil", + "functionName": "ceiling", "arguments": [{ "unresolvedAttribute": { "unparsedIdentifier": "b" diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_ceiling_scale.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_ceiling_scale.proto.bin index 8db402ac167e0..30efc42b9d2bc 100644 Binary files a/connector/connect/common/src/test/resources/query-tests/queries/function_ceiling_scale.proto.bin and b/connector/connect/common/src/test/resources/query-tests/queries/function_ceiling_scale.proto.bin differ diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_printf.json b/connector/connect/common/src/test/resources/query-tests/queries/function_printf.json index dc7ca880c4b09..73ca595e8650b 100644 --- a/connector/connect/common/src/test/resources/query-tests/queries/function_printf.json +++ b/connector/connect/common/src/test/resources/query-tests/queries/function_printf.json @@ -13,7 +13,7 @@ }, "expressions": [{ "unresolvedFunction": { - "functionName": "format_string", + "functionName": "printf", "arguments": [{ "unresolvedAttribute": { "unparsedIdentifier": "g" diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_printf.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_printf.proto.bin index 7ebdda6cac10d..3fb3862f44d91 100644 Binary files a/connector/connect/common/src/test/resources/query-tests/queries/function_printf.proto.bin and b/connector/connect/common/src/test/resources/query-tests/queries/function_printf.proto.bin differ diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_sign.json b/connector/connect/common/src/test/resources/query-tests/queries/function_sign.json index bcf6ad7eb174d..34451969078b0 100644 --- a/connector/connect/common/src/test/resources/query-tests/queries/function_sign.json +++ b/connector/connect/common/src/test/resources/query-tests/queries/function_sign.json @@ -13,7 +13,7 @@ }, "expressions": [{ "unresolvedFunction": { - "functionName": "signum", + "functionName": "sign", "arguments": [{ "unresolvedAttribute": { "unparsedIdentifier": "b" diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_sign.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_sign.proto.bin index af52abfb7f25b..ff866c97303ed 100644 Binary files a/connector/connect/common/src/test/resources/query-tests/queries/function_sign.proto.bin and b/connector/connect/common/src/test/resources/query-tests/queries/function_sign.proto.bin differ diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_std.json b/connector/connect/common/src/test/resources/query-tests/queries/function_std.json index 1403817886ca0..cbdb4ea9e5e83 100644 --- a/connector/connect/common/src/test/resources/query-tests/queries/function_std.json +++ b/connector/connect/common/src/test/resources/query-tests/queries/function_std.json @@ -13,7 +13,7 @@ }, "expressions": [{ "unresolvedFunction": { - "functionName": "stddev", + "functionName": "std", "arguments": [{ "unresolvedAttribute": { "unparsedIdentifier": "a" diff --git a/connector/connect/common/src/test/resources/query-tests/queries/function_std.proto.bin b/connector/connect/common/src/test/resources/query-tests/queries/function_std.proto.bin index 8d214eea8e74e..7e34b0427c23b 100644 Binary files a/connector/connect/common/src/test/resources/query-tests/queries/function_std.proto.bin and b/connector/connect/common/src/test/resources/query-tests/queries/function_std.proto.bin differ diff --git a/connector/connect/server/pom.xml b/connector/connect/server/pom.xml index 34c919bc61417..e98b8da8e5c07 100644 --- a/connector/connect/server/pom.xml +++ b/connector/connect/server/pom.xml @@ -248,13 +248,6 @@ - - - kr.motd.maven - os-maven-plugin - 1.6.2 - - target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes @@ -410,87 +403,6 @@ - - - org.apache.maven.plugins - maven-antrun-plugin - - - process-test-sources - - - - - - - run - - - - - - - default-protoc - - true - - - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - 0.6.1 - - com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} - src/test/protobuf - - - - - test-compile - - - - - - - - - user-defined-protoc - - ${env.SPARK_PROTOC_EXEC_PATH} - - - - - org.xolstice.maven.plugins - protobuf-maven-plugin - 0.6.1 - - ${spark.protoc.executable.path} - src/test/protobuf - - - - - test-compile - - - - - - - - diff --git a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/artifact/SparkConnectArtifactManager.scala b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/artifact/SparkConnectArtifactManager.scala index a2df11eeb5832..fee99532bd55f 100644 --- a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/artifact/SparkConnectArtifactManager.scala +++ b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/artifact/SparkConnectArtifactManager.scala @@ -208,12 +208,14 @@ class SparkConnectArtifactManager(sessionHolder: SessionHolder) extends Logging s"sessionId: ${sessionHolder.sessionId}") // Clean up added files - sessionHolder.session.sparkContext.addedFiles.remove(state.uuid) - sessionHolder.session.sparkContext.addedArchives.remove(state.uuid) - sessionHolder.session.sparkContext.addedJars.remove(state.uuid) + val fileserver = SparkEnv.get.rpcEnv.fileServer + val sparkContext = sessionHolder.session.sparkContext + sparkContext.addedFiles.remove(state.uuid).foreach(_.keys.foreach(fileserver.removeFile)) + sparkContext.addedArchives.remove(state.uuid).foreach(_.keys.foreach(fileserver.removeFile)) + sparkContext.addedJars.remove(state.uuid).foreach(_.keys.foreach(fileserver.removeJar)) // Clean up cached relations - val blockManager = sessionHolder.session.sparkContext.env.blockManager + val blockManager = sparkContext.env.blockManager blockManager.removeCache(sessionHolder.userId, sessionHolder.sessionId) // Clean up artifacts folder diff --git a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala index fbe877b454764..579b378d09f65 100644 --- a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala +++ b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala @@ -2464,35 +2464,35 @@ class SparkConnectPlanner(val sessionHolder: SessionHolder) extends Logging { case _ => Seq.empty } - // Convert the results to Arrow. - val schema = df.schema - val maxBatchSize = (SparkEnv.get.conf.get(CONNECT_GRPC_ARROW_MAX_BATCH_SIZE) * 0.7).toLong - val timeZoneId = session.sessionState.conf.sessionLocalTimeZone - - // Convert the data. - val bytes = if (rows.isEmpty) { - ArrowConverters.createEmptyArrowBatch( - schema, - timeZoneId, - errorOnDuplicatedFieldNames = false) - } else { - val batches = ArrowConverters.toBatchWithSchemaIterator( - rowIter = rows.iterator, - schema = schema, - maxRecordsPerBatch = -1, - maxEstimatedBatchSize = maxBatchSize, - timeZoneId = timeZoneId, - errorOnDuplicatedFieldNames = false) - assert(batches.hasNext) - val bytes = batches.next() - assert(!batches.hasNext, s"remaining batches: ${batches.size}") - bytes - } - // To avoid explicit handling of the result on the client, we build the expected input // of the relation on the server. The client has to simply forward the result. val result = SqlCommandResult.newBuilder() if (isCommand) { + // Convert the results to Arrow. + val schema = df.schema + val maxBatchSize = (SparkEnv.get.conf.get(CONNECT_GRPC_ARROW_MAX_BATCH_SIZE) * 0.7).toLong + val timeZoneId = session.sessionState.conf.sessionLocalTimeZone + + // Convert the data. + val bytes = if (rows.isEmpty) { + ArrowConverters.createEmptyArrowBatch( + schema, + timeZoneId, + errorOnDuplicatedFieldNames = false) + } else { + val batches = ArrowConverters.toBatchWithSchemaIterator( + rowIter = rows.iterator, + schema = schema, + maxRecordsPerBatch = -1, + maxEstimatedBatchSize = maxBatchSize, + timeZoneId = timeZoneId, + errorOnDuplicatedFieldNames = false) + assert(batches.hasNext) + val bytes = batches.next() + assert(!batches.hasNext, s"remaining batches: ${batches.size}") + bytes + } + result.setRelation( proto.Relation .newBuilder() @@ -2900,7 +2900,9 @@ class SparkConnectPlanner(val sessionHolder: SessionHolder) extends Logging { SparkConnectService.streamingSessionManager.registerNewStreamingQuery(sessionHolder, query) // Register the runner with the query if Python foreachBatch is enabled. foreachBatchRunnerCleaner.foreach { cleaner => - sessionHolder.streamingRunnerCleanerCache.registerCleanerForQuery(query, cleaner) + sessionHolder.streamingForeachBatchRunnerCleanerCache.registerCleanerForQuery( + query, + cleaner) } executeHolder.eventsManager.postFinished() diff --git a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/service/ExecuteHolder.scala b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/service/ExecuteHolder.scala index bce0713339228..fb22893598505 100644 --- a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/service/ExecuteHolder.scala +++ b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/service/ExecuteHolder.scala @@ -172,7 +172,7 @@ private[connect] class ExecuteHolder( } } - def removeGrpcResponseSender[_](sender: ExecuteGrpcResponseSender[_]): Unit = synchronized { + def removeGrpcResponseSender(sender: ExecuteGrpcResponseSender[_]): Unit = synchronized { // if closed, we are shutting down and interrupting all senders already if (closedTime.isEmpty) { grpcResponseSenders -= diff --git a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SessionHolder.scala b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SessionHolder.scala index 2034a97fce940..1cef02d7e3466 100644 --- a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SessionHolder.scala +++ b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/service/SessionHolder.scala @@ -57,7 +57,7 @@ case class SessionHolder(userId: String, sessionId: String, session: SparkSessio new ConcurrentHashMap() // Handles Python process clean up for streaming queries. Initialized on first use in a query. - private[connect] lazy val streamingRunnerCleanerCache = + private[connect] lazy val streamingForeachBatchRunnerCleanerCache = new StreamingForeachBatchHelper.CleanerCache(this) /** Add ExecuteHolder to this session. Called only by SparkConnectExecutionManager. */ @@ -160,7 +160,8 @@ case class SessionHolder(userId: String, sessionId: String, session: SparkSessio eventManager.postClosed() // Clean up running queries SparkConnectService.streamingSessionManager.cleanupRunningQueries(this) - streamingRunnerCleanerCache.cleanUpAll() // Clean up any streaming workers. + streamingForeachBatchRunnerCleanerCache.cleanUpAll() // Clean up any streaming workers. + removeAllListeners() // removes all listener and stop python listener processes if necessary. } /** @@ -237,11 +238,21 @@ case class SessionHolder(userId: String, sessionId: String, session: SparkSessio * Spark Connect PythonStreamingQueryListener. */ private[connect] def removeCachedListener(id: String): Unit = { - listenerCache.get(id) match { - case pyListener: PythonStreamingQueryListener => pyListener.stopListenerProcess() + Option(listenerCache.remove(id)) match { + case Some(pyListener: PythonStreamingQueryListener) => pyListener.stopListenerProcess() case _ => // do nothing } - listenerCache.remove(id) + } + + /** + * Stop all streaming listener threads, and removes all python process if applicable. Only + * called when session is expired. + */ + private def removeAllListeners(): Unit = { + listenerCache.forEach((id, listener) => { + session.streams.removeListener(listener) + removeCachedListener(id) + }) } /** diff --git a/connector/connect/server/src/test/protobuf/test.proto b/connector/connect/server/src/test/protobuf/test.proto deleted file mode 100644 index 844f89ba81f47..0000000000000 --- a/connector/connect/server/src/test/protobuf/test.proto +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -syntax = "proto3"; -package org.apache.spark.sql.protobuf.protos; - -option java_multiple_files = true; -option java_outer_classname = "TestProto"; - -message TestProtoObj { - int64 v1 = 1; - int32 v2 = 2; -} diff --git a/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala b/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala index 648532faa3a1c..1e80eb66dc520 100644 --- a/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala +++ b/core/src/main/scala/org/apache/spark/metrics/ExecutorMetricType.scala @@ -110,10 +110,12 @@ case object GarbageCollectionMetrics extends ExecutorMetricType with Logging { "MinorGCTime", "MajorGCCount", "MajorGCTime", - "TotalGCTime" + "TotalGCTime", + "ConcurrentGCCount", + "ConcurrentGCTime" ) - /* We builtin some common GC collectors which categorized as young generation and old */ + /* We builtin some common GC collectors */ private[spark] val YOUNG_GENERATION_BUILTIN_GARBAGE_COLLECTORS = Seq( "Copy", "PS Scavenge", @@ -128,6 +130,8 @@ case object GarbageCollectionMetrics extends ExecutorMetricType with Logging { "G1 Old Generation" ) + private[spark] val BUILTIN_CONCURRENT_GARBAGE_COLLECTOR = "G1 Concurrent GC" + private lazy val youngGenerationGarbageCollector: Seq[String] = { SparkEnv.get.conf.get(config.EVENT_LOG_GC_METRICS_YOUNG_GENERATION_GARBAGE_COLLECTORS) } @@ -147,6 +151,9 @@ case object GarbageCollectionMetrics extends ExecutorMetricType with Logging { } else if (oldGenerationGarbageCollector.contains(mxBean.getName)) { gcMetrics(2) = mxBean.getCollectionCount gcMetrics(3) = mxBean.getCollectionTime + } else if (BUILTIN_CONCURRENT_GARBAGE_COLLECTOR.equals(mxBean.getName)) { + gcMetrics(5) = mxBean.getCollectionCount + gcMetrics(6) = mxBean.getCollectionTime } else if (!nonBuiltInCollectors.contains(mxBean.getName)) { nonBuiltInCollectors = mxBean.getName +: nonBuiltInCollectors // log it when first seen diff --git a/core/src/main/scala/org/apache/spark/rpc/RpcEnv.scala b/core/src/main/scala/org/apache/spark/rpc/RpcEnv.scala index 2fce2889c0977..2575cffdeb3b5 100644 --- a/core/src/main/scala/org/apache/spark/rpc/RpcEnv.scala +++ b/core/src/main/scala/org/apache/spark/rpc/RpcEnv.scala @@ -206,6 +206,19 @@ private[spark] trait RpcEnvFileServer { fixedBaseUri } + /** + * Removes a file from this RpcEnv. + * + * @param key Local file to remove. + */ + def removeFile(key: String): Unit + + /** + * Removes a jar to from this RpcEnv. + * + * @param key Local jar to remove. + */ + def removeJar(key: String): Unit } private[spark] case class RpcEnvConfig( diff --git a/core/src/main/scala/org/apache/spark/rpc/netty/NettyStreamManager.scala b/core/src/main/scala/org/apache/spark/rpc/netty/NettyStreamManager.scala index 57243133aba92..9ac14f3483683 100644 --- a/core/src/main/scala/org/apache/spark/rpc/netty/NettyStreamManager.scala +++ b/core/src/main/scala/org/apache/spark/rpc/netty/NettyStreamManager.scala @@ -43,6 +43,10 @@ private[netty] class NettyStreamManager(rpcEnv: NettyRpcEnv) private val jars = new ConcurrentHashMap[String, File]() private val dirs = new ConcurrentHashMap[String, File]() + override def removeFile(key: String): Unit = files.remove(key) + + override def removeJar(key: String): Unit = jars.remove(key) + override def getChunk(streamId: Long, chunkIndex: Int): ManagedBuffer = { throw new UnsupportedOperationException() } diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala index 9658e5e627724..ca088dc80550b 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala @@ -97,10 +97,10 @@ private[v1] class PrometheusResource extends ApiRequestContext { names.foreach { name => sb.append(s"$prefix${name}_bytes$labels ${m.getMetricValue(name)}\n") } - Seq("MinorGCCount", "MajorGCCount").foreach { name => + Seq("MinorGCCount", "MajorGCCount", "ConcurrentGCCount").foreach { name => sb.append(s"$prefix${name}_total$labels ${m.getMetricValue(name)}\n") } - Seq("MinorGCTime", "MajorGCTime").foreach { name => + Seq("MinorGCTime", "MajorGCTime", "ConcurrentGCTime").foreach { name => sb.append(s"$prefix${name}_seconds_total$labels ${m.getMetricValue(name) * 0.001}\n") } } diff --git a/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json index 850c3777ec4d8..ac0f2ce26051d 100644 --- a/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json @@ -76,7 +76,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } }, { "status" : "COMPLETE", @@ -156,7 +158,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } }, { "status" : "COMPLETE", @@ -236,6 +240,8 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } } ] diff --git a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json index ed4ed9ad87185..d614bb000e4ce 100644 --- a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_for_stage_expectation.json @@ -887,7 +887,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : true }, @@ -928,7 +930,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -956,6 +960,8 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } } diff --git a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json index f96a59fae5378..475dee00a2654 100644 --- a/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/excludeOnFailure_node_for_stage_expectation.json @@ -1021,7 +1021,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : true }, @@ -1062,7 +1064,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : true }, @@ -1103,7 +1107,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false }, @@ -1144,7 +1150,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false }, @@ -1185,7 +1193,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : true } @@ -1213,6 +1223,8 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } } diff --git a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json index ec3fc280b0a5e..a860682ca2e24 100644 --- a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json @@ -42,7 +42,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, diff --git a/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json index 9b7498d9e9145..2833cdcfde5dd 100644 --- a/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/executor_list_with_executor_metrics_json_expectation.json @@ -48,7 +48,9 @@ "MinorGCTime" : 55, "MajorGCCount" : 3, "MajorGCTime" : 144, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -108,7 +110,9 @@ "MinorGCTime" : 145, "MajorGCCount" : 2, "MajorGCTime" : 63, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { "NM_HTTP_ADDRESS" : "test-3.vpc.company.com:8042", @@ -178,7 +182,9 @@ "MinorGCTime" : 106, "MajorGCCount" : 2, "MajorGCTime" : 75, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { "NM_HTTP_ADDRESS" : "test-4.vpc.company.com:8042", @@ -248,7 +254,9 @@ "MinorGCTime" : 140, "MajorGCCount" : 2, "MajorGCTime" : 60, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { "NM_HTTP_ADDRESS" : "test-2.vpc.company.com:8042", diff --git a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json index fbb7b6631f02a..8a96858a2014a 100644 --- a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json @@ -85,7 +85,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -145,7 +147,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -205,7 +209,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -265,7 +271,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, diff --git a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json index fbb7b6631f02a..8a96858a2014a 100644 --- a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_expectation.json @@ -85,7 +85,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -145,7 +147,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -205,7 +209,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -265,7 +271,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, diff --git a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_unexcluding_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_unexcluding_expectation.json index b72ed0a625420..0e5e73f36fabd 100644 --- a/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_unexcluding_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/executor_node_excludeOnFailure_unexcluding_expectation.json @@ -73,7 +73,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -127,7 +129,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -181,7 +185,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, @@ -235,7 +241,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "attributes" : { }, "resources" : { }, diff --git a/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json index fee7377f18134..dc1bcd6a39625 100644 --- a/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json @@ -75,7 +75,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json index 9e390a995c36c..e24ac4f82b8a8 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_details_with_failed_task_expectation.json @@ -92,7 +92,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -120,6 +122,8 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } } diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json index 887d2678e6160..659e3c41d9289 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json @@ -595,7 +595,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -625,6 +627,8 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } } diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json index 3bb59aaf5b507..f84cf26fcf1d6 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json @@ -595,7 +595,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -625,6 +627,8 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } } ] diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json index b688b72b04d50..564f3eadd1cc2 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_details_expectation.json @@ -597,7 +597,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -625,6 +627,8 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } } ] diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_partitionId_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_partitionId_expectation.json index 83ffb7da8e77f..2bf7f34803775 100644 --- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_partitionId_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_with_partitionId_expectation.json @@ -721,7 +721,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -749,7 +751,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 diff --git a/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json index e3cd980943450..8df41bfcc8d78 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json @@ -74,7 +74,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 @@ -155,7 +157,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 @@ -235,7 +239,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 @@ -315,7 +321,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 diff --git a/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json index e4caffcf10787..730df3fbd5341 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json @@ -78,7 +78,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 diff --git a/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json index d3459be777d48..16d92244ee0d0 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_list_with_peak_metrics_expectation.json @@ -74,7 +74,9 @@ "MinorGCTime" : 115, "MajorGCCount" : 4, "MajorGCTime" : 339, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 @@ -155,7 +157,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 @@ -236,7 +240,9 @@ "MinorGCTime" : 33, "MajorGCCount" : 3, "MajorGCTime" : 110, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json index 3880818a7b5df..e38741b7bf6e7 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json @@ -639,7 +639,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -667,7 +669,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json index d3eb7d55e0e1d..630b0512e8f98 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_with_peak_metrics_expectation.json @@ -1147,7 +1147,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false }, @@ -1188,7 +1190,9 @@ "MinorGCTime" : 115, "MajorGCCount" : 4, "MajorGCTime" : 339, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -1216,7 +1220,9 @@ "MinorGCTime" : 115, "MajorGCCount" : 4, "MajorGCTime" : 339, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isShufflePushEnabled" : false, "shuffleMergersCount" : 0 diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json index 3ad18f816fbe5..23770480ad62a 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_with_speculation_summary_expectation.json @@ -424,7 +424,9 @@ "MinorGCTime" : 280, "MajorGCCount" : 2, "MajorGCTime" : 1116, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false }, @@ -465,7 +467,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false }, @@ -506,7 +510,9 @@ "MinorGCTime" : 587, "MajorGCCount" : 2, "MajorGCTime" : 906, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false }, @@ -547,7 +553,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -584,6 +592,8 @@ "MinorGCTime" : 587, "MajorGCCount" : 2, "MajorGCTime" : 1116, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 } } diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json index c89b82caf3818..c8458a409589e 100644 --- a/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json +++ b/core/src/test/resources/HistoryServerExpectations/stage_with_summaries_expectation.json @@ -1147,7 +1147,9 @@ "MinorGCTime" : 0, "MajorGCCount" : 0, "MajorGCTime" : 0, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false }, @@ -1188,7 +1190,9 @@ "MinorGCTime" : 115, "MajorGCCount" : 4, "MajorGCTime" : 339, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "isExcludedForStage" : false } @@ -1216,7 +1220,9 @@ "MinorGCTime" : 115, "MajorGCCount" : 4, "MajorGCTime" : 339, - "TotalGCTime" : 0 + "TotalGCTime" : 0, + "ConcurrentGCCount" : 0, + "ConcurrentGCTime" : 0 }, "taskMetricsDistributions" : { "quantiles" : [ 0.0, 0.25, 0.5, 0.75, 1.0 ], @@ -1306,7 +1312,9 @@ "MinorGCTime" : [ 0.0, 0.0, 115.0, 115.0, 115.0 ], "MajorGCCount" : [ 0.0, 0.0, 4.0, 4.0, 4.0 ], "MajorGCTime" : [ 0.0, 0.0, 339.0, 339.0, 339.0 ], - "TotalGCTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ] + "TotalGCTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], + "ConcurrentGCCount" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ], + "ConcurrentGCTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ] } }, "isShufflePushEnabled" : false, diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala index 8105df64705a4..e8d41c4d46e21 100644 --- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala +++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala @@ -58,21 +58,21 @@ class JsonProtocolSuite extends SparkFunSuite { makeTaskInfo(123L, 234, 67, 234, 345L, false), new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L, 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L, - 0, 0, 0, 0, 80001L)), + 0, 0, 0, 0, 80001L, 3, 3)), makeTaskMetrics(300L, 400L, 500L, 600L, 700, 800, 0, hasHadoopInput = false, hasOutput = false)) val taskEndWithHadoopInput = SparkListenerTaskEnd(1, 0, "ShuffleMapTask", Success, makeTaskInfo(123L, 234, 67, 234, 345L, false), new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L, 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L, - 0, 0, 0, 0, 80001L)), + 0, 0, 0, 0, 80001L, 3, 3)), makeTaskMetrics(300L, 400L, 500L, 600L, 700, 800, 0, hasHadoopInput = true, hasOutput = false)) val taskEndWithOutput = SparkListenerTaskEnd(1, 0, "ResultTask", Success, makeTaskInfo(123L, 234, 67, 234, 345L, false), new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L, 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, 30364L, 15182L, - 0, 0, 0, 0, 80001L)), + 0, 0, 0, 0, 80001L, 3, 3)), makeTaskMetrics(300L, 400L, 500L, 600L, 700, 800, 0, hasHadoopInput = true, hasOutput = true)) val jobStart = { @@ -136,7 +136,7 @@ class JsonProtocolSuite extends SparkFunSuite { val executorUpdates = new ExecutorMetrics( Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L, 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, - 30364L, 15182L, 10L, 90L, 2L, 20L, 80001L)) + 30364L, 15182L, 10L, 90L, 2L, 20L, 80001L, 3, 3)) SparkListenerExecutorMetricsUpdate("exec3", Seq((1L, 2, 3, accumUpdates)), Map((0, 0) -> executorUpdates)) } @@ -147,7 +147,7 @@ class JsonProtocolSuite extends SparkFunSuite { SparkListenerStageExecutorMetrics("1", 2, 3, new ExecutorMetrics(Array(543L, 123456L, 12345L, 1234L, 123L, 12L, 432L, 321L, 654L, 765L, 256912L, 123456L, 123456L, 61728L, - 30364L, 15182L, 10L, 90L, 2L, 20L, 80001L))) + 30364L, 15182L, 10L, 90L, 2L, 20L, 80001L, 3, 3))) val rprofBuilder = new ResourceProfileBuilder() val taskReq = new TaskResourceRequests() .cpus(1) @@ -1754,7 +1754,9 @@ private[spark] object JsonProtocolSuite extends Assertions { | "MinorGCTime" : 0, | "MajorGCCount" : 0, | "MajorGCTime" : 0, - | "TotalGCTime" : 80001 + | "TotalGCTime": 80001, + | "ConcurrentGCCount" : 3, + | "ConcurrentGCTime" : 3 | }, | "Task Metrics": { | "Executor Deserialize Time": 300, @@ -1893,7 +1895,9 @@ private[spark] object JsonProtocolSuite extends Assertions { | "MinorGCTime" : 0, | "MajorGCCount" : 0, | "MajorGCTime" : 0, - | "TotalGCTime" : 80001 + | "TotalGCTime": 80001, + | "ConcurrentGCCount" : 3, + | "ConcurrentGCTime" : 3 | }, | "Task Metrics": { | "Executor Deserialize Time": 300, @@ -2032,7 +2036,9 @@ private[spark] object JsonProtocolSuite extends Assertions { | "MinorGCTime" : 0, | "MajorGCCount" : 0, | "MajorGCTime" : 0, - | "TotalGCTime" : 80001 + | "TotalGCTime": 80001, + | "ConcurrentGCCount" : 3, + | "ConcurrentGCTime" : 3 | }, | "Task Metrics": { | "Executor Deserialize Time": 300, @@ -2933,7 +2939,9 @@ private[spark] object JsonProtocolSuite extends Assertions { | "MinorGCTime": 90, | "MajorGCCount": 2, | "MajorGCTime": 20, - | "TotalGCTime" : 80001 + | "TotalGCTime": 80001, + | "ConcurrentGCCount" : 3, + | "ConcurrentGCTime" : 3 | } | } | ] @@ -2968,7 +2976,9 @@ private[spark] object JsonProtocolSuite extends Assertions { | "MinorGCTime": 90, | "MajorGCCount": 2, | "MajorGCTime": 20, - | "TotalGCTime" : 80001 + | "TotalGCTime": 80001, + | "ConcurrentGCCount" : 3, + | "ConcurrentGCTime" : 3 | } |} """.stripMargin diff --git a/dev/.scalafmt.conf b/dev/.scalafmt.conf index c3b26002a7690..721dec289900b 100644 --- a/dev/.scalafmt.conf +++ b/dev/.scalafmt.conf @@ -32,4 +32,4 @@ fileOverride { runner.dialect = scala213 } } -version = 3.7.5 +version = 3.7.13 diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index 94c999ba4b6d8..59164c1f8f441 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -128,8 +128,8 @@ jersey-container-servlet/2.40//jersey-container-servlet-2.40.jar jersey-hk2/2.40//jersey-hk2-2.40.jar jersey-server/2.40//jersey-server-2.40.jar jettison/1.5.4//jettison-1.5.4.jar -jetty-util-ajax/9.4.51.v20230217//jetty-util-ajax-9.4.51.v20230217.jar -jetty-util/9.4.51.v20230217//jetty-util-9.4.51.v20230217.jar +jetty-util-ajax/9.4.52.v20230823//jetty-util-ajax-9.4.52.v20230823.jar +jetty-util/9.4.52.v20230823//jetty-util-9.4.52.v20230823.jar jline/2.14.6//jline-2.14.6.jar joda-time/2.12.5//joda-time-2.12.5.jar jodd-core/3.5.2//jodd-core-3.5.2.jar diff --git a/docs/Gemfile b/docs/Gemfile index 6c35201296480..6c6760371163c 100644 --- a/docs/Gemfile +++ b/docs/Gemfile @@ -18,7 +18,7 @@ source "https://rubygems.org" gem "ffi", "1.15.5" -gem "jekyll", "4.2.1" +gem "jekyll", "4.3.2" gem "rouge", "3.26.0" gem "jekyll-redirect-from", "0.16.0" -gem "webrick", "1.7" +gem "webrick", "1.8.1" diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock index 6654e6c47c615..eda31f857476e 100644 --- a/docs/Gemfile.lock +++ b/docs/Gemfile.lock @@ -1,74 +1,78 @@ GEM remote: https://rubygems.org/ specs: - addressable (2.8.0) - public_suffix (>= 2.0.2, < 5.0) + addressable (2.8.5) + public_suffix (>= 2.0.2, < 6.0) colorator (1.1.0) - concurrent-ruby (1.1.9) - em-websocket (0.5.2) + concurrent-ruby (1.2.2) + em-websocket (0.5.3) eventmachine (>= 0.12.9) - http_parser.rb (~> 0.6.0) + http_parser.rb (~> 0) eventmachine (1.2.7) ffi (1.15.5) forwardable-extended (2.6.0) - http_parser.rb (0.6.0) - i18n (1.8.11) + google-protobuf (3.24.2) + http_parser.rb (0.8.0) + i18n (1.14.1) concurrent-ruby (~> 1.0) - jekyll (4.2.1) + jekyll (4.3.2) addressable (~> 2.4) colorator (~> 1.0) em-websocket (~> 0.5) i18n (~> 1.0) - jekyll-sass-converter (~> 2.0) + jekyll-sass-converter (>= 2.0, < 4.0) jekyll-watch (~> 2.0) - kramdown (~> 2.3) + kramdown (~> 2.3, >= 2.3.1) kramdown-parser-gfm (~> 1.0) liquid (~> 4.0) - mercenary (~> 0.4.0) + mercenary (>= 0.3.6, < 0.5) pathutil (~> 0.9) - rouge (~> 3.0) + rouge (>= 3.0, < 5.0) safe_yaml (~> 1.0) - terminal-table (~> 2.0) + terminal-table (>= 1.8, < 4.0) + webrick (~> 1.7) jekyll-redirect-from (0.16.0) jekyll (>= 3.3, < 5.0) - jekyll-sass-converter (2.1.0) - sassc (> 2.0.1, < 3.0) + jekyll-sass-converter (3.0.0) + sass-embedded (~> 1.54) jekyll-watch (2.2.1) listen (~> 3.0) - kramdown (2.3.1) + kramdown (2.4.0) rexml kramdown-parser-gfm (1.1.0) kramdown (~> 2.0) - liquid (4.0.3) - listen (3.7.0) + liquid (4.0.4) + listen (3.8.0) rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) mercenary (0.4.0) pathutil (0.16.2) forwardable-extended (~> 2.6) - public_suffix (4.0.6) - rb-fsevent (0.11.0) + public_suffix (5.0.3) + rake (13.0.6) + rb-fsevent (0.11.2) rb-inotify (0.10.1) ffi (~> 1.0) - rexml (3.2.5) + rexml (3.2.6) rouge (3.26.0) safe_yaml (1.0.5) - sassc (2.4.0) - ffi (~> 1.9) - terminal-table (2.0.0) - unicode-display_width (~> 1.1, >= 1.1.1) - unicode-display_width (1.8.0) - webrick (1.7.0) + sass-embedded (1.63.6) + google-protobuf (~> 3.23) + rake (>= 13.0.0) + terminal-table (3.0.2) + unicode-display_width (>= 1.1.1, < 3) + unicode-display_width (2.4.2) + webrick (1.8.1) PLATFORMS ruby DEPENDENCIES ffi (= 1.15.5) - jekyll (= 4.2.1) + jekyll (= 4.3.2) jekyll-redirect-from (= 0.16.0) rouge (= 3.26.0) - webrick (= 1.7) + webrick (= 1.8.1) BUNDLED WITH 2.3.8 diff --git a/docs/sql-error-conditions-incompatible-data-for-table-error-class.md b/docs/sql-error-conditions-incompatible-data-for-table-error-class.md index f70b69ba6c5bd..0dd28e9d55c50 100644 --- a/docs/sql-error-conditions-incompatible-data-for-table-error-class.md +++ b/docs/sql-error-conditions-incompatible-data-for-table-error-class.md @@ -37,6 +37,10 @@ Cannot find data for the output column ``. Cannot safely cast `` `` to ``. +## EXTRA_COLUMNS + +Cannot write extra columns ``. + ## EXTRA_STRUCT_FIELDS Cannot write extra fields `` to the struct ``. diff --git a/docs/sql-error-conditions.md b/docs/sql-error-conditions.md index 89c27f72ea093..e25ef384a75cb 100644 --- a/docs/sql-error-conditions.md +++ b/docs/sql-error-conditions.md @@ -1305,6 +1305,12 @@ Cannot call function `` because named argument references are not It is not allowed to use an aggregate function in the argument of another aggregate function. Please use the inner aggregate function in a sub-query. +### NON_FOLDABLE_ARGUMENT + +[SQLSTATE: 22024](sql-error-conditions-sqlstates.html#class-22-data-exception) + +The function `` requires the parameter `` to be a foldable expression of the type ``, but the actual argument is a non-foldable. + ### NON_LAST_MATCHED_CLAUSE_OMIT_CONDITION [SQLSTATE: 42613](sql-error-conditions-sqlstates.html#class-42-syntax-error-or-access-rule-violation) @@ -1758,6 +1764,18 @@ SQLSTATE: none assigned Failed to analyze the Python user defined table function: `` +### TABLE_VALUED_FUNCTION_REQUIRED_METADATA_INCOMPATIBLE_WITH_CALL + +[SQLSTATE: 22023](sql-error-conditions-sqlstates.html#class-22-data-exception) + +Failed to evaluate the table function `` because its table metadata ``, but the function call ``. + +### TABLE_VALUED_FUNCTION_REQUIRED_METADATA_INVALID + +[SQLSTATE: 22023](sql-error-conditions-sqlstates.html#class-22-data-exception) + +Failed to evaluate the table function `` because its table metadata was invalid; ``. + ### TABLE_VALUED_FUNCTION_TOO_MANY_TABLE_ARGUMENTS SQLSTATE: none assigned diff --git a/pom.xml b/pom.xml index b64a0ab15acd2..8edc3fd550c2e 100644 --- a/pom.xml +++ b/pom.xml @@ -143,7 +143,7 @@ 1.13.1 1.9.1 shaded-protobuf - 9.4.51.v20230217 + 9.4.52.v20230823 4.0.3 0.10.0