From 0e32d0a8b6c21267af763e3d331339d2677d231c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Osipiuk?= Date: Mon, 8 Jul 2024 18:34:22 +0200 Subject: [PATCH] Bump default value of max split count per task for FTE With limit of 256 splits per task it was possible to end up with very small tasks and possibly even reach limit of number of tasks per stage (2^^15). The problem appeared when splits were very small. Task sizing should be mostly done based on split size; max number of splits should be treated as extra limit to be reached only in rare situations. This PR bumps default value 8x to 2048. --- .../src/main/java/io/trino/execution/QueryManagerConfig.java | 2 +- .../test/java/io/trino/execution/TestQueryManagerConfig.java | 2 +- docs/src/main/sphinx/admin/fault-tolerant-execution.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryManagerConfig.java b/core/trino-main/src/main/java/io/trino/execution/QueryManagerConfig.java index d8442f45724a..d8c1da219ca1 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryManagerConfig.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryManagerConfig.java @@ -132,7 +132,7 @@ public class QueryManagerConfig private int faultTolerantExecutionHashDistributionWriteTaskTargetMaxCount = 2000; private DataSize faultTolerantExecutionStandardSplitSize = DataSize.of(64, MEGABYTE); - private int faultTolerantExecutionMaxTaskSplitCount = 256; + private int faultTolerantExecutionMaxTaskSplitCount = 2048; private DataSize faultTolerantExecutionTaskDescriptorStorageMaxMemory = DataSize.ofBytes(round(AVAILABLE_HEAP_MEMORY * 0.15)); private int faultTolerantExecutionMaxPartitionCount = 50; private int faultTolerantExecutionMinPartitionCount = 4; diff --git a/core/trino-main/src/test/java/io/trino/execution/TestQueryManagerConfig.java b/core/trino-main/src/test/java/io/trino/execution/TestQueryManagerConfig.java index 3b7126852aa1..f5ef3873e1f4 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestQueryManagerConfig.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestQueryManagerConfig.java @@ -98,7 +98,7 @@ public void testDefaults() .setFaultTolerantExecutionHashDistributionWriteTasksToNodesMinRatio(2.0) .setFaultTolerantExecutionHashDistributionWriteTaskTargetMaxCount(2000) .setFaultTolerantExecutionStandardSplitSize(DataSize.of(64, MEGABYTE)) - .setFaultTolerantExecutionMaxTaskSplitCount(256) + .setFaultTolerantExecutionMaxTaskSplitCount(2048) .setFaultTolerantExecutionTaskDescriptorStorageMaxMemory(DataSize.ofBytes(round(AVAILABLE_HEAP_MEMORY * 0.15))) .setFaultTolerantExecutionMaxPartitionCount(50) .setFaultTolerantExecutionMinPartitionCount(4) diff --git a/docs/src/main/sphinx/admin/fault-tolerant-execution.md b/docs/src/main/sphinx/admin/fault-tolerant-execution.md index 3d732a1d7008..1c4e043a74b9 100644 --- a/docs/src/main/sphinx/admin/fault-tolerant-execution.md +++ b/docs/src/main/sphinx/admin/fault-tolerant-execution.md @@ -237,7 +237,7 @@ properties only apply to a `TASK` retry policy. May be overridden for the current session with the `fault_tolerant_execution_max_task_split_count` [session property](session-properties-definition). - - `256` + - `2048` * - `fault-tolerant-execution-arbitrary-distribution-compute-task-target-size-growth-period` - The number of tasks created for any given non-writer stage of arbitrary distribution before task size is increased.