From 5033623e8a29842dd9f4f6cf65e6935a2597e4c3 Mon Sep 17 00:00:00 2001 From: Kamil Endruszkiewicz Date: Tue, 3 Jan 2023 13:22:43 +0100 Subject: [PATCH] Increase node scheduler config parameters --- .../execution/scheduler/NodeSchedulerConfig.java | 4 ++-- .../io/trino/execution/TestNodeSchedulerConfig.java | 4 ++-- .../main/sphinx/admin/properties-node-scheduler.rst | 13 ++++++++++++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/core/trino-main/src/main/java/io/trino/execution/scheduler/NodeSchedulerConfig.java b/core/trino-main/src/main/java/io/trino/execution/scheduler/NodeSchedulerConfig.java index 739920553812..11b54b099514 100644 --- a/core/trino-main/src/main/java/io/trino/execution/scheduler/NodeSchedulerConfig.java +++ b/core/trino-main/src/main/java/io/trino/execution/scheduler/NodeSchedulerConfig.java @@ -47,11 +47,11 @@ public enum SplitsBalancingPolicy private boolean includeCoordinator = true; private int maxSplitsPerNode = 100; private int minPendingSplitsPerTask = 10; - private int maxAdjustedPendingSplitsWeightPerTask = 500; + private int maxAdjustedPendingSplitsWeightPerTask = 2000; private NodeSchedulerPolicy nodeSchedulerPolicy = NodeSchedulerPolicy.UNIFORM; private boolean optimizedLocalScheduling = true; private SplitsBalancingPolicy splitsBalancingPolicy = SplitsBalancingPolicy.STAGE; - private int maxUnacknowledgedSplitsPerTask = 500; + private int maxUnacknowledgedSplitsPerTask = 2000; private Duration allowedNoMatchingNodePeriod = new Duration(2, TimeUnit.MINUTES); private NodeAllocatorType nodeAllocatorType = NodeAllocatorType.BIN_PACKING; diff --git a/core/trino-main/src/test/java/io/trino/execution/TestNodeSchedulerConfig.java b/core/trino-main/src/test/java/io/trino/execution/TestNodeSchedulerConfig.java index 0a038b83e41b..ad718fef56b4 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestNodeSchedulerConfig.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestNodeSchedulerConfig.java @@ -37,8 +37,8 @@ public void testDefaults() .setMinCandidates(10) .setMaxSplitsPerNode(100) .setMinPendingSplitsPerTask(10) - .setMaxAdjustedPendingSplitsWeightPerTask(500) - .setMaxUnacknowledgedSplitsPerTask(500) + .setMaxAdjustedPendingSplitsWeightPerTask(2000) + .setMaxUnacknowledgedSplitsPerTask(2000) .setIncludeCoordinator(true) .setSplitsBalancingPolicy(NodeSchedulerConfig.SplitsBalancingPolicy.STAGE) .setOptimizedLocalScheduling(true) diff --git a/docs/src/main/sphinx/admin/properties-node-scheduler.rst b/docs/src/main/sphinx/admin/properties-node-scheduler.rst index 4a9782fee133..33a0495fef39 100644 --- a/docs/src/main/sphinx/admin/properties-node-scheduler.rst +++ b/docs/src/main/sphinx/admin/properties-node-scheduler.rst @@ -47,7 +47,7 @@ and has similar drawbacks if set too high. ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ * **Type:** :ref:`prop-type-integer` -* **Default value:** ``500`` +* **Default value:** ``2000`` The maximum number of outstanding splits with the standard split weight guaranteed to be scheduled on a node (even when the node is already at the limit for total number of splits) for a single task given the task has remaining splits to process. @@ -61,6 +61,17 @@ if set too high. Only applies for ``uniform`` :ref:`scheduler policy `. +``node-scheduler.max-unacknowledged-splits-per-task`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* **Type:** :ref:`prop-type-integer` +* **Default value:** ``2000`` + +Maximum number of splits that are either queued on the coordinator, but not yet sent or confirmed to have been received by +the worker. This limit enforcement takes precedence over other existing split limit configurations +like ``node-scheduler.max-splits-per-node`` or ``node-scheduler.max-adjusted-pending-splits-per-task`` +and is designed to prevent large task update requests that might cause a query to fail. + ``node-scheduler.min-candidates`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^