Increase node scheduler config parameters

trinodb · Feb 1, 2023 · 5033623 · 5033623
1 parent a5a6960
commit 5033623
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 5 deletions.
diff --git a/core/trino-main/src/main/java/io/trino/execution/scheduler/NodeSchedulerConfig.java b/core/trino-main/src/main/java/io/trino/execution/scheduler/NodeSchedulerConfig.java
@@ -47,11 +47,11 @@ public enum SplitsBalancingPolicy
     private boolean includeCoordinator = true;
     private int maxSplitsPerNode = 100;
     private int minPendingSplitsPerTask = 10;
-    private int maxAdjustedPendingSplitsWeightPerTask = 500;
+    private int maxAdjustedPendingSplitsWeightPerTask = 2000;
     private NodeSchedulerPolicy nodeSchedulerPolicy = NodeSchedulerPolicy.UNIFORM;
     private boolean optimizedLocalScheduling = true;
     private SplitsBalancingPolicy splitsBalancingPolicy = SplitsBalancingPolicy.STAGE;
-    private int maxUnacknowledgedSplitsPerTask = 500;
+    private int maxUnacknowledgedSplitsPerTask = 2000;
     private Duration allowedNoMatchingNodePeriod = new Duration(2, TimeUnit.MINUTES);
     private NodeAllocatorType nodeAllocatorType = NodeAllocatorType.BIN_PACKING;
 

diff --git a/core/trino-main/src/test/java/io/trino/execution/TestNodeSchedulerConfig.java b/core/trino-main/src/test/java/io/trino/execution/TestNodeSchedulerConfig.java
@@ -37,8 +37,8 @@ public void testDefaults()
                 .setMinCandidates(10)
                 .setMaxSplitsPerNode(100)
                 .setMinPendingSplitsPerTask(10)
-                .setMaxAdjustedPendingSplitsWeightPerTask(500)
-                .setMaxUnacknowledgedSplitsPerTask(500)
+                .setMaxAdjustedPendingSplitsWeightPerTask(2000)
+                .setMaxUnacknowledgedSplitsPerTask(2000)
                 .setIncludeCoordinator(true)
                 .setSplitsBalancingPolicy(NodeSchedulerConfig.SplitsBalancingPolicy.STAGE)
                 .setOptimizedLocalScheduling(true)

diff --git a/docs/src/main/sphinx/admin/properties-node-scheduler.rst b/docs/src/main/sphinx/admin/properties-node-scheduler.rst
@@ -47,7 +47,7 @@ and has similar drawbacks if set too high.
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 * **Type:** :ref:`prop-type-integer`
-* **Default value:** ``500``
+* **Default value:** ``2000``
 
 The maximum number of outstanding splits with the standard split weight guaranteed to be scheduled on a node (even when the node
 is already at the limit for total number of splits) for a single task given the task has remaining splits to process.
@@ -61,6 +61,17 @@ if set too high.
 
     Only applies for ``uniform`` :ref:`scheduler policy <node-scheduler-policy>`.
 
+``node-scheduler.max-unacknowledged-splits-per-task``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+* **Type:** :ref:`prop-type-integer`
+* **Default value:** ``2000``
+
+Maximum number of splits that are either queued on the coordinator, but not yet sent or confirmed to have been received by
+the worker. This limit enforcement takes precedence over other existing split limit configurations
+like ``node-scheduler.max-splits-per-node`` or ``node-scheduler.max-adjusted-pending-splits-per-task``
+and is designed to prevent large task update requests that might cause a query to fail.
+
 ``node-scheduler.min-candidates``
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^