From 179be3069be48f87abe7f20d23bffd3364a3baf5 Mon Sep 17 00:00:00 2001 From: aishikbh Date: Tue, 19 Nov 2024 01:59:05 +0530 Subject: [PATCH] add support for prefix/suffix for taskname (#14478) --- .../core/minion/PinotHelixTaskResourceManager.java | 11 ++++++++++- .../spi/ingestion/batch/BatchConfigProperties.java | 2 ++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotHelixTaskResourceManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotHelixTaskResourceManager.java index a28799314e91..f8b258265a6c 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotHelixTaskResourceManager.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/minion/PinotHelixTaskResourceManager.java @@ -60,6 +60,7 @@ import org.apache.pinot.controller.util.CompletionServiceHelper; import org.apache.pinot.core.common.MinionConstants; import org.apache.pinot.core.minion.PinotTaskConfig; +import org.apache.pinot.spi.ingestion.batch.BatchConfigProperties; import org.apache.pinot.spi.utils.CommonConstants; import org.apache.pinot.spi.utils.CommonConstants.Helix; import org.apache.pinot.spi.utils.JsonUtils; @@ -262,7 +263,15 @@ public synchronized String submitTask(List pinotTaskConfigs, St Preconditions.checkState(numConcurrentTasksPerInstance > 0); String taskType = pinotTaskConfigs.get(0).getTaskType(); - String parentTaskName = getParentTaskName(taskType, UUID.randomUUID() + "_" + System.currentTimeMillis()); + + // Get task name prefix and suffix from the first task config. + String taskNamePrefix = pinotTaskConfigs.get(0).getConfigs() + .getOrDefault(BatchConfigProperties.TASK_NAME_PREFIX_KEY, UUID.randomUUID().toString()); + String taskNameSuffix = + pinotTaskConfigs.get(0).getConfigs().getOrDefault(BatchConfigProperties.TASK_NAME_SUFFIX_KEY, ""); + + String parentTaskName = + getParentTaskName(taskType, taskNamePrefix + "_" + System.currentTimeMillis() + taskNameSuffix); return submitTask(parentTaskName, pinotTaskConfigs, minionInstanceTag, taskTimeoutMs, numConcurrentTasksPerInstance, maxAttemptsPerTask); } diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/BatchConfigProperties.java b/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/BatchConfigProperties.java index 011bc0f1096c..fb7134a598c0 100644 --- a/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/BatchConfigProperties.java +++ b/pinot-spi/src/main/java/org/apache/pinot/spi/ingestion/batch/BatchConfigProperties.java @@ -66,6 +66,8 @@ private BatchConfigProperties() { public static final String APPEND_UUID_TO_SEGMENT_NAME = "append.uuid.to.segment.name"; public static final String EXCLUDE_TIME_IN_SEGMENT_NAME = "exclude.time.in.segment.name"; public static final String BATCH_SEGMENT_UPLOAD = "batchSegmentUpload"; + public static final String TASK_NAME_PREFIX_KEY = "taskNamePrefix"; + public static final String TASK_NAME_SUFFIX_KEY = "taskNameSuffix"; public static final String OUTPUT_SEGMENT_DIR_URI = "output.segment.dir.uri";