From 4a8b8fb2bde036803db270b6aedfe1011fe5b913 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Sat, 7 Dec 2024 17:41:45 +0000 Subject: [PATCH] sets default limits on bulk imports files fixes #5138 --- .../java/org/apache/accumulo/core/conf/Property.java | 11 +++++------ .../apache/accumulo/test/CountNameNodeOpsBulkIT.java | 1 + 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java index f3302499982..d88840cd2c4 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@ -921,13 +921,12 @@ public enum Property { "The maximum amount of memory that will be used to cache results of a client query/scan. " + "Once this limit is reached, the buffered data is sent to the client.", "1.3.5"), - TABLE_BULK_MAX_TABLETS("table.bulk.max.tablets", "0", PropertyType.COUNT, - "The maximum number of tablets allowed for one bulk import file. Value of 0 is Unlimited. " - + "This property is only enforced in the new bulk import API.", + TABLE_BULK_MAX_TABLETS("table.bulk.max.tablets", "100", PropertyType.COUNT, + "The maximum number of tablets allowed for one bulk import file. Value of 0 is Unlimited. ", "2.1.0"), - TABLE_BULK_MAX_TABLET_FILES("table.bulk.max.tablet.files", "0", PropertyType.COUNT, + TABLE_BULK_MAX_TABLET_FILES("table.bulk.max.tablet.files", "100", PropertyType.COUNT, "The maximum number of files a bulk import can add to a single tablet. When this property " - + "is exceeded for any tablet the entire bulk import operation will fail before any making " + + "is exceeded for any tablet the entire bulk import operation will fail before making any " + "changes. Value of 0 is unlimited.", "4.0.0"), TABLE_FILE_TYPE("table.file.type", RFile.EXTENSION, PropertyType.FILENAME_EXT, @@ -964,7 +963,7 @@ public enum Property { + "to 0 will make it default to tserver.scan.files.open.max-1, this will prevent a tablet" + " from having more RFiles than can be opened by a scan.", "1.4.0"), - TABLE_FILE_PAUSE("table.file.pause", "0", PropertyType.COUNT, + TABLE_FILE_PAUSE("table.file.pause", "100", PropertyType.COUNT, "When a tablet has more than this number of files, bulk imports and minor compactions " + "will wait until the tablet has less files before proceeding. This will cause back " + "pressure on bulk imports and writes to tables when compactions are not keeping up. " diff --git a/test/src/main/java/org/apache/accumulo/test/CountNameNodeOpsBulkIT.java b/test/src/main/java/org/apache/accumulo/test/CountNameNodeOpsBulkIT.java index 0ba5ae64b1f..d13d92237c6 100644 --- a/test/src/main/java/org/apache/accumulo/test/CountNameNodeOpsBulkIT.java +++ b/test/src/main/java/org/apache/accumulo/test/CountNameNodeOpsBulkIT.java @@ -103,6 +103,7 @@ public void compareOldNewBulkImportTest() throws Exception { Map props = new HashMap<>(); props.put(Property.TABLE_MAJC_RATIO.getKey(), "2000"); props.put(Property.TABLE_FILE_MAX.getKey(), "2000"); + props.put(Property.TABLE_FILE_PAUSE.getKey(), "2000"); // splits to slow down bulk import SortedSet splits = new TreeSet<>(); for (int i = 1; i < 0xf; i++) {