Skip to content

Commit

Permalink
Increase PAGE_SIZE_IN_BYTES limits in PageProcessor
Browse files Browse the repository at this point in the history
Orc and parquet readers are configured to produce upto
16MB pages by default
  • Loading branch information
raunaqmorarka committed Aug 4, 2023
1 parent a71f59c commit 0811c1a
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 7 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,8 @@
public class PageProcessor
{
public static final int MAX_BATCH_SIZE = 8 * 1024;
static final int MAX_PAGE_SIZE_IN_BYTES = 4 * 1024 * 1024;
static final int MIN_PAGE_SIZE_IN_BYTES = 1024 * 1024;
static final int MAX_PAGE_SIZE_IN_BYTES = 16 * 1024 * 1024;
static final int MIN_PAGE_SIZE_IN_BYTES = 4 * 1024 * 1024;

private final ExpressionProfiler expressionProfiler;
private final DictionarySourceIdFunction dictionarySourceIdFunction = new DictionarySourceIdFunction();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,7 @@ public void testAdaptiveBatchSize()

// process large page which will reduce batch size
Slice[] slices = new Slice[(int) (MAX_BATCH_SIZE * 2.5)];
Arrays.fill(slices, Slices.allocate(1024));
Arrays.fill(slices, Slices.allocate(4096));
Page inputPage = new Page(createSlicesBlock(slices));

Iterator<Optional<Page>> output = processAndAssertRetainedPageSize(pageProcessor, new DriverYieldSignal(), inputPage);
Expand Down Expand Up @@ -319,7 +319,7 @@ public void testOptimisticProcessing()

// process large page which will reduce batch size
Slice[] slices = new Slice[(int) (MAX_BATCH_SIZE * 2.5)];
Arrays.fill(slices, Slices.allocate(1024));
Arrays.fill(slices, Slices.allocate(4096));
Page inputPage = new Page(createSlicesBlock(slices));

Iterator<Optional<Page>> output = processAndAssertRetainedPageSize(pageProcessor, inputPage);
Expand Down Expand Up @@ -359,9 +359,9 @@ public void testRetainedSize()
ImmutableList.of(new InputPageProjection(0, VARCHAR), new InputPageProjection(1, VARCHAR)),
OptionalInt.of(MAX_BATCH_SIZE));

// create 2 columns X 800 rows of strings with each string's size = 10KB
// this can force previouslyComputedResults to be saved given the page is 16MB in size
String value = join("", nCopies(10_000, "a"));
// create 2 columns X 800 rows of strings with each string's size = 30KB
// this can force previouslyComputedResults to be saved given the page is 48MB in size
String value = join("", nCopies(30_000, "a"));
List<String> values = nCopies(800, value);
Page inputPage = new Page(createStringsBlock(values), createStringsBlock(values));

Expand Down

0 comments on commit 0811c1a

Please sign in to comment.