Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve WindowCheck memory usage. #5197

Merged
merged 15 commits into from
Feb 28, 2024
Merged
Show file tree
Hide file tree
Changes from 13 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions engine/table/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ dependencies {
implementation 'com.tdunning:t-digest:3.2'
implementation 'com.squareup:javapoet:1.13.0'
implementation 'io.github.classgraph:classgraph:4.8.165'
implementation 'it.unimi.dsi:fastutil:8.5.13'
cpwright marked this conversation as resolved.
Show resolved Hide resolved

implementation project(':plugin')
implementation depCommonsLang3
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,12 @@
import java.util.List;

/**
* This will filter a table for the most recent N nanoseconds (must be on a date time column).
* This will filter a table for the most recent N nanoseconds (must be on an {@link Instant} column).
*
* <p>
* Note, this filter rescans the source table. You should prefer to use {@link io.deephaven.engine.util.WindowCheck}
* instead.
* </p>
*/
public class TimeSeriesFilter
extends WhereFilterLivenessArtifactImpl
Expand Down
731 changes: 613 additions & 118 deletions engine/table/src/main/java/io/deephaven/engine/util/WindowCheck.java

Large diffs are not rendered by default.

254 changes: 244 additions & 10 deletions engine/table/src/test/java/io/deephaven/engine/util/TestWindowCheck.java

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion engine/test-utils/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ dependencies {
implementation depCommonsLang3
implementation depTrove3

implementation 'it.unimi.dsi:fastutil:8.5.11'
implementation 'it.unimi.dsi:fastutil:8.5.13'

Classpaths.inheritJUnitClassic(project, 'implementation')
Classpaths.inheritJUnitPlatform(project, 'implementation')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,15 @@ private void validateGroup(int... opts) {

public static final SimulationProfile DEFAULT_PROFILE = new SimulationProfile();

public static final SimulationProfile NO_SHIFT_PROFILE =
new SimulationProfile() {
{
SHIFT_10_PERCENT_KEY_SPACE = 0;
SHIFT_10_PERCENT_POS_SPACE = 0;
SHIFT_AGGRESSIVELY = 0;
}
};

public static void generateShiftAwareTableUpdates(final SimulationProfile profile, final int targetUpdateSize,
final Random random, final QueryTable table,
final ColumnInfo<?, ?>[] columnInfo) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,21 +23,13 @@
public abstract class QueryTableTestBase extends RefreshingTableTestCase {
public final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");

private static final GenerateTableUpdates.SimulationProfile NO_SHIFT_PROFILE =
new GenerateTableUpdates.SimulationProfile() {
{
SHIFT_10_PERCENT_KEY_SPACE = 0;
SHIFT_10_PERCENT_POS_SPACE = 0;
SHIFT_AGGRESSIVELY = 0;
}
};

public final JoinIncrement leftStep = new JoinIncrement() {
@Override
public void step(int leftSize, int rightSize, QueryTable leftTable, QueryTable rightTable,
ColumnInfo<?, ?>[] leftColumnInfo, ColumnInfo<?, ?>[] rightColumnInfo, EvalNuggetInterface[] en,
Random random) {
simulateShiftAwareStep(NO_SHIFT_PROFILE, toString(), leftSize, random, leftTable, leftColumnInfo, en);
simulateShiftAwareStep(GenerateTableUpdates.NO_SHIFT_PROFILE, toString(), leftSize, random, leftTable,
leftColumnInfo, en);
}

@Override
Expand All @@ -63,7 +55,8 @@ public String toString() {
public void step(int leftSize, int rightSize, QueryTable leftTable, QueryTable rightTable,
ColumnInfo<?, ?>[] leftColumnInfo, ColumnInfo<?, ?>[] rightColumnInfo, EvalNuggetInterface[] en,
Random random) {
simulateShiftAwareStep(NO_SHIFT_PROFILE, toString(), rightSize, random, rightTable, rightColumnInfo, en);
simulateShiftAwareStep(GenerateTableUpdates.NO_SHIFT_PROFILE, toString(), rightSize, random, rightTable,
rightColumnInfo, en);
}

@Override
Expand All @@ -89,8 +82,10 @@ public String toString() {
public void step(int leftSize, int rightSize, QueryTable leftTable, QueryTable rightTable,
ColumnInfo<?, ?>[] leftColumnInfo, ColumnInfo<?, ?>[] rightColumnInfo, EvalNuggetInterface[] en,
Random random) {
simulateShiftAwareStep(NO_SHIFT_PROFILE, toString(), leftSize, random, leftTable, leftColumnInfo, en);
simulateShiftAwareStep(NO_SHIFT_PROFILE, toString(), rightSize, random, rightTable, rightColumnInfo, en);
simulateShiftAwareStep(GenerateTableUpdates.NO_SHIFT_PROFILE, toString(), leftSize, random, leftTable,
leftColumnInfo, en);
simulateShiftAwareStep(GenerateTableUpdates.NO_SHIFT_PROFILE, toString(), rightSize, random, rightTable,
rightColumnInfo, en);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ public static void simulateShiftAwareStep(final String ctxt, int targetUpdateSiz
en);
}

protected static void simulateShiftAwareStep(final GenerateTableUpdates.SimulationProfile simulationProfile,
public static void simulateShiftAwareStep(final GenerateTableUpdates.SimulationProfile simulationProfile,
final String ctxt, int targetUpdateSize, Random random, QueryTable table, ColumnInfo[] columnInfo,
EvalNuggetInterface[] en) {
final ControlledUpdateGraph updateGraph = ExecutionContext.getContext().getUpdateGraph().cast();
Expand Down
Loading