From 4c744ca8d3968c31bcc43cfa977b323feeb1eefa Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Thu, 3 Mar 2022 15:40:45 -0800 Subject: [PATCH 01/47] working for data tests, failing others --- .../client/examples/SubscribeExampleBase.java | 2 +- .../barrage/BarrageMessageProducer.java | 368 +++++++++++++----- .../barrage/BarrageStreamGenerator.java | 9 +- 3 files changed, 269 insertions(+), 110 deletions(-) diff --git a/java-client/barrage-examples/src/main/java/io/deephaven/client/examples/SubscribeExampleBase.java b/java-client/barrage-examples/src/main/java/io/deephaven/client/examples/SubscribeExampleBase.java index ca5d442c08e..d5b80904e1c 100644 --- a/java-client/barrage-examples/src/main/java/io/deephaven/client/examples/SubscribeExampleBase.java +++ b/java-client/barrage-examples/src/main/java/io/deephaven/client/examples/SubscribeExampleBase.java @@ -53,7 +53,7 @@ protected void execute(final BarrageSession client) throws Exception { final BarrageTable table; if (headerSize > 0) { // create a table subscription with forward viewport of the specified size - table = subscription.partialTable(RowSetFactory.flat(headerSize), null, true); + table = subscription.partialTable(RowSetFactory.flat(headerSize), null, false); } else if (tailSize > 0) { // create a table subscription with reverse viewport of the specified size table = subscription.partialTable(RowSetFactory.flat(tailSize), null, true); diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index 3c3bb104bce..fd9fc8db86c 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -223,7 +223,7 @@ public MemoizedOperationKey getMemoizedOperationKey() { public Result> initialize(final boolean usePrev, final long beforeClock) { final BarrageMessageProducer result = new BarrageMessageProducer<>( - scheduler, streamGeneratorFactory, parent, updateIntervalMs, onGetSnapshot); + scheduler, streamGeneratorFactory, parent, updateIntervalMs, onGetSnapshot); return new Result<>(result, result.constructListener()); } } @@ -325,14 +325,15 @@ public void close() { private RowSet activeViewport = null; private RowSet activeReverseViewport = null; - private RowSet postSnapshotViewport = null; - private RowSet postSnapshotReverseViewport = null; + private WritableRowSet postSnapshotViewport = null; + private WritableRowSet postSnapshotReverseViewport = null; private final BitSet activeColumns = new BitSet(); private final BitSet postSnapshotColumns = new BitSet(); private final BitSet objectColumnsToClear = new BitSet(); private long numFullSubscriptions = 0; + private long numGrowingSubscriptions = 0; private List pendingSubscriptions = new ArrayList<>(); private final ArrayList activeSubscriptions = new ArrayList<>(); @@ -432,14 +433,21 @@ private class Subscription { boolean pendingDelete = false; // is this subscription deleted as far as the client is concerned? boolean hasPendingUpdate = false; // is this subscription in our pending list? boolean pendingInitialSnapshot = true; // do we need to send the initial snapshot? + RowSet pendingViewport; // if an update is pending this is our new viewport boolean pendingReverseViewport; // is the pending viewport reversed (indexed from end of table) BitSet pendingColumns; // if an update is pending this is our new column subscription set - RowSet snapshotViewport = null; // captured viewport during snapshot portion of propagation job + WritableRowSet snapshotViewport = null; // captured viewport during snapshot portion of propagation job BitSet snapshotColumns = null; // captured column during snapshot portion of propagation job boolean snapshotReverseViewport = false; // captured setting during snapshot portion of propagation job + RowSet targetViewport; // target viewport for a growing subscription + boolean targetReverseViewport; // is the target viewport reversed (indexed from end of table) + BitSet targetColumns; // target column subscription set + + boolean isGrowingViewport; // does this have a growing viewport? + private Subscription(final StreamObserver listener, final BarrageSubscriptionOptions options, final BitSet subscribedColumns, @@ -511,20 +519,20 @@ private boolean findAndUpdateSubscription(final StreamObserver list final Consumer updateSubscription) { final Function, Boolean> findAndUpdate = (List subscriptions) -> { for (final Subscription sub : subscriptions) { - if (sub.listener == listener) { - updateSubscription.accept(sub); - if (!sub.hasPendingUpdate) { - sub.hasPendingUpdate = true; - pendingSubscriptions.add(sub); - } + if (sub.listener == listener) { + updateSubscription.accept(sub); + if (!sub.hasPendingUpdate) { + sub.hasPendingUpdate = true; + pendingSubscriptions.add(sub); + } - updatePropagationJob.scheduleImmediately(); - return true; - } - } + updatePropagationJob.scheduleImmediately(); + return true; + } + } - return false; - }; + return false; + }; synchronized (this) { return findAndUpdate.apply(activeSubscriptions) || findAndUpdate.apply(pendingSubscriptions); @@ -988,17 +996,12 @@ private void updateSubscriptionsSnapshotAndPropagate() { log.info().append(logPrefix).append("Starting update job at " + lastUpdateTime).endl(); } - boolean needsSnapshot = false; - boolean needsFullSnapshot = false; boolean firstSubscription = false; - BitSet snapshotColumns = null; - RowSetBuilderRandom snapshotRows = null; - RowSetBuilderRandom reverseSnapshotRows = null; - - List updatedSubscriptions = null; - // first, we take out any new subscriptions (under the lock) + // check for pending changes (under the lock) synchronized (this) { + List updatedSubscriptions = null; + if (!pendingSubscriptions.isEmpty()) { updatedSubscriptions = this.pendingSubscriptions; pendingSubscriptions = new ArrayList<>(); @@ -1012,15 +1015,14 @@ private void updateSubscriptionsSnapshotAndPropagate() { } catch (final Exception ignored) { // ignore races on cancellation } + // remove this from the "growing" list (if present) + subscription.isGrowingViewport = false; continue; } - if (!needsSnapshot) { - needsSnapshot = true; - snapshotColumns = new BitSet(); - snapshotRows = RowSetFactory.builderRandom(); - reverseSnapshotRows = RowSetFactory.builderRandom(); - } + // add this subscription to the "growing" list to handle snapshot creation + subscription.isGrowingViewport = true; + ++numGrowingSubscriptions; subscription.hasPendingUpdate = false; if (!subscription.isActive) { @@ -1032,75 +1034,67 @@ private void updateSubscriptionsSnapshotAndPropagate() { if (!subscription.isViewport()) { ++numFullSubscriptions; - needsFullSnapshot = true; } } if (subscription.pendingViewport != null) { - subscription.snapshotViewport = subscription.pendingViewport; + subscription.targetViewport = subscription.pendingViewport; subscription.pendingViewport = null; - if (!needsFullSnapshot) { - // track forward and reverse viewport rows separately - if (subscription.pendingReverseViewport) { - reverseSnapshotRows.addRowSet(subscription.snapshotViewport); - } else { - snapshotRows.addRowSet(subscription.snapshotViewport); - } - } } if (subscription.pendingColumns != null) { - subscription.snapshotColumns = subscription.pendingColumns; + subscription.targetColumns = subscription.pendingColumns; subscription.pendingColumns = null; - snapshotColumns.or(subscription.snapshotColumns); - if (!subscription.isViewport()) { - needsFullSnapshot = true; - } } - subscription.snapshotReverseViewport = subscription.pendingReverseViewport; - } // end updatedSubscriptions loop - - boolean haveViewport = false; - postSnapshotColumns.clear(); + subscription.targetReverseViewport = subscription.pendingReverseViewport; - final RowSetBuilderRandom postSnapshotViewportBuilder = RowSetFactory.builderRandom(); - final RowSetBuilderRandom postSnapshotReverseViewportBuilder = RowSetFactory.builderRandom(); + } + // remove deleted subscriptions while we still hold the lock for (int i = 0; i < activeSubscriptions.size(); ++i) { final Subscription sub = activeSubscriptions.get(i); if (sub.pendingDelete) { if (!sub.isViewport()) { --numFullSubscriptions; } + if (sub.isGrowingViewport) { + --numGrowingSubscriptions; + } activeSubscriptions.set(i, activeSubscriptions.get(activeSubscriptions.size() - 1)); activeSubscriptions.remove(activeSubscriptions.size() - 1); --i; continue; } + } + } - if (sub.isViewport()) { - haveViewport = true; - // handle forward and reverse snapshots separately - if (sub.snapshotReverseViewport) { - postSnapshotReverseViewportBuilder - .addRowSet(sub.snapshotViewport != null ? sub.snapshotViewport : sub.viewport); - } else { - postSnapshotViewportBuilder - .addRowSet(sub.snapshotViewport != null ? sub.snapshotViewport : sub.viewport); - } + // need to rebuild the forward/reverse viewports and column sets + final RowSetBuilderRandom postSnapshotViewportBuilder = RowSetFactory.builderRandom(); + final RowSetBuilderRandom postSnapshotReverseViewportBuilder = RowSetFactory.builderRandom(); + + postSnapshotColumns.clear(); + for (final Subscription sub : activeSubscriptions) { + postSnapshotColumns.or(sub.snapshotColumns != null ? sub.snapshotColumns : sub.subscribedColumns); + if (sub.isViewport()) { + // handle forward and reverse snapshots separately + if (sub.snapshotReverseViewport) { + postSnapshotReverseViewportBuilder + .addRowSet(sub.viewport); + } else { + postSnapshotViewportBuilder + .addRowSet(sub.viewport); } - postSnapshotColumns.or(sub.snapshotColumns != null ? sub.snapshotColumns : sub.subscribedColumns); } + } - postSnapshotViewport = haveViewport ? postSnapshotViewportBuilder.build() : null; - postSnapshotReverseViewport = haveViewport ? postSnapshotReverseViewportBuilder.build() : null; + postSnapshotViewport = postSnapshotViewportBuilder.build(); + postSnapshotReverseViewport = postSnapshotReverseViewportBuilder.build(); - if (!needsSnapshot) { - // i.e. We have only removed subscriptions; we can update this state immediately. - promoteSnapshotToActive(); - } + if (numGrowingSubscriptions == 0) { + // i.e. We have only removed subscriptions; we can update this state immediately. + promoteSnapshotToActive(); } } @@ -1109,18 +1103,136 @@ private void updateSubscriptionsSnapshotAndPropagate() { BarrageMessage snapshot = null; BarrageMessage postSnapshot = null; - // then we spend the effort to take a snapshot - if (needsSnapshot) { - try (final RowSet snapshotRowSet = snapshotRows.build(); - final RowSet reverseSnapshotRowSet = reverseSnapshotRows.build()) { - snapshot = - getSnapshot(updatedSubscriptions, snapshotColumns, needsFullSnapshot ? null : snapshotRowSet, - needsFullSnapshot ? null : reverseSnapshotRowSet); + BitSet snapshotColumns = null; + + ArrayList growingSubscriptions = + new ArrayList<>(); + + if (numGrowingSubscriptions > 0) { + snapshotColumns = new BitSet(); + + ArrayList reverse = new ArrayList<>(); + ArrayList forward = new ArrayList<>(); + ArrayList full = new ArrayList<>(); + + for (final Subscription subscription : activeSubscriptions) { + if (subscription.isGrowingViewport) { + // build the column set from all columns needed by the growing subscriptions + snapshotColumns.or(subscription.targetColumns); + + if (subscription.targetViewport == null) { + full.add(subscription); + } else if (subscription.targetReverseViewport) { + reverse.add(subscription); + } else { + forward.add(subscription); + } + } + } + + // have to determine priorities between reverse / forward / full snapshots + growingSubscriptions.addAll(reverse); + growingSubscriptions.addAll(forward); + growingSubscriptions.addAll(full); + + // TODO: determine this by LTM usage percentage + long MAX_CELLS = 100; + long rowsRemaining = MAX_CELLS / snapshotColumns.cardinality(); + + // some builders to help generate the rowsets we need + RowSetBuilderRandom viewportBuilder = RowSetFactory.builderRandom(); + RowSetBuilderRandom reverseViewportBuilder = RowSetFactory.builderRandom(); + + try (final WritableRowSet snapshotRowSet = RowSetFactory.empty(); + final WritableRowSet reverseSnapshotRowSet = RowSetFactory.empty()) { + + final long GLOBAL_MAX = Long.MAX_VALUE - 1; + + // satisfy the subscriptions in priority order + for (final Subscription subscription : growingSubscriptions) { + try (final WritableRowSet missingRows = subscription.targetViewport == null + ? RowSetFactory.flat(parent.size()) + : subscription.targetViewport.copy()) { + + // TODO: check this logic, if column set changes then previous viewport is also invalid + final boolean viewportValid = + subscription.reverseViewport == subscription.targetReverseViewport; + + if (!viewportValid) { + System.out.println(); + } + + // if we haven't reversed viewports, can exclude current viewport + if (viewportValid && subscription.viewport != null) { + missingRows.remove(subscription.viewport); + } + + // remove rows that we already have selected for the upcoming snapshot + missingRows.remove(subscription.targetReverseViewport ? reverseSnapshotRowSet : snapshotRowSet); + + // shrink this to the `rowsRemaining` value + if (missingRows.size() > rowsRemaining) { + final long key = missingRows.get(rowsRemaining); + missingRows.removeRange(key, GLOBAL_MAX); + } + + // "grow" the snapshot viewport (snapshotViewport is always null) + subscription.snapshotViewport = viewportValid && subscription.viewport != null + ? subscription.viewport.copy() + : RowSetFactory.empty(); + subscription.snapshotViewport.insert(missingRows); + + // add all the rows from the current snapshot + subscription.snapshotViewport + .insert(subscription.targetReverseViewport ? reverseSnapshotRowSet : snapshotRowSet); + + if (subscription.targetReverseViewport) { + if (subscription.targetViewport != null) { + // add this set to the BMP viewport + reverseViewportBuilder.addRowSet(missingRows); + + // retain only the rows that apply to this sub + subscription.snapshotViewport.retain(subscription.targetViewport); + } + + // add the new rows to the upcoming snapshot + reverseSnapshotRowSet.insert(missingRows); + } else { + if (subscription.targetViewport != null) { + // add this set to the BMP viewport + viewportBuilder.addRowSet(missingRows); + + // retain only the rows that apply to this sub + subscription.snapshotViewport.retain(subscription.targetViewport); + } + + // add the new rows to the upcoming snapshot + snapshotRowSet.insert(missingRows); + } + + subscription.snapshotColumns = (BitSet) subscription.targetColumns.clone(); + + rowsRemaining -= missingRows.size(); + + subscription.snapshotReverseViewport = subscription.targetReverseViewport; + } + } + + // update the postSnapshot viewports/columns to include the new viewports (excluding `full`) + try (final RowSet vp = viewportBuilder.build(); final RowSet rvp = reverseViewportBuilder.build()) { + postSnapshotViewport.insert(vp); + postSnapshotReverseViewport.insert(rvp); + } + + postSnapshotColumns.or(snapshotColumns); + + // finally, grab the snapshot + snapshot = getSnapshot(growingSubscriptions, snapshotColumns, snapshotRowSet, reverseSnapshotRowSet); } } synchronized (this) { - if (!needsSnapshot && pendingDeltas.isEmpty() && pendingError == null) { + if (growingSubscriptions.size() == 0 && pendingDeltas.isEmpty() && pendingError == null) { return; } @@ -1136,7 +1248,7 @@ private void updateSubscriptionsSnapshotAndPropagate() { // flip snapshot state so that we build the preSnapshot using previous viewports/columns if (snapshot != null && deltaSplitIdx > 0) { - flipSnapshotStateForSubscriptions(updatedSubscriptions); + flipSnapshotStateForSubscriptions(growingSubscriptions); } if (!firstSubscription && deltaSplitIdx > 0) { @@ -1154,7 +1266,7 @@ private void updateSubscriptionsSnapshotAndPropagate() { // flip back for the UGP thread's processing before releasing the lock if (snapshot != null && deltaSplitIdx > 0) { - flipSnapshotStateForSubscriptions(updatedSubscriptions); + flipSnapshotStateForSubscriptions(growingSubscriptions); } if (deltaSplitIdx < pendingDeltas.size()) { @@ -1163,7 +1275,7 @@ private void updateSubscriptionsSnapshotAndPropagate() { // cleanup for next iteration clearObjectDeltaColumns(objectColumnsToClear); - if (updatedSubscriptions != null) { + if (growingSubscriptions.size() > 0) { objectColumnsToClear.clear(); objectColumnsToClear.or(objectColumns); objectColumnsToClear.and(activeColumns); @@ -1184,7 +1296,7 @@ private void updateSubscriptionsSnapshotAndPropagate() { if (snapshot != null) { try (final StreamGenerator snapshotGenerator = streamGeneratorFactory.newGenerator(snapshot)) { - for (final Subscription subscription : updatedSubscriptions) { + for (final Subscription subscription : growingSubscriptions) { if (subscription.pendingDelete) { continue; } @@ -1206,6 +1318,10 @@ private void updateSubscriptionsSnapshotAndPropagate() { } } + if (numGrowingSubscriptions > 0) { + updatePropagationJob.scheduleImmediately(); + } + lastUpdateTime = scheduler.currentTime().getMillis(); if (DEBUG) { log.info().append(logPrefix).append("Completed Propagation: " + lastUpdateTime); @@ -1214,7 +1330,8 @@ private void updateSubscriptionsSnapshotAndPropagate() { private void propagateToSubscribers(final BarrageMessage message, final RowSet propRowSetForMessage) { // message is released via transfer to stream generator (as it must live until all view's are closed) - try (final StreamGenerator generator = streamGeneratorFactory.newGenerator(message)) { + try (final StreamGenerator generator = + streamGeneratorFactory.newGenerator(message)) { for (final Subscription subscription : activeSubscriptions) { if (subscription.pendingInitialSnapshot || subscription.pendingDelete) { continue; @@ -1264,7 +1381,8 @@ private void clearObjectDeltaColumns(@NotNull final BitSet objectColumnsToClear) } } - private void propagateSnapshotForSubscription(final Subscription subscription, + private void propagateSnapshotForSubscription( + final Subscription subscription, final StreamGenerator snapshotGenerator) { boolean needsSnapshot = subscription.pendingInitialSnapshot; @@ -1635,14 +1753,16 @@ private static void applyRedirMapping(final RowSet keys, final RowSet values, fi }); } - private void flipSnapshotStateForSubscriptions(final List subscriptions) { + private void flipSnapshotStateForSubscriptions( + final List subscriptions) { for (final Subscription subscription : subscriptions) { if (subscription.snapshotViewport != null) { final RowSet tmp = subscription.viewport; subscription.viewport = subscription.snapshotViewport; subscription.reverseViewport = subscription.snapshotReverseViewport; - subscription.snapshotViewport = tmp; + subscription.snapshotViewport = (WritableRowSet) tmp; } + if (subscription.snapshotColumns != null) { final BitSet tmp = subscription.subscribedColumns; subscription.subscribedColumns = subscription.snapshotColumns; @@ -1651,34 +1771,70 @@ private void flipSnapshotStateForSubscriptions(final List subscrip } } + private void finalizeSnapshotForSubscriptions( + final List subscriptions) { + for (final Subscription subscription : subscriptions) { + // test whether we have completed the viewport + if (subscription.targetViewport == null) { // full subscription + // are all the rows in parent covered by the current viewport? + try (final RowSet keysInViewport = parent.getRowSet().subSetForPositions(subscription.viewport)) { + if (parent.getRowSet().subsetOf(keysInViewport)) { + // remove this subscription from the growing list + subscription.isGrowingViewport = false; + --numGrowingSubscriptions; + + // change the viewport to `null` to signify full subscription + try (final RowSet ignored = subscription.viewport) { + subscription.viewport = null; + } + } + + } + } else { + if (subscription.targetViewport.subsetOf(subscription.viewport)) { + // remove this subscription from the growing list + subscription.isGrowingViewport = false; + --numGrowingSubscriptions; + } + } + } + } + private void promoteSnapshotToActive() { Assert.holdsLock(this, "promoteSnapshotToActive must hold lock!"); - if (this.activeViewport != null) { - this.activeViewport.close(); + if (activeViewport != null) { + activeViewport.close(); } - if (this.activeReverseViewport != null) { - this.activeReverseViewport.close(); + if (activeReverseViewport != null) { + activeReverseViewport.close(); } - this.activeViewport = this.postSnapshotViewport == null || this.postSnapshotViewport.isEmpty() ? null - : this.postSnapshotViewport; + activeViewport = postSnapshotViewport == null || postSnapshotViewport.isEmpty() ? null + : postSnapshotViewport; - this.activeReverseViewport = - this.postSnapshotReverseViewport == null || this.postSnapshotReverseViewport.isEmpty() ? null - : this.postSnapshotReverseViewport; + activeReverseViewport = + postSnapshotReverseViewport == null || postSnapshotReverseViewport.isEmpty() ? null + : postSnapshotReverseViewport; - this.postSnapshotViewport = null; - this.postSnapshotReverseViewport = null; + if (postSnapshotViewport != null && postSnapshotViewport.isEmpty()) { + postSnapshotViewport.close(); + } + postSnapshotViewport = null; + + if (postSnapshotReverseViewport != null && postSnapshotReverseViewport.isEmpty()) { + postSnapshotReverseViewport.close(); + } + postSnapshotReverseViewport = null; // Pre-condition: activeObjectColumns == objectColumns & activeColumns - this.objectColumnsToClear.or(postSnapshotColumns); - this.objectColumnsToClear.and(objectColumns); + objectColumnsToClear.or(postSnapshotColumns); + objectColumnsToClear.and(objectColumns); // Post-condition: activeObjectColumns == objectColumns & (activeColumns | postSnapshotColumns) - this.activeColumns.clear(); - this.activeColumns.or(this.postSnapshotColumns); - this.postSnapshotColumns.clear(); + activeColumns.clear(); + activeColumns.or(postSnapshotColumns); + postSnapshotColumns.clear(); } private synchronized long getLastIndexClockStep() { @@ -1690,7 +1846,8 @@ private class SnapshotControl implements ConstructSnapshot.SnapshotControl { long step = -1; final List snapshotSubscriptions; - SnapshotControl(final List snapshotSubscriptions) { + SnapshotControl( + final List snapshotSubscriptions) { this.snapshotSubscriptions = snapshotSubscriptions; } @@ -1735,6 +1892,7 @@ public boolean snapshotCompletedConsistently(final long afterClockValue, final b step = -1; } else { flipSnapshotStateForSubscriptions(snapshotSubscriptions); + finalizeSnapshotForSubscriptions(snapshotSubscriptions); promoteSnapshotToActive(); } } @@ -1747,7 +1905,8 @@ public boolean snapshotCompletedConsistently(final long afterClockValue, final b } @VisibleForTesting - BarrageMessage getSnapshot(final List snapshotSubscriptions, + BarrageMessage getSnapshot( + final List snapshotSubscriptions, final BitSet columnsToSnapshot, final RowSet positionsToSnapshot, final RowSet reversePositionsToSnapshot) { @@ -1757,7 +1916,8 @@ BarrageMessage getSnapshot(final List snapshotSubscriptions, // TODO: Use *this* as snapshot tick source for fail fast. // TODO: Let notification-indifferent use cases skip notification test - final SnapshotControl snapshotControl = new SnapshotControl(snapshotSubscriptions); + final SnapshotControl snapshotControl = + new SnapshotControl(snapshotSubscriptions); return ConstructSnapshot.constructBackplaneSnapshotInPositionSpace( this, parent, columnsToSnapshot, positionsToSnapshot, reversePositionsToSnapshot, snapshotControl); diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java index a1ecef5e325..02695c3fecc 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java @@ -532,9 +532,6 @@ private long appendAddColumns(final View view, try (WritableRowSet intersect = view.keyspaceViewport().intersect(rowsIncluded.original)) { myAddedOffsets = rowsIncluded.original.invert(intersect); } - } else if (!rowsAdded.original.equals(rowsIncluded.original)) { - // there are scoped rows included in the chunks that need to be removed - myAddedOffsets = rowsIncluded.original.invert(rowsAdded.original); } else { // use chunk data as-is myAddedOffsets = null; @@ -550,7 +547,7 @@ private long appendAddColumns(final View view, // Add the drainable last as it is allowed to immediately close a row set the visitors need addStream.accept(drainableColumn); } - return myAddedOffsets != null ? myAddedOffsets.size() : rowsAdded.original.size(); + return myAddedOffsets != null ? myAddedOffsets.size() : rowsIncluded.original.size(); } finally { if (myAddedOffsets != null) { myAddedOffsets.close(); @@ -640,7 +637,9 @@ private ByteBuffer getSubscriptionMetadata(final SubView view) throws IOExceptio // Added Chunk Data: int addedRowsIncludedOffset = 0; - if (view.isViewport()) { + if (view.keyspaceViewport == null) { + addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(metadata); + } else { addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(view.keyspaceViewport, metadata); } From c8b4bc9052ce2cbad8de5e26a393b4dfe07c8c46 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Thu, 3 Mar 2022 16:23:49 -0800 Subject: [PATCH 02/47] spotless checkin --- .../barrage/BarrageMessageProducer.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index fd9fc8db86c..841b7bb9171 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -223,7 +223,7 @@ public MemoizedOperationKey getMemoizedOperationKey() { public Result> initialize(final boolean usePrev, final long beforeClock) { final BarrageMessageProducer result = new BarrageMessageProducer<>( - scheduler, streamGeneratorFactory, parent, updateIntervalMs, onGetSnapshot); + scheduler, streamGeneratorFactory, parent, updateIntervalMs, onGetSnapshot); return new Result<>(result, result.constructListener()); } } @@ -519,20 +519,20 @@ private boolean findAndUpdateSubscription(final StreamObserver list final Consumer updateSubscription) { final Function, Boolean> findAndUpdate = (List subscriptions) -> { for (final Subscription sub : subscriptions) { - if (sub.listener == listener) { - updateSubscription.accept(sub); - if (!sub.hasPendingUpdate) { - sub.hasPendingUpdate = true; - pendingSubscriptions.add(sub); - } - - updatePropagationJob.scheduleImmediately(); - return true; - } + if (sub.listener == listener) { + updateSubscription.accept(sub); + if (!sub.hasPendingUpdate) { + sub.hasPendingUpdate = true; + pendingSubscriptions.add(sub); } - return false; - }; + updatePropagationJob.scheduleImmediately(); + return true; + } + } + + return false; + }; synchronized (this) { return findAndUpdate.apply(activeSubscriptions) || findAndUpdate.apply(pendingSubscriptions); From c006ab305bfdfa1479e5801b0450e11efbc41153 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Fri, 4 Mar 2022 12:33:30 -0800 Subject: [PATCH 03/47] woohoo, _all_ tests passing --- .../barrage/BarrageMessageProducer.java | 10 +++--- .../barrage/BarrageStreamGenerator.java | 35 +++++++++++++++++-- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index 841b7bb9171..f9cf97969e0 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -997,6 +997,7 @@ private void updateSubscriptionsSnapshotAndPropagate() { } boolean firstSubscription = false; + boolean pendingChanges = false; // check for pending changes (under the lock) synchronized (this) { @@ -1005,6 +1006,7 @@ private void updateSubscriptionsSnapshotAndPropagate() { if (!pendingSubscriptions.isEmpty()) { updatedSubscriptions = this.pendingSubscriptions; pendingSubscriptions = new ArrayList<>(); + pendingChanges = true; } if (updatedSubscriptions != null) { @@ -1146,12 +1148,10 @@ private void updateSubscriptionsSnapshotAndPropagate() { try (final WritableRowSet snapshotRowSet = RowSetFactory.empty(); final WritableRowSet reverseSnapshotRowSet = RowSetFactory.empty()) { - final long GLOBAL_MAX = Long.MAX_VALUE - 1; - // satisfy the subscriptions in priority order for (final Subscription subscription : growingSubscriptions) { try (final WritableRowSet missingRows = subscription.targetViewport == null - ? RowSetFactory.flat(parent.size()) + ? RowSetFactory.flat(Long.MAX_VALUE) : subscription.targetViewport.copy()) { // TODO: check this logic, if column set changes then previous viewport is also invalid @@ -1173,7 +1173,7 @@ private void updateSubscriptionsSnapshotAndPropagate() { // shrink this to the `rowsRemaining` value if (missingRows.size() > rowsRemaining) { final long key = missingRows.get(rowsRemaining); - missingRows.removeRange(key, GLOBAL_MAX); + missingRows.removeRange(key, Long.MAX_VALUE - 1); } // "grow" the snapshot viewport (snapshotViewport is always null) @@ -1275,7 +1275,7 @@ private void updateSubscriptionsSnapshotAndPropagate() { // cleanup for next iteration clearObjectDeltaColumns(objectColumnsToClear); - if (growingSubscriptions.size() > 0) { + if (pendingChanges) { objectColumnsToClear.clear(); objectColumnsToClear.or(objectColumns); objectColumnsToClear.and(activeColumns); diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java index 02695c3fecc..23a137e944b 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java @@ -69,6 +69,8 @@ public interface View { boolean isViewport(); + boolean isInitialSnapshot(); + StreamReaderOptions options(); RowSet keyspaceViewport(); @@ -269,6 +271,10 @@ public boolean isViewport() { return this.viewport != null; } + public boolean isInitialSnapshot() { + return this.isInitialSnapshot; + } + public final StreamReaderOptions options() { return options; } @@ -345,6 +351,10 @@ public boolean isViewport() { return this.viewport != null; } + public boolean isInitialSnapshot() { + return true; + } + public final StreamReaderOptions options() { return options; } @@ -374,6 +384,10 @@ public boolean isViewport() { return false; } + public boolean isInitialSnapshot() { + return false; + } + @Override public StreamReaderOptions options() { return null; @@ -533,8 +547,15 @@ private long appendAddColumns(final View view, myAddedOffsets = rowsIncluded.original.invert(intersect); } } else { - // use chunk data as-is - myAddedOffsets = null; + if (view.isInitialSnapshot()) { + // use chunk data as-is, rely on snapshot process to avoid duplicates + myAddedOffsets = null; + } else { + // there may be scoped rows included in the chunks that need to be removed + try (WritableRowSet intersect = rowsIncluded.original.intersect(rowsAdded.original)) { + myAddedOffsets = rowsIncluded.original.invert(intersect); + } + } } // add the add-column streams @@ -638,7 +659,15 @@ private ByteBuffer getSubscriptionMetadata(final SubView view) throws IOExceptio // Added Chunk Data: int addedRowsIncludedOffset = 0; if (view.keyspaceViewport == null) { - addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(metadata); + if (view.isInitialSnapshot()) { + // use chunk data as-is, rely on snapshot process to avoid duplicates + addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(metadata); + } else { + // there may be scoped rows included in the chunks that need to be removed + try (WritableRowSet intersect = rowsIncluded.original.intersect(rowsAdded.original)) { + addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(intersect, metadata); + } + } } else { addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(view.keyspaceViewport, metadata); } From 554cff172d91043a697bf36d5021277b723c6e85 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Tue, 8 Mar 2022 12:17:52 -0800 Subject: [PATCH 04/47] time slicing added (beta code) --- .../client/examples/SubscribeExampleBase.java | 6 +- .../barrage/BarrageMessageProducer.java | 67 ++++++++++++------- 2 files changed, 47 insertions(+), 26 deletions(-) diff --git a/java-client/barrage-examples/src/main/java/io/deephaven/client/examples/SubscribeExampleBase.java b/java-client/barrage-examples/src/main/java/io/deephaven/client/examples/SubscribeExampleBase.java index d5b80904e1c..670f5c5ea93 100644 --- a/java-client/barrage-examples/src/main/java/io/deephaven/client/examples/SubscribeExampleBase.java +++ b/java-client/barrage-examples/src/main/java/io/deephaven/client/examples/SubscribeExampleBase.java @@ -64,7 +64,7 @@ protected void execute(final BarrageSession client) throws Exception { final CountDownLatch countDownLatch = new CountDownLatch(1); - table.listenForUpdates(new InstrumentedTableUpdateListener("example-listener") { + InstrumentedTableUpdateListener listener = new InstrumentedTableUpdateListener("example-listener") { @Override protected void onFailureInternal(final Throwable originalException, final Entry sourceEntry) { System.out.println("exiting due to onFailureInternal:"); @@ -77,7 +77,9 @@ public void onUpdate(final TableUpdate upstream) { System.out.println("Received table update:"); System.out.println(upstream); } - }); + }; + + table.listenForUpdates(listener); countDownLatch.await(); } diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index f9cf97969e0..3c2d5a623b9 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -84,6 +84,15 @@ public class BarrageMessageProducer extends LivenessArtifact private static final Logger log = LoggerFactory.getLogger(BarrageMessageProducer.class); + private static final long TARGET_SNAPSHOT_MILLIS = + Configuration.getInstance().getLongForClassWithDefault(BarrageMessageProducer.class, + "target_snapshot_millis", 25); + private static final long MIN_SNAPSHOT_ROWS = + Configuration.getInstance().getLongForClassWithDefault(BarrageMessageProducer.class, + "min_snapshot_rows", 1000); + + private long nextCellsTarget; + /** * A StreamGenerator takes a BarrageMessage and re-uses portions of the serialized payload across different * subscribers that may subscribe to different viewports and columns. @@ -1107,39 +1116,33 @@ private void updateSubscriptionsSnapshotAndPropagate() { BitSet snapshotColumns = null; - ArrayList growingSubscriptions = - new ArrayList<>(); + // create a prioritized list for the subscriptions + TreeMap growingSubscriptionsMap = new TreeMap<>(Collections.reverseOrder()); + List growingSubscriptions = new ArrayList<>();; if (numGrowingSubscriptions > 0) { snapshotColumns = new BitSet(); - ArrayList reverse = new ArrayList<>(); - ArrayList forward = new ArrayList<>(); - ArrayList full = new ArrayList<>(); - for (final Subscription subscription : activeSubscriptions) { if (subscription.isGrowingViewport) { // build the column set from all columns needed by the growing subscriptions snapshotColumns.or(subscription.targetColumns); if (subscription.targetViewport == null) { - full.add(subscription); - } else if (subscription.targetReverseViewport) { - reverse.add(subscription); + growingSubscriptionsMap.put(0, subscription); // full, lowest priority + } else if (!subscription.targetReverseViewport) { + growingSubscriptionsMap.put(1, subscription); // forward VP, medium priority } else { - forward.add(subscription); + growingSubscriptionsMap.put(2, subscription); // reverse VP, high priority } } } - // have to determine priorities between reverse / forward / full snapshots - growingSubscriptions.addAll(reverse); - growingSubscriptions.addAll(forward); - growingSubscriptions.addAll(full); + growingSubscriptions.addAll(growingSubscriptionsMap.values()); - // TODO: determine this by LTM usage percentage - long MAX_CELLS = 100; - long rowsRemaining = MAX_CELLS / snapshotColumns.cardinality(); + // MIN_SNAPSHOT_ROWS helps prevent viewports from getting partial delivery (breaking js client) + final long minCells = MIN_SNAPSHOT_ROWS * snapshotColumns.cardinality(); + long rowsRemaining = Math.max(minCells, nextCellsTarget / Math.max(1, snapshotColumns.cardinality())); // some builders to help generate the rowsets we need RowSetBuilderRandom viewportBuilder = RowSetFactory.builderRandom(); @@ -1154,13 +1157,16 @@ private void updateSubscriptionsSnapshotAndPropagate() { ? RowSetFactory.flat(Long.MAX_VALUE) : subscription.targetViewport.copy()) { - // TODO: check this logic, if column set changes then previous viewport is also invalid - final boolean viewportValid = - subscription.reverseViewport == subscription.targetReverseViewport; + // TODO: check this logic + // if viewport direction changes then previous viewport is invalid, also if subscribed columns + // change and new columns are added then viewport is invalid - if (!viewportValid) { - System.out.println(); - } + BitSet missingColumns = (BitSet) subscription.targetColumns.clone(); + missingColumns.andNot(subscription.subscribedColumns); + + final boolean viewportValid = + subscription.reverseViewport == subscription.targetReverseViewport + && missingColumns.isEmpty(); // if we haven't reversed viewports, can exclude current viewport if (viewportValid && subscription.viewport != null) { @@ -1226,8 +1232,21 @@ private void updateSubscriptionsSnapshotAndPropagate() { postSnapshotColumns.or(snapshotColumns); - // finally, grab the snapshot + // finally, grab the snapshot and measure elapsed time for next projections + long start = scheduler.currentTime().getNanos(); snapshot = getSnapshot(growingSubscriptions, snapshotColumns, snapshotRowSet, reverseSnapshotRowSet); + long elapsed = scheduler.currentTime().getNanos() - start; + + if (snapshot.rowsIncluded.size() > 0) { + // TODO: determine this by LTM usage percentage + long nanosPerCell = elapsed / (snapshot.rowsIncluded.size() * snapshotColumns.cardinality()); + // capped until we get auto-chunked delivery + if (nanosPerCell > 0) { + nextCellsTarget = Math.min((TARGET_SNAPSHOT_MILLIS * 1000000) / nanosPerCell, 400000L); + nextCellsTarget = Math.max(nextCellsTarget, 1000 * snapshotColumns.cardinality()); + } + } + // System.out.println("nextCellsTarget: " + nextCellsTarget); } } From 37a25cf9d67b17d7935bbcbc82b3b6a62c0a4309 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Wed, 9 Mar 2022 11:46:22 -0800 Subject: [PATCH 05/47] wip, BarrageSubscriptionImpl has blocking subscription --- .../barrage/table/BarrageTable.java | 12 ++ .../client/impl/BarrageSnapshotImpl.java | 1 - .../client/impl/BarrageSubscription.java | 2 +- .../client/impl/BarrageSubscriptionImpl.java | 96 +++++++++++- .../barrage/BarrageMessageProducer.java | 141 ++++++++++-------- 5 files changed, 177 insertions(+), 75 deletions(-) diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java index 1a446578e06..22c2be022ad 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java @@ -166,6 +166,18 @@ public Class[] getWireComponentTypes() { return Arrays.stream(destSources).map(ColumnSource::getComponentType).toArray(Class[]::new); } + public RowSet getServerViewport() { + return serverViewport; + } + + public boolean getServerReverseViewport() { + return serverReverseViewport; + } + + public BitSet getServerColumns() { + return serverColumns; + } + /** * Invoke sealTable to prevent further updates from being processed and to mark this source table as static. * diff --git a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSnapshotImpl.java b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSnapshotImpl.java index b23a93481fe..81b57a5a16a 100644 --- a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSnapshotImpl.java +++ b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSnapshotImpl.java @@ -205,7 +205,6 @@ public synchronized BarrageTable partialTable(RowSet viewport, BitSet columns, b } } - @Override protected void destroy() { super.destroy(); diff --git a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscription.java b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscription.java index a3569db77c6..8f0ac0172ad 100644 --- a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscription.java +++ b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscription.java @@ -45,7 +45,7 @@ BarrageSubscription subscribe(TableSpec tableSpec, BarrageSubscriptionOptions op * * @return the {@code BarrageTable} */ - BarrageTable entireTable(); + BarrageTable entireTable() throws InterruptedException; // TODO (deephaven-core#712): java-client viewport support /** diff --git a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java index 58987a17891..275cf68a3a0 100644 --- a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java +++ b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java @@ -15,8 +15,11 @@ import io.deephaven.engine.liveness.ReferenceCountedLivenessNode; import io.deephaven.engine.rowset.RowSet; import io.deephaven.engine.table.TableDefinition; +import io.deephaven.engine.table.TableUpdate; +import io.deephaven.engine.table.impl.InstrumentedTableUpdateListener; import io.deephaven.engine.table.impl.util.BarrageMessage; import io.deephaven.engine.table.impl.util.BarrageMessage.Listener; +import io.deephaven.engine.updategraph.UpdateGraphProcessor; import io.deephaven.extensions.barrage.BarrageSubscriptionOptions; import io.deephaven.extensions.barrage.table.BarrageTable; import io.deephaven.extensions.barrage.util.*; @@ -37,6 +40,7 @@ import java.io.InputStream; import java.nio.ByteBuffer; import java.util.BitSet; +import java.util.concurrent.locks.Condition; public class BarrageSubscriptionImpl extends ReferenceCountedLivenessNode implements BarrageSubscription { private static final Logger log = LoggerFactory.getLogger(BarrageSubscriptionImpl.class); @@ -48,9 +52,15 @@ public class BarrageSubscriptionImpl extends ReferenceCountedLivenessNode implem private BarrageTable resultTable; + private volatile Condition completedCondition; + private volatile boolean completed = false; + private volatile Throwable exceptionWhileCompleting = null; + private boolean subscribed = false; private volatile boolean connected = true; + private RowSet targetViewport = null; + /** * Represents a BarrageSubscription. * @@ -133,40 +143,106 @@ public void onCompleted() { } @Override - public BarrageTable entireTable() { + public BarrageTable entireTable() throws InterruptedException { return partialTable(null, null, false); } @Override - public synchronized BarrageTable partialTable(RowSet viewport, BitSet columns) { + public synchronized BarrageTable partialTable(RowSet viewport, BitSet columns) throws InterruptedException { return partialTable(viewport, columns, false); } @Override - public synchronized BarrageTable partialTable(RowSet viewport, BitSet columns, boolean reverseViewport) { + public synchronized BarrageTable partialTable(RowSet viewport, BitSet columns, boolean reverseViewport) + throws InterruptedException { if (!connected) { throw new UncheckedDeephavenException( this + " is no longer an active subscription and cannot be retained further"); } if (!subscribed) { + + // test lock conditions + if (UpdateGraphProcessor.DEFAULT.sharedLock().isHeldByCurrentThread()) { + throw new UnsupportedOperationException( + "Cannot create subscription while holding the UpdateGraphProcessor shared lock"); + } + + if (UpdateGraphProcessor.DEFAULT.exclusiveLock().isHeldByCurrentThread()) { + completedCondition = UpdateGraphProcessor.DEFAULT.exclusiveLock().newCondition(); + } + // Send the initial subscription: observer.onNext(FlightData.newBuilder() .setAppMetadata( ByteStringAccess.wrap(makeRequestInternal(viewport, columns, reverseViewport, options))) .build()); subscribed = true; + + // use a listener to decide when the table is complete + InstrumentedTableUpdateListener listener = new InstrumentedTableUpdateListener("example-listener") { + @Override + protected void onFailureInternal(final Throwable originalException, final Entry sourceEntry) { + exceptionWhileCompleting = originalException; + if (completedCondition != null) { + UpdateGraphProcessor.DEFAULT.requestSignal(completedCondition); + } else { + synchronized (BarrageSubscriptionImpl.this) { + BarrageSubscriptionImpl.this.notifyAll(); + } + } + } + + @Override + public void onUpdate(final TableUpdate upstream) { + // test to see if the viewport matches the requested + if (viewport == null && resultTable.getServerViewport() == null) { + completed = true; + } else if (viewport != null && resultTable.getServerViewport() != null) { + completed = viewport.subsetOf(resultTable.getServerViewport()); + } else { + completed = false; + } + + if (completed) { + if (completedCondition != null) { + UpdateGraphProcessor.DEFAULT.requestSignal(completedCondition); + } else { + synchronized (BarrageSubscriptionImpl.this) { + BarrageSubscriptionImpl.this.notifyAll(); + } + } + } + } + }; + + resultTable.listenForUpdates(listener); + + while (!completed && exceptionWhileCompleting == null) { + // handle the condition where this function may have the exclusive lock + if (completedCondition != null) { + completedCondition.await(); + } else { + wait(); // barragesubscriptionimpl lock + } + } + + resultTable.removeUpdateListener(listener); } - return resultTable; + if (exceptionWhileCompleting == null) { + return resultTable; + } else { + throw new UncheckedDeephavenException("Error while handling subscription:", exceptionWhileCompleting); + } } @Override - protected synchronized void destroy() { + protected void destroy() { super.destroy(); close(); } - private synchronized void handleDisconnect() { + private void handleDisconnect() { if (!connected) { return; } @@ -175,7 +251,7 @@ private synchronized void handleDisconnect() { } @Override - public synchronized void close() { + public void close() { if (!connected) { return; } @@ -186,6 +262,8 @@ public synchronized void close() { private void cleanup() { this.connected = false; this.tableHandle.close(); + if (targetViewport != null) + targetViewport.close(); resultTable = null; } @@ -200,6 +278,10 @@ private ByteBuffer makeRequestInternal( @Nullable final BitSet columns, boolean reverseViewport, @Nullable BarrageSubscriptionOptions options) { + + // store the viewport for comparison to received data to determine completion of growing snapshot + targetViewport = viewport != null ? viewport.copy() : null; + final FlatBufferBuilder metadata = new FlatBufferBuilder(); int colOffset = 0; diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index 3c2d5a623b9..4ca2aef28ad 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -31,6 +31,7 @@ import io.deephaven.engine.table.impl.util.UpdateCoalescer; import io.deephaven.engine.updategraph.DynamicNode; import io.deephaven.engine.updategraph.LogicalClock; +import io.deephaven.engine.updategraph.UpdateGraphProcessor; import io.deephaven.extensions.barrage.BarrageSnapshotOptions; import io.deephaven.extensions.barrage.BarrageSubscriptionOptions; import io.deephaven.extensions.barrage.util.GrpcUtil; @@ -38,6 +39,7 @@ import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; import io.deephaven.server.util.Scheduler; +import io.deephaven.tablelogger.Row; import io.deephaven.time.DateTimeUtils; import io.deephaven.util.SafeCloseable; import io.deephaven.util.SafeCloseableArray; @@ -84,14 +86,19 @@ public class BarrageMessageProducer extends LivenessArtifact private static final Logger log = LoggerFactory.getLogger(BarrageMessageProducer.class); - private static final long TARGET_SNAPSHOT_MILLIS = - Configuration.getInstance().getLongForClassWithDefault(BarrageMessageProducer.class, - "target_snapshot_millis", 25); - private static final long MIN_SNAPSHOT_ROWS = + private static final boolean SUBSCRIPTION_GROWTH_ENABLED = + Configuration.getInstance().getBooleanForClassWithDefault(BarrageMessageProducer.class, + "subscriptionGrowthEnabled", false); + + private static final double TARGET_SNAPSHOT_PERCENTAGE = + Configuration.getInstance().getDoubleForClassWithDefault(BarrageMessageProducer.class, + "targetSnapshotPercentage", 0.25); + private static final long INITIAL_SNAPSHOT_CELL_COUNT = Configuration.getInstance().getLongForClassWithDefault(BarrageMessageProducer.class, - "min_snapshot_rows", 1000); + "initialSnapshotCellCount", 10000); - private long nextCellsTarget; + private long snapshotTargetCellCount = INITIAL_SNAPSHOT_CELL_COUNT; + private double snapshotNanosPerCell = 0; /** * A StreamGenerator takes a BarrageMessage and re-uses portions of the serialized payload across different @@ -1129,7 +1136,7 @@ private void updateSubscriptionsSnapshotAndPropagate() { snapshotColumns.or(subscription.targetColumns); if (subscription.targetViewport == null) { - growingSubscriptionsMap.put(0, subscription); // full, lowest priority + growingSubscriptionsMap.put(0, subscription); // full, low priority } else if (!subscription.targetReverseViewport) { growingSubscriptionsMap.put(1, subscription); // forward VP, medium priority } else { @@ -1140,9 +1147,14 @@ private void updateSubscriptionsSnapshotAndPropagate() { growingSubscriptions.addAll(growingSubscriptionsMap.values()); - // MIN_SNAPSHOT_ROWS helps prevent viewports from getting partial delivery (breaking js client) - final long minCells = MIN_SNAPSHOT_ROWS * snapshotColumns.cardinality(); - long rowsRemaining = Math.max(minCells, nextCellsTarget / Math.max(1, snapshotColumns.cardinality())); + // we want to limit the size of the snapshot to keep the UGP responsive + long rowsRemaining; + if (SUBSCRIPTION_GROWTH_ENABLED) { + rowsRemaining = snapshotTargetCellCount / Math.max(1, snapshotColumns.cardinality()); + } else { + // growth is disabled, allow unlimited snapshot size + rowsRemaining = Long.MAX_VALUE; + } // some builders to help generate the rowsets we need RowSetBuilderRandom viewportBuilder = RowSetFactory.builderRandom(); @@ -1157,7 +1169,6 @@ private void updateSubscriptionsSnapshotAndPropagate() { ? RowSetFactory.flat(Long.MAX_VALUE) : subscription.targetViewport.copy()) { - // TODO: check this logic // if viewport direction changes then previous viewport is invalid, also if subscribed columns // change and new columns are added then viewport is invalid @@ -1168,7 +1179,7 @@ private void updateSubscriptionsSnapshotAndPropagate() { subscription.reverseViewport == subscription.targetReverseViewport && missingColumns.isEmpty(); - // if we haven't reversed viewports, can exclude current viewport + // if we have a valid viewport, can exclude current viewport from requested rows if (viewportValid && subscription.viewport != null) { missingRows.remove(subscription.viewport); } @@ -1182,45 +1193,41 @@ private void updateSubscriptionsSnapshotAndPropagate() { missingRows.removeRange(key, Long.MAX_VALUE - 1); } - // "grow" the snapshot viewport (snapshotViewport is always null) subscription.snapshotViewport = viewportValid && subscription.viewport != null ? subscription.viewport.copy() : RowSetFactory.empty(); - subscription.snapshotViewport.insert(missingRows); - // add all the rows from the current snapshot + // add the new rows to the upcoming snapshot + if (subscription.targetReverseViewport) { + reverseSnapshotRowSet.insert(missingRows); + } else { + snapshotRowSet.insert(missingRows); + } + + // "grow" the snapshot viewport with rows from the current snapshot subscription.snapshotViewport .insert(subscription.targetReverseViewport ? reverseSnapshotRowSet : snapshotRowSet); - if (subscription.targetReverseViewport) { - if (subscription.targetViewport != null) { - // add this set to the BMP viewport + // extra bookkeeping for viewport subscriptions + if (subscription.targetViewport != null) { + if (subscription.targetReverseViewport) { + // add this set to the BMP reverse viewport reverseViewportBuilder.addRowSet(missingRows); - - // retain only the rows that apply to this sub - subscription.snapshotViewport.retain(subscription.targetViewport); - } - - // add the new rows to the upcoming snapshot - reverseSnapshotRowSet.insert(missingRows); - } else { - if (subscription.targetViewport != null) { - // add this set to the BMP viewport + } else { + // add this set to the BMP forward viewport viewportBuilder.addRowSet(missingRows); - - // retain only the rows that apply to this sub - subscription.snapshotViewport.retain(subscription.targetViewport); } - - // add the new rows to the upcoming snapshot - snapshotRowSet.insert(missingRows); + // retain only the rows that apply to this subscription + subscription.snapshotViewport.retain(subscription.targetViewport); } + // save the column set subscription.snapshotColumns = (BitSet) subscription.targetColumns.clone(); - rowsRemaining -= missingRows.size(); - + // save the forward/reverse viewport setting subscription.snapshotReverseViewport = subscription.targetReverseViewport; + + rowsRemaining -= missingRows.size(); } } @@ -1233,20 +1240,30 @@ private void updateSubscriptionsSnapshotAndPropagate() { postSnapshotColumns.or(snapshotColumns); // finally, grab the snapshot and measure elapsed time for next projections - long start = scheduler.currentTime().getNanos(); + long start = System.nanoTime(); snapshot = getSnapshot(growingSubscriptions, snapshotColumns, snapshotRowSet, reverseSnapshotRowSet); - long elapsed = scheduler.currentTime().getNanos() - start; - - if (snapshot.rowsIncluded.size() > 0) { - // TODO: determine this by LTM usage percentage - long nanosPerCell = elapsed / (snapshot.rowsIncluded.size() * snapshotColumns.cardinality()); - // capped until we get auto-chunked delivery - if (nanosPerCell > 0) { - nextCellsTarget = Math.min((TARGET_SNAPSHOT_MILLIS * 1000000) / nanosPerCell, 400000L); - nextCellsTarget = Math.max(nextCellsTarget, 1000 * snapshotColumns.cardinality()); + long elapsed = System.nanoTime() - start; + + if (SUBSCRIPTION_GROWTH_ENABLED) { + // very simplistic logic to take the last snapshot and extrapolate max number of rows that will + // not exceed the target UGP processing time percentage + if (snapshot.rowsIncluded.size() > 0) { + long targetNanos = (long)(TARGET_SNAPSHOT_PERCENTAGE + * UpdateGraphProcessor.DEFAULT.getTargetCycleDurationMillis() * 1000000); + long nanosPerCell = elapsed / (snapshot.rowsIncluded.size() * snapshotColumns.cardinality()); + if (nanosPerCell > 0) { + // apply an exponential moving average to filter the data + if (snapshotNanosPerCell == 0) { + snapshotNanosPerCell = nanosPerCell; // initialize to first value + } else { + // EMA smoothing factor is 0.1 (N = 10) + snapshotNanosPerCell = (snapshotNanosPerCell * 0.9) + (nanosPerCell * 0.1); + } + // TODO: remove this when auto-chunking BarrageMessages exists + snapshotTargetCellCount = Math.min((long)(targetNanos / snapshotNanosPerCell), 400000L); + } } } - // System.out.println("nextCellsTarget: " + nextCellsTarget); } } @@ -1793,27 +1810,19 @@ private void flipSnapshotStateForSubscriptions( private void finalizeSnapshotForSubscriptions( final List subscriptions) { for (final Subscription subscription : subscriptions) { - // test whether we have completed the viewport - if (subscription.targetViewport == null) { // full subscription - // are all the rows in parent covered by the current viewport? - try (final RowSet keysInViewport = parent.getRowSet().subSetForPositions(subscription.viewport)) { - if (parent.getRowSet().subsetOf(keysInViewport)) { - // remove this subscription from the growing list - subscription.isGrowingViewport = false; - --numGrowingSubscriptions; - - // change the viewport to `null` to signify full subscription - try (final RowSet ignored = subscription.viewport) { - subscription.viewport = null; - } - } - - } - } else { - if (subscription.targetViewport.subsetOf(subscription.viewport)) { - // remove this subscription from the growing list + // are there any outstanding rows in the parent that should belong to this subscription? + try (final RowSet remaining = subscription.targetViewport == null + ? RowSetFactory.flat(Long.MAX_VALUE).minus(subscription.viewport) + : subscription.targetViewport.minus(subscription.viewport); + final RowSet remainingKeys = parent.getRowSet().subSetForPositions(remaining)) { + + if (remainingKeys.size() == 0) { + // this subscription is complete, remove it from the growing list subscription.isGrowingViewport = false; --numGrowingSubscriptions; + + // set the active viewport to the target viewport + subscription.viewport = subscription.targetViewport; } } } From ea2e87f588b40688ba0d00e7128ef08b60b1fc79 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Wed, 9 Mar 2022 11:47:11 -0800 Subject: [PATCH 06/47] spotless apply --- .../deephaven/server/barrage/BarrageMessageProducer.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index 4ca2aef28ad..702219c13a0 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -1152,7 +1152,7 @@ private void updateSubscriptionsSnapshotAndPropagate() { if (SUBSCRIPTION_GROWTH_ENABLED) { rowsRemaining = snapshotTargetCellCount / Math.max(1, snapshotColumns.cardinality()); } else { - // growth is disabled, allow unlimited snapshot size + // growth is disabled, allow unlimited snapshot size rowsRemaining = Long.MAX_VALUE; } @@ -1248,7 +1248,7 @@ private void updateSubscriptionsSnapshotAndPropagate() { // very simplistic logic to take the last snapshot and extrapolate max number of rows that will // not exceed the target UGP processing time percentage if (snapshot.rowsIncluded.size() > 0) { - long targetNanos = (long)(TARGET_SNAPSHOT_PERCENTAGE + long targetNanos = (long) (TARGET_SNAPSHOT_PERCENTAGE * UpdateGraphProcessor.DEFAULT.getTargetCycleDurationMillis() * 1000000); long nanosPerCell = elapsed / (snapshot.rowsIncluded.size() * snapshotColumns.cardinality()); if (nanosPerCell > 0) { @@ -1260,7 +1260,7 @@ private void updateSubscriptionsSnapshotAndPropagate() { snapshotNanosPerCell = (snapshotNanosPerCell * 0.9) + (nanosPerCell * 0.1); } // TODO: remove this when auto-chunking BarrageMessages exists - snapshotTargetCellCount = Math.min((long)(targetNanos / snapshotNanosPerCell), 400000L); + snapshotTargetCellCount = Math.min((long) (targetNanos / snapshotNanosPerCell), 400000L); } } } @@ -1814,7 +1814,7 @@ private void finalizeSnapshotForSubscriptions( try (final RowSet remaining = subscription.targetViewport == null ? RowSetFactory.flat(Long.MAX_VALUE).minus(subscription.viewport) : subscription.targetViewport.minus(subscription.viewport); - final RowSet remainingKeys = parent.getRowSet().subSetForPositions(remaining)) { + final RowSet remainingKeys = parent.getRowSet().subSetForPositions(remaining)) { if (remainingKeys.size() == 0) { // this subscription is complete, remove it from the growing list From 4b0623a1bf85ccf24cd5db1e807c07cfda8bc015 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Wed, 9 Mar 2022 13:34:42 -0800 Subject: [PATCH 07/47] updated BarrageSubscriptionImpl to block while subscription grows --- .../client/impl/BarrageSubscription.java | 44 +++++++++++++-- .../client/impl/BarrageSubscriptionImpl.java | 54 ++++++++++++++----- .../barrage/BarrageMessageProducer.java | 5 ++ 3 files changed, 86 insertions(+), 17 deletions(-) diff --git a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscription.java b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscription.java index 8f0ac0172ad..bf1b01bbfc7 100644 --- a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscription.java +++ b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscription.java @@ -39,18 +39,41 @@ BarrageSubscription subscribe(TableSpec tableSpec, BarrageSubscriptionOptions op BarrageSubscription subscribe(TableHandle tableHandle, BarrageSubscriptionOptions options); } + /** + * This call will return false until all rows for the subscribed table are available. + * + * @return true when all rows for the subscribed table are available, false otherwise + */ + public boolean isCompleted(); + + /** + * This call will block until all rows for the subscribed table are available. + * + */ + public void waitForCompletion() throws InterruptedException; + /** * Request a full subscription of the data and populate a {@link BarrageTable} with the incrementally updating data - * that is received. + * that is received. This call will block until all rows for the subscribed table are available. * * @return the {@code BarrageTable} */ BarrageTable entireTable() throws InterruptedException; + /** + * Request a full subscription of the data and populate a {@link BarrageTable} with the incrementally updating data + * that is received. + * + * @param blockUntilComplete block execution until all rows for the subscribed table are available + * + * @return the {@code BarrageTable} + */ + BarrageTable entireTable(boolean blockUntilComplete) throws InterruptedException; + // TODO (deephaven-core#712): java-client viewport support /** * Request a partial subscription of the data limited by viewport or column set and populate a {@link BarrageTable} - * with the data that is received. + * with the data that is received. This call will block until the subscribed table viewport is satisfied. * * @param viewport the position-space viewport to use for the subscription * @param columns the columns to include in the subscription @@ -61,7 +84,8 @@ BarrageSubscription subscribe(TableSpec tableSpec, BarrageSubscriptionOptions op /** * Request a partial subscription of the data limited by viewport or column set and populate a {@link BarrageTable} - * with the data that is received. Allows the viewport to be reversed. + * with the data that is received. Allows the viewport to be reversed. This call will block until the subscribed + * table viewport is satisfied. * * @param viewport the position-space viewport to use for the subscription * @param columns the columns to include in the subscription @@ -71,4 +95,18 @@ BarrageSubscription subscribe(TableSpec tableSpec, BarrageSubscriptionOptions op */ BarrageTable partialTable(RowSet viewport, BitSet columns, boolean reverseViewport) throws InterruptedException; + /** + * Request a partial subscription of the data limited by viewport or column set and populate a {@link BarrageTable} + * with the data that is received. Allows the viewport to be reversed. + * + * @param viewport the position-space viewport to use for the subscription + * @param columns the columns to include in the subscription + * @param reverseViewport Whether to treat {@code posRowSet} as offsets from {@link #size()} rather than {@code 0} + * @param blockUntilComplete block execution until the subscribed table viewport is satisfied + * + * @return the {@code BarrageTable} + */ + BarrageTable partialTable(RowSet viewport, BitSet columns, boolean reverseViewport, boolean blockUntilComplete) + throws InterruptedException; + } diff --git a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java index 275cf68a3a0..9ba4cd22c8b 100644 --- a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java +++ b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java @@ -55,6 +55,7 @@ public class BarrageSubscriptionImpl extends ReferenceCountedLivenessNode implem private volatile Condition completedCondition; private volatile boolean completed = false; private volatile Throwable exceptionWhileCompleting = null; + private InstrumentedTableUpdateListener listener = null; private boolean subscribed = false; private volatile boolean connected = true; @@ -142,19 +143,47 @@ public void onCompleted() { } } + @Override + public boolean isCompleted() { + return completed; + } + + @Override + public synchronized void waitForCompletion() throws InterruptedException { + while (!completed && exceptionWhileCompleting == null) { + // handle the condition where this function may have the exclusive lock + if (completedCondition != null) { + completedCondition.await(); + } else { + wait(); // barragesnapshotimpl lock + } + } + } + @Override public BarrageTable entireTable() throws InterruptedException { - return partialTable(null, null, false); + return entireTable(true); } @Override - public synchronized BarrageTable partialTable(RowSet viewport, BitSet columns) throws InterruptedException { - return partialTable(viewport, columns, false); + public BarrageTable entireTable(boolean blockUntilComplete) throws InterruptedException { + return partialTable(null, null, false, blockUntilComplete); } @Override - public synchronized BarrageTable partialTable(RowSet viewport, BitSet columns, boolean reverseViewport) + public BarrageTable partialTable(RowSet viewport, BitSet columns) throws InterruptedException { + return partialTable(viewport, columns, false, true); + } + + @Override + public BarrageTable partialTable(RowSet viewport, BitSet columns, boolean reverseViewport) throws InterruptedException { + return partialTable(viewport, columns, reverseViewport, true); + } + + @Override + public synchronized BarrageTable partialTable(RowSet viewport, BitSet columns, boolean reverseViewport, + boolean blockUntilComplete) throws InterruptedException { if (!connected) { throw new UncheckedDeephavenException( this + " is no longer an active subscription and cannot be retained further"); @@ -179,7 +208,7 @@ public synchronized BarrageTable partialTable(RowSet viewport, BitSet columns, b subscribed = true; // use a listener to decide when the table is complete - InstrumentedTableUpdateListener listener = new InstrumentedTableUpdateListener("example-listener") { + listener = new InstrumentedTableUpdateListener("example-listener") { @Override protected void onFailureInternal(final Throwable originalException, final Entry sourceEntry) { exceptionWhileCompleting = originalException; @@ -211,22 +240,19 @@ public void onUpdate(final TableUpdate upstream) { BarrageSubscriptionImpl.this.notifyAll(); } } + + // no longer need to listen for completion + resultTable.removeUpdateListener(this); + listener = null; } } }; resultTable.listenForUpdates(listener); - while (!completed && exceptionWhileCompleting == null) { - // handle the condition where this function may have the exclusive lock - if (completedCondition != null) { - completedCondition.await(); - } else { - wait(); // barragesubscriptionimpl lock - } + if (blockUntilComplete) { + waitForCompletion(); } - - resultTable.removeUpdateListener(listener); } if (exceptionWhileCompleting == null) { diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index 702219c13a0..b2f5531bd67 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -1810,6 +1810,11 @@ private void flipSnapshotStateForSubscriptions( private void finalizeSnapshotForSubscriptions( final List subscriptions) { for (final Subscription subscription : subscriptions) { + + // TODO: parent.getRowSet().subSetForPositions is inefficient in this context, would prefer some method that + // short circuits if any matching key of parent.getRowSet() is found. Something like: + // long firstKeyforPositions(final RowSequence positions); // returns -1 for no match + // are there any outstanding rows in the parent that should belong to this subscription? try (final RowSet remaining = subscription.targetViewport == null ? RowSetFactory.flat(Long.MAX_VALUE).minus(subscription.viewport) From b04dbb0d869761f0d85e71f618a217c1e98dd95e Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Wed, 9 Mar 2022 13:38:47 -0800 Subject: [PATCH 08/47] minor fix --- .../java/io/deephaven/client/impl/BarrageSubscriptionImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java index 9ba4cd22c8b..cc09796acab 100644 --- a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java +++ b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java @@ -226,7 +226,8 @@ public void onUpdate(final TableUpdate upstream) { // test to see if the viewport matches the requested if (viewport == null && resultTable.getServerViewport() == null) { completed = true; - } else if (viewport != null && resultTable.getServerViewport() != null) { + } else if (viewport != null && resultTable.getServerViewport() != null + && reverseViewport == resultTable.getServerReverseViewport()) { completed = viewport.subsetOf(resultTable.getServerViewport()); } else { completed = false; From 9223c7a4585dc516e9a5778aa57fcf2201bcbf71 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Wed, 9 Mar 2022 13:58:33 -0800 Subject: [PATCH 09/47] cosmetic fixes --- .../io/deephaven/client/impl/BarrageSubscriptionImpl.java | 7 ------- .../deephaven/server/barrage/BarrageMessageProducer.java | 3 +-- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java index cc09796acab..d6a8c802517 100644 --- a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java +++ b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java @@ -60,8 +60,6 @@ public class BarrageSubscriptionImpl extends ReferenceCountedLivenessNode implem private boolean subscribed = false; private volatile boolean connected = true; - private RowSet targetViewport = null; - /** * Represents a BarrageSubscription. * @@ -289,8 +287,6 @@ public void close() { private void cleanup() { this.connected = false; this.tableHandle.close(); - if (targetViewport != null) - targetViewport.close(); resultTable = null; } @@ -306,9 +302,6 @@ private ByteBuffer makeRequestInternal( boolean reverseViewport, @Nullable BarrageSubscriptionOptions options) { - // store the viewport for comparison to received data to determine completion of growing snapshot - targetViewport = viewport != null ? viewport.copy() : null; - final FlatBufferBuilder metadata = new FlatBufferBuilder(); int colOffset = 0; diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index b2f5531bd67..627cf764526 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -1879,8 +1879,7 @@ private class SnapshotControl implements ConstructSnapshot.SnapshotControl { long step = -1; final List snapshotSubscriptions; - SnapshotControl( - final List snapshotSubscriptions) { + SnapshotControl(final List snapshotSubscriptions) { this.snapshotSubscriptions = snapshotSubscriptions; } From 80009233897b1bfac1f73edaba324379d63171eb Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Thu, 17 Mar 2022 16:28:49 -0700 Subject: [PATCH 10/47] tests pass except testAllUniqueXXX --- .../barrage/BarrageMessageProducer.java | 174 ++++++++++-------- .../barrage/BarrageStreamGenerator.java | 30 +-- 2 files changed, 106 insertions(+), 98 deletions(-) diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index 2bd63b1f41f..abdd6888a95 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -40,7 +40,6 @@ import io.deephaven.internal.log.LoggerFactory; import io.deephaven.io.logger.Logger; import io.deephaven.server.util.Scheduler; -import io.deephaven.tablelogger.Row; import io.deephaven.time.DateTimeUtils; import io.deephaven.util.SafeCloseable; import io.deephaven.util.SafeCloseableArray; @@ -458,11 +457,13 @@ private class Subscription { BitSet snapshotColumns = null; // captured column during snapshot portion of propagation job boolean snapshotReverseViewport = false; // captured setting during snapshot portion of propagation job - RowSet targetViewport; // target viewport for a growing subscription + RowSet targetViewport = null; // target viewport for a growing subscription boolean targetReverseViewport; // is the target viewport reversed (indexed from end of table) BitSet targetColumns; // target column subscription set boolean isGrowingViewport; // does this have a growing viewport? + WritableRowSet growRemainingViewport = null; // remaining viewport rows for a growing subscription + WritableRowSet growIncrementalViewport = null; // remaining viewport rows for a growing subscription private Subscription(final StreamObserver listener, final BarrageSubscriptionOptions options, @@ -1037,13 +1038,18 @@ private void updateSubscriptionsSnapshotAndPropagate() { subscription.pendingViewport = null; } - if (subscription.pendingColumns != null) { - subscription.targetColumns = subscription.pendingColumns; - subscription.pendingColumns = null; - } + subscription.targetColumns = subscription.pendingColumns; + subscription.pendingColumns = null; subscription.targetReverseViewport = subscription.pendingReverseViewport; + // get the set of remaining rows for this subscription + if (subscription.growRemainingViewport != null) { + subscription.growRemainingViewport.close(); + } + subscription.growRemainingViewport = subscription.targetViewport == null + ? RowSetFactory.flat(Long.MAX_VALUE) + : subscription.targetViewport.copy(); } // remove deleted subscriptions while we still hold the lock @@ -1076,7 +1082,7 @@ private void updateSubscriptionsSnapshotAndPropagate() { // handle forward and reverse snapshots separately if (sub.snapshotReverseViewport) { postSnapshotReverseViewportBuilder - .addRowSet(sub.viewport); + .addRowSet(sub.viewport); } else { postSnapshotViewportBuilder .addRowSet(sub.viewport); @@ -1101,8 +1107,7 @@ private void updateSubscriptionsSnapshotAndPropagate() { BitSet snapshotColumns = null; // create a prioritized list for the subscriptions - TreeMap growingSubscriptionsMap = new TreeMap<>(Collections.reverseOrder()); - List growingSubscriptions = new ArrayList<>();; + LinkedList growingSubscriptions = new LinkedList<>();; if (numGrowingSubscriptions > 0) { snapshotColumns = new BitSet(); @@ -1113,21 +1118,18 @@ private void updateSubscriptionsSnapshotAndPropagate() { snapshotColumns.or(subscription.targetColumns); if (subscription.targetViewport == null) { - growingSubscriptionsMap.put(0, subscription); // full, low priority - } else if (!subscription.targetReverseViewport) { - growingSubscriptionsMap.put(1, subscription); // forward VP, medium priority + growingSubscriptions.addLast(subscription); // full gets low priority } else { - growingSubscriptionsMap.put(2, subscription); // reverse VP, high priority + growingSubscriptions.addFirst(subscription); // viewport gets higher priority } } } - growingSubscriptions.addAll(growingSubscriptionsMap.values()); - // we want to limit the size of the snapshot to keep the UGP responsive long rowsRemaining; if (SUBSCRIPTION_GROWTH_ENABLED) { rowsRemaining = snapshotTargetCellCount / Math.max(1, snapshotColumns.cardinality()); + rowsRemaining = 4; } else { // growth is disabled, allow unlimited snapshot size rowsRemaining = Long.MAX_VALUE; @@ -1140,72 +1142,79 @@ private void updateSubscriptionsSnapshotAndPropagate() { try (final WritableRowSet snapshotRowSet = RowSetFactory.empty(); final WritableRowSet reverseSnapshotRowSet = RowSetFactory.empty()) { - // satisfy the subscriptions in priority order + // satisfy the subscriptions in order for (final Subscription subscription : growingSubscriptions) { - try (final WritableRowSet missingRows = subscription.targetViewport == null - ? RowSetFactory.flat(Long.MAX_VALUE) - : subscription.targetViewport.copy()) { - // if viewport direction changes then previous viewport is invalid, also if subscribed columns - // change and new columns are added then viewport is invalid + // we need to determine if the `activeViewport` is valid. if the viewport direction changes or + // columns were added, the client viewport is invalid - BitSet missingColumns = (BitSet) subscription.targetColumns.clone(); - missingColumns.andNot(subscription.subscribedColumns); + BitSet addedCols = (BitSet) subscription.targetColumns.clone(); + addedCols.andNot(subscription.subscribedColumns); - final boolean viewportValid = - subscription.reverseViewport == subscription.targetReverseViewport - && missingColumns.isEmpty(); + final boolean viewportValid = + subscription.reverseViewport == subscription.targetReverseViewport + && addedCols.isEmpty(); - // if we have a valid viewport, can exclude current viewport from requested rows - if (viewportValid && subscription.viewport != null) { - missingRows.remove(subscription.viewport); - } + // don't request valid rows that the client already possesses + if (viewportValid && subscription.viewport != null) { + subscription.growRemainingViewport.remove(subscription.viewport); + } - // remove rows that we already have selected for the upcoming snapshot - missingRows.remove(subscription.targetReverseViewport ? reverseSnapshotRowSet : snapshotRowSet); + // get the current set for this viewport direction + final WritableRowSet currentSet = + subscription.targetReverseViewport ? reverseSnapshotRowSet : snapshotRowSet; - // shrink this to the `rowsRemaining` value - if (missingRows.size() > rowsRemaining) { - final long key = missingRows.get(rowsRemaining); - missingRows.removeRange(key, Long.MAX_VALUE - 1); - } + // get the rows that we need that are already in the snapshot + try (final RowSet overlap = subscription.growRemainingViewport.extract(currentSet)) { + if (rowsRemaining <= 0) { + subscription.growIncrementalViewport = overlap.copy(); + } else { + try (final WritableRowSet additional = subscription.growRemainingViewport.copy()) { - subscription.snapshotViewport = viewportValid && subscription.viewport != null - ? subscription.viewport.copy() - : RowSetFactory.empty(); + // shrink the set of new rows to <= `rowsRemaining` size + if (additional.size() > rowsRemaining) { + final long key = additional.get(rowsRemaining); + additional.removeRange(key, Long.MAX_VALUE - 1); + } - // add the new rows to the upcoming snapshot - if (subscription.targetReverseViewport) { - reverseSnapshotRowSet.insert(missingRows); - } else { - snapshotRowSet.insert(missingRows); - } + // store the rowset for this subscription for this snapshot + subscription.growIncrementalViewport = overlap.union(additional); - // "grow" the snapshot viewport with rows from the current snapshot - subscription.snapshotViewport - .insert(subscription.targetReverseViewport ? reverseSnapshotRowSet : snapshotRowSet); + // add the new rows to the upcoming snapshot + currentSet.insert(additional); - // extra bookkeeping for viewport subscriptions - if (subscription.targetViewport != null) { - if (subscription.targetReverseViewport) { - // add this set to the BMP reverse viewport - reverseViewportBuilder.addRowSet(missingRows); - } else { - // add this set to the BMP forward viewport - viewportBuilder.addRowSet(missingRows); + // extra bookkeeping for viewport subscriptions + if (subscription.targetViewport != null) { + if (subscription.targetReverseViewport) { + // add this set to the BMP reverse viewport + reverseViewportBuilder.addRowSet(additional); + } else { + // add this set to the BMP forward viewport + viewportBuilder.addRowSet(additional); + } + } + + // decrement the remaining row count + rowsRemaining -= additional.size(); } - // retain only the rows that apply to this subscription - subscription.snapshotViewport.retain(subscription.targetViewport); } + } + + // "grow" the snapshot viewport with rows from the current snapshot + if (viewportValid && subscription.viewport != null) { + subscription.snapshotViewport = subscription.viewport.union(subscription.growIncrementalViewport); + } else { + subscription.snapshotViewport = subscription.growIncrementalViewport.copy(); + } - // save the column set - subscription.snapshotColumns = (BitSet) subscription.targetColumns.clone(); + // remove the rows we are about to snapshot from the remaining set + subscription.growRemainingViewport.remove(subscription.growIncrementalViewport); - // save the forward/reverse viewport setting - subscription.snapshotReverseViewport = subscription.targetReverseViewport; + // save the column set + subscription.snapshotColumns = (BitSet) subscription.targetColumns.clone(); - rowsRemaining -= missingRows.size(); - } + // save the forward/reverse viewport setting + subscription.snapshotReverseViewport = subscription.targetReverseViewport; } // update the postSnapshot viewports/columns to include the new viewports (excluding `full`) @@ -1404,15 +1413,14 @@ private void propagateSnapshotForSubscription( // the parent table listener needs to be recording data as if we've already sent the successful snapshot. if (subscription.snapshotViewport != null) { + subscription.snapshotViewport.close(); + subscription.snapshotViewport = null; needsSnapshot = true; - try (final RowSet ignored = subscription.snapshotViewport) { - subscription.snapshotViewport = null; - } } if (subscription.snapshotColumns != null) { - needsSnapshot = true; subscription.snapshotColumns = null; + needsSnapshot = true; } if (needsSnapshot) { @@ -1421,12 +1429,14 @@ private void propagateSnapshotForSubscription( .append(System.identityHashCode(subscription)).endl(); } - final boolean isViewport = subscription.viewport != null; - try (final RowSet keySpaceViewport = - isViewport - ? snapshotGenerator.getMessage().rowsAdded - .subSetForPositions(subscription.viewport, subscription.reverseViewport) - : null) { + try (final RowSet keySpaceViewport = snapshotGenerator.getMessage().rowsAdded + .subSetForPositions(subscription.growIncrementalViewport, subscription.reverseViewport)) { +// final boolean isViewport = subscription.viewport != null; +// try (final RowSet keySpaceViewport = +// isViewport +// ? snapshotGenerator.getMessage().rowsAdded +// .subSetForPositions(subscription.viewport, subscription.reverseViewport) +// : null) { if (subscription.pendingInitialSnapshot) { // Send schema metadata to this new client. subscription.listener.onNext(streamGeneratorFactory.getSchemaView( @@ -1444,6 +1454,11 @@ private void propagateSnapshotForSubscription( } } + if (subscription.growIncrementalViewport != null) { + subscription.growIncrementalViewport.close(); + subscription.growIncrementalViewport = null; + } + subscription.pendingInitialSnapshot = false; } @@ -1805,6 +1820,15 @@ private void finalizeSnapshotForSubscriptions( // set the active viewport to the target viewport subscription.viewport = subscription.targetViewport; + + // expand the postSnapshot viewports to include this entire snapshot + if (subscription.isViewport()) { + if (subscription.reverseViewport) { + postSnapshotReverseViewport.insert(subscription.viewport); + } else { + postSnapshotViewport.insert(subscription.viewport); + } + } } } } diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java index b7c0a9dbc62..1e8bcbd447b 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java @@ -288,16 +288,8 @@ public SubView(final BarrageStreamGenerator generator, addRowOffsets = generator.rowsIncluded.original.invert(intersect); } } else { - if (isInitialSnapshot) { - // use chunk data as-is, rely on snapshot process to avoid duplicates - addRowOffsets = RowSetFactory.flat(generator.rowsAdded.original.size()); - } else { - // there may be scoped rows included in the chunks that need to be removed - try (WritableRowSet intersect = - generator.rowsIncluded.original.intersect(generator.rowsAdded.original)) { - addRowOffsets = generator.rowsIncluded.original.invert(intersect); - } - } + // use chunk data as-is, rely on snapshot process to avoid duplicates + addRowOffsets = RowSetFactory.flat(generator.rowsIncluded.original.size()); } // require an add batch if there are no mod batches @@ -662,7 +654,7 @@ private long appendAddColumns(final View view, final long startRange, final long // Add the drainable last as it is allowed to immediately close a row set the visitors need addStream.accept(drainableColumn); } - return myAddedOffsets != null ? myAddedOffsets.size() : rowsAdded.original.size(); + return myAddedOffsets.size(); } } @@ -734,25 +726,17 @@ private ByteBuffer getSubscriptionMetadata(final SubView view) throws IOExceptio // Added Chunk Data: int addedRowsIncludedOffset = 0; - if (view.keyspaceViewport == null) { - if (view.isInitialSnapshot) { - // use chunk data as-is, rely on snapshot process to avoid duplicates - addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(metadata); - } else { - // there may be scoped rows included in the chunks that need to be removed - try (WritableRowSet intersect = rowsIncluded.original.intersect(rowsAdded.original)) { - addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(intersect, metadata); - } - } - } else { + if (view.keyspaceViewport != null) { addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(view.keyspaceViewport, metadata); + } else { + addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(metadata); } // now add mod-column streams, and write the mod column indexes TIntArrayList modOffsets = new TIntArrayList(modColumnData.length); for (final ModColumnData mcd : modColumnData) { final int myModRowOffset; - if (view.isViewport()) { + if (view.keyspaceViewport != null) { myModRowOffset = mcd.rowsModified.addToFlatBuffer(view.keyspaceViewport, metadata); } else { myModRowOffset = mcd.rowsModified.addToFlatBuffer(metadata); From ac7cae24555af361b957dffb8343b7d5f3de6fce Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Mon, 21 Mar 2022 11:29:04 -0700 Subject: [PATCH 11/47] all tests passing --- .../barrage/table/BarrageTable.java | 11 +- .../client/impl/BarrageSubscription.java | 1 - .../barrage/BarrageMessageProducer.java | 296 ++++++++++-------- .../barrage/BarrageStreamGenerator.java | 21 +- 4 files changed, 188 insertions(+), 141 deletions(-) diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java index 22c2be022ad..dbeb91f8f58 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java @@ -166,14 +166,17 @@ public Class[] getWireComponentTypes() { return Arrays.stream(destSources).map(ColumnSource::getComponentType).toArray(Class[]::new); } + @VisibleForTesting public RowSet getServerViewport() { return serverViewport; } + @VisibleForTesting public boolean getServerReverseViewport() { return serverReverseViewport; } + @VisibleForTesting public BitSet getServerColumns() { return serverColumns; } @@ -229,7 +232,13 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC beginLog(LogLevel.INFO).append(": Processing delta updates ") .append(update.firstSeq).append("-").append(update.lastSeq) - .append(" update=").append(up).endl(); + .append(" update=").append(up) + .append(" included=").append(update.rowsIncluded) + .append(" rowset=").append(this.getRowSet()) + .append(" isSnapshot=").append(update.isSnapshot) + .append(" snapshotRowSet=").append(update.snapshotRowSet) + .append(" snapshotRowSetIsReversed=").append(update.snapshotRowSetIsReversed) + .endl(); mods.close(); } diff --git a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscription.java b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscription.java index bf1b01bbfc7..26d3e26ea13 100644 --- a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscription.java +++ b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscription.java @@ -48,7 +48,6 @@ BarrageSubscription subscribe(TableSpec tableSpec, BarrageSubscriptionOptions op /** * This call will block until all rows for the subscribed table are available. - * */ public void waitForCompletion() throws InterruptedException; diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index abdd6888a95..66b57f8beb0 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -92,11 +92,15 @@ public class BarrageMessageProducer extends LivenessArtifact private static final double TARGET_SNAPSHOT_PERCENTAGE = Configuration.getInstance().getDoubleForClassWithDefault(BarrageMessageProducer.class, "targetSnapshotPercentage", 0.25); - private static final long INITIAL_SNAPSHOT_CELL_COUNT = + + private static final long MIN_SNAPSHOT_CELL_COUNT = + Configuration.getInstance().getLongForClassWithDefault(BarrageMessageProducer.class, + "minSnapshotCellCount", 50000); + private static final long MAX_SNAPSHOT_CELL_COUNT = Configuration.getInstance().getLongForClassWithDefault(BarrageMessageProducer.class, - "initialSnapshotCellCount", 10000); + "maxSnapshotCellCount", Long.MAX_VALUE); - private long snapshotTargetCellCount = INITIAL_SNAPSHOT_CELL_COUNT; + private long snapshotTargetCellCount = MIN_SNAPSHOT_CELL_COUNT; private double snapshotNanosPerCell = 0; /** @@ -280,6 +284,9 @@ public int hashCode() { // We keep this RowSet in-sync with deltas being propagated to subscribers. private final WritableRowSet propagationRowSet; + // this holds the size of the current table, refreshed with each update + private long parentTableSize; + /** * On every update we compute which subset of rows need to be recorded dependent on our active subscriptions. We * compute two sets, which rows were added (or need to be scoped into viewports) and which rows were modified. For @@ -370,6 +377,8 @@ public BarrageMessageProducer(final Scheduler scheduler, this.updateIntervalMs = updateIntervalMs; this.onGetSnapshot = onGetSnapshot; + this.parentTableSize = parent.size(); + if (DEBUG) { log.info().append(logPrefix).append("Creating new BarrageMessageProducer for ") .append(System.identityHashCode(parent)).append(" with an interval of ") @@ -462,8 +471,8 @@ private class Subscription { BitSet targetColumns; // target column subscription set boolean isGrowingViewport; // does this have a growing viewport? - WritableRowSet growRemainingViewport = null; // remaining viewport rows for a growing subscription - WritableRowSet growIncrementalViewport = null; // remaining viewport rows for a growing subscription + WritableRowSet growingRemainingViewport = null; // remaining viewport rows for a growing subscription + WritableRowSet growingIncrementalViewport = null; // incremental viewport rows for a growing subscription private Subscription(final StreamObserver listener, final BarrageSubscriptionOptions options, @@ -624,6 +633,7 @@ public void onUpdate(final TableUpdate upstream) { enqueueUpdate(upstream); schedulePropagation(); } + parentTableSize = parent.size(); // mark when the last indices are from, so that terminal notifications can make use of them if required lastIndexClockStep = LogicalClock.DEFAULT.currentStep(); @@ -826,8 +836,10 @@ private void enqueueUpdate(final TableUpdate upstream) { .append(activeSubscriptions.size()) .append(", numFullSubscriptions=").append(numFullSubscriptions).append(", addsToRecord=") .append(addsToRecord) - .append(", modsToRecord=").append(modsToRecord).append(", columns=") - .append(FormatBitSet.formatBitSet(activeColumns)).endl(); + .append(", modsToRecord=").append(modsToRecord) + .append(", activeViewport=").append(activeViewport) + .append(", activeReverseViewport=").append(activeReverseViewport) + .append(", columns=").append(FormatBitSet.formatBitSet(activeColumns)).endl(); } // Now append any data that we need to save for later. @@ -992,6 +1004,7 @@ private void updateSubscriptionsSnapshotAndPropagate() { boolean firstSubscription = false; boolean pendingChanges = false; + boolean pendingDeletes = false; // check for pending changes (under the lock) synchronized (this) { @@ -1006,19 +1019,20 @@ private void updateSubscriptionsSnapshotAndPropagate() { if (updatedSubscriptions != null) { for (final Subscription subscription : updatedSubscriptions) { if (subscription.pendingDelete) { + pendingDeletes = true; try { subscription.listener.onCompleted(); } catch (final Exception ignored) { // ignore races on cancellation } - // remove this from the "growing" list (if present) - subscription.isGrowingViewport = false; continue; } // add this subscription to the "growing" list to handle snapshot creation - subscription.isGrowingViewport = true; - ++numGrowingSubscriptions; + if (!subscription.isGrowingViewport) { + subscription.isGrowingViewport = true; + ++numGrowingSubscriptions; + } subscription.hasPendingUpdate = false; if (!subscription.isActive) { @@ -1044,57 +1058,37 @@ private void updateSubscriptionsSnapshotAndPropagate() { subscription.targetReverseViewport = subscription.pendingReverseViewport; // get the set of remaining rows for this subscription - if (subscription.growRemainingViewport != null) { - subscription.growRemainingViewport.close(); + if (subscription.growingRemainingViewport != null) { + subscription.growingRemainingViewport.close(); } - subscription.growRemainingViewport = subscription.targetViewport == null + subscription.growingRemainingViewport = subscription.targetViewport == null ? RowSetFactory.flat(Long.MAX_VALUE) : subscription.targetViewport.copy(); } - // remove deleted subscriptions while we still hold the lock - for (int i = 0; i < activeSubscriptions.size(); ++i) { - final Subscription sub = activeSubscriptions.get(i); - if (sub.pendingDelete) { - if (!sub.isViewport()) { - --numFullSubscriptions; - } - if (sub.isGrowingViewport) { - --numGrowingSubscriptions; - } - - activeSubscriptions.set(i, activeSubscriptions.get(activeSubscriptions.size() - 1)); - activeSubscriptions.remove(activeSubscriptions.size() - 1); - --i; - continue; - } - } - } + if (pendingDeletes) { + // remove deleted subscriptions while we still hold the lock + for (int i = 0; i < activeSubscriptions.size(); ++i) { + final Subscription sub = activeSubscriptions.get(i); + if (sub.pendingDelete) { + if (!sub.isViewport()) { + --numFullSubscriptions; + } + if (sub.isGrowingViewport) { + --numGrowingSubscriptions; + } - // need to rebuild the forward/reverse viewports and column sets - final RowSetBuilderRandom postSnapshotViewportBuilder = RowSetFactory.builderRandom(); - final RowSetBuilderRandom postSnapshotReverseViewportBuilder = RowSetFactory.builderRandom(); - - postSnapshotColumns.clear(); - for (final Subscription sub : activeSubscriptions) { - postSnapshotColumns.or(sub.snapshotColumns != null ? sub.snapshotColumns : sub.subscribedColumns); - if (sub.isViewport()) { - // handle forward and reverse snapshots separately - if (sub.snapshotReverseViewport) { - postSnapshotReverseViewportBuilder - .addRowSet(sub.viewport); - } else { - postSnapshotViewportBuilder - .addRowSet(sub.viewport); + activeSubscriptions.set(i, activeSubscriptions.get(activeSubscriptions.size() - 1)); + activeSubscriptions.remove(activeSubscriptions.size() - 1); + --i; + } } } } - postSnapshotViewport = postSnapshotViewportBuilder.build(); - postSnapshotReverseViewport = postSnapshotReverseViewportBuilder.build(); - - if (numGrowingSubscriptions == 0) { + if (pendingDeletes && numGrowingSubscriptions == 0) { // i.e. We have only removed subscriptions; we can update this state immediately. + buildPostSnapshotViewports(); promoteSnapshotToActive(); } } @@ -1107,9 +1101,12 @@ private void updateSubscriptionsSnapshotAndPropagate() { BitSet snapshotColumns = null; // create a prioritized list for the subscriptions - LinkedList growingSubscriptions = new LinkedList<>();; + LinkedList growingSubscriptions = new LinkedList<>(); if (numGrowingSubscriptions > 0) { + // start with fresh viewports + buildPostSnapshotViewports(); + snapshotColumns = new BitSet(); for (final Subscription subscription : activeSubscriptions) { @@ -1126,10 +1123,13 @@ private void updateSubscriptionsSnapshotAndPropagate() { } // we want to limit the size of the snapshot to keep the UGP responsive + final long columnCount = Math.max(1, snapshotColumns.cardinality()); + long rowsRemaining; if (SUBSCRIPTION_GROWTH_ENABLED) { - rowsRemaining = snapshotTargetCellCount / Math.max(1, snapshotColumns.cardinality()); - rowsRemaining = 4; + final long cellCount = + Math.max(MIN_SNAPSHOT_CELL_COUNT, Math.min(snapshotTargetCellCount, MAX_SNAPSHOT_CELL_COUNT)); + rowsRemaining = cellCount / columnCount; } else { // growth is disabled, allow unlimited snapshot size rowsRemaining = Long.MAX_VALUE; @@ -1151,13 +1151,12 @@ private void updateSubscriptionsSnapshotAndPropagate() { BitSet addedCols = (BitSet) subscription.targetColumns.clone(); addedCols.andNot(subscription.subscribedColumns); - final boolean viewportValid = - subscription.reverseViewport == subscription.targetReverseViewport - && addedCols.isEmpty(); + final boolean viewportValid = subscription.reverseViewport == subscription.targetReverseViewport + && addedCols.isEmpty(); // don't request valid rows that the client already possesses if (viewportValid && subscription.viewport != null) { - subscription.growRemainingViewport.remove(subscription.viewport); + subscription.growingRemainingViewport.remove(subscription.viewport); } // get the current set for this viewport direction @@ -1165,11 +1164,13 @@ private void updateSubscriptionsSnapshotAndPropagate() { subscription.targetReverseViewport ? reverseSnapshotRowSet : snapshotRowSet; // get the rows that we need that are already in the snapshot - try (final RowSet overlap = subscription.growRemainingViewport.extract(currentSet)) { + try (final RowSet overlap = subscription.growingRemainingViewport.extract(currentSet)) { if (rowsRemaining <= 0) { - subscription.growIncrementalViewport = overlap.copy(); + // we can't request additional rows, but can use other rows in this snapshot + subscription.growingIncrementalViewport = overlap.copy(); } else { - try (final WritableRowSet additional = subscription.growRemainingViewport.copy()) { + // request as many rows as we are allowed + try (final WritableRowSet additional = subscription.growingRemainingViewport.copy()) { // shrink the set of new rows to <= `rowsRemaining` size if (additional.size() > rowsRemaining) { @@ -1177,13 +1178,13 @@ private void updateSubscriptionsSnapshotAndPropagate() { additional.removeRange(key, Long.MAX_VALUE - 1); } - // store the rowset for this subscription for this snapshot - subscription.growIncrementalViewport = overlap.union(additional); + // store the rowset that applies for this exact snapshot + subscription.growingIncrementalViewport = overlap.union(additional); // add the new rows to the upcoming snapshot currentSet.insert(additional); - // extra bookkeeping for viewport subscriptions + // extra bookkeeping (for viewport subscriptions when there are no full subs) if (subscription.targetViewport != null) { if (subscription.targetReverseViewport) { // add this set to the BMP reverse viewport @@ -1198,17 +1199,24 @@ private void updateSubscriptionsSnapshotAndPropagate() { rowsRemaining -= additional.size(); } } - } + } // "grow" the snapshot viewport with rows from the current snapshot if (viewportValid && subscription.viewport != null) { - subscription.snapshotViewport = subscription.viewport.union(subscription.growIncrementalViewport); + // we can use the current viewport as a start + subscription.snapshotViewport = + subscription.viewport.union(subscription.growingIncrementalViewport); + + // the current viewport might contain rows that we don't want (from a previous viewport, e.g.) + if (subscription.targetViewport != null) { + subscription.snapshotViewport.retain(subscription.targetViewport); + } } else { - subscription.snapshotViewport = subscription.growIncrementalViewport.copy(); + subscription.snapshotViewport = subscription.growingIncrementalViewport.copy(); } // remove the rows we are about to snapshot from the remaining set - subscription.growRemainingViewport.remove(subscription.growIncrementalViewport); + subscription.growingRemainingViewport.remove(subscription.growingIncrementalViewport); // save the column set subscription.snapshotColumns = (BitSet) subscription.targetColumns.clone(); @@ -1230,25 +1238,23 @@ private void updateSubscriptionsSnapshotAndPropagate() { snapshot = getSnapshot(growingSubscriptions, snapshotColumns, snapshotRowSet, reverseSnapshotRowSet); long elapsed = System.nanoTime() - start; - if (SUBSCRIPTION_GROWTH_ENABLED) { + if (SUBSCRIPTION_GROWTH_ENABLED && snapshot.rowsIncluded.size() > 0) { // very simplistic logic to take the last snapshot and extrapolate max number of rows that will // not exceed the target UGP processing time percentage - if (snapshot.rowsIncluded.size() > 0) { - long targetNanos = (long) (TARGET_SNAPSHOT_PERCENTAGE - * UpdateGraphProcessor.DEFAULT.getTargetCycleDurationMillis() * 1000000); - long nanosPerCell = elapsed / (snapshot.rowsIncluded.size() * snapshotColumns.cardinality()); - if (nanosPerCell > 0) { - // apply an exponential moving average to filter the data - if (snapshotNanosPerCell == 0) { - snapshotNanosPerCell = nanosPerCell; // initialize to first value - } else { - // EMA smoothing factor is 0.1 (N = 10) - snapshotNanosPerCell = (snapshotNanosPerCell * 0.9) + (nanosPerCell * 0.1); - } - // TODO: remove this when auto-chunking BarrageMessages exists - snapshotTargetCellCount = Math.min((long) (targetNanos / snapshotNanosPerCell), 400000L); - } + long targetNanos = (long) (TARGET_SNAPSHOT_PERCENTAGE + * UpdateGraphProcessor.DEFAULT.getTargetCycleDurationMillis() * 1000000); + + long nanosPerCell = elapsed / (snapshot.rowsIncluded.size() * columnCount); + + // apply an exponential moving average to filter the data + if (snapshotNanosPerCell == 0) { + snapshotNanosPerCell = nanosPerCell; // initialize to first value + } else { + // EMA smoothing factor is 0.1 (N = 10) + snapshotNanosPerCell = (snapshotNanosPerCell * 0.9) + (nanosPerCell * 0.1); } + + snapshotTargetCellCount = (long) (targetNanos / Math.max(1, snapshotNanosPerCell)); } } } @@ -1352,28 +1358,28 @@ private void updateSubscriptionsSnapshotAndPropagate() { private void propagateToSubscribers(final BarrageMessage message, final RowSet propRowSetForMessage) { // message is released via transfer to stream generator (as it must live until all view's are closed) - try (final StreamGenerator generator = - streamGeneratorFactory.newGenerator(message)) { + try (final StreamGenerator generator = streamGeneratorFactory.newGenerator(message)) { for (final Subscription subscription : activeSubscriptions) { if (subscription.pendingInitialSnapshot || subscription.pendingDelete) { continue; } - // There are three messages that might be sent this update: + // There are two messages that might be sent this update: // - pre-snapshot: snapshotViewport/snapshotColumn values apply during this phase - // - snapshot: here we close and clear the snapshotViewport/snapshotColumn values; officially we - // recognize the subscription change // - post-snapshot: now we use the viewport/subscribedColumn values (these are the values the UGP // listener uses) - final RowSet vp = - subscription.snapshotViewport != null ? subscription.snapshotViewport : subscription.viewport; - final BitSet cols = subscription.snapshotColumns != null ? subscription.snapshotColumns - : subscription.subscribedColumns; + + final boolean isPreSnapshot = subscription.snapshotViewport != null; + + final RowSet vp = isPreSnapshot ? subscription.snapshotViewport : subscription.viewport; + final BitSet cols = isPreSnapshot ? subscription.snapshotColumns : subscription.subscribedColumns; try (final RowSet clientView = - subscription.isViewport() - ? propRowSetForMessage.subSetForPositions(vp, subscription.reverseViewport) - : null) { + isPreSnapshot + ? propRowSetForMessage.subSetForPositions(vp, subscription.snapshotReverseViewport) + : subscription.viewport != null + ? propRowSetForMessage.subSetForPositions(vp, subscription.reverseViewport) + : null) { subscription.listener .onNext(generator.getSubView(subscription.options, false, vp, subscription.reverseViewport, clientView, cols)); @@ -1429,14 +1435,11 @@ private void propagateSnapshotForSubscription( .append(System.identityHashCode(subscription)).endl(); } + // limit the rows included by this message to the subset of rows in this snapshot that this subscription + // requested (exclude rows needed by other subscribers but not this one) try (final RowSet keySpaceViewport = snapshotGenerator.getMessage().rowsAdded - .subSetForPositions(subscription.growIncrementalViewport, subscription.reverseViewport)) { -// final boolean isViewport = subscription.viewport != null; -// try (final RowSet keySpaceViewport = -// isViewport -// ? snapshotGenerator.getMessage().rowsAdded -// .subSetForPositions(subscription.viewport, subscription.reverseViewport) -// : null) { + .subSetForPositions(subscription.growingIncrementalViewport, subscription.reverseViewport)) { + if (subscription.pendingInitialSnapshot) { // Send schema metadata to this new client. subscription.listener.onNext(streamGeneratorFactory.getSchemaView( @@ -1444,19 +1447,21 @@ private void propagateSnapshotForSubscription( parent.getAttributes())); } + // some messages may be empty of rows, but we need to update the client viewport and column set subscription.listener .onNext(snapshotGenerator.getSubView(subscription.options, subscription.pendingInitialSnapshot, subscription.viewport, subscription.reverseViewport, keySpaceViewport, subscription.subscribedColumns)); + } catch (final Exception e) { GrpcUtil.safelyExecute(() -> subscription.listener.onError(GrpcUtil.securelyWrapError(log, e))); removeSubscription(subscription.listener); } } - if (subscription.growIncrementalViewport != null) { - subscription.growIncrementalViewport.close(); - subscription.growIncrementalViewport = null; + if (subscription.growingIncrementalViewport != null) { + subscription.growingIncrementalViewport.close(); + subscription.growingIncrementalViewport = null; } subscription.pendingInitialSnapshot = false; @@ -1787,10 +1792,13 @@ private void flipSnapshotStateForSubscriptions( if (subscription.snapshotViewport != null) { final RowSet tmp = subscription.viewport; subscription.viewport = subscription.snapshotViewport; - subscription.reverseViewport = subscription.snapshotReverseViewport; subscription.snapshotViewport = (WritableRowSet) tmp; } + boolean tmpDirection = subscription.reverseViewport; + subscription.reverseViewport = subscription.snapshotReverseViewport; + subscription.snapshotReverseViewport = tmpDirection; + if (subscription.snapshotColumns != null) { final BitSet tmp = subscription.subscribedColumns; subscription.subscribedColumns = subscription.snapshotColumns; @@ -1799,41 +1807,69 @@ private void flipSnapshotStateForSubscriptions( } } - private void finalizeSnapshotForSubscriptions( - final List subscriptions) { + private void finalizeSnapshotForSubscriptions(final List subscriptions) { for (final Subscription subscription : subscriptions) { - // TODO: parent.getRowSet().subSetForPositions is inefficient in this context, would prefer some method that - // short circuits if any matching key of parent.getRowSet() is found. Something like: - // long firstKeyforPositions(final RowSequence positions); // returns -1 for no match + boolean isComplete; + if (subscription.targetViewport == null) { + isComplete = subscription.viewport.containsRange(0, parentTableSize - 1); + } else { + // compare the viewports up to the size of the parent table + try (WritableRowSet viewWithinTable = subscription.viewport.copy(); + WritableRowSet targetWithinTable = subscription.targetViewport.copy()) { - // are there any outstanding rows in the parent that should belong to this subscription? - try (final RowSet remaining = subscription.targetViewport == null - ? RowSetFactory.flat(Long.MAX_VALUE).minus(subscription.viewport) - : subscription.targetViewport.minus(subscription.viewport); - final RowSet remainingKeys = parent.getRowSet().subSetForPositions(remaining)) { + viewWithinTable.removeRange(parentTableSize, Long.MAX_VALUE); + targetWithinTable.removeRange(parentTableSize, Long.MAX_VALUE); - if (remainingKeys.size() == 0) { - // this subscription is complete, remove it from the growing list - subscription.isGrowingViewport = false; - --numGrowingSubscriptions; + isComplete = viewWithinTable.equals(targetWithinTable); + } + } - // set the active viewport to the target viewport - subscription.viewport = subscription.targetViewport; + if (isComplete) { + // this subscription is complete, remove it from the growing list + subscription.isGrowingViewport = false; + --numGrowingSubscriptions; - // expand the postSnapshot viewports to include this entire snapshot - if (subscription.isViewport()) { - if (subscription.reverseViewport) { - postSnapshotReverseViewport.insert(subscription.viewport); - } else { - postSnapshotViewport.insert(subscription.viewport); - } + // set the active viewport to the target viewport + if (subscription.viewport != null) { + subscription.viewport.close(); + } + + subscription.viewport = subscription.targetViewport; + + // expand the postSnapshot viewports to include this entire snapshot + if (subscription.viewport != null) { + if (subscription.reverseViewport) { + postSnapshotReverseViewport.insert(subscription.viewport); + } else { + postSnapshotViewport.insert(subscription.viewport); } } } } } + private void buildPostSnapshotViewports() { + final RowSetBuilderRandom postSnapshotViewportBuilder = RowSetFactory.builderRandom(); + final RowSetBuilderRandom postSnapshotReverseViewportBuilder = RowSetFactory.builderRandom(); + + postSnapshotColumns.clear(); + for (final Subscription sub : activeSubscriptions) { + postSnapshotColumns.or(sub.subscribedColumns); + if (sub.isViewport()) { + // handle forward and reverse snapshots separately + if (sub.snapshotReverseViewport) { + postSnapshotReverseViewportBuilder.addRowSet(sub.viewport); + } else { + postSnapshotViewportBuilder.addRowSet(sub.viewport); + } + } + } + + postSnapshotViewport = postSnapshotViewportBuilder.build(); + postSnapshotReverseViewport = postSnapshotReverseViewportBuilder.build(); + } + private void promoteSnapshotToActive() { Assert.holdsLock(this, "promoteSnapshotToActive must hold lock!"); diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java index 1e8bcbd447b..bba2eefd76d 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java @@ -283,13 +283,18 @@ public SubView(final BarrageStreamGenerator generator, numModBatches = (numModRows + batchSize - 1) / batchSize; // precompute add row offsets - if (keyspaceViewport != null) { - try (WritableRowSet intersect = keyspaceViewport.intersect(generator.rowsIncluded.original)) { - addRowOffsets = generator.rowsIncluded.original.invert(intersect); - } - } else { + if (generator.isSnapshot) { // use chunk data as-is, rely on snapshot process to avoid duplicates addRowOffsets = RowSetFactory.flat(generator.rowsIncluded.original.size()); + } else { + if (keyspaceViewport != null) { + try (WritableRowSet intersect = keyspaceViewport.intersect(generator.rowsIncluded.original)) { + addRowOffsets = generator.rowsIncluded.original.invert(intersect); + } + } else { + // there may be scoped rows included in the chunks that need to be removed + addRowOffsets = generator.rowsIncluded.original.invert(generator.rowsAdded.original); + } } // require an add batch if there are no mod batches @@ -726,10 +731,8 @@ private ByteBuffer getSubscriptionMetadata(final SubView view) throws IOExceptio // Added Chunk Data: int addedRowsIncludedOffset = 0; - if (view.keyspaceViewport != null) { - addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(view.keyspaceViewport, metadata); - } else { - addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(metadata); + try (final RowSet myAddedKeys = rowsIncluded.original.subSetForPositions(view.addRowOffsets())) { + addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(myAddedKeys, metadata); } // now add mod-column streams, and write the mod column indexes From bf79c58aa2e70a2ee00f9669a33db920e8b6824d Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Mon, 21 Mar 2022 16:00:44 -0700 Subject: [PATCH 12/47] nearly complete --- .../barrage/BarrageMessageProducer.java | 131 ++++++++++-------- 1 file changed, 71 insertions(+), 60 deletions(-) diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index 66b57f8beb0..3586b568252 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -834,8 +834,8 @@ private void enqueueUpdate(final TableUpdate upstream) { log.info().append(logPrefix).append("step=").append(LogicalClock.DEFAULT.currentStep()) .append(", upstream=").append(upstream).append(", activeSubscriptions=") .append(activeSubscriptions.size()) - .append(", numFullSubscriptions=").append(numFullSubscriptions).append(", addsToRecord=") - .append(addsToRecord) + .append(", numFullSubscriptions=").append(numFullSubscriptions) + .append(", addsToRecord=").append(addsToRecord) .append(", modsToRecord=").append(modsToRecord) .append(", activeViewport=").append(activeViewport) .append(", activeReverseViewport=").append(activeReverseViewport) @@ -1003,8 +1003,8 @@ private void updateSubscriptionsSnapshotAndPropagate() { } boolean firstSubscription = false; - boolean pendingChanges = false; boolean pendingDeletes = false; + boolean pendingChanges = false; // check for pending changes (under the lock) synchronized (this) { @@ -1013,20 +1013,44 @@ private void updateSubscriptionsSnapshotAndPropagate() { if (!pendingSubscriptions.isEmpty()) { updatedSubscriptions = this.pendingSubscriptions; pendingSubscriptions = new ArrayList<>(); - pendingChanges = true; } if (updatedSubscriptions != null) { - for (final Subscription subscription : updatedSubscriptions) { - if (subscription.pendingDelete) { + // remove deleted subscriptions while we still hold the lock + for (int i = 0; i < activeSubscriptions.size(); ++i) { + final Subscription sub = activeSubscriptions.get(i); + if (sub.pendingDelete) { pendingDeletes = true; + + if (!sub.isViewport()) { + --numFullSubscriptions; + } + if (sub.isGrowingViewport) { + --numGrowingSubscriptions; + } + try { - subscription.listener.onCompleted(); + sub.listener.onCompleted(); } catch (final Exception ignored) { // ignore races on cancellation } - continue; + + activeSubscriptions.set(i, activeSubscriptions.get(activeSubscriptions.size() - 1)); + activeSubscriptions.remove(activeSubscriptions.size() - 1); + + updatedSubscriptions.set(i, updatedSubscriptions.get(updatedSubscriptions.size() - 1)); + updatedSubscriptions.remove(updatedSubscriptions.size() - 1); + + --i; } + } + + // rebuild the viewports, since there are pending changes. This function excludes active subscriptions + // with pending changes because the snapshot process will add those to the active viewports + buildPostSnapshotViewports(); + + for (final Subscription subscription : updatedSubscriptions) { + pendingChanges = true; // add this subscription to the "growing" list to handle snapshot creation if (!subscription.isGrowingViewport) { @@ -1065,30 +1089,10 @@ private void updateSubscriptionsSnapshotAndPropagate() { ? RowSetFactory.flat(Long.MAX_VALUE) : subscription.targetViewport.copy(); } - - if (pendingDeletes) { - // remove deleted subscriptions while we still hold the lock - for (int i = 0; i < activeSubscriptions.size(); ++i) { - final Subscription sub = activeSubscriptions.get(i); - if (sub.pendingDelete) { - if (!sub.isViewport()) { - --numFullSubscriptions; - } - if (sub.isGrowingViewport) { - --numGrowingSubscriptions; - } - - activeSubscriptions.set(i, activeSubscriptions.get(activeSubscriptions.size() - 1)); - activeSubscriptions.remove(activeSubscriptions.size() - 1); - --i; - } - } - } } - if (pendingDeletes && numGrowingSubscriptions == 0) { + if (pendingDeletes && !pendingChanges) { // i.e. We have only removed subscriptions; we can update this state immediately. - buildPostSnapshotViewports(); promoteSnapshotToActive(); } } @@ -1104,8 +1108,11 @@ private void updateSubscriptionsSnapshotAndPropagate() { LinkedList growingSubscriptions = new LinkedList<>(); if (numGrowingSubscriptions > 0) { - // start with fresh viewports - buildPostSnapshotViewports(); + // use the current active columns and viewport for the starting point of this post-snapshot view + postSnapshotViewport = activeViewport != null ? activeViewport.copy() : RowSetFactory.empty(); + postSnapshotReverseViewport = + activeReverseViewport != null ? activeReverseViewport.copy() : RowSetFactory.empty(); + postSnapshotColumns.or(activeColumns); snapshotColumns = new BitSet(); @@ -1184,15 +1191,12 @@ private void updateSubscriptionsSnapshotAndPropagate() { // add the new rows to the upcoming snapshot currentSet.insert(additional); - // extra bookkeeping (for viewport subscriptions when there are no full subs) - if (subscription.targetViewport != null) { - if (subscription.targetReverseViewport) { - // add this set to the BMP reverse viewport - reverseViewportBuilder.addRowSet(additional); - } else { - // add this set to the BMP forward viewport - viewportBuilder.addRowSet(additional); - } + if (subscription.targetReverseViewport) { + // add this set to the global reverse viewport (for scoping) + reverseViewportBuilder.addRowSet(additional); + } else { + // add this set to the global forward viewport (for scoping) + viewportBuilder.addRowSet(additional); } // decrement the remaining row count @@ -1230,7 +1234,6 @@ private void updateSubscriptionsSnapshotAndPropagate() { postSnapshotViewport.insert(vp); postSnapshotReverseViewport.insert(rvp); } - postSnapshotColumns.or(snapshotColumns); // finally, grab the snapshot and measure elapsed time for next projections @@ -1375,9 +1378,9 @@ private void propagateToSubscribers(final BarrageMessage message, final RowSet p final BitSet cols = isPreSnapshot ? subscription.snapshotColumns : subscription.subscribedColumns; try (final RowSet clientView = - isPreSnapshot - ? propRowSetForMessage.subSetForPositions(vp, subscription.snapshotReverseViewport) - : subscription.viewport != null + isPreSnapshot + ? propRowSetForMessage.subSetForPositions(vp, subscription.snapshotReverseViewport) + : subscription.viewport != null ? propRowSetForMessage.subSetForPositions(vp, subscription.reverseViewport) : null) { subscription.listener @@ -1808,20 +1811,26 @@ private void flipSnapshotStateForSubscriptions( } private void finalizeSnapshotForSubscriptions(final List subscriptions) { - for (final Subscription subscription : subscriptions) { + boolean rebuildViewport = false; + for (final Subscription subscription : subscriptions) { boolean isComplete; if (subscription.targetViewport == null) { isComplete = subscription.viewport.containsRange(0, parentTableSize - 1); } else { - // compare the viewports up to the size of the parent table - try (WritableRowSet viewWithinTable = subscription.viewport.copy(); - WritableRowSet targetWithinTable = subscription.targetViewport.copy()) { + // any more rows to gather? + if (subscription.growingRemainingViewport.isEmpty()) { + isComplete = true; + } else { + // compare the viewports up to the size of the parent table + try (WritableRowSet viewWithinTable = subscription.viewport.copy(); + WritableRowSet targetWithinTable = subscription.targetViewport.copy()) { - viewWithinTable.removeRange(parentTableSize, Long.MAX_VALUE); - targetWithinTable.removeRange(parentTableSize, Long.MAX_VALUE); + viewWithinTable.removeRange(parentTableSize, Long.MAX_VALUE); + targetWithinTable.removeRange(parentTableSize, Long.MAX_VALUE); - isComplete = viewWithinTable.equals(targetWithinTable); + isComplete = viewWithinTable.equals(targetWithinTable); + } } } @@ -1837,28 +1846,30 @@ private void finalizeSnapshotForSubscriptions(final List subscript subscription.viewport = subscription.targetViewport; - // expand the postSnapshot viewports to include this entire snapshot - if (subscription.viewport != null) { - if (subscription.reverseViewport) { - postSnapshotReverseViewport.insert(subscription.viewport); - } else { - postSnapshotViewport.insert(subscription.viewport); - } - } + // after each satisfied subscription, we need to rebuild the active viewports so the temporary + // growing viewports are not included in `scoped` rows in later updates. + rebuildViewport = true; } } + if (rebuildViewport) { + buildPostSnapshotViewports(); + } } private void buildPostSnapshotViewports() { + // rebuild the viewports for the active snapshots, but exclude any that have pending changes. final RowSetBuilderRandom postSnapshotViewportBuilder = RowSetFactory.builderRandom(); final RowSetBuilderRandom postSnapshotReverseViewportBuilder = RowSetFactory.builderRandom(); postSnapshotColumns.clear(); for (final Subscription sub : activeSubscriptions) { + if (sub.hasPendingUpdate) { + continue; + } postSnapshotColumns.or(sub.subscribedColumns); if (sub.isViewport()) { // handle forward and reverse snapshots separately - if (sub.snapshotReverseViewport) { + if (sub.reverseViewport) { postSnapshotReverseViewportBuilder.addRowSet(sub.viewport); } else { postSnapshotViewportBuilder.addRowSet(sub.viewport); From 529701c1d233b6ca3f3e93d61b6619ede570768e Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Tue, 22 Mar 2022 13:17:30 -0700 Subject: [PATCH 13/47] Minor changes, fixed a memory leak in buildPostSnapshotViewports --- .../barrage/BarrageMessageProducer.java | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index 3586b568252..5ca60f6bafc 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -451,7 +451,7 @@ private class Subscription { RowSet viewport; // active viewport BitSet subscribedColumns; // active subscription columns - boolean reverseViewport = false; // is the active viewport reversed (indexed from end of table) + boolean reverseViewport; // is the active viewport reversed (indexed from end of table) boolean isActive = false; // is this subscription in our active list? boolean pendingDelete = false; // is this subscription deleted as far as the client is concerned? @@ -577,9 +577,8 @@ public boolean updateSubscription(final StreamObserver listener, fi if (sub.pendingViewport != null) { sub.pendingViewport.close(); } - sub.pendingViewport = newViewport.copy(); + sub.pendingViewport = newViewport != null ? newViewport.copy() : null; sub.pendingReverseViewport = newReverseViewport; - final BitSet cols; if (columnsToSubscribe == null) { cols = new BitSet(sourceColumns.length); @@ -1035,17 +1034,16 @@ private void updateSubscriptionsSnapshotAndPropagate() { // ignore races on cancellation } + // remove this deleted subscription from future consideration activeSubscriptions.set(i, activeSubscriptions.get(activeSubscriptions.size() - 1)); activeSubscriptions.remove(activeSubscriptions.size() - 1); - updatedSubscriptions.set(i, updatedSubscriptions.get(updatedSubscriptions.size() - 1)); updatedSubscriptions.remove(updatedSubscriptions.size() - 1); - --i; } } - // rebuild the viewports, since there are pending changes. This function excludes active subscriptions + // rebuild the viewports since there are pending changes. This function excludes active subscriptions // with pending changes because the snapshot process will add those to the active viewports buildPostSnapshotViewports(); @@ -1092,7 +1090,7 @@ private void updateSubscriptionsSnapshotAndPropagate() { } if (pendingDeletes && !pendingChanges) { - // i.e. We have only removed subscriptions; we can update this state immediately. + // we have only removed subscriptions; we can update this state immediately. promoteSnapshotToActive(); } } @@ -1102,7 +1100,7 @@ private void updateSubscriptionsSnapshotAndPropagate() { BarrageMessage snapshot = null; BarrageMessage postSnapshot = null; - BitSet snapshotColumns = null; + BitSet snapshotColumns; // create a prioritized list for the subscriptions LinkedList growingSubscriptions = new LinkedList<>(); @@ -1122,9 +1120,9 @@ private void updateSubscriptionsSnapshotAndPropagate() { snapshotColumns.or(subscription.targetColumns); if (subscription.targetViewport == null) { - growingSubscriptions.addLast(subscription); // full gets low priority + growingSubscriptions.addLast(subscription); // full sub gets low priority } else { - growingSubscriptions.addFirst(subscription); // viewport gets higher priority + growingSubscriptions.addFirst(subscription); // viewport sub gets higher priority } } } @@ -1154,10 +1152,8 @@ private void updateSubscriptionsSnapshotAndPropagate() { // we need to determine if the `activeViewport` is valid. if the viewport direction changes or // columns were added, the client viewport is invalid - BitSet addedCols = (BitSet) subscription.targetColumns.clone(); addedCols.andNot(subscription.subscribedColumns); - final boolean viewportValid = subscription.reverseViewport == subscription.targetReverseViewport && addedCols.isEmpty(); @@ -1173,10 +1169,9 @@ private void updateSubscriptionsSnapshotAndPropagate() { // get the rows that we need that are already in the snapshot try (final RowSet overlap = subscription.growingRemainingViewport.extract(currentSet)) { if (rowsRemaining <= 0) { - // we can't request additional rows, but can use other rows in this snapshot + // can't request additional rows, but can use snapshot rows that match our viewport subscription.growingIncrementalViewport = overlap.copy(); } else { - // request as many rows as we are allowed try (final WritableRowSet additional = subscription.growingRemainingViewport.copy()) { // shrink the set of new rows to <= `rowsRemaining` size @@ -1598,11 +1593,9 @@ private BarrageMessage aggregateUpdatesInRange(final int startDelta, final int e } // One drawback of the ModifiedColumnSet, is that our adds must include data for all columns. However, - // column - // specific data may be updated and we only write down that single changed column. So, the computation of - // mapping - // output rows to input data may be different per Column. We can re-use calculations where the set of deltas - // that modify column A are the same as column B. + // column specific data may be updated and we only write down that single changed column. So, the + // computation of mapping output rows to input data may be different per Column. We can re-use calculations + // where the set of deltas that modify column A are the same as column B. final class ColumnInfo { final WritableRowSet modified = RowSetFactory.empty(); final WritableRowSet recordedMods = RowSetFactory.empty(); @@ -1877,6 +1870,12 @@ private void buildPostSnapshotViewports() { } } + if (postSnapshotViewport != null) { + postSnapshotViewport.close(); + } + if (postSnapshotReverseViewport != null) { + postSnapshotReverseViewport.close(); + } postSnapshotViewport = postSnapshotViewportBuilder.build(); postSnapshotReverseViewport = postSnapshotReverseViewportBuilder.build(); } From 9391ad502ffc2e3b02643563fd7d64a5e252e138 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Tue, 22 Mar 2022 14:27:22 -0700 Subject: [PATCH 14/47] Attempted to avoid a deadlock --- .../io/deephaven/client/impl/BarrageSubscriptionImpl.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java index a93aaf70c96..7846845ac9e 100644 --- a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java +++ b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java @@ -196,6 +196,9 @@ public synchronized BarrageTable partialTable(RowSet viewport, BitSet columns, b if (UpdateGraphProcessor.DEFAULT.exclusiveLock().isHeldByCurrentThread()) { completedCondition = UpdateGraphProcessor.DEFAULT.exclusiveLock().newCondition(); + + // wake up any active listeners and convert them to holding the exclusive lock + notifyAll(); } // Send the initial subscription: @@ -276,7 +279,7 @@ private void handleDisconnect() { } @Override - public void close() { + public synchronized void close() { if (!connected) { return; } From df25d400f4d9fcab3a3feca17f706c68d6e0964a Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Wed, 30 Mar 2022 13:54:40 -0700 Subject: [PATCH 15/47] most comments done, tests still passing --- .../barrage/BarrageMessageProducer.java | 207 +++++++++--------- 1 file changed, 106 insertions(+), 101 deletions(-) diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index 5ca60f6bafc..e43ea26be74 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -473,6 +473,7 @@ private class Subscription { boolean isGrowingViewport; // does this have a growing viewport? WritableRowSet growingRemainingViewport = null; // remaining viewport rows for a growing subscription WritableRowSet growingIncrementalViewport = null; // incremental viewport rows for a growing subscription + boolean isFirstSnapshot; // is this the first snapshot in a growing snapshot? private Subscription(final StreamObserver listener, final BarrageSubscriptionOptions options, @@ -482,7 +483,7 @@ private Subscription(final StreamObserver listener, this.options = options; this.listener = listener; this.logPrefix = "Sub{" + Integer.toHexString(System.identityHashCode(listener)) + "}: "; - this.viewport = initialViewport != null ? RowSetFactory.empty() : null; + this.viewport = RowSetFactory.empty(); this.subscribedColumns = new BitSet(); this.pendingColumns = subscribedColumns; this.pendingViewport = initialViewport; @@ -1037,55 +1038,51 @@ private void updateSubscriptionsSnapshotAndPropagate() { // remove this deleted subscription from future consideration activeSubscriptions.set(i, activeSubscriptions.get(activeSubscriptions.size() - 1)); activeSubscriptions.remove(activeSubscriptions.size() - 1); - updatedSubscriptions.set(i, updatedSubscriptions.get(updatedSubscriptions.size() - 1)); - updatedSubscriptions.remove(updatedSubscriptions.size() - 1); --i; } } // rebuild the viewports since there are pending changes. This function excludes active subscriptions // with pending changes because the snapshot process will add those to the active viewports - buildPostSnapshotViewports(); + buildPostSnapshotViewports(true); for (final Subscription subscription : updatedSubscriptions) { - pendingChanges = true; + if (!subscription.pendingDelete) { + pendingChanges = true; - // add this subscription to the "growing" list to handle snapshot creation - if (!subscription.isGrowingViewport) { - subscription.isGrowingViewport = true; - ++numGrowingSubscriptions; - } - - subscription.hasPendingUpdate = false; - if (!subscription.isActive) { - firstSubscription |= activeSubscriptions.isEmpty(); + // add this subscription to the "growing" list to handle snapshot creation + if (!subscription.isGrowingViewport) { + subscription.isGrowingViewport = true; + ++numGrowingSubscriptions; + } - // Note that initial subscriptions have empty viewports and no subscribed columns. - subscription.isActive = true; - activeSubscriptions.add(subscription); + subscription.hasPendingUpdate = false; + if (!subscription.isActive) { + firstSubscription |= activeSubscriptions.isEmpty(); - if (!subscription.isViewport()) { - ++numFullSubscriptions; + // Note that initial subscriptions have empty viewports and no subscribed columns. + subscription.isActive = true; + activeSubscriptions.add(subscription); } - } - if (subscription.pendingViewport != null) { subscription.targetViewport = subscription.pendingViewport; subscription.pendingViewport = null; - } - subscription.targetColumns = subscription.pendingColumns; - subscription.pendingColumns = null; + subscription.targetColumns = subscription.pendingColumns; + subscription.pendingColumns = null; - subscription.targetReverseViewport = subscription.pendingReverseViewport; + subscription.targetReverseViewport = subscription.pendingReverseViewport; - // get the set of remaining rows for this subscription - if (subscription.growingRemainingViewport != null) { - subscription.growingRemainingViewport.close(); + subscription.isFirstSnapshot = true; + + // get the set of remaining rows for this subscription + if (subscription.growingRemainingViewport != null) { + subscription.growingRemainingViewport.close(); + } + subscription.growingRemainingViewport = subscription.targetViewport == null + ? RowSetFactory.flat(Long.MAX_VALUE) + : subscription.targetViewport.copy(); } - subscription.growingRemainingViewport = subscription.targetViewport == null - ? RowSetFactory.flat(Long.MAX_VALUE) - : subscription.targetViewport.copy(); } } @@ -1106,11 +1103,13 @@ private void updateSubscriptionsSnapshotAndPropagate() { LinkedList growingSubscriptions = new LinkedList<>(); if (numGrowingSubscriptions > 0) { - // use the current active columns and viewport for the starting point of this post-snapshot view - postSnapshotViewport = activeViewport != null ? activeViewport.copy() : RowSetFactory.empty(); - postSnapshotReverseViewport = - activeReverseViewport != null ? activeReverseViewport.copy() : RowSetFactory.empty(); - postSnapshotColumns.or(activeColumns); + if (!pendingChanges) { + // use the current active columns and viewport for the starting point of this post-snapshot view + postSnapshotViewport = activeViewport != null ? activeViewport.copy() : RowSetFactory.empty(); + postSnapshotReverseViewport = + activeReverseViewport != null ? activeReverseViewport.copy() : RowSetFactory.empty(); + postSnapshotColumns.or(activeColumns); + } snapshotColumns = new BitSet(); @@ -1157,65 +1156,80 @@ private void updateSubscriptionsSnapshotAndPropagate() { final boolean viewportValid = subscription.reverseViewport == subscription.targetReverseViewport && addedCols.isEmpty(); - // don't request valid rows that the client already possesses if (viewportValid && subscription.viewport != null) { - subscription.growingRemainingViewport.remove(subscription.viewport); + // handle the first snapshot of a growing subscription differently + if (subscription.isFirstSnapshot) { + // identify rows in the both the current viewport and the remaining viewport + subscription.snapshotViewport = subscription.growingRemainingViewport.extract(subscription.viewport); + + // add these to the global viewports (for scoping) + if (subscription.targetReverseViewport) { + reverseViewportBuilder.addRowSet(subscription.snapshotViewport); + } else { + viewportBuilder.addRowSet(subscription.snapshotViewport); + } + } else { + // after the first snapshot, we can use the valid viewport + subscription.snapshotViewport = subscription.viewport.copy(); + } + } else { + subscription.snapshotViewport = RowSetFactory.empty(); } + subscription.isFirstSnapshot = false; + // get the current set for this viewport direction final WritableRowSet currentSet = subscription.targetReverseViewport ? reverseSnapshotRowSet : snapshotRowSet; // get the rows that we need that are already in the snapshot - try (final RowSet overlap = subscription.growingRemainingViewport.extract(currentSet)) { - if (rowsRemaining <= 0) { - // can't request additional rows, but can use snapshot rows that match our viewport - subscription.growingIncrementalViewport = overlap.copy(); - } else { - try (final WritableRowSet additional = subscription.growingRemainingViewport.copy()) { + subscription.growingIncrementalViewport = subscription.growingRemainingViewport.extract(currentSet); + if (rowsRemaining > 0) { + try (final WritableRowSet additional = subscription.growingRemainingViewport.copy()) { - // shrink the set of new rows to <= `rowsRemaining` size - if (additional.size() > rowsRemaining) { - final long key = additional.get(rowsRemaining); - additional.removeRange(key, Long.MAX_VALUE - 1); - } + // shrink the set of new rows to <= `rowsRemaining` size + if (additional.size() > rowsRemaining) { + final long key = additional.get(rowsRemaining); + additional.removeRange(key, Long.MAX_VALUE - 1); - // store the rowset that applies for this exact snapshot - subscription.growingIncrementalViewport = overlap.union(additional); + // update the rows remaining + subscription.growingRemainingViewport.removeRange(0, key - 1); + } else { + // all rows are satisfied + subscription.growingRemainingViewport.clear(); + } - // add the new rows to the upcoming snapshot - currentSet.insert(additional); + // store the rowset that applies for this exact snapshot + subscription.growingIncrementalViewport.insert(additional); - if (subscription.targetReverseViewport) { - // add this set to the global reverse viewport (for scoping) - reverseViewportBuilder.addRowSet(additional); - } else { - // add this set to the global forward viewport (for scoping) - viewportBuilder.addRowSet(additional); - } + // add the new rows to the upcoming snapshot + currentSet.insert(additional); - // decrement the remaining row count - rowsRemaining -= additional.size(); + if (subscription.targetReverseViewport) { + // add this set to the global reverse viewport (for scoping) + reverseViewportBuilder.addRowSet(additional); + } else { + // add this set to the global forward viewport (for scoping) + viewportBuilder.addRowSet(additional); } + + // decrement the remaining row count + rowsRemaining -= additional.size(); } } // "grow" the snapshot viewport with rows from the current snapshot if (viewportValid && subscription.viewport != null) { // we can use the current viewport as a start - subscription.snapshotViewport = - subscription.viewport.union(subscription.growingIncrementalViewport); + subscription.snapshotViewport.insert(subscription.viewport); // the current viewport might contain rows that we don't want (from a previous viewport, e.g.) if (subscription.targetViewport != null) { subscription.snapshotViewport.retain(subscription.targetViewport); } - } else { - subscription.snapshotViewport = subscription.growingIncrementalViewport.copy(); } - // remove the rows we are about to snapshot from the remaining set - subscription.growingRemainingViewport.remove(subscription.growingIncrementalViewport); + subscription.snapshotViewport.insert(subscription.growingIncrementalViewport); // save the column set subscription.snapshotColumns = (BitSet) subscription.targetColumns.clone(); @@ -1301,7 +1315,7 @@ private void updateSubscriptionsSnapshotAndPropagate() { // cleanup for next iteration clearObjectDeltaColumns(objectColumnsToClear); - if (pendingChanges) { + if (pendingDeletes || pendingChanges) { objectColumnsToClear.clear(); objectColumnsToClear.or(objectColumns); objectColumnsToClear.and(activeColumns); @@ -1362,8 +1376,10 @@ private void propagateToSubscribers(final BarrageMessage message, final RowSet p continue; } - // There are two messages that might be sent this update: + // There are three messages that might be sent this update: // - pre-snapshot: snapshotViewport/snapshotColumn values apply during this phase + // - snapshot: here we close and clear the snapshotViewport/snapshotColumn values; officially we + // recognize the subscription change // - post-snapshot: now we use the viewport/subscribedColumn values (these are the values the UGP // listener uses) @@ -1371,13 +1387,10 @@ private void propagateToSubscribers(final BarrageMessage message, final RowSet p final RowSet vp = isPreSnapshot ? subscription.snapshotViewport : subscription.viewport; final BitSet cols = isPreSnapshot ? subscription.snapshotColumns : subscription.subscribedColumns; + final boolean isReversed = + isPreSnapshot ? subscription.snapshotReverseViewport : subscription.reverseViewport; - try (final RowSet clientView = - isPreSnapshot - ? propRowSetForMessage.subSetForPositions(vp, subscription.snapshotReverseViewport) - : subscription.viewport != null - ? propRowSetForMessage.subSetForPositions(vp, subscription.reverseViewport) - : null) { + try (final RowSet clientView = vp != null ? propRowSetForMessage.subSetForPositions(vp, isReversed) : null) { subscription.listener .onNext(generator.getSubView(subscription.options, false, vp, subscription.reverseViewport, clientView, cols)); @@ -1807,25 +1820,8 @@ private void finalizeSnapshotForSubscriptions(final List subscript boolean rebuildViewport = false; for (final Subscription subscription : subscriptions) { - boolean isComplete; - if (subscription.targetViewport == null) { - isComplete = subscription.viewport.containsRange(0, parentTableSize - 1); - } else { - // any more rows to gather? - if (subscription.growingRemainingViewport.isEmpty()) { - isComplete = true; - } else { - // compare the viewports up to the size of the parent table - try (WritableRowSet viewWithinTable = subscription.viewport.copy(); - WritableRowSet targetWithinTable = subscription.targetViewport.copy()) { - - viewWithinTable.removeRange(parentTableSize, Long.MAX_VALUE); - targetWithinTable.removeRange(parentTableSize, Long.MAX_VALUE); - - isComplete = viewWithinTable.equals(targetWithinTable); - } - } - } + boolean isComplete = subscription.growingRemainingViewport.isEmpty() + || subscription.growingRemainingViewport.firstRowKey() >= parentTableSize; if (isComplete) { // this subscription is complete, remove it from the growing list @@ -1836,27 +1832,36 @@ private void finalizeSnapshotForSubscriptions(final List subscript if (subscription.viewport != null) { subscription.viewport.close(); } - subscription.viewport = subscription.targetViewport; - // after each satisfied subscription, we need to rebuild the active viewports so the temporary - // growing viewports are not included in `scoped` rows in later updates. + if (subscription.viewport == null) { + // track active `full` subscriptions + ++numFullSubscriptions; + } + + subscription.growingRemainingViewport.close(); + subscription.growingRemainingViewport = null; + + // after each satisfied subscription, we need to rebuild the active viewports: + // - full subscriptions should no longer be considered viewports + // - viewports that were satisfied via the table size check are not yet fully included rebuildViewport = true; } } if (rebuildViewport) { - buildPostSnapshotViewports(); + // don't exclude subscriptions with pending changes here + buildPostSnapshotViewports(false); } } - private void buildPostSnapshotViewports() { + private void buildPostSnapshotViewports(boolean ignorePending) { // rebuild the viewports for the active snapshots, but exclude any that have pending changes. final RowSetBuilderRandom postSnapshotViewportBuilder = RowSetFactory.builderRandom(); final RowSetBuilderRandom postSnapshotReverseViewportBuilder = RowSetFactory.builderRandom(); postSnapshotColumns.clear(); for (final Subscription sub : activeSubscriptions) { - if (sub.hasPendingUpdate) { + if (ignorePending && sub.hasPendingUpdate) { continue; } postSnapshotColumns.or(sub.subscribedColumns); From ac6308bb31a1e206809d0bbcf4e2ba799767cac6 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Wed, 30 Mar 2022 15:00:00 -0700 Subject: [PATCH 16/47] final comments addressed --- .../barrage/BarrageStreamGenerator.java | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java index bba2eefd76d..983e9493d4c 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java @@ -282,19 +282,15 @@ public SubView(final BarrageStreamGenerator generator, } numModBatches = (numModRows + batchSize - 1) / batchSize; - // precompute add row offsets - if (generator.isSnapshot) { - // use chunk data as-is, rely on snapshot process to avoid duplicates - addRowOffsets = RowSetFactory.flat(generator.rowsIncluded.original.size()); - } else { - if (keyspaceViewport != null) { - try (WritableRowSet intersect = keyspaceViewport.intersect(generator.rowsIncluded.original)) { - addRowOffsets = generator.rowsIncluded.original.invert(intersect); - } - } else { - // there may be scoped rows included in the chunks that need to be removed - addRowOffsets = generator.rowsIncluded.original.invert(generator.rowsAdded.original); + if (keyspaceViewport != null) { + try (WritableRowSet intersect = keyspaceViewport.intersect(generator.rowsIncluded.original)) { + addRowOffsets = generator.rowsIncluded.original.invert(intersect); } + } else if (!generator.rowsAdded.original.equals(generator.rowsIncluded.original)) { + // there are scoped rows included in the chunks that need to be removed + addRowOffsets = generator.rowsIncluded.original.invert(generator.rowsAdded.original); + } else { + addRowOffsets = RowSetFactory.flat(generator.rowsAdded.original.size()); } // require an add batch if there are no mod batches @@ -731,8 +727,8 @@ private ByteBuffer getSubscriptionMetadata(final SubView view) throws IOExceptio // Added Chunk Data: int addedRowsIncludedOffset = 0; - try (final RowSet myAddedKeys = rowsIncluded.original.subSetForPositions(view.addRowOffsets())) { - addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(myAddedKeys, metadata); + if (view.keyspaceViewport != null) { + addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(view.keyspaceViewport, metadata); } // now add mod-column streams, and write the mod column indexes From 9460baca04fe92802851ad24f0686467b23c4848 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Wed, 30 Mar 2022 15:06:56 -0700 Subject: [PATCH 17/47] spotless checks --- .../io/deephaven/server/barrage/BarrageMessageProducer.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index e43ea26be74..15e8395b808 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -1160,7 +1160,8 @@ private void updateSubscriptionsSnapshotAndPropagate() { // handle the first snapshot of a growing subscription differently if (subscription.isFirstSnapshot) { // identify rows in the both the current viewport and the remaining viewport - subscription.snapshotViewport = subscription.growingRemainingViewport.extract(subscription.viewport); + subscription.snapshotViewport = + subscription.growingRemainingViewport.extract(subscription.viewport); // add these to the global viewports (for scoping) if (subscription.targetReverseViewport) { @@ -1390,7 +1391,8 @@ private void propagateToSubscribers(final BarrageMessage message, final RowSet p final boolean isReversed = isPreSnapshot ? subscription.snapshotReverseViewport : subscription.reverseViewport; - try (final RowSet clientView = vp != null ? propRowSetForMessage.subSetForPositions(vp, isReversed) : null) { + try (final RowSet clientView = + vp != null ? propRowSetForMessage.subSetForPositions(vp, isReversed) : null) { subscription.listener .onNext(generator.getSubView(subscription.options, false, vp, subscription.reverseViewport, clientView, cols)); From e9e9421f273580c1e470a5c177b8a03f2e99e2b6 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Wed, 30 Mar 2022 17:22:34 -0700 Subject: [PATCH 18/47] final PR comments addressed --- .../client/impl/BarrageSubscription.java | 5 ---- .../client/impl/BarrageSubscriptionImpl.java | 30 ++++++++----------- 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscription.java b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscription.java index 26d3e26ea13..ddac6b5e8ce 100644 --- a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscription.java +++ b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscription.java @@ -46,11 +46,6 @@ BarrageSubscription subscribe(TableSpec tableSpec, BarrageSubscriptionOptions op */ public boolean isCompleted(); - /** - * This call will block until all rows for the subscribed table are available. - */ - public void waitForCompletion() throws InterruptedException; - /** * Request a full subscription of the data and populate a {@link BarrageTable} with the incrementally updating data * that is received. This call will block until all rows for the subscribed table are available. diff --git a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java index 7846845ac9e..007f552a93e 100644 --- a/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java +++ b/java-client/barrage/src/main/java/io/deephaven/client/impl/BarrageSubscriptionImpl.java @@ -146,18 +146,6 @@ public boolean isCompleted() { return completed; } - @Override - public synchronized void waitForCompletion() throws InterruptedException { - while (!completed && exceptionWhileCompleting == null) { - // handle the condition where this function may have the exclusive lock - if (completedCondition != null) { - completedCondition.await(); - } else { - wait(); // barragesnapshotimpl lock - } - } - } - @Override public BarrageTable entireTable() throws InterruptedException { return entireTable(true); @@ -186,8 +174,10 @@ public synchronized BarrageTable partialTable(RowSet viewport, BitSet columns, b throw new UncheckedDeephavenException( this + " is no longer an active subscription and cannot be retained further"); } - if (!subscribed) { - + if (subscribed) { + throw new UncheckedDeephavenException( + "BarrageSubscription objects cannot be reused."); + } else { // test lock conditions if (UpdateGraphProcessor.DEFAULT.sharedLock().isHeldByCurrentThread()) { throw new UnsupportedOperationException( @@ -196,9 +186,6 @@ public synchronized BarrageTable partialTable(RowSet viewport, BitSet columns, b if (UpdateGraphProcessor.DEFAULT.exclusiveLock().isHeldByCurrentThread()) { completedCondition = UpdateGraphProcessor.DEFAULT.exclusiveLock().newCondition(); - - // wake up any active listeners and convert them to holding the exclusive lock - notifyAll(); } // Send the initial subscription: @@ -253,7 +240,14 @@ public void onUpdate(final TableUpdate upstream) { resultTable.listenForUpdates(listener); if (blockUntilComplete) { - waitForCompletion(); + while (!completed && exceptionWhileCompleting == null) { + // handle the condition where this function may have the exclusive lock + if (completedCondition != null) { + completedCondition.await(); + } else { + wait(); // barragesubscriptionimpl lock + } + } } } From 6a12d57ed10298c9c1151ac5617876f4d599cd2f Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Thu, 31 Mar 2022 11:51:41 -0700 Subject: [PATCH 19/47] added tracking for addRowKeys --- .../server/barrage/BarrageStreamGenerator.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java index 406c1835e19..bf9a5303616 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java @@ -241,6 +241,7 @@ public static class SubView implements View { public final long numAddBatches; public final long numModBatches; public final RowSet addRowOffsets; + public final RowSet addRowKeys; public final RowSet[] modRowOffsets; public SubView(final BarrageStreamGenerator generator, @@ -283,13 +284,14 @@ public SubView(final BarrageStreamGenerator generator, numModBatches = (numModRows + batchSize - 1) / batchSize; if (keyspaceViewport != null) { - try (WritableRowSet intersect = keyspaceViewport.intersect(generator.rowsIncluded.original)) { - addRowOffsets = generator.rowsIncluded.original.invert(intersect); - } + addRowKeys = keyspaceViewport.intersect(generator.rowsIncluded.original); + addRowOffsets = generator.rowsIncluded.original.invert(addRowKeys); } else if (!generator.rowsAdded.original.equals(generator.rowsIncluded.original)) { // there are scoped rows included in the chunks that need to be removed + addRowKeys = generator.rowsAdded.original.copy(); addRowOffsets = generator.rowsIncluded.original.invert(generator.rowsAdded.original); } else { + addRowKeys = generator.rowsAdded.original.copy(); addRowOffsets = RowSetFactory.flat(generator.rowsAdded.original.size()); } @@ -319,6 +321,7 @@ public void forEachStream(Consumer visitor) throws IOException { // clean up the helper indexes addRowOffsets.close(); + addRowKeys.close(); if (modRowOffsets != null) { for (final RowSet modViewport : modRowOffsets) { modViewport.close(); @@ -727,9 +730,10 @@ private ByteBuffer getSubscriptionMetadata(final SubView view) throws IOExceptio // Added Chunk Data: int addedRowsIncludedOffset = 0; + // don't send `rowsIncluded` when identical to `rowsAdded`, client will infer they are the same - if (view.keyspaceViewport != null && !rowsIncluded.original.equals(rowsAdded.original)) { - addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(view.keyspaceViewport, metadata); + if (isSnapshot || !view.addRowKeys.equals(rowsAdded.original)) { + addedRowsIncludedOffset = rowsIncluded.addToFlatBuffer(view.addRowKeys, metadata); } // now add mod-column streams, and write the mod column indexes From 5ae5cfcbffb5eff6d38f35a8cfcb8713b20b1ee3 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Tue, 5 Apr 2022 09:22:02 -0700 Subject: [PATCH 20/47] docs and PR comments --- .../barrage/BarrageMessageProducer.java | 172 ++++++++++-------- 1 file changed, 98 insertions(+), 74 deletions(-) diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index 80d26363ec4..947a6aacf78 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -245,11 +245,6 @@ public Result> initialize(final boolean useP scheduler, streamGeneratorFactory, parent, updateIntervalMs, onGetSnapshot); return new Result<>(result, result.constructListener()); } - - @Override - public boolean snapshotNeeded() { - return false; - } } private static class MyMemoKey extends MemoizedOperationKey { @@ -437,10 +432,9 @@ public TableDefinition getTableDefinition() { * 3) The BMP's update propagation job runs. All pendingSubscriptions (new or updated) will have their pending * viewport / columns requests accepted. All pendingSubscriptions move to the activeSubscription list if they * were brand new. The pendingSubscription list is cleared. At this stage, the `pending` variables are nulled - * and their contents move to the variables prefixed with `snapshot`. If a viewport's subscribedColumns change - * when the viewport remains the same, we copy the reference from `viewport` to `snapshotViewport`. The - * propagation job is responsible for building the snapshot and sending it to the client. Finally, the - * `snapshot` variables are nulled and promoted to `viewport` and `subscribedColumns`. + * and their contents move to the variables prefixed with `target`. The propagation job is responsible for + * building the snapshot and sending it to the client. When the snapshot is complete, the `target` variables + * are nulled and promoted to `viewport` and `subscribedColumns`. * 4) If a subscription is updated during or after stage 3, it will be added back to the pendingSubscription list, * and the updated requests will sit in the `pending` variables until the next time the update propagation job * executes. It will NOT be removed from the activeSubscription list. A given subscription will exist no more @@ -470,18 +464,18 @@ private class Subscription { boolean pendingReverseViewport; // is the pending viewport reversed (indexed from end of table) BitSet pendingColumns; // if an update is pending this is our new column subscription set - WritableRowSet snapshotViewport = null; // captured viewport during snapshot portion of propagation job - BitSet snapshotColumns = null; // captured column during snapshot portion of propagation job - boolean snapshotReverseViewport = false; // captured setting during snapshot portion of propagation job + WritableRowSet snapshotViewport = null; // promoted to `active` viewport by the snapshot process + BitSet snapshotColumns = null; // promoted to `active` columns by the snapshot process + boolean snapshotReverseViewport = false; // promoted to `active` viewport direction by the snapshot process - RowSet targetViewport = null; // target viewport for a growing subscription - boolean targetReverseViewport; // is the target viewport reversed (indexed from end of table) - BitSet targetColumns; // target column subscription set + RowSet targetViewport = null; // the final viewport for a changed (new or updated) subscription + BitSet targetColumns; // the final set of columns for a changed subscription + boolean targetReverseViewport; // the final viewport direction for a changed subscription - boolean isGrowingViewport; // does this have a growing viewport? - WritableRowSet growingRemainingViewport = null; // remaining viewport rows for a growing subscription - WritableRowSet growingIncrementalViewport = null; // incremental viewport rows for a growing subscription - boolean isFirstSnapshot; // is this the first snapshot in a growing snapshot? + boolean isGrowingViewport; // is this subscription actively growing + WritableRowSet growingRemainingViewport = null; // rows still needed to satisfy this subscription viewport + WritableRowSet growingIncrementalViewport = null; // rows to be sent to the client from the current snapshot + boolean isFirstSnapshot; // is this the first snapshot after a change to a subscriptions private Subscription(final StreamObserver listener, final BarrageSubscriptionOptions options, @@ -625,7 +619,6 @@ private class DeltaListener extends InstrumentedTableUpdateListener { Assert.assertion(parentIsRefreshing, "parent.isRefreshing()"); manage(parent); addParentReference(this); - parent.listenForUpdates(this); } @Override @@ -1004,6 +997,41 @@ public void scheduleAt(final long nextRunTime) { } } + /** + * @formatter:off + * Here is how a subscription viewport `grows` over multiple snapshots: + * 1) When a subscription is updated (on creation or after a change to columns or viewport), a new snapshot must + * be created and transmitted to the client. The `growing` snapshot algorithm attempts to keep the UGP + * responsive by creating snapshots that consume no more than a certain percentage of the UGP cycle time. In + * addition, GUI responsiveness is improved by prioritizing viewport subscription client requests over full + * subscription clients. + * NOTE: All updated subscriptions are considered to be `growing` subscriptions even if they can be satisfied in + * a single snapshot. + * 2) When the `BarrageMessageProducer` is ready to provide a new snapshot to an updated subscription, it will + * transfer the `pending` values (viewport rowset and direction, columns) to `target` values which are the + * final goals toward which the viewport grows. In addition, the `growingRemainingViewport` is created which + * will hold all the outstanding rows the client needs to receive in the upcoming snapshot(s). If the updated + * (or new) subscription is a `full` subscription, this viewport is set to range (0, Long.MAX_VALUE). + * 3) If a client has changed viewports, it is possible that the new viewport overlaps with the previous and some + * rows may not need to be requested. This can only happen on the first snapshot after the change, so the + * `isFirstSnapshot` flag is used to add these rows to the viewport on the first snapshot. + * 4) To generate the full rowset for the snapshot, a maximum number of rows to snapshot is computed and the + * subscriptions are processed in a prioritized order, placing viewport above full subscriptions. For each + * subscription (while not exceeding the snapshot maximum number of rows), rows are extracted from the + * `growingRemainingViewport` into `growingIncrementalViewport`. Each subscription can also leverage rows + * already selected for this cycle by previous subscriptions (where the direction of the viewport matches). + * Additionally, the `snapshotViewport` is expanded by the additional rows this client will receive this + * cycle. When a snapshot is successfully created, this `snapshotViewport` will be promoted to the + * `activeViewport` for this subscription. + * 5) When the parent table is smaller than the viewport, it is possible to snapshot all rows in the table before + * exhausting `growingRemainingViewport`. During the snapshot call and while the lock is held, + * `finalizeSnapshotForSubscriptions()` is called which will detect when the subscription is complete and will + * perform some clean up as well as updating the subscription `activeViewport` to match the initially set + * `targetViewport`. When the final snapshot message is sent, the client will see that the `activeViewport` + * matches the requested `targetViewport` and the subscription snapshotting process is complete. + * @formatter:on + */ + private void updateSubscriptionsSnapshotAndPropagate() { lastUpdateTime = scheduler.currentTime().getMillis(); if (DEBUG) { @@ -1027,27 +1055,29 @@ private void updateSubscriptionsSnapshotAndPropagate() { // remove deleted subscriptions while we still hold the lock for (int i = 0; i < activeSubscriptions.size(); ++i) { final Subscription sub = activeSubscriptions.get(i); - if (sub.pendingDelete) { - pendingDeletes = true; - - if (!sub.isViewport()) { - --numFullSubscriptions; - } - if (sub.isGrowingViewport) { - --numGrowingSubscriptions; - } + if (!sub.pendingDelete) { + continue; + } + pendingDeletes = true; - try { - sub.listener.onCompleted(); - } catch (final Exception ignored) { - // ignore races on cancellation - } + if (!sub.isViewport()) { + --numFullSubscriptions; + } + if (sub.isGrowingViewport) { + --numGrowingSubscriptions; + } - // remove this deleted subscription from future consideration - activeSubscriptions.set(i, activeSubscriptions.get(activeSubscriptions.size() - 1)); - activeSubscriptions.remove(activeSubscriptions.size() - 1); - --i; + try { + sub.listener.onCompleted(); + } catch (final Exception ignored) { + // ignore races on cancellation } + + // remove this deleted subscription from future consideration + activeSubscriptions.set(i, activeSubscriptions.get(activeSubscriptions.size() - 1)); + activeSubscriptions.remove(activeSubscriptions.size() - 1); + --i; + } // rebuild the viewports since there are pending changes. This function excludes active subscriptions @@ -1055,42 +1085,45 @@ private void updateSubscriptionsSnapshotAndPropagate() { buildPostSnapshotViewports(true); for (final Subscription subscription : updatedSubscriptions) { - if (!subscription.pendingDelete) { - pendingChanges = true; + if (subscription.pendingDelete) { + continue; + } + pendingChanges = true; - // add this subscription to the "growing" list to handle snapshot creation - if (!subscription.isGrowingViewport) { - subscription.isGrowingViewport = true; - ++numGrowingSubscriptions; - } + // add this subscription to the "growing" list to handle snapshot creation + if (!subscription.isGrowingViewport) { + subscription.isGrowingViewport = true; + ++numGrowingSubscriptions; + } - subscription.hasPendingUpdate = false; - if (!subscription.isActive) { - firstSubscription |= activeSubscriptions.isEmpty(); + subscription.hasPendingUpdate = false; + if (!subscription.isActive) { + firstSubscription |= activeSubscriptions.isEmpty(); - // Note that initial subscriptions have empty viewports and no subscribed columns. - subscription.isActive = true; - activeSubscriptions.add(subscription); - } + // Note that initial subscriptions have empty viewports and no subscribed columns. + subscription.isActive = true; + activeSubscriptions.add(subscription); + } + try (RowSet ignored = subscription.targetViewport) { subscription.targetViewport = subscription.pendingViewport; subscription.pendingViewport = null; + } - subscription.targetColumns = subscription.pendingColumns; - subscription.pendingColumns = null; + subscription.targetColumns = subscription.pendingColumns; + subscription.pendingColumns = null; - subscription.targetReverseViewport = subscription.pendingReverseViewport; + subscription.targetReverseViewport = subscription.pendingReverseViewport; - subscription.isFirstSnapshot = true; + subscription.isFirstSnapshot = true; - // get the set of remaining rows for this subscription - if (subscription.growingRemainingViewport != null) { - subscription.growingRemainingViewport.close(); - } - subscription.growingRemainingViewport = subscription.targetViewport == null - ? RowSetFactory.flat(Long.MAX_VALUE) - : subscription.targetViewport.copy(); + // get the set of remaining rows for this subscription + if (subscription.growingRemainingViewport != null) { + subscription.growingRemainingViewport.close(); } + subscription.growingRemainingViewport = subscription.targetViewport == null + ? RowSetFactory.flat(Long.MAX_VALUE) + : subscription.targetViewport.copy(); } } @@ -1116,6 +1149,7 @@ private void updateSubscriptionsSnapshotAndPropagate() { postSnapshotViewport = activeViewport != null ? activeViewport.copy() : RowSetFactory.empty(); postSnapshotReverseViewport = activeReverseViewport != null ? activeReverseViewport.copy() : RowSetFactory.empty(); + postSnapshotColumns.clear(); postSnapshotColumns.or(activeColumns); } @@ -1227,17 +1261,6 @@ private void updateSubscriptionsSnapshotAndPropagate() { } } - // "grow" the snapshot viewport with rows from the current snapshot - if (viewportValid && subscription.viewport != null) { - // we can use the current viewport as a start - subscription.snapshotViewport.insert(subscription.viewport); - - // the current viewport might contain rows that we don't want (from a previous viewport, e.g.) - if (subscription.targetViewport != null) { - subscription.snapshotViewport.retain(subscription.targetViewport); - } - } - subscription.snapshotViewport.insert(subscription.growingIncrementalViewport); // save the column set @@ -1843,6 +1866,7 @@ private void finalizeSnapshotForSubscriptions(final List subscript subscription.viewport.close(); } subscription.viewport = subscription.targetViewport; + subscription.targetViewport = null; if (subscription.viewport == null) { // track active `full` subscriptions From 34eb31bcfccabd04190075c1f5b47a4a43299477 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Wed, 6 Apr 2022 10:18:19 -0700 Subject: [PATCH 21/47] improved listener visibility in example --- .../client/examples/SubscribeExampleBase.java | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/java-client/barrage-examples/src/main/java/io/deephaven/client/examples/SubscribeExampleBase.java b/java-client/barrage-examples/src/main/java/io/deephaven/client/examples/SubscribeExampleBase.java index 670f5c5ea93..a569a2e5ff3 100644 --- a/java-client/barrage-examples/src/main/java/io/deephaven/client/examples/SubscribeExampleBase.java +++ b/java-client/barrage-examples/src/main/java/io/deephaven/client/examples/SubscribeExampleBase.java @@ -10,16 +10,20 @@ import io.deephaven.client.impl.TableHandleManager; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.table.TableUpdate; +import io.deephaven.engine.table.TableUpdateListener; import io.deephaven.engine.table.impl.InstrumentedTableUpdateListener; import io.deephaven.extensions.barrage.BarrageSubscriptionOptions; import io.deephaven.extensions.barrage.table.BarrageTable; import io.deephaven.qst.TableCreationLogic; +import io.deephaven.util.annotations.ReferentialIntegrity; import picocli.CommandLine; import java.util.concurrent.CountDownLatch; abstract class SubscribeExampleBase extends BarrageClientExampleBase { + TableUpdateListener listener; + @CommandLine.Option(names = {"--tail"}, required = false, description = "Tail viewport size") int tailSize = 0; @@ -64,7 +68,14 @@ protected void execute(final BarrageSession client) throws Exception { final CountDownLatch countDownLatch = new CountDownLatch(1); - InstrumentedTableUpdateListener listener = new InstrumentedTableUpdateListener("example-listener") { + table.listenForUpdates(listener = new InstrumentedTableUpdateListener("example-listener") { + @ReferentialIntegrity + final BarrageTable tableRef = table; + { + // Maintain a liveness ownership relationship with table for the lifetime of the listener + manage(tableRef); + } + @Override protected void onFailureInternal(final Throwable originalException, final Entry sourceEntry) { System.out.println("exiting due to onFailureInternal:"); @@ -77,11 +88,13 @@ public void onUpdate(final TableUpdate upstream) { System.out.println("Received table update:"); System.out.println(upstream); } - }; - - table.listenForUpdates(listener); + }); countDownLatch.await(); + + // For a "real" implementation, we would use liveness tracking for the listener, and ensure that it was + // destroyed and unreachable when we no longer needed it. + listener = null; } } } From 1f04db8fb9a49b73d3a6aa394090a6d26058e259 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Thu, 7 Apr 2022 12:20:05 -0700 Subject: [PATCH 22/47] updated documentation --- .../barrage/BarrageMessageProducer.java | 75 +++++++++++-------- 1 file changed, 43 insertions(+), 32 deletions(-) diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index 947a6aacf78..75809ce63c2 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -422,29 +422,35 @@ public TableDefinition getTableDefinition() { ///////////////////////////////////// /** - * @formatter:off - * Here is the typical lifecycle of a subscription: - * 1) The new subscription is added to pendingSubscriptions. It is not active and its viewport / subscribed - * columns are empty. - * 2) If a subscription is updated before the initial snapshot is prepared, we overwrite the viewport / columns + * Here is the typical lifecycle of a subscription:

+ *
    + *
  1. The new subscription is added to pendingSubscriptions. It is not active and its viewport / subscribed + * columns are empty.
    2. + *
  2. If a subscription is updated before the initial snapshot is prepared, we overwrite the viewport / columns * stored in the variables prefixed with `pending`. These variables will always contain the most recently - * requested viewport / columns that have not yet been acknowledged by the BMP. - * 3) The BMP's update propagation job runs. All pendingSubscriptions (new or updated) will have their pending + * requested viewport / columns that have not yet been acknowledged by the BMP.
    3. + *
  3. The BMP's update propagation job runs. All pendingSubscriptions (new or updated) will have their pending * viewport / columns requests accepted. All pendingSubscriptions move to the activeSubscription list if they * were brand new. The pendingSubscription list is cleared. At this stage, the `pending` variables are nulled * and their contents move to the variables prefixed with `target`. The propagation job is responsible for - * building the snapshot and sending it to the client. When the snapshot is complete, the `target` variables - * are nulled and promoted to `viewport` and `subscribedColumns`. - * 4) If a subscription is updated during or after stage 3, it will be added back to the pendingSubscription list, + * building the snapshot(s) and sending to the client. When each snapshot is complete, the `snapshot` variables + * are flipped to `viewport` and `subscribedColumns`.
    4. + *
  4. While the subscription viewport is growing, it may receive deltas on the rows that have already been + * snapshotted and sent to the client. This ensures consistency is maintained through the propagation process. + * When the client has received the entire contents of the `target` viewport, the growing subscription is + * complete. The `target` variables are promoted to `viewport` and `subscribedColumns` and the subscription is + * removed from the list of growing subscriptions. Only deltas will be sent to this subscriber until a change + * of viewport or columns is requested by the client.
    5. + *
  5. If a subscription is updated during or after stage 3, it will be added back to the pendingSubscription list, * and the updated requests will sit in the `pending` variables until the next time the update propagation job * executes. It will NOT be removed from the activeSubscription list. A given subscription will exist no more - * than once in either subscription list. - * 5) Finally, when a subscription is removed we mark it as having a `pendingDelete` and add it to the + * than once in either subscription list.
    6. + *
  6. Finally, when a subscription is removed we mark it as having a `pendingDelete` and add it to the * pendingSubscription list. Any subscription requests/updates that re-use this handleId will ignore this * instance of Subscription and be allowed to construct a new Subscription starting from step 1. When the * update propagation job is run we clean up deleted subscriptions and rebuild any state that is used to filter - * recorded updates. - * @formatter:on + * recorded updates.
    7. + *
*/ private class Subscription { final BarrageSubscriptionOptions options; @@ -473,7 +479,7 @@ private class Subscription { boolean targetReverseViewport; // the final viewport direction for a changed subscription boolean isGrowingViewport; // is this subscription actively growing - WritableRowSet growingRemainingViewport = null; // rows still needed to satisfy this subscription viewport + WritableRowSet growingRemainingViewport = null; // rows still needed to satisfy this subscription target viewport WritableRowSet growingIncrementalViewport = null; // rows to be sent to the client from the current snapshot boolean isFirstSnapshot; // is this the first snapshot after a change to a subscriptions @@ -998,38 +1004,43 @@ public void scheduleAt(final long nextRunTime) { } /** - * @formatter:off - * Here is how a subscription viewport `grows` over multiple snapshots: - * 1) When a subscription is updated (on creation or after a change to columns or viewport), a new snapshot must - * be created and transmitted to the client. The `growing` snapshot algorithm attempts to keep the UGP - * responsive by creating snapshots that consume no more than a certain percentage of the UGP cycle time. In - * addition, GUI responsiveness is improved by prioritizing viewport subscription client requests over full - * subscription clients. - * NOTE: All updated subscriptions are considered to be `growing` subscriptions even if they can be satisfied in - * a single snapshot. - * 2) When the `BarrageMessageProducer` is ready to provide a new snapshot to an updated subscription, it will + * Handles updates to subscriptions and propagates snapshots and deltas to subscribed clients. Manages `growing` + * viewports, where a subscription receives initial data in one or more snapshots that are assembled client-side + * into the complete dataset. + * + *

Here is how a subscription viewport `grows` over multiple snapshots: + *

    + *
  1. When a subscription is updated (on creation or after a change to columns or viewport), a new snapshot must + * be created and transmitted to the client. The `growing` snapshot algorithm attempts to keep the UGP + * responsive by creating snapshots that consume no more than a certain percentage of the UGP cycle time. In + * addition, GUI responsiveness is improved by prioritizing viewport subscription client requests over full + * subscription clients. + * + *

    NOTE: All subscriptions are initially considered to be `growing` subscriptions even if they can + * be satisfied in a single snapshot.

    2. + *
  2. When the `BarrageMessageProducer` is ready to provide a new snapshot to an updated subscription, it will * transfer the `pending` values (viewport rowset and direction, columns) to `target` values which are the * final goals toward which the viewport grows. In addition, the `growingRemainingViewport` is created which * will hold all the outstanding rows the client needs to receive in the upcoming snapshot(s). If the updated * (or new) subscription is a `full` subscription, this viewport is set to range (0, Long.MAX_VALUE). - * 3) If a client has changed viewports, it is possible that the new viewport overlaps with the previous and some + *
  3. If a client has changed viewports, it is possible that the new viewport overlaps with the previous and some * rows may not need to be requested. This can only happen on the first snapshot after the change, so the - * `isFirstSnapshot` flag is used to add these rows to the viewport on the first snapshot. - * 4) To generate the full rowset for the snapshot, a maximum number of rows to snapshot is computed and the + * `isFirstSnapshot` flag is used to add these rows to the viewport on the first snapshot.
    4. + *
  4. To generate the full rowset for the snapshot, a maximum number of rows to snapshot is computed and the * subscriptions are processed in a prioritized order, placing viewport above full subscriptions. For each * subscription (while not exceeding the snapshot maximum number of rows), rows are extracted from the * `growingRemainingViewport` into `growingIncrementalViewport`. Each subscription can also leverage rows * already selected for this cycle by previous subscriptions (where the direction of the viewport matches). * Additionally, the `snapshotViewport` is expanded by the additional rows this client will receive this * cycle. When a snapshot is successfully created, this `snapshotViewport` will be promoted to the - * `activeViewport` for this subscription. - * 5) When the parent table is smaller than the viewport, it is possible to snapshot all rows in the table before + * `activeViewport` for this subscription.
    5. + *
  5. When the parent table is smaller than the viewport, it is possible to snapshot all rows in the table before * exhausting `growingRemainingViewport`. During the snapshot call and while the lock is held, * `finalizeSnapshotForSubscriptions()` is called which will detect when the subscription is complete and will * perform some clean up as well as updating the subscription `activeViewport` to match the initially set * `targetViewport`. When the final snapshot message is sent, the client will see that the `activeViewport` - * matches the requested `targetViewport` and the subscription snapshotting process is complete. - * @formatter:on + * matches the requested `targetViewport` and the subscription snapshotting process is complete.
    6. + *
*/ private void updateSubscriptionsSnapshotAndPropagate() { From a0fa9ee7a9277ec0f328b4eefd5a74542ae49e3f Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Thu, 7 Apr 2022 13:25:16 -0700 Subject: [PATCH 23/47] spotless apply --- .../barrage/BarrageMessageProducer.java | 147 ++++++++++-------- 1 file changed, 79 insertions(+), 68 deletions(-) diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index 75809ce63c2..366f70a31b6 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -422,34 +422,33 @@ public TableDefinition getTableDefinition() { ///////////////////////////////////// /** - * Here is the typical lifecycle of a subscription:

+ * Here is the typical lifecycle of a subscription: *
    - *
  1. The new subscription is added to pendingSubscriptions. It is not active and its viewport / subscribed - * columns are empty.
    2. - *
  2. If a subscription is updated before the initial snapshot is prepared, we overwrite the viewport / columns - * stored in the variables prefixed with `pending`. These variables will always contain the most recently - * requested viewport / columns that have not yet been acknowledged by the BMP.
    3. - *
  3. The BMP's update propagation job runs. All pendingSubscriptions (new or updated) will have their pending - * viewport / columns requests accepted. All pendingSubscriptions move to the activeSubscription list if they - * were brand new. The pendingSubscription list is cleared. At this stage, the `pending` variables are nulled - * and their contents move to the variables prefixed with `target`. The propagation job is responsible for - * building the snapshot(s) and sending to the client. When each snapshot is complete, the `snapshot` variables - * are flipped to `viewport` and `subscribedColumns`.
    4. - *
  4. While the subscription viewport is growing, it may receive deltas on the rows that have already been - * snapshotted and sent to the client. This ensures consistency is maintained through the propagation process. - * When the client has received the entire contents of the `target` viewport, the growing subscription is - * complete. The `target` variables are promoted to `viewport` and `subscribedColumns` and the subscription is - * removed from the list of growing subscriptions. Only deltas will be sent to this subscriber until a change - * of viewport or columns is requested by the client.
    5. - *
  5. If a subscription is updated during or after stage 3, it will be added back to the pendingSubscription list, - * and the updated requests will sit in the `pending` variables until the next time the update propagation job - * executes. It will NOT be removed from the activeSubscription list. A given subscription will exist no more - * than once in either subscription list.
    6. - *
  6. Finally, when a subscription is removed we mark it as having a `pendingDelete` and add it to the - * pendingSubscription list. Any subscription requests/updates that re-use this handleId will ignore this - * instance of Subscription and be allowed to construct a new Subscription starting from step 1. When the - * update propagation job is run we clean up deleted subscriptions and rebuild any state that is used to filter - * recorded updates.
    7. + *
  7. The new subscription is added to pendingSubscriptions. It is not active and its viewport / subscribed columns + * are empty.
    8. + *
  8. If a subscription is updated before the initial snapshot is prepared, we overwrite the viewport / columns + * stored in the variables prefixed with `pending`. These variables will always contain the most recently requested + * viewport / columns that have not yet been acknowledged by the BMP.
    9. + *
  9. The BMP's update propagation job runs. All pendingSubscriptions (new or updated) will have their pending + * viewport / columns requests accepted. All pendingSubscriptions move to the activeSubscription list if they were + * brand new. The pendingSubscription list is cleared. At this stage, the `pending` variables are nulled and their + * contents move to the variables prefixed with `target`. The propagation job is responsible for building the + * snapshot(s) and sending to the client. When each snapshot is complete, the `snapshot` variables are flipped to + * `viewport` and `subscribedColumns`.
    10. + *
  10. While the subscription viewport is growing, it may receive deltas on the rows that have already been + * snapshotted and sent to the client. This ensures consistency is maintained through the propagation process. When + * the client has received the entire contents of the `target` viewport, the growing subscription is complete. The + * `target` variables are promoted to `viewport` and `subscribedColumns` and the subscription is removed from the + * list of growing subscriptions. Only deltas will be sent to this subscriber until a change of viewport or columns + * is requested by the client.
    11. + *
  11. If a subscription is updated during or after stage 3, it will be added back to the pendingSubscription list, + * and the updated requests will sit in the `pending` variables until the next time the update propagation job + * executes. It will NOT be removed from the activeSubscription list. A given subscription will exist no more than + * once in either subscription list.
    12. + *
  12. Finally, when a subscription is removed we mark it as having a `pendingDelete` and add it to the + * pendingSubscription list. Any subscription requests/updates that re-use this handleId will ignore this instance + * of Subscription and be allowed to construct a new Subscription starting from step 1. When the update propagation + * job is run we clean up deleted subscriptions and rebuild any state that is used to filter recorded updates.
    13. *
*/ private class Subscription { @@ -479,7 +478,8 @@ private class Subscription { boolean targetReverseViewport; // the final viewport direction for a changed subscription boolean isGrowingViewport; // is this subscription actively growing - WritableRowSet growingRemainingViewport = null; // rows still needed to satisfy this subscription target viewport + WritableRowSet growingRemainingViewport = null; // rows still needed to satisfy this subscription target + // viewport WritableRowSet growingIncrementalViewport = null; // rows to be sent to the client from the current snapshot boolean isFirstSnapshot; // is this the first snapshot after a change to a subscriptions @@ -1008,38 +1008,39 @@ public void scheduleAt(final long nextRunTime) { * viewports, where a subscription receives initial data in one or more snapshots that are assembled client-side * into the complete dataset. * - *

Here is how a subscription viewport `grows` over multiple snapshots: + *

+ * Here is how a subscription viewport `grows` over multiple snapshots: *

    - *
  1. When a subscription is updated (on creation or after a change to columns or viewport), a new snapshot must - * be created and transmitted to the client. The `growing` snapshot algorithm attempts to keep the UGP - * responsive by creating snapshots that consume no more than a certain percentage of the UGP cycle time. In - * addition, GUI responsiveness is improved by prioritizing viewport subscription client requests over full - * subscription clients. + *
  2. When a subscription is updated (on creation or after a change to columns or viewport), a new snapshot must be + * created and transmitted to the client. The `growing` snapshot algorithm attempts to keep the UGP responsive by + * creating snapshots that consume no more than a certain percentage of the UGP cycle time. In addition, GUI + * responsiveness is improved by prioritizing viewport subscription client requests over full subscription clients. * - *

    NOTE: All subscriptions are initially considered to be `growing` subscriptions even if they can - * be satisfied in a single snapshot.

    3. - *
  3. When the `BarrageMessageProducer` is ready to provide a new snapshot to an updated subscription, it will - * transfer the `pending` values (viewport rowset and direction, columns) to `target` values which are the - * final goals toward which the viewport grows. In addition, the `growingRemainingViewport` is created which - * will hold all the outstanding rows the client needs to receive in the upcoming snapshot(s). If the updated - * (or new) subscription is a `full` subscription, this viewport is set to range (0, Long.MAX_VALUE). - *
  4. If a client has changed viewports, it is possible that the new viewport overlaps with the previous and some - * rows may not need to be requested. This can only happen on the first snapshot after the change, so the - * `isFirstSnapshot` flag is used to add these rows to the viewport on the first snapshot.
    5. - *
  5. To generate the full rowset for the snapshot, a maximum number of rows to snapshot is computed and the - * subscriptions are processed in a prioritized order, placing viewport above full subscriptions. For each - * subscription (while not exceeding the snapshot maximum number of rows), rows are extracted from the - * `growingRemainingViewport` into `growingIncrementalViewport`. Each subscription can also leverage rows - * already selected for this cycle by previous subscriptions (where the direction of the viewport matches). - * Additionally, the `snapshotViewport` is expanded by the additional rows this client will receive this - * cycle. When a snapshot is successfully created, this `snapshotViewport` will be promoted to the - * `activeViewport` for this subscription.
    6. - *
  6. When the parent table is smaller than the viewport, it is possible to snapshot all rows in the table before - * exhausting `growingRemainingViewport`. During the snapshot call and while the lock is held, - * `finalizeSnapshotForSubscriptions()` is called which will detect when the subscription is complete and will - * perform some clean up as well as updating the subscription `activeViewport` to match the initially set - * `targetViewport`. When the final snapshot message is sent, the client will see that the `activeViewport` - * matches the requested `targetViewport` and the subscription snapshotting process is complete.
    7. + *

    + * NOTE: All subscriptions are initially considered to be `growing` subscriptions even if they can be + * satisfied in a single snapshot. + *

  7. When the `BarrageMessageProducer` is ready to provide a new snapshot to an updated subscription, it will + * transfer the `pending` values (viewport rowset and direction, columns) to `target` values which are the final + * goals toward which the viewport grows. In addition, the `growingRemainingViewport` is created which will hold all + * the outstanding rows the client needs to receive in the upcoming snapshot(s). If the updated (or new) + * subscription is a `full` subscription, this viewport is set to range (0, Long.MAX_VALUE). + *
  8. If a client has changed viewports, it is possible that the new viewport overlaps with the previous and some + * rows may not need to be requested. This can only happen on the first snapshot after the change, so the + * `isFirstSnapshot` flag is used to add these rows to the viewport on the first snapshot.
    9. + *
  9. To generate the full rowset for the snapshot, a maximum number of rows to snapshot is computed and the + * subscriptions are processed in a prioritized order, placing viewport above full subscriptions. For each + * subscription (while not exceeding the snapshot maximum number of rows), rows are extracted from the + * `growingRemainingViewport` into `growingIncrementalViewport`. Each subscription can also leverage rows already + * selected for this cycle by previous subscriptions (where the direction of the viewport matches). Additionally, + * the `snapshotViewport` is expanded by the additional rows this client will receive this cycle. When a snapshot is + * successfully created, this `snapshotViewport` will be promoted to the `activeViewport` for this + * subscription.
    10. + *
  10. When the parent table is smaller than the viewport, it is possible to snapshot all rows in the table before + * exhausting `growingRemainingViewport`. During the snapshot call and while the lock is held, + * `finalizeSnapshotForSubscriptions()` is called which will detect when the subscription is complete and will + * perform some clean up as well as updating the subscription `activeViewport` to match the initially set + * `targetViewport`. When the final snapshot message is sent, the client will see that the `activeViewport` matches + * the requested `targetViewport` and the subscription snapshotting process is complete.
    11. *
*/ @@ -1050,9 +1051,10 @@ private void updateSubscriptionsSnapshotAndPropagate() { } boolean firstSubscription = false; - boolean pendingDeletes = false; boolean pendingChanges = false; + List deletedSubscriptions = null; + // check for pending changes (under the lock) synchronized (this) { List updatedSubscriptions = null; @@ -1069,7 +1071,12 @@ private void updateSubscriptionsSnapshotAndPropagate() { if (!sub.pendingDelete) { continue; } - pendingDeletes = true; + + // save this for later deletion + if (deletedSubscriptions == null) { + deletedSubscriptions = new ArrayList<>(); + } + deletedSubscriptions.add(sub); if (!sub.isViewport()) { --numFullSubscriptions; @@ -1078,12 +1085,6 @@ private void updateSubscriptionsSnapshotAndPropagate() { --numGrowingSubscriptions; } - try { - sub.listener.onCompleted(); - } catch (final Exception ignored) { - // ignore races on cancellation - } - // remove this deleted subscription from future consideration activeSubscriptions.set(i, activeSubscriptions.get(activeSubscriptions.size() - 1)); activeSubscriptions.remove(activeSubscriptions.size() - 1); @@ -1138,7 +1139,7 @@ private void updateSubscriptionsSnapshotAndPropagate() { } } - if (pendingDeletes && !pendingChanges) { + if (deletedSubscriptions != null && !pendingChanges) { // we have only removed subscriptions; we can update this state immediately. promoteSnapshotToActive(); } @@ -1358,7 +1359,7 @@ private void updateSubscriptionsSnapshotAndPropagate() { // cleanup for next iteration clearObjectDeltaColumns(objectColumnsToClear); - if (pendingDeletes || pendingChanges) { + if (deletedSubscriptions != null || pendingChanges) { objectColumnsToClear.clear(); objectColumnsToClear.or(objectColumns); objectColumnsToClear.and(activeColumns); @@ -1393,6 +1394,16 @@ private void updateSubscriptionsSnapshotAndPropagate() { propagateToSubscribers(postSnapshot, propagationRowSet); } + if (deletedSubscriptions != null) { + for (final Subscription subscription : deletedSubscriptions) { + try { + subscription.listener.onCompleted(); + } catch (final Exception ignored) { + // ignore races on cancellation + } + } + } + // propagate any error notifying listeners there are no more updates incoming if (pendingError != null) { for (final Subscription subscription : activeSubscriptions) { From 02cb7b81cf29652cd6b8a4a91f8645c9c34aa3ba Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Tue, 19 Apr 2022 08:15:49 -0700 Subject: [PATCH 24/47] initial push --- .../java/io/deephaven/chunk/ChunkList.java | 88 ++++++++ .../table/impl/remote/ConstructSnapshot.java | 52 ++++- .../table/impl/util/BarrageMessage.java | 17 +- .../VarBinaryChunkInputStreamGenerator.java | 207 ++++++++++++++---- .../barrage/table/BarrageTable.java | 173 +++++++++++---- .../barrage/util/BarrageStreamReader.java | 137 +++++++++++- .../server/arrow/ArrowFlightUtil.java | 14 +- .../barrage/BarrageMessageProducer.java | 49 +++-- .../barrage/BarrageStreamGenerator.java | 203 ++++++++++++++--- 9 files changed, 778 insertions(+), 162 deletions(-) create mode 100644 engine/chunk/src/main/java/io/deephaven/chunk/ChunkList.java diff --git a/engine/chunk/src/main/java/io/deephaven/chunk/ChunkList.java b/engine/chunk/src/main/java/io/deephaven/chunk/ChunkList.java new file mode 100644 index 00000000000..66491b726e2 --- /dev/null +++ b/engine/chunk/src/main/java/io/deephaven/chunk/ChunkList.java @@ -0,0 +1,88 @@ +package io.deephaven.chunk; + +import io.deephaven.chunk.attributes.Values; +import io.deephaven.chunk.util.pools.PoolableChunk; +import io.deephaven.util.SafeCloseable; + +import java.util.ArrayList; + +public class ChunkList implements SafeCloseable { + // rows in position space for each chunk + public final ArrayList startIndex = new ArrayList<>(); + public final ArrayList endIndex = new ArrayList<>(); + public final ArrayList> chunks = new ArrayList<>(); + + public void addChunk(Chunk nextChunk, long start, long end) { + startIndex.add(start); + endIndex.add(end); + chunks.add(nextChunk); + } + + public void addChunk(Chunk nextChunk) { + final long start; + final long end; + + if (endIndex.isEmpty()) { + start = 0L; + end = (long)nextChunk.size() - 1; + } else { + start = lastEndIndex() + 1; + end = start + (long)nextChunk.size() - 1; + } + + addChunk(nextChunk, start, end); + } + + public int size() { + return startIndex.size(); + } + + public Long chunkSize(int ci) { + if (ci < startIndex.size() && ci < endIndex.size()) { + return endIndex.get(ci) - startIndex.get(ci); + } + return null; + } + + public Chunk firstChunk() { + if (chunks.size() == 0) { + return null; + } + return chunks.get(0); + } + + public Chunk lastChunk() { + if (chunks.size() == 0) { + return null; + } + return chunks.get(chunks.size() -1); + } + + public Long lastStartIndex() { + if (startIndex.size() == 0) { + return null; + } + return startIndex.get(startIndex.size() -1); + } + + public Long lastEndIndex() { + if (endIndex.size() == 0) { + return null; + } + return endIndex.get(endIndex.size() -1); + } + + public Long lastChunkSize() { + return chunkSize(endIndex.size() -1); + } + + + @Override + public void close() { + for (Chunk chunk : chunks){ + if (chunk instanceof PoolableChunk) { + ((PoolableChunk) chunk).close(); + } + } + } +} \ No newline at end of file diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java index c1b3e82e1b7..54640ad6668 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java @@ -45,6 +45,8 @@ import io.deephaven.chunk.attributes.Values; +import static io.deephaven.engine.table.impl.sources.InMemoryColumnSource.TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD; + /** * A Set of static utilities for computing values from a table while avoiding the use of the UGP lock. This class * supports snapshots in both position space and key space. @@ -1316,7 +1318,7 @@ public static boolean serializeAllTable(final boolean usePrev, snapshot.rowsIncluded = snapshot.rowsAdded.copy(); } - LongSizedDataStructure.intSize("construct snapshot", snapshot.rowsIncluded.size()); + //LongSizedDataStructure.intSize("construct snapshot", snapshot.rowsIncluded.size()); final Map sourceMap = table.getColumnSourceMap(); final String[] columnSources = sourceMap.keySet().toArray(CollectionUtil.ZERO_LENGTH_STRING_ARRAY); @@ -1340,14 +1342,14 @@ public static boolean serializeAllTable(final boolean usePrev, final RowSet rows = columnIsEmpty ? RowSetFactory.empty() : snapshot.rowsIncluded; // Note: cannot use shared context across several calls of differing lengths and no sharing necessary // when empty - acd.data = getSnapshotDataAsChunk(columnSource, columnIsEmpty ? null : sharedContext, rows, usePrev); + acd.data = getSnapshotDataAsChunkSet(columnSource, columnIsEmpty ? null : sharedContext, rows, usePrev); acd.type = columnSource.getType(); acd.componentType = columnSource.getComponentType(); final BarrageMessage.ModColumnData mcd = new BarrageMessage.ModColumnData(); snapshot.modColumnData[ii] = mcd; mcd.rowsModified = RowSetFactory.empty(); - mcd.data = getSnapshotDataAsChunk(columnSource, null, RowSetFactory.empty(), usePrev); + mcd.data = getSnapshotDataAsChunkSet(columnSource, null, RowSetFactory.empty(), usePrev); mcd.type = acd.type; mcd.componentType = acd.componentType; } @@ -1430,6 +1432,50 @@ private static WritableChunk getSnapshotDataAsChunk(final ColumnSour } } + private static ChunkList getSnapshotDataAsChunkSet(final ColumnSource columnSource, + final SharedContext sharedContext, final RowSet rowSet, final boolean usePrev) { + final ColumnSource sourceToUse = ReinterpretUtils.maybeConvertToPrimitive(columnSource); + long offset = 0; + final long size = rowSet.size(); + final ChunkList result = new ChunkList(); + + while (offset < rowSet.size()) { + final int chunkSize; + if (size - offset > TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD) { + chunkSize = TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD; + } else { + chunkSize = (int)(size - offset); + } + + final long keyStart = rowSet.get(offset); + final long keyEnd = rowSet.get(offset + chunkSize - 1); + + try (final ColumnSource.FillContext context = sharedContext != null + ? sourceToUse.makeFillContext(chunkSize, sharedContext) + : sourceToUse.makeFillContext(chunkSize); + final WritableRowSet reducedRowSet = rowSet.subSetByKeyRange(keyStart, keyEnd)) { + + final ChunkType chunkType = sourceToUse.getChunkType(); + + // create a new chunk + WritableChunk currentChunk = chunkType.makeWritableChunk(chunkSize); + + if (usePrev) { + sourceToUse.fillPrevChunk(context, currentChunk, reducedRowSet); + } else { + sourceToUse.fillChunk(context, currentChunk, reducedRowSet); + } + + // add the chunk to the current list + result.addChunk(currentChunk, offset, offset + currentChunk.size() - 1); + + // increment the offset for the next chunk (using the actual values written) + offset += currentChunk.size(); + } + } + return result; + } + /** * Estimate the size of a complete table snapshot in bytes. * diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/util/BarrageMessage.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/util/BarrageMessage.java index c2b9ff4be1a..3acc106d792 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/util/BarrageMessage.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/util/BarrageMessage.java @@ -5,12 +5,16 @@ package io.deephaven.engine.table.impl.util; import io.deephaven.chunk.Chunk; +import io.deephaven.chunk.ChunkList; import io.deephaven.chunk.attributes.Values; import io.deephaven.chunk.util.pools.PoolableChunk; import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.rowset.RowSetShiftData; +import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.util.SafeCloseable; +import java.util.ArrayList; import java.util.BitSet; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; @@ -28,13 +32,13 @@ public static class ModColumnData { public RowSet rowsModified; public Class type; public Class componentType; - public Chunk data; + public ChunkList data; } public static class AddColumnData { public Class type; public Class componentType; - public Chunk data; + public ChunkList data; } public long firstSeq = -1; @@ -94,9 +98,7 @@ public void close() { continue; } - if (acd.data instanceof PoolableChunk) { - ((PoolableChunk) acd.data).close(); - } + acd.data.close(); } } if (modColumnData != null) { @@ -108,9 +110,8 @@ public void close() { if (mcd.rowsModified != null) { mcd.rowsModified.close(); } - if (mcd.data instanceof PoolableChunk) { - ((PoolableChunk) mcd.data).close(); - } + + mcd.data.close(); } } } diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java index 4650e1652db..e22955ee515 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java @@ -12,7 +12,6 @@ import io.deephaven.chunk.attributes.Values; import io.deephaven.extensions.barrage.util.StreamReaderOptions; import io.deephaven.util.datastructures.LongSizedDataStructure; -import io.deephaven.chunk.IntChunk; import io.deephaven.chunk.ObjectChunk; import io.deephaven.chunk.WritableIntChunk; import io.deephaven.chunk.WritableLongChunk; @@ -27,6 +26,7 @@ import java.io.DataInput; import java.io.IOException; import java.io.OutputStream; +import java.util.ArrayList; import java.util.Iterator; public class VarBinaryChunkInputStreamGenerator extends BaseChunkInputStreamGenerator> { @@ -34,8 +34,58 @@ public class VarBinaryChunkInputStreamGenerator extends BaseChunkInputStreamG private final Appender appendItem; - private byte[] bytes; - private WritableIntChunk offsets; + public static class ByteStorage { + + private final WritableIntChunk offsets; + private final ArrayList byteArrays; + private final ArrayList byteArraySizes; + private final ArrayList byteArrayStartIndex; + + public ByteStorage(int size) { + offsets = WritableIntChunk.makeWritableChunk(size); + + byteArrays = new ArrayList<>(); + byteArraySizes = new ArrayList<>(); + byteArrayStartIndex = new ArrayList<>(); + } + + public int size() { + return byteArrays.size(); + } + + void addByteArray(byte[] arr, int startIndex, int size) { + byteArrays.add(arr); + byteArrayStartIndex.add(startIndex); + byteArraySizes.add(size); + } + + public boolean isEmpty() { + return byteArrays.isEmpty(); + } + + public Integer getByteArrayIndex(int pos) { + // optimize for most common case + if (byteArrays.size() == 1) { + return 0; + } + for (int i = byteArrayStartIndex.size() - 1; i > 0; --i) { + if (byteArrayStartIndex.get(i) <= pos) { + return i; + } + } + return 0; + } + + public byte[] getByteArray(int arrayIdx) { + return byteArrays.get(arrayIdx); + } + + public int getByteArraySize(int arrayIdx) { + return byteArraySizes.get(arrayIdx); + } + }; + + private ByteStorage byteStorage = null; public interface Appender { void append(OutputStream out, T item) throws IOException; @@ -52,24 +102,43 @@ public interface Mapper { } private synchronized void computePayload() throws IOException { - if (bytes != null) { + if (byteStorage != null) { return; } + byteStorage = new ByteStorage(chunk.size() == 0 ? 0 : (chunk.size() + 1)); - offsets = WritableIntChunk.makeWritableChunk(chunk.size() == 0 ? 0 : (chunk.size() + 1)); + BarrageProtoUtil.ExposedByteArrayOutputStream baos = new BarrageProtoUtil.ExposedByteArrayOutputStream(); + if (chunk.size() > 0) { + byteStorage.offsets.set(0, 0); + } + int baosSize = 0; + int startIndex = 0; - try (final BarrageProtoUtil.ExposedByteArrayOutputStream baos = new BarrageProtoUtil.ExposedByteArrayOutputStream()) { - if (chunk.size() > 0) { - offsets.set(0, 0); - } - for (int i = 0; i < chunk.size(); ++i) { - if (chunk.get(i) != null) { + for (int i = 0; i < chunk.size(); ++i) { + if (chunk.get(i) != null) { + try { + appendItem.append(baos, chunk.get(i)); + baosSize = baos.size(); + } catch (OutOfMemoryError ex) { + // we overran the buffer on this item, the output stream probably has junk from the failed write + // so we use the stored output stream size instead of querying + byteStorage.addByteArray(baos.peekBuffer(), startIndex, baosSize); + + // close the old output stream and create a new one + baos.close(); + baos = new BarrageProtoUtil.ExposedByteArrayOutputStream(); + + // add the item to this buffer appendItem.append(baos, chunk.get(i)); + baosSize = baos.size(); + startIndex = i; + byteStorage.offsets.set(i, 0); } - offsets.set(i + 1, baos.size()); + byteStorage.offsets.set(i + 1, baosSize); } - bytes = baos.peekBuffer(); } + byteStorage.addByteArray(baos.peekBuffer(), startIndex, baosSize); + baos.close(); } @Override @@ -78,8 +147,8 @@ public void close() { if (chunk instanceof PoolableChunk) { ((PoolableChunk) chunk).close(); } - if (offsets != null) { - offsets.close(); + if (byteStorage != null) { + byteStorage.offsets.close(); } } } @@ -87,21 +156,18 @@ public void close() { @Override public DrainableColumn getInputStream(final StreamReaderOptions options, final @Nullable RowSet subset) throws IOException { computePayload(); - return new ObjectChunkInputStream(options, offsets, bytes, subset); + return new ObjectChunkInputStream(options, byteStorage, subset); } private class ObjectChunkInputStream extends BaseChunkInputStream { private int cachedSize = -1; - private final byte[] myBytes; - private final IntChunk myOffsets; + private final ByteStorage myByteStorage; private ObjectChunkInputStream( final StreamReaderOptions options, - final IntChunk myOffsets, - final byte[] myBytes, final RowSet subset) { + final ByteStorage myByteStorage, final RowSet subset) { super(chunk, options, subset); - this.myBytes = myBytes; - this.myOffsets = myOffsets; + this.myByteStorage = myByteStorage; } private int cachedNullCount = -1; @@ -141,9 +207,23 @@ public void visitBuffers(final BufferListener listener) { // payload final MutableLong numPayloadBytes = new MutableLong(); subset.forAllRowKeyRanges((s, e) -> { - // account for payload - numPayloadBytes.add(myOffsets.get(LongSizedDataStructure.intSize(DEBUG_NAME, e + 1))); - numPayloadBytes.subtract(myOffsets.get(LongSizedDataStructure.intSize(DEBUG_NAME, s))); + // account for payload, we have already int-size verified all rows in the RowSet + final int startArrayIndex = myByteStorage.getByteArrayIndex((int)s); + final int startOffset = myByteStorage.offsets.get((int) s); + + final int endArrayIndex = myByteStorage.getByteArrayIndex((int)e); + final int endOffset = myByteStorage.offsets.get((int) e + 1); + + if (startArrayIndex == endArrayIndex) { // same byte array, can optimize + numPayloadBytes.add(endOffset - startOffset); + } else { + // need to span multiple byte arrays + numPayloadBytes.add(myByteStorage.getByteArraySize(startArrayIndex) - startOffset); + for (int midArrayIndex = startArrayIndex + 1; midArrayIndex < endArrayIndex; midArrayIndex++) { + numPayloadBytes.add(myByteStorage.getByteArraySize(midArrayIndex)); + } + numPayloadBytes.add(endOffset); + } }); final long payloadExtended = numPayloadBytes.longValue() & REMAINDER_MOD_8_MASK; if (payloadExtended > 0) { @@ -161,17 +241,34 @@ protected int getRawSize() { } // there are n+1 offsets; it is not assumed first offset is zero - if (!subset.isEmpty() && subset.size() == myOffsets.size() - 1) { - cachedSize += myOffsets.size() * Integer.BYTES; - cachedSize += myOffsets.get(subset.intSize(DEBUG_NAME)) - myOffsets.get(0); + if (!subset.isEmpty() && subset.size() == myByteStorage.offsets.size() - 1) { + cachedSize += myByteStorage.offsets.size() * Integer.BYTES; + for (int i = 0; i < myByteStorage.size(); i++) { + cachedSize += myByteStorage.getByteArraySize(i); + } } else { cachedSize += subset.isEmpty() ? 0 : Integer.BYTES; // account for the n+1 offset subset.forAllRowKeyRanges((s, e) -> { // account for offsets cachedSize += (e - s + 1) * Integer.BYTES; + // account for payload - cachedSize += myOffsets.get(LongSizedDataStructure.intSize(DEBUG_NAME, e + 1)); - cachedSize -= myOffsets.get(LongSizedDataStructure.intSize(DEBUG_NAME, s)); + final int startArrayIndex = myByteStorage.getByteArrayIndex((int)s); + final int startOffset = myByteStorage.offsets.get((int) s); + + final int endArrayIndex = myByteStorage.getByteArrayIndex((int)e); + final int endOffset = myByteStorage.offsets.get((int) e + 1); + + if (startArrayIndex == endArrayIndex) { // same byte array, can optimize + cachedSize += endOffset - startOffset; + } else { + // need to span multiple byte arrays + cachedSize += (myByteStorage.getByteArraySize(startArrayIndex) - startOffset); + for (int midArrayIndex = startArrayIndex + 1; midArrayIndex < endArrayIndex; midArrayIndex++) { + cachedSize += myByteStorage.getByteArraySize(midArrayIndex); + } + cachedSize += endOffset; + } }); } @@ -223,11 +320,24 @@ public int drainTo(final OutputStream outputStream) throws IOException { dos.writeInt(0); final MutableInt logicalSize = new MutableInt(); - subset.forAllRowKeys((rawRow) -> { + subset.forAllRowKeys((idx) -> { try { - final int rowEnd = LongSizedDataStructure.intSize(DEBUG_NAME, rawRow + 1); - final int size = myOffsets.get(rowEnd) - myOffsets.get(rowEnd - 1); - logicalSize.add(size); + final int startArrayIndex = myByteStorage.getByteArrayIndex((int)idx); + final int startOffset = myByteStorage.offsets.get((int)idx); + + // is this the last row in the chunk? + if (idx == chunk.size() - 1) { + logicalSize.add(myByteStorage.getByteArraySize(startArrayIndex) - startOffset); + } else { + final int endArrayIndex = myByteStorage.getByteArrayIndex((int)idx + 1); + final int endOffset = myByteStorage.offsets.get((int) idx + 1); + + if (startArrayIndex == endArrayIndex) { // same byte array, can optimize + logicalSize.add(endOffset - startOffset); + } else { + logicalSize.add(myByteStorage.getByteArraySize(startArrayIndex) - startOffset); + } + } dos.writeInt(logicalSize.intValue()); } catch (final IOException e) { throw new UncheckedDeephavenException("couldn't drain data to OutputStream", e); @@ -244,11 +354,30 @@ public int drainTo(final OutputStream outputStream) throws IOException { final MutableLong payloadLen = new MutableLong(); subset.forAllRowKeyRanges((s, e) -> { try { - // we have already int-size verified all rows in the RowSet - final int startOffset = myOffsets.get((int) s); - final int endOffset = myOffsets.get((int) e + 1); - dos.write(myBytes, startOffset, endOffset - startOffset); - payloadLen.add(endOffset - startOffset); + final int startArrayIndex = myByteStorage.getByteArrayIndex((int)s); + final int startOffset = myByteStorage.offsets.get((int) s); + + final int endArrayIndex = myByteStorage.getByteArrayIndex((int)e); + final int endOffset = myByteStorage.offsets.get((int) e + 1); + + if (startArrayIndex == endArrayIndex) { // same byte array, can optimize + dos.write(myByteStorage.byteArrays.get(startArrayIndex), startOffset, endOffset - startOffset); + payloadLen.add(endOffset - startOffset); + } else { + // need to span multiple byte arrays + int firstSize = myByteStorage.byteArraySizes.get(startArrayIndex) - startOffset; + dos.write(myByteStorage.byteArrays.get(startArrayIndex), startOffset, firstSize); + payloadLen.add(firstSize); + + for (int midArrayIndex = startArrayIndex + 1; midArrayIndex < endArrayIndex; midArrayIndex++) { + int midSize = myByteStorage.getByteArraySize(midArrayIndex); + dos.write(myByteStorage.byteArrays.get(midArrayIndex), 0, midSize); + payloadLen.add(midSize); + } + + dos.write(myByteStorage.byteArrays.get(endArrayIndex), 0, endOffset); + payloadLen.add(endOffset); + } } catch (final IOException err) { throw new UncheckedDeephavenException("couldn't drain data to OutputStream", err); } diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java index dbeb91f8f58..9509b600d02 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java @@ -9,6 +9,7 @@ import gnu.trove.list.linked.TLongLinkedList; import io.deephaven.base.verify.Assert; import io.deephaven.chunk.attributes.Values; +import io.deephaven.chunk.util.pools.ChunkPoolConstants; import io.deephaven.configuration.Configuration; import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.table.*; @@ -39,6 +40,8 @@ import java.util.*; import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import static io.deephaven.engine.table.impl.sources.InMemoryColumnSource.TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD; + /** * A client side {@link Table} that mirrors an upstream/server side {@code Table}. * @@ -47,7 +50,7 @@ public class BarrageTable extends QueryTable implements BarrageMessage.Listener, Runnable { public static final boolean DEBUG_ENABLED = - Configuration.getInstance().getBooleanWithDefault("BarrageTable.debug", false); + Configuration.getInstance().getBooleanWithDefault("BarrageTable.debug", true); private static final Logger log = LoggerFactory.getLogger(BarrageTable.class); @@ -57,7 +60,7 @@ public class BarrageTable extends QueryTable implements BarrageMessage.Listener, private final PerformanceEntry refreshEntry; /** the capacity that the destSources been set to */ - private int capacity = 0; + private long capacity = 0; /** the reinterpreted destination writable sources */ private final WritableColumnSource[] destSources; /** we compact the parent table's key-space and instead redirect; ideal for viewport */ @@ -281,27 +284,71 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC totalMods.insert(column.rowsModified); } + int maxChunkSize = 1 << ChunkPoolConstants.LARGEST_POOLED_CHUNK_LOG2_CAPACITY; + if (update.rowsIncluded.isNonempty()) { - try (final ChunkSink.FillFromContext redirContext = - rowRedirection.makeFillFromContext(update.rowsIncluded.intSize()); - final RowSet destinationRowSet = getFreeRows(update.rowsIncluded.size())) { - // Update redirection mapping: - rowRedirection.fillFromChunk(redirContext, destinationRowSet.asRowKeyChunk(), - update.rowsIncluded); - - // Update data chunk-wise: - for (int ii = 0; ii < update.addColumnData.length; ++ii) { - if (isSubscribedColumn(ii)) { - final Chunk data = update.addColumnData[ii].data; - Assert.eq(data.size(), "delta.includedAdditions.size()", destinationRowSet.size(), - "destinationRowSet.size()"); - try (final ChunkSink.FillFromContext ctxt = - destSources[ii].makeFillFromContext(destinationRowSet.intSize())) { - destSources[ii].fillFromChunk(ctxt, data, destinationRowSet); + long start = System.nanoTime(); + long elapsed; + + if (mightBeInitialSnapshot) { + // ensure the data sources have at least the incoming capacity. The sources will auto-resize but + // we know the initial snapshot size and can optimize + capacity = update.rowsIncluded.size(); + for (final WritableColumnSource source : destSources) { + source.ensureCapacity(capacity); + } + freeset.insertRange(0, capacity - 1); + } + + elapsed = System.nanoTime() - start; start = System.nanoTime(); + System.out.println("BT ensureCapacity (ms): " + (double)elapsed / 1_000_000.0); + + // this will hold all the free rows allocated for the included rows + final WritableRowSet destinationRowSet = RowSetFactory.empty(); + + // update the rowRedirection with the rowsIncluded set + try (final RowSequence.Iterator rowsIncludedIterator = update.rowsIncluded.getRowSequenceIterator()) { + while (rowsIncludedIterator.hasMore()) { + final RowSequence chunkRowsToFree = + rowsIncludedIterator.getNextRowSequenceWithLength(maxChunkSize); + try (final ChunkSink.FillFromContext redirContext = + rowRedirection.makeFillFromContext(chunkRowsToFree.intSize()); + final RowSet newRows = getFreeRows(chunkRowsToFree.intSize());) { + + // Update redirection mapping: + rowRedirection.fillFromChunk(redirContext, newRows.asRowKeyChunk(), chunkRowsToFree); + + // add these rows to the final destination set + destinationRowSet.insert(newRows); + } + } + } + + elapsed = System.nanoTime() - start; start = System.nanoTime(); + System.out.println("BT rowRedirection (ms): " + (double)elapsed / 1_000_000.0); + + // update the column sources + for (int ii = 0; ii < update.addColumnData.length; ++ii) { + if (isSubscribedColumn(ii)) { + final BarrageMessage.AddColumnData column = update.addColumnData[ii]; + // grab the matching rows from each chunk + for (int chunkIndex = 0; chunkIndex < column.data.size(); ++chunkIndex) { + try (final RowSet chunkRows + = RowSetFactory.fromRange( + column.data.startIndex.get(chunkIndex), + column.data.endIndex.get(chunkIndex)); + final RowSet chunkDestSet = destinationRowSet.subSetForPositions(chunkRows); + final ChunkSink.FillFromContext ctxt = + destSources[ii].makeFillFromContext(chunkDestSet.intSize())) { + final Chunk chunkData = column.data.chunks.get(chunkIndex); + destSources[ii].fillFromChunk(ctxt, chunkData, chunkDestSet); } } } } + + elapsed = System.nanoTime() - start; start = System.nanoTime(); + System.out.println("BT fill column data (ms): " + (double)elapsed / 1_000_000.0); } modifiedColumnSet.clear(); @@ -313,20 +360,50 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC modifiedColumnSet.setColumnWithIndex(ii); - try (final ChunkSource.FillContext redirContext = - rowRedirection.makeFillContext(column.rowsModified.intSize(), null); - final WritableLongChunk keys = - WritableLongChunk.makeWritableChunk(column.rowsModified.intSize())) { - rowRedirection.fillChunk(redirContext, keys, column.rowsModified); - for (int i = 0; i < keys.size(); ++i) { - Assert.notEquals(keys.get(i), "keys[i]", RowSequence.NULL_ROW_KEY, "RowSet.NULL_ROW_KEY"); - } + // grab the matching rows from each chunk + for (int chunkIndex = 0; chunkIndex < column.data.size(); ++chunkIndex) { + try (final RowSet chunkRows + = RowSetFactory.fromRange( + column.data.startIndex.get(chunkIndex), + column.data.endIndex.get(chunkIndex))) { + final Chunk chunkData = column.data.chunks.get(chunkIndex); + + try (final ChunkSource.FillContext redirContext = + rowRedirection.makeFillContext(chunkRows.intSize(), null); + final WritableLongChunk keys = + WritableLongChunk.makeWritableChunk(chunkRows.intSize()); + final RowSet chunkKeys = column.rowsModified.subSetForPositions(chunkRows)) { + + // fill the key chunk with the keys from this chunk + rowRedirection.fillChunk(redirContext, keys, chunkKeys); + for (int i = 0; i < keys.size(); ++i) { + Assert.notEquals(keys.get(i), "keys[i]", RowSequence.NULL_ROW_KEY, "RowSet.NULL_ROW_KEY"); + } - try (final ChunkSink.FillFromContext ctxt = - destSources[ii].makeFillFromContext(keys.size())) { - destSources[ii].fillFromChunkUnordered(ctxt, column.data, keys); + // fill the column with the data from this chunk + try (final ChunkSink.FillFromContext ctxt = + destSources[ii].makeFillFromContext(keys.size())) { + destSources[ii].fillFromChunkUnordered(ctxt, chunkData, keys); + } + } } } + +// try (final ChunkSource.FillContext redirContext = +// rowRedirection.makeFillContext(column.rowsModified.intSize(), null); +// final WritableLongChunk keys = +// WritableLongChunk.makeWritableChunk(column.rowsModified.intSize())) { +// rowRedirection.fillChunk(redirContext, keys, column.rowsModified); +// for (int i = 0; i < keys.size(); ++i) { +// Assert.notEquals(keys.get(i), "keys[i]", RowSequence.NULL_ROW_KEY, "RowSet.NULL_ROW_KEY"); +// } +// +// try (final ChunkSink.FillFromContext ctxt = +// destSources[ii].makeFillFromContext(keys.size())) { +// destSources[ii].fillFromChunkUnordered(ctxt, column.data, keys); +// } +// } + } // remove all data outside of our viewport @@ -368,7 +445,7 @@ private RowSet getFreeRows(long size) { needsResizing = true; } else if (freeset.size() < size) { int usedSlots = (int) (capacity - freeset.size()); - int prevCapacity = capacity; + long prevCapacity = capacity; do { capacity *= 2; } while ((capacity - usedSlots) < size); @@ -394,18 +471,26 @@ private void freeRows(final RowSet rowsToFree) { } // Note: these are NOT OrderedRowKeys until after the call to .sort() - try (final WritableLongChunk redirectedRows = - WritableLongChunk.makeWritableChunk(rowsToFree.intSize("BarrageTable"))) { - redirectedRows.setSize(0); - - rowsToFree.forAllRowKeys(next -> { - final long prevIndex = rowRedirection.remove(next); - Assert.assertion(prevIndex != -1, "prevIndex != -1", prevIndex, "prevIndex", next, "next"); - redirectedRows.add(prevIndex); - }); - - redirectedRows.sort(); // now they're truly ordered - freeset.insert(redirectedRows, 0, redirectedRows.size()); + final int chunkSize = (int) Math.min(rowsToFree.size(), TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD); + + try (final WritableLongChunk redirectedRows = WritableLongChunk.makeWritableChunk(chunkSize); + final RowSequence.Iterator rowsToFreeIterator = rowsToFree.getRowSequenceIterator()) { + + while (rowsToFreeIterator.hasMore()) { + + final RowSequence chunkRowsToFree = rowsToFreeIterator.getNextRowSequenceWithLength(chunkSize); + + redirectedRows.setSize(0); + + chunkRowsToFree.forAllRowKeys(next -> { + final long prevIndex = rowRedirection.remove(next); + Assert.assertion(prevIndex != -1, "prevIndex != -1", prevIndex, "prevIndex", next, "next"); + redirectedRows.add(prevIndex); + }); + + redirectedRows.sort(); // now they're truly ordered + freeset.insert(redirectedRows, 0, redirectedRows.size()); + } } } @@ -541,7 +626,9 @@ public static BarrageTable make(final UpdateSourceRegistrar registrar, final boolean isViewPort) { final ColumnDefinition[] columns = tableDefinition.getColumns(); final WritableColumnSource[] writableSources = new WritableColumnSource[columns.length]; - final WritableRowRedirection rowRedirection = WritableRowRedirection.FACTORY.createRowRedirection(8); +// final WritableRowRedirection rowRedirection = WritableRowRedirection.FACTORY.createRowRedirection(8); + final WritableRowRedirection rowRedirection = new LongColumnSourceWritableRowRedirection(new LongSparseArraySource()); +// final WritableRowRedirection rowRedirection = new LongColumnSourceWritableRowRedirection(new LongArraySource()); final LinkedHashMap> finalColumns = makeColumns(columns, writableSources, rowRedirection); diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java index 6dc0fd9c7a7..30e9c542327 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java @@ -13,9 +13,11 @@ import io.deephaven.barrage.flatbuf.BarrageMessageWrapper; import io.deephaven.barrage.flatbuf.BarrageModColumnMetadata; import io.deephaven.barrage.flatbuf.BarrageUpdateMetadata; +import io.deephaven.chunk.ChunkList; import io.deephaven.chunk.WritableChunk; import io.deephaven.chunk.attributes.Values; import io.deephaven.engine.rowset.RowSet; +import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.rowset.impl.ExternalizableRowSetUtils; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.rowset.RowSetShiftData; @@ -36,12 +38,14 @@ import java.util.BitSet; import java.util.Iterator; +import static io.deephaven.engine.table.impl.sources.InMemoryColumnSource.TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD; + public class BarrageStreamReader implements StreamReader { private static final Logger log = LoggerFactory.getLogger(BarrageStreamReader.class); - private int numAddRowsRead = 0; - private int numModRowsRead = 0; + private long numAddRowsRead = 0; + private long numModRowsRead = 0; private int numAddBatchesRemaining = 0; private int numModBatchesRemaining = 0; private BarrageMessage msg = null; @@ -122,6 +126,11 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, msg.addColumnData[ci] = new BarrageMessage.AddColumnData(); msg.addColumnData[ci].type = columnTypes[ci]; msg.addColumnData[ci].componentType = componentTypes[ci]; + msg.addColumnData[ci].data = new ChunkList(); + + // create an initial chunk of the correct size + final int chunkSize = (int)(Math.min(msg.rowsIncluded.size(), TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); + msg.addColumnData[ci].data.addChunk(columnChunkTypes[ci].makeWritableChunk(chunkSize), 0, 0); } // if this message is a snapshot response (vs. subscription) then mod columns may be empty @@ -130,9 +139,14 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, msg.modColumnData[ci] = new BarrageMessage.ModColumnData(); msg.modColumnData[ci].type = columnTypes[ci]; msg.modColumnData[ci].componentType = componentTypes[ci]; + msg.modColumnData[ci].data = new ChunkList(); final BarrageModColumnMetadata mcd = metadata.modColumnNodes(ci); msg.modColumnData[ci].rowsModified = extractIndex(mcd.modifiedRowsAsByteBuffer()); + + // create an initial chunk of the correct size + final int chunkSize = (int)(Math.min(msg.modColumnData[ci].rowsModified.size(), TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); + msg.modColumnData[ci].data.addChunk(columnChunkTypes[ci].makeWritableChunk(chunkSize), 0, 0); } } @@ -170,6 +184,11 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, msg.addColumnData[ci] = new BarrageMessage.AddColumnData(); msg.addColumnData[ci].type = columnTypes[ci]; msg.addColumnData[ci].componentType = componentTypes[ci]; + msg.addColumnData[ci].data = new ChunkList(); + + // create an initial chunk of the correct size + final int chunkSize = (int)(Math.min(msg.rowsIncluded.size(), TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); + msg.addColumnData[ci].data.addChunk(columnChunkTypes[ci].makeWritableChunk(chunkSize), 0, 0); } // no mod column data @@ -219,22 +238,116 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, } if (isAddBatch) { - final int numRowsTotal = msg.rowsIncluded.intSize("BarrageStreamReader"); for (int ci = 0; ci < msg.addColumnData.length; ++ci) { - msg.addColumnData[ci].data = ChunkInputStreamGenerator.extractChunkFromInputStream(options, - columnChunkTypes[ci], columnTypes[ci], componentTypes[ci], fieldNodeIter, - bufferInfoIter, ois, (WritableChunk) msg.addColumnData[ci].data, - numAddRowsRead, numRowsTotal); + final BarrageMessage.AddColumnData acd = msg.addColumnData[ci]; + + // need to add the batch row data to the column chunks + WritableChunk chunk = (WritableChunk)acd.data.lastChunk(); + int chunkSize = chunk.size(); + + final int chunkOffset; + if (acd.data.lastChunkSize() == 0) { + chunkOffset = 0; + } else { + // reading the rows from this might overflow the existing chunk + if (acd.data.lastChunkSize() + batch.length() > chunkSize) { + // create a new chunk before trying to write again + chunkSize = (int)(Math.min(msg.rowsIncluded.size() - numAddRowsRead, TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); + chunk = columnChunkTypes[ci].makeWritableChunk(chunkSize); + acd.data.addChunk(chunk, numAddRowsRead, numAddRowsRead); + + chunkOffset = 0; + } else { + chunkOffset = (int) (numAddRowsRead - acd.data.lastStartIndex()); + } + } + +// System.out.println("numAddRowsRead: " + numAddRowsRead + ", chunkSize: " + chunkSize + ", chunkOffset: " + chunkOffset); + System.out.println("BSR percentage complete: " + ((double)numAddRowsRead / (double)msg.rowsIncluded.size()) * 100.0); + + // fill the chunk, but catch overrun exceptions + try { + chunk = ChunkInputStreamGenerator.extractChunkFromInputStream(options, + columnChunkTypes[ci], columnTypes[ci], componentTypes[ci], fieldNodeIter, + bufferInfoIter, ois, chunk, + chunkOffset, chunkSize); + + // add the batch rows to this chunk rowset + acd.data.endIndex.set(acd.data.size() - 1, numAddRowsRead + batch.length() - 1); + + } catch (Exception ex) { + // create a new chunk and write this batch into that chunk + chunkSize = (int)(Math.min(msg.rowsIncluded.size() - numAddRowsRead, TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); + chunk = columnChunkTypes[ci].makeWritableChunk(chunkSize); + + acd.data.addChunk(chunk, numAddRowsRead, numAddRowsRead); + + chunk = ChunkInputStreamGenerator.extractChunkFromInputStream(options, + columnChunkTypes[ci], columnTypes[ci], componentTypes[ci], fieldNodeIter, + bufferInfoIter, ois, chunk, + 0, chunkSize); + + // add the batch rows to this chunk rowset + acd.data.endIndex.set(acd.data.size() - 1, numAddRowsRead + batch.length() - 1); + + System.out.println(ex.toString()); + } } numAddRowsRead += batch.length(); } else { for (int ci = 0; ci < msg.modColumnData.length; ++ci) { final BarrageMessage.ModColumnData mcd = msg.modColumnData[ci]; - final int numModdedRows = mcd.rowsModified.intSize("BarrageStreamReader"); - mcd.data = ChunkInputStreamGenerator.extractChunkFromInputStream(options, - columnChunkTypes[ci], columnTypes[ci], componentTypes[ci], fieldNodeIter, - bufferInfoIter, ois, (WritableChunk) mcd.data, numModRowsRead, - numModdedRows); + + // need to add the batch row data to the column chunks + WritableChunk chunk = (WritableChunk)mcd.data.lastChunk(); + int chunkSize = chunk.size(); + + final int chunkOffset; + if (mcd.data.lastChunkSize() == 0) { + chunkOffset = 0; + } else { + // these row will overflow + if (mcd.data.lastChunkSize() + batch.length() > chunkSize) { + // create a new chunk before trying to write again + chunkSize = (int)(Math.min(mcd.rowsModified.size() - numModRowsRead, TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); + chunk = columnChunkTypes[ci].makeWritableChunk(chunkSize); + + mcd.data.addChunk(chunk, numModRowsRead, numModRowsRead); + + chunkOffset = 0; + } else { + chunkOffset = (int) (numModRowsRead - mcd.data.lastStartIndex()); + } + } + + // fill the chunk, but catch overrun exceptions + try { + chunk = ChunkInputStreamGenerator.extractChunkFromInputStream(options, + columnChunkTypes[ci], columnTypes[ci], componentTypes[ci], fieldNodeIter, + bufferInfoIter, ois, chunk, + chunkOffset, chunkSize); + + // add the batch rows to this chunk rowset + mcd.data.endIndex.set(mcd.data.size() - 1, numModRowsRead + batch.length() - 1); + + } catch (Exception ex) { + // create a new chunk and write this batch into that chunk + // create a new chunk before trying to write again + chunkSize = (int)(Math.min(mcd.rowsModified.size() - numModRowsRead, TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); + chunk = columnChunkTypes[ci].makeWritableChunk(chunkSize); + + mcd.data.addChunk(chunk, numModRowsRead, numModRowsRead); + + chunk = ChunkInputStreamGenerator.extractChunkFromInputStream(options, + columnChunkTypes[ci], columnTypes[ci], componentTypes[ci], fieldNodeIter, + bufferInfoIter, ois, chunk, + 0, chunkSize); + + // add the batch rows to this chunk rowset + mcd.data.endIndex.set(mcd.data.size() - 1, numModRowsRead + batch.length() - 1); + + System.out.println(ex.toString()); + } } numModRowsRead += batch.length(); } diff --git a/server/src/main/java/io/deephaven/server/arrow/ArrowFlightUtil.java b/server/src/main/java/io/deephaven/server/arrow/ArrowFlightUtil.java index 65b7c94b38a..e5530197c41 100644 --- a/server/src/main/java/io/deephaven/server/arrow/ArrowFlightUtil.java +++ b/server/src/main/java/io/deephaven/server/arrow/ArrowFlightUtil.java @@ -169,13 +169,13 @@ public void onNext(final InputStream request) { final BarrageMessage.AddColumnData acd = new BarrageMessage.AddColumnData(); msg.addColumnData[ci] = acd; final int factor = (columnConversionFactors == null) ? 1 : columnConversionFactors[ci]; - try { - acd.data = ChunkInputStreamGenerator.extractChunkFromInputStream(options, factor, - columnChunkTypes[ci], columnTypes[ci], componentTypes[ci], fieldNodeIter, - bufferInfoIter, mi.inputStream, null, 0, 0); - } catch (final IOException unexpected) { - throw new UncheckedDeephavenException(unexpected); - } +// try { +// acd.data = ChunkInputStreamGenerator.extractChunkFromInputStream(options, factor, +// columnChunkTypes[ci], columnTypes[ci], componentTypes[ci], fieldNodeIter, +// bufferInfoIter, mi.inputStream, null, 0, 0); +// } catch (final IOException unexpected) { +// throw new UncheckedDeephavenException(unexpected); +// } if (acd.data.size() != numRowsAdded) { throw GrpcUtil.statusRuntimeException(Code.INVALID_ARGUMENT, diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index 366f70a31b6..333c4c6b4ef 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -11,6 +11,7 @@ import dagger.assisted.AssistedInject; import io.deephaven.base.formatters.FormatBitSet; import io.deephaven.base.verify.Assert; +import io.deephaven.chunk.ChunkList; import io.deephaven.chunk.LongChunk; import io.deephaven.chunk.ResettableWritableObjectChunk; import io.deephaven.chunk.WritableChunk; @@ -1580,6 +1581,8 @@ private BarrageMessage aggregateUpdatesInRange(final int startDelta, final int e for (int ci = 0; ci < downstream.addColumnData.length; ++ci) { final ColumnSource deltaColumn = deltaColumns[ci]; final BarrageMessage.AddColumnData adds = new BarrageMessage.AddColumnData(); + adds.data = new ChunkList(); + downstream.addColumnData[ci] = adds; if (addColumnSet.get(ci)) { @@ -1589,9 +1592,9 @@ private BarrageMessage aggregateUpdatesInRange(final int startDelta, final int e try (final ChunkSource.FillContext fc = deltaColumn.makeFillContext(chunkCapacity)) { deltaColumn.fillChunk(fc, chunk, localAdded); } - adds.data = chunk; + adds.data.addChunk(chunk); } else { - adds.data = deltaColumn.getChunkType().getEmptyChunk(); + adds.data.addChunk(deltaColumn.getChunkType().getEmptyChunk()); } adds.type = deltaColumn.getType(); @@ -1600,11 +1603,12 @@ private BarrageMessage aggregateUpdatesInRange(final int startDelta, final int e for (int ci = 0; ci < downstream.modColumnData.length; ++ci) { final ColumnSource deltaColumn = deltaColumns[ci]; - final BarrageMessage.ModColumnData modifications = new BarrageMessage.ModColumnData(); - downstream.modColumnData[ci] = modifications; + final BarrageMessage.ModColumnData mods = new BarrageMessage.ModColumnData(); + mods.data = new ChunkList(); + downstream.modColumnData[ci] = mods; if (modColumnSet.get(ci)) { - modifications.rowsModified = firstDelta.recordedMods.copy(); + mods.rowsModified = firstDelta.recordedMods.copy(); final int chunkCapacity = localModified.intSize("serializeItems"); final WritableChunk chunk = @@ -1612,14 +1616,14 @@ private BarrageMessage aggregateUpdatesInRange(final int startDelta, final int e try (final ChunkSource.FillContext fc = deltaColumn.makeFillContext(chunkCapacity)) { deltaColumn.fillChunk(fc, chunk, localModified); } - modifications.data = chunk; + mods.data.addChunk(chunk); } else { - modifications.rowsModified = RowSetFactory.empty(); - modifications.data = deltaColumn.getChunkType().getEmptyChunk(); + mods.rowsModified = RowSetFactory.empty(); + mods.data.addChunk(deltaColumn.getChunkType().getEmptyChunk()); } - modifications.type = deltaColumn.getType(); - modifications.componentType = deltaColumn.getComponentType(); + mods.type = deltaColumn.getType(); + mods.componentType = deltaColumn.getComponentType(); } } else { // We must coalesce these updates. @@ -1782,6 +1786,8 @@ final class ColumnInfo { for (int ci = 0; ci < downstream.addColumnData.length; ++ci) { final ColumnSource deltaColumn = deltaColumns[ci]; final BarrageMessage.AddColumnData adds = new BarrageMessage.AddColumnData(); + adds.data = new ChunkList(); + downstream.addColumnData[ci] = adds; if (addColumnSet.get(ci)) { @@ -1792,9 +1798,9 @@ final class ColumnInfo { ((FillUnordered) deltaColumn).fillChunkUnordered(fc, chunk, LongChunk.chunkWrap(info.addedMapping)); } - adds.data = chunk; + adds.data.addChunk(chunk); } else { - adds.data = deltaColumn.getChunkType().getEmptyChunk(); + adds.data.addChunk(deltaColumn.getChunkType().getEmptyChunk()); } adds.type = deltaColumn.getType(); @@ -1804,12 +1810,14 @@ final class ColumnInfo { int numActualModCols = 0; for (int i = 0; i < downstream.modColumnData.length; ++i) { final ColumnSource sourceColumn = deltaColumns[i]; - final BarrageMessage.ModColumnData modifications = new BarrageMessage.ModColumnData(); - downstream.modColumnData[numActualModCols++] = modifications; + final BarrageMessage.ModColumnData mods = new BarrageMessage.ModColumnData(); + mods.data = new ChunkList(); + + downstream.modColumnData[numActualModCols++] = mods; if (modColumnSet.get(i)) { final ColumnInfo info = getColumnInfo.apply(i); - modifications.rowsModified = info.recordedMods.copy(); + mods.rowsModified = info.recordedMods.copy(); final WritableChunk chunk = sourceColumn.getChunkType().makeWritableChunk(info.modifiedMapping.length); @@ -1817,15 +1825,14 @@ final class ColumnInfo { ((FillUnordered) sourceColumn).fillChunkUnordered(fc, chunk, LongChunk.chunkWrap(info.modifiedMapping)); } - - modifications.data = chunk; + mods.data.addChunk(chunk); } else { - modifications.rowsModified = RowSetFactory.empty(); - modifications.data = sourceColumn.getChunkType().getEmptyChunk(); + mods.rowsModified = RowSetFactory.empty(); + mods.data.addChunk(sourceColumn.getChunkType().getEmptyChunk()); } - modifications.type = sourceColumn.getType(); - modifications.componentType = sourceColumn.getComponentType(); + mods.type = sourceColumn.getType(); + mods.componentType = sourceColumn.getComponentType(); } } diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java index 4eda9f508ff..5fb6e598035 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java @@ -28,6 +28,7 @@ import io.deephaven.engine.table.impl.util.BarrageMessage; import io.deephaven.extensions.barrage.BarrageSubscriptionOptions; import io.deephaven.extensions.barrage.BarrageSnapshotOptions; +import io.deephaven.extensions.barrage.chunk.BaseChunkInputStreamGenerator; import io.deephaven.extensions.barrage.chunk.ChunkInputStreamGenerator; import io.deephaven.extensions.barrage.util.BarrageProtoUtil.ExposedByteArrayOutputStream; import io.deephaven.extensions.barrage.util.BarrageUtil; @@ -53,13 +54,11 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; -import java.util.ArrayDeque; -import java.util.BitSet; -import java.util.Map; -import java.util.Objects; +import java.util.*; import java.util.function.Consumer; import static io.deephaven.extensions.barrage.chunk.BaseChunkInputStreamGenerator.PADDING_BUFFER; +import static io.deephaven.util.QueryConstants.NULL_INT; public class BarrageStreamGenerator implements BarrageMessageProducer.StreamGenerator { @@ -104,14 +103,130 @@ public View getSchemaView(final TableDefinition table, } } + public static class ChunkSetInputStreamGenerator extends BaseChunkInputStreamGenerator implements SafeCloseable { + public RowSet[] rowSets; + public ChunkInputStreamGenerator[] generators; + + private class ChunkSetInputStream extends DrainableColumn { + + private final ChunkSetInputStreamGenerator generator; + private final StreamReaderOptions options; + private final RowSet subset; + + private int cachedNullCount = - 1; + + public ChunkSetInputStream(ChunkSetInputStreamGenerator generator, StreamReaderOptions options, RowSet subset) { + this.generator = generator; + this.options = options; + this.subset = subset; + } + + @Override + public void visitFieldNodes(FieldNodeListener listener) { + listener.noteLogicalFieldNode(subset.intSize("ChunkSetInputStreamGenerator"), nullCount()); + } + + @Override + public void visitBuffers(BufferListener listener) { + + } + + @Override + public int nullCount() { + if (options.useDeephavenNulls()) { + return 0; + } + if (cachedNullCount == -1) { + cachedNullCount = 0; + + // look through each chunk for this calculation + for (int i = 0; i < generator.size(); ++i) { + + final long shiftAmount = -generator.rowSets[i].firstRowKey(); + + // get an offset rowset for each chunk in the set + try (final WritableRowSet adjustedOffsets = subset.intersect(generator.rowSets[i])) { + // normalize this to the chunk offsets + adjustedOffsets.shiftInPlace(shiftAmount); + + adjustedOffsets.forAllRowKeys(row -> { + generator.generators[i].chunk + if (chunk.get((int) row) == NULL_INT) { + ++cachedNullCount; + } + }); + } catch (Exception ex) { + System.out.println(ex.getMessage()); + } + + + + } + } + return cachedNullCount; + } + + @Override + public int drainTo(OutputStream target) throws IOException { + return 0; + } + } + + ChunkSetInputStreamGenerator(BarrageMessage.AddColumnData acd) { + // create an input stream generator for each chunk + rowSets = new RowSet[acd.data.size()]; + generators = new ChunkInputStreamGenerator[acd.data.size()]; + + for (int i = 0; i < acd.data.size(); ++i) { + rowSets[i] = RowSetFactory.fromRange(acd.data.startIndex.get(i), acd.data.endIndex.get(i)); + + generators[i] = ChunkInputStreamGenerator.makeInputStreamGenerator( + acd.data.chunks.get(i).getChunkType(), acd.type, acd.componentType, acd.data.chunks.get(i)); + } + } + + ChunkSetInputStreamGenerator(BarrageMessage.ModColumnData mcd) { + // create an input stream generator for each chunk + rowSets = new RowSet[mcd.data.size()]; + generators = new ChunkInputStreamGenerator[mcd.data.size()]; + + for (int i = 0; i < mcd.data.size(); ++i) { + rowSets[i] = RowSetFactory.fromRange(mcd.data.startIndex.get(i), mcd.data.endIndex.get(i)); + + generators[i] = ChunkInputStreamGenerator.makeInputStreamGenerator( + mcd.data.chunks.get(i).getChunkType(), mcd.type, mcd.componentType, mcd.data.chunks.get(i)); + } + } + + public int size() { + return rowSets.length; + } + + @Override + public void close() { + for (int i = 0; i < generators.length; i++) { + generators[i].close(); + generators[i] = null; + } + for (int i = 0; i < rowSets.length; i++) { + rowSets[i].close(); + rowSets[i] = null; + } + } + + @Override + public DrainableColumn getInputStream(StreamReaderOptions options, @Nullable RowSet subset) throws IOException { + return new ChunkSetInputStream(this, options, subset); + } + } + public static class ModColumnData { public final RowSetGenerator rowsModified; - public final ChunkInputStreamGenerator data; + public final ChunkSetInputStreamGenerator data; ModColumnData(final BarrageMessage.ModColumnData col) throws IOException { rowsModified = new RowSetGenerator(col.rowsModified); - data = ChunkInputStreamGenerator.makeInputStreamGenerator( - col.data.getChunkType(), col.type, col.componentType, col.data); + data = new ChunkSetInputStreamGenerator(col); } } @@ -128,7 +243,7 @@ public static class ModColumnData { public final RowSetGenerator rowsRemoved; public final RowSetShiftDataGenerator shifted; - public final ChunkInputStreamGenerator[] addColumnData; + public final ChunkSetInputStreamGenerator[] addColumnData; public final ModColumnData[] modColumnData; /** @@ -149,11 +264,9 @@ public BarrageStreamGenerator(final BarrageMessage message) { rowsRemoved = new RowSetGenerator(message.rowsRemoved); shifted = new RowSetShiftDataGenerator(message.shifted); - addColumnData = new ChunkInputStreamGenerator[message.addColumnData.length]; + addColumnData = new ChunkSetInputStreamGenerator[message.addColumnData.length]; for (int i = 0; i < message.addColumnData.length; ++i) { - final BarrageMessage.AddColumnData acd = message.addColumnData[i]; - addColumnData[i] = ChunkInputStreamGenerator.makeInputStreamGenerator( - acd.data.getChunkType(), acd.type, acd.componentType, acd.data); + addColumnData[i] = new ChunkSetInputStreamGenerator(message.addColumnData[i]); } modColumnData = new ModColumnData[message.modColumnData.length]; for (int i = 0; i < modColumnData.length; ++i) { @@ -184,7 +297,7 @@ public void close() { rowsRemoved.close(); if (addColumnData != null) { - for (final ChunkInputStreamGenerator in : addColumnData) { + for (final ChunkSetInputStreamGenerator in : addColumnData) { in.close(); } } @@ -648,17 +761,37 @@ private long appendAddColumns(final View view, final long startRange, final long final ChunkInputStreamGenerator.FieldNodeListener fieldNodeListener, final ChunkInputStreamGenerator.BufferListener bufferListener) throws IOException { - - try (final RowSet myAddedOffsets = view.addRowOffsets().subSetByPositionRange(startRange, endRange)) { + try (final WritableRowSet myAddedOffsets = view.addRowOffsets().subSetByPositionRange(startRange, endRange)) { // add the add-column streams - for (final ChunkInputStreamGenerator col : addColumnData) { - final ChunkInputStreamGenerator.DrainableColumn drainableColumn = - col.getInputStream(view.options(), myAddedOffsets); - drainableColumn.visitFieldNodes(fieldNodeListener); - drainableColumn.visitBuffers(bufferListener); - - // Add the drainable last as it is allowed to immediately close a row set the visitors need - addStream.accept(drainableColumn); + for (final ChunkSetInputStreamGenerator chunkSetGen : addColumnData) { + + if (chunkSetGen.size() == 0) { + System.out.println("oops"); + } + // iterate through each chunk + for (int i = 0; i < chunkSetGen.size(); ++i) { + final ChunkInputStreamGenerator generator = chunkSetGen.generators[i]; + + final long shiftAmount = -chunkSetGen.rowSets[i].firstRowKey(); + + // get an offset rowset for each chunk in the set + try (final WritableRowSet adjustedOffsets = myAddedOffsets.intersect(chunkSetGen.rowSets[i])) { + // normalize this to the chunk offsets + adjustedOffsets.shiftInPlace(shiftAmount); + +// System.out.println("myAddedOffsets: " + myAddedOffsets + ", adjustedOffsets: " + adjustedOffsets); + + final ChunkInputStreamGenerator.DrainableColumn drainableColumn = + generator.getInputStream(view.options(), adjustedOffsets); + drainableColumn.visitFieldNodes(fieldNodeListener); + drainableColumn.visitBuffers(bufferListener); + + // Add the drainable last as it is allowed to immediately close a row set the visitors need + addStream.accept(drainableColumn); + } catch (Exception ex) { + System.out.println(ex.getMessage()); + } + } } return myAddedOffsets.size(); } @@ -689,14 +822,26 @@ private long appendModColumns(final View view, final long startRange, final long numRows = Math.max(numRows, myModOffsets.size()); try { - final ChunkInputStreamGenerator.DrainableColumn drainableColumn = - mcd.data.getInputStream(view.options(), myModOffsets); + // iterate through each chunk + for (int i = 0; i < mcd.data.size(); ++i) { + final ChunkInputStreamGenerator generator = mcd.data.generators[i]; + + final long shiftAmount = -mcd.data.rowSets[i].firstRowKey(); - drainableColumn.visitFieldNodes(fieldNodeListener); - drainableColumn.visitBuffers(bufferListener); + // get an offset rowset for each chunk in the set + try (final WritableRowSet adjustedOffsets = myModOffsets.intersect(mcd.data.rowSets[i])) { + // normalize this to the chunk offsets + adjustedOffsets.shiftInPlace(shiftAmount); - // See comment in appendAddColumns - addStream.accept(drainableColumn); + final ChunkInputStreamGenerator.DrainableColumn drainableColumn = + generator.getInputStream(view.options(), adjustedOffsets); + drainableColumn.visitFieldNodes(fieldNodeListener); + drainableColumn.visitBuffers(bufferListener); + + // Add the drainable last as it is allowed to immediately close a row set the visitors need + addStream.accept(drainableColumn); + } + } } finally { myModOffsets.close(); } From 9c5db5522a01ff09701638c3ff823b36745fb823 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Tue, 19 Apr 2022 16:46:16 -0700 Subject: [PATCH 25/47] updated architecture, removed ChunkList structure --- .../java/io/deephaven/chunk/ChunkList.java | 88 ---------- .../table/impl/remote/ConstructSnapshot.java | 17 +- .../table/impl/util/BarrageMessage.java | 23 ++- .../engine/table/impl/QueryTableTest.java | 4 +- .../VarBinaryChunkInputStreamGenerator.java | 2 +- .../barrage/table/BarrageTable.java | 74 ++++----- .../barrage/util/BarrageStreamReader.java | 95 ++++------- .../barrage/BarrageMessageProducer.java | 25 ++- .../barrage/BarrageStreamGenerator.java | 155 ++++++------------ 9 files changed, 158 insertions(+), 325 deletions(-) delete mode 100644 engine/chunk/src/main/java/io/deephaven/chunk/ChunkList.java diff --git a/engine/chunk/src/main/java/io/deephaven/chunk/ChunkList.java b/engine/chunk/src/main/java/io/deephaven/chunk/ChunkList.java deleted file mode 100644 index 66491b726e2..00000000000 --- a/engine/chunk/src/main/java/io/deephaven/chunk/ChunkList.java +++ /dev/null @@ -1,88 +0,0 @@ -package io.deephaven.chunk; - -import io.deephaven.chunk.attributes.Values; -import io.deephaven.chunk.util.pools.PoolableChunk; -import io.deephaven.util.SafeCloseable; - -import java.util.ArrayList; - -public class ChunkList implements SafeCloseable { - // rows in position space for each chunk - public final ArrayList startIndex = new ArrayList<>(); - public final ArrayList endIndex = new ArrayList<>(); - public final ArrayList> chunks = new ArrayList<>(); - - public void addChunk(Chunk nextChunk, long start, long end) { - startIndex.add(start); - endIndex.add(end); - chunks.add(nextChunk); - } - - public void addChunk(Chunk nextChunk) { - final long start; - final long end; - - if (endIndex.isEmpty()) { - start = 0L; - end = (long)nextChunk.size() - 1; - } else { - start = lastEndIndex() + 1; - end = start + (long)nextChunk.size() - 1; - } - - addChunk(nextChunk, start, end); - } - - public int size() { - return startIndex.size(); - } - - public Long chunkSize(int ci) { - if (ci < startIndex.size() && ci < endIndex.size()) { - return endIndex.get(ci) - startIndex.get(ci); - } - return null; - } - - public Chunk firstChunk() { - if (chunks.size() == 0) { - return null; - } - return chunks.get(0); - } - - public Chunk lastChunk() { - if (chunks.size() == 0) { - return null; - } - return chunks.get(chunks.size() -1); - } - - public Long lastStartIndex() { - if (startIndex.size() == 0) { - return null; - } - return startIndex.get(startIndex.size() -1); - } - - public Long lastEndIndex() { - if (endIndex.size() == 0) { - return null; - } - return endIndex.get(endIndex.size() -1); - } - - public Long lastChunkSize() { - return chunkSize(endIndex.size() -1); - } - - - @Override - public void close() { - for (Chunk chunk : chunks){ - if (chunk instanceof PoolableChunk) { - ((PoolableChunk) chunk).close(); - } - } - } -} \ No newline at end of file diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java index 54640ad6668..c38bef09ac2 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java @@ -1342,14 +1342,14 @@ public static boolean serializeAllTable(final boolean usePrev, final RowSet rows = columnIsEmpty ? RowSetFactory.empty() : snapshot.rowsIncluded; // Note: cannot use shared context across several calls of differing lengths and no sharing necessary // when empty - acd.data = getSnapshotDataAsChunkSet(columnSource, columnIsEmpty ? null : sharedContext, rows, usePrev); + acd.data = getSnapshotDataAsChunkList(columnSource, columnIsEmpty ? null : sharedContext, rows, usePrev); acd.type = columnSource.getType(); acd.componentType = columnSource.getComponentType(); final BarrageMessage.ModColumnData mcd = new BarrageMessage.ModColumnData(); snapshot.modColumnData[ii] = mcd; mcd.rowsModified = RowSetFactory.empty(); - mcd.data = getSnapshotDataAsChunkSet(columnSource, null, RowSetFactory.empty(), usePrev); + mcd.data = getSnapshotDataAsChunkList(columnSource, null, RowSetFactory.empty(), usePrev); mcd.type = acd.type; mcd.componentType = acd.componentType; } @@ -1432,12 +1432,17 @@ private static WritableChunk getSnapshotDataAsChunk(final ColumnSour } } - private static ChunkList getSnapshotDataAsChunkSet(final ColumnSource columnSource, - final SharedContext sharedContext, final RowSet rowSet, final boolean usePrev) { + private static ArrayList> getSnapshotDataAsChunkList(final ColumnSource columnSource, + final SharedContext sharedContext, final RowSet rowSet, final boolean usePrev) { final ColumnSource sourceToUse = ReinterpretUtils.maybeConvertToPrimitive(columnSource); long offset = 0; final long size = rowSet.size(); - final ChunkList result = new ChunkList(); + final ArrayList> result = new ArrayList<>(); + + if (rowSet.size() == 0) { + result.add(sourceToUse.getChunkType().getEmptyChunk()); + return result; + } while (offset < rowSet.size()) { final int chunkSize; @@ -1467,7 +1472,7 @@ private static ChunkList getSnapshotDataAsChunkSet(final ColumnSource col } // add the chunk to the current list - result.addChunk(currentChunk, offset, offset + currentChunk.size() - 1); + result.add(currentChunk); // increment the offset for the next chunk (using the actual values written) offset += currentChunk.size(); diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/util/BarrageMessage.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/util/BarrageMessage.java index 3acc106d792..4ac53b93e8b 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/util/BarrageMessage.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/util/BarrageMessage.java @@ -5,13 +5,10 @@ package io.deephaven.engine.table.impl.util; import io.deephaven.chunk.Chunk; -import io.deephaven.chunk.ChunkList; import io.deephaven.chunk.attributes.Values; import io.deephaven.chunk.util.pools.PoolableChunk; import io.deephaven.engine.rowset.RowSet; -import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.rowset.RowSetShiftData; -import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.util.SafeCloseable; import java.util.ArrayList; @@ -32,13 +29,13 @@ public static class ModColumnData { public RowSet rowsModified; public Class type; public Class componentType; - public ChunkList data; + public ArrayList> data; } public static class AddColumnData { public Class type; public Class componentType; - public ChunkList data; + public ArrayList> data; } public long firstSeq = -1; @@ -98,7 +95,13 @@ public void close() { continue; } - acd.data.close(); + for (Chunk chunk : acd.data) { + if (chunk instanceof PoolableChunk) { + ((PoolableChunk) chunk).close(); + } + } + + acd.data.clear(); } } if (modColumnData != null) { @@ -107,11 +110,13 @@ public void close() { continue; } - if (mcd.rowsModified != null) { - mcd.rowsModified.close(); + for (Chunk chunk : mcd.data) { + if (chunk instanceof PoolableChunk) { + ((PoolableChunk) chunk).close(); + } } - mcd.data.close(); + mcd.data.clear(); } } } diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableTest.java index fa3fd5e1cda..50d0dc94d7a 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableTest.java @@ -2390,8 +2390,8 @@ public void testUngroupConstructSnapshotOfBoxedNull() { try (final BarrageMessage snap = ConstructSnapshot.constructBackplaneSnapshot(this, (BaseTable) ungrouped)) { assertEquals(snap.rowsAdded, i(0, 1, 2)); - assertEquals(snap.addColumnData[0].data.asIntChunk().get(0), io.deephaven.util.QueryConstants.NULL_INT); - assertEquals(snap.addColumnData[1].data.asIntChunk().get(2), io.deephaven.util.QueryConstants.NULL_INT); + assertEquals(snap.addColumnData[0].data.get(0).asIntChunk().get(0), io.deephaven.util.QueryConstants.NULL_INT); + assertEquals(snap.addColumnData[1].data.get(0).asIntChunk().get(2), io.deephaven.util.QueryConstants.NULL_INT); } } diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java index e22955ee515..69621c7c743 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java @@ -128,7 +128,7 @@ private synchronized void computePayload() throws IOException { baos.close(); baos = new BarrageProtoUtil.ExposedByteArrayOutputStream(); - // add the item to this buffer + // add the item to the new buffer appendItem.append(baos, chunk.get(i)); baosSize = baos.size(); startIndex = i; diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java index 9509b600d02..314d18316f4 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java @@ -50,7 +50,7 @@ public class BarrageTable extends QueryTable implements BarrageMessage.Listener, Runnable { public static final boolean DEBUG_ENABLED = - Configuration.getInstance().getBooleanWithDefault("BarrageTable.debug", true); + Configuration.getInstance().getBooleanWithDefault("BarrageTable.debug", false); private static final Logger log = LoggerFactory.getLogger(BarrageTable.class); @@ -301,7 +301,7 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC } elapsed = System.nanoTime() - start; start = System.nanoTime(); - System.out.println("BT ensureCapacity (ms): " + (double)elapsed / 1_000_000.0); +// System.out.println("BT ensureCapacity (ms): " + (double)elapsed / 1_000_000.0); // this will hold all the free rows allocated for the included rows final WritableRowSet destinationRowSet = RowSetFactory.empty(); @@ -325,30 +325,30 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC } elapsed = System.nanoTime() - start; start = System.nanoTime(); - System.out.println("BT rowRedirection (ms): " + (double)elapsed / 1_000_000.0); +// System.out.println("BT rowRedirection (ms): " + (double)elapsed / 1_000_000.0); // update the column sources for (int ii = 0; ii < update.addColumnData.length; ++ii) { if (isSubscribedColumn(ii)) { final BarrageMessage.AddColumnData column = update.addColumnData[ii]; // grab the matching rows from each chunk + long offset = 0; for (int chunkIndex = 0; chunkIndex < column.data.size(); ++chunkIndex) { + final Chunk chunk = column.data.get(chunkIndex); try (final RowSet chunkRows - = RowSetFactory.fromRange( - column.data.startIndex.get(chunkIndex), - column.data.endIndex.get(chunkIndex)); + = RowSetFactory.fromRange(offset, offset + chunk.size() - 1); final RowSet chunkDestSet = destinationRowSet.subSetForPositions(chunkRows); final ChunkSink.FillFromContext ctxt = destSources[ii].makeFillFromContext(chunkDestSet.intSize())) { - final Chunk chunkData = column.data.chunks.get(chunkIndex); - destSources[ii].fillFromChunk(ctxt, chunkData, chunkDestSet); + destSources[ii].fillFromChunk(ctxt, chunk, chunkDestSet); } + offset += chunk.size(); } } } elapsed = System.nanoTime() - start; start = System.nanoTime(); - System.out.println("BT fill column data (ms): " + (double)elapsed / 1_000_000.0); +// System.out.println("BT fill column data (ms): " + (double)elapsed / 1_000_000.0); } modifiedColumnSet.clear(); @@ -361,49 +361,31 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC modifiedColumnSet.setColumnWithIndex(ii); // grab the matching rows from each chunk + long offset = 0; for (int chunkIndex = 0; chunkIndex < column.data.size(); ++chunkIndex) { + final Chunk chunk = column.data.get(chunkIndex); try (final RowSet chunkRows - = RowSetFactory.fromRange( - column.data.startIndex.get(chunkIndex), - column.data.endIndex.get(chunkIndex))) { - final Chunk chunkData = column.data.chunks.get(chunkIndex); - - try (final ChunkSource.FillContext redirContext = - rowRedirection.makeFillContext(chunkRows.intSize(), null); - final WritableLongChunk keys = - WritableLongChunk.makeWritableChunk(chunkRows.intSize()); - final RowSet chunkKeys = column.rowsModified.subSetForPositions(chunkRows)) { - - // fill the key chunk with the keys from this chunk - rowRedirection.fillChunk(redirContext, keys, chunkKeys); - for (int i = 0; i < keys.size(); ++i) { - Assert.notEquals(keys.get(i), "keys[i]", RowSequence.NULL_ROW_KEY, "RowSet.NULL_ROW_KEY"); - } + = RowSetFactory.fromRange(offset, offset + chunk.size() - 1); + final ChunkSource.FillContext redirContext = + rowRedirection.makeFillContext(chunkRows.intSize(), null); + final WritableLongChunk keys = + WritableLongChunk.makeWritableChunk(chunkRows.intSize()); + final RowSet chunkKeys = column.rowsModified.subSetForPositions(chunkRows)) { + + // fill the key chunk with the keys from this chunk + rowRedirection.fillChunk(redirContext, keys, chunkKeys); + for (int i = 0; i < keys.size(); ++i) { + Assert.notEquals(keys.get(i), "keys[i]", RowSequence.NULL_ROW_KEY, "RowSet.NULL_ROW_KEY"); + } - // fill the column with the data from this chunk - try (final ChunkSink.FillFromContext ctxt = - destSources[ii].makeFillFromContext(keys.size())) { - destSources[ii].fillFromChunkUnordered(ctxt, chunkData, keys); - } + // fill the column with the data from this chunk + try (final ChunkSink.FillFromContext ctxt = + destSources[ii].makeFillFromContext(keys.size())) { + destSources[ii].fillFromChunkUnordered(ctxt, chunk, keys); } } + offset += chunk.size(); } - -// try (final ChunkSource.FillContext redirContext = -// rowRedirection.makeFillContext(column.rowsModified.intSize(), null); -// final WritableLongChunk keys = -// WritableLongChunk.makeWritableChunk(column.rowsModified.intSize())) { -// rowRedirection.fillChunk(redirContext, keys, column.rowsModified); -// for (int i = 0; i < keys.size(); ++i) { -// Assert.notEquals(keys.get(i), "keys[i]", RowSequence.NULL_ROW_KEY, "RowSet.NULL_ROW_KEY"); -// } -// -// try (final ChunkSink.FillFromContext ctxt = -// destSources[ii].makeFillFromContext(keys.size())) { -// destSources[ii].fillFromChunkUnordered(ctxt, column.data, keys); -// } -// } - } // remove all data outside of our viewport diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java index 30e9c542327..2d730593871 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java @@ -13,11 +13,9 @@ import io.deephaven.barrage.flatbuf.BarrageMessageWrapper; import io.deephaven.barrage.flatbuf.BarrageModColumnMetadata; import io.deephaven.barrage.flatbuf.BarrageUpdateMetadata; -import io.deephaven.chunk.ChunkList; import io.deephaven.chunk.WritableChunk; import io.deephaven.chunk.attributes.Values; import io.deephaven.engine.rowset.RowSet; -import io.deephaven.engine.rowset.WritableRowSet; import io.deephaven.engine.rowset.impl.ExternalizableRowSetUtils; import io.deephaven.engine.rowset.RowSetFactory; import io.deephaven.engine.rowset.RowSetShiftData; @@ -35,6 +33,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.BitSet; import java.util.Iterator; @@ -48,6 +47,8 @@ public class BarrageStreamReader implements StreamReader { private long numModRowsRead = 0; private int numAddBatchesRemaining = 0; private int numModBatchesRemaining = 0; + private long lastAddStartIndex = 0; + private long lastModStartIndex = 0; private BarrageMessage msg = null; @Override @@ -126,11 +127,11 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, msg.addColumnData[ci] = new BarrageMessage.AddColumnData(); msg.addColumnData[ci].type = columnTypes[ci]; msg.addColumnData[ci].componentType = componentTypes[ci]; - msg.addColumnData[ci].data = new ChunkList(); + msg.addColumnData[ci].data = new ArrayList<>(); // create an initial chunk of the correct size final int chunkSize = (int)(Math.min(msg.rowsIncluded.size(), TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); - msg.addColumnData[ci].data.addChunk(columnChunkTypes[ci].makeWritableChunk(chunkSize), 0, 0); + msg.addColumnData[ci].data.add(columnChunkTypes[ci].makeWritableChunk(chunkSize)); } // if this message is a snapshot response (vs. subscription) then mod columns may be empty @@ -139,14 +140,14 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, msg.modColumnData[ci] = new BarrageMessage.ModColumnData(); msg.modColumnData[ci].type = columnTypes[ci]; msg.modColumnData[ci].componentType = componentTypes[ci]; - msg.modColumnData[ci].data = new ChunkList(); + msg.modColumnData[ci].data = new ArrayList<>(); final BarrageModColumnMetadata mcd = metadata.modColumnNodes(ci); msg.modColumnData[ci].rowsModified = extractIndex(mcd.modifiedRowsAsByteBuffer()); // create an initial chunk of the correct size final int chunkSize = (int)(Math.min(msg.modColumnData[ci].rowsModified.size(), TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); - msg.modColumnData[ci].data.addChunk(columnChunkTypes[ci].makeWritableChunk(chunkSize), 0, 0); + msg.modColumnData[ci].data.add(columnChunkTypes[ci].makeWritableChunk(chunkSize)); } } @@ -184,11 +185,11 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, msg.addColumnData[ci] = new BarrageMessage.AddColumnData(); msg.addColumnData[ci].type = columnTypes[ci]; msg.addColumnData[ci].componentType = componentTypes[ci]; - msg.addColumnData[ci].data = new ChunkList(); + msg.addColumnData[ci].data = new ArrayList<>(); // create an initial chunk of the correct size final int chunkSize = (int)(Math.min(msg.rowsIncluded.size(), TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); - msg.addColumnData[ci].data.addChunk(columnChunkTypes[ci].makeWritableChunk(chunkSize), 0, 0); + msg.addColumnData[ci].data.add(columnChunkTypes[ci].makeWritableChunk(chunkSize)); } // no mod column data @@ -241,29 +242,34 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, for (int ci = 0; ci < msg.addColumnData.length; ++ci) { final BarrageMessage.AddColumnData acd = msg.addColumnData[ci]; + final int lastChunkIndex = acd.data.size() - 1; + // need to add the batch row data to the column chunks - WritableChunk chunk = (WritableChunk)acd.data.lastChunk(); + WritableChunk chunk = (WritableChunk)acd.data.get(lastChunkIndex); int chunkSize = chunk.size(); final int chunkOffset; - if (acd.data.lastChunkSize() == 0) { + if (chunkSize == 0) { chunkOffset = 0; } else { - // reading the rows from this might overflow the existing chunk - if (acd.data.lastChunkSize() + batch.length() > chunkSize) { + long rowOffset = numAddRowsRead - lastAddStartIndex; + // reading the rows from this batch might overflow the existing chunk + if (rowOffset + batch.length() > chunkSize) { + lastAddStartIndex += chunkSize; + // create a new chunk before trying to write again chunkSize = (int)(Math.min(msg.rowsIncluded.size() - numAddRowsRead, TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); chunk = columnChunkTypes[ci].makeWritableChunk(chunkSize); - acd.data.addChunk(chunk, numAddRowsRead, numAddRowsRead); + acd.data.add(chunk); chunkOffset = 0; } else { - chunkOffset = (int) (numAddRowsRead - acd.data.lastStartIndex()); + chunkOffset = (int)rowOffset; } } // System.out.println("numAddRowsRead: " + numAddRowsRead + ", chunkSize: " + chunkSize + ", chunkOffset: " + chunkOffset); - System.out.println("BSR percentage complete: " + ((double)numAddRowsRead / (double)msg.rowsIncluded.size()) * 100.0); +// System.out.println("BSR percentage complete: " + ((double)numAddRowsRead / (double)msg.rowsIncluded.size()) * 100.0); // fill the chunk, but catch overrun exceptions try { @@ -271,25 +277,8 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, columnChunkTypes[ci], columnTypes[ci], componentTypes[ci], fieldNodeIter, bufferInfoIter, ois, chunk, chunkOffset, chunkSize); - - // add the batch rows to this chunk rowset - acd.data.endIndex.set(acd.data.size() - 1, numAddRowsRead + batch.length() - 1); - } catch (Exception ex) { - // create a new chunk and write this batch into that chunk - chunkSize = (int)(Math.min(msg.rowsIncluded.size() - numAddRowsRead, TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); - chunk = columnChunkTypes[ci].makeWritableChunk(chunkSize); - - acd.data.addChunk(chunk, numAddRowsRead, numAddRowsRead); - - chunk = ChunkInputStreamGenerator.extractChunkFromInputStream(options, - columnChunkTypes[ci], columnTypes[ci], componentTypes[ci], fieldNodeIter, - bufferInfoIter, ois, chunk, - 0, chunkSize); - - // add the batch rows to this chunk rowset - acd.data.endIndex.set(acd.data.size() - 1, numAddRowsRead + batch.length() - 1); - + // Should never happen, might be able to remove this exception handler System.out.println(ex.toString()); } } @@ -298,25 +287,30 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, for (int ci = 0; ci < msg.modColumnData.length; ++ci) { final BarrageMessage.ModColumnData mcd = msg.modColumnData[ci]; + final int lastChunkIndex = mcd.data.size() - 1; + // need to add the batch row data to the column chunks - WritableChunk chunk = (WritableChunk)mcd.data.lastChunk(); + WritableChunk chunk = (WritableChunk)mcd.data.get(lastChunkIndex); int chunkSize = chunk.size(); final int chunkOffset; - if (mcd.data.lastChunkSize() == 0) { + if (chunkSize == 0) { chunkOffset = 0; } else { - // these row will overflow - if (mcd.data.lastChunkSize() + batch.length() > chunkSize) { + long remaining = mcd.rowsModified.size() - numModRowsRead; + long rowOffset = numModRowsRead - lastModStartIndex; + // this batch might overflow the chunk + if (rowOffset + Math.min(remaining, batch.length()) > chunkSize) { + lastModStartIndex += chunkSize; + // create a new chunk before trying to write again - chunkSize = (int)(Math.min(mcd.rowsModified.size() - numModRowsRead, TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); + chunkSize = (int)(Math.min(remaining, TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); chunk = columnChunkTypes[ci].makeWritableChunk(chunkSize); - - mcd.data.addChunk(chunk, numModRowsRead, numModRowsRead); + mcd.data.add(chunk); chunkOffset = 0; } else { - chunkOffset = (int) (numModRowsRead - mcd.data.lastStartIndex()); + chunkOffset = (int)rowOffset; } } @@ -327,25 +321,8 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, bufferInfoIter, ois, chunk, chunkOffset, chunkSize); - // add the batch rows to this chunk rowset - mcd.data.endIndex.set(mcd.data.size() - 1, numModRowsRead + batch.length() - 1); - } catch (Exception ex) { - // create a new chunk and write this batch into that chunk - // create a new chunk before trying to write again - chunkSize = (int)(Math.min(mcd.rowsModified.size() - numModRowsRead, TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); - chunk = columnChunkTypes[ci].makeWritableChunk(chunkSize); - - mcd.data.addChunk(chunk, numModRowsRead, numModRowsRead); - - chunk = ChunkInputStreamGenerator.extractChunkFromInputStream(options, - columnChunkTypes[ci], columnTypes[ci], componentTypes[ci], fieldNodeIter, - bufferInfoIter, ois, chunk, - 0, chunkSize); - - // add the batch rows to this chunk rowset - mcd.data.endIndex.set(mcd.data.size() - 1, numModRowsRead + batch.length() - 1); - + // Should never happen, might be able to remove this exception handler System.out.println(ex.toString()); } } diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index 333c4c6b4ef..f2245ce551c 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -11,7 +11,6 @@ import dagger.assisted.AssistedInject; import io.deephaven.base.formatters.FormatBitSet; import io.deephaven.base.verify.Assert; -import io.deephaven.chunk.ChunkList; import io.deephaven.chunk.LongChunk; import io.deephaven.chunk.ResettableWritableObjectChunk; import io.deephaven.chunk.WritableChunk; @@ -1581,7 +1580,7 @@ private BarrageMessage aggregateUpdatesInRange(final int startDelta, final int e for (int ci = 0; ci < downstream.addColumnData.length; ++ci) { final ColumnSource deltaColumn = deltaColumns[ci]; final BarrageMessage.AddColumnData adds = new BarrageMessage.AddColumnData(); - adds.data = new ChunkList(); + adds.data = new ArrayList<>(); downstream.addColumnData[ci] = adds; @@ -1592,9 +1591,9 @@ private BarrageMessage aggregateUpdatesInRange(final int startDelta, final int e try (final ChunkSource.FillContext fc = deltaColumn.makeFillContext(chunkCapacity)) { deltaColumn.fillChunk(fc, chunk, localAdded); } - adds.data.addChunk(chunk); + adds.data.add(chunk); } else { - adds.data.addChunk(deltaColumn.getChunkType().getEmptyChunk()); + adds.data.add(deltaColumn.getChunkType().getEmptyChunk()); } adds.type = deltaColumn.getType(); @@ -1604,7 +1603,7 @@ private BarrageMessage aggregateUpdatesInRange(final int startDelta, final int e for (int ci = 0; ci < downstream.modColumnData.length; ++ci) { final ColumnSource deltaColumn = deltaColumns[ci]; final BarrageMessage.ModColumnData mods = new BarrageMessage.ModColumnData(); - mods.data = new ChunkList(); + mods.data = new ArrayList<>(); downstream.modColumnData[ci] = mods; if (modColumnSet.get(ci)) { @@ -1616,10 +1615,10 @@ private BarrageMessage aggregateUpdatesInRange(final int startDelta, final int e try (final ChunkSource.FillContext fc = deltaColumn.makeFillContext(chunkCapacity)) { deltaColumn.fillChunk(fc, chunk, localModified); } - mods.data.addChunk(chunk); + mods.data.add(chunk); } else { mods.rowsModified = RowSetFactory.empty(); - mods.data.addChunk(deltaColumn.getChunkType().getEmptyChunk()); + mods.data.add(deltaColumn.getChunkType().getEmptyChunk()); } mods.type = deltaColumn.getType(); @@ -1786,7 +1785,7 @@ final class ColumnInfo { for (int ci = 0; ci < downstream.addColumnData.length; ++ci) { final ColumnSource deltaColumn = deltaColumns[ci]; final BarrageMessage.AddColumnData adds = new BarrageMessage.AddColumnData(); - adds.data = new ChunkList(); + adds.data = new ArrayList<>(); downstream.addColumnData[ci] = adds; @@ -1798,9 +1797,9 @@ final class ColumnInfo { ((FillUnordered) deltaColumn).fillChunkUnordered(fc, chunk, LongChunk.chunkWrap(info.addedMapping)); } - adds.data.addChunk(chunk); + adds.data.add(chunk); } else { - adds.data.addChunk(deltaColumn.getChunkType().getEmptyChunk()); + adds.data.add(deltaColumn.getChunkType().getEmptyChunk()); } adds.type = deltaColumn.getType(); @@ -1811,7 +1810,7 @@ final class ColumnInfo { for (int i = 0; i < downstream.modColumnData.length; ++i) { final ColumnSource sourceColumn = deltaColumns[i]; final BarrageMessage.ModColumnData mods = new BarrageMessage.ModColumnData(); - mods.data = new ChunkList(); + mods.data = new ArrayList<>(); downstream.modColumnData[numActualModCols++] = mods; @@ -1825,10 +1824,10 @@ final class ColumnInfo { ((FillUnordered) sourceColumn).fillChunkUnordered(fc, chunk, LongChunk.chunkWrap(info.modifiedMapping)); } - mods.data.addChunk(chunk); + mods.data.add(chunk); } else { mods.rowsModified = RowSetFactory.empty(); - mods.data.addChunk(sourceColumn.getChunkType().getEmptyChunk()); + mods.data.add(sourceColumn.getChunkType().getEmptyChunk()); } mods.type = sourceColumn.getType(); diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java index 5fb6e598035..337d59e83b8 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java @@ -11,6 +11,7 @@ import gnu.trove.list.array.TIntArrayList; import io.deephaven.UncheckedDeephavenException; import io.deephaven.barrage.flatbuf.*; +import io.deephaven.chunk.Chunk; import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.WritableChunk; import io.deephaven.chunk.WritableLongChunk; @@ -28,7 +29,6 @@ import io.deephaven.engine.table.impl.util.BarrageMessage; import io.deephaven.extensions.barrage.BarrageSubscriptionOptions; import io.deephaven.extensions.barrage.BarrageSnapshotOptions; -import io.deephaven.extensions.barrage.chunk.BaseChunkInputStreamGenerator; import io.deephaven.extensions.barrage.chunk.ChunkInputStreamGenerator; import io.deephaven.extensions.barrage.util.BarrageProtoUtil.ExposedByteArrayOutputStream; import io.deephaven.extensions.barrage.util.BarrageUtil; @@ -58,7 +58,6 @@ import java.util.function.Consumer; import static io.deephaven.extensions.barrage.chunk.BaseChunkInputStreamGenerator.PADDING_BUFFER; -import static io.deephaven.util.QueryConstants.NULL_INT; public class BarrageStreamGenerator implements BarrageMessageProducer.StreamGenerator { @@ -103,103 +102,31 @@ public View getSchemaView(final TableDefinition table, } } - public static class ChunkSetInputStreamGenerator extends BaseChunkInputStreamGenerator implements SafeCloseable { - public RowSet[] rowSets; + public static class ChunkListInputStreamGenerator implements SafeCloseable { public ChunkInputStreamGenerator[] generators; - private class ChunkSetInputStream extends DrainableColumn { - - private final ChunkSetInputStreamGenerator generator; - private final StreamReaderOptions options; - private final RowSet subset; - - private int cachedNullCount = - 1; - - public ChunkSetInputStream(ChunkSetInputStreamGenerator generator, StreamReaderOptions options, RowSet subset) { - this.generator = generator; - this.options = options; - this.subset = subset; - } - - @Override - public void visitFieldNodes(FieldNodeListener listener) { - listener.noteLogicalFieldNode(subset.intSize("ChunkSetInputStreamGenerator"), nullCount()); - } - - @Override - public void visitBuffers(BufferListener listener) { - - } - - @Override - public int nullCount() { - if (options.useDeephavenNulls()) { - return 0; - } - if (cachedNullCount == -1) { - cachedNullCount = 0; - - // look through each chunk for this calculation - for (int i = 0; i < generator.size(); ++i) { - - final long shiftAmount = -generator.rowSets[i].firstRowKey(); - - // get an offset rowset for each chunk in the set - try (final WritableRowSet adjustedOffsets = subset.intersect(generator.rowSets[i])) { - // normalize this to the chunk offsets - adjustedOffsets.shiftInPlace(shiftAmount); - - adjustedOffsets.forAllRowKeys(row -> { - generator.generators[i].chunk - if (chunk.get((int) row) == NULL_INT) { - ++cachedNullCount; - } - }); - } catch (Exception ex) { - System.out.println(ex.getMessage()); - } - - - - } - } - return cachedNullCount; - } - - @Override - public int drainTo(OutputStream target) throws IOException { - return 0; - } - } - - ChunkSetInputStreamGenerator(BarrageMessage.AddColumnData acd) { + ChunkListInputStreamGenerator(BarrageMessage.AddColumnData acd) { // create an input stream generator for each chunk - rowSets = new RowSet[acd.data.size()]; generators = new ChunkInputStreamGenerator[acd.data.size()]; for (int i = 0; i < acd.data.size(); ++i) { - rowSets[i] = RowSetFactory.fromRange(acd.data.startIndex.get(i), acd.data.endIndex.get(i)); - generators[i] = ChunkInputStreamGenerator.makeInputStreamGenerator( - acd.data.chunks.get(i).getChunkType(), acd.type, acd.componentType, acd.data.chunks.get(i)); + acd.data.get(i).getChunkType(), acd.type, acd.componentType, acd.data.get(i)); } } - ChunkSetInputStreamGenerator(BarrageMessage.ModColumnData mcd) { + ChunkListInputStreamGenerator(BarrageMessage.ModColumnData mcd) { // create an input stream generator for each chunk - rowSets = new RowSet[mcd.data.size()]; generators = new ChunkInputStreamGenerator[mcd.data.size()]; for (int i = 0; i < mcd.data.size(); ++i) { - rowSets[i] = RowSetFactory.fromRange(mcd.data.startIndex.get(i), mcd.data.endIndex.get(i)); - generators[i] = ChunkInputStreamGenerator.makeInputStreamGenerator( - mcd.data.chunks.get(i).getChunkType(), mcd.type, mcd.componentType, mcd.data.chunks.get(i)); + mcd.data.get(i).getChunkType(), mcd.type, mcd.componentType, mcd.data.get(i)); } } public int size() { - return rowSets.length; + return generators.length; } @Override @@ -208,25 +135,26 @@ public void close() { generators[i].close(); generators[i] = null; } - for (int i = 0; i < rowSets.length; i++) { - rowSets[i].close(); - rowSets[i] = null; - } - } - - @Override - public DrainableColumn getInputStream(StreamReaderOptions options, @Nullable RowSet subset) throws IOException { - return new ChunkSetInputStream(this, options, subset); } } public static class ModColumnData { public final RowSetGenerator rowsModified; - public final ChunkSetInputStreamGenerator data; + public final ChunkListInputStreamGenerator data; + public final RowSet[] modChunkRowSets; ModColumnData(final BarrageMessage.ModColumnData col) throws IOException { rowsModified = new RowSetGenerator(col.rowsModified); - data = new ChunkSetInputStreamGenerator(col); + data = new ChunkListInputStreamGenerator(col); + + // build the row offsets for this column chunks + long offset = 0; + modChunkRowSets = new RowSet[col.data.size()]; + for (int chunkIdx = 0; chunkIdx < col.data.size(); ++chunkIdx) { + int chunkSize = col.data.get(chunkIdx).size(); + modChunkRowSets[chunkIdx] = RowSetFactory.fromRange(offset, offset + chunkSize - 1); + offset += chunkSize; + } } } @@ -243,9 +171,11 @@ public static class ModColumnData { public final RowSetGenerator rowsRemoved; public final RowSetShiftDataGenerator shifted; - public final ChunkSetInputStreamGenerator[] addColumnData; + public final ChunkListInputStreamGenerator[] addColumnData; public final ModColumnData[] modColumnData; + public final RowSet[] addChunkRowSets; + /** * Create a barrage stream generator that can slice and dice the barrage message for delivery to clients. * @@ -264,10 +194,28 @@ public BarrageStreamGenerator(final BarrageMessage message) { rowsRemoved = new RowSetGenerator(message.rowsRemoved); shifted = new RowSetShiftDataGenerator(message.shifted); - addColumnData = new ChunkSetInputStreamGenerator[message.addColumnData.length]; + boolean firstColumnWithChunks = true; + addColumnData = new ChunkListInputStreamGenerator[message.addColumnData.length]; + + // build the row sets for the add column data. NOTE: all populated columns will have the same + // number of chunks and equal row counts in each chunk so we can use any to create the rowsets + RowSet[] tmpAddChunkRowSets = new RowSet[0]; + for (int i = 0; i < message.addColumnData.length; ++i) { - addColumnData[i] = new ChunkSetInputStreamGenerator(message.addColumnData[i]); + if (firstColumnWithChunks && message.addColumnData[i].data.size() > 0) { + long offset = 0; + tmpAddChunkRowSets = new RowSet[message.addColumnData[i].data.size()]; + for (int chunkIdx = 0; chunkIdx < message.addColumnData[i].data.size(); ++chunkIdx) { + int chunkSize = message.addColumnData[i].data.get(chunkIdx).size(); + tmpAddChunkRowSets[chunkIdx] = RowSetFactory.fromRange(offset, offset + chunkSize - 1); + offset += chunkSize; + } + firstColumnWithChunks = false; + } + addColumnData[i] = new ChunkListInputStreamGenerator(message.addColumnData[i]); } + addChunkRowSets = tmpAddChunkRowSets; + modColumnData = new ModColumnData[message.modColumnData.length]; for (int i = 0; i < modColumnData.length; ++i) { modColumnData[i] = new ModColumnData(message.modColumnData[i]); @@ -297,7 +245,7 @@ public void close() { rowsRemoved.close(); if (addColumnData != null) { - for (final ChunkSetInputStreamGenerator in : addColumnData) { + for (final ChunkListInputStreamGenerator in : addColumnData) { in.close(); } } @@ -417,7 +365,10 @@ public SubView(final BarrageStreamGenerator generator, public void forEachStream(Consumer visitor) throws IOException { ByteBuffer metadata = generator.getSubscriptionMetadata(this); long offset = 0; + // batch size is maximum, can go smaller when needed final long batchSize = batchSize(); + + for (long ii = 0; ii < numAddBatches; ++ii) { visitor.accept(generator.getInputStream( this, offset, offset + batchSize, metadata, generator::appendAddColumns)); @@ -639,7 +590,7 @@ long visit(final View view, final long startRange, final long endRange, * @return an InputStream ready to be drained by GRPC */ private InputStream getInputStream(final View view, final long startRange, final long endRange, - final ByteBuffer metadata, final ColumnVisitor columnVisitor) throws IOException { + final ByteBuffer metadata, final ColumnVisitor columnVisitor) throws IOException { final ArrayDeque streams = new ArrayDeque<>(); final MutableInt size = new MutableInt(); @@ -763,19 +714,21 @@ private long appendAddColumns(final View view, final long startRange, final long try (final WritableRowSet myAddedOffsets = view.addRowOffsets().subSetByPositionRange(startRange, endRange)) { // add the add-column streams - for (final ChunkSetInputStreamGenerator chunkSetGen : addColumnData) { + for (final ChunkListInputStreamGenerator chunkSetGen : addColumnData) { if (chunkSetGen.size() == 0) { System.out.println("oops"); + // need to write an empty column here + } // iterate through each chunk for (int i = 0; i < chunkSetGen.size(); ++i) { final ChunkInputStreamGenerator generator = chunkSetGen.generators[i]; - final long shiftAmount = -chunkSetGen.rowSets[i].firstRowKey(); + final long shiftAmount = -addChunkRowSets[i].firstRowKey(); // get an offset rowset for each chunk in the set - try (final WritableRowSet adjustedOffsets = myAddedOffsets.intersect(chunkSetGen.rowSets[i])) { + try (final WritableRowSet adjustedOffsets = myAddedOffsets.intersect(addChunkRowSets[i])) { // normalize this to the chunk offsets adjustedOffsets.shiftInPlace(shiftAmount); @@ -826,10 +779,10 @@ private long appendModColumns(final View view, final long startRange, final long for (int i = 0; i < mcd.data.size(); ++i) { final ChunkInputStreamGenerator generator = mcd.data.generators[i]; - final long shiftAmount = -mcd.data.rowSets[i].firstRowKey(); + final long shiftAmount = -mcd.modChunkRowSets[i].firstRowKey(); // get an offset rowset for each chunk in the set - try (final WritableRowSet adjustedOffsets = myModOffsets.intersect(mcd.data.rowSets[i])) { + try (final WritableRowSet adjustedOffsets = myModOffsets.intersect(mcd.modChunkRowSets[i])) { // normalize this to the chunk offsets adjustedOffsets.shiftInPlace(shiftAmount); From 7d8b737e753e4119a0396bc710f52463bf0788c3 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Wed, 20 Apr 2022 17:54:50 -0700 Subject: [PATCH 26/47] passes tests --- .../VarBinaryChunkInputStreamGenerator.java | 149 +++++++++--------- .../barrage/util/BarrageStreamReader.java | 36 ++--- .../barrage/BarrageStreamGenerator.java | 113 +++++++++---- 3 files changed, 170 insertions(+), 128 deletions(-) diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java index 69621c7c743..14c68bd4286 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java @@ -63,6 +63,27 @@ public boolean isEmpty() { return byteArrays.isEmpty(); } + public long getByteOffset(int s, int e) { + // account for payload from s to e (inclusive) + final int startArrayIndex = getByteArrayIndex(s); + final int startOffset = offsets.get(s); + + final int endArrayIndex = getByteArrayIndex(e); + final int endOffset = offsets.get(e + 1); + + if (startArrayIndex == endArrayIndex) { // same byte array, can optimize + return endOffset - startOffset; + } else { + // need to span multiple byte arrays + long byteCount = getByteArraySize(startArrayIndex) - startOffset; + for (int midArrayIndex = startArrayIndex + 1; midArrayIndex < endArrayIndex; midArrayIndex++) { + byteCount += getByteArraySize(midArrayIndex); + } + byteCount += endOffset; + return byteCount; + } + } + public Integer getByteArrayIndex(int pos) { // optimize for most common case if (byteArrays.size() == 1) { @@ -83,6 +104,36 @@ public byte[] getByteArray(int arrayIdx) { public int getByteArraySize(int arrayIdx) { return byteArraySizes.get(arrayIdx); } + + public long writeByteData(LittleEndianDataOutputStream dos, int s, int e) throws IOException { + final int startArrayIndex = getByteArrayIndex(s); + final int startOffset = offsets.get(s); + + final int endArrayIndex = getByteArrayIndex(e); + final int endOffset = offsets.get(e + 1); + + long writeLen = 0; + + if (startArrayIndex == endArrayIndex) { // same byte array, can optimize + dos.write(byteArrays.get(startArrayIndex), startOffset, endOffset - startOffset); + writeLen += endOffset - startOffset; + } else { + // need to span multiple byte arrays + int firstSize = byteArraySizes.get(startArrayIndex) - startOffset; + dos.write(byteArrays.get(startArrayIndex), startOffset, firstSize); + writeLen += firstSize; + + for (int midArrayIndex = startArrayIndex + 1; midArrayIndex < endArrayIndex; midArrayIndex++) { + int midSize = getByteArraySize(midArrayIndex); + dos.write(byteArrays.get(midArrayIndex), 0, midSize); + writeLen += midSize; + } + + dos.write(byteArrays.get(endArrayIndex), 0, endOffset); + writeLen += endOffset; + } + return writeLen; + } }; private ByteStorage byteStorage = null; @@ -207,23 +258,7 @@ public void visitBuffers(final BufferListener listener) { // payload final MutableLong numPayloadBytes = new MutableLong(); subset.forAllRowKeyRanges((s, e) -> { - // account for payload, we have already int-size verified all rows in the RowSet - final int startArrayIndex = myByteStorage.getByteArrayIndex((int)s); - final int startOffset = myByteStorage.offsets.get((int) s); - - final int endArrayIndex = myByteStorage.getByteArrayIndex((int)e); - final int endOffset = myByteStorage.offsets.get((int) e + 1); - - if (startArrayIndex == endArrayIndex) { // same byte array, can optimize - numPayloadBytes.add(endOffset - startOffset); - } else { - // need to span multiple byte arrays - numPayloadBytes.add(myByteStorage.getByteArraySize(startArrayIndex) - startOffset); - for (int midArrayIndex = startArrayIndex + 1; midArrayIndex < endArrayIndex; midArrayIndex++) { - numPayloadBytes.add(myByteStorage.getByteArraySize(midArrayIndex)); - } - numPayloadBytes.add(endOffset); - } + numPayloadBytes.add(myByteStorage.getByteOffset((int)s, (int)e)); }); final long payloadExtended = numPayloadBytes.longValue() & REMAINDER_MOD_8_MASK; if (payloadExtended > 0) { @@ -235,47 +270,33 @@ public void visitBuffers(final BufferListener listener) { @Override protected int getRawSize() { if (cachedSize == -1) { - cachedSize = 0; + MutableLong totalCachedSize = new MutableLong(0L); if (sendValidityBuffer()) { - cachedSize += getValidityMapSerializationSizeFor(subset.intSize(DEBUG_NAME)); + totalCachedSize.add(getValidityMapSerializationSizeFor(subset.intSize(DEBUG_NAME))); } // there are n+1 offsets; it is not assumed first offset is zero if (!subset.isEmpty() && subset.size() == myByteStorage.offsets.size() - 1) { - cachedSize += myByteStorage.offsets.size() * Integer.BYTES; + totalCachedSize.add(myByteStorage.offsets.size() * Integer.BYTES); for (int i = 0; i < myByteStorage.size(); i++) { - cachedSize += myByteStorage.getByteArraySize(i); + totalCachedSize.add(myByteStorage.getByteArraySize(i)); } } else { - cachedSize += subset.isEmpty() ? 0 : Integer.BYTES; // account for the n+1 offset + totalCachedSize.add(subset.isEmpty() ? 0 : Integer.BYTES); // account for the n+1 offset subset.forAllRowKeyRanges((s, e) -> { // account for offsets - cachedSize += (e - s + 1) * Integer.BYTES; + totalCachedSize.add((e - s + 1) * Integer.BYTES); // account for payload - final int startArrayIndex = myByteStorage.getByteArrayIndex((int)s); - final int startOffset = myByteStorage.offsets.get((int) s); - - final int endArrayIndex = myByteStorage.getByteArrayIndex((int)e); - final int endOffset = myByteStorage.offsets.get((int) e + 1); - - if (startArrayIndex == endArrayIndex) { // same byte array, can optimize - cachedSize += endOffset - startOffset; - } else { - // need to span multiple byte arrays - cachedSize += (myByteStorage.getByteArraySize(startArrayIndex) - startOffset); - for (int midArrayIndex = startArrayIndex + 1; midArrayIndex < endArrayIndex; midArrayIndex++) { - cachedSize += myByteStorage.getByteArraySize(midArrayIndex); - } - cachedSize += endOffset; - } + totalCachedSize.add(myByteStorage.getByteOffset((int)s, (int)e)); }); } if (!subset.isEmpty() && (subset.size() & 0x1) == 0) { // then we must also align offset array - cachedSize += Integer.BYTES; + totalCachedSize.add(Integer.BYTES); } + cachedSize = LongSizedDataStructure.intSize("SortHelper.makeAndFillValues", totalCachedSize.longValue()); } return cachedSize; } @@ -322,21 +343,22 @@ public int drainTo(final OutputStream outputStream) throws IOException { final MutableInt logicalSize = new MutableInt(); subset.forAllRowKeys((idx) -> { try { - final int startArrayIndex = myByteStorage.getByteArrayIndex((int)idx); - final int startOffset = myByteStorage.offsets.get((int)idx); - // is this the last row in the chunk? if (idx == chunk.size() - 1) { + final int startArrayIndex = myByteStorage.getByteArrayIndex((int)idx); + final int startOffset = myByteStorage.offsets.get((int)idx); + logicalSize.add(myByteStorage.getByteArraySize(startArrayIndex) - startOffset); } else { - final int endArrayIndex = myByteStorage.getByteArrayIndex((int)idx + 1); - final int endOffset = myByteStorage.offsets.get((int) idx + 1); - - if (startArrayIndex == endArrayIndex) { // same byte array, can optimize - logicalSize.add(endOffset - startOffset); - } else { - logicalSize.add(myByteStorage.getByteArraySize(startArrayIndex) - startOffset); - } + logicalSize.add(myByteStorage.getByteOffset((int)idx,(int)idx)); +// final int endArrayIndex = myByteStorage.getByteArrayIndex((int)idx + 1); +// final int endOffset = myByteStorage.offsets.get((int) idx + 1); +// +// if (startArrayIndex == endArrayIndex) { // same byte array, can optimize +// logicalSize.add(endOffset - startOffset); +// } else { +// logicalSize.add(myByteStorage.getByteArraySize(startArrayIndex) - startOffset); +// } } dos.writeInt(logicalSize.intValue()); } catch (final IOException e) { @@ -354,30 +376,7 @@ public int drainTo(final OutputStream outputStream) throws IOException { final MutableLong payloadLen = new MutableLong(); subset.forAllRowKeyRanges((s, e) -> { try { - final int startArrayIndex = myByteStorage.getByteArrayIndex((int)s); - final int startOffset = myByteStorage.offsets.get((int) s); - - final int endArrayIndex = myByteStorage.getByteArrayIndex((int)e); - final int endOffset = myByteStorage.offsets.get((int) e + 1); - - if (startArrayIndex == endArrayIndex) { // same byte array, can optimize - dos.write(myByteStorage.byteArrays.get(startArrayIndex), startOffset, endOffset - startOffset); - payloadLen.add(endOffset - startOffset); - } else { - // need to span multiple byte arrays - int firstSize = myByteStorage.byteArraySizes.get(startArrayIndex) - startOffset; - dos.write(myByteStorage.byteArrays.get(startArrayIndex), startOffset, firstSize); - payloadLen.add(firstSize); - - for (int midArrayIndex = startArrayIndex + 1; midArrayIndex < endArrayIndex; midArrayIndex++) { - int midSize = myByteStorage.getByteArraySize(midArrayIndex); - dos.write(myByteStorage.byteArrays.get(midArrayIndex), 0, midSize); - payloadLen.add(midSize); - } - - dos.write(myByteStorage.byteArrays.get(endArrayIndex), 0, endOffset); - payloadLen.add(endOffset); - } + payloadLen.add(myByteStorage.writeByteData(dos, (int) s, (int) e)); } catch (final IOException err) { throw new UncheckedDeephavenException("couldn't drain data to OutputStream", err); } diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java index 2d730593871..445f4e8a0d0 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java @@ -44,9 +44,10 @@ public class BarrageStreamReader implements StreamReader { private static final Logger log = LoggerFactory.getLogger(BarrageStreamReader.class); private long numAddRowsRead = 0; + private long numAddRowsTotal = 0; private long numModRowsRead = 0; - private int numAddBatchesRemaining = 0; - private int numModBatchesRemaining = 0; + private long numModRowsTotal = 0; + private long lastAddStartIndex = 0; private long lastModStartIndex = 0; private BarrageMessage msg = null; @@ -80,8 +81,8 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, } else if (wrapper.msgType() == BarrageMessageType.BarrageUpdateMetadata) { if (msg != null) { throw new IllegalStateException( - "Previous message was not complete; pending " + numAddBatchesRemaining - + " add batches and " + numModBatchesRemaining + " mod batches"); + "Previous message was not complete; pending " + (numAddRowsTotal - numAddRowsRead) + + " add rows and " + (numModRowsTotal - numModRowsRead) + " mod rows"); } final BarrageUpdateMetadata metadata = @@ -94,14 +95,8 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, numAddRowsRead = 0; numModRowsRead = 0; - numAddBatchesRemaining = metadata.numAddBatches(); - numModBatchesRemaining = metadata.numModBatches(); - if (numAddBatchesRemaining < 0 || numModBatchesRemaining < 0) { - throw new IllegalStateException( - "Found negative number of record batches in barrage metadata: " - + numAddBatchesRemaining + " add batches and " + numModBatchesRemaining - + " mod batches"); - } + lastAddStartIndex = 0; + lastModStartIndex = 0; if (msg.isSnapshot) { final ByteBuffer effectiveViewport = metadata.effectiveViewportAsByteBuffer(); @@ -133,8 +128,10 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, final int chunkSize = (int)(Math.min(msg.rowsIncluded.size(), TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); msg.addColumnData[ci].data.add(columnChunkTypes[ci].makeWritableChunk(chunkSize)); } + numAddRowsTotal = msg.rowsIncluded.size(); // if this message is a snapshot response (vs. subscription) then mod columns may be empty + numModRowsTotal = 0; msg.modColumnData = new BarrageMessage.ModColumnData[metadata.modColumnNodesLength()]; for (int ci = 0; ci < msg.modColumnData.length; ++ci) { msg.modColumnData[ci] = new BarrageMessage.ModColumnData(); @@ -148,6 +145,8 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, // create an initial chunk of the correct size final int chunkSize = (int)(Math.min(msg.modColumnData[ci].rowsModified.size(), TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); msg.modColumnData[ci].data.add(columnChunkTypes[ci].makeWritableChunk(chunkSize)); + + numModRowsTotal = Math.max(numModRowsTotal, msg.modColumnData[ci].rowsModified.size()); } } @@ -200,7 +199,6 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, msg.shifted = RowSetShiftData.EMPTY; msg.isSnapshot = true; - numAddBatchesRemaining = 1; } bodyParsed = true; @@ -231,14 +229,10 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, } final TLongIterator bufferInfoIter = bufferInfo.iterator(); - final boolean isAddBatch = numAddBatchesRemaining > 0; - if (isAddBatch) { - --numAddBatchesRemaining; - } else { - --numModBatchesRemaining; - } + // add and mod rows are never combined in a batch. all added rows must be received before the first + // mod rows will be received - if (isAddBatch) { + if (numAddRowsRead < numAddRowsTotal) { for (int ci = 0; ci < msg.addColumnData.length; ++ci) { final BarrageMessage.AddColumnData acd = msg.addColumnData[ci]; @@ -340,7 +334,7 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, throw new IllegalStateException("Missing body tag"); } - if (numAddBatchesRemaining + numModBatchesRemaining == 0) { + if (numAddRowsRead == numAddRowsTotal && numModRowsRead == numModRowsTotal) { final BarrageMessage retval = msg; msg = null; return retval; diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java index 337d59e83b8..07ebd4e3b72 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java @@ -57,6 +57,7 @@ import java.util.*; import java.util.function.Consumer; +import static io.deephaven.engine.table.impl.sources.InMemoryColumnSource.TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD; import static io.deephaven.extensions.barrage.chunk.BaseChunkInputStreamGenerator.PADDING_BUFFER; public class BarrageStreamGenerator implements @@ -299,8 +300,8 @@ public static class SubView implements View { public final boolean reverseViewport; public final RowSet keyspaceViewport; public final BitSet subscribedColumns; - public final long numAddBatches; - public final long numModBatches; + public final long numAddRows; + public final long numModRows; public final RowSet addRowOffsets; public final RowSet addRowKeys; public final RowSet[] modRowOffsets; @@ -342,7 +343,7 @@ public SubView(final BarrageStreamGenerator generator, numModRows = Math.max(numModRows, mcd.rowsModified.original.size()); } } - numModBatches = (numModRows + batchSize - 1) / batchSize; + this.numModRows = numModRows; if (keyspaceViewport != null) { addRowKeys = keyspaceViewport.intersect(generator.rowsIncluded.original); @@ -356,31 +357,62 @@ public SubView(final BarrageStreamGenerator generator, addRowOffsets = RowSetFactory.flat(generator.rowsAdded.original.size()); } - // require an add batch if there are no mod batches - final long needsAddBatch = this.numModBatches == 0 ? 1 : 0; - numAddBatches = Math.max(needsAddBatch, (addRowOffsets.size() + batchSize - 1) / batchSize); + this.numAddRows = addRowOffsets.size(); } @Override public void forEachStream(Consumer visitor) throws IOException { ByteBuffer metadata = generator.getSubscriptionMetadata(this); - long offset = 0; + // batch size is maximum, can go smaller when needed - final long batchSize = batchSize(); + final int batchSize = batchSize(); + // there may be multiple chunk generators and we need to ensure that a single batch rowset does not cross + // boundaries and that we honor the batch size requests - for (long ii = 0; ii < numAddBatches; ++ii) { + if (numAddRows == 0 && numModRows == 0) { + // we still need to send a message containing just the metadata visitor.accept(generator.getInputStream( - this, offset, offset + batchSize, metadata, generator::appendAddColumns)); - offset += batchSize; - metadata = null; - } - offset = 0; - for (long ii = 0; ii < numModBatches; ++ii) { - visitor.accept(generator.getInputStream( - this, offset, offset + batchSize, metadata, generator::appendModColumns)); - offset += batchSize; - metadata = null; + this, 0, 0, metadata, generator::appendAddColumns)); + } else { + long offset = 0; + long chunkOffset = 0; + + while (offset < numAddRows) { + long effectiveBatchSize = batchSize; + + // make sure we are not about to cross a chunk boundary + if (chunkOffset + batchSize > TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD) { + effectiveBatchSize = TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD - chunkOffset; + chunkOffset = 0; + } + + visitor.accept(generator.getInputStream( + this, offset, offset + effectiveBatchSize, metadata, generator::appendAddColumns)); + + offset += effectiveBatchSize; + chunkOffset += effectiveBatchSize; + metadata = null; + } + + offset = 0; + chunkOffset = 0; + while (offset < numModRows) { + long effectiveBatchSize = batchSize; + + // make sure we are not about to cross a chunk boundary + if (chunkOffset + batchSize > TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD) { + effectiveBatchSize = TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD - chunkOffset; + chunkOffset = 0; + } + + visitor.accept(generator.getInputStream( + this, offset, offset + effectiveBatchSize, metadata, generator::appendModColumns)); + + offset += effectiveBatchSize; + chunkOffset += effectiveBatchSize; + metadata = null; + } } // clean up the helper indexes @@ -459,7 +491,7 @@ public static class SnapshotView implements View { public final boolean reverseViewport; public final RowSet keyspaceViewport; public final BitSet subscribedColumns; - public final long numAddBatches; + public final long numAddRows; public final RowSet addRowOffsets; public SnapshotView(final BarrageStreamGenerator generator, @@ -488,19 +520,39 @@ public SnapshotView(final BarrageStreamGenerator generator, } // require a batch to at least send the metadata - numAddBatches = Math.max(1, (addRowOffsets.size() + batchSize - 1) / batchSize); + numAddRows = addRowOffsets.size(); +// numAddBatches = Math.max(1, (addRowOffsets.size() + batchSize - 1) / batchSize); } @Override public void forEachStream(Consumer visitor) throws IOException { ByteBuffer metadata = generator.getSnapshotMetadata(this); - long offset = 0; final long batchSize = batchSize(); - for (long ii = 0; ii < numAddBatches; ++ii) { + + if (numAddRows == 0) { + // we still need to send a message containing just the metadata visitor.accept(generator.getInputStream( - this, offset, offset + batchSize, metadata, generator::appendAddColumns)); - offset += batchSize; - metadata = null; + this, 0, 0, metadata, generator::appendAddColumns)); + } else { + long offset = 0; + long chunkOffset = 0; + + while (offset < numAddRows) { + long effectiveBatchSize = batchSize; + + // make sure we are not about to cross a chunk boundary + if (chunkOffset + batchSize > TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD) { + effectiveBatchSize = TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD - chunkOffset; + chunkOffset = 0; + } + + visitor.accept(generator.getInputStream( + this, offset, offset + effectiveBatchSize, metadata, generator::appendAddColumns)); + + offset += effectiveBatchSize; + chunkOffset += effectiveBatchSize; + metadata = null; + } } addRowOffsets.close(); @@ -856,10 +908,8 @@ private ByteBuffer getSubscriptionMetadata(final SubView view) throws IOExceptio final int nodesOffset = metadata.endVector(); BarrageUpdateMetadata.startBarrageUpdateMetadata(metadata); - BarrageUpdateMetadata.addNumAddBatches(metadata, - LongSizedDataStructure.intSize("BarrageStreamGenerator", view.numAddBatches)); - BarrageUpdateMetadata.addNumModBatches(metadata, - LongSizedDataStructure.intSize("BarrageStreamGenerator", view.numModBatches)); + BarrageUpdateMetadata.addNumAddBatches(metadata, LongSizedDataStructure.intSize("BarrageStreamGenerator", (view.numAddRows + view.batchSize() - 1) / view.batchSize())); + BarrageUpdateMetadata.addNumModBatches(metadata, LongSizedDataStructure.intSize("BarrageStreamGenerator", (view.numModRows + view.batchSize() - 1) / view.batchSize())); BarrageUpdateMetadata.addIsSnapshot(metadata, isSnapshot); BarrageUpdateMetadata.addFirstSeq(metadata, firstSeq); BarrageUpdateMetadata.addLastSeq(metadata, lastSeq); @@ -911,8 +961,7 @@ private ByteBuffer getSnapshotMetadata(final SnapshotView view) throws IOExcepti } BarrageUpdateMetadata.startBarrageUpdateMetadata(metadata); - BarrageUpdateMetadata.addNumAddBatches(metadata, - LongSizedDataStructure.intSize("BarrageStreamGenerator", view.numAddBatches)); + BarrageUpdateMetadata.addNumAddBatches(metadata, LongSizedDataStructure.intSize("BarrageStreamGenerator", view.numAddRows + view.batchSize() - 1) / view.batchSize()); BarrageUpdateMetadata.addNumModBatches(metadata, 0); BarrageUpdateMetadata.addIsSnapshot(metadata, isSnapshot); BarrageUpdateMetadata.addFirstSeq(metadata, firstSeq); From 3a64e9cb280725affa0ba8ad49610287d695561e Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Thu, 21 Apr 2022 09:52:42 -0700 Subject: [PATCH 27/47] spotless applied --- .../table/impl/remote/ConstructSnapshot.java | 11 ++- .../engine/table/impl/QueryTableTest.java | 6 +- .../VarBinaryChunkInputStreamGenerator.java | 68 ++++++++++--------- .../barrage/table/BarrageTable.java | 57 +++++++--------- .../barrage/util/BarrageStreamReader.java | 26 ++++--- .../server/arrow/ArrowFlightUtil.java | 14 ++-- .../barrage/BarrageMessageProducer.java | 16 ++--- .../barrage/BarrageStreamGenerator.java | 19 +++--- 8 files changed, 104 insertions(+), 113 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java index c38bef09ac2..068cc89750a 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java @@ -1318,8 +1318,6 @@ public static boolean serializeAllTable(final boolean usePrev, snapshot.rowsIncluded = snapshot.rowsAdded.copy(); } - //LongSizedDataStructure.intSize("construct snapshot", snapshot.rowsIncluded.size()); - final Map sourceMap = table.getColumnSourceMap(); final String[] columnSources = sourceMap.keySet().toArray(CollectionUtil.ZERO_LENGTH_STRING_ARRAY); @@ -1342,7 +1340,8 @@ public static boolean serializeAllTable(final boolean usePrev, final RowSet rows = columnIsEmpty ? RowSetFactory.empty() : snapshot.rowsIncluded; // Note: cannot use shared context across several calls of differing lengths and no sharing necessary // when empty - acd.data = getSnapshotDataAsChunkList(columnSource, columnIsEmpty ? null : sharedContext, rows, usePrev); + acd.data = + getSnapshotDataAsChunkList(columnSource, columnIsEmpty ? null : sharedContext, rows, usePrev); acd.type = columnSource.getType(); acd.componentType = columnSource.getComponentType(); @@ -1433,7 +1432,7 @@ private static WritableChunk getSnapshotDataAsChunk(final ColumnSour } private static ArrayList> getSnapshotDataAsChunkList(final ColumnSource columnSource, - final SharedContext sharedContext, final RowSet rowSet, final boolean usePrev) { + final SharedContext sharedContext, final RowSet rowSet, final boolean usePrev) { final ColumnSource sourceToUse = ReinterpretUtils.maybeConvertToPrimitive(columnSource); long offset = 0; final long size = rowSet.size(); @@ -1444,12 +1443,12 @@ private static ArrayList> getSnapshotDataAsChunkList(final Col return result; } - while (offset < rowSet.size()) { + while (offset < size) { final int chunkSize; if (size - offset > TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD) { chunkSize = TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD; } else { - chunkSize = (int)(size - offset); + chunkSize = (int) (size - offset); } final long keyStart = rowSet.get(offset); diff --git a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableTest.java b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableTest.java index e9e06e8f496..0d9a55e8b58 100644 --- a/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableTest.java +++ b/engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableTest.java @@ -2390,8 +2390,10 @@ public void testUngroupConstructSnapshotOfBoxedNull() { try (final BarrageMessage snap = ConstructSnapshot.constructBackplaneSnapshot(this, (BaseTable) ungrouped)) { assertEquals(snap.rowsAdded, i(0, 1, 2)); - assertEquals(snap.addColumnData[0].data.get(0).asIntChunk().get(0), io.deephaven.util.QueryConstants.NULL_INT); - assertEquals(snap.addColumnData[1].data.get(0).asIntChunk().get(2), io.deephaven.util.QueryConstants.NULL_INT); + assertEquals(snap.addColumnData[0].data.get(0).asIntChunk().get(0), + io.deephaven.util.QueryConstants.NULL_INT); + assertEquals(snap.addColumnData[1].data.get(0).asIntChunk().get(2), + io.deephaven.util.QueryConstants.NULL_INT); } } diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java index 14c68bd4286..4532a862d52 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java @@ -63,13 +63,19 @@ public boolean isEmpty() { return byteArrays.isEmpty(); } - public long getByteOffset(int s, int e) { - // account for payload from s to e (inclusive) - final int startArrayIndex = getByteArrayIndex(s); - final int startOffset = offsets.get(s); - - final int endArrayIndex = getByteArrayIndex(e); - final int endOffset = offsets.get(e + 1); + /*** + * computes the size of the payload from sPos to ePos (inclusive) + * + * @param sPos the first data item to include in this payload + * @param ePos the last data item to include in this payload + * @return number of bytes in the payload + */ + public long getPayloadSize(int sPos, int ePos) { + final int startArrayIndex = getByteArrayIndex(sPos); + final int startOffset = offsets.get(sPos); + + final int endArrayIndex = getByteArrayIndex(ePos + 1); + final int endOffset = offsets.get(ePos + 1); if (startArrayIndex == endArrayIndex) { // same byte array, can optimize return endOffset - startOffset; @@ -97,20 +103,25 @@ public Integer getByteArrayIndex(int pos) { return 0; } - public byte[] getByteArray(int arrayIdx) { - return byteArrays.get(arrayIdx); - } - public int getByteArraySize(int arrayIdx) { return byteArraySizes.get(arrayIdx); } - public long writeByteData(LittleEndianDataOutputStream dos, int s, int e) throws IOException { - final int startArrayIndex = getByteArrayIndex(s); - final int startOffset = offsets.get(s); - - final int endArrayIndex = getByteArrayIndex(e); - final int endOffset = offsets.get(e + 1); + /*** + * write payload from sPos to ePos (inclusive) to the output stream + * + * @param dos the data output stream to populate with data + * @param sPos the first data item to include in this payload + * @param ePos the last data item to include in this payload + * @return number of bytes written to the outputstream + * @throws IOException if there is a problem writing to the output stream + */ + public long writePayload(LittleEndianDataOutputStream dos, int sPos, int ePos) throws IOException { + final int startArrayIndex = getByteArrayIndex(sPos); + final int startOffset = offsets.get(sPos); + + final int endArrayIndex = getByteArrayIndex(ePos + 1); + final int endOffset = offsets.get(ePos + 1); long writeLen = 0; @@ -171,8 +182,11 @@ private synchronized void computePayload() throws IOException { appendItem.append(baos, chunk.get(i)); baosSize = baos.size(); } catch (OutOfMemoryError ex) { - // we overran the buffer on this item, the output stream probably has junk from the failed write - // so we use the stored output stream size instead of querying + // we overran the buffer on this item and the output stream probably has junk from the failed write + // but it is no more than the size of a single data item. We use the stored output stream size + // though instead of querying the size of the output stream (since that includes junk bytes) + + // add the buffer to storage byteStorage.addByteArray(baos.peekBuffer(), startIndex, baosSize); // close the old output stream and create a new one @@ -258,7 +272,7 @@ public void visitBuffers(final BufferListener listener) { // payload final MutableLong numPayloadBytes = new MutableLong(); subset.forAllRowKeyRanges((s, e) -> { - numPayloadBytes.add(myByteStorage.getByteOffset((int)s, (int)e)); + numPayloadBytes.add(myByteStorage.getPayloadSize((int)s, (int)e)); }); final long payloadExtended = numPayloadBytes.longValue() & REMAINDER_MOD_8_MASK; if (payloadExtended > 0) { @@ -288,7 +302,7 @@ protected int getRawSize() { totalCachedSize.add((e - s + 1) * Integer.BYTES); // account for payload - totalCachedSize.add(myByteStorage.getByteOffset((int)s, (int)e)); + totalCachedSize.add(myByteStorage.getPayloadSize((int)s, (int)e)); }); } @@ -350,15 +364,7 @@ public int drainTo(final OutputStream outputStream) throws IOException { logicalSize.add(myByteStorage.getByteArraySize(startArrayIndex) - startOffset); } else { - logicalSize.add(myByteStorage.getByteOffset((int)idx,(int)idx)); -// final int endArrayIndex = myByteStorage.getByteArrayIndex((int)idx + 1); -// final int endOffset = myByteStorage.offsets.get((int) idx + 1); -// -// if (startArrayIndex == endArrayIndex) { // same byte array, can optimize -// logicalSize.add(endOffset - startOffset); -// } else { -// logicalSize.add(myByteStorage.getByteArraySize(startArrayIndex) - startOffset); -// } + logicalSize.add(myByteStorage.getPayloadSize((int)idx,(int)idx)); } dos.writeInt(logicalSize.intValue()); } catch (final IOException e) { @@ -376,7 +382,7 @@ public int drainTo(final OutputStream outputStream) throws IOException { final MutableLong payloadLen = new MutableLong(); subset.forAllRowKeyRanges((s, e) -> { try { - payloadLen.add(myByteStorage.writeByteData(dos, (int) s, (int) e)); + payloadLen.add(myByteStorage.writePayload(dos, (int) s, (int) e)); } catch (final IOException err) { throw new UncheckedDeephavenException("couldn't drain data to OutputStream", err); } diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java index 314d18316f4..98abf337c89 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java @@ -284,14 +284,12 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC totalMods.insert(column.rowsModified); } + // we are going to some potentially large operations, but we want to do the work in batches int maxChunkSize = 1 << ChunkPoolConstants.LARGEST_POOLED_CHUNK_LOG2_CAPACITY; if (update.rowsIncluded.isNonempty()) { - long start = System.nanoTime(); - long elapsed; - if (mightBeInitialSnapshot) { - // ensure the data sources have at least the incoming capacity. The sources will auto-resize but + // ensure the data sources have at least the incoming capacity. The sources will auto-resize but // we know the initial snapshot size and can optimize capacity = update.rowsIncluded.size(); for (final WritableColumnSource source : destSources) { @@ -300,20 +298,17 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC freeset.insertRange(0, capacity - 1); } - elapsed = System.nanoTime() - start; start = System.nanoTime(); -// System.out.println("BT ensureCapacity (ms): " + (double)elapsed / 1_000_000.0); - // this will hold all the free rows allocated for the included rows final WritableRowSet destinationRowSet = RowSetFactory.empty(); - // update the rowRedirection with the rowsIncluded set + // update the rowRedirection with the rowsIncluded set (in manageable batch sizes) try (final RowSequence.Iterator rowsIncludedIterator = update.rowsIncluded.getRowSequenceIterator()) { while (rowsIncludedIterator.hasMore()) { final RowSequence chunkRowsToFree = rowsIncludedIterator.getNextRowSequenceWithLength(maxChunkSize); try (final ChunkSink.FillFromContext redirContext = - rowRedirection.makeFillFromContext(chunkRowsToFree.intSize()); - final RowSet newRows = getFreeRows(chunkRowsToFree.intSize());) { + rowRedirection.makeFillFromContext(chunkRowsToFree.intSize()); + final RowSet newRows = getFreeRows(chunkRowsToFree.intSize());) { // Update redirection mapping: rowRedirection.fillFromChunk(redirContext, newRows.asRowKeyChunk(), chunkRowsToFree); @@ -324,9 +319,6 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC } } - elapsed = System.nanoTime() - start; start = System.nanoTime(); -// System.out.println("BT rowRedirection (ms): " + (double)elapsed / 1_000_000.0); - // update the column sources for (int ii = 0; ii < update.addColumnData.length; ++ii) { if (isSubscribedColumn(ii)) { @@ -335,20 +327,16 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC long offset = 0; for (int chunkIndex = 0; chunkIndex < column.data.size(); ++chunkIndex) { final Chunk chunk = column.data.get(chunkIndex); - try (final RowSet chunkRows - = RowSetFactory.fromRange(offset, offset + chunk.size() - 1); - final RowSet chunkDestSet = destinationRowSet.subSetForPositions(chunkRows); - final ChunkSink.FillFromContext ctxt = - destSources[ii].makeFillFromContext(chunkDestSet.intSize())) { - destSources[ii].fillFromChunk(ctxt, chunk, chunkDestSet); + try (final RowSet chunkRows = RowSetFactory.fromRange(offset, offset + chunk.size() - 1); + final RowSet chunkDestSet = destinationRowSet.subSetForPositions(chunkRows); + final ChunkSink.FillFromContext ctxt = + destSources[ii].makeFillFromContext(chunkDestSet.intSize())) { + destSources[ii].fillFromChunk(ctxt, chunk, chunkDestSet); } offset += chunk.size(); } } } - - elapsed = System.nanoTime() - start; start = System.nanoTime(); -// System.out.println("BT fill column data (ms): " + (double)elapsed / 1_000_000.0); } modifiedColumnSet.clear(); @@ -364,13 +352,12 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC long offset = 0; for (int chunkIndex = 0; chunkIndex < column.data.size(); ++chunkIndex) { final Chunk chunk = column.data.get(chunkIndex); - try (final RowSet chunkRows - = RowSetFactory.fromRange(offset, offset + chunk.size() - 1); - final ChunkSource.FillContext redirContext = - rowRedirection.makeFillContext(chunkRows.intSize(), null); - final WritableLongChunk keys = - WritableLongChunk.makeWritableChunk(chunkRows.intSize()); - final RowSet chunkKeys = column.rowsModified.subSetForPositions(chunkRows)) { + try (final RowSet chunkRows = RowSetFactory.fromRange(offset, offset + chunk.size() - 1); + final ChunkSource.FillContext redirContext = + rowRedirection.makeFillContext(chunkRows.intSize(), null); + final WritableLongChunk keys = + WritableLongChunk.makeWritableChunk(chunkRows.intSize()); + final RowSet chunkKeys = column.rowsModified.subSetForPositions(chunkRows)) { // fill the key chunk with the keys from this chunk rowRedirection.fillChunk(redirContext, keys, chunkKeys); @@ -380,7 +367,7 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC // fill the column with the data from this chunk try (final ChunkSink.FillFromContext ctxt = - destSources[ii].makeFillFromContext(keys.size())) { + destSources[ii].makeFillFromContext(keys.size())) { destSources[ii].fillFromChunkUnordered(ctxt, chunk, keys); } } @@ -456,7 +443,7 @@ private void freeRows(final RowSet rowsToFree) { final int chunkSize = (int) Math.min(rowsToFree.size(), TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD); try (final WritableLongChunk redirectedRows = WritableLongChunk.makeWritableChunk(chunkSize); - final RowSequence.Iterator rowsToFreeIterator = rowsToFree.getRowSequenceIterator()) { + final RowSequence.Iterator rowsToFreeIterator = rowsToFree.getRowSequenceIterator()) { while (rowsToFreeIterator.hasMore()) { @@ -608,9 +595,11 @@ public static BarrageTable make(final UpdateSourceRegistrar registrar, final boolean isViewPort) { final ColumnDefinition[] columns = tableDefinition.getColumns(); final WritableColumnSource[] writableSources = new WritableColumnSource[columns.length]; -// final WritableRowRedirection rowRedirection = WritableRowRedirection.FACTORY.createRowRedirection(8); - final WritableRowRedirection rowRedirection = new LongColumnSourceWritableRowRedirection(new LongSparseArraySource()); -// final WritableRowRedirection rowRedirection = new LongColumnSourceWritableRowRedirection(new LongArraySource()); + // final WritableRowRedirection rowRedirection = WritableRowRedirection.FACTORY.createRowRedirection(8); + final WritableRowRedirection rowRedirection = + new LongColumnSourceWritableRowRedirection(new LongSparseArraySource()); + // final WritableRowRedirection rowRedirection = new LongColumnSourceWritableRowRedirection(new + // LongArraySource()); final LinkedHashMap> finalColumns = makeColumns(columns, writableSources, rowRedirection); diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java index 445f4e8a0d0..2c1fa2b90cc 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java @@ -125,7 +125,8 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, msg.addColumnData[ci].data = new ArrayList<>(); // create an initial chunk of the correct size - final int chunkSize = (int)(Math.min(msg.rowsIncluded.size(), TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); + final int chunkSize = + (int) (Math.min(msg.rowsIncluded.size(), TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); msg.addColumnData[ci].data.add(columnChunkTypes[ci].makeWritableChunk(chunkSize)); } numAddRowsTotal = msg.rowsIncluded.size(); @@ -143,7 +144,8 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, msg.modColumnData[ci].rowsModified = extractIndex(mcd.modifiedRowsAsByteBuffer()); // create an initial chunk of the correct size - final int chunkSize = (int)(Math.min(msg.modColumnData[ci].rowsModified.size(), TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); + final int chunkSize = (int) (Math.min(msg.modColumnData[ci].rowsModified.size(), + TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); msg.modColumnData[ci].data.add(columnChunkTypes[ci].makeWritableChunk(chunkSize)); numModRowsTotal = Math.max(numModRowsTotal, msg.modColumnData[ci].rowsModified.size()); @@ -186,9 +188,7 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, msg.addColumnData[ci].componentType = componentTypes[ci]; msg.addColumnData[ci].data = new ArrayList<>(); - // create an initial chunk of the correct size - final int chunkSize = (int)(Math.min(msg.rowsIncluded.size(), TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); - msg.addColumnData[ci].data.add(columnChunkTypes[ci].makeWritableChunk(chunkSize)); + msg.addColumnData[ci].data.add(null); } // no mod column data @@ -239,7 +239,7 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, final int lastChunkIndex = acd.data.size() - 1; // need to add the batch row data to the column chunks - WritableChunk chunk = (WritableChunk)acd.data.get(lastChunkIndex); + WritableChunk chunk = (WritableChunk) acd.data.get(lastChunkIndex); int chunkSize = chunk.size(); final int chunkOffset; @@ -252,19 +252,17 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, lastAddStartIndex += chunkSize; // create a new chunk before trying to write again - chunkSize = (int)(Math.min(msg.rowsIncluded.size() - numAddRowsRead, TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); + chunkSize = (int) (Math.min(msg.rowsIncluded.size() - numAddRowsRead, + TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); chunk = columnChunkTypes[ci].makeWritableChunk(chunkSize); acd.data.add(chunk); chunkOffset = 0; } else { - chunkOffset = (int)rowOffset; + chunkOffset = (int) rowOffset; } } -// System.out.println("numAddRowsRead: " + numAddRowsRead + ", chunkSize: " + chunkSize + ", chunkOffset: " + chunkOffset); -// System.out.println("BSR percentage complete: " + ((double)numAddRowsRead / (double)msg.rowsIncluded.size()) * 100.0); - // fill the chunk, but catch overrun exceptions try { chunk = ChunkInputStreamGenerator.extractChunkFromInputStream(options, @@ -284,7 +282,7 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, final int lastChunkIndex = mcd.data.size() - 1; // need to add the batch row data to the column chunks - WritableChunk chunk = (WritableChunk)mcd.data.get(lastChunkIndex); + WritableChunk chunk = (WritableChunk) mcd.data.get(lastChunkIndex); int chunkSize = chunk.size(); final int chunkOffset; @@ -298,13 +296,13 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, lastModStartIndex += chunkSize; // create a new chunk before trying to write again - chunkSize = (int)(Math.min(remaining, TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); + chunkSize = (int) (Math.min(remaining, TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); chunk = columnChunkTypes[ci].makeWritableChunk(chunkSize); mcd.data.add(chunk); chunkOffset = 0; } else { - chunkOffset = (int)rowOffset; + chunkOffset = (int) rowOffset; } } diff --git a/server/src/main/java/io/deephaven/server/arrow/ArrowFlightUtil.java b/server/src/main/java/io/deephaven/server/arrow/ArrowFlightUtil.java index e5530197c41..344a579343e 100644 --- a/server/src/main/java/io/deephaven/server/arrow/ArrowFlightUtil.java +++ b/server/src/main/java/io/deephaven/server/arrow/ArrowFlightUtil.java @@ -169,13 +169,13 @@ public void onNext(final InputStream request) { final BarrageMessage.AddColumnData acd = new BarrageMessage.AddColumnData(); msg.addColumnData[ci] = acd; final int factor = (columnConversionFactors == null) ? 1 : columnConversionFactors[ci]; -// try { -// acd.data = ChunkInputStreamGenerator.extractChunkFromInputStream(options, factor, -// columnChunkTypes[ci], columnTypes[ci], componentTypes[ci], fieldNodeIter, -// bufferInfoIter, mi.inputStream, null, 0, 0); -// } catch (final IOException unexpected) { -// throw new UncheckedDeephavenException(unexpected); -// } + try { + acd.data.add(ChunkInputStreamGenerator.extractChunkFromInputStream(options, factor, + columnChunkTypes[ci], columnTypes[ci], componentTypes[ci], fieldNodeIter, + bufferInfoIter, mi.inputStream, null, 0, 0)); + } catch (final IOException unexpected) { + throw new UncheckedDeephavenException(unexpected); + } if (acd.data.size() != numRowsAdded) { throw GrpcUtil.statusRuntimeException(Code.INVALID_ARGUMENT, diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index f2245ce551c..4bf647734b0 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -1859,21 +1859,17 @@ private static void applyRedirMapping(final RowSet keys, final RowSet values, fi private void flipSnapshotStateForSubscriptions( final List subscriptions) { for (final Subscription subscription : subscriptions) { - if (subscription.snapshotViewport != null) { - final RowSet tmp = subscription.viewport; - subscription.viewport = subscription.snapshotViewport; - subscription.snapshotViewport = (WritableRowSet) tmp; - } + final RowSet tmp = subscription.viewport; + subscription.viewport = subscription.snapshotViewport; + subscription.snapshotViewport = (WritableRowSet) tmp; boolean tmpDirection = subscription.reverseViewport; subscription.reverseViewport = subscription.snapshotReverseViewport; subscription.snapshotReverseViewport = tmpDirection; - if (subscription.snapshotColumns != null) { - final BitSet tmp = subscription.subscribedColumns; - subscription.subscribedColumns = subscription.snapshotColumns; - subscription.snapshotColumns = tmp; - } + final BitSet tmpCols = subscription.subscribedColumns; + subscription.subscribedColumns = subscription.snapshotColumns; + subscription.snapshotColumns = tmpCols; } } diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java index 07ebd4e3b72..a5802aecaf5 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java @@ -198,7 +198,7 @@ public BarrageStreamGenerator(final BarrageMessage message) { boolean firstColumnWithChunks = true; addColumnData = new ChunkListInputStreamGenerator[message.addColumnData.length]; - // build the row sets for the add column data. NOTE: all populated columns will have the same + // build the row sets for the add column data. NOTE: all populated columns will have the same // number of chunks and equal row counts in each chunk so we can use any to create the rowsets RowSet[] tmpAddChunkRowSets = new RowSet[0]; @@ -508,8 +508,6 @@ public SnapshotView(final BarrageStreamGenerator generator, this.keyspaceViewport = keyspaceViewport; this.subscribedColumns = subscribedColumns; - final int batchSize = batchSize(); - // precompute add row offsets if (keyspaceViewport != null) { try (WritableRowSet intersect = keyspaceViewport.intersect(generator.rowsIncluded.original)) { @@ -521,7 +519,6 @@ public SnapshotView(final BarrageStreamGenerator generator, // require a batch to at least send the metadata numAddRows = addRowOffsets.size(); -// numAddBatches = Math.max(1, (addRowOffsets.size() + batchSize - 1) / batchSize); } @Override @@ -642,7 +639,7 @@ long visit(final View view, final long startRange, final long endRange, * @return an InputStream ready to be drained by GRPC */ private InputStream getInputStream(final View view, final long startRange, final long endRange, - final ByteBuffer metadata, final ColumnVisitor columnVisitor) throws IOException { + final ByteBuffer metadata, final ColumnVisitor columnVisitor) throws IOException { final ArrayDeque streams = new ArrayDeque<>(); final MutableInt size = new MutableInt(); @@ -784,7 +781,8 @@ private long appendAddColumns(final View view, final long startRange, final long // normalize this to the chunk offsets adjustedOffsets.shiftInPlace(shiftAmount); -// System.out.println("myAddedOffsets: " + myAddedOffsets + ", adjustedOffsets: " + adjustedOffsets); + // System.out.println("myAddedOffsets: " + myAddedOffsets + ", adjustedOffsets: " + + // adjustedOffsets); final ChunkInputStreamGenerator.DrainableColumn drainableColumn = generator.getInputStream(view.options(), adjustedOffsets); @@ -908,8 +906,10 @@ private ByteBuffer getSubscriptionMetadata(final SubView view) throws IOExceptio final int nodesOffset = metadata.endVector(); BarrageUpdateMetadata.startBarrageUpdateMetadata(metadata); - BarrageUpdateMetadata.addNumAddBatches(metadata, LongSizedDataStructure.intSize("BarrageStreamGenerator", (view.numAddRows + view.batchSize() - 1) / view.batchSize())); - BarrageUpdateMetadata.addNumModBatches(metadata, LongSizedDataStructure.intSize("BarrageStreamGenerator", (view.numModRows + view.batchSize() - 1) / view.batchSize())); + BarrageUpdateMetadata.addNumAddBatches(metadata, LongSizedDataStructure.intSize("BarrageStreamGenerator", + (view.numAddRows + view.batchSize() - 1) / view.batchSize())); + BarrageUpdateMetadata.addNumModBatches(metadata, LongSizedDataStructure.intSize("BarrageStreamGenerator", + (view.numModRows + view.batchSize() - 1) / view.batchSize())); BarrageUpdateMetadata.addIsSnapshot(metadata, isSnapshot); BarrageUpdateMetadata.addFirstSeq(metadata, firstSeq); BarrageUpdateMetadata.addLastSeq(metadata, lastSeq); @@ -961,7 +961,8 @@ private ByteBuffer getSnapshotMetadata(final SnapshotView view) throws IOExcepti } BarrageUpdateMetadata.startBarrageUpdateMetadata(metadata); - BarrageUpdateMetadata.addNumAddBatches(metadata, LongSizedDataStructure.intSize("BarrageStreamGenerator", view.numAddRows + view.batchSize() - 1) / view.batchSize()); + BarrageUpdateMetadata.addNumAddBatches(metadata, LongSizedDataStructure.intSize("BarrageStreamGenerator", + (view.numAddRows + view.batchSize() - 1) / view.batchSize())); BarrageUpdateMetadata.addNumModBatches(metadata, 0); BarrageUpdateMetadata.addIsSnapshot(metadata, isSnapshot); BarrageUpdateMetadata.addFirstSeq(metadata, firstSeq); From 33b75ba0f8fbfdd5cf2a65e5e7280ade0f1dbc9a Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Thu, 21 Apr 2022 13:04:18 -0700 Subject: [PATCH 28/47] addressed many PR comments --- .../VarBinaryChunkInputStreamGenerator.java | 34 ++++++++++++++----- .../barrage/table/BarrageTable.java | 11 ++---- .../server/arrow/ArrowFlightUtil.java | 8 ++--- .../barrage/BarrageStreamGenerator.java | 20 +++++------ 4 files changed, 39 insertions(+), 34 deletions(-) diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java index 4532a862d52..0b162dad8af 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java @@ -41,6 +41,11 @@ public static class ByteStorage { private final ArrayList byteArraySizes; private final ArrayList byteArrayStartIndex; + // low-budget memoization + private long lastIdx = -1; + private int lastIdxArrayIdx = -1; + private int lastIdxOffset = -1; + public ByteStorage(int size) { offsets = WritableIntChunk.makeWritableChunk(size); @@ -71,21 +76,32 @@ public boolean isEmpty() { * @return number of bytes in the payload */ public long getPayloadSize(int sPos, int ePos) { - final int startArrayIndex = getByteArrayIndex(sPos); - final int startOffset = offsets.get(sPos); + final int startArrayIndex; + final int startOffset; - final int endArrayIndex = getByteArrayIndex(ePos + 1); - final int endOffset = offsets.get(ePos + 1); + // might already have these start offsets saved + if (sPos == lastIdx) { + startArrayIndex = lastIdxArrayIdx; + startOffset = lastIdxOffset; + } else { + startArrayIndex = getByteArrayIndex(sPos); + startOffset = offsets.get(sPos); + } - if (startArrayIndex == endArrayIndex) { // same byte array, can optimize - return endOffset - startOffset; + // store these for current and later re-use + lastIdx = ePos + 1; + lastIdxArrayIdx = getByteArrayIndex((int)lastIdx); + lastIdxOffset = offsets.get(ePos + 1); + + if (startArrayIndex == lastIdxArrayIdx) { // same byte array, can optimize + return lastIdxOffset - startOffset; } else { // need to span multiple byte arrays long byteCount = getByteArraySize(startArrayIndex) - startOffset; - for (int midArrayIndex = startArrayIndex + 1; midArrayIndex < endArrayIndex; midArrayIndex++) { + for (int midArrayIndex = startArrayIndex + 1; midArrayIndex < lastIdxArrayIdx; midArrayIndex++) { byteCount += getByteArraySize(midArrayIndex); } - byteCount += endOffset; + byteCount += lastIdxOffset; return byteCount; } } @@ -310,7 +326,7 @@ protected int getRawSize() { // then we must also align offset array totalCachedSize.add(Integer.BYTES); } - cachedSize = LongSizedDataStructure.intSize("SortHelper.makeAndFillValues", totalCachedSize.longValue()); + cachedSize = LongSizedDataStructure.intSize("VarBinaryChunkInputStreamGenerator.getRawSize", totalCachedSize.longValue()); } return cachedSize; } diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java index 09e9f7a6ef6..548744193dc 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java @@ -318,13 +318,10 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC totalMods.insert(column.rowsModified); } - // we are going to some potentially large operations, but we want to do the work in batches - int maxChunkSize = 1 << ChunkPoolConstants.LARGEST_POOLED_CHUNK_LOG2_CAPACITY; - if (update.rowsIncluded.isNonempty()) { if (mightBeInitialSnapshot) { - // ensure the data sources have at least the incoming capacity. The sources will auto-resize but - // we know the initial snapshot size and can optimize + // ensure the data sources have at least the incoming capacity. The sources can auto-resize but + // we know the initial snapshot size and resize immediately capacity = update.rowsIncluded.size(); for (final WritableColumnSource source : destSources) { source.ensureCapacity(capacity); @@ -336,6 +333,7 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC final WritableRowSet destinationRowSet = RowSetFactory.empty(); // update the rowRedirection with the rowsIncluded set (in manageable batch sizes) + int maxChunkSize = 1 << ChunkPoolConstants.LARGEST_POOLED_CHUNK_LOG2_CAPACITY; try (final RowSequence.Iterator rowsIncludedIterator = update.rowsIncluded.getRowSequenceIterator()) { while (rowsIncludedIterator.hasMore()) { final RowSequence chunkRowsToFree = @@ -644,11 +642,8 @@ public static BarrageTable make( final boolean isViewPort) { final ColumnDefinition[] columns = tableDefinition.getColumns(); final WritableColumnSource[] writableSources = new WritableColumnSource[columns.length]; - // final WritableRowRedirection rowRedirection = WritableRowRedirection.FACTORY.createRowRedirection(8); final WritableRowRedirection rowRedirection = new LongColumnSourceWritableRowRedirection(new LongSparseArraySource()); - // final WritableRowRedirection rowRedirection = new LongColumnSourceWritableRowRedirection(new - // LongArraySource()); final LinkedHashMap> finalColumns = makeColumns(columns, writableSources, rowRedirection); diff --git a/server/src/main/java/io/deephaven/server/arrow/ArrowFlightUtil.java b/server/src/main/java/io/deephaven/server/arrow/ArrowFlightUtil.java index f8d16dbe085..156ea7bfa51 100644 --- a/server/src/main/java/io/deephaven/server/arrow/ArrowFlightUtil.java +++ b/server/src/main/java/io/deephaven/server/arrow/ArrowFlightUtil.java @@ -52,10 +52,7 @@ import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.util.ArrayDeque; -import java.util.BitSet; -import java.util.Iterator; -import java.util.Queue; +import java.util.*; import java.util.concurrent.ScheduledExecutorService; import static io.deephaven.extensions.barrage.util.BarrageProtoUtil.DEFAULT_SER_OPTIONS; @@ -173,6 +170,7 @@ public void onNext(final InputStream request) { for (int ci = 0; ci < numColumns; ++ci) { final BarrageMessage.AddColumnData acd = new BarrageMessage.AddColumnData(); msg.addColumnData[ci] = acd; + msg.addColumnData[ci].data = new ArrayList<>(); final int factor = (columnConversionFactors == null) ? 1 : columnConversionFactors[ci]; try { acd.data.add(ChunkInputStreamGenerator.extractChunkFromInputStream(options, factor, @@ -182,7 +180,7 @@ public void onNext(final InputStream request) { throw new UncheckedDeephavenException(unexpected); } - if (acd.data.size() != numRowsAdded) { + if (acd.data.get(0).size() != numRowsAdded) { throw GrpcUtil.statusRuntimeException(Code.INVALID_ARGUMENT, "Inconsistent num records per column: " + numRowsAdded + " != " + acd.data.size()); } diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java index 01b05c6e6f2..1c31b865024 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java @@ -252,11 +252,18 @@ public void close() { for (final ChunkListInputStreamGenerator in : addColumnData) { in.close(); } + for (final RowSet rs : addChunkRowSets) { + rs.close(); + } + } if (modColumnData != null) { for (final ModColumnData mcd : modColumnData) { mcd.rowsModified.close(); mcd.data.close(); + for (final RowSet rs : mcd.modChunkRowSets) { + rs.close(); + } } } } @@ -324,8 +331,6 @@ public SubView(final BarrageStreamGenerator generator, this.keyspaceViewport = keyspaceViewport; this.subscribedColumns = subscribedColumns; - final int batchSize = batchSize(); - if (keyspaceViewport != null) { this.modRowOffsets = new WritableRowSet[generator.modColumnData.length]; } else { @@ -776,26 +781,17 @@ private long appendAddColumns(final View view, final long startRange, final long try (final WritableRowSet myAddedOffsets = view.addRowOffsets().subSetByPositionRange(startRange, endRange)) { // add the add-column streams for (final ChunkListInputStreamGenerator chunkSetGen : addColumnData) { - - if (chunkSetGen.size() == 0) { - System.out.println("oops"); - // need to write an empty column here - - } // iterate through each chunk for (int i = 0; i < chunkSetGen.size(); ++i) { final ChunkInputStreamGenerator generator = chunkSetGen.generators[i]; final long shiftAmount = -addChunkRowSets[i].firstRowKey(); - // get an offset rowset for each chunk in the set + // get an offset RowSet for each chunk in the set try (final WritableRowSet adjustedOffsets = myAddedOffsets.intersect(addChunkRowSets[i])) { // normalize this to the chunk offsets adjustedOffsets.shiftInPlace(shiftAmount); - // System.out.println("myAddedOffsets: " + myAddedOffsets + ", adjustedOffsets: " + - // adjustedOffsets); - final ChunkInputStreamGenerator.DrainableColumn drainableColumn = generator.getInputStream(view.options(), adjustedOffsets); drainableColumn.visitFieldNodes(fieldNodeListener); From 68ebcc98ba9114c7cf8f8b98d7f92b78777e1e86 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Thu, 21 Apr 2022 15:18:56 -0700 Subject: [PATCH 29/47] fixed bug in VarBinary stream generator --- .../chunk/VarBinaryChunkInputStreamGenerator.java | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java index 0b162dad8af..9da3fa38d7a 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java @@ -215,8 +215,8 @@ private synchronized void computePayload() throws IOException { startIndex = i; byteStorage.offsets.set(i, 0); } - byteStorage.offsets.set(i + 1, baosSize); } + byteStorage.offsets.set(i + 1, baosSize); } byteStorage.addByteArray(baos.peekBuffer(), startIndex, baosSize); baos.close(); @@ -373,15 +373,7 @@ public int drainTo(final OutputStream outputStream) throws IOException { final MutableInt logicalSize = new MutableInt(); subset.forAllRowKeys((idx) -> { try { - // is this the last row in the chunk? - if (idx == chunk.size() - 1) { - final int startArrayIndex = myByteStorage.getByteArrayIndex((int)idx); - final int startOffset = myByteStorage.offsets.get((int)idx); - - logicalSize.add(myByteStorage.getByteArraySize(startArrayIndex) - startOffset); - } else { - logicalSize.add(myByteStorage.getPayloadSize((int)idx,(int)idx)); - } + logicalSize.add(myByteStorage.getPayloadSize((int)idx,(int)idx)); dos.writeInt(logicalSize.intValue()); } catch (final IOException e) { throw new UncheckedDeephavenException("couldn't drain data to OutputStream", e); From 8d8c167b59bfe4897a1ca7c9f51f8f3e00be47ef Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Thu, 21 Apr 2022 17:20:33 -0700 Subject: [PATCH 30/47] fixed bug in BSG that wrote multiple column data --- .../barrage/BarrageStreamGenerator.java | 104 +++++++++++------- 1 file changed, 64 insertions(+), 40 deletions(-) diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java index 1c31b865024..076a40fe18c 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java @@ -11,6 +11,7 @@ import gnu.trove.list.array.TIntArrayList; import io.deephaven.UncheckedDeephavenException; import io.deephaven.barrage.flatbuf.*; +import io.deephaven.base.verify.Assert; import io.deephaven.chunk.Chunk; import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.WritableChunk; @@ -153,7 +154,11 @@ public static class ModColumnData { modChunkRowSets = new RowSet[col.data.size()]; for (int chunkIdx = 0; chunkIdx < col.data.size(); ++chunkIdx) { int chunkSize = col.data.get(chunkIdx).size(); - modChunkRowSets[chunkIdx] = RowSetFactory.fromRange(offset, offset + chunkSize - 1); + if (chunkSize == 0) { + modChunkRowSets[chunkIdx] = RowSetFactory.empty(); + } else { + modChunkRowSets[chunkIdx] = RowSetFactory.fromRange(offset, offset + chunkSize - 1); + } offset += chunkSize; } } @@ -211,7 +216,11 @@ public BarrageStreamGenerator(final BarrageMessage message, final WriteMetricsCo tmpAddChunkRowSets = new RowSet[message.addColumnData[i].data.size()]; for (int chunkIdx = 0; chunkIdx < message.addColumnData[i].data.size(); ++chunkIdx) { int chunkSize = message.addColumnData[i].data.get(chunkIdx).size(); - tmpAddChunkRowSets[chunkIdx] = RowSetFactory.fromRange(offset, offset + chunkSize - 1); + if (chunkSize == 0) { + tmpAddChunkRowSets[chunkIdx] = RowSetFactory.empty(); + } else { + tmpAddChunkRowSets[chunkIdx] = RowSetFactory.fromRange(offset, offset + chunkSize - 1); + } offset += chunkSize; } firstColumnWithChunks = false; @@ -779,30 +788,39 @@ private long appendAddColumns(final View view, final long startRange, final long final ChunkInputStreamGenerator.BufferListener bufferListener) throws IOException { try (final WritableRowSet myAddedOffsets = view.addRowOffsets().subSetByPositionRange(startRange, endRange)) { - // add the add-column streams + // every column must write to the stream for (final ChunkListInputStreamGenerator chunkSetGen : addColumnData) { - // iterate through each chunk - for (int i = 0; i < chunkSetGen.size(); ++i) { - final ChunkInputStreamGenerator generator = chunkSetGen.generators[i]; - - final long shiftAmount = -addChunkRowSets[i].firstRowKey(); - - // get an offset RowSet for each chunk in the set - try (final WritableRowSet adjustedOffsets = myAddedOffsets.intersect(addChunkRowSets[i])) { - // normalize this to the chunk offsets - adjustedOffsets.shiftInPlace(shiftAmount); - - final ChunkInputStreamGenerator.DrainableColumn drainableColumn = - generator.getInputStream(view.options(), adjustedOffsets); - drainableColumn.visitFieldNodes(fieldNodeListener); - drainableColumn.visitBuffers(bufferListener); - - // Add the drainable last as it is allowed to immediately close a row set the visitors need - addStream.accept(drainableColumn); - } catch (Exception ex) { - System.out.println(ex.getMessage()); + boolean columnWritten = false; + + // iterate through each chunk and find the one that contains the requested data + for (int i = 0; i < chunkSetGen.size() && !columnWritten; ++i) { + // if the set of data is empty + if (addChunkRowSets[i].isEmpty() || startRange <= addChunkRowSets[i].lastRowKey()) { + final ChunkInputStreamGenerator generator = chunkSetGen.generators[i]; + + // shift this into the chunk position space + final long shiftAmount = -addChunkRowSets[i].firstRowKey(); + + // get an offset RowSet for each chunk in the set + try (final WritableRowSet adjustedOffsets = myAddedOffsets.intersect(addChunkRowSets[i])) { + // normalize this to the chunk offsets + adjustedOffsets.shiftInPlace(shiftAmount); + + final ChunkInputStreamGenerator.DrainableColumn drainableColumn = + generator.getInputStream(view.options(), adjustedOffsets); + drainableColumn.visitFieldNodes(fieldNodeListener); + drainableColumn.visitBuffers(bufferListener); + + // Add the drainable last as it is allowed to immediately close a row set the visitors need + addStream.accept(drainableColumn); + } catch (Exception ex) { + System.out.println(ex.getMessage()); + } + // no need to test other chunks + columnWritten = true; } } + Assert.assertion(columnWritten, "appendAddColumns - range not contained in a single chunk"); } return myAddedOffsets.size(); } @@ -833,26 +851,32 @@ private long appendModColumns(final View view, final long startRange, final long numRows = Math.max(numRows, myModOffsets.size()); try { + boolean columnWritten = false; // iterate through each chunk - for (int i = 0; i < mcd.data.size(); ++i) { - final ChunkInputStreamGenerator generator = mcd.data.generators[i]; - - final long shiftAmount = -mcd.modChunkRowSets[i].firstRowKey(); - - // get an offset rowset for each chunk in the set - try (final WritableRowSet adjustedOffsets = myModOffsets.intersect(mcd.modChunkRowSets[i])) { - // normalize this to the chunk offsets - adjustedOffsets.shiftInPlace(shiftAmount); - - final ChunkInputStreamGenerator.DrainableColumn drainableColumn = - generator.getInputStream(view.options(), adjustedOffsets); - drainableColumn.visitFieldNodes(fieldNodeListener); - drainableColumn.visitBuffers(bufferListener); - - // Add the drainable last as it is allowed to immediately close a row set the visitors need - addStream.accept(drainableColumn); + for (int i = 0; i < mcd.data.size() && !columnWritten; ++i) { + if (mcd.modChunkRowSets[i].isEmpty() || startRange <= mcd.modChunkRowSets[i].lastRowKey()) { + final ChunkInputStreamGenerator generator = mcd.data.generators[i]; + + final long shiftAmount = -mcd.modChunkRowSets[i].firstRowKey(); + + // get an offset rowset for each chunk in the set + try (final WritableRowSet adjustedOffsets = myModOffsets.intersect(mcd.modChunkRowSets[i])) { + // normalize this to the chunk offsets + adjustedOffsets.shiftInPlace(shiftAmount); + + final ChunkInputStreamGenerator.DrainableColumn drainableColumn = + generator.getInputStream(view.options(), adjustedOffsets); + drainableColumn.visitFieldNodes(fieldNodeListener); + drainableColumn.visitBuffers(bufferListener); + + // Add the drainable last as it is allowed to immediately close a row set the visitors need + addStream.accept(drainableColumn); + } + // no need to test other chunks + columnWritten = true; } } + Assert.assertion(columnWritten, "appendModColumns - range not contained in a single chunk"); } finally { myModOffsets.close(); } From 238226b0a1569ecb7723c784ae8a67ec5b3169cf Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Fri, 22 Apr 2022 07:36:56 -0700 Subject: [PATCH 31/47] corrected bug in BSG column generation --- .../barrage/BarrageStreamGenerator.java | 38 +++++++++++++++---- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java index 076a40fe18c..ca6f77505e6 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java @@ -794,14 +794,14 @@ private long appendAddColumns(final View view, final long startRange, final long // iterate through each chunk and find the one that contains the requested data for (int i = 0; i < chunkSetGen.size() && !columnWritten; ++i) { - // if the set of data is empty - if (addChunkRowSets[i].isEmpty() || startRange <= addChunkRowSets[i].lastRowKey()) { + // if the set of data is empty or the requested rows start in this chunk + if (startRange <= addChunkRowSets[i].lastRowKey()) { final ChunkInputStreamGenerator generator = chunkSetGen.generators[i]; // shift this into the chunk position space final long shiftAmount = -addChunkRowSets[i].firstRowKey(); - // get an offset RowSet for each chunk in the set + // get an offset RowSet for each row in this chunk try (final WritableRowSet adjustedOffsets = myAddedOffsets.intersect(addChunkRowSets[i])) { // normalize this to the chunk offsets adjustedOffsets.shiftInPlace(shiftAmount); @@ -820,7 +820,18 @@ private long appendAddColumns(final View view, final long startRange, final long columnWritten = true; } } - Assert.assertion(columnWritten, "appendAddColumns - range not contained in a single chunk"); + // must write column data, just write an empty column + if (!columnWritten) { + try (final RowSet empty = RowSetFactory.empty()){ + final ChunkInputStreamGenerator.DrainableColumn drainableColumn = + chunkSetGen.generators[0].getInputStream(view.options(), empty); + drainableColumn.visitFieldNodes(fieldNodeListener); + drainableColumn.visitBuffers(bufferListener); + + // Add the drainable last as it is allowed to immediately close a row set the visitors need + addStream.accept(drainableColumn); + } + } } return myAddedOffsets.size(); } @@ -852,14 +863,14 @@ private long appendModColumns(final View view, final long startRange, final long try { boolean columnWritten = false; - // iterate through each chunk + // iterate through each chunk and find the one that contains the requested data for (int i = 0; i < mcd.data.size() && !columnWritten; ++i) { - if (mcd.modChunkRowSets[i].isEmpty() || startRange <= mcd.modChunkRowSets[i].lastRowKey()) { + if (startRange <= mcd.modChunkRowSets[i].lastRowKey()) { final ChunkInputStreamGenerator generator = mcd.data.generators[i]; final long shiftAmount = -mcd.modChunkRowSets[i].firstRowKey(); - // get an offset rowset for each chunk in the set + // get an offset rowset for each row in this chunk try (final WritableRowSet adjustedOffsets = myModOffsets.intersect(mcd.modChunkRowSets[i])) { // normalize this to the chunk offsets adjustedOffsets.shiftInPlace(shiftAmount); @@ -876,7 +887,18 @@ private long appendModColumns(final View view, final long startRange, final long columnWritten = true; } } - Assert.assertion(columnWritten, "appendModColumns - range not contained in a single chunk"); + // must write column data, just write an empty column + if (!columnWritten) { + try (final RowSet empty = RowSetFactory.empty()){ + final ChunkInputStreamGenerator.DrainableColumn drainableColumn = + mcd.data.generators[0].getInputStream(view.options(), empty); + drainableColumn.visitFieldNodes(fieldNodeListener); + drainableColumn.visitBuffers(bufferListener); + + // Add the drainable last as it is allowed to immediately close a row set the visitors need + addStream.accept(drainableColumn); + } + } } finally { myModOffsets.close(); } From 1b52162fdd3106b085de54e1beca10d7199741dc Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Fri, 22 Apr 2022 12:03:14 -0700 Subject: [PATCH 32/47] output message size enforcing --- .../barrage/BarrageStreamGenerator.java | 178 +++++++++++++----- 1 file changed, 134 insertions(+), 44 deletions(-) diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java index ca6f77505e6..6631528309f 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java @@ -10,9 +10,10 @@ import com.google.protobuf.WireFormat; import gnu.trove.list.array.TIntArrayList; import io.deephaven.UncheckedDeephavenException; -import io.deephaven.barrage.flatbuf.*; -import io.deephaven.base.verify.Assert; -import io.deephaven.chunk.Chunk; +import io.deephaven.barrage.flatbuf.BarrageMessageType; +import io.deephaven.barrage.flatbuf.BarrageMessageWrapper; +import io.deephaven.barrage.flatbuf.BarrageModColumnMetadata; +import io.deephaven.barrage.flatbuf.BarrageUpdateMetadata; import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.WritableChunk; import io.deephaven.chunk.WritableLongChunk; @@ -20,16 +21,12 @@ import io.deephaven.chunk.sized.SizedChunk; import io.deephaven.chunk.sized.SizedLongChunk; import io.deephaven.configuration.Configuration; -import io.deephaven.engine.rowset.RowSet; -import io.deephaven.engine.rowset.RowSetBuilderSequential; -import io.deephaven.engine.rowset.RowSetFactory; -import io.deephaven.engine.rowset.RowSetShiftData; -import io.deephaven.engine.rowset.WritableRowSet; +import io.deephaven.engine.rowset.*; import io.deephaven.engine.rowset.impl.ExternalizableRowSetUtils; import io.deephaven.engine.table.TableDefinition; import io.deephaven.engine.table.impl.util.BarrageMessage; -import io.deephaven.extensions.barrage.BarrageSubscriptionOptions; import io.deephaven.extensions.barrage.BarrageSnapshotOptions; +import io.deephaven.extensions.barrage.BarrageSubscriptionOptions; import io.deephaven.extensions.barrage.chunk.ChunkInputStreamGenerator; import io.deephaven.extensions.barrage.util.BarrageProtoUtil.ExposedByteArrayOutputStream; import io.deephaven.extensions.barrage.util.BarrageUtil; @@ -40,6 +37,7 @@ import io.deephaven.proto.flight.util.MessageHelper; import io.deephaven.util.SafeCloseable; import io.deephaven.util.datastructures.LongSizedDataStructure; +import io.deephaven.util.datastructures.SizeException; import io.grpc.Drainable; import org.apache.arrow.flatbuf.Buffer; import org.apache.arrow.flatbuf.FieldNode; @@ -55,7 +53,10 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; -import java.util.*; +import java.util.ArrayDeque; +import java.util.BitSet; +import java.util.Map; +import java.util.Objects; import java.util.function.Consumer; import static io.deephaven.engine.table.impl.sources.InMemoryColumnSource.TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD; @@ -69,6 +70,10 @@ public class BarrageStreamGenerator implements private static final int DEFAULT_BATCH_SIZE = Configuration.getInstance() .getIntegerForClassWithDefault(BarrageStreamGenerator.class, "batchSize", Integer.MAX_VALUE); + // default to 100MB to match java-client and w2w incoming limits + private static final int DEFAULT_MESSAGE_SIZE_LIMIT = Configuration.getInstance() + .getIntegerForClassWithDefault(BarrageStreamGenerator.class, "messageSizeLimit", 100 * 1024 * 1024); + public interface View { void forEachStream(Consumer visitor) throws IOException; @@ -383,14 +388,15 @@ public void forEachStream(Consumer visitor) throws IOException { long bytesWritten = 0; ByteBuffer metadata = generator.getSubscriptionMetadata(this); - // batch size is maximum, can go smaller when needed - final int batchSize = batchSize(); + // batch size is maximum, will write fewer when needed. If we are enforcing a message size limit, then + // reasonably we will require at least one byte per row and restrict rows accordingly + int batchSizeLimit = Math.min(DEFAULT_MESSAGE_SIZE_LIMIT, batchSize()); // there may be multiple chunk generators and we need to ensure that a single batch rowset does not cross // boundaries and that we honor the batch size requests if (numAddRows == 0 && numModRows == 0) { - // we still need to send a message containing just the metadata + // we still need to send a message containing metadata when there are no rows final InputStream is = generator.getInputStream( this, 0, 0, metadata, generator::appendAddColumns); bytesWritten += is.available(); @@ -400,43 +406,98 @@ public void forEachStream(Consumer visitor) throws IOException { long chunkOffset = 0; while (offset < numAddRows) { - long effectiveBatchSize = batchSize; + long rowsRemaining = numAddRows - offset; + + int batchSize = (int) Math.min(rowsRemaining, batchSizeLimit); // make sure we are not about to cross a chunk boundary if (chunkOffset + batchSize > TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD) { - effectiveBatchSize = TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD - chunkOffset; + batchSize = (int) (TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD - chunkOffset); chunkOffset = 0; } - final InputStream is = generator.getInputStream( - this, offset, offset + effectiveBatchSize, metadata, generator::appendAddColumns); - bytesWritten += is.available(); - visitor.accept(is); + try { + final InputStream is = generator.getInputStream( + this, offset, offset + batchSize, metadata, generator::appendAddColumns); + int bytesToWrite = is.available(); + + // treat this as a hard limit, exceeding fails a client or w2w (unless we are sending a single + // row then we must send and let it potentially fail) + if (bytesToWrite > DEFAULT_MESSAGE_SIZE_LIMIT && batchSize > 1) { + // can't write this, so close the input stream and retry + is.close(); + } else { + bytesWritten += bytesToWrite; + + // let's write the data + visitor.accept(is); - offset += effectiveBatchSize; - chunkOffset += effectiveBatchSize; - metadata = null; + offset += batchSize; + chunkOffset += batchSize; + metadata = null; + } + // recompute the batch limit for the next message (or retry) + int bytesPerRow = bytesToWrite / batchSize; + if (bytesPerRow > 0) { + int rowLimit = DEFAULT_MESSAGE_SIZE_LIMIT / bytesPerRow; + + // add some margin for abnormal cell contents + batchSizeLimit = Math.min(batchSize(), Math.max(1, (int) ((double) rowLimit * 0.9))); + } + } catch (SizeException ex) { + // was an overflow in the ChunkInputStream generator (probably VarBinary). We can't compute the + // correct number of rows from this failure, so cut batch size in half and try again. This may + // occur multiple times until the size is restricted properly + batchSizeLimit = Math.max(1, batchSizeLimit / 2); + } } offset = 0; chunkOffset = 0; while (offset < numModRows) { - long effectiveBatchSize = batchSize; + long rowsRemaining = numModRows - offset; + + int batchSize = (int) Math.min(rowsRemaining, batchSizeLimit); // make sure we are not about to cross a chunk boundary if (chunkOffset + batchSize > TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD) { - effectiveBatchSize = TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD - chunkOffset; + batchSize = (int) (TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD - chunkOffset); chunkOffset = 0; } - final InputStream is = generator.getInputStream( - this, offset, offset + effectiveBatchSize, metadata, generator::appendModColumns); - bytesWritten += is.available(); - visitor.accept(is); + try { + final InputStream is = generator.getInputStream( + this, offset, offset + batchSize, metadata, generator::appendModColumns); + int bytesToWrite = is.available(); + + // treat this as a hard limit, exceeding will fail a client or w2w + if (bytesToWrite > DEFAULT_MESSAGE_SIZE_LIMIT) { + // can't write this, so close the input stream and retry + is.close(); + } else { + bytesWritten += bytesToWrite; - offset += effectiveBatchSize; - chunkOffset += effectiveBatchSize; - metadata = null; + // let's write the data + visitor.accept(is); + + offset += batchSize; + chunkOffset += batchSize; + metadata = null; + } + // recompute the batch limit for the next message (or retry) + int bytesPerRow = bytesToWrite / batchSize; + if (bytesPerRow > 0) { + int rowLimit = DEFAULT_MESSAGE_SIZE_LIMIT / bytesPerRow; + + // add some margin for abnormal cell contents + batchSizeLimit = Math.min(batchSize(), Math.max(1, (int) ((double) rowLimit * 0.9))); + } + } catch (SizeException ex) { + // was an overflow in the ChunkInputStream generator (probably VarBinary). We can't compute the + // correct number of rows from this failure, so cut batch size in half and try again. This may + // occur multiple times until the size is restricted properly + batchSizeLimit = Math.max(1, batchSizeLimit / 2); + } } } @@ -550,10 +611,13 @@ public SnapshotView(final BarrageStreamGenerator generator, @Override public void forEachStream(Consumer visitor) throws IOException { ByteBuffer metadata = generator.getSnapshotMetadata(this); - final long batchSize = batchSize(); + + // batch size is maximum, will write fewer when needed. If we are enforcing a message size limit, then + // reasonably we will require at least one byte per row and restrict rows accordingly + int batchSizeLimit = Math.min(DEFAULT_MESSAGE_SIZE_LIMIT, batchSize()); if (numAddRows == 0) { - // we still need to send a message containing just the metadata + // we still need to send a message containing metadata when there are no rows visitor.accept(generator.getInputStream( this, 0, 0, metadata, generator::appendAddColumns)); } else { @@ -561,20 +625,48 @@ public void forEachStream(Consumer visitor) throws IOException { long chunkOffset = 0; while (offset < numAddRows) { - long effectiveBatchSize = batchSize; + long rowsRemaining = numAddRows - offset; + + int batchSize = (int) Math.min(rowsRemaining, batchSizeLimit); // make sure we are not about to cross a chunk boundary if (chunkOffset + batchSize > TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD) { - effectiveBatchSize = TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD - chunkOffset; + batchSize = (int) (TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD - chunkOffset); chunkOffset = 0; } - visitor.accept(generator.getInputStream( - this, offset, offset + effectiveBatchSize, metadata, generator::appendAddColumns)); + try { + final InputStream is = generator.getInputStream( + this, offset, offset + batchSize, metadata, generator::appendAddColumns); + int bytesToWrite = is.available(); - offset += effectiveBatchSize; - chunkOffset += effectiveBatchSize; - metadata = null; + // treat this as a hard limit, exceeding fails a client or w2w (unless we are sending a single + // row then we must send and let it potentially fail) + if (bytesToWrite > DEFAULT_MESSAGE_SIZE_LIMIT && batchSize > 1) { + // can't write this, so close the input stream and retry + is.close(); + } else { + // let's write the data + visitor.accept(is); + + offset += batchSize; + chunkOffset += batchSize; + metadata = null; + } + // recompute the batch limit for the next message (or retry) + int bytesPerRow = bytesToWrite / batchSize; + if (bytesPerRow > 0) { + int rowLimit = DEFAULT_MESSAGE_SIZE_LIMIT / bytesPerRow; + + // add some margin for abnormal cell contents + batchSizeLimit = Math.min(batchSize(), Math.max(1, (int) ((double) rowLimit * 0.9))); + } + } catch (SizeException ex) { + // was an overflow in the ChunkInputStream generator (probably VarBinary). We can't compute the + // correct number of rows from this failure, so cut batch size in half and try again. This may + // occur multiple times until the size is restricted properly + batchSizeLimit = Math.max(1, batchSizeLimit / 2); + } } } @@ -813,8 +905,6 @@ private long appendAddColumns(final View view, final long startRange, final long // Add the drainable last as it is allowed to immediately close a row set the visitors need addStream.accept(drainableColumn); - } catch (Exception ex) { - System.out.println(ex.getMessage()); } // no need to test other chunks columnWritten = true; @@ -822,7 +912,7 @@ private long appendAddColumns(final View view, final long startRange, final long } // must write column data, just write an empty column if (!columnWritten) { - try (final RowSet empty = RowSetFactory.empty()){ + try (final RowSet empty = RowSetFactory.empty()) { final ChunkInputStreamGenerator.DrainableColumn drainableColumn = chunkSetGen.generators[0].getInputStream(view.options(), empty); drainableColumn.visitFieldNodes(fieldNodeListener); @@ -889,7 +979,7 @@ private long appendModColumns(final View view, final long startRange, final long } // must write column data, just write an empty column if (!columnWritten) { - try (final RowSet empty = RowSetFactory.empty()){ + try (final RowSet empty = RowSetFactory.empty()) { final ChunkInputStreamGenerator.DrainableColumn drainableColumn = mcd.data.generators[0].getInputStream(view.options(), empty); drainableColumn.visitFieldNodes(fieldNodeListener); From cdd15bb1819b78ddd16575817283a83a1fd6d535 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Fri, 22 Apr 2022 12:11:27 -0700 Subject: [PATCH 33/47] renamed config item to be consisten with others --- .../io/deephaven/server/barrage/BarrageStreamGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java index 6631528309f..382af1f8913 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java @@ -72,7 +72,7 @@ public class BarrageStreamGenerator implements // default to 100MB to match java-client and w2w incoming limits private static final int DEFAULT_MESSAGE_SIZE_LIMIT = Configuration.getInstance() - .getIntegerForClassWithDefault(BarrageStreamGenerator.class, "messageSizeLimit", 100 * 1024 * 1024); + .getIntegerForClassWithDefault(BarrageStreamGenerator.class, "maxOutboundMessageSize", 100 * 1024 * 1024); public interface View { void forEachStream(Consumer visitor) throws IOException; From e5c4ac714e7e2948fd26982eb1f7cbd5e1f710dc Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Mon, 25 Apr 2022 10:01:33 -0700 Subject: [PATCH 34/47] better comments --- .../server/barrage/BarrageStreamGenerator.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java index 382af1f8913..d41d8b3d877 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java @@ -388,7 +388,7 @@ public void forEachStream(Consumer visitor) throws IOException { long bytesWritten = 0; ByteBuffer metadata = generator.getSubscriptionMetadata(this); - // batch size is maximum, will write fewer when needed. If we are enforcing a message size limit, then + // batch size is maximum, will write fewer rows when needed. If we are enforcing a message size limit, then // reasonably we will require at least one byte per row and restrict rows accordingly int batchSizeLimit = Math.min(DEFAULT_MESSAGE_SIZE_LIMIT, batchSize()); @@ -436,7 +436,7 @@ public void forEachStream(Consumer visitor) throws IOException { chunkOffset += batchSize; metadata = null; } - // recompute the batch limit for the next message (or retry) + // recompute the batch limit for the next message int bytesPerRow = bytesToWrite / batchSize; if (bytesPerRow > 0) { int rowLimit = DEFAULT_MESSAGE_SIZE_LIMIT / bytesPerRow; @@ -484,7 +484,7 @@ public void forEachStream(Consumer visitor) throws IOException { chunkOffset += batchSize; metadata = null; } - // recompute the batch limit for the next message (or retry) + // recompute the batch limit for the next message int bytesPerRow = bytesToWrite / batchSize; if (bytesPerRow > 0) { int rowLimit = DEFAULT_MESSAGE_SIZE_LIMIT / bytesPerRow; @@ -612,7 +612,7 @@ public SnapshotView(final BarrageStreamGenerator generator, public void forEachStream(Consumer visitor) throws IOException { ByteBuffer metadata = generator.getSnapshotMetadata(this); - // batch size is maximum, will write fewer when needed. If we are enforcing a message size limit, then + // batch size is maximum, will write fewer rows when needed. If we are enforcing a message size limit, then // reasonably we will require at least one byte per row and restrict rows accordingly int batchSizeLimit = Math.min(DEFAULT_MESSAGE_SIZE_LIMIT, batchSize()); @@ -653,7 +653,7 @@ public void forEachStream(Consumer visitor) throws IOException { chunkOffset += batchSize; metadata = null; } - // recompute the batch limit for the next message (or retry) + // recompute the batch limit for the next message int bytesPerRow = bytesToWrite / batchSize; if (bytesPerRow > 0) { int rowLimit = DEFAULT_MESSAGE_SIZE_LIMIT / bytesPerRow; From 02c14712711e9890986c2f3bd48ed2c64209b05d Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Fri, 29 Apr 2022 14:18:26 -0700 Subject: [PATCH 35/47] PR comments partially addressed --- .../table/impl/remote/ConstructSnapshot.java | 33 +- .../table/impl/util/BarrageMessage.java | 3 + .../barrage/table/BarrageTable.java | 108 +++-- .../barrage/util/BarrageStreamReader.java | 89 ++-- .../client/examples/SubscribeExampleBase.java | 4 +- .../barrage/BarrageMessageProducer.java | 132 ++++-- .../barrage/BarrageStreamGenerator.java | 434 +++++++----------- 7 files changed, 379 insertions(+), 424 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java index 068cc89750a..67b6e4d31db 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java @@ -1344,6 +1344,7 @@ public static boolean serializeAllTable(final boolean usePrev, getSnapshotDataAsChunkList(columnSource, columnIsEmpty ? null : sharedContext, rows, usePrev); acd.type = columnSource.getType(); acd.componentType = columnSource.getComponentType(); + acd.chunkType = columnSource.getChunkType(); final BarrageMessage.ModColumnData mcd = new BarrageMessage.ModColumnData(); snapshot.modColumnData[ii] = mcd; @@ -1351,6 +1352,7 @@ public static boolean serializeAllTable(final boolean usePrev, mcd.data = getSnapshotDataAsChunkList(columnSource, null, RowSetFactory.empty(), usePrev); mcd.type = acd.type; mcd.componentType = acd.componentType; + mcd.chunkType = columnSource.getChunkType(); } } @@ -1438,27 +1440,17 @@ private static ArrayList> getSnapshotDataAsChunkList(final Col final long size = rowSet.size(); final ArrayList> result = new ArrayList<>(); - if (rowSet.size() == 0) { - result.add(sourceToUse.getChunkType().getEmptyChunk()); + if (size == 0) { return result; } - while (offset < size) { - final int chunkSize; - if (size - offset > TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD) { - chunkSize = TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD; - } else { - chunkSize = (int) (size - offset); - } - - final long keyStart = rowSet.get(offset); - final long keyEnd = rowSet.get(offset + chunkSize - 1); - - try (final ColumnSource.FillContext context = sharedContext != null - ? sourceToUse.makeFillContext(chunkSize, sharedContext) - : sourceToUse.makeFillContext(chunkSize); - final WritableRowSet reducedRowSet = rowSet.subSetByKeyRange(keyStart, keyEnd)) { + final int initialChunkSize = (int) Math.min(size, TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD); + try (final ColumnSource.FillContext context = sourceToUse.makeFillContext(initialChunkSize, sharedContext); + final RowSequence.Iterator it = rowSet.getRowSequenceIterator()) { + int chunkSize = initialChunkSize; + while (it.hasMore()) { + final RowSequence reducedRowSet = it.getNextRowSequenceWithLength(chunkSize); final ChunkType chunkType = sourceToUse.getChunkType(); // create a new chunk @@ -1475,6 +1467,13 @@ private static ArrayList> getSnapshotDataAsChunkList(final Col // increment the offset for the next chunk (using the actual values written) offset += currentChunk.size(); + + // recompute the size of the next chunk + if (size - offset > TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD) { + chunkSize = TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD; + } else { + chunkSize = (int) (size - offset); + } } } return result; diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/util/BarrageMessage.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/util/BarrageMessage.java index 4ac53b93e8b..fb331bce18f 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/util/BarrageMessage.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/util/BarrageMessage.java @@ -5,6 +5,7 @@ package io.deephaven.engine.table.impl.util; import io.deephaven.chunk.Chunk; +import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.attributes.Values; import io.deephaven.chunk.util.pools.PoolableChunk; import io.deephaven.engine.rowset.RowSet; @@ -30,12 +31,14 @@ public static class ModColumnData { public Class type; public Class componentType; public ArrayList> data; + public ChunkType chunkType; } public static class AddColumnData { public Class type; public Class componentType; public ArrayList> data; + public ChunkType chunkType; } public long firstSeq = -1; diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java index 548744193dc..6e1af76eb4d 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java @@ -318,10 +318,14 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC totalMods.insert(column.rowsModified); } + // perform the addition operations in batches for efficiency + final int addBatchSize = (int) Math.min(update.rowsIncluded.size(), + 1 << ChunkPoolConstants.LARGEST_POOLED_CHUNK_LOG2_CAPACITY); + if (update.rowsIncluded.isNonempty()) { if (mightBeInitialSnapshot) { // ensure the data sources have at least the incoming capacity. The sources can auto-resize but - // we know the initial snapshot size and resize immediately + // we know the initial snapshot size and can resize immediately capacity = update.rowsIncluded.size(); for (final WritableColumnSource source : destSources) { source.ensureCapacity(capacity); @@ -332,40 +336,44 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC // this will hold all the free rows allocated for the included rows final WritableRowSet destinationRowSet = RowSetFactory.empty(); - // update the rowRedirection with the rowsIncluded set (in manageable batch sizes) - int maxChunkSize = 1 << ChunkPoolConstants.LARGEST_POOLED_CHUNK_LOG2_CAPACITY; - try (final RowSequence.Iterator rowsIncludedIterator = update.rowsIncluded.getRowSequenceIterator()) { + // update the table with the rowsIncluded set (in manageable batch sizes) + try (final RowSequence.Iterator rowsIncludedIterator = update.rowsIncluded.getRowSequenceIterator(); + final ChunkSink.FillFromContext redirContext = + rowRedirection.makeFillFromContext(addBatchSize)) { while (rowsIncludedIterator.hasMore()) { - final RowSequence chunkRowsToFree = - rowsIncludedIterator.getNextRowSequenceWithLength(maxChunkSize); - try (final ChunkSink.FillFromContext redirContext = - rowRedirection.makeFillFromContext(chunkRowsToFree.intSize()); - final RowSet newRows = getFreeRows(chunkRowsToFree.intSize());) { - + final RowSequence rowsToRedirect = + rowsIncludedIterator.getNextRowSequenceWithLength(addBatchSize); + try (final RowSet newRows = getFreeRows(rowsToRedirect.intSize())) { // Update redirection mapping: - rowRedirection.fillFromChunk(redirContext, newRows.asRowKeyChunk(), chunkRowsToFree); - + rowRedirection.fillFromChunk(redirContext, newRows.asRowKeyChunk(), rowsToRedirect); // add these rows to the final destination set destinationRowSet.insert(newRows); } } } - // update the column sources + // update the column sources (in manageable batch sizes) for (int ii = 0; ii < update.addColumnData.length; ++ii) { if (isSubscribedColumn(ii)) { final BarrageMessage.AddColumnData column = update.addColumnData[ii]; - // grab the matching rows from each chunk - long offset = 0; - for (int chunkIndex = 0; chunkIndex < column.data.size(); ++chunkIndex) { - final Chunk chunk = column.data.get(chunkIndex); - try (final RowSet chunkRows = RowSetFactory.fromRange(offset, offset + chunk.size() - 1); - final RowSet chunkDestSet = destinationRowSet.subSetForPositions(chunkRows); - final ChunkSink.FillFromContext ctxt = - destSources[ii].makeFillFromContext(chunkDestSet.intSize())) { - destSources[ii].fillFromChunk(ctxt, chunk, chunkDestSet); + try (final ChunkSink.FillFromContext fillContext = + destSources[ii].makeFillFromContext(addBatchSize); + final RowSequence.Iterator destIterator = destinationRowSet.getRowSequenceIterator()) { + // grab the matching rows from each chunk + for (final Chunk chunk : column.data) { + // track where we are in the current chunk + int chunkOffset = 0; + while (chunkOffset < chunk.size()) { + // don't overrun the chunk boundary + int effectiveBatchSize = Math.min(addBatchSize, chunk.size() - chunkOffset); + final RowSequence chunkKeys = + destIterator.getNextRowSequenceWithLength(effectiveBatchSize); + Chunk slicedChunk = chunk.slice(chunkOffset, effectiveBatchSize); + destSources[ii].fillFromChunk(fillContext, slicedChunk, chunkKeys); + chunkOffset += effectiveBatchSize; + } } - offset += chunk.size(); + Assert.assertion(!destIterator.hasMore(), "not all rowsIncluded were processed"); } } } @@ -378,32 +386,34 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC continue; } + // perform the modification operations in batches for efficiency + final int modBatchSize = (int) Math.min(column.rowsModified.size(), + 1 << ChunkPoolConstants.LARGEST_POOLED_CHUNK_LOG2_CAPACITY); modifiedColumnSet.setColumnWithIndex(ii); - // grab the matching rows from each chunk - long offset = 0; - for (int chunkIndex = 0; chunkIndex < column.data.size(); ++chunkIndex) { - final Chunk chunk = column.data.get(chunkIndex); - try (final RowSet chunkRows = RowSetFactory.fromRange(offset, offset + chunk.size() - 1); - final ChunkSource.FillContext redirContext = - rowRedirection.makeFillContext(chunkRows.intSize(), null); - final WritableLongChunk keys = - WritableLongChunk.makeWritableChunk(chunkRows.intSize()); - final RowSet chunkKeys = column.rowsModified.subSetForPositions(chunkRows)) { - - // fill the key chunk with the keys from this chunk - rowRedirection.fillChunk(redirContext, keys, chunkKeys); - for (int i = 0; i < keys.size(); ++i) { - Assert.notEquals(keys.get(i), "keys[i]", RowSequence.NULL_ROW_KEY, "RowSet.NULL_ROW_KEY"); - } - - // fill the column with the data from this chunk - try (final ChunkSink.FillFromContext ctxt = - destSources[ii].makeFillFromContext(keys.size())) { - destSources[ii].fillFromChunkUnordered(ctxt, chunk, keys); + try (final ChunkSource.FillContext redirContext = rowRedirection.makeFillContext(modBatchSize, null); + final ChunkSink.FillFromContext fillContext = destSources[ii].makeFillFromContext(modBatchSize); + final WritableLongChunk keys = WritableLongChunk.makeWritableChunk(modBatchSize); + final RowSequence.Iterator destIterator = column.rowsModified.getRowSequenceIterator()) { + + // grab the matching rows from each chunk + for (final Chunk chunk : column.data) { + // track where we are in the current chunk + int chunkOffset = 0; + while (chunkOffset < chunk.size()) { + // don't overrun the chunk boundary + int effectiveBatchSize = Math.min(modBatchSize, chunk.size() - chunkOffset); + final RowSequence chunkKeys = destIterator.getNextRowSequenceWithLength(effectiveBatchSize); + // fill the key chunk with the keys from this rowset + rowRedirection.fillChunk(redirContext, keys, chunkKeys); + Chunk slicedChunk = chunk.slice(chunkOffset, effectiveBatchSize); + + destSources[ii].fillFromChunkUnordered(fillContext, slicedChunk, keys); + + chunkOffset += effectiveBatchSize; } } - offset += chunk.size(); + Assert.assertion(!destIterator.hasMore(), "not all rowsModified were processed"); } } @@ -438,15 +448,15 @@ private RowSet getFreeRows(long size) { if (size <= 0) { return RowSetFactory.empty(); } - boolean needsResizing = false; if (capacity == 0) { - capacity = Integer.highestOneBit((int) Math.max(size * 2, 8)); + capacity = Long.highestOneBit(Math.max(size * 2, 8)); freeset = RowSetFactory.flat(capacity); needsResizing = true; } else if (freeset.size() < size) { - int usedSlots = (int) (capacity - freeset.size()); + long usedSlots = capacity - freeset.size(); long prevCapacity = capacity; + do { capacity *= 2; } while ((capacity - usedSlots) < size); @@ -460,7 +470,7 @@ private RowSet getFreeRows(long size) { } } - final RowSet result = freeset.subSetByPositionRange(0, (int) size); + final RowSet result = freeset.subSetByPositionRange(0, size); Assert.assertion(result.size() == size, "result.size() == size"); freeset.removeRange(0, result.lastRowKey()); return result; diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java index e504df3f548..f65393bc0a6 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java @@ -241,7 +241,7 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, // add and mod rows are never combined in a batch. all added rows must be received before the first // mod rows will be received - if (numAddRowsRead < numAddRowsTotal) { + if (numAddRowsRead < numAddRowsTotal || (numAddRowsTotal == 0 && numModRowsTotal == 0)) { for (int ci = 0; ci < msg.addColumnData.length; ++ci) { final BarrageMessage.AddColumnData acd = msg.addColumnData[ci]; @@ -252,36 +252,27 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, int chunkSize = chunk.size(); final int chunkOffset; - if (chunkSize == 0) { + long rowOffset = numAddRowsRead - lastAddStartIndex; + // reading the rows from this batch might overflow the existing chunk + if (rowOffset + batch.length() > chunkSize) { + lastAddStartIndex += chunkSize; + + // create a new chunk before trying to write again + chunkSize = (int) (Math.min(msg.rowsIncluded.size() - numAddRowsRead, + TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); + chunk = columnChunkTypes[ci].makeWritableChunk(chunkSize); + acd.data.add(chunk); + chunkOffset = 0; } else { - long rowOffset = numAddRowsRead - lastAddStartIndex; - // reading the rows from this batch might overflow the existing chunk - if (rowOffset + batch.length() > chunkSize) { - lastAddStartIndex += chunkSize; - - // create a new chunk before trying to write again - chunkSize = (int) (Math.min(msg.rowsIncluded.size() - numAddRowsRead, - TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); - chunk = columnChunkTypes[ci].makeWritableChunk(chunkSize); - acd.data.add(chunk); - - chunkOffset = 0; - } else { - chunkOffset = (int) rowOffset; - } + chunkOffset = (int) rowOffset; } - // fill the chunk, but catch overrun exceptions - try { - chunk = ChunkInputStreamGenerator.extractChunkFromInputStream(options, - columnChunkTypes[ci], columnTypes[ci], componentTypes[ci], fieldNodeIter, - bufferInfoIter, ois, chunk, - chunkOffset, chunkSize); - } catch (Exception ex) { - // Should never happen, might be able to remove this exception handler - System.out.println(ex.toString()); - } + // fill the chunk with data and assign back into the array + acd.data.set(lastChunkIndex, + ChunkInputStreamGenerator.extractChunkFromInputStream(options, columnChunkTypes[ci], + columnTypes[ci], componentTypes[ci], fieldNodeIter, bufferInfoIter, ois, + chunk, chunkOffset, chunkSize)); } numAddRowsRead += batch.length(); } else { @@ -295,37 +286,27 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, int chunkSize = chunk.size(); final int chunkOffset; - if (chunkSize == 0) { + long remaining = mcd.rowsModified.size() - numModRowsRead; + long rowOffset = numModRowsRead - lastModStartIndex; + // this batch might overflow the chunk + if (rowOffset + Math.min(remaining, batch.length()) > chunkSize) { + lastModStartIndex += chunkSize; + + // create a new chunk before trying to write again + chunkSize = (int) (Math.min(remaining, TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); + chunk = columnChunkTypes[ci].makeWritableChunk(chunkSize); + mcd.data.add(chunk); + chunkOffset = 0; } else { - long remaining = mcd.rowsModified.size() - numModRowsRead; - long rowOffset = numModRowsRead - lastModStartIndex; - // this batch might overflow the chunk - if (rowOffset + Math.min(remaining, batch.length()) > chunkSize) { - lastModStartIndex += chunkSize; - - // create a new chunk before trying to write again - chunkSize = (int) (Math.min(remaining, TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); - chunk = columnChunkTypes[ci].makeWritableChunk(chunkSize); - mcd.data.add(chunk); - - chunkOffset = 0; - } else { - chunkOffset = (int) rowOffset; - } + chunkOffset = (int) rowOffset; } - // fill the chunk, but catch overrun exceptions - try { - chunk = ChunkInputStreamGenerator.extractChunkFromInputStream(options, - columnChunkTypes[ci], columnTypes[ci], componentTypes[ci], fieldNodeIter, - bufferInfoIter, ois, chunk, - chunkOffset, chunkSize); - - } catch (Exception ex) { - // Should never happen, might be able to remove this exception handler - System.out.println(ex.toString()); - } + // fill the chunk with data and assign back into the array + mcd.data.set(lastChunkIndex, + ChunkInputStreamGenerator.extractChunkFromInputStream(options, columnChunkTypes[ci], + columnTypes[ci], componentTypes[ci], fieldNodeIter, bufferInfoIter, ois, + chunk, chunkOffset, chunkSize)); } numModRowsRead += batch.length(); } diff --git a/java-client/barrage-examples/src/main/java/io/deephaven/client/examples/SubscribeExampleBase.java b/java-client/barrage-examples/src/main/java/io/deephaven/client/examples/SubscribeExampleBase.java index a569a2e5ff3..9790c199d1f 100644 --- a/java-client/barrage-examples/src/main/java/io/deephaven/client/examples/SubscribeExampleBase.java +++ b/java-client/barrage-examples/src/main/java/io/deephaven/client/examples/SubscribeExampleBase.java @@ -25,10 +25,10 @@ abstract class SubscribeExampleBase extends BarrageClientExampleBase { TableUpdateListener listener; @CommandLine.Option(names = {"--tail"}, required = false, description = "Tail viewport size") - int tailSize = 0; + long tailSize = 0; @CommandLine.Option(names = {"--head"}, required = false, description = "Header viewport size") - int headerSize = 0; + long headerSize = 0; static class Mode { @CommandLine.Option(names = {"-b", "--batch"}, required = true, description = "Batch mode") diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index ccdf3653f64..0d9ae337d17 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -49,6 +49,8 @@ import io.deephaven.util.datastructures.LongSizedDataStructure; import io.grpc.StatusRuntimeException; import io.grpc.stub.StreamObserver; +import org.apache.commons.lang3.mutable.MutableInt; +import org.apache.commons.lang3.mutable.MutableLong; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.HdrHistogram.Histogram; @@ -62,6 +64,8 @@ import java.util.function.Function; import java.util.function.IntFunction; +import static io.deephaven.engine.table.impl.sources.InMemoryColumnSource.TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD; + /** * The server-side implementation of a Barrage replication source. * @@ -1620,19 +1624,24 @@ private BarrageMessage aggregateUpdatesInRange(final int startDelta, final int e final ColumnSource deltaColumn = deltaColumns[ci]; final BarrageMessage.AddColumnData adds = new BarrageMessage.AddColumnData(); adds.data = new ArrayList<>(); + adds.chunkType = deltaColumn.getChunkType(); downstream.addColumnData[ci] = adds; if (addColumnSet.get(ci)) { - final int chunkCapacity = localAdded.intSize("serializeItems"); - final WritableChunk chunk = - deltaColumn.getChunkType().makeWritableChunk(chunkCapacity); - try (final ChunkSource.FillContext fc = deltaColumn.makeFillContext(chunkCapacity)) { - deltaColumn.fillChunk(fc, chunk, localAdded); + // create data chunk(s) for the added row data + try (final RowSequence.Iterator it = localAdded.getRowSequenceIterator()) { + while (it.hasMore()) { + final RowSequence rs = + it.getNextRowSequenceWithLength(TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD); + final int chunkCapacity = rs.intSize("serializeItems"); + final WritableChunk chunk = adds.chunkType.makeWritableChunk(chunkCapacity); + try (final ChunkSource.FillContext fc = deltaColumn.makeFillContext(chunkCapacity)) { + deltaColumn.fillChunk(fc, chunk, rs); + } + adds.data.add(chunk); + } } - adds.data.add(chunk); - } else { - adds.data.add(deltaColumn.getChunkType().getEmptyChunk()); } adds.type = deltaColumn.getType(); @@ -1643,21 +1652,27 @@ private BarrageMessage aggregateUpdatesInRange(final int startDelta, final int e final ColumnSource deltaColumn = deltaColumns[ci]; final BarrageMessage.ModColumnData mods = new BarrageMessage.ModColumnData(); mods.data = new ArrayList<>(); + mods.chunkType = deltaColumn.getChunkType(); downstream.modColumnData[ci] = mods; if (modColumnSet.get(ci)) { mods.rowsModified = firstDelta.recordedMods.copy(); - final int chunkCapacity = localModified.intSize("serializeItems"); - final WritableChunk chunk = - deltaColumn.getChunkType().makeWritableChunk(chunkCapacity); - try (final ChunkSource.FillContext fc = deltaColumn.makeFillContext(chunkCapacity)) { - deltaColumn.fillChunk(fc, chunk, localModified); + // create data chunk(s) for the added row data + try (final RowSequence.Iterator it = localModified.getRowSequenceIterator()) { + while (it.hasMore()) { + final RowSequence rs = + it.getNextRowSequenceWithLength(TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD); + final int chunkCapacity = rs.intSize("serializeItems"); + final WritableChunk chunk = mods.chunkType.makeWritableChunk(chunkCapacity); + try (final ChunkSource.FillContext fc = deltaColumn.makeFillContext(chunkCapacity)) { + deltaColumn.fillChunk(fc, chunk, rs); + } + mods.data.add(chunk); + } } - mods.data.add(chunk); } else { mods.rowsModified = RowSetFactory.empty(); - mods.data.add(deltaColumn.getChunkType().getEmptyChunk()); } mods.type = deltaColumn.getType(); @@ -1709,8 +1724,8 @@ private BarrageMessage aggregateUpdatesInRange(final int startDelta, final int e final class ColumnInfo { final WritableRowSet modified = RowSetFactory.empty(); final WritableRowSet recordedMods = RowSetFactory.empty(); - long[] addedMapping; - long[] modifiedMapping; + ArrayList addedMappings = new ArrayList<>(); + ArrayList modifiedMappings = new ArrayList<>(); } final HashMap infoCache = new HashMap<>(); @@ -1743,15 +1758,28 @@ final class ColumnInfo { retval.modified.remove(coalescer.added); retval.recordedMods.remove(coalescer.added); - retval.addedMapping = new long[localAdded.intSize()]; - retval.modifiedMapping = new long[retval.recordedMods.intSize()]; - Arrays.fill(retval.addedMapping, RowSequence.NULL_ROW_KEY); - Arrays.fill(retval.modifiedMapping, RowSequence.NULL_ROW_KEY); + try (final RowSequence.Iterator it = localAdded.getRowSequenceIterator()) { + while (it.hasMore()) { + final RowSequence rs = it.getNextRowSequenceWithLength(TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD); + long[] addedMapping = new long[rs.intSize()]; + Arrays.fill(addedMapping, RowSequence.NULL_ROW_KEY); + retval.addedMappings.add(addedMapping); + } + } + + try (final RowSequence.Iterator it = retval.recordedMods.getRowSequenceIterator()) { + while (it.hasMore()) { + final RowSequence rs = it.getNextRowSequenceWithLength(TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD); + long[] modifiedMapping = new long[rs.intSize()]; + Arrays.fill(modifiedMapping, RowSequence.NULL_ROW_KEY); + retval.modifiedMappings.add(modifiedMapping); + } + } final WritableRowSet unfilledAdds = localAdded.isEmpty() ? RowSetFactory.empty() - : RowSetFactory.fromRange(0, retval.addedMapping.length - 1); + : RowSetFactory.flat(localAdded.size()); final WritableRowSet unfilledMods = retval.recordedMods.isEmpty() ? RowSetFactory.empty() - : RowSetFactory.fromRange(0, retval.modifiedMapping.length - 1); + : RowSetFactory.flat(retval.recordedMods.size()); final WritableRowSet addedRemaining = localAdded.copy(); final WritableRowSet modifiedRemaining = retval.recordedMods.copy(); @@ -1781,7 +1809,7 @@ final class ColumnInfo { } applyRedirMapping(rowsToFill, sourceRows, - addedMapping ? retval.addedMapping : retval.modifiedMapping); + addedMapping ? retval.addedMappings : retval.modifiedMappings); } }; @@ -1825,20 +1853,20 @@ final class ColumnInfo { final ColumnSource deltaColumn = deltaColumns[ci]; final BarrageMessage.AddColumnData adds = new BarrageMessage.AddColumnData(); adds.data = new ArrayList<>(); + adds.chunkType = deltaColumn.getChunkType(); downstream.addColumnData[ci] = adds; if (addColumnSet.get(ci)) { final ColumnInfo info = getColumnInfo.apply(ci); - final WritableChunk chunk = - deltaColumn.getChunkType().makeWritableChunk(info.addedMapping.length); - try (final ChunkSource.FillContext fc = deltaColumn.makeFillContext(info.addedMapping.length)) { - ((FillUnordered) deltaColumn).fillChunkUnordered(fc, chunk, - LongChunk.chunkWrap(info.addedMapping)); + for (long[] addedMapping : info.addedMappings) { + final WritableChunk chunk = adds.chunkType.makeWritableChunk(addedMapping.length); + try (final ChunkSource.FillContext fc = deltaColumn.makeFillContext(addedMapping.length)) { + ((FillUnordered) deltaColumn).fillChunkUnordered(fc, chunk, + LongChunk.chunkWrap(addedMapping)); + } + adds.data.add(chunk); } - adds.data.add(chunk); - } else { - adds.data.add(deltaColumn.getChunkType().getEmptyChunk()); } adds.type = deltaColumn.getType(); @@ -1850,23 +1878,23 @@ final class ColumnInfo { final ColumnSource sourceColumn = deltaColumns[i]; final BarrageMessage.ModColumnData mods = new BarrageMessage.ModColumnData(); mods.data = new ArrayList<>(); + mods.chunkType = sourceColumn.getChunkType(); downstream.modColumnData[numActualModCols++] = mods; if (modColumnSet.get(i)) { final ColumnInfo info = getColumnInfo.apply(i); mods.rowsModified = info.recordedMods.copy(); - - final WritableChunk chunk = - sourceColumn.getChunkType().makeWritableChunk(info.modifiedMapping.length); - try (final ChunkSource.FillContext fc = sourceColumn.makeFillContext(info.modifiedMapping.length)) { - ((FillUnordered) sourceColumn).fillChunkUnordered(fc, chunk, - LongChunk.chunkWrap(info.modifiedMapping)); + for (long[] modifiedMapping : info.modifiedMappings) { + final WritableChunk chunk = mods.chunkType.makeWritableChunk(modifiedMapping.length); + try (final ChunkSource.FillContext fc = sourceColumn.makeFillContext(modifiedMapping.length)) { + ((FillUnordered) sourceColumn).fillChunkUnordered(fc, chunk, + LongChunk.chunkWrap(modifiedMapping)); + } + mods.data.add(chunk); } - mods.data.add(chunk); } else { mods.rowsModified = RowSetFactory.empty(); - mods.data.add(sourceColumn.getChunkType().getEmptyChunk()); } mods.type = sourceColumn.getType(); @@ -1884,14 +1912,28 @@ final class ColumnInfo { } // Updates provided mapping so that mapping[i] returns values.get(i) for all i in keys. - private static void applyRedirMapping(final RowSet keys, final RowSet values, final long[] mapping) { + private static void applyRedirMapping(final RowSet keys, final RowSet values, final ArrayList mappings) { Assert.eq(keys.size(), "keys.size()", values.size(), "values.size()"); - Assert.leq(keys.size(), "keys.size()", mapping.length, "mapping.length"); + MutableLong mapCount = new MutableLong(0L); + mappings.forEach((arr) -> mapCount.add(arr.length)); + Assert.leq(keys.size(), "keys.size()", mapCount.longValue(), "mapping.length"); + + // we need to track our progress through multiple mapping arrays + MutableLong arrOffset = new MutableLong(0L); + MutableInt arrIdx = new MutableInt(0); + final RowSet.Iterator vit = values.iterator(); keys.forAllRowKeys(lkey -> { - final int key = LongSizedDataStructure.intSize("applyRedirMapping", lkey); - Assert.eq(mapping[key], "mapping[key]", RowSequence.NULL_ROW_KEY, "RowSet.NULL_ROW_KEY"); - mapping[key] = vit.nextLong(); + long[] mapping = mappings.get(arrIdx.intValue()); + int keyIdx = LongSizedDataStructure.intSize("applyRedirMapping", lkey - arrOffset.longValue()); + + Assert.eq(mapping[keyIdx], "mapping[keyIdx]", RowSequence.NULL_ROW_KEY, "RowSet.NULL_ROW_KEY"); + mapping[keyIdx] = vit.nextLong(); + + if (keyIdx == mapping.length - 1) { + arrOffset.add(mapping.length); + arrIdx.add(1); + } }); } diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java index d41d8b3d877..bd7ff165564 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java @@ -14,6 +14,7 @@ import io.deephaven.barrage.flatbuf.BarrageMessageWrapper; import io.deephaven.barrage.flatbuf.BarrageModColumnMetadata; import io.deephaven.barrage.flatbuf.BarrageUpdateMetadata; +import io.deephaven.base.verify.Assert; import io.deephaven.chunk.ChunkType; import io.deephaven.chunk.WritableChunk; import io.deephaven.chunk.WritableLongChunk; @@ -53,10 +54,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; -import java.util.ArrayDeque; -import java.util.BitSet; -import java.util.Map; -import java.util.Objects; +import java.util.*; import java.util.function.Consumer; import static io.deephaven.engine.table.impl.sources.InMemoryColumnSource.TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD; @@ -111,6 +109,7 @@ public View getSchemaView(final TableDefinition table, public static class ChunkListInputStreamGenerator implements SafeCloseable { public ChunkInputStreamGenerator[] generators; + public ChunkInputStreamGenerator emptyGenerator; ChunkListInputStreamGenerator(BarrageMessage.AddColumnData acd) { // create an input stream generator for each chunk @@ -120,6 +119,8 @@ public static class ChunkListInputStreamGenerator implements SafeCloseable { generators[i] = ChunkInputStreamGenerator.makeInputStreamGenerator( acd.data.get(i).getChunkType(), acd.type, acd.componentType, acd.data.get(i)); } + emptyGenerator = ChunkInputStreamGenerator.makeInputStreamGenerator( + acd.chunkType, acd.type, acd.componentType, acd.chunkType.getEmptyChunk()); } ChunkListInputStreamGenerator(BarrageMessage.ModColumnData mcd) { @@ -128,8 +129,10 @@ public static class ChunkListInputStreamGenerator implements SafeCloseable { for (int i = 0; i < mcd.data.size(); ++i) { generators[i] = ChunkInputStreamGenerator.makeInputStreamGenerator( - mcd.data.get(i).getChunkType(), mcd.type, mcd.componentType, mcd.data.get(i)); + mcd.chunkType, mcd.type, mcd.componentType, mcd.data.get(i)); } + emptyGenerator = ChunkInputStreamGenerator.makeInputStreamGenerator( + mcd.chunkType, mcd.type, mcd.componentType, mcd.chunkType.getEmptyChunk()); } public int size() { @@ -142,13 +145,14 @@ public void close() { generators[i].close(); generators[i] = null; } + emptyGenerator.close(); } } public static class ModColumnData { public final RowSetGenerator rowsModified; public final ChunkListInputStreamGenerator data; - public final RowSet[] modChunkRowSets; + public final long[] modChunkRowOffsets; ModColumnData(final BarrageMessage.ModColumnData col) throws IOException { rowsModified = new RowSetGenerator(col.rowsModified); @@ -156,16 +160,13 @@ public static class ModColumnData { // build the row offsets for this column chunks long offset = 0; - modChunkRowSets = new RowSet[col.data.size()]; + modChunkRowOffsets = new long[col.data.size() + 1]; for (int chunkIdx = 0; chunkIdx < col.data.size(); ++chunkIdx) { + modChunkRowOffsets[chunkIdx] = offset; int chunkSize = col.data.get(chunkIdx).size(); - if (chunkSize == 0) { - modChunkRowSets[chunkIdx] = RowSetFactory.empty(); - } else { - modChunkRowSets[chunkIdx] = RowSetFactory.fromRange(offset, offset + chunkSize - 1); - } offset += chunkSize; } + modChunkRowOffsets[modChunkRowOffsets.length - 1] = offset; } } @@ -186,7 +187,7 @@ public static class ModColumnData { public final ChunkListInputStreamGenerator[] addColumnData; public final ModColumnData[] modColumnData; - public final RowSet[] addChunkRowSets; + public final long[] addChunkRowOffsets; /** * Create a barrage stream generator that can slice and dice the barrage message for delivery to clients. @@ -211,28 +212,25 @@ public BarrageStreamGenerator(final BarrageMessage message, final WriteMetricsCo boolean firstColumnWithChunks = true; addColumnData = new ChunkListInputStreamGenerator[message.addColumnData.length]; - // build the row sets for the add column data. NOTE: all populated columns will have the same - // number of chunks and equal row counts in each chunk so we can use any to create the rowsets - RowSet[] tmpAddChunkRowSets = new RowSet[0]; + // build the row offsets for the add column data. NOTE: all populated columns will have the same + // number of chunks and equal row counts in each chunk so we can use any to create the row offsets + long[] tmpAddChunkRowOffsets = new long[0]; for (int i = 0; i < message.addColumnData.length; ++i) { if (firstColumnWithChunks && message.addColumnData[i].data.size() > 0) { long offset = 0; - tmpAddChunkRowSets = new RowSet[message.addColumnData[i].data.size()]; + tmpAddChunkRowOffsets = new long[message.addColumnData[i].data.size() + 1]; for (int chunkIdx = 0; chunkIdx < message.addColumnData[i].data.size(); ++chunkIdx) { + tmpAddChunkRowOffsets[chunkIdx] = offset; int chunkSize = message.addColumnData[i].data.get(chunkIdx).size(); - if (chunkSize == 0) { - tmpAddChunkRowSets[chunkIdx] = RowSetFactory.empty(); - } else { - tmpAddChunkRowSets[chunkIdx] = RowSetFactory.fromRange(offset, offset + chunkSize - 1); - } offset += chunkSize; } + tmpAddChunkRowOffsets[tmpAddChunkRowOffsets.length - 1] = offset; firstColumnWithChunks = false; } addColumnData[i] = new ChunkListInputStreamGenerator(message.addColumnData[i]); } - addChunkRowSets = tmpAddChunkRowSets; + addChunkRowOffsets = tmpAddChunkRowOffsets; modColumnData = new ModColumnData[message.modColumnData.length]; for (int i = 0; i < modColumnData.length; ++i) { @@ -266,18 +264,11 @@ public void close() { for (final ChunkListInputStreamGenerator in : addColumnData) { in.close(); } - for (final RowSet rs : addChunkRowSets) { - rs.close(); - } - } if (modColumnData != null) { for (final ModColumnData mcd : modColumnData) { mcd.rowsModified.close(); mcd.data.close(); - for (final RowSet rs : mcd.modChunkRowSets) { - rs.close(); - } } } } @@ -385,121 +376,32 @@ public SubView(final BarrageStreamGenerator generator, @Override public void forEachStream(Consumer visitor) throws IOException { final long startTm = System.nanoTime(); - long bytesWritten = 0; ByteBuffer metadata = generator.getSubscriptionMetadata(this); + MutableLong bytesWritten = new MutableLong(0L); // batch size is maximum, will write fewer rows when needed. If we are enforcing a message size limit, then // reasonably we will require at least one byte per row and restrict rows accordingly - int batchSizeLimit = Math.min(DEFAULT_MESSAGE_SIZE_LIMIT, batchSize()); + int maxBatchSize = Math.min(DEFAULT_MESSAGE_SIZE_LIMIT, batchSize()); - // there may be multiple chunk generators and we need to ensure that a single batch rowset does not cross - // boundaries and that we honor the batch size requests + final MutableInt actualBatchSize = new MutableInt(); if (numAddRows == 0 && numModRows == 0) { // we still need to send a message containing metadata when there are no rows final InputStream is = generator.getInputStream( - this, 0, 0, metadata, generator::appendAddColumns); - bytesWritten += is.available(); + this, 0, 0, actualBatchSize, metadata, generator::appendAddColumns); + bytesWritten.add(is.available()); visitor.accept(is); - } else { - long offset = 0; - long chunkOffset = 0; - - while (offset < numAddRows) { - long rowsRemaining = numAddRows - offset; - - int batchSize = (int) Math.min(rowsRemaining, batchSizeLimit); - - // make sure we are not about to cross a chunk boundary - if (chunkOffset + batchSize > TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD) { - batchSize = (int) (TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD - chunkOffset); - chunkOffset = 0; - } - - try { - final InputStream is = generator.getInputStream( - this, offset, offset + batchSize, metadata, generator::appendAddColumns); - int bytesToWrite = is.available(); - - // treat this as a hard limit, exceeding fails a client or w2w (unless we are sending a single - // row then we must send and let it potentially fail) - if (bytesToWrite > DEFAULT_MESSAGE_SIZE_LIMIT && batchSize > 1) { - // can't write this, so close the input stream and retry - is.close(); - } else { - bytesWritten += bytesToWrite; - - // let's write the data - visitor.accept(is); - - offset += batchSize; - chunkOffset += batchSize; - metadata = null; - } - // recompute the batch limit for the next message - int bytesPerRow = bytesToWrite / batchSize; - if (bytesPerRow > 0) { - int rowLimit = DEFAULT_MESSAGE_SIZE_LIMIT / bytesPerRow; - - // add some margin for abnormal cell contents - batchSizeLimit = Math.min(batchSize(), Math.max(1, (int) ((double) rowLimit * 0.9))); - } - } catch (SizeException ex) { - // was an overflow in the ChunkInputStream generator (probably VarBinary). We can't compute the - // correct number of rows from this failure, so cut batch size in half and try again. This may - // occur multiple times until the size is restricted properly - batchSizeLimit = Math.max(1, batchSizeLimit / 2); - } - } - - offset = 0; - chunkOffset = 0; - while (offset < numModRows) { - long rowsRemaining = numModRows - offset; + generator.writeConsumer.onWrite(bytesWritten.longValue(), System.nanoTime() - startTm); + return; + } - int batchSize = (int) Math.min(rowsRemaining, batchSizeLimit); + // send the add batches (if any) + generator.processBatches(visitor, this, numAddRows, maxBatchSize, metadata, generator::appendAddColumns, + bytesWritten); - // make sure we are not about to cross a chunk boundary - if (chunkOffset + batchSize > TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD) { - batchSize = (int) (TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD - chunkOffset); - chunkOffset = 0; - } - - try { - final InputStream is = generator.getInputStream( - this, offset, offset + batchSize, metadata, generator::appendModColumns); - int bytesToWrite = is.available(); - - // treat this as a hard limit, exceeding will fail a client or w2w - if (bytesToWrite > DEFAULT_MESSAGE_SIZE_LIMIT) { - // can't write this, so close the input stream and retry - is.close(); - } else { - bytesWritten += bytesToWrite; - - // let's write the data - visitor.accept(is); - - offset += batchSize; - chunkOffset += batchSize; - metadata = null; - } - // recompute the batch limit for the next message - int bytesPerRow = bytesToWrite / batchSize; - if (bytesPerRow > 0) { - int rowLimit = DEFAULT_MESSAGE_SIZE_LIMIT / bytesPerRow; - - // add some margin for abnormal cell contents - batchSizeLimit = Math.min(batchSize(), Math.max(1, (int) ((double) rowLimit * 0.9))); - } - } catch (SizeException ex) { - // was an overflow in the ChunkInputStream generator (probably VarBinary). We can't compute the - // correct number of rows from this failure, so cut batch size in half and try again. This may - // occur multiple times until the size is restricted properly - batchSizeLimit = Math.max(1, batchSizeLimit / 2); - } - } - } + // send the mod batches (if any) but don't send metadata twice + generator.processBatches(visitor, this, numModRows, maxBatchSize, numAddRows > 0 ? null : metadata, + generator::appendModColumns, bytesWritten); // clean up the helper indexes addRowOffsets.close(); @@ -509,7 +411,7 @@ public void forEachStream(Consumer visitor) throws IOException { modViewport.close(); } } - generator.writeConsumer.onWrite(bytesWritten, System.nanoTime() - startTm); + generator.writeConsumer.onWrite(bytesWritten.longValue(), System.nanoTime() - startTm); } private int batchSize() { @@ -611,63 +513,21 @@ public SnapshotView(final BarrageStreamGenerator generator, @Override public void forEachStream(Consumer visitor) throws IOException { ByteBuffer metadata = generator.getSnapshotMetadata(this); + MutableLong bytesWritten = new MutableLong(0L); // batch size is maximum, will write fewer rows when needed. If we are enforcing a message size limit, then // reasonably we will require at least one byte per row and restrict rows accordingly - int batchSizeLimit = Math.min(DEFAULT_MESSAGE_SIZE_LIMIT, batchSize()); + int maxBatchSize = Math.min(DEFAULT_MESSAGE_SIZE_LIMIT, batchSize()); + final MutableInt actualBatchSize = new MutableInt(); if (numAddRows == 0) { // we still need to send a message containing metadata when there are no rows visitor.accept(generator.getInputStream( - this, 0, 0, metadata, generator::appendAddColumns)); + this, 0, 0, actualBatchSize, metadata, generator::appendAddColumns)); } else { - long offset = 0; - long chunkOffset = 0; - - while (offset < numAddRows) { - long rowsRemaining = numAddRows - offset; - - int batchSize = (int) Math.min(rowsRemaining, batchSizeLimit); - - // make sure we are not about to cross a chunk boundary - if (chunkOffset + batchSize > TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD) { - batchSize = (int) (TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD - chunkOffset); - chunkOffset = 0; - } - - try { - final InputStream is = generator.getInputStream( - this, offset, offset + batchSize, metadata, generator::appendAddColumns); - int bytesToWrite = is.available(); - - // treat this as a hard limit, exceeding fails a client or w2w (unless we are sending a single - // row then we must send and let it potentially fail) - if (bytesToWrite > DEFAULT_MESSAGE_SIZE_LIMIT && batchSize > 1) { - // can't write this, so close the input stream and retry - is.close(); - } else { - // let's write the data - visitor.accept(is); - - offset += batchSize; - chunkOffset += batchSize; - metadata = null; - } - // recompute the batch limit for the next message - int bytesPerRow = bytesToWrite / batchSize; - if (bytesPerRow > 0) { - int rowLimit = DEFAULT_MESSAGE_SIZE_LIMIT / bytesPerRow; - - // add some margin for abnormal cell contents - batchSizeLimit = Math.min(batchSize(), Math.max(1, (int) ((double) rowLimit * 0.9))); - } - } catch (SizeException ex) { - // was an overflow in the ChunkInputStream generator (probably VarBinary). We can't compute the - // correct number of rows from this failure, so cut batch size in half and try again. This may - // occur multiple times until the size is restricted properly - batchSizeLimit = Math.max(1, batchSizeLimit / 2); - } - } + // send the add batches + generator.processBatches(visitor, this, numAddRows, maxBatchSize, metadata, generator::appendAddColumns, + bytesWritten); } addRowOffsets.close(); @@ -740,24 +600,27 @@ public RowSet modRowOffsets(int col) { @FunctionalInterface private interface ColumnVisitor { - long visit(final View view, final long startRange, final long endRange, + long visit(final View view, final long startRange, final int targetBatchSize, final Consumer addStream, final ChunkInputStreamGenerator.FieldNodeListener fieldNodeListener, final ChunkInputStreamGenerator.BufferListener bufferListener) throws IOException; } /** - * Returns an InputStream of the message filtered to the viewport. + * Returns an InputStream of the message filtered to the viewport. This function accepts `targetBatchSize` but may + * actually write fewer rows than the target (when crossing an internal chunk boundary, e.g.) * * @param view the view of the overall chunk to generate a RecordBatch for - * @param startRange the start of the batch in position space w.r.t. the view (inclusive) - * @param endRange the end of the batch in position space w.r.t. the view (exclusive) + * @param offset the start of the batch in position space w.r.t. the view (inclusive) + * @param targetBatchSize the target (and maximum) batch size to use for this message + * @param actualBatchSize the number of rows actually sent in this batch (will be <= targetBatchSize) * @param metadata the optional flight data metadata to attach to the message * @param columnVisitor the helper method responsible for appending the payload columns to the RecordBatch * @return an InputStream ready to be drained by GRPC */ - private InputStream getInputStream(final View view, final long startRange, final long endRange, - final ByteBuffer metadata, final ColumnVisitor columnVisitor) throws IOException { + private InputStream getInputStream(final View view, final long offset, final int targetBatchSize, + final MutableInt actualBatchSize, final ByteBuffer metadata, final ColumnVisitor columnVisitor) + throws IOException { final ArrayDeque streams = new ArrayDeque<>(); final MutableInt size = new MutableInt(); @@ -809,7 +672,9 @@ private InputStream getInputStream(final View view, final long startRange, final bufferInfos.ensureCapacityPreserve(bufferInfos.get().size() + 1); bufferInfos.get().add(length); }; - numRows = columnVisitor.visit(view, startRange, endRange, addStream, fieldNodeListener, bufferListener); + + numRows = columnVisitor.visit(view, offset, targetBatchSize, addStream, fieldNodeListener, bufferListener); + actualBatchSize.setValue(numRows); final WritableChunk noChunk = nodeOffsets.get(); RecordBatch.startNodesVector(header, noChunk.size()); @@ -874,63 +739,126 @@ private static int createByteVector(final FlatBufferBuilder builder, final byte[ return builder.endVector(); } - private long appendAddColumns(final View view, final long startRange, final long endRange, - final Consumer addStream, - final ChunkInputStreamGenerator.FieldNodeListener fieldNodeListener, - final ChunkInputStreamGenerator.BufferListener bufferListener) throws IOException { + private void processBatches(Consumer visitor, final View view, + final long numRows, final int maxBatchSize, ByteBuffer metadata, + final ColumnVisitor columnVisitor, final MutableLong bytesWritten) throws IOException { + long offset = 0; + MutableInt actualBatchSize = new MutableInt(); + int batchSize = maxBatchSize; + + while (offset < numRows) { + try { + final InputStream is = getInputStream( + view, offset, batchSize, actualBatchSize, metadata, columnVisitor); + int bytesToWrite = is.available(); + + // treat this as a hard limit, exceeding fails a client or w2w (unless we are sending a single + // row then we must send and let it potentially fail) + if (bytesToWrite > DEFAULT_MESSAGE_SIZE_LIMIT && batchSize > 1) { + // can't write this, so close the input stream and retry + is.close(); + } else { + bytesWritten.add(bytesToWrite); - try (final WritableRowSet myAddedOffsets = view.addRowOffsets().subSetByPositionRange(startRange, endRange)) { + // let's write the data + visitor.accept(is); + + offset += actualBatchSize.intValue(); + metadata = null; + } + // recompute the batch limit for the next message + int bytesPerRow = bytesToWrite / actualBatchSize.intValue(); + if (bytesPerRow > 0) { + int rowLimit = DEFAULT_MESSAGE_SIZE_LIMIT / bytesPerRow; + + // add some margin for abnormal cell contents + batchSize = Math.min(maxBatchSize, Math.max(1, (int) ((double) rowLimit * 0.9))); + } + } catch (SizeException ex) { + // was an overflow in the ChunkInputStream generator (probably VarBinary). We can't compute the + // correct number of rows from this failure, so cut batch size in half and try again. This may + // occur multiple times until the size is restricted properly + batchSize = Math.max(1, batchSize / 2); + } + } + } + + private long appendAddColumns(final View view, final long startRange, final int targetBatchSize, + final Consumer addStream, final ChunkInputStreamGenerator.FieldNodeListener fieldNodeListener, + final ChunkInputStreamGenerator.BufferListener bufferListener) throws IOException { + long endRange = startRange + targetBatchSize; + + // identify the chunk that holds this startRange (NOTE: will be same for all columns) + int chunkIdx = 0; + if (addChunkRowOffsets.length > 1) { + chunkIdx = Arrays.binarySearch(addChunkRowOffsets, startRange); + if (chunkIdx < 0) { + chunkIdx = -(chunkIdx) - 1; + } + // assert the startRange value lies within the selected chunk + Assert.assertion(startRange >= 0 && chunkIdx < addChunkRowOffsets.length - 1, + "appendAddColumns - chunk lookup failed"); + // adjust the batch size if we would cross a chunk boundary + endRange = Math.min(addChunkRowOffsets[chunkIdx] + TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD, endRange); + } + try (final WritableRowSet myAddedOffsets = view.addRowOffsets().subSetByPositionRange(startRange, endRange); + final RowSet adjustedOffsets = chunkIdx == 0 ? myAddedOffsets.copy() + : myAddedOffsets.shift(-addChunkRowOffsets[chunkIdx])) { // every column must write to the stream for (final ChunkListInputStreamGenerator chunkSetGen : addColumnData) { - boolean columnWritten = false; - - // iterate through each chunk and find the one that contains the requested data - for (int i = 0; i < chunkSetGen.size() && !columnWritten; ++i) { - // if the set of data is empty or the requested rows start in this chunk - if (startRange <= addChunkRowSets[i].lastRowKey()) { - final ChunkInputStreamGenerator generator = chunkSetGen.generators[i]; - - // shift this into the chunk position space - final long shiftAmount = -addChunkRowSets[i].firstRowKey(); - - // get an offset RowSet for each row in this chunk - try (final WritableRowSet adjustedOffsets = myAddedOffsets.intersect(addChunkRowSets[i])) { - // normalize this to the chunk offsets - adjustedOffsets.shiftInPlace(shiftAmount); - - final ChunkInputStreamGenerator.DrainableColumn drainableColumn = - generator.getInputStream(view.options(), adjustedOffsets); - drainableColumn.visitFieldNodes(fieldNodeListener); - drainableColumn.visitBuffers(bufferListener); - - // Add the drainable last as it is allowed to immediately close a row set the visitors need - addStream.accept(drainableColumn); - } - // no need to test other chunks - columnWritten = true; - } - } - // must write column data, just write an empty column - if (!columnWritten) { + if (myAddedOffsets.isEmpty() || chunkSetGen.generators.length == 0) { + // use an empty generator to publish the column data try (final RowSet empty = RowSetFactory.empty()) { final ChunkInputStreamGenerator.DrainableColumn drainableColumn = - chunkSetGen.generators[0].getInputStream(view.options(), empty); + chunkSetGen.emptyGenerator.getInputStream(view.options(), empty); drainableColumn.visitFieldNodes(fieldNodeListener); drainableColumn.visitBuffers(bufferListener); // Add the drainable last as it is allowed to immediately close a row set the visitors need addStream.accept(drainableColumn); } + } else { + final ChunkInputStreamGenerator generator = chunkSetGen.generators[chunkIdx]; + final ChunkInputStreamGenerator.DrainableColumn drainableColumn = + generator.getInputStream(view.options(), adjustedOffsets); + drainableColumn.visitFieldNodes(fieldNodeListener); + drainableColumn.visitBuffers(bufferListener); + + // Add the drainable last as it is allowed to immediately close a row set the visitors need + addStream.accept(drainableColumn); } } return myAddedOffsets.size(); } } - private long appendModColumns(final View view, final long startRange, final long endRange, + private long appendModColumns(final View view, final long startRange, final int targetBatchSize, final Consumer addStream, final ChunkInputStreamGenerator.FieldNodeListener fieldNodeListener, final ChunkInputStreamGenerator.BufferListener bufferListener) throws IOException { + long endRange = startRange + targetBatchSize; + int[] columnChunkIdx = new int[modColumnData.length]; + + // for each column identify the chunk that holds this startRange + for (int ii = 0; ii < modColumnData.length; ++ii) { + final ModColumnData mcd = modColumnData[ii]; + int chunkIdx = 0; + if (mcd.modChunkRowOffsets.length > 1) { + // identify the chunk that holds this startRange + chunkIdx = Arrays.binarySearch(mcd.modChunkRowOffsets, startRange); + if (chunkIdx < 0) { + chunkIdx = -(chunkIdx) - 1; + } + // assert the startRange value lies within the selected chunk + Assert.assertion(startRange >= 0 && chunkIdx < mcd.modChunkRowOffsets.length - 1, + "appendModColumns - chunk lookup failed"); + // shorten the batch size if we would cross a chunk boundary + endRange = + Math.min(mcd.modChunkRowOffsets[chunkIdx] + TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD, endRange); + } + columnChunkIdx[ii] = chunkIdx; + } + // now add mod-column streams, and write the mod column indexes long numRows = 0; for (int ii = 0; ii < modColumnData.length; ++ii) { @@ -952,36 +880,28 @@ private long appendModColumns(final View view, final long startRange, final long numRows = Math.max(numRows, myModOffsets.size()); try { - boolean columnWritten = false; - // iterate through each chunk and find the one that contains the requested data - for (int i = 0; i < mcd.data.size() && !columnWritten; ++i) { - if (startRange <= mcd.modChunkRowSets[i].lastRowKey()) { - final ChunkInputStreamGenerator generator = mcd.data.generators[i]; - - final long shiftAmount = -mcd.modChunkRowSets[i].firstRowKey(); - - // get an offset rowset for each row in this chunk - try (final WritableRowSet adjustedOffsets = myModOffsets.intersect(mcd.modChunkRowSets[i])) { - // normalize this to the chunk offsets - adjustedOffsets.shiftInPlace(shiftAmount); - - final ChunkInputStreamGenerator.DrainableColumn drainableColumn = - generator.getInputStream(view.options(), adjustedOffsets); - drainableColumn.visitFieldNodes(fieldNodeListener); - drainableColumn.visitBuffers(bufferListener); - - // Add the drainable last as it is allowed to immediately close a row set the visitors need - addStream.accept(drainableColumn); - } - // no need to test other chunks - columnWritten = true; - } - } - // must write column data, just write an empty column - if (!columnWritten) { + if (myModOffsets.isEmpty() || mcd.data.generators.length == 0) { + // use the empty generator to publish the column data try (final RowSet empty = RowSetFactory.empty()) { + final ChunkInputStreamGenerator.DrainableColumn drainableColumn = - mcd.data.generators[0].getInputStream(view.options(), empty); + mcd.data.emptyGenerator.getInputStream(view.options(), empty); + drainableColumn.visitFieldNodes(fieldNodeListener); + drainableColumn.visitBuffers(bufferListener); + + // Add the drainable last as it is allowed to immediately close a row set the visitors need + addStream.accept(drainableColumn); + } + } else { + + final int chunkIdx = columnChunkIdx[ii]; + final ChunkInputStreamGenerator generator = mcd.data.generators[chunkIdx]; + + // normalize to the chunk offsets + try (final WritableRowSet adjustedOffsets = chunkIdx == 0 ? myModOffsets.copy() : + myModOffsets.shift(-mcd.modChunkRowOffsets[chunkIdx])) { + final ChunkInputStreamGenerator.DrainableColumn drainableColumn = + generator.getInputStream(view.options(), adjustedOffsets); drainableColumn.visitFieldNodes(fieldNodeListener); drainableColumn.visitBuffers(bufferListener); @@ -1319,12 +1239,12 @@ public int drainTo(final OutputStream outputStream) throws IOException { } @Override - public int available() throws IOException { + public int available() throws SizeException, IOException { int total = 0; for (final InputStream stream : streams) { total += stream.available(); if (total < 0) { - throw new IllegalStateException("drained message is too large; exceeds Integer.MAX_VALUE"); + throw new SizeException("drained message is too large; exceeds Integer.MAX_VALUE", total); } } return total; From cfdba3c889a2d109171ec5de4a951a6be47baec6 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Fri, 29 Apr 2022 14:19:00 -0700 Subject: [PATCH 36/47] spotless applied --- .../io/deephaven/server/barrage/BarrageStreamGenerator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java index bd7ff165564..0f96649e3f1 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java @@ -898,8 +898,8 @@ private long appendModColumns(final View view, final long startRange, final int final ChunkInputStreamGenerator generator = mcd.data.generators[chunkIdx]; // normalize to the chunk offsets - try (final WritableRowSet adjustedOffsets = chunkIdx == 0 ? myModOffsets.copy() : - myModOffsets.shift(-mcd.modChunkRowOffsets[chunkIdx])) { + try (final WritableRowSet adjustedOffsets = chunkIdx == 0 ? myModOffsets.copy() + : myModOffsets.shift(-mcd.modChunkRowOffsets[chunkIdx])) { final ChunkInputStreamGenerator.DrainableColumn drainableColumn = generator.getInputStream(view.options(), adjustedOffsets); drainableColumn.visitFieldNodes(fieldNodeListener); From 275d1d8ba7a2d36a26b5361639a7cd6700d36e30 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Fri, 29 Apr 2022 15:30:21 -0700 Subject: [PATCH 37/47] fixed bug --- .../extensions/barrage/util/BarrageStreamReader.java | 6 ++++-- .../io/deephaven/server/barrage/BarrageStreamGenerator.java | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java index f65393bc0a6..92059c9b2ba 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java @@ -245,7 +245,7 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, for (int ci = 0; ci < msg.addColumnData.length; ++ci) { final BarrageMessage.AddColumnData acd = msg.addColumnData[ci]; - final int lastChunkIndex = acd.data.size() - 1; + int lastChunkIndex = acd.data.size() - 1; // need to add the batch row data to the column chunks WritableChunk chunk = (WritableChunk) acd.data.get(lastChunkIndex); @@ -264,6 +264,7 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, acd.data.add(chunk); chunkOffset = 0; + ++lastChunkIndex; } else { chunkOffset = (int) rowOffset; } @@ -279,7 +280,7 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, for (int ci = 0; ci < msg.modColumnData.length; ++ci) { final BarrageMessage.ModColumnData mcd = msg.modColumnData[ci]; - final int lastChunkIndex = mcd.data.size() - 1; + int lastChunkIndex = mcd.data.size() - 1; // need to add the batch row data to the column chunks WritableChunk chunk = (WritableChunk) mcd.data.get(lastChunkIndex); @@ -298,6 +299,7 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, mcd.data.add(chunk); chunkOffset = 0; + ++lastChunkIndex; } else { chunkOffset = (int) rowOffset; } diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java index 0f96649e3f1..29d198fb7f8 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java @@ -793,7 +793,7 @@ private long appendAddColumns(final View view, final long startRange, final int if (addChunkRowOffsets.length > 1) { chunkIdx = Arrays.binarySearch(addChunkRowOffsets, startRange); if (chunkIdx < 0) { - chunkIdx = -(chunkIdx) - 1; + chunkIdx = -(chunkIdx) - 2; // point to the chunk that contains this (offset < startRange) } // assert the startRange value lies within the selected chunk Assert.assertion(startRange >= 0 && chunkIdx < addChunkRowOffsets.length - 1, @@ -847,7 +847,7 @@ private long appendModColumns(final View view, final long startRange, final int // identify the chunk that holds this startRange chunkIdx = Arrays.binarySearch(mcd.modChunkRowOffsets, startRange); if (chunkIdx < 0) { - chunkIdx = -(chunkIdx) - 1; + chunkIdx = -(chunkIdx) - 2; // point to the chunk that contains this (offset < startRange) } // assert the startRange value lies within the selected chunk Assert.assertion(startRange >= 0 && chunkIdx < mcd.modChunkRowOffsets.length - 1, From fb6c561cf8bcb6f84aca3f95549b66e863a76737 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Fri, 29 Apr 2022 16:48:18 -0700 Subject: [PATCH 38/47] tests passing, still work to do --- .../barrage/BarrageStreamGenerator.java | 86 +++++-------------- 1 file changed, 20 insertions(+), 66 deletions(-) diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java index 29d198fb7f8..06cd3df9ff2 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java @@ -135,10 +135,6 @@ public static class ChunkListInputStreamGenerator implements SafeCloseable { mcd.chunkType, mcd.type, mcd.componentType, mcd.chunkType.getEmptyChunk()); } - public int size() { - return generators.length; - } - @Override public void close() { for (int i = 0; i < generators.length; i++) { @@ -152,21 +148,10 @@ public void close() { public static class ModColumnData { public final RowSetGenerator rowsModified; public final ChunkListInputStreamGenerator data; - public final long[] modChunkRowOffsets; ModColumnData(final BarrageMessage.ModColumnData col) throws IOException { rowsModified = new RowSetGenerator(col.rowsModified); data = new ChunkListInputStreamGenerator(col); - - // build the row offsets for this column chunks - long offset = 0; - modChunkRowOffsets = new long[col.data.size() + 1]; - for (int chunkIdx = 0; chunkIdx < col.data.size(); ++chunkIdx) { - modChunkRowOffsets[chunkIdx] = offset; - int chunkSize = col.data.get(chunkIdx).size(); - offset += chunkSize; - } - modChunkRowOffsets[modChunkRowOffsets.length - 1] = offset; } } @@ -185,10 +170,9 @@ public static class ModColumnData { public final RowSetShiftDataGenerator shifted; public final ChunkListInputStreamGenerator[] addColumnData; + public int addGeneratorCount = 0; public final ModColumnData[] modColumnData; - public final long[] addChunkRowOffsets; - /** * Create a barrage stream generator that can slice and dice the barrage message for delivery to clients. * @@ -209,28 +193,12 @@ public BarrageStreamGenerator(final BarrageMessage message, final WriteMetricsCo rowsRemoved = new RowSetGenerator(message.rowsRemoved); shifted = new RowSetShiftDataGenerator(message.shifted); - boolean firstColumnWithChunks = true; addColumnData = new ChunkListInputStreamGenerator[message.addColumnData.length]; - // build the row offsets for the add column data. NOTE: all populated columns will have the same - // number of chunks and equal row counts in each chunk so we can use any to create the row offsets - long[] tmpAddChunkRowOffsets = new long[0]; - for (int i = 0; i < message.addColumnData.length; ++i) { - if (firstColumnWithChunks && message.addColumnData[i].data.size() > 0) { - long offset = 0; - tmpAddChunkRowOffsets = new long[message.addColumnData[i].data.size() + 1]; - for (int chunkIdx = 0; chunkIdx < message.addColumnData[i].data.size(); ++chunkIdx) { - tmpAddChunkRowOffsets[chunkIdx] = offset; - int chunkSize = message.addColumnData[i].data.get(chunkIdx).size(); - offset += chunkSize; - } - tmpAddChunkRowOffsets[tmpAddChunkRowOffsets.length - 1] = offset; - firstColumnWithChunks = false; - } addColumnData[i] = new ChunkListInputStreamGenerator(message.addColumnData[i]); + addGeneratorCount = Math.max(addGeneratorCount, addColumnData[i].generators.length); } - addChunkRowOffsets = tmpAddChunkRowOffsets; modColumnData = new ModColumnData[message.modColumnData.length]; for (int i = 0; i < modColumnData.length; ++i) { @@ -788,29 +756,26 @@ private long appendAddColumns(final View view, final long startRange, final int final ChunkInputStreamGenerator.BufferListener bufferListener) throws IOException { long endRange = startRange + targetBatchSize; - // identify the chunk that holds this startRange (NOTE: will be same for all columns) int chunkIdx = 0; - if (addChunkRowOffsets.length > 1) { - chunkIdx = Arrays.binarySearch(addChunkRowOffsets, startRange); - if (chunkIdx < 0) { - chunkIdx = -(chunkIdx) - 2; // point to the chunk that contains this (offset < startRange) - } - // assert the startRange value lies within the selected chunk - Assert.assertion(startRange >= 0 && chunkIdx < addChunkRowOffsets.length - 1, - "appendAddColumns - chunk lookup failed"); + if (addGeneratorCount > 0) { + // identify the chunk that holds this startRange (NOTE: will be same for all columns) + chunkIdx = (int) (startRange / TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD); + // verify the selected chunk index is valid + Assert.assertion(chunkIdx >= 0 && chunkIdx < addGeneratorCount, "appendAddColumns - chunk lookup failed"); // adjust the batch size if we would cross a chunk boundary - endRange = Math.min(addChunkRowOffsets[chunkIdx] + TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD, endRange); + endRange = Math.min((long) (chunkIdx + 1) * TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD, endRange); } + try (final WritableRowSet myAddedOffsets = view.addRowOffsets().subSetByPositionRange(startRange, endRange); - final RowSet adjustedOffsets = chunkIdx == 0 ? myAddedOffsets.copy() - : myAddedOffsets.shift(-addChunkRowOffsets[chunkIdx])) { + final RowSet adjustedOffsets = + myAddedOffsets.shift((long) chunkIdx * -TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)) { // every column must write to the stream - for (final ChunkListInputStreamGenerator chunkSetGen : addColumnData) { - if (myAddedOffsets.isEmpty() || chunkSetGen.generators.length == 0) { + for (final ChunkListInputStreamGenerator data : addColumnData) { + if (myAddedOffsets.isEmpty() || data.generators.length == 0) { // use an empty generator to publish the column data try (final RowSet empty = RowSetFactory.empty()) { final ChunkInputStreamGenerator.DrainableColumn drainableColumn = - chunkSetGen.emptyGenerator.getInputStream(view.options(), empty); + data.emptyGenerator.getInputStream(view.options(), empty); drainableColumn.visitFieldNodes(fieldNodeListener); drainableColumn.visitBuffers(bufferListener); @@ -818,7 +783,7 @@ private long appendAddColumns(final View view, final long startRange, final int addStream.accept(drainableColumn); } } else { - final ChunkInputStreamGenerator generator = chunkSetGen.generators[chunkIdx]; + final ChunkInputStreamGenerator generator = data.generators[chunkIdx]; final ChunkInputStreamGenerator.DrainableColumn drainableColumn = generator.getInputStream(view.options(), adjustedOffsets); drainableColumn.visitFieldNodes(fieldNodeListener); @@ -842,20 +807,9 @@ private long appendModColumns(final View view, final long startRange, final int // for each column identify the chunk that holds this startRange for (int ii = 0; ii < modColumnData.length; ++ii) { final ModColumnData mcd = modColumnData[ii]; - int chunkIdx = 0; - if (mcd.modChunkRowOffsets.length > 1) { - // identify the chunk that holds this startRange - chunkIdx = Arrays.binarySearch(mcd.modChunkRowOffsets, startRange); - if (chunkIdx < 0) { - chunkIdx = -(chunkIdx) - 2; // point to the chunk that contains this (offset < startRange) - } - // assert the startRange value lies within the selected chunk - Assert.assertion(startRange >= 0 && chunkIdx < mcd.modChunkRowOffsets.length - 1, - "appendModColumns - chunk lookup failed"); - // shorten the batch size if we would cross a chunk boundary - endRange = - Math.min(mcd.modChunkRowOffsets[chunkIdx] + TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD, endRange); - } + int chunkIdx = (int) (startRange / TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD); + // adjust the batch size if we would cross a chunk boundary + endRange = Math.min((long) (chunkIdx + 1) * TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD, endRange); columnChunkIdx[ii] = chunkIdx; } @@ -898,8 +852,8 @@ private long appendModColumns(final View view, final long startRange, final int final ChunkInputStreamGenerator generator = mcd.data.generators[chunkIdx]; // normalize to the chunk offsets - try (final WritableRowSet adjustedOffsets = chunkIdx == 0 ? myModOffsets.copy() - : myModOffsets.shift(-mcd.modChunkRowOffsets[chunkIdx])) { + try (final WritableRowSet adjustedOffsets = + myModOffsets.shift((long) chunkIdx * -TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)) { final ChunkInputStreamGenerator.DrainableColumn drainableColumn = generator.getInputStream(view.options(), adjustedOffsets); drainableColumn.visitFieldNodes(fieldNodeListener); From bf509e5da2325cd48a2fd2585c9cf510c75b1a24 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Mon, 2 May 2022 09:58:20 -0700 Subject: [PATCH 39/47] updated ByteStorage class to extend OutputStream --- .../VarBinaryChunkInputStreamGenerator.java | 269 +++++++++--------- 1 file changed, 139 insertions(+), 130 deletions(-) diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java index 9da3fa38d7a..c0535277650 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java @@ -6,6 +6,7 @@ import com.google.common.io.LittleEndianDataOutputStream; import gnu.trove.iterator.TLongIterator; +import gnu.trove.list.array.TLongArrayList; import io.deephaven.UncheckedDeephavenException; import io.deephaven.chunk.WritableChunk; import io.deephaven.chunk.attributes.ChunkPositions; @@ -29,45 +30,121 @@ import java.util.ArrayList; import java.util.Iterator; +import static io.deephaven.engine.table.impl.sources.InMemoryColumnSource.TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD; + public class VarBinaryChunkInputStreamGenerator extends BaseChunkInputStreamGenerator> { private static final String DEBUG_NAME = "ObjectChunkInputStream Serialization"; private final Appender appendItem; - public static class ByteStorage { + public static class ByteStorage extends OutputStream { - private final WritableIntChunk offsets; + private final WritableLongChunk offsets; private final ArrayList byteArrays; - private final ArrayList byteArraySizes; - private final ArrayList byteArrayStartIndex; + private final TLongArrayList byteArrayOffsets; + + private BarrageProtoUtil.ExposedByteArrayOutputStream baos; - // low-budget memoization - private long lastIdx = -1; - private int lastIdxArrayIdx = -1; - private int lastIdxOffset = -1; + private long writtenByteCount; public ByteStorage(int size) { - offsets = WritableIntChunk.makeWritableChunk(size); + offsets = WritableLongChunk.makeWritableChunk(size); byteArrays = new ArrayList<>(); - byteArraySizes = new ArrayList<>(); - byteArrayStartIndex = new ArrayList<>(); - } + byteArrayOffsets = new TLongArrayList(); + byteArrayOffsets.add(0L); - public int size() { - return byteArrays.size(); - } + this.writtenByteCount = 0L; - void addByteArray(byte[] arr, int startIndex, int size) { - byteArrays.add(arr); - byteArrayStartIndex.add(startIndex); - byteArraySizes.add(size); + baos = new BarrageProtoUtil.ExposedByteArrayOutputStream(); } public boolean isEmpty() { return byteArrays.isEmpty(); } + /** + * Writes the specified byte to the underlying {@code ByteArrayOutputStream}. + * + * @param b the byte to be written. + */ + public synchronized void write(int b) throws IOException { + if (baos.size() + 1 > TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD) { + // close the current stream and add to the + finish(); + // create a new output stream + baos = new BarrageProtoUtil.ExposedByteArrayOutputStream(); + } + // do the write + baos.write(b); + // increment the offset + writtenByteCount += 1; + } + + /** + * Writes {@code len} bytes from the specified byte array + * starting at offset {@code off} to the underlying {@code ByteArrayOutputStream}. + * + * @param b the data. + * @param off the start offset in the data. + * @param len the number of bytes to write. + * @throws NullPointerException if {@code b} is {@code null}. + * @throws IndexOutOfBoundsException if {@code off} is negative, + * {@code len} is negative, or {@code len} is greater than + * {@code b.length - off} + */ + public synchronized void write(byte b[], int off, int len) throws IOException { + if (baos.size() + len > TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD) { + finish(); + // create a new output stream + baos = new BarrageProtoUtil.ExposedByteArrayOutputStream(); + } + // do the write + baos.write(b, off, len); + // increment the offset + writtenByteCount += len; + } + + /** + * Writes the complete contents of the specified byte array + * to the underlying {@code ByteArrayOutputStream}. + * + * @apiNote + * This method is equivalent to {@link #write(byte[],int,int) + * write(b, 0, b.length)}. + * + * @param b the data. + * @throws NullPointerException if {@code b} is {@code null}. + * @since 11 + */ + public void writeBytes(byte b[]) throws IOException { + if (baos.size() + b.length > TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD) { + finish(); + // create a new output stream + baos = new BarrageProtoUtil.ExposedByteArrayOutputStream(); + } + // do the write + baos.write(b, 0, b.length); + // increment the offset + writtenByteCount += b.length; + } + + public long size() { + return writtenByteCount; + } + + /*** + * completes the creation of the internal byte[] and closes open output streams + */ + public void finish() throws IOException { + // add the buffer to storage + byteArrays.add(baos.peekBuffer()); + // set the offset of the next buffer + byteArrayOffsets.add(byteArrayOffsets.get(byteArrayOffsets.size() - 1) + baos.size()); + // close the old output stream + baos.close(); + } + /*** * computes the size of the payload from sPos to ePos (inclusive) * @@ -76,51 +153,20 @@ public boolean isEmpty() { * @return number of bytes in the payload */ public long getPayloadSize(int sPos, int ePos) { - final int startArrayIndex; - final int startOffset; - - // might already have these start offsets saved - if (sPos == lastIdx) { - startArrayIndex = lastIdxArrayIdx; - startOffset = lastIdxOffset; - } else { - startArrayIndex = getByteArrayIndex(sPos); - startOffset = offsets.get(sPos); - } - - // store these for current and later re-use - lastIdx = ePos + 1; - lastIdxArrayIdx = getByteArrayIndex((int)lastIdx); - lastIdxOffset = offsets.get(ePos + 1); - - if (startArrayIndex == lastIdxArrayIdx) { // same byte array, can optimize - return lastIdxOffset - startOffset; - } else { - // need to span multiple byte arrays - long byteCount = getByteArraySize(startArrayIndex) - startOffset; - for (int midArrayIndex = startArrayIndex + 1; midArrayIndex < lastIdxArrayIdx; midArrayIndex++) { - byteCount += getByteArraySize(midArrayIndex); - } - byteCount += lastIdxOffset; - return byteCount; - } + return offsets.get(ePos + 1) - offsets.get(sPos); } - public Integer getByteArrayIndex(int pos) { + private int getByteArrayIndex(long bytePos) { // optimize for most common case if (byteArrays.size() == 1) { return 0; } - for (int i = byteArrayStartIndex.size() - 1; i > 0; --i) { - if (byteArrayStartIndex.get(i) <= pos) { - return i; - } + int idx = byteArrayOffsets.binarySearch(bytePos); + if (idx < 0) { + // return the index that contains this byte position + idx = -idx - 2; } - return 0; - } - - public int getByteArraySize(int arrayIdx) { - return byteArraySizes.get(arrayIdx); + return idx; } /*** @@ -133,35 +179,25 @@ public int getByteArraySize(int arrayIdx) { * @throws IOException if there is a problem writing to the output stream */ public long writePayload(LittleEndianDataOutputStream dos, int sPos, int ePos) throws IOException { - final int startArrayIndex = getByteArrayIndex(sPos); - final int startOffset = offsets.get(sPos); - - final int endArrayIndex = getByteArrayIndex(ePos + 1); - final int endOffset = offsets.get(ePos + 1); - - long writeLen = 0; - - if (startArrayIndex == endArrayIndex) { // same byte array, can optimize - dos.write(byteArrays.get(startArrayIndex), startOffset, endOffset - startOffset); - writeLen += endOffset - startOffset; - } else { - // need to span multiple byte arrays - int firstSize = byteArraySizes.get(startArrayIndex) - startOffset; - dos.write(byteArrays.get(startArrayIndex), startOffset, firstSize); - writeLen += firstSize; - - for (int midArrayIndex = startArrayIndex + 1; midArrayIndex < endArrayIndex; midArrayIndex++) { - int midSize = getByteArraySize(midArrayIndex); - dos.write(byteArrays.get(midArrayIndex), 0, midSize); - writeLen += midSize; - } - - dos.write(byteArrays.get(endArrayIndex), 0, endOffset); - writeLen += endOffset; + final long writeLen = getPayloadSize(sPos, ePos); + long remainingBytes = writeLen; + + long startBytePos = offsets.get(sPos); + while (remainingBytes > 0) { + final int arrayIdx = getByteArrayIndex(startBytePos); + // find the offset in the current byte[] + final int offset = + LongSizedDataStructure.intSize(DEBUG_NAME, startBytePos - byteArrayOffsets.get(arrayIdx)); + // compute the length (don't exceed end of the current byte[]) + final int len = (int) Math.min(remainingBytes, byteArrayOffsets.get(arrayIdx + 1) - startBytePos); + // do the write + dos.write(byteArrays.get(arrayIdx), offset, len); + startBytePos += len; + remainingBytes -= len; } return writeLen; } - }; + } private ByteStorage byteStorage = null; @@ -185,41 +221,17 @@ private synchronized void computePayload() throws IOException { } byteStorage = new ByteStorage(chunk.size() == 0 ? 0 : (chunk.size() + 1)); - BarrageProtoUtil.ExposedByteArrayOutputStream baos = new BarrageProtoUtil.ExposedByteArrayOutputStream(); if (chunk.size() > 0) { byteStorage.offsets.set(0, 0); } - int baosSize = 0; - int startIndex = 0; - for (int i = 0; i < chunk.size(); ++i) { if (chunk.get(i) != null) { - try { - appendItem.append(baos, chunk.get(i)); - baosSize = baos.size(); - } catch (OutOfMemoryError ex) { - // we overran the buffer on this item and the output stream probably has junk from the failed write - // but it is no more than the size of a single data item. We use the stored output stream size - // though instead of querying the size of the output stream (since that includes junk bytes) - - // add the buffer to storage - byteStorage.addByteArray(baos.peekBuffer(), startIndex, baosSize); - - // close the old output stream and create a new one - baos.close(); - baos = new BarrageProtoUtil.ExposedByteArrayOutputStream(); - - // add the item to the new buffer - appendItem.append(baos, chunk.get(i)); - baosSize = baos.size(); - startIndex = i; - byteStorage.offsets.set(i, 0); - } + appendItem.append(byteStorage, chunk.get(i)); } - byteStorage.offsets.set(i + 1, baosSize); + byteStorage.offsets.set(i + 1, byteStorage.size()); } - byteStorage.addByteArray(baos.peekBuffer(), startIndex, baosSize); - baos.close(); + // must call this function after writes are complete + byteStorage.finish(); } @Override @@ -235,20 +247,19 @@ public void close() { } @Override - public DrainableColumn getInputStream(final StreamReaderOptions options, final @Nullable RowSet subset) throws IOException { + public DrainableColumn getInputStream(final StreamReaderOptions options, final @Nullable RowSet subset) + throws IOException { computePayload(); - return new ObjectChunkInputStream(options, byteStorage, subset); + return new ObjectChunkInputStream(options, subset); } private class ObjectChunkInputStream extends BaseChunkInputStream { + private int cachedSize = -1; - private final ByteStorage myByteStorage; private ObjectChunkInputStream( - final StreamReaderOptions options, - final ByteStorage myByteStorage, final RowSet subset) { + final StreamReaderOptions options, final RowSet subset) { super(chunk, options, subset); - this.myByteStorage = myByteStorage; } private int cachedNullCount = -1; @@ -258,7 +269,7 @@ public int nullCount() { if (cachedNullCount == -1) { cachedNullCount = 0; subset.forAllRowKeys(i -> { - if (chunk.get((int)i) == null) { + if (chunk.get((int) i) == null) { ++cachedNullCount; } }); @@ -278,7 +289,7 @@ public void visitBuffers(final BufferListener listener) { listener.noteLogicalBuffer(sendValidityBuffer() ? getValidityMapSerializationSizeFor(numElements) : 0); // offsets - long numOffsetBytes = Integer.BYTES * (((long)numElements) + (numElements > 0 ? 1 : 0)); + long numOffsetBytes = Integer.BYTES * (((long) numElements) + (numElements > 0 ? 1 : 0)); final long bytesExtended = numOffsetBytes & REMAINDER_MOD_8_MASK; if (bytesExtended > 0) { numOffsetBytes += 8 - bytesExtended; @@ -288,7 +299,7 @@ public void visitBuffers(final BufferListener listener) { // payload final MutableLong numPayloadBytes = new MutableLong(); subset.forAllRowKeyRanges((s, e) -> { - numPayloadBytes.add(myByteStorage.getPayloadSize((int)s, (int)e)); + numPayloadBytes.add(byteStorage.getPayloadSize((int) s, (int) e)); }); final long payloadExtended = numPayloadBytes.longValue() & REMAINDER_MOD_8_MASK; if (payloadExtended > 0) { @@ -306,19 +317,17 @@ protected int getRawSize() { } // there are n+1 offsets; it is not assumed first offset is zero - if (!subset.isEmpty() && subset.size() == myByteStorage.offsets.size() - 1) { - totalCachedSize.add(myByteStorage.offsets.size() * Integer.BYTES); - for (int i = 0; i < myByteStorage.size(); i++) { - totalCachedSize.add(myByteStorage.getByteArraySize(i)); - } - } else { + if (!subset.isEmpty() && subset.size() == byteStorage.offsets.size() - 1) { + totalCachedSize.add(byteStorage.offsets.size() * Integer.BYTES); + totalCachedSize.add(byteStorage.size()); + } else { totalCachedSize.add(subset.isEmpty() ? 0 : Integer.BYTES); // account for the n+1 offset subset.forAllRowKeyRanges((s, e) -> { // account for offsets totalCachedSize.add((e - s + 1) * Integer.BYTES); // account for payload - totalCachedSize.add(myByteStorage.getPayloadSize((int)s, (int)e)); + totalCachedSize.add(byteStorage.getPayloadSize((int)s, (int)e)); }); } @@ -326,7 +335,7 @@ protected int getRawSize() { // then we must also align offset array totalCachedSize.add(Integer.BYTES); } - cachedSize = LongSizedDataStructure.intSize("VarBinaryChunkInputStreamGenerator.getRawSize", totalCachedSize.longValue()); + cachedSize = LongSizedDataStructure.intSize(DEBUG_NAME, totalCachedSize.longValue()); } return cachedSize; } @@ -373,7 +382,7 @@ public int drainTo(final OutputStream outputStream) throws IOException { final MutableInt logicalSize = new MutableInt(); subset.forAllRowKeys((idx) -> { try { - logicalSize.add(myByteStorage.getPayloadSize((int)idx,(int)idx)); + logicalSize.add(byteStorage.getPayloadSize((int) idx, (int) idx)); dos.writeInt(logicalSize.intValue()); } catch (final IOException e) { throw new UncheckedDeephavenException("couldn't drain data to OutputStream", e); @@ -390,7 +399,7 @@ public int drainTo(final OutputStream outputStream) throws IOException { final MutableLong payloadLen = new MutableLong(); subset.forAllRowKeyRanges((s, e) -> { try { - payloadLen.add(myByteStorage.writePayload(dos, (int) s, (int) e)); + payloadLen.add(byteStorage.writePayload(dos, (int) s, (int) e)); } catch (final IOException err) { throw new UncheckedDeephavenException("couldn't drain data to OutputStream", err); } From 4d19767e952abc3ebbe5f832fce26ff39b8060c6 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Mon, 2 May 2022 12:14:53 -0700 Subject: [PATCH 40/47] PR comments addressed, ready for review --- .../table/impl/remote/ConstructSnapshot.java | 6 +-- .../VarBinaryChunkInputStreamGenerator.java | 6 +-- .../barrage/table/BarrageTable.java | 8 ++-- .../barrage/util/BarrageStreamReader.java | 6 +-- .../client/examples/GetDirectTable.java | 6 ++- .../barrage/BarrageStreamGenerator.java | 42 +++++++------------ 6 files changed, 34 insertions(+), 40 deletions(-) diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java index 67b6e4d31db..6baf21ca6e7 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java @@ -1444,11 +1444,11 @@ private static ArrayList> getSnapshotDataAsChunkList(final Col return result; } - final int initialChunkSize = (int) Math.min(size, TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD); + final int maxChunkSize = (int) Math.min(size, TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD); - try (final ColumnSource.FillContext context = sourceToUse.makeFillContext(initialChunkSize, sharedContext); + try (final ColumnSource.FillContext context = sourceToUse.makeFillContext(maxChunkSize, sharedContext); final RowSequence.Iterator it = rowSet.getRowSequenceIterator()) { - int chunkSize = initialChunkSize; + int chunkSize = maxChunkSize; while (it.hasMore()) { final RowSequence reducedRowSet = it.getNextRowSequenceWithLength(chunkSize); final ChunkType chunkType = sourceToUse.getChunkType(); diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java index c0535277650..75a12f34f0c 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java @@ -69,7 +69,7 @@ public boolean isEmpty() { * @param b the byte to be written. */ public synchronized void write(int b) throws IOException { - if (baos.size() + 1 > TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD) { + if ((long) baos.size() + 1 > TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD) { // close the current stream and add to the finish(); // create a new output stream @@ -94,7 +94,7 @@ public synchronized void write(int b) throws IOException { * {@code b.length - off} */ public synchronized void write(byte b[], int off, int len) throws IOException { - if (baos.size() + len > TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD) { + if ((long) baos.size() + len > TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD) { finish(); // create a new output stream baos = new BarrageProtoUtil.ExposedByteArrayOutputStream(); @@ -118,7 +118,7 @@ public synchronized void write(byte b[], int off, int len) throws IOException { * @since 11 */ public void writeBytes(byte b[]) throws IOException { - if (baos.size() + b.length > TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD) { + if ((long) baos.size() + b.length > TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD) { finish(); // create a new output stream baos = new BarrageProtoUtil.ExposedByteArrayOutputStream(); diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java index 6e1af76eb4d..5ca36134bae 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java @@ -318,11 +318,11 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC totalMods.insert(column.rowsModified); } - // perform the addition operations in batches for efficiency - final int addBatchSize = (int) Math.min(update.rowsIncluded.size(), - 1 << ChunkPoolConstants.LARGEST_POOLED_CHUNK_LOG2_CAPACITY); - if (update.rowsIncluded.isNonempty()) { + // perform the addition operations in batches for efficiency + final int addBatchSize = (int) Math.min(update.rowsIncluded.size(), + 1 << ChunkPoolConstants.LARGEST_POOLED_CHUNK_LOG2_CAPACITY); + if (mightBeInitialSnapshot) { // ensure the data sources have at least the incoming capacity. The sources can auto-resize but // we know the initial snapshot size and can resize immediately diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java index 92059c9b2ba..e9fbf70fe41 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java @@ -248,7 +248,7 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, int lastChunkIndex = acd.data.size() - 1; // need to add the batch row data to the column chunks - WritableChunk chunk = (WritableChunk) acd.data.get(lastChunkIndex); + WritableChunk chunk = (WritableChunk) acd.data.get(lastChunkIndex); int chunkSize = chunk.size(); final int chunkOffset; @@ -258,7 +258,7 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, lastAddStartIndex += chunkSize; // create a new chunk before trying to write again - chunkSize = (int) (Math.min(msg.rowsIncluded.size() - numAddRowsRead, + chunkSize = (int) (Math.min(numAddRowsTotal - numAddRowsRead, TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); chunk = columnChunkTypes[ci].makeWritableChunk(chunkSize); acd.data.add(chunk); @@ -283,7 +283,7 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, int lastChunkIndex = mcd.data.size() - 1; // need to add the batch row data to the column chunks - WritableChunk chunk = (WritableChunk) mcd.data.get(lastChunkIndex); + WritableChunk chunk = (WritableChunk) mcd.data.get(lastChunkIndex); int chunkSize = chunk.size(); final int chunkOffset; diff --git a/java-client/flight-examples/src/main/java/io/deephaven/client/examples/GetDirectTable.java b/java-client/flight-examples/src/main/java/io/deephaven/client/examples/GetDirectTable.java index 62a6087b955..946791ff6ea 100644 --- a/java-client/flight-examples/src/main/java/io/deephaven/client/examples/GetDirectTable.java +++ b/java-client/flight-examples/src/main/java/io/deephaven/client/examples/GetDirectTable.java @@ -16,9 +16,13 @@ class GetDirectTable extends FlightExampleBase { protected void execute(FlightSession flight) throws Exception { try (final FlightStream stream = flight.stream(ticket)) { System.out.println(stream.getSchema()); + long tableRows = 0L; while (stream.next()) { - System.out.println(stream.getRoot().contentToTSVString()); + int batchRows = stream.getRoot().getRowCount(); + System.out.println(" batch received: " + batchRows + " rows"); + tableRows += batchRows; } + System.out.println("Table received: " + tableRows + " rows"); } } diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java index 06cd3df9ff2..8bcfe65d367 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java @@ -68,9 +68,12 @@ public class BarrageStreamGenerator implements private static final int DEFAULT_BATCH_SIZE = Configuration.getInstance() .getIntegerForClassWithDefault(BarrageStreamGenerator.class, "batchSize", Integer.MAX_VALUE); - // default to 100MB to match java-client and w2w incoming limits + private static final int DEFAULT_INITIAL_BATCH_SIZE = Configuration.getInstance() + .getIntegerForClassWithDefault(BarrageStreamGenerator.class, "initialBatchSize", 4096); + + // default to 99MB to match 100MB java-client and w2w default incoming limits and compensate for GRPC overhead private static final int DEFAULT_MESSAGE_SIZE_LIMIT = Configuration.getInstance() - .getIntegerForClassWithDefault(BarrageStreamGenerator.class, "maxOutboundMessageSize", 100 * 1024 * 1024); + .getIntegerForClassWithDefault(BarrageStreamGenerator.class, "maxOutboundMessageSize", 99 * 1024 * 1024); public interface View { void forEachStream(Consumer visitor) throws IOException; @@ -347,9 +350,8 @@ public void forEachStream(Consumer visitor) throws IOException { ByteBuffer metadata = generator.getSubscriptionMetadata(this); MutableLong bytesWritten = new MutableLong(0L); - // batch size is maximum, will write fewer rows when needed. If we are enforcing a message size limit, then - // reasonably we will require at least one byte per row and restrict rows accordingly - int maxBatchSize = Math.min(DEFAULT_MESSAGE_SIZE_LIMIT, batchSize()); + // batch size is maximum, will write fewer rows when needed + int maxBatchSize = batchSize(); final MutableInt actualBatchSize = new MutableInt(); @@ -483,9 +485,8 @@ public void forEachStream(Consumer visitor) throws IOException { ByteBuffer metadata = generator.getSnapshotMetadata(this); MutableLong bytesWritten = new MutableLong(0L); - // batch size is maximum, will write fewer rows when needed. If we are enforcing a message size limit, then - // reasonably we will require at least one byte per row and restrict rows accordingly - int maxBatchSize = Math.min(DEFAULT_MESSAGE_SIZE_LIMIT, batchSize()); + // batch size is maximum, will write fewer rows when needed + int maxBatchSize = batchSize(); final MutableInt actualBatchSize = new MutableInt(); if (numAddRows == 0) { @@ -712,7 +713,8 @@ private void processBatches(Consumer visitor, final View view, final ColumnVisitor columnVisitor, final MutableLong bytesWritten) throws IOException { long offset = 0; MutableInt actualBatchSize = new MutableInt(); - int batchSize = maxBatchSize; + + int batchSize = DEFAULT_INITIAL_BATCH_SIZE; while (offset < numRows) { try { @@ -722,17 +724,16 @@ private void processBatches(Consumer visitor, final View view, // treat this as a hard limit, exceeding fails a client or w2w (unless we are sending a single // row then we must send and let it potentially fail) - if (bytesToWrite > DEFAULT_MESSAGE_SIZE_LIMIT && batchSize > 1) { - // can't write this, so close the input stream and retry - is.close(); - } else { - bytesWritten.add(bytesToWrite); - + if (bytesToWrite < DEFAULT_MESSAGE_SIZE_LIMIT || batchSize == 1) { // let's write the data visitor.accept(is); + bytesWritten.add(bytesToWrite); offset += actualBatchSize.intValue(); metadata = null; + } else { + // can't write this, so close the input stream and retry + is.close(); } // recompute the batch limit for the next message int bytesPerRow = bytesToWrite / actualBatchSize.intValue(); @@ -755,7 +756,6 @@ private long appendAddColumns(final View view, final long startRange, final int final Consumer addStream, final ChunkInputStreamGenerator.FieldNodeListener fieldNodeListener, final ChunkInputStreamGenerator.BufferListener bufferListener) throws IOException { long endRange = startRange + targetBatchSize; - int chunkIdx = 0; if (addGeneratorCount > 0) { // identify the chunk that holds this startRange (NOTE: will be same for all columns) @@ -765,7 +765,6 @@ private long appendAddColumns(final View view, final long startRange, final int // adjust the batch size if we would cross a chunk boundary endRange = Math.min((long) (chunkIdx + 1) * TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD, endRange); } - try (final WritableRowSet myAddedOffsets = view.addRowOffsets().subSetByPositionRange(startRange, endRange); final RowSet adjustedOffsets = myAddedOffsets.shift((long) chunkIdx * -TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)) { @@ -788,7 +787,6 @@ private long appendAddColumns(final View view, final long startRange, final int generator.getInputStream(view.options(), adjustedOffsets); drainableColumn.visitFieldNodes(fieldNodeListener); drainableColumn.visitBuffers(bufferListener); - // Add the drainable last as it is allowed to immediately close a row set the visitors need addStream.accept(drainableColumn); } @@ -803,7 +801,6 @@ private long appendModColumns(final View view, final long startRange, final int final ChunkInputStreamGenerator.BufferListener bufferListener) throws IOException { long endRange = startRange + targetBatchSize; int[] columnChunkIdx = new int[modColumnData.length]; - // for each column identify the chunk that holds this startRange for (int ii = 0; ii < modColumnData.length; ++ii) { final ModColumnData mcd = modColumnData[ii]; @@ -812,7 +809,6 @@ private long appendModColumns(final View view, final long startRange, final int endRange = Math.min((long) (chunkIdx + 1) * TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD, endRange); columnChunkIdx[ii] = chunkIdx; } - // now add mod-column streams, and write the mod column indexes long numRows = 0; for (int ii = 0; ii < modColumnData.length; ++ii) { @@ -832,25 +828,20 @@ private long appendModColumns(final View view, final long startRange, final int } } numRows = Math.max(numRows, myModOffsets.size()); - try { if (myModOffsets.isEmpty() || mcd.data.generators.length == 0) { // use the empty generator to publish the column data try (final RowSet empty = RowSetFactory.empty()) { - final ChunkInputStreamGenerator.DrainableColumn drainableColumn = mcd.data.emptyGenerator.getInputStream(view.options(), empty); drainableColumn.visitFieldNodes(fieldNodeListener); drainableColumn.visitBuffers(bufferListener); - // Add the drainable last as it is allowed to immediately close a row set the visitors need addStream.accept(drainableColumn); } } else { - final int chunkIdx = columnChunkIdx[ii]; final ChunkInputStreamGenerator generator = mcd.data.generators[chunkIdx]; - // normalize to the chunk offsets try (final WritableRowSet adjustedOffsets = myModOffsets.shift((long) chunkIdx * -TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)) { @@ -858,7 +849,6 @@ private long appendModColumns(final View view, final long startRange, final int generator.getInputStream(view.options(), adjustedOffsets); drainableColumn.visitFieldNodes(fieldNodeListener); drainableColumn.visitBuffers(bufferListener); - // Add the drainable last as it is allowed to immediately close a row set the visitors need addStream.accept(drainableColumn); } From 0667ad0724334da3a319da61231db71958d56409 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Mon, 2 May 2022 12:56:00 -0700 Subject: [PATCH 41/47] reverted the outbound message limit to 100 (from 99) MB --- .../io/deephaven/server/barrage/BarrageStreamGenerator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java index 8bcfe65d367..08018cf9b25 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java @@ -71,9 +71,9 @@ public class BarrageStreamGenerator implements private static final int DEFAULT_INITIAL_BATCH_SIZE = Configuration.getInstance() .getIntegerForClassWithDefault(BarrageStreamGenerator.class, "initialBatchSize", 4096); - // default to 99MB to match 100MB java-client and w2w default incoming limits and compensate for GRPC overhead + // default to 100MB to match 100MB java-client and w2w default incoming limits private static final int DEFAULT_MESSAGE_SIZE_LIMIT = Configuration.getInstance() - .getIntegerForClassWithDefault(BarrageStreamGenerator.class, "maxOutboundMessageSize", 99 * 1024 * 1024); + .getIntegerForClassWithDefault(BarrageStreamGenerator.class, "maxOutboundMessageSize", 100 * 1024 * 1024); public interface View { void forEachStream(Consumer visitor) throws IOException; From e2e188f7824b69c3f5faf38563a9dce5a4cbf076 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Thu, 12 May 2022 13:48:37 -0700 Subject: [PATCH 42/47] finishing up PR --- .../chunk/ByteChunkToOutputStreamAdapter.java | 12 ++ .../table/impl/remote/ConstructSnapshot.java | 11 +- .../VarBinaryChunkInputStreamGenerator.java | 186 ++++++++---------- .../barrage/table/BarrageTable.java | 5 +- .../barrage/util/BarrageStreamReader.java | 21 +- .../barrage/BarrageMessageProducer.java | 10 +- .../barrage/BarrageStreamGenerator.java | 19 +- 7 files changed, 131 insertions(+), 133 deletions(-) create mode 100644 engine/chunk/src/main/java/io/deephaven/chunk/ByteChunkToOutputStreamAdapter.java diff --git a/engine/chunk/src/main/java/io/deephaven/chunk/ByteChunkToOutputStreamAdapter.java b/engine/chunk/src/main/java/io/deephaven/chunk/ByteChunkToOutputStreamAdapter.java new file mode 100644 index 00000000000..ea2ab47def2 --- /dev/null +++ b/engine/chunk/src/main/java/io/deephaven/chunk/ByteChunkToOutputStreamAdapter.java @@ -0,0 +1,12 @@ +package io.deephaven.chunk; + +import io.deephaven.chunk.attributes.Any; + +import java.io.IOException; +import java.io.OutputStream; + +public class ByteChunkToOutputStreamAdapter { + public static void write(OutputStream stream, ByteChunk chunk, int srcOffset, int length) throws IOException { + stream.write(chunk.data, chunk.offset + srcOffset, length); + } +} diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java index 6baf21ca6e7..3a5d6c06b28 100644 --- a/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java +++ b/engine/table/src/main/java/io/deephaven/engine/table/impl/remote/ConstructSnapshot.java @@ -8,6 +8,7 @@ import io.deephaven.base.log.LogOutput; import io.deephaven.base.log.LogOutputAppendable; import io.deephaven.base.verify.Assert; +import io.deephaven.chunk.util.pools.ChunkPoolConstants; import io.deephaven.configuration.Configuration; import io.deephaven.datastructures.util.CollectionUtil; import io.deephaven.engine.rowset.*; @@ -45,8 +46,6 @@ import io.deephaven.chunk.attributes.Values; -import static io.deephaven.engine.table.impl.sources.InMemoryColumnSource.TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD; - /** * A Set of static utilities for computing values from a table while avoiding the use of the UGP lock. This class * supports snapshots in both position space and key space. @@ -81,6 +80,8 @@ public NoSnapshotAllowedException(String reason) { private static final int MAX_CONCURRENT_ATTEMPT_DURATION_MILLIS = Configuration.getInstance() .getIntegerWithDefault("ConstructSnapshot.maxConcurrentAttemptDurationMillis", 5000); + public static final int SNAPSHOT_CHUNK_SIZE = 1 << ChunkPoolConstants.LARGEST_POOLED_CHUNK_LOG2_CAPACITY; + /** * Holder for thread-local state. */ @@ -1444,7 +1445,7 @@ private static ArrayList> getSnapshotDataAsChunkList(final Col return result; } - final int maxChunkSize = (int) Math.min(size, TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD); + final int maxChunkSize = (int) Math.min(size, SNAPSHOT_CHUNK_SIZE); try (final ColumnSource.FillContext context = sourceToUse.makeFillContext(maxChunkSize, sharedContext); final RowSequence.Iterator it = rowSet.getRowSequenceIterator()) { @@ -1469,8 +1470,8 @@ private static ArrayList> getSnapshotDataAsChunkList(final Col offset += currentChunk.size(); // recompute the size of the next chunk - if (size - offset > TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD) { - chunkSize = TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD; + if (size - offset > maxChunkSize) { + chunkSize = maxChunkSize; } else { chunkSize = (int) (size - offset); } diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java index 75a12f34f0c..17ac27b59b4 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java @@ -6,22 +6,19 @@ import com.google.common.io.LittleEndianDataOutputStream; import gnu.trove.iterator.TLongIterator; -import gnu.trove.list.array.TLongArrayList; import io.deephaven.UncheckedDeephavenException; -import io.deephaven.chunk.WritableChunk; +import io.deephaven.chunk.*; import io.deephaven.chunk.attributes.ChunkPositions; import io.deephaven.chunk.attributes.Values; +import io.deephaven.chunk.util.pools.ChunkPoolConstants; import io.deephaven.extensions.barrage.util.StreamReaderOptions; +import io.deephaven.util.SafeCloseable; import io.deephaven.util.datastructures.LongSizedDataStructure; -import io.deephaven.chunk.ObjectChunk; -import io.deephaven.chunk.WritableIntChunk; -import io.deephaven.chunk.WritableLongChunk; -import io.deephaven.chunk.WritableObjectChunk; import io.deephaven.chunk.util.pools.PoolableChunk; import io.deephaven.engine.rowset.RowSet; -import io.deephaven.extensions.barrage.util.BarrageProtoUtil; import org.apache.commons.lang3.mutable.MutableInt; import org.apache.commons.lang3.mutable.MutableLong; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.io.DataInput; @@ -30,119 +27,95 @@ import java.util.ArrayList; import java.util.Iterator; -import static io.deephaven.engine.table.impl.sources.InMemoryColumnSource.TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD; - public class VarBinaryChunkInputStreamGenerator extends BaseChunkInputStreamGenerator> { private static final String DEBUG_NAME = "ObjectChunkInputStream Serialization"; + private static final int BYTE_CHUNK_SIZE = 1 << ChunkPoolConstants.LARGEST_POOLED_CHUNK_LOG2_CAPACITY; private final Appender appendItem; - public static class ByteStorage extends OutputStream { + public static class ByteStorage extends OutputStream implements SafeCloseable { private final WritableLongChunk offsets; - private final ArrayList byteArrays; - private final TLongArrayList byteArrayOffsets; - - private BarrageProtoUtil.ExposedByteArrayOutputStream baos; + private final ArrayList> byteChunks; - private long writtenByteCount; + /** + * The total number of bytes written to this output stream + */ + private long writtenTotalByteCount = 0L; + /** + * The total number of bytes written to the current ByteChunk + */ + private int activeChunkByteCount = 0; + /** + * The ByteChunk to which we are currently writing + */ + private WritableByteChunk activeChunk = null; public ByteStorage(int size) { offsets = WritableLongChunk.makeWritableChunk(size); + byteChunks = new ArrayList<>(); - byteArrays = new ArrayList<>(); - byteArrayOffsets = new TLongArrayList(); - byteArrayOffsets.add(0L); - - this.writtenByteCount = 0L; - - baos = new BarrageProtoUtil.ExposedByteArrayOutputStream(); + // create an initial chunk for data storage. it might not be needed, but eliminates testing on every + // write operation and the costs for creating and disposing from the pool are minimal + byteChunks.add(activeChunk = WritableByteChunk.makeWritableChunk(BYTE_CHUNK_SIZE)); } public boolean isEmpty() { - return byteArrays.isEmpty(); + return writtenTotalByteCount == 0; } /** - * Writes the specified byte to the underlying {@code ByteArrayOutputStream}. + * Writes the specified byte to the underlying {@code ByteChunk}. * * @param b the byte to be written. */ public synchronized void write(int b) throws IOException { - if ((long) baos.size() + 1 > TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD) { - // close the current stream and add to the - finish(); - // create a new output stream - baos = new BarrageProtoUtil.ExposedByteArrayOutputStream(); + // test for overflow + if (activeChunkByteCount + 1 > BYTE_CHUNK_SIZE) { + byteChunks.add(activeChunk = WritableByteChunk.makeWritableChunk(BYTE_CHUNK_SIZE)); + activeChunkByteCount = 0; } // do the write - baos.write(b); + activeChunk.set(activeChunkByteCount++, (byte)b); + // increment the offset - writtenByteCount += 1; + writtenTotalByteCount += 1; } /** * Writes {@code len} bytes from the specified byte array - * starting at offset {@code off} to the underlying {@code ByteArrayOutputStream}. + * starting at offset {@code off} to the underlying {@code ByteChunk}. * * @param b the data. * @param off the start offset in the data. * @param len the number of bytes to write. - * @throws NullPointerException if {@code b} is {@code null}. * @throws IndexOutOfBoundsException if {@code off} is negative, * {@code len} is negative, or {@code len} is greater than * {@code b.length - off} */ - public synchronized void write(byte b[], int off, int len) throws IOException { - if ((long) baos.size() + len > TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD) { - finish(); - // create a new output stream - baos = new BarrageProtoUtil.ExposedByteArrayOutputStream(); - } - // do the write - baos.write(b, off, len); - // increment the offset - writtenByteCount += len; - } + public synchronized void write(@NotNull byte[] b, int off, int len) throws IOException { + int remaining = len; + while (remaining > 0) { + final int writeLen = Math.min(remaining, BYTE_CHUNK_SIZE - activeChunkByteCount); - /** - * Writes the complete contents of the specified byte array - * to the underlying {@code ByteArrayOutputStream}. - * - * @apiNote - * This method is equivalent to {@link #write(byte[],int,int) - * write(b, 0, b.length)}. - * - * @param b the data. - * @throws NullPointerException if {@code b} is {@code null}. - * @since 11 - */ - public void writeBytes(byte b[]) throws IOException { - if ((long) baos.size() + b.length > TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD) { - finish(); - // create a new output stream - baos = new BarrageProtoUtil.ExposedByteArrayOutputStream(); + // do the write + activeChunk.copyFromTypedArray(b, off, activeChunkByteCount, writeLen); + + // increment the counts + writtenTotalByteCount += writeLen; + activeChunkByteCount += writeLen; + remaining -= writeLen; + + // allocate a new chunk when needed + if (activeChunkByteCount == BYTE_CHUNK_SIZE) { + byteChunks.add(activeChunk = WritableByteChunk.makeWritableChunk(BYTE_CHUNK_SIZE)); + activeChunkByteCount = 0; + } } - // do the write - baos.write(b, 0, b.length); - // increment the offset - writtenByteCount += b.length; } public long size() { - return writtenByteCount; - } - - /*** - * completes the creation of the internal byte[] and closes open output streams - */ - public void finish() throws IOException { - // add the buffer to storage - byteArrays.add(baos.peekBuffer()); - // set the offset of the next buffer - byteArrayOffsets.add(byteArrayOffsets.get(byteArrayOffsets.size() - 1) + baos.size()); - // close the old output stream - baos.close(); + return writtenTotalByteCount; } /*** @@ -156,19 +129,6 @@ public long getPayloadSize(int sPos, int ePos) { return offsets.get(ePos + 1) - offsets.get(sPos); } - private int getByteArrayIndex(long bytePos) { - // optimize for most common case - if (byteArrays.size() == 1) { - return 0; - } - int idx = byteArrayOffsets.binarySearch(bytePos); - if (idx < 0) { - // return the index that contains this byte position - idx = -idx - 2; - } - return idx; - } - /*** * write payload from sPos to ePos (inclusive) to the output stream * @@ -184,19 +144,41 @@ public long writePayload(LittleEndianDataOutputStream dos, int sPos, int ePos) t long startBytePos = offsets.get(sPos); while (remainingBytes > 0) { - final int arrayIdx = getByteArrayIndex(startBytePos); - // find the offset in the current byte[] - final int offset = - LongSizedDataStructure.intSize(DEBUG_NAME, startBytePos - byteArrayOffsets.get(arrayIdx)); - // compute the length (don't exceed end of the current byte[]) - final int len = (int) Math.min(remainingBytes, byteArrayOffsets.get(arrayIdx + 1) - startBytePos); - // do the write - dos.write(byteArrays.get(arrayIdx), offset, len); + final int chunkIdx = (int)(startBytePos / BYTE_CHUNK_SIZE); + final int byteIdx = (int)(startBytePos % BYTE_CHUNK_SIZE); + + final ByteChunk chunk = byteChunks.get(chunkIdx); + + final int len = (int) Math.min(remainingBytes, BYTE_CHUNK_SIZE - byteIdx); + + // do the write (using the stream adapter utility) + ByteChunkToOutputStreamAdapter.write(dos, chunk, byteIdx, len); + + // increment the offsets startBytePos += len; remainingBytes -= len; } return writeLen; } + + @Override + public void close() { + try { + super.close(); + } catch (IOException e) { + // ignore this error + } + + // close the offset and byte chunks + if (offsets instanceof PoolableChunk) { + ((PoolableChunk) offsets).close(); + } + for (ByteChunk chunk : byteChunks) { + if (chunk instanceof PoolableChunk) { + ((PoolableChunk) chunk).close(); + } + } + } } private ByteStorage byteStorage = null; @@ -230,8 +212,6 @@ private synchronized void computePayload() throws IOException { } byteStorage.offsets.set(i + 1, byteStorage.size()); } - // must call this function after writes are complete - byteStorage.finish(); } @Override @@ -241,7 +221,7 @@ public void close() { ((PoolableChunk) chunk).close(); } if (byteStorage != null) { - byteStorage.offsets.close(); + byteStorage.close(); } } } @@ -318,7 +298,7 @@ protected int getRawSize() { // there are n+1 offsets; it is not assumed first offset is zero if (!subset.isEmpty() && subset.size() == byteStorage.offsets.size() - 1) { - totalCachedSize.add(byteStorage.offsets.size() * Integer.BYTES); + totalCachedSize.add(byteStorage.offsets.size() * (long) Integer.BYTES); totalCachedSize.add(byteStorage.size()); } else { totalCachedSize.add(subset.isEmpty() ? 0 : Integer.BYTES); // account for the n+1 offset diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java index b7871198884..9d4f810f976 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java @@ -51,8 +51,6 @@ import java.util.function.Function; import java.util.function.LongConsumer; -import static io.deephaven.engine.table.impl.sources.InMemoryColumnSource.TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD; - /** * A client side {@link Table} that mirrors an upstream/server side {@code Table}. * @@ -478,7 +476,8 @@ private void freeRows(final RowSet rowsToFree) { } // Note: these are NOT OrderedRowKeys until after the call to .sort() - final int chunkSize = (int) Math.min(rowsToFree.size(), TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD); + final int chunkSize = + (int) Math.min(rowsToFree.size(), 1 << ChunkPoolConstants.LARGEST_POOLED_CHUNK_LOG2_CAPACITY); try (final WritableLongChunk redirectedRows = WritableLongChunk.makeWritableChunk(chunkSize); final RowSequence.Iterator rowsToFreeIterator = rowsToFree.getRowSequenceIterator()) { diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java index e9fbf70fe41..e25b68729d7 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java @@ -37,14 +37,14 @@ import java.util.BitSet; import java.util.Iterator; import java.util.function.LongConsumer; -import java.util.function.LongConsumer; - -import static io.deephaven.engine.table.impl.sources.InMemoryColumnSource.TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD; public class BarrageStreamReader implements StreamReader { private static final Logger log = LoggerFactory.getLogger(BarrageStreamReader.class); + // We would like to use jdk.internal.util.ArraysSupport.MAX_ARRAY_LENGTH, but it is not exported + private static final int MAX_CHUNK_SIZE = Integer.MAX_VALUE - 8; + private final LongConsumer deserializeTmConsumer; private long numAddRowsRead = 0; @@ -134,8 +134,7 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, msg.addColumnData[ci].data = new ArrayList<>(); // create an initial chunk of the correct size - final int chunkSize = - (int) (Math.min(msg.rowsIncluded.size(), TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); + final int chunkSize = (int) (Math.min(msg.rowsIncluded.size(), MAX_CHUNK_SIZE)); msg.addColumnData[ci].data.add(columnChunkTypes[ci].makeWritableChunk(chunkSize)); } numAddRowsTotal = msg.rowsIncluded.size(); @@ -154,7 +153,7 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, // create an initial chunk of the correct size final int chunkSize = (int) (Math.min(msg.modColumnData[ci].rowsModified.size(), - TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); + MAX_CHUNK_SIZE)); msg.modColumnData[ci].data.add(columnChunkTypes[ci].makeWritableChunk(chunkSize)); numModRowsTotal = Math.max(numModRowsTotal, msg.modColumnData[ci].rowsModified.size()); @@ -249,7 +248,7 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, // need to add the batch row data to the column chunks WritableChunk chunk = (WritableChunk) acd.data.get(lastChunkIndex); - int chunkSize = chunk.size(); + int chunkSize = chunk == null ? 0 : chunk.size(); final int chunkOffset; long rowOffset = numAddRowsRead - lastAddStartIndex; @@ -258,8 +257,10 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, lastAddStartIndex += chunkSize; // create a new chunk before trying to write again - chunkSize = (int) (Math.min(numAddRowsTotal - numAddRowsRead, - TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); + chunkSize = (int) (Math.min(numAddRowsTotal - numAddRowsRead, MAX_CHUNK_SIZE)); + // make sure the chunk will hold this batch (DoPut won't populate numAddRowsTotal) + chunkSize = Math.max((int) batch.length(), chunkSize); + chunk = columnChunkTypes[ci].makeWritableChunk(chunkSize); acd.data.add(chunk); @@ -294,7 +295,7 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, lastModStartIndex += chunkSize; // create a new chunk before trying to write again - chunkSize = (int) (Math.min(remaining, TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)); + chunkSize = (int) (Math.min(remaining, MAX_CHUNK_SIZE)); chunk = columnChunkTypes[ci].makeWritableChunk(chunkSize); mcd.data.add(chunk); diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java index bf3d2033dcd..0a6131438d9 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageMessageProducer.java @@ -64,7 +64,7 @@ import java.util.function.IntFunction; import java.util.stream.Stream; -import static io.deephaven.engine.table.impl.sources.InMemoryColumnSource.TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD; +import static io.deephaven.engine.table.impl.remote.ConstructSnapshot.SNAPSHOT_CHUNK_SIZE; /** * The server-side implementation of a Barrage replication source. @@ -1627,7 +1627,7 @@ private BarrageMessage aggregateUpdatesInRange(final int startDelta, final int e try (final RowSequence.Iterator it = localAdded.getRowSequenceIterator()) { while (it.hasMore()) { final RowSequence rs = - it.getNextRowSequenceWithLength(TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD); + it.getNextRowSequenceWithLength(SNAPSHOT_CHUNK_SIZE); final int chunkCapacity = rs.intSize("serializeItems"); final WritableChunk chunk = adds.chunkType.makeWritableChunk(chunkCapacity); try (final ChunkSource.FillContext fc = deltaColumn.makeFillContext(chunkCapacity)) { @@ -1656,7 +1656,7 @@ private BarrageMessage aggregateUpdatesInRange(final int startDelta, final int e try (final RowSequence.Iterator it = localModified.getRowSequenceIterator()) { while (it.hasMore()) { final RowSequence rs = - it.getNextRowSequenceWithLength(TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD); + it.getNextRowSequenceWithLength(SNAPSHOT_CHUNK_SIZE); final int chunkCapacity = rs.intSize("serializeItems"); final WritableChunk chunk = mods.chunkType.makeWritableChunk(chunkCapacity); try (final ChunkSource.FillContext fc = deltaColumn.makeFillContext(chunkCapacity)) { @@ -1754,7 +1754,7 @@ final class ColumnInfo { try (final RowSequence.Iterator it = localAdded.getRowSequenceIterator()) { while (it.hasMore()) { - final RowSequence rs = it.getNextRowSequenceWithLength(TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD); + final RowSequence rs = it.getNextRowSequenceWithLength(SNAPSHOT_CHUNK_SIZE); long[] addedMapping = new long[rs.intSize()]; Arrays.fill(addedMapping, RowSequence.NULL_ROW_KEY); retval.addedMappings.add(addedMapping); @@ -1763,7 +1763,7 @@ final class ColumnInfo { try (final RowSequence.Iterator it = retval.recordedMods.getRowSequenceIterator()) { while (it.hasMore()) { - final RowSequence rs = it.getNextRowSequenceWithLength(TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD); + final RowSequence rs = it.getNextRowSequenceWithLength(SNAPSHOT_CHUNK_SIZE); long[] modifiedMapping = new long[rs.intSize()]; Arrays.fill(modifiedMapping, RowSequence.NULL_ROW_KEY); retval.modifiedMappings.add(modifiedMapping); diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java index a94c7d5f361..10ac80615e8 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java @@ -59,7 +59,7 @@ import java.util.*; import java.util.function.Consumer; -import static io.deephaven.engine.table.impl.sources.InMemoryColumnSource.TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD; +import static io.deephaven.engine.table.impl.remote.ConstructSnapshot.SNAPSHOT_CHUNK_SIZE; import static io.deephaven.extensions.barrage.chunk.BaseChunkInputStreamGenerator.PADDING_BUFFER; public class BarrageStreamGenerator implements @@ -70,6 +70,7 @@ public class BarrageStreamGenerator implements private static final int DEFAULT_BATCH_SIZE = Configuration.getInstance() .getIntegerForClassWithDefault(BarrageStreamGenerator.class, "batchSize", Integer.MAX_VALUE); + // defaults to a small value that is likely to succeed and provide data for following batches private static final int DEFAULT_INITIAL_BATCH_SIZE = Configuration.getInstance() .getIntegerForClassWithDefault(BarrageStreamGenerator.class, "initialBatchSize", 4096); @@ -750,6 +751,10 @@ private void processBatches(Consumer visitor, final View view, // was an overflow in the ChunkInputStream generator (probably VarBinary). We can't compute the // correct number of rows from this failure, so cut batch size in half and try again. This may // occur multiple times until the size is restricted properly + if (batchSize == 1) { + // this row exceeds internal limits and can never be sent + throw(new UncheckedDeephavenException("BarrageStreamGenerator could not send", ex)); + } batchSize = Math.max(1, batchSize / 2); } } @@ -762,15 +767,15 @@ private long appendAddColumns(final View view, final long startRange, final int int chunkIdx = 0; if (addGeneratorCount > 0) { // identify the chunk that holds this startRange (NOTE: will be same for all columns) - chunkIdx = (int) (startRange / TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD); + chunkIdx = (int) (startRange / SNAPSHOT_CHUNK_SIZE); // verify the selected chunk index is valid Assert.assertion(chunkIdx >= 0 && chunkIdx < addGeneratorCount, "appendAddColumns - chunk lookup failed"); // adjust the batch size if we would cross a chunk boundary - endRange = Math.min((long) (chunkIdx + 1) * TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD, endRange); + endRange = Math.min((long) (chunkIdx + 1) * SNAPSHOT_CHUNK_SIZE, endRange); } try (final WritableRowSet myAddedOffsets = view.addRowOffsets().subSetByPositionRange(startRange, endRange); final RowSet adjustedOffsets = - myAddedOffsets.shift((long) chunkIdx * -TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)) { + myAddedOffsets.shift((long) chunkIdx * -SNAPSHOT_CHUNK_SIZE)) { // every column must write to the stream for (final ChunkListInputStreamGenerator data : addColumnData) { if (myAddedOffsets.isEmpty() || data.generators.length == 0) { @@ -807,9 +812,9 @@ private long appendModColumns(final View view, final long startRange, final int // for each column identify the chunk that holds this startRange for (int ii = 0; ii < modColumnData.length; ++ii) { final ModColumnData mcd = modColumnData[ii]; - int chunkIdx = (int) (startRange / TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD); + int chunkIdx = (int) (startRange / SNAPSHOT_CHUNK_SIZE); // adjust the batch size if we would cross a chunk boundary - endRange = Math.min((long) (chunkIdx + 1) * TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD, endRange); + endRange = Math.min((long) (chunkIdx + 1) * SNAPSHOT_CHUNK_SIZE, endRange); columnChunkIdx[ii] = chunkIdx; } // now add mod-column streams, and write the mod column indexes @@ -847,7 +852,7 @@ private long appendModColumns(final View view, final long startRange, final int final ChunkInputStreamGenerator generator = mcd.data.generators[chunkIdx]; // normalize to the chunk offsets try (final WritableRowSet adjustedOffsets = - myModOffsets.shift((long) chunkIdx * -TWO_DIMENSIONAL_COLUMN_SOURCE_THRESHOLD)) { + myModOffsets.shift((long) chunkIdx * -SNAPSHOT_CHUNK_SIZE)) { final ChunkInputStreamGenerator.DrainableColumn drainableColumn = generator.getInputStream(view.options(), adjustedOffsets); drainableColumn.visitFieldNodes(fieldNodeListener); From bf76b45be951e9e257cf4df7a0ef39c28b194878 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Thu, 12 May 2022 13:55:08 -0700 Subject: [PATCH 43/47] consolidation of constants --- .../extensions/barrage/table/BarrageTable.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java index 9d4f810f976..881da43e36f 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/table/BarrageTable.java @@ -63,6 +63,8 @@ public class BarrageTable extends QueryTable implements BarrageMessage.Listener, private static final Logger log = LoggerFactory.getLogger(BarrageTable.class); + private static final int BATCH_SIZE = 1 << ChunkPoolConstants.LARGEST_POOLED_CHUNK_LOG2_CAPACITY; + private final UpdateSourceRegistrar registrar; private final NotificationQueue notificationQueue; private final ScheduledExecutorService executorService; @@ -314,8 +316,7 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC if (update.rowsIncluded.isNonempty()) { // perform the addition operations in batches for efficiency - final int addBatchSize = (int) Math.min(update.rowsIncluded.size(), - 1 << ChunkPoolConstants.LARGEST_POOLED_CHUNK_LOG2_CAPACITY); + final int addBatchSize = (int) Math.min(update.rowsIncluded.size(), BATCH_SIZE); if (mightBeInitialSnapshot) { // ensure the data sources have at least the incoming capacity. The sources can auto-resize but @@ -381,8 +382,7 @@ private UpdateCoalescer processUpdate(final BarrageMessage update, final UpdateC } // perform the modification operations in batches for efficiency - final int modBatchSize = (int) Math.min(column.rowsModified.size(), - 1 << ChunkPoolConstants.LARGEST_POOLED_CHUNK_LOG2_CAPACITY); + final int modBatchSize = (int) Math.min(column.rowsModified.size(), BATCH_SIZE); modifiedColumnSet.setColumnWithIndex(ii); try (final ChunkSource.FillContext redirContext = rowRedirection.makeFillContext(modBatchSize, null); @@ -476,8 +476,7 @@ private void freeRows(final RowSet rowsToFree) { } // Note: these are NOT OrderedRowKeys until after the call to .sort() - final int chunkSize = - (int) Math.min(rowsToFree.size(), 1 << ChunkPoolConstants.LARGEST_POOLED_CHUNK_LOG2_CAPACITY); + final int chunkSize = (int) Math.min(rowsToFree.size(), BATCH_SIZE); try (final WritableLongChunk redirectedRows = WritableLongChunk.makeWritableChunk(chunkSize); final RowSequence.Iterator rowsToFreeIterator = rowsToFree.getRowSequenceIterator()) { From 2a8434b7d61c122676a822d90b84d5ab9599cf08 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Thu, 12 May 2022 14:07:59 -0700 Subject: [PATCH 44/47] spotless and exception message correction --- .../io/deephaven/server/barrage/BarrageStreamGenerator.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java index 10ac80615e8..7e41dfa7582 100644 --- a/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java +++ b/server/src/main/java/io/deephaven/server/barrage/BarrageStreamGenerator.java @@ -753,7 +753,8 @@ private void processBatches(Consumer visitor, final View view, // occur multiple times until the size is restricted properly if (batchSize == 1) { // this row exceeds internal limits and can never be sent - throw(new UncheckedDeephavenException("BarrageStreamGenerator could not send", ex)); + throw (new UncheckedDeephavenException( + "BarrageStreamGenerator - single row (" + offset + ") exceeds transmissible size", ex)); } batchSize = Math.max(1, batchSize / 2); } From 2d0836d487fac77b1467fbe45468150fb6660820 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Thu, 12 May 2022 16:31:47 -0700 Subject: [PATCH 45/47] bug fix in VarBinaryChunkInputStreamGenerator.java' --- .../chunk/VarBinaryChunkInputStreamGenerator.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java index 17ac27b59b4..30f2f22e985 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java @@ -70,16 +70,17 @@ public boolean isEmpty() { * @param b the byte to be written. */ public synchronized void write(int b) throws IOException { - // test for overflow - if (activeChunkByteCount + 1 > BYTE_CHUNK_SIZE) { - byteChunks.add(activeChunk = WritableByteChunk.makeWritableChunk(BYTE_CHUNK_SIZE)); - activeChunkByteCount = 0; - } // do the write activeChunk.set(activeChunkByteCount++, (byte)b); // increment the offset writtenTotalByteCount += 1; + + // allocate a new chunk when needed + if (activeChunkByteCount == BYTE_CHUNK_SIZE) { + byteChunks.add(activeChunk = WritableByteChunk.makeWritableChunk(BYTE_CHUNK_SIZE)); + activeChunkByteCount = 0; + } } /** @@ -104,6 +105,8 @@ public synchronized void write(@NotNull byte[] b, int off, int len) throws IOExc // increment the counts writtenTotalByteCount += writeLen; activeChunkByteCount += writeLen; + off += writeLen; + remaining -= writeLen; // allocate a new chunk when needed From 41f9166b9cdd7608941d547b9ab03ab6abbfee64 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Tue, 17 May 2022 09:08:03 -0700 Subject: [PATCH 46/47] PR comments addressed --- .../VarBinaryChunkInputStreamGenerator.java | 10 ++-- .../barrage/util/BarrageStreamReader.java | 53 ++++++------------- 2 files changed, 20 insertions(+), 43 deletions(-) diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java index 30f2f22e985..d0f71863615 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/chunk/VarBinaryChunkInputStreamGenerator.java @@ -173,13 +173,9 @@ public void close() { } // close the offset and byte chunks - if (offsets instanceof PoolableChunk) { - ((PoolableChunk) offsets).close(); - } - for (ByteChunk chunk : byteChunks) { - if (chunk instanceof PoolableChunk) { - ((PoolableChunk) chunk).close(); - } + offsets.close(); + for (WritableByteChunk chunk : byteChunks) { + chunk.close(); } } } diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java index e25b68729d7..62ef0fd41df 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java @@ -181,32 +181,10 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, throw new IllegalStateException("Only know how to decode Schema/BarrageRecordBatch messages"); } - // snapshots may not provide metadata, generate it now + // throw an error when no app metadata (snapshots now provide by default) if (msg == null) { - msg = new BarrageMessage(); - - // generate a default set of column selectors - msg.snapshotColumns = expectedColumns; - - // create and fill the add column metadata from the schema - msg.addColumnData = new BarrageMessage.AddColumnData[columnTypes.length]; - for (int ci = 0; ci < msg.addColumnData.length; ++ci) { - msg.addColumnData[ci] = new BarrageMessage.AddColumnData(); - msg.addColumnData[ci].type = columnTypes[ci]; - msg.addColumnData[ci].componentType = componentTypes[ci]; - msg.addColumnData[ci].data = new ArrayList<>(); - - msg.addColumnData[ci].data.add(null); - } - - // no mod column data - msg.modColumnData = new BarrageMessage.ModColumnData[0]; - - // generate empty row sets - msg.rowsRemoved = RowSetFactory.empty(); - msg.shifted = RowSetShiftData.EMPTY; - - msg.isSnapshot = true; + throw new IllegalStateException( + "Missing app metadata tag; cannot decode using BarrageStreamReader"); } bodyParsed = true; @@ -238,17 +216,22 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, final TLongIterator bufferInfoIter = bufferInfo.iterator(); // add and mod rows are never combined in a batch. all added rows must be received before the first - // mod rows will be received - - if (numAddRowsRead < numAddRowsTotal || (numAddRowsTotal == 0 && numModRowsTotal == 0)) { + // mod rows will be received. If no metadata is provided, assume there will be only one record + // batch + if (numAddRowsRead < numAddRowsTotal) { for (int ci = 0; ci < msg.addColumnData.length; ++ci) { final BarrageMessage.AddColumnData acd = msg.addColumnData[ci]; - int lastChunkIndex = acd.data.size() - 1; + final long remaining = numAddRowsTotal - numAddRowsRead; + if (batch.length() > remaining) { + throw new IllegalStateException( + "Batch length exceeded the expected number of rows from app metadata"); + } - // need to add the batch row data to the column chunks + // select the current chunk size and read the size + int lastChunkIndex = acd.data.size() - 1; WritableChunk chunk = (WritableChunk) acd.data.get(lastChunkIndex); - int chunkSize = chunk == null ? 0 : chunk.size(); + int chunkSize = acd.data.get(lastChunkIndex).size(); final int chunkOffset; long rowOffset = numAddRowsRead - lastAddStartIndex; @@ -257,9 +240,7 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, lastAddStartIndex += chunkSize; // create a new chunk before trying to write again - chunkSize = (int) (Math.min(numAddRowsTotal - numAddRowsRead, MAX_CHUNK_SIZE)); - // make sure the chunk will hold this batch (DoPut won't populate numAddRowsTotal) - chunkSize = Math.max((int) batch.length(), chunkSize); + chunkSize = (int) (Math.min(remaining, MAX_CHUNK_SIZE)); chunk = columnChunkTypes[ci].makeWritableChunk(chunkSize); acd.data.add(chunk); @@ -281,14 +262,14 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, for (int ci = 0; ci < msg.modColumnData.length; ++ci) { final BarrageMessage.ModColumnData mcd = msg.modColumnData[ci]; - int lastChunkIndex = mcd.data.size() - 1; + long remaining = mcd.rowsModified.size() - numModRowsRead; // need to add the batch row data to the column chunks + int lastChunkIndex = mcd.data.size() - 1; WritableChunk chunk = (WritableChunk) mcd.data.get(lastChunkIndex); int chunkSize = chunk.size(); final int chunkOffset; - long remaining = mcd.rowsModified.size() - numModRowsRead; long rowOffset = numModRowsRead - lastModStartIndex; // this batch might overflow the chunk if (rowOffset + Math.min(remaining, batch.length()) > chunkSize) { From 6228a5f8532505b37d9e51e52b14d7fabdab74b6 Mon Sep 17 00:00:00 2001 From: Larry Booker Date: Tue, 17 May 2022 09:16:12 -0700 Subject: [PATCH 47/47] minor edit of comment --- .../deephaven/extensions/barrage/util/BarrageStreamReader.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java index 62ef0fd41df..360d65f9347 100644 --- a/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java +++ b/extensions/barrage/src/main/java/io/deephaven/extensions/barrage/util/BarrageStreamReader.java @@ -216,8 +216,7 @@ public BarrageMessage safelyParseFrom(final StreamReaderOptions options, final TLongIterator bufferInfoIter = bufferInfo.iterator(); // add and mod rows are never combined in a batch. all added rows must be received before the first - // mod rows will be received. If no metadata is provided, assume there will be only one record - // batch + // mod rows will be received. if (numAddRowsRead < numAddRowsTotal) { for (int ci = 0; ci < msg.addColumnData.length; ++ci) { final BarrageMessage.AddColumnData acd = msg.addColumnData[ci];