DH-18300: Improve DataIndex performance.

DataIndex, particularly when used for where() filters had missing
parallelization opportunities; and would read more data than strictly
necessary to satisfy the filter.

Statistics have been added to various operations, the existing Value class
was not thread safe.  The internal state has been updated to use volatiles
and AtomicLongFieldUpdaters.

The following Configuration properties have been added:
- AbstractColumnSource.usePartialDataIndex
- AbstractColumnSource.useParallelIndexBuild
- QueryTable.useDataIndexForAggregation
- MergedDataIndex.useParallelLazyFetch

Base/src/main/java/io/deephaven/base/stats/Value.java

@@ -3,14 +3,22 @@
 //
 package io.deephaven.base.stats;
 
-public abstract class Value {
+import java.text.DecimalFormat;
+import java.util.concurrent.atomic.AtomicLongFieldUpdater;
 
-    protected long n = 0;
-    protected long last = 0;
-    protected long sum = 0;
-    protected long sum2 = 0;
-    protected long max = Long.MIN_VALUE;
-    protected long min = Long.MAX_VALUE;
+public abstract class Value {
+    private static final AtomicLongFieldUpdater<Value> N_UPDATER = AtomicLongFieldUpdater.newUpdater(Value.class, "n");
+    private static final AtomicLongFieldUpdater<Value> SUM_UPDATER =
+            AtomicLongFieldUpdater.newUpdater(Value.class, "sum");
+    private static final AtomicLongFieldUpdater<Value> SUM2_UPDATER =
+            AtomicLongFieldUpdater.newUpdater(Value.class, "sum2");
+
+    volatile protected long n = 0;
+    volatile protected long last = 0;
+    volatile protected long sum = 0;
+    volatile protected long sum2 = 0;
+    volatile protected long max = Long.MIN_VALUE;
+    volatile protected long min = Long.MAX_VALUE;
 
     private boolean alwaysUpdated = false;
 
@@ -52,10 +60,10 @@ protected Value(History history) {
     }
 
     public void sample(long x) {
-        n++;
+        N_UPDATER.incrementAndGet(this);
+        SUM_UPDATER.addAndGet(this, x);
+        SUM2_UPDATER.addAndGet(this, x * x);
         last = x;
-        sum += x;
-        sum2 += x * x;
         if (x > max) {
             max = x;
         }
@@ -99,4 +107,23 @@ public void update(Item item, ItemUpdateListener listener, long logInterval, lon
             }
         }
     }
+
+    @Override
+    public String toString() {
+        final DecimalFormat format = new DecimalFormat("#,###");
+        final DecimalFormat avgFormat = new DecimalFormat("#,###.###");
+
+        final double variance = n > 1 ? (sum2 - ((double) sum * sum / (double) n)) / (n - 1) : Double.NaN;
+
+        return "Value{" +
+                "n=" + format.format(n) +
+                (n > 0 ? ", sum=" + format.format(sum) +
+                        ", max=" + format.format(max) +
+                        ", min=" + format.format(min) +
+                        ", avg=" + avgFormat.format((n > 0 ? (double) sum / n : Double.NaN)) +
+                        ", std=" + avgFormat.format(Math.sqrt(variance))
+                        : "")
+                +
+                '}';
+    }
 }

engine/api/src/main/java/io/deephaven/engine/table/BasicDataIndex.java

@@ -96,16 +96,43 @@ default ColumnSource<?>[] keyColumns(@NotNull final ColumnSource<?>[] indexedCol
     @FinalDefault
     @NotNull
     default ColumnSource<RowSet> rowSetColumn() {
-        return table().getColumnSource(rowSetColumnName(), RowSet.class);
+        return rowSetColumn(DataIndexOptions.DEFAULT);
+    }
+
+    /**
+     * Get the {@link RowSet} {@link ColumnSource} of the index {@link #table() table}.
+     *
+     * @return The {@link RowSet} {@link ColumnSource}
+     */
+    @FinalDefault
+    @NotNull
+    default ColumnSource<RowSet> rowSetColumn(final DataIndexOptions options) {
+        return table(options).getColumnSource(rowSetColumnName(), RowSet.class);
     }
 
     /**
      * Get the {@link Table} backing this data index.
+     *
+     * <p>
+     * The returned table is fully in-memory, equivalent to {@link #table(DataIndexOptions)} with default options.
+     * </p>
      * 
      * @return The {@link Table}
      */
     @NotNull
-    Table table();
+    default Table table() {
+        return table(DataIndexOptions.DEFAULT);
+    }
+
+    /**
+     * Get the {@link Table} backing this data index.
+     *
+     * @param options parameters to control the returned table
+     *
+     * @return The {@link Table}
+     */
+    @NotNull
+    Table table(DataIndexOptions options);
 
     /**
      * Whether the index {@link #table()} {@link Table#isRefreshing() is refreshing}. Some transformations will force
@@ -115,4 +142,5 @@ default ColumnSource<RowSet> rowSetColumn() {
      *         otherwise
      */
     boolean isRefreshing();
+
 }

engine/api/src/main/java/io/deephaven/engine/table/DataIndex.java

@@ -48,7 +48,12 @@ interface RowKeyLookup {
      * @return A function that provides map-like lookup of index {@link #table()} row keys from an index lookup key
      */
     @NotNull
-    RowKeyLookup rowKeyLookup();
+    default RowKeyLookup rowKeyLookup() {
+        return rowKeyLookup(DataIndexOptions.DEFAULT);
+    }
+
+    @NotNull
+    RowKeyLookup rowKeyLookup(DataIndexOptions options);
 
     /**
      * Return a {@link RowKeyLookup lookup function} function of index row keys for this index. If

engine/api/src/main/java/io/deephaven/engine/table/DataIndexOptions.java

@@ -0,0 +1,65 @@
+//
+// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending
+//
+package io.deephaven.engine.table;
+
+import io.deephaven.annotations.BuildableStyle;
+import io.deephaven.api.filter.Filter;
+import org.immutables.value.Value;
+
+/**
+ * Options for controlling the function of a {@link DataIndex}.
+ *
+ * <p>
+ * Presently, this is used for the {@link Table#where(Filter)} operation to more efficiently handle data index matches,
+ * without necessarily reading all RowSet information from disk across partitions.
+ * </p>
+ */
+@Value.Immutable
+@BuildableStyle
+public interface DataIndexOptions {
+    DataIndexOptions DEFAULT = DataIndexOptions.builder().build();
+
+    /**
+     * Does this operation use only a subset of the DataIndex?
+     *
+     * <p>
+     * The DataIndex implementation may use this hint to defer work for some row sets.
+     * </p>
+     *
+     * @return if this operation is only going to use a subset of this data index
+     */
+    @Value.Default
+    default boolean operationUsesPartialTable() {
+        return false;
+    }
+
+    /**
+     * Create a new builder for a {@link DataIndexOptions}.
+     * 
+     * @return
+     */
+    static Builder builder() {
+        return ImmutableDataIndexOptions.builder();
+    }
+
+    /**
+     * The builder interface to construct a {@link DataIndexOptions}.
+     */
+    interface Builder {
+        /**
+         * Set whether this operation only uses a subset of the data index.
+         *
+         * @param usesPartialTable true if this operation only uses a partial table
+         * @return this builder
+         */
+        Builder operationUsesPartialTable(boolean usesPartialTable);
+
+        /**
+         * Build the {@link DataIndexOptions}.
+         * 
+         * @return an immutable DataIndexOptions structure.
+         */
+        DataIndexOptions build();
+    }
+}