Merge branch 'deephaven:main' into fix-java-coverage-enabled-in-build

Base/src/main/java/io/deephaven/base/AtomicUtil.java

@@ -5,6 +5,7 @@
 
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicLongFieldUpdater;
 
 public abstract class AtomicUtil {
 
@@ -267,4 +268,44 @@ public static int atomicAndNot(AtomicInteger i, int mask) {
         } while (!i.compareAndSet(expect, update));
         return update;
     }
+
+    /**
+     * Sets the field to the minimum of the current value and the passed in value
+     * 
+     * @param o the object to update
+     * @param fu the field updater
+     * @param value the value that is a candidate for the minumum
+     * @return true if the minimum was set
+     * @param <T> the type of o
+     */
+    public static <T> boolean setMin(final T o, final AtomicLongFieldUpdater<T> fu, final long value) {
+        long current = fu.get(o);
+        while (current > value) {
+            if (fu.compareAndSet(o, current, value)) {
+                return true;
+            }
+            current = fu.get(o);
+        }
+        return false;
+    }
+
+    /**
+     * Sets the field to the maximum of the current value and the passed in value
+     * 
+     * @param o the object to update
+     * @param fu the field updater
+     * @param value the value that is a candidate for the maximum
+     * @return true if the maximum was set
+     * @param <T> the type of o
+     */
+    public static <T> boolean setMax(final T o, final AtomicLongFieldUpdater<T> fu, final long value) {
+        long current = fu.get(o);
+        while (value > current) {
+            if (fu.compareAndSet(o, current, value)) {
+                return true;
+            }
+            current = fu.get(o);
+        }
+        return false;
+    }
 }

Base/src/main/java/io/deephaven/base/stats/ThreadSafeCounter.java

@@ -0,0 +1,30 @@
+//
+// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending
+//
+package io.deephaven.base.stats;
+
+import java.util.function.LongFunction;
+
+// --------------------------------------------------------------------
+
+/**
+ * A statistic where each value represents an additive quantity, and thus the sum of the values <U>does</U> have
+ * meaning. Examples include event counts and processing duration. If the sum of the values <I>does not</I> have a
+ * useful interpretation, use {@link State} instead.
+ * <UL>
+ * <LI>{@link #increment} updates the counter, recording a single value. This is the most common usage. ({@link #sample}
+ * does exactly the same thing but is a poor verb to use with a Counter.)
+ * </UL>
+ */
+public class ThreadSafeCounter extends ThreadSafeValue {
+
+    public ThreadSafeCounter(final long now) {
+        super(now);
+    }
+
+    public char getTypeTag() {
+        return Counter.TYPE_TAG;
+    }
+
+    public static final LongFunction<ThreadSafeCounter> FACTORY = ThreadSafeCounter::new;
+}

Base/src/main/java/io/deephaven/base/stats/ThreadSafeValue.java

@@ -0,0 +1,37 @@
+//
+// Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending
+//
+package io.deephaven.base.stats;
+
+import io.deephaven.base.AtomicUtil;
+
+import java.util.concurrent.atomic.AtomicLongFieldUpdater;
+
+/**
+ * A thread-safe extension of the {@link Value} class.
+ *
+ * <p>
+ * The {@link #sample(long)} method is synchronized, so may introduce contention compared to the unsafe Value version of
+ * sample.
+ * </p>
+ */
+public abstract class ThreadSafeValue extends Value {
+
+    public ThreadSafeValue(long now) {
+        super(now);
+    }
+
+    protected ThreadSafeValue(History history) {
+        super(history);
+    }
+
+    @Override
+    public synchronized void sample(final long x) {
+        super.sample(x);
+    }
+
+    @Override
+    public synchronized String toString() {
+        return super.toString();
+    }
+}

Base/src/main/java/io/deephaven/base/stats/Value.java

@@ -4,7 +4,10 @@
 package io.deephaven.base.stats;
 
 public abstract class Value {
-
+    /**
+     * The sample(long) method is not thread safe; and you can get wrong answers out of it. If you require safety, you
+     * should instead use a ThreadSafeValue.
+     */
     protected long n = 0;
     protected long last = 0;
     protected long sum = 0;
@@ -51,7 +54,7 @@ protected Value(History history) {
         this.history = history;
     }
 
-    public void sample(long x) {
+    public void sample(final long x) {
         n++;
         last = x;
         sum += x;
@@ -99,4 +102,15 @@ public void update(Item item, ItemUpdateListener listener, long logInterval, lon
             }
         }
     }
+
+    @Override
+    public String toString() {
+        if (n > 0) {
+            final double std = Math.sqrt(n > 1 ? (sum2 - ((double) sum * sum / (double) n)) / (n - 1) : Double.NaN);
+            final double avg = (double) sum / n;
+            return String.format("Value{n=%,d, sum=%,d, max=%,d, min=%,d, avg=%,.3f, std=%,.3f}", n, sum, max, min, avg,
+                    std);
+        }
+        return String.format("Value{n=%,d}", n);
+    }
 }

Base/src/test/java/io/deephaven/base/stats/TestValue.java

@@ -5,6 +5,7 @@
 
 import junit.framework.TestCase;
 
+import java.util.concurrent.Semaphore;
 import java.util.function.LongFunction;
 
 // --------------------------------------------------------------------
@@ -49,6 +50,44 @@ public void testCounter() {
         checkValue(Counter.FACTORY);
     }
 
+    public void testToString() {
+        // this is purposefully a heisentest, this should make it fail if it is really broken
+        for (int ii = 0; ii < 10; ++ii) {
+            doTestToString();
+        }
+    }
+
+    public void doTestToString() {
+        // we are testing toString, but also creating a pile of threads to exercise some of the AtomicFieldUpdater
+        // behavior of the value
+        final Value counter = ThreadSafeCounter.FACTORY.apply(0L);
+
+        final String emptyString = counter.toString();
+        assertEquals("Value{n=0}", emptyString);
+
+        final Semaphore semaphore = new Semaphore(0);
+        final Semaphore completion = new Semaphore(0);
+        final int n = 100;
+        for (int ii = 0; ii < n; ++ii) {
+            final int fii = ii;
+            new Thread(() -> {
+                semaphore.acquireUninterruptibly();
+                counter.sample(fii);
+                completion.release();
+            }).start();
+        }
+        semaphore.release(100);
+        completion.acquireUninterruptibly(100);
+
+        assertEquals(n, counter.getN());
+        assertEquals(((n - 1) * n) / 2, counter.getSum());
+        assertEquals(0, counter.getMin());
+        assertEquals(99, counter.getMax());
+
+        final String asString = counter.toString();
+        assertEquals("Value{n=100, sum=4,950, max=99, min=0, avg=49.500, std=29.011}", asString);
+    }
+
     // ----------------------------------------------------------------
     private void checkValue(LongFunction<? extends Value> factory) {
         Value value = factory.apply(1000L);

CONTRIBUTING.md

@@ -54,6 +54,11 @@ For more information, see:
 * [gh pr create](https://cli.github.com/manual/gh_pr_create)
 * [CLI In Use](https://cli.github.com/manual/examples.html)
 
+## Labels
+
+- Each pull request must have a `ReleaseNotesNeeded` label if a user can perceive the changes.  Build or testing-only changes should have a `NoReleaseNotesNeeded` label.
+- Each pull request must have a `DocumentationNeeded` label if the user guide should be updated.  Pull requests that do not require a user guide update should have a `NoDocumentationNeeded` label.
+
 ## Styleguide
 The [styleguide](style/README.md) is applied globally to the entire project, except for generated code that gets checked in.
 To apply the styleguide, run `./gradlew spotlessApply`.

Util/src/main/java/io/deephaven/util/annotations/InternalUseOnly.java

@@ -9,10 +9,11 @@
 import java.lang.annotation.Target;
 
 /**
- * Indicates that a particular method is for internal use only and should not be used by client code. It is subject to
- * change/removal at any time.
+ * Indicates that a particular {@link ElementType#METHOD method}, {@link ElementType#CONSTRUCTOR constructor},
+ * {@link ElementType#TYPE type}, or {@link ElementType#PACKAGE package} is for internal use only and should not be used
+ * by client code. It is subject to change/removal at any time.
  */
-@Target({ElementType.METHOD, ElementType.CONSTRUCTOR, ElementType.TYPE})
+@Target({ElementType.METHOD, ElementType.CONSTRUCTOR, ElementType.TYPE, ElementType.PACKAGE})
 @Inherited
 @Documented
 public @interface InternalUseOnly {

docker/registry/cpp-clients-multi-base/gradle.properties

@@ -1,5 +1,5 @@
 io.deephaven.project.ProjectType=DOCKER_REGISTRY
 deephaven.registry.imageName=ghcr.io/deephaven/cpp-clients-multi-base:latest
 # When the image version changes, remember to update cpp-client/build-dependencies.sh to match the image's content
-deephaven.registry.imageId=ghcr.io/deephaven/cpp-clients-multi-base@sha256:9a5efc669bf239d9c9f2ad6f934201219efa0112b93864f1afaa8bf510dedf49
+deephaven.registry.imageId=ghcr.io/deephaven/cpp-clients-multi-base@sha256:5d7dce6defbd30b3181a42d05988b763750467f2258e341b7431b378303cca4b
 deephaven.registry.platform=linux/amd64

docker/registry/go/gradle.properties

@@ -1,3 +1,3 @@
 io.deephaven.project.ProjectType=DOCKER_REGISTRY
 deephaven.registry.imageName=golang:1
-deephaven.registry.imageId=golang@sha256:73f06be4578c9987ce560087e2e2ea6485fb605e3910542cadd8fa09fc5f3e31
+deephaven.registry.imageId=golang@sha256:51a6466e8dbf3e00e422eb0f7a97ac450b2d57b33617bbe8d2ee0bddcd9d0d37

docker/registry/manylinux2014_x86_64/gradle.properties

@@ -1,4 +1,4 @@
 io.deephaven.project.ProjectType=DOCKER_REGISTRY
 deephaven.registry.imageName=quay.io/pypa/manylinux2014_x86_64:latest
-deephaven.registry.imageId=quay.io/pypa/manylinux2014_x86_64@sha256:24858373a047b97fd3a8fe2ee28709479a828fa0ed88719158b728947eb53270
+deephaven.registry.imageId=quay.io/pypa/manylinux2014_x86_64@sha256:52199398003bb163291b6822eea54457db8dd0550381b6b90a506f8f1cfb96b9
 deephaven.registry.platform=linux/amd64

docker/registry/minio/gradle.properties

@@ -1,3 +1,3 @@
 io.deephaven.project.ProjectType=DOCKER_REGISTRY
 deephaven.registry.imageName=minio/minio:latest
-deephaven.registry.imageId=minio/minio@sha256:ac591851803a79aee64bc37f66d77c56b0a4b6e12d9e5356380f4105510f2332
+deephaven.registry.imageId=minio/minio@sha256:1dce27c494a16bae114774f1cec295493f3613142713130c2d22dd5696be6ad3

docker/registry/protoc-base/gradle.properties

@@ -1,5 +1,5 @@
 io.deephaven.project.ProjectType=DOCKER_REGISTRY
 deephaven.registry.imageName=ghcr.io/deephaven/protoc-base:latest
-deephaven.registry.imageId=ghcr.io/deephaven/protoc-base@sha256:3cb04f768cfdb66b47154568a23772d42dcc55076c7cbb94ae950309d350a82a
+deephaven.registry.imageId=ghcr.io/deephaven/protoc-base@sha256:90f2a29c3b28d5e5a6cdc37d58d36fb413e4e6ce7964bd00f1d4e696fca46e18
 # TODO(deephaven-base-images#54): arm64 native image for cpp-client-base
 deephaven.registry.platform=linux/amd64

docker/registry/python/gradle.properties

@@ -1,3 +1,3 @@
 io.deephaven.project.ProjectType=DOCKER_REGISTRY
 deephaven.registry.imageName=python:3.10
-deephaven.registry.imageId=python@sha256:941b0bfddbf17d809fd1f457acbf55dfca014e3e0e3d592b1c9070491681bc02
+deephaven.registry.imageId=python@sha256:eb7df628043d68aa30019fed02052bd27f1431c3a0abe9299d1e4d804d4b11e0

docker/registry/selenium/gradle.properties

@@ -1,3 +1,3 @@
 io.deephaven.project.ProjectType=DOCKER_REGISTRY
 deephaven.registry.imageName=selenium/standalone-firefox:4.27.0
-deephaven.registry.imageId=selenium/standalone-firefox@sha256:287cd73a59d241b7264fe5fab7dbd31f92d56b8ec81ba74f53ecc4997f218dff
+deephaven.registry.imageId=selenium/standalone-firefox@sha256:29f5f0e96c32b219ad1101d259374346cc96d3b9b1820ca700dfdcac5bc332b9

docker/registry/server-base/gradle.properties

@@ -1,3 +1,3 @@
 io.deephaven.project.ProjectType=DOCKER_REGISTRY
 deephaven.registry.imageName=ghcr.io/deephaven/server-base:edge
-deephaven.registry.imageId=ghcr.io/deephaven/server-base@sha256:a904aba9cd5f91b6e2f19c4e024702700dfa37f549b73d24e9821f8f02061b45
+deephaven.registry.imageId=ghcr.io/deephaven/server-base@sha256:270e1a42772d5c68d0c49af1078cfa1bdeb659de786f8d04f7e78deb6fc85836

docker/registry/slim-base/gradle.properties

@@ -1,3 +1,3 @@
 io.deephaven.project.ProjectType=DOCKER_REGISTRY
 deephaven.registry.imageName=ghcr.io/deephaven/server-slim-base:edge
-deephaven.registry.imageId=ghcr.io/deephaven/server-slim-base@sha256:941e783bde091416f4b0359cc2a9df893f8340ada9bf4681f84875a91dfcebd1
+deephaven.registry.imageId=ghcr.io/deephaven/server-slim-base@sha256:14c6a88b60463f256496e3c582d7a00ce3ab7672912216582dde9adf3e33eeac

docker/server-jetty/src/main/server-jetty/requirements.txt

@@ -12,7 +12,7 @@ numba==0.60.0
 numpy==2.0.2
 pandas==2.2.3
 parso==0.8.4
-pyarrow==18.1.0
+pyarrow==19.0.0
 python-dateutil==2.9.0.post0
 pytz==2024.2
 six==1.17.0

docker/server/src/main/server-netty/requirements.txt

@@ -12,7 +12,7 @@ numba==0.60.0
 numpy==2.0.2
 pandas==2.2.3
 parso==0.8.4
-pyarrow==18.1.0
+pyarrow==19.0.0
 python-dateutil==2.9.0.post0
 pytz==2024.2
 six==1.17.0

engine/api/src/main/java/io/deephaven/engine/table/BasicDataIndex.java

@@ -96,16 +96,46 @@ default ColumnSource<?>[] keyColumns(@NotNull final ColumnSource<?>[] indexedCol
     @FinalDefault
     @NotNull
     default ColumnSource<RowSet> rowSetColumn() {
-        return table().getColumnSource(rowSetColumnName(), RowSet.class);
+        return rowSetColumn(DataIndexOptions.DEFAULT);
+    }
+
+    /**
+     * Get the {@link RowSet} {@link ColumnSource} of the index {@link #table() table}.
+     *
+     * @param options parameters for controlling how the the table will be built (if necessary) in order to retrieve the
+     *        result {@link RowSet} {@link ColumnSource}
+     *
+     * @return The {@link RowSet} {@link ColumnSource}
+     */
+    @FinalDefault
+    @NotNull
+    default ColumnSource<RowSet> rowSetColumn(final DataIndexOptions options) {
+        return table(options).getColumnSource(rowSetColumnName(), RowSet.class);
     }
 
     /**
      * Get the {@link Table} backing this data index.
+     *
+     * <p>
+     * The returned table is fully in-memory, equivalent to {@link #table(DataIndexOptions)} with default options.
+     * </p>
      * 
      * @return The {@link Table}
      */
     @NotNull
-    Table table();
+    default Table table() {
+        return table(DataIndexOptions.DEFAULT);
+    }
+
+    /**
+     * Get the {@link Table} backing this data index.
+     *
+     * @param options parameters to control the returned table
+     *
+     * @return The {@link Table}
+     */
+    @NotNull
+    Table table(DataIndexOptions options);
 
     /**
      * Whether the index {@link #table()} {@link Table#isRefreshing() is refreshing}. Some transformations will force
@@ -115,4 +145,5 @@ default ColumnSource<RowSet> rowSetColumn() {
      *         otherwise
      */
     boolean isRefreshing();
+
 }

engine/api/src/main/java/io/deephaven/engine/table/DataIndex.java

@@ -48,7 +48,21 @@ interface RowKeyLookup {
      * @return A function that provides map-like lookup of index {@link #table()} row keys from an index lookup key
      */
     @NotNull
-    RowKeyLookup rowKeyLookup();
+    default RowKeyLookup rowKeyLookup() {
+        return rowKeyLookup(DataIndexOptions.DEFAULT);
+    }
+
+    /**
+     * Build a {@link RowKeyLookup lookup function} of row keys for this index. If {@link #isRefreshing()} is
+     * {@code true}, this lookup function is only guaranteed to be accurate for the current cycle. Lookup keys should be
+     * in the order of the index's key columns.
+     *
+     * @param options parameters for building the table, if required by this RowKeyLookup
+     *
+     * @return A function that provides map-like lookup of index {@link #table()} row keys from an index lookup key
+     */
+    @NotNull
+    RowKeyLookup rowKeyLookup(DataIndexOptions options);
 
     /**
      * Return a {@link RowKeyLookup lookup function} function of index row keys for this index. If