Introduce APIs for list_recordings and update_metadata (#8223)

### Related
- rerun-io/dataplatform#17

### What
- Python API support for the list / update APIs
- Example script that uses them for a simple CLI app

Cargo.lock

@@ -6970,6 +6970,7 @@ dependencies = [
  "re_web_viewer_server",
  "re_ws_comms",
  "tokio",
+ "tokio-stream",
  "tonic",
  "url",
  "uuid",

examples/python/remote/metadata.py

@@ -0,0 +1,41 @@
+"""Script to show how to interact with a remote storage node via python APIs."""
+
+from __future__ import annotations
+
+import argparse
+
+import polars as pl
+import pyarrow as pa
+import rerun as rr
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+
+    subparsers = parser.add_subparsers(dest="subcommand")
+
+    print_cmd = subparsers.add_parser("print", help="Print everything")
+    update_cmd = subparsers.add_parser("update", help="Update metadata for a recording")
+
+    update_cmd.add_argument("id", help="ID of the recording to update")
+    update_cmd.add_argument("key", help="Key of the metadata to update")
+    update_cmd.add_argument("value", help="Value of the metadata to update")
+
+    args = parser.parse_args()
+
+    # Register the new rrd
+    conn = rr.remote.connect("http://0.0.0.0:51234")
+
+    catalog = pl.from_arrow(conn.list_recordings())
+
+    if args.subcommand == "print":
+        print(catalog)
+
+    if args.subcommand == "update":
+        id = catalog.filter(catalog["id"].str.starts_with(args.id)).select(pl.first("id")).item()
+
+        if id is None:
+            print("ID not found")
+            exit(1)
+        print(f"Updating metadata for {id}")
+
+        conn.update_metadata(id, {args.key: pa.array([args.value])})

pixi.toml

@@ -605,6 +605,7 @@ python = "=3.11"
 [feature.examples-common.pypi-dependencies]
 # External deps
 jupyter = ">=1.0"
+polars = ">=0.12.0"
 
 segment-anything = { git = "https://github.com/facebookresearch/segment-anything.git" }
 mesh-to-sdf = { git = "https://github.com/marian42/mesh_to_sdf.git" }

rerun_py/Cargo.toml

@@ -40,6 +40,7 @@ remote = [
   "dep:re_protos",
   "dep:re_ws_comms",
   "dep:tokio",
+  "dep:tokio-stream",
   "dep:tonic",
   "dep:url",
 ]
@@ -87,6 +88,7 @@ uuid.workspace = true
 object_store = { workspace = true, optional = true, features = ["aws"] }
 re_protos = { workspace = true, optional = true }
 tokio = { workspace = true, optional = true }
+tokio-stream = { workspace = true, optional = true }
 # Not used yet, but we will need it when we start streaming data
 #tokio-stream = { workspace = true, optional = true }
 tonic = { workspace = true, default-features = false, features = [

rerun_py/rerun_bindings/rerun_bindings.pyi

@@ -3,7 +3,7 @@ from typing import Iterator, Optional, Sequence, Union
 
 import pyarrow as pa
 
-from .types import AnyColumn, AnyComponentColumn, ComponentLike, IndexValuesLike, ViewContentsLike
+from .types import AnyColumn, AnyComponentColumn, ComponentLike, IndexValuesLike, MetadataLike, ViewContentsLike
 
 class IndexColumnDescriptor:
     """
@@ -567,3 +567,80 @@ def load_archive(path_to_rrd: str | os.PathLike) -> RRDArchive:
 
     """
     ...
+
+class StorageNodeClient:
+    """
+    A client for interfacing with a Rerun storage node.
+
+    Required-feature: `remote`
+    """
+
+    def list_recordings(self) -> pa.RecordBatchReader:
+        """Get the metadata for all recordings in the storage node."""
+        ...
+
+    def register(self, storage_url: str, metadata: Optional[dict[str, MetadataLike]] = None) -> str:
+        """
+        Register a recording along with some metadata.
+
+        Parameters
+        ----------
+        storage_url : str
+            The URL to the storage location.
+        metadata : dict[str, MetadataLike]
+            A dictionary where the keys are the metadata columns and the values are pyarrow arrays.
+
+        """
+        ...
+
+    def update_metadata(self, id: str, metadata: dict[str, MetadataLike]) -> None:
+        """
+        Update the metadata for the recording with the given id.
+
+        Parameters
+        ----------
+        id : str
+            The id of the recording to update.
+        metadata : dict[str, MetadataLike]
+            A dictionary where the keys are the metadata columns and the values are pyarrow arrays.
+
+        """
+        ...
+
+    def open_recording(self, id: str) -> Recording:
+        """
+        Open a [`Recording`][rerun.dataframe.Recording] by id to use with the dataframe APIs.
+
+        This currently downloads the full recording to the local machine.
+
+        Parameters
+        ----------
+        id : str
+            The id of the recording to open.
+
+        Returns
+        -------
+        Recording
+            The opened recording.
+
+        """
+        ...
+
+def connect(addr: str) -> StorageNodeClient:
+    """
+    Load a rerun archive from an RRD file.
+
+    Required-feature: `remote`
+
+    Parameters
+    ----------
+    addr : str
+        The address of the storage node to connect to.
+
+    Returns
+    -------
+    StorageNodeClient
+        The connected client.
+
+    """
+    ...

rerun_py/rerun_bindings/types.py

@@ -67,3 +67,5 @@
 
 This can be any numpy-compatible array of integers, or a [`pa.Int64Array`][]
 """
+
+MetadataLike: TypeAlias = pa.Array

rerun_py/rerun_sdk/rerun/remote.py

@@ -2,9 +2,10 @@
 
 try:
     from rerun_bindings import (
+        StorageNodeClient as StorageNodeClient,
         connect as connect,
     )
 except ImportError:
 
-    def connect(url: str) -> None:
+    def connect(addr: str) -> StorageNodeClient:
         raise NotImplementedError("Rerun SDK was built without the `remote` feature enabled.")

rerun_py/src/dataframe.rs

@@ -565,8 +565,8 @@ impl PySchema {
 /// to retrieve the data.
 #[pyclass(name = "Recording")]
 pub struct PyRecording {
-    store: ChunkStoreHandle,
-    cache: re_dataframe::QueryCacheHandle,
+    pub(crate) store: ChunkStoreHandle,
+    pub(crate) cache: re_dataframe::QueryCacheHandle,
 }
 
 /// A view of a recording restricted to a given index, containing a specific set of entities and components.