Add region property to DAG

.github/workflows/tiledb-cloud-py.yaml

@@ -139,6 +139,14 @@ jobs:
             echo ".test_durations file does not exist."
           fi
 
+      - name: Run tests for vendored cloudpickle
+        # Test pinned dependencies on commit / tag and fresh installs on nightlies
+        if: (matrix.dependencies == 'fresh') == (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
+        run: |
+          pip install psutil
+          pip install src/tiledb/cloud/_vendor/cloudpickle/tests/cloudpickle_testpkg
+          pytest -sv src/tiledb/cloud/_vendor/cloudpickle
+
       - name: Run tests
         # Test pinned dependencies on commit / tag and fresh installs on nightlies
         if: (matrix.dependencies == 'fresh') == (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')

.gitignore

@@ -105,3 +105,4 @@ venv.bak/
 
 .idea
 src/tiledb/cloud/version.py
+.DS_Store

README.md

@@ -43,7 +43,7 @@ Documentation uses [Quarto](https://quarto.org/) style documentation.
 This package contains modules and functions that will be run in the TileDB Cloud as UDFs. Local ingestor changes can be tested in the cloud by using a feature of [cloudpickle](https://github.com/cloudpipe/cloudpickle?tab=readme-ov-file#overriding-pickles-serialization-mechanism-for-importable-constructs). Below is a runnable example. `TILEDB_NAMESPACE` is your TileDB namespace ("TileDB-Inc", for example). `TILEDB_ACCESS_CREDENTIAL_NAME` is the name of the stored credentials for accessing `AWS_BUCKET`. `IMAGE_FILE_KEY` is the key for an object in that bucket and `OUTPUT_GROUP_KEY` is the key to be used for the group that the `ingest()` UDF will create in `AWS_BUCKET`.
 
 ```python
-import cloudpickle
+from tiledb.cloud._vendor import cloudpickle
 
 import tiledb.cloud.bioimg
 
@@ -59,7 +59,7 @@ tiledb.cloud.bioimg.ingest(
 )
 ```
 
-In this case `tiledb.cloud.bioimg.ingest()` uses cloudpickle to send a local function to TileDB Cloud, and `cloudpickle.register_pickle_by_value(tiledb.cloud.bioimg)` directs cloudpickle to bring the currently imported `tiledb.cloud.bioimg` module along with the function. Your local version of the module will be used instead of the version currently deployed in TileDB Cloud.
+In this case `tiledb.cloud.bioimg.ingest()` uses cloudpickle to send a local function to TileDB Cloud, and `tdbcp.register_pickle_by_value(tiledb.cloud.bioimg)` directs cloudpickle to bring the currently imported `tiledb.cloud.bioimg` module along with the function. Your local version of the module will be used instead of the version currently deployed in TileDB Cloud.
 
 Note: your local changes to the Cloud-Py package will need to be installed in order for cloudpickle to serialize them, as cloudpickle needs to find them at runtime.
 

ci/requirements-geospatial-py3.9.txt

@@ -1,6 +1,5 @@
 attrs==23.1.0
 certifi==2023.7.22
-cloudpickle==2.2.1
 importlib-metadata==6.8.0
 fiona==1.10b1
 numpy==1.26.1

ci/requirements-py3.9.txt

@@ -1,6 +1,6 @@
 attrs==23.2.0
 certifi==2024.2.2
-cloudpickle==2.2.1
+cloudpickle
 importlib-metadata==7.0.1
 numpy==1.26.4
 packaging==23.2

docs/_quarto.yaml

@@ -30,10 +30,10 @@ website:
             href: reference/array.qmd
           - text: asset
             href: reference/asset.qmd
-          - text: files
-            href: reference/files.qmd
           - text: notebook
             href: reference/notebook.qmd
+          - text: dashboard
+            href: reference/dashboard.qmd
           - text: udf
             href: reference/udf.qmd
 
@@ -52,8 +52,6 @@ website:
         contents:
           - text: bioimg.exportation
             href: reference/bioimg.exportation.qmd
-          - text: bioimg.helpers
-            href: reference/bioimg.helpers.qmd
           - text: bioimg.ingestion
             href: reference/bioimg.ingestion.qmd
           - text: files.indexing
@@ -72,7 +70,7 @@ website:
             href: reference/soma.mapper.qmd
           - text: vcf.allele_frequency
             href: reference/vcf.allele_frequency.qmd
-          - text: vcf.ingestionq
+          - text: vcf.ingestion
             href: reference/vcf.ingestion.qmd
           - text: vcf.query
             href: reference/vcf.query.qmd
@@ -91,12 +89,18 @@ website:
             href: reference/dag.dag.qmd
           - text: dag.mode
             href: reference/dag.mode.qmd
-          - text: dag.status
-            href: reference/dag.status.qmd
           - text: dag.visualization
             href: reference/dag.visualization.qmd
           - text: tasks
             href: reference/tasks.qmd
+          - text: taskgraphs.builder
+            href: reference/taskgraphs.builder.qmd
+          - text: taskgraphs.executor
+            href: reference/taskgraphs.executor.qmd
+          - text: taskgraphs.registration
+            href: reference/taskgraphs.registration.qmd
+          - text: taskgraphs.types
+            href: reference/taskgraphs.types.qmd
           - text: udf
             href: reference/udf.qmd
           - text: utilities.consolidate
@@ -129,8 +133,8 @@ quartodoc:
         - groups
         - array
         - asset
-        - files
         - notebook
+        - dashboard
         - udf
 
     - title: Collaborate
@@ -147,7 +151,6 @@ quartodoc:
     - subtitle: Bio Imaging
       contents:
         - bioimg.exportation
-        - bioimg.helpers
         - bioimg.ingestion
 
     - subtitle: Files
@@ -181,9 +184,12 @@ quartodoc:
         - compute.delayed
         - dag.dag
         - dag.mode
-        - dag.status
         - dag.visualization
         - tasks
+        - taskgraphs.builder
+        - taskgraphs.executor
+        - taskgraphs.registration
+        - taskgraphs.types
         - udf
         - utilities.consolidate
         - utilities.profiler

pyproject.toml

@@ -9,7 +9,6 @@ dependencies = [
     "anndata",
     "attrs>=21.4.0",
     "certifi",
-    "cloudpickle>=1.4.1,<3",
     "importlib-metadata",
     "packaging",
     "pandas>=1.2.4",
@@ -66,6 +65,7 @@ write_to = "src/tiledb/cloud/version.py"
 
 [tool.ruff]
 extend-select = ["I"]
+exclude = ["src/tiledb/cloud/_vendor"]
 
 [tool.ruff.isort]
 known-first-party = ["tiledb"]
@@ -82,3 +82,6 @@ single-line-exclusions = ["typing", "typing_extensions"]
 # Certain tests have long JSON strings and deep structures.
 "tests/taskgraphs/test_builder.py" = ["E501"]
 "tests/test_geospatial.py" = ["E501"]
+
+[tool.black]
+force-exclude = 'src\/tiledb\/cloud\/_vendor\/.*'

src/tiledb/cloud/_common/pickle_compat.py

@@ -14,11 +14,12 @@
 import sys
 import types
 
-import cloudpickle.cloudpickle as cpcp
 import importlib_metadata
 import numpy
 import packaging.version as pkgver
 
+import tiledb.cloud._vendor.cloudpickle.cloudpickle as cpcp
+
 
 def patch_cloudpickle() -> None:
     """Make older cloudpickle versions able to unpickle new function pickles."""

src/tiledb/cloud/_common/utils.py

@@ -7,10 +7,10 @@
 from enum import Enum
 from typing import Any, Callable, Optional, Tuple, Type, TypeVar, Union
 
-import cloudpickle
 import urllib3
 
 from tiledb.cloud._common import functions
+from tiledb.cloud._vendor import cloudpickle as tdbcp
 
 TILEDB_CLOUD_PROTOCOL = 4
 PYTHON_VERSION = ".".join(map(str, sys.version_info[:3]))
@@ -103,7 +103,7 @@ def split_uri(uri: str) -> Tuple[str, str]:
 
 def b64_pickle(obj: Any) -> str:
     """Pickles the given object, then base64 encodes the pickle."""
-    pickle = cloudpickle.dumps(obj, protocol=TILEDB_CLOUD_PROTOCOL)
+    pickle = tdbcp.dumps(obj, protocol=TILEDB_CLOUD_PROTOCOL)
     return base64.b64encode(pickle).decode("ascii")
 
 

src/tiledb/cloud/_results/codecs.py

@@ -5,11 +5,12 @@
 from typing import TYPE_CHECKING, Any, Generic, Tuple, Type, TypeVar
 
 import attrs
-import cloudpickle
 import pyarrow
 import urllib3
 from typing_extensions import Self, TypeGuard
 
+from tiledb.cloud._vendor import cloudpickle as tdbcp
+
 # This is a circular dependency since we need to be able to decode `tiledb_json`
 # format data.
 from . import tiledb_json
@@ -148,11 +149,11 @@ class PickleCodec(Codec[object]):
 
     @classmethod
     def encode(cls, obj: object) -> bytes:
-        return cloudpickle.dumps(obj, protocol=_PICKLE_PROTOCOL)
+        return tdbcp.dumps(obj, protocol=_PICKLE_PROTOCOL)
 
     @classmethod
     def decode(cls, data: bytes) -> object:
-        return cloudpickle.loads(data)
+        return tdbcp.loads(data)
 
 
 class TileDBJSONCodec(Codec[object]):

src/tiledb/cloud/_vendor/cloudpickle/LICENSE

@@ -0,0 +1,32 @@
+This module was extracted from the `cloud` package, developed by
+PiCloud, Inc.
+
+Copyright (c) 2015, Cloudpickle contributors.
+Copyright (c) 2012, Regents of the University of California.
+Copyright (c) 2009 PiCloud, Inc. http://www.picloud.com.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+    * Neither the name of the University of California, Berkeley nor the
+      names of its contributors may be used to endorse or promote
+      products derived from this software without specific prior written
+      permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.