Merge pull request #35 from theislab/feature/simplfy_10ktp

Simplify 10ktp

.github/workflows/build.yaml

@@ -14,11 +14,11 @@ jobs:
   package:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.11"
           cache: "pip"
           cache-dependency-path: "**/pyproject.toml"
       - name: Install build dependencies

.github/workflows/test.yaml

@@ -26,9 +26,9 @@ jobs:
           - os: ubuntu-latest
             python: "3.10"
           - os: ubuntu-latest
-            python: "3.12"
+            python: "3.11"
           - os: ubuntu-latest
-            python: "3.12"
+            python: "3.11"
             pip-flags: "--pre"
             name: PRE-RELEASE DEPENDENCIES
 
@@ -39,20 +39,20 @@ jobs:
       PYTHON: ${{ matrix.python }}
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
           cache: "pip"
           cache-dependency-path: "**/pyproject.toml"
 
       - name: Install test dependencies
         run: |
-          python -m pip install --upgrade pip wheel
+          python -m pip install --upgrade uv wheel
       - name: Install dependencies
         run: |
-          pip install ${{ matrix.pip-flags }} ".[dev,test]"
+          uv pip install --system ${{ matrix.pip-flags }} ".[dev,test]"
       - name: Test
         env:
           MPLBACKEND: agg

.gitignore

@@ -28,3 +28,4 @@ __pycache__/
 *.pye
 
 Sandbox.ipynb
+test.ipynb

data/10ktp__10X__Visium__Mouse__brain__20200623__v1.1.0/10ktp_make_vitessce.py

@@ -1,18 +1,8 @@
-# ADJUST
-uid = "10ktp"
-
-# Setup before running this script for the first time
-#   1) Ensure you have a lamin.ai account
-#   2) Run `lamin login` to authenticate
-#   3) Activate lamin db connection with `lamin load scverse/spatialdata-db`
-#   4) Execute script once and replace the stem_uid for this file
+theislab_uid = "10ktp"
 
 import lamindb as ln
 
-ln.settings.transform.stem_uid = "5PYtTxDE7LvE"
-ln.settings.transform.version = "1"
-
-ln.track()
+ln.context.track("5PYtTxDE7LvE")
 
 from vitessce import (
     VitessceConfig,
@@ -31,7 +21,7 @@
 from lamindb.integrations import save_vitessce_config
 
 
-artifact = ln.Artifact.filter(ulabels__name=uid).one()
+artifact = ln.Artifact.filter(ulabels__name=theislab_uid).one()
 
 vc = VitessceConfig(
     schema_version="1.0.16",
@@ -49,4 +39,4 @@
 dataset = vc.add_dataset(name='Visium demo').add_object(wrapper)
 spatial = vc.add_view("spatialBeta", dataset=dataset)
 vc.layout(spatial)
-view = save_vitessce_config(vc, description="Visium HD demo vitessce config")
+view = save_vitessce_config(vc, description="Visium HD demo vitessce config")

data/10ktp__10X__Visium__Mouse__brain__20200623__v1.1.0/10ktp_to_lamin.py

@@ -1,42 +1,31 @@
-# ADJUST
-uid = "10ktp"
-
-# Setup before running this script for the first time
-#   1) Ensure you have a lamin.ai account
-#   2) Run `lamin login` to authenticate
-#   3) Activate lamin db connection with `lamin load scverse/spatialdata-db`
-#   4) Execute script once and replace the stem_uid for this file
+theislab_uid = "10ktp"
 
 import lamindb as ln
+from spatialdata_db import load_10x_metadata
 
-ln.settings.transform.stem_uid = "KgGzOw8PUYKO"
-ln.settings.transform.version = "3"
 ln.track("KgGzOw8PUYKO7CpM")
+
 try:
-    artifact = ln.Artifact.filter(ulabels__name=uid).one()
+    artifact = ln.Artifact.filter(ulabels__name=theislab_uid).one()
     artifact.delete(permanent=True)
 except:
     pass
 
-### From here on we assume Lamin is set up correctly
 DATASET_PATH = "/lustre/groups/ml01/projects/2024_spatialdata_db/data/10ktp__10X__Visium__Mouse__brain__20200623__v1.1.0/10ktp__10X__Visium__Mouse__brain__20200623__v1.1.0.zarr"
 
 artifact = ln.Artifact(DATASET_PATH, description="10X, Visium, Mouse, Brain")
-artifact.save() # transfers to Lamin
+artifact.save()
 
 # Associate ID with the artifact so we can retrieve afterwards
 tuid_parent = ln.ULabel.filter(name="theislab unique ID").one()
-tuid = ln.ULabel(name=uid).save()
+tuid = ln.ULabel(name=theislab_uid).save()
 tuid.parents.add(tuid_parent)
 artifact.labels.add(tuid)
 
 # load 10X metadata we have on disk
-import pandas as pd
-from spatialdata_db import load_10x_metadata
-
 all_metadata_10x = load_10x_metadata()
-assert len(all_metadata_10x.query(f"uid == '{uid}'")) == 1
-metadata = all_metadata_10x.query(f"uid == '{uid}'").iloc[0]
+assert len(all_metadata_10x.query(f"uid == '{theislab_uid}'")) == 1
+metadata = all_metadata_10x.query(f"uid == '{theislab_uid}'").iloc[0]
 
 # Associate metadata as features
 import bionty as bt
@@ -45,39 +34,18 @@
 organism_lo = bt.Organism.public().lookup()
 tissue_lo = bt.Tissue.public().lookup()
 
-# Species --
-
-if metadata["Species"].lower() == "mouse":
-    feature_organism = bt.Organism.from_public(name=organism_lo.mouse.name)
-elif metadata["Species"].lower() == "human":
-    feature_organism = bt.Organism.from_public(name=organism_lo.human.name)
-else:
-    raise NotImplementedError("Unknown species!")
-
-feature_organism.save()
-
-# Tissue
+# Species (Organism)
+organisms = bt.Organism.from_values(metadata["Species"].lower(), field="name")
+ln.save(organisms)
 
+# Orgna (Tissue)
 if metadata["organ"].lower() == "olfactory_bulb":
     metadata["organ"] = "olfactorybulb"
-
-if metadata["organ"].lower() == "brain":
-    feature_tissue = bt.Tissue.from_public(name=tissue_lo.brain.name)
-elif metadata["organ"].lower() == "kidney":
-    feature_tissue = bt.Tissue.from_public(name=tissue_lo.kidney.name)
-elif metadata["organ"].lower() == "colon":
-    feature_tissue = bt.Tissue.from_public(name=tissue_lo.colon.name)
-else:
-    raise NotImplementedError("Unknown tissue!")
-
-feature_tissue.save()
+tissues = bt.Tissue.from_values(metadata["organ"].lower(), field="name")
+ln.save(tissues)
 
 # Associate bionty terms to artifact
+artifact.organism.set(organisms)
+artifact.tissue.set(tissues)
 
-artifact.labels.add(feature_organism, feature=feature_lo.organism)
-artifact.labels.add(feature_tissue, feature=feature_lo.tissue)
-
-
-# finish for tracing in Lamin
-
-ln.finish()
+ln.context.finish()

pyproject.toml

@@ -14,18 +14,23 @@ maintainers = [
 authors = [
   { name = "Tim Treis" },
 ]
-requires-python = ">=3.9"
+requires-python = ">=3.10, <3.12"
+# 3.12 attempts to install an old version of dask-image which isn't 3.12 compatible
+# See https://github.com/theislab/spatialdata-db/actions/runs/11230700843/job/31218561107?pr=35
 classifiers = [
   "Programming Language :: Python :: 3 :: Only",
-  "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
 ]
 dependencies = [
   "anndata",
   # for debug logging (referenced from the issue template)
   "session-info",
-  "lamindb[aws,bionty,jupyter]"
+  "lamindb[aws,bionty,jupyter]",
+  "vitessce",
+  "spatialdata",
+  "spatialdata-io",
+  "spatialdata-plot"
 ]
 optional-dependencies.dev = [
   "pre-commit",

src/spatialdata_db/utils/scripts/create_uids.ipynb → scripts/create_uids.ipynb

@@ -198,7 +198,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "lamin",
+   "display_name": "spatialdatadb",
    "language": "python",
    "name": "python3"
   },
@@ -212,7 +212,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.8"
+   "version": "3.11.10"
   }
  },
  "nbformat": 4,

src/spatialdata_db/utils/scripts/download_10X_data.ipynb → scripts/download_10X_data.ipynb

@@ -175,7 +175,7 @@
     }
    ],
    "source": [
-    "data = pd.read_csv(\"/home/icb/tim.treis/projects/spatialdata-db/utils/data/10x_datasets.csv\")[:2]\n",
+    "data = pd.read_csv(\"/home/icb/tim.treis/projects/spatialdata-db/scripts/data/10x_datasets.csv\")[:2]\n",
     "data"
    ]
   },

src/spatialdata_db/utils/scripts/download_10X_data.py → scripts/download_10X_data.py

@@ -22,7 +22,7 @@
 # In[2]:
 
 
-data = pd.read_csv("/home/icb/tim.treis/projects/spatialdata-db/utils/data/10x_datasets.csv")[:2]
+data = pd.read_csv("/home/icb/tim.treis/projects/spatialdata-db/scripts/data/10x_datasets.csv")[:2]
 data
 
 

src/spatialdata_db/utils/scripts/rename_on_lustre.ipynb → scripts/rename_on_lustre.ipynb

@@ -18,7 +18,7 @@
     "import os\n",
     "from concurrent.futures import ProcessPoolExecutor, as_completed\n",
     "\n",
-    "tenx = pd.read_csv(\"../data/datasets_10x.csv\", sep=\";\")"
+    "tenx = pd.read_csv(\"data/datasets_10x.csv\", sep=\";\")"
    ]
   },
   {

src/spatialdata_db/utils/scripts/update_uid_master.py → scripts/update_uid_master.py

@@ -30,9 +30,9 @@ def update_master_with_ids(uid_master_csv_path, file_paths):
 
 
 if __name__ == "__main__":
-    master_csv_path = "../data/uid_master.csv" 
+    master_csv_path = "data/uid_master.csv" 
     dataset_paths = [
-        "../data/datasets_10x.csv",
+        "data/datasets_10x.csv",
         # tbd
     ]