From 8996cdc3408c58f0846554426ddb64ee8f0995c1 Mon Sep 17 00:00:00 2001 From: Raymond Lim Date: Thu, 24 Aug 2023 11:53:23 -0700 Subject: [PATCH 1/2] feat: add singularity runner --- poetry.lock | 192 +++++++++--------- pyproject.toml | 1 + src/luna/common/runners.py | 96 +++++++++ .../cli/run_stardist_cell_detection.py | 128 ++++++++---- 4 files changed, 283 insertions(+), 134 deletions(-) create mode 100644 src/luna/common/runners.py diff --git a/poetry.lock b/poetry.lock index 9a9e8155..44d528c4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -560,11 +560,11 @@ test-no-codebase = ["matplotlib", "pillow", "pytest"] [[package]] name = "coverage" -version = "7.2.7" +version = "7.3.0" description = "Code coverage measurement for Python" category = "dev" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" [package.dependencies] tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} @@ -1880,59 +1880,59 @@ python-versions = ">=3.7" [[package]] name = "large-image" -version = "1.23.2" +version = "1.23.3" description = "Python modules to work with large, multiresolution images." category = "main" optional = false python-versions = ">=3.6" [package.dependencies] -cachetools = ">=3.0.0" -large-image-source-pil = {version = ">=1.23.2", optional = true, markers = "extra == \"pil\""} -numpy = ">=1.10.4" +cachetools = "*" +large-image-source-pil = {version = ">=1.23.3", optional = true, markers = "extra == \"pil\""} +numpy = "*" palettable = "*" Pillow = "*" [package.extras] -all = ["large-image-converter (>=1.23.2)", "large-image-source-bioformats (>=1.23.2)", "large-image-source-deepzoom (>=1.23.2)", "large-image-source-dicom (>=1.23.2)", "large-image-source-dummy (>=1.23.2)", "large-image-source-gdal (>=1.23.2)", "large-image-source-mapnik (>=1.23.2)", "large-image-source-multi (>=1.23.2)", "large-image-source-nd2 (>=1.23.2)", "large-image-source-ometiff (>=1.23.2)", "large-image-source-openjpeg (>=1.23.2)", "large-image-source-openslide (>=1.23.2)", "large-image-source-pil (>=1.23.2)", "large-image-source-pil[all] (>=1.23.2)", "large-image-source-rasterio (>=1.23.2)", "large-image-source-rasterio[all] (>=1.23.2)", "large-image-source-test (>=1.23.2)", "large-image-source-tiff (>=1.23.2)", "large-image-source-tifffile (>=1.23.2)", "large-image-source-vips (>=1.23.2)", "matplotlib", "psutil (>=4.2.0)", "pylibmc (>=1.5.1)", "pyvips", "simplejpeg", "simplejpeg (<1.6.6)"] -bioformats = ["large-image-source-bioformats (>=1.23.2)"] +all = ["large-image-converter (>=1.23.3)", "large-image-source-bioformats (>=1.23.3)", "large-image-source-deepzoom (>=1.23.3)", "large-image-source-dicom (>=1.23.3)", "large-image-source-dummy (>=1.23.3)", "large-image-source-gdal (>=1.23.3)", "large-image-source-mapnik (>=1.23.3)", "large-image-source-multi (>=1.23.3)", "large-image-source-nd2 (>=1.23.3)", "large-image-source-ometiff (>=1.23.3)", "large-image-source-openjpeg (>=1.23.3)", "large-image-source-openslide (>=1.23.3)", "large-image-source-pil (>=1.23.3)", "large-image-source-pil[all] (>=1.23.3)", "large-image-source-rasterio (>=1.23.3)", "large-image-source-rasterio[all] (>=1.23.3)", "large-image-source-test (>=1.23.3)", "large-image-source-tiff (>=1.23.3)", "large-image-source-tifffile (>=1.23.3)", "large-image-source-vips (>=1.23.3)", "matplotlib", "psutil (>=4.2.0)", "pylibmc (>=1.5.1)", "pyvips", "simplejpeg", "simplejpeg (<1.6.6)"] +bioformats = ["large-image-source-bioformats (>=1.23.3)"] colormaps = ["matplotlib"] -converter = ["large-image-converter (>=1.23.2)"] -deepzoom = ["large-image-source-deepzoom (>=1.23.2)"] -dicom = ["large-image-source-dicom (>=1.23.2)"] -dummy = ["large-image-source-dummy (>=1.23.2)"] -gdal = ["large-image-source-gdal (>=1.23.2)"] -mapnik = ["large-image-source-mapnik (>=1.23.2)"] +converter = ["large-image-converter (>=1.23.3)"] +deepzoom = ["large-image-source-deepzoom (>=1.23.3)"] +dicom = ["large-image-source-dicom (>=1.23.3)"] +dummy = ["large-image-source-dummy (>=1.23.3)"] +gdal = ["large-image-source-gdal (>=1.23.3)"] +mapnik = ["large-image-source-mapnik (>=1.23.3)"] memcached = ["pylibmc (>=1.5.1)"] -multi = ["large-image-source-multi (>=1.23.2)"] -nd2 = ["large-image-source-nd2 (>=1.23.2)"] -ometiff = ["large-image-source-ometiff (>=1.23.2)"] -openjpeg = ["large-image-source-openjpeg (>=1.23.2)"] -openslide = ["large-image-source-openslide (>=1.23.2)"] +multi = ["large-image-source-multi (>=1.23.3)"] +nd2 = ["large-image-source-nd2 (>=1.23.3)"] +ometiff = ["large-image-source-ometiff (>=1.23.3)"] +openjpeg = ["large-image-source-openjpeg (>=1.23.3)"] +openslide = ["large-image-source-openslide (>=1.23.3)"] performance = ["psutil (>=4.2.0)", "simplejpeg", "simplejpeg (<1.6.6)"] -pil = ["large-image-source-pil (>=1.23.2)"] -rasterio = ["large-image-source-rasterio (>=1.23.2)"] -sources = ["large-image-source-bioformats (>=1.23.2)", "large-image-source-deepzoom (>=1.23.2)", "large-image-source-dicom (>=1.23.2)", "large-image-source-dummy (>=1.23.2)", "large-image-source-gdal (>=1.23.2)", "large-image-source-mapnik (>=1.23.2)", "large-image-source-multi (>=1.23.2)", "large-image-source-nd2 (>=1.23.2)", "large-image-source-ometiff (>=1.23.2)", "large-image-source-openjpeg (>=1.23.2)", "large-image-source-openslide (>=1.23.2)", "large-image-source-pil (>=1.23.2)", "large-image-source-rasterio (>=1.23.2)", "large-image-source-test (>=1.23.2)", "large-image-source-tiff (>=1.23.2)", "large-image-source-tifffile (>=1.23.2)", "large-image-source-vips (>=1.23.2)"] -test = ["large-image-source-test (>=1.23.2)"] -tiff = ["large-image-source-tiff (>=1.23.2)"] -tifffile = ["large-image-source-tifffile (>=1.23.2)"] +pil = ["large-image-source-pil (>=1.23.3)"] +rasterio = ["large-image-source-rasterio (>=1.23.3)"] +sources = ["large-image-source-bioformats (>=1.23.3)", "large-image-source-deepzoom (>=1.23.3)", "large-image-source-dicom (>=1.23.3)", "large-image-source-dummy (>=1.23.3)", "large-image-source-gdal (>=1.23.3)", "large-image-source-mapnik (>=1.23.3)", "large-image-source-multi (>=1.23.3)", "large-image-source-nd2 (>=1.23.3)", "large-image-source-ometiff (>=1.23.3)", "large-image-source-openjpeg (>=1.23.3)", "large-image-source-openslide (>=1.23.3)", "large-image-source-pil (>=1.23.3)", "large-image-source-rasterio (>=1.23.3)", "large-image-source-test (>=1.23.3)", "large-image-source-tiff (>=1.23.3)", "large-image-source-tifffile (>=1.23.3)", "large-image-source-vips (>=1.23.3)"] +test = ["large-image-source-test (>=1.23.3)"] +tiff = ["large-image-source-tiff (>=1.23.3)"] +tifffile = ["large-image-source-tifffile (>=1.23.3)"] tiledoutput = ["pyvips"] -vips = ["large-image-source-vips (>=1.23.2)"] +vips = ["large-image-source-vips (>=1.23.3)"] [[package]] name = "large-image-source-pil" -version = "1.23.2" +version = "1.23.3" description = "A Pillow tilesource for large_image." category = "main" optional = false python-versions = ">=3.6" [package.dependencies] -large-image = ">=1.23.2" +large-image = ">=1.23.3" [package.extras] all = ["pillow-heif", "rawpy"] -girder = ["girder-large-image (>=1.23.2)"] +girder = ["girder-large-image (>=1.23.3)"] [[package]] name = "locket" @@ -4068,6 +4068,14 @@ python-versions = ">=3.5" lint = ["docutils-stubs", "flake8", "mypy"] test = ["pytest"] +[[package]] +name = "spython" +version = "0.3.0" +description = "Command line python tool for working with singularity." +category = "main" +optional = false +python-versions = "*" + [[package]] name = "sqlalchemy" version = "1.4.45" @@ -4736,7 +4744,7 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools" [metadata] lock-version = "1.1" python-versions = ">=3.9,<3.12" -content-hash = "8d56668aa2e91617cf9cf7eab91481c67a9dde3b89d306aaa45ed7aa52332ee5" +content-hash = "17db7d244fbc8cd3cc3b0648eae216100903ed3882c37f7bf1074e17f6bd24ec" [metadata.files] aiobotocore = [ @@ -5164,66 +5172,58 @@ contourpy = [ {file = "contourpy-1.0.6.tar.gz", hash = "sha256:6e459ebb8bb5ee4c22c19cc000174f8059981971a33ce11e17dddf6aca97a142"}, ] coverage = [ - {file = "coverage-7.2.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d39b5b4f2a66ccae8b7263ac3c8170994b65266797fb96cbbfd3fb5b23921db8"}, - {file = "coverage-7.2.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6d040ef7c9859bb11dfeb056ff5b3872436e3b5e401817d87a31e1750b9ae2fb"}, - {file = "coverage-7.2.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba90a9563ba44a72fda2e85302c3abc71c5589cea608ca16c22b9804262aaeb6"}, - {file = "coverage-7.2.7-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7d9405291c6928619403db1d10bd07888888ec1abcbd9748fdaa971d7d661b2"}, - {file = "coverage-7.2.7-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31563e97dae5598556600466ad9beea39fb04e0229e61c12eaa206e0aa202063"}, - {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ebba1cd308ef115925421d3e6a586e655ca5a77b5bf41e02eb0e4562a111f2d1"}, - {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:cb017fd1b2603ef59e374ba2063f593abe0fc45f2ad9abdde5b4d83bd922a353"}, - {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d62a5c7dad11015c66fbb9d881bc4caa5b12f16292f857842d9d1871595f4495"}, - {file = "coverage-7.2.7-cp310-cp310-win32.whl", hash = "sha256:ee57190f24fba796e36bb6d3aa8a8783c643d8fa9760c89f7a98ab5455fbf818"}, - {file = "coverage-7.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:f75f7168ab25dd93110c8a8117a22450c19976afbc44234cbf71481094c1b850"}, - {file = "coverage-7.2.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:06a9a2be0b5b576c3f18f1a241f0473575c4a26021b52b2a85263a00f034d51f"}, - {file = "coverage-7.2.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5baa06420f837184130752b7c5ea0808762083bf3487b5038d68b012e5937dbe"}, - {file = "coverage-7.2.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdec9e8cbf13a5bf63290fc6013d216a4c7232efb51548594ca3631a7f13c3a3"}, - {file = "coverage-7.2.7-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:52edc1a60c0d34afa421c9c37078817b2e67a392cab17d97283b64c5833f427f"}, - {file = "coverage-7.2.7-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63426706118b7f5cf6bb6c895dc215d8a418d5952544042c8a2d9fe87fcf09cb"}, - {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:afb17f84d56068a7c29f5fa37bfd38d5aba69e3304af08ee94da8ed5b0865833"}, - {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:48c19d2159d433ccc99e729ceae7d5293fbffa0bdb94952d3579983d1c8c9d97"}, - {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0e1f928eaf5469c11e886fe0885ad2bf1ec606434e79842a879277895a50942a"}, - {file = "coverage-7.2.7-cp311-cp311-win32.whl", hash = "sha256:33d6d3ea29d5b3a1a632b3c4e4f4ecae24ef170b0b9ee493883f2df10039959a"}, - {file = "coverage-7.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:5b7540161790b2f28143191f5f8ec02fb132660ff175b7747b95dcb77ac26562"}, - {file = "coverage-7.2.7-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f2f67fe12b22cd130d34d0ef79206061bfb5eda52feb6ce0dba0644e20a03cf4"}, - {file = "coverage-7.2.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a342242fe22407f3c17f4b499276a02b01e80f861f1682ad1d95b04018e0c0d4"}, - {file = "coverage-7.2.7-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:171717c7cb6b453aebac9a2ef603699da237f341b38eebfee9be75d27dc38e01"}, - {file = "coverage-7.2.7-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49969a9f7ffa086d973d91cec8d2e31080436ef0fb4a359cae927e742abfaaa6"}, - {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b46517c02ccd08092f4fa99f24c3b83d8f92f739b4657b0f146246a0ca6a831d"}, - {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:a3d33a6b3eae87ceaefa91ffdc130b5e8536182cd6dfdbfc1aa56b46ff8c86de"}, - {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:976b9c42fb2a43ebf304fa7d4a310e5f16cc99992f33eced91ef6f908bd8f33d"}, - {file = "coverage-7.2.7-cp312-cp312-win32.whl", hash = "sha256:8de8bb0e5ad103888d65abef8bca41ab93721647590a3f740100cd65c3b00511"}, - {file = "coverage-7.2.7-cp312-cp312-win_amd64.whl", hash = "sha256:9e31cb64d7de6b6f09702bb27c02d1904b3aebfca610c12772452c4e6c21a0d3"}, - {file = "coverage-7.2.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:58c2ccc2f00ecb51253cbe5d8d7122a34590fac9646a960d1430d5b15321d95f"}, - {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d22656368f0e6189e24722214ed8d66b8022db19d182927b9a248a2a8a2f67eb"}, - {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a895fcc7b15c3fc72beb43cdcbdf0ddb7d2ebc959edac9cef390b0d14f39f8a9"}, - {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e84606b74eb7de6ff581a7915e2dab7a28a0517fbe1c9239eb227e1354064dcd"}, - {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0a5f9e1dbd7fbe30196578ca36f3fba75376fb99888c395c5880b355e2875f8a"}, - {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:419bfd2caae268623dd469eff96d510a920c90928b60f2073d79f8fe2bbc5959"}, - {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2aee274c46590717f38ae5e4650988d1af340fe06167546cc32fe2f58ed05b02"}, - {file = "coverage-7.2.7-cp37-cp37m-win32.whl", hash = "sha256:61b9a528fb348373c433e8966535074b802c7a5d7f23c4f421e6c6e2f1697a6f"}, - {file = "coverage-7.2.7-cp37-cp37m-win_amd64.whl", hash = "sha256:b1c546aca0ca4d028901d825015dc8e4d56aac4b541877690eb76490f1dc8ed0"}, - {file = "coverage-7.2.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:54b896376ab563bd38453cecb813c295cf347cf5906e8b41d340b0321a5433e5"}, - {file = "coverage-7.2.7-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3d376df58cc111dc8e21e3b6e24606b5bb5dee6024f46a5abca99124b2229ef5"}, - {file = "coverage-7.2.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e330fc79bd7207e46c7d7fd2bb4af2963f5f635703925543a70b99574b0fea9"}, - {file = "coverage-7.2.7-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e9d683426464e4a252bf70c3498756055016f99ddaec3774bf368e76bbe02b6"}, - {file = "coverage-7.2.7-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d13c64ee2d33eccf7437961b6ea7ad8673e2be040b4f7fd4fd4d4d28d9ccb1e"}, - {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b7aa5f8a41217360e600da646004f878250a0d6738bcdc11a0a39928d7dc2050"}, - {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8fa03bce9bfbeeef9f3b160a8bed39a221d82308b4152b27d82d8daa7041fee5"}, - {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:245167dd26180ab4c91d5e1496a30be4cd721a5cf2abf52974f965f10f11419f"}, - {file = "coverage-7.2.7-cp38-cp38-win32.whl", hash = "sha256:d2c2db7fd82e9b72937969bceac4d6ca89660db0a0967614ce2481e81a0b771e"}, - {file = "coverage-7.2.7-cp38-cp38-win_amd64.whl", hash = "sha256:2e07b54284e381531c87f785f613b833569c14ecacdcb85d56b25c4622c16c3c"}, - {file = "coverage-7.2.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:537891ae8ce59ef63d0123f7ac9e2ae0fc8b72c7ccbe5296fec45fd68967b6c9"}, - {file = "coverage-7.2.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:06fb182e69f33f6cd1d39a6c597294cff3143554b64b9825d1dc69d18cc2fff2"}, - {file = "coverage-7.2.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:201e7389591af40950a6480bd9edfa8ed04346ff80002cec1a66cac4549c1ad7"}, - {file = "coverage-7.2.7-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f6951407391b639504e3b3be51b7ba5f3528adbf1a8ac3302b687ecababf929e"}, - {file = "coverage-7.2.7-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f48351d66575f535669306aa7d6d6f71bc43372473b54a832222803eb956fd1"}, - {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b29019c76039dc3c0fd815c41392a044ce555d9bcdd38b0fb60fb4cd8e475ba9"}, - {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:81c13a1fc7468c40f13420732805a4c38a105d89848b7c10af65a90beff25250"}, - {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:975d70ab7e3c80a3fe86001d8751f6778905ec723f5b110aed1e450da9d4b7f2"}, - {file = "coverage-7.2.7-cp39-cp39-win32.whl", hash = "sha256:7ee7d9d4822c8acc74a5e26c50604dff824710bc8de424904c0982e25c39c6cb"}, - {file = "coverage-7.2.7-cp39-cp39-win_amd64.whl", hash = "sha256:eb393e5ebc85245347950143969b241d08b52b88a3dc39479822e073a1a8eb27"}, - {file = "coverage-7.2.7-pp37.pp38.pp39-none-any.whl", hash = "sha256:b7b4c971f05e6ae490fef852c218b0e79d4e52f79ef0c8475566584a8fb3e01d"}, - {file = "coverage-7.2.7.tar.gz", hash = "sha256:924d94291ca674905fe9481f12294eb11f2d3d3fd1adb20314ba89e94f44ed59"}, + {file = "coverage-7.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:db76a1bcb51f02b2007adacbed4c88b6dee75342c37b05d1822815eed19edee5"}, + {file = "coverage-7.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c02cfa6c36144ab334d556989406837336c1d05215a9bdf44c0bc1d1ac1cb637"}, + {file = "coverage-7.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:477c9430ad5d1b80b07f3c12f7120eef40bfbf849e9e7859e53b9c93b922d2af"}, + {file = "coverage-7.3.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce2ee86ca75f9f96072295c5ebb4ef2a43cecf2870b0ca5e7a1cbdd929cf67e1"}, + {file = "coverage-7.3.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68d8a0426b49c053013e631c0cdc09b952d857efa8f68121746b339912d27a12"}, + {file = "coverage-7.3.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b3eb0c93e2ea6445b2173da48cb548364f8f65bf68f3d090404080d338e3a689"}, + {file = "coverage-7.3.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:90b6e2f0f66750c5a1178ffa9370dec6c508a8ca5265c42fbad3ccac210a7977"}, + {file = "coverage-7.3.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:96d7d761aea65b291a98c84e1250cd57b5b51726821a6f2f8df65db89363be51"}, + {file = "coverage-7.3.0-cp310-cp310-win32.whl", hash = "sha256:63c5b8ecbc3b3d5eb3a9d873dec60afc0cd5ff9d9f1c75981d8c31cfe4df8527"}, + {file = "coverage-7.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:97c44f4ee13bce914272589b6b41165bbb650e48fdb7bd5493a38bde8de730a1"}, + {file = "coverage-7.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:74c160285f2dfe0acf0f72d425f3e970b21b6de04157fc65adc9fd07ee44177f"}, + {file = "coverage-7.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b543302a3707245d454fc49b8ecd2c2d5982b50eb63f3535244fd79a4be0c99d"}, + {file = "coverage-7.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad0f87826c4ebd3ef484502e79b39614e9c03a5d1510cfb623f4a4a051edc6fd"}, + {file = "coverage-7.3.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:13c6cbbd5f31211d8fdb477f0f7b03438591bdd077054076eec362cf2207b4a7"}, + {file = "coverage-7.3.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fac440c43e9b479d1241fe9d768645e7ccec3fb65dc3a5f6e90675e75c3f3e3a"}, + {file = "coverage-7.3.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:3c9834d5e3df9d2aba0275c9f67989c590e05732439b3318fa37a725dff51e74"}, + {file = "coverage-7.3.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4c8e31cf29b60859876474034a83f59a14381af50cbe8a9dbaadbf70adc4b214"}, + {file = "coverage-7.3.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7a9baf8e230f9621f8e1d00c580394a0aa328fdac0df2b3f8384387c44083c0f"}, + {file = "coverage-7.3.0-cp311-cp311-win32.whl", hash = "sha256:ccc51713b5581e12f93ccb9c5e39e8b5d4b16776d584c0f5e9e4e63381356482"}, + {file = "coverage-7.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:887665f00ea4e488501ba755a0e3c2cfd6278e846ada3185f42d391ef95e7e70"}, + {file = "coverage-7.3.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d000a739f9feed900381605a12a61f7aaced6beae832719ae0d15058a1e81c1b"}, + {file = "coverage-7.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:59777652e245bb1e300e620ce2bef0d341945842e4eb888c23a7f1d9e143c446"}, + {file = "coverage-7.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9737bc49a9255d78da085fa04f628a310c2332b187cd49b958b0e494c125071"}, + {file = "coverage-7.3.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5247bab12f84a1d608213b96b8af0cbb30d090d705b6663ad794c2f2a5e5b9fe"}, + {file = "coverage-7.3.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2ac9a1de294773b9fa77447ab7e529cf4fe3910f6a0832816e5f3d538cfea9a"}, + {file = "coverage-7.3.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:85b7335c22455ec12444cec0d600533a238d6439d8d709d545158c1208483873"}, + {file = "coverage-7.3.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:36ce5d43a072a036f287029a55b5c6a0e9bd73db58961a273b6dc11a2c6eb9c2"}, + {file = "coverage-7.3.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:211a4576e984f96d9fce61766ffaed0115d5dab1419e4f63d6992b480c2bd60b"}, + {file = "coverage-7.3.0-cp312-cp312-win32.whl", hash = "sha256:56afbf41fa4a7b27f6635bc4289050ac3ab7951b8a821bca46f5b024500e6321"}, + {file = "coverage-7.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:7f297e0c1ae55300ff688568b04ff26b01c13dfbf4c9d2b7d0cb688ac60df479"}, + {file = "coverage-7.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ac0dec90e7de0087d3d95fa0533e1d2d722dcc008bc7b60e1143402a04c117c1"}, + {file = "coverage-7.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:438856d3f8f1e27f8e79b5410ae56650732a0dcfa94e756df88c7e2d24851fcd"}, + {file = "coverage-7.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1084393c6bda8875c05e04fce5cfe1301a425f758eb012f010eab586f1f3905e"}, + {file = "coverage-7.3.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49ab200acf891e3dde19e5aa4b0f35d12d8b4bd805dc0be8792270c71bd56c54"}, + {file = "coverage-7.3.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a67e6bbe756ed458646e1ef2b0778591ed4d1fcd4b146fc3ba2feb1a7afd4254"}, + {file = "coverage-7.3.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8f39c49faf5344af36042b293ce05c0d9004270d811c7080610b3e713251c9b0"}, + {file = "coverage-7.3.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:7df91fb24c2edaabec4e0eee512ff3bc6ec20eb8dccac2e77001c1fe516c0c84"}, + {file = "coverage-7.3.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:34f9f0763d5fa3035a315b69b428fe9c34d4fc2f615262d6be3d3bf3882fb985"}, + {file = "coverage-7.3.0-cp38-cp38-win32.whl", hash = "sha256:bac329371d4c0d456e8d5f38a9b0816b446581b5f278474e416ea0c68c47dcd9"}, + {file = "coverage-7.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:b859128a093f135b556b4765658d5d2e758e1fae3e7cc2f8c10f26fe7005e543"}, + {file = "coverage-7.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fc0ed8d310afe013db1eedd37176d0839dc66c96bcfcce8f6607a73ffea2d6ba"}, + {file = "coverage-7.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61260ec93f99f2c2d93d264b564ba912bec502f679793c56f678ba5251f0393"}, + {file = "coverage-7.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:97af9554a799bd7c58c0179cc8dbf14aa7ab50e1fd5fa73f90b9b7215874ba28"}, + {file = "coverage-7.3.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3558e5b574d62f9c46b76120a5c7c16c4612dc2644c3d48a9f4064a705eaee95"}, + {file = "coverage-7.3.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37d5576d35fcb765fca05654f66aa71e2808d4237d026e64ac8b397ffa66a56a"}, + {file = "coverage-7.3.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:07ea61bcb179f8f05ffd804d2732b09d23a1238642bf7e51dad62082b5019b34"}, + {file = "coverage-7.3.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:80501d1b2270d7e8daf1b64b895745c3e234289e00d5f0e30923e706f110334e"}, + {file = "coverage-7.3.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4eddd3153d02204f22aef0825409091a91bf2a20bce06fe0f638f5c19a85de54"}, + {file = "coverage-7.3.0-cp39-cp39-win32.whl", hash = "sha256:2d22172f938455c156e9af2612650f26cceea47dc86ca048fa4e0b2d21646ad3"}, + {file = "coverage-7.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:60f64e2007c9144375dd0f480a54d6070f00bb1a28f65c408370544091c9bc9e"}, + {file = "coverage-7.3.0-pp38.pp39.pp310-none-any.whl", hash = "sha256:5492a6ce3bdb15c6ad66cb68a0244854d9917478877a25671d70378bdc8562d0"}, + {file = "coverage-7.3.0.tar.gz", hash = "sha256:49dbb19cdcafc130f597d9e04a29d0a032ceedf729e41b181f51cd170e6ee865"}, ] crashtest = [ {file = "crashtest-0.3.1-py3-none-any.whl", hash = "sha256:300f4b0825f57688b47b6d70c6a31de33512eb2fa1ac614f780939aa0cf91680"}, @@ -6136,12 +6136,12 @@ kiwisolver = [ {file = "kiwisolver-1.4.4.tar.gz", hash = "sha256:d41997519fcba4a1e46eb4a2fe31bc12f0ff957b2b81bac28db24744f333e955"}, ] large-image = [ - {file = "large-image-1.23.2.tar.gz", hash = "sha256:04caf63872e255fcbf3810cbe3da397279bdf5eb7ee071a9005151bea9f2ffcd"}, - {file = "large_image-1.23.2-py3-none-any.whl", hash = "sha256:835df31ee987b6c2dd823e5a816ada2115f8a4210a379969258505956e2deb81"}, + {file = "large-image-1.23.3.tar.gz", hash = "sha256:f176d51b3cefd05c93e4ac6386b5e573a57b9009a3141ac48eec1823c0dc20c9"}, + {file = "large_image-1.23.3-py3-none-any.whl", hash = "sha256:7669bb12983e8c4a40303af5ce560360f31af6d33174931c42b6ae6508a120a5"}, ] large-image-source-pil = [ - {file = "large-image-source-pil-1.23.2.tar.gz", hash = "sha256:49bc3ea663abd02d976221c95186ff2175b6dbcb54092a0d9b673915df329c60"}, - {file = "large_image_source_pil-1.23.2-py3-none-any.whl", hash = "sha256:d2c2c61be6f774f272f464af5f2d99fdffb08e7af78f4656ca983cbae6164e5c"}, + {file = "large-image-source-pil-1.23.3.tar.gz", hash = "sha256:e6c97dbd6a7aa3ea0958a5b745dcb79f3eb07e42779846bb0076e71cf9b8aa4a"}, + {file = "large_image_source_pil-1.23.3-py3-none-any.whl", hash = "sha256:eb4b6233acf5eb506eda651627958ca8fb8be0f9b6f5739d28dd61ba9136b003"}, ] locket = [ {file = "locket-1.0.0-py2.py3-none-any.whl", hash = "sha256:b6c819a722f7b6bd955b80781788e4a66a55628b858d347536b7e81325a3a5e3"}, @@ -7854,6 +7854,10 @@ sphinxcontrib-serializinghtml = [ {file = "sphinxcontrib-serializinghtml-1.1.5.tar.gz", hash = "sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952"}, {file = "sphinxcontrib_serializinghtml-1.1.5-py2.py3-none-any.whl", hash = "sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd"}, ] +spython = [ + {file = "spython-0.3.0-py3-none-any.whl", hash = "sha256:a6819e6c82b759536ce6855647cdcf62177408d7818acb4360a95d64c1ad7ef6"}, + {file = "spython-0.3.0.tar.gz", hash = "sha256:1bbadb18829aaf18e68bcdb9850e6e8c1e1be9066fbd4b71e1fff88f7d80b76f"}, +] sqlalchemy = [ {file = "SQLAlchemy-1.4.45-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:f1d3fb02a4d0b07d1351a4a52f159e5e7b3045c903468b7e9349ebf0020ffdb9"}, {file = "SQLAlchemy-1.4.45-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:9b7025d46aba946272f6b6b357a22f3787473ef27451f342df1a2a6de23743e3"}, diff --git a/pyproject.toml b/pyproject.toml index ff194a8f..6985da12 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,7 @@ trimesh = "^3.22.0" python-dotenv = "^1.0.0" geopandas = "^0.13.2" Rtree = "^1.0.1" +spython = "^0.3.0" [tool.poetry.dev-dependencies] pytest = "^7.2.0" diff --git a/src/luna/common/runners.py b/src/luna/common/runners.py new file mode 100644 index 00000000..312e9c24 --- /dev/null +++ b/src/luna/common/runners.py @@ -0,0 +1,96 @@ +import spython.main + +import docker + + +class DockerRunner: + def __init__(self, image, command, volumes_map, num_cores, max_heap_size): + self._image = image + self._command = command + self._volumes = {} + self._num_cores = num_cores + self._java_options = f"-Xmx{max_heap_size}" + for k, v in volumes_map.items(): + self._volumes[k] = {"bind": v, "mode": "rw"} + + def run(self): + client = docker.from_env() + container = client.containers.run( + volumes=self._volumes, + nano_cpus=int(self._num_cores * 1e9), + image=self._image, + command=self._command, + environment={"_JAVA_OPTIONS": self._java_options}, + detach=False, + stream=True, + ) + return container + + +class DockerRunnerBuilder: + def __init__(self): + self._instance = None + + def __call__(self, image, command, volumes_map, num_cores, max_heap_size, **_ignored): + if not self._instance: + self._instance = DockerRunner(image, command, volumes_map, num_cores, max_heap_size) + return self._instance + + +class SingularityRunner: + def __init__(self, image, command, volumes_map, num_cores, use_gpu, max_heap_size): + self._image = image + self._command = command + self._num_cores = num_cores + self._use_gpu = use_gpu + self._volumes = [] + self._java_options = f"-XX:ActiveProcessorCount={num_cores} -Xmx{max_heap_size}" + for k, v in volumes_map.items(): + self._volumes.append(f"{k}:{v}") + + def run(self): + executor = spython.main.Client.execute( + image=self._image, + command=self._command, + bind=self._volumes, + nv=self._use_gpu, + options=["--env", f"_JAVA_OPTIONS={self._java_options}"], + stream=True, + ) + return executor + + +class SingularityRunnerBuilder: + def __init__(self): + self._instance = None + + def __call__(self, image, command, volumes_map, num_cores, use_gpu, max_heap_size, **_ignored): + if not self._instance: + self._instance = SingularityRunner( + image, command, volumes_map, num_cores, use_gpu, max_heap_size + ) + return self._instance + + +class RunnerFactory: + def __init__(self): + self._builders = {} + + def register_builder(self, key, builder): + self._builders[key] = builder + + def create(self, key, **kwargs): + builder = self._builders.get(key) + if not builder: + raise ValueError(key) + return builder(**kwargs) + + +class RunnerProvider(RunnerFactory): + def get(self, runner_type, **kwargs): + return self.create(runner_type, **kwargs) + + +runner_provider = RunnerProvider() +runner_provider.register_builder("DOCKER", DockerRunnerBuilder()) +runner_provider.register_builder("SINGULARITY", SingularityRunnerBuilder()) diff --git a/src/luna/pathology/cli/run_stardist_cell_detection.py b/src/luna/pathology/cli/run_stardist_cell_detection.py index ad802dff..5c3eef68 100644 --- a/src/luna/pathology/cli/run_stardist_cell_detection.py +++ b/src/luna/pathology/cli/run_stardist_cell_detection.py @@ -6,7 +6,7 @@ import pandas as pd from loguru import logger -import docker +from luna.common.runners import runner_provider from luna.common.utils import get_config, local_cache_urlpath, save_metadata, timed @@ -19,6 +19,9 @@ def stardist_simple( output_urlpath: str = ".", debug_opts: str = "", num_cores: int = 1, + image: str = "mskmind/qupath-stardist:0.4.3", + use_singularity: bool = False, + max_heap_size: str = "64G", storage_options: dict = {}, output_storage_options: dict = {}, local_config: str = "", @@ -32,8 +35,12 @@ def stardist_simple( image_type (str): qupath image type (BRIGHTFIELD_H_DAB) output_urlpath (str): output url/path debug_opts (str): debug options passed as arguments to groovy script + image (str): docker/singularity image + use_singularity (bool): use singularity instead of docker + max_heap_size (str): maximum heap size to pass to java options storage_options (dict): storage options to pass to reading functions output_storage_options (dict): storage options to pass to writing functions + local_config (str): local config yaml file Returns: pd.DataFrame: metadata about function call @@ -56,6 +63,9 @@ def stardist_simple( config["output_urlpath"], config["debug_opts"], config["num_cores"], + config["image"], + config["use_singularity"], + config['max_heap_size'], config["storage_options"], config["output_storage_options"], ) @@ -66,7 +76,7 @@ def stardist_simple( logger.info("generated cell data:") logger.info(df) - output_geojson_file = Path(output_path) / f"cell_detections.geojson" + output_geojson_file = Path(output_path) / "cell_detections.geojson" properties = { "cell_objects": str(output_header_file), @@ -91,6 +101,9 @@ def stardist_simple_main( output_urlpath: str, debug_opts: str, num_cores: int, + image: str, + use_singularity: bool, + max_heap_size: str, storage_options: dict, output_storage_options: dict, ) -> pd.DataFrame: @@ -103,6 +116,9 @@ def stardist_simple_main( image_type (str): qupath image type (BRIGHTFIELD_H_DAB) output_urlpath (str): output url/path debug_opts (str): debug options passed as arguments to groovy script + image (str): docker/singularity image + use_singularity (bool): use singularity instead of docker + max_heap_size (str): maximum heap size to pass to java options storage_options (dict): storage options to pass to reading functions output_storage_options (dict): storage options to pass to writing functions @@ -112,31 +128,39 @@ def stardist_simple_main( fs, slide_path = fsspec.core.url_to_fs(slide_urlpath, **storage_options) ofs, output_path = fsspec.core.url_to_fs(output_urlpath, **output_storage_options) + if ofs.protocol == 'file' and not ofs.exists(output_path): + ofs.mkdir(output_path) + + runner_type = "DOCKER" + if use_singularity: + runner_type = "SINGULARITY" + slide_filename = Path(slide_path).name - docker_image = "mskmind/qupath-stardist:current" command = f"QuPath script --image /inputs/{slide_filename} --args [cellSize={cell_expansion_size},imageType={image_type},{debug_opts}] /scripts/stardist_simple.groovy" - logger.info(f"Launching QuPath via {docker_image} ...") + logger.info(f"Launching QuPath via {runner_type}:{image} ...") logger.info( f"\tvolumes={slide_urlpath}:'/inputs/{slide_filename}', {slide_path}:'/output_dir'" ) logger.info(f"\tnano_cpus={int(num_cores * 1e9)}") - logger.info(f"\timage='{docker_image}'") + logger.info(f"\timage='{image}'") logger.info(f"\tcommand={command}") - client = docker.from_env() - container = client.containers.run( - volumes={ - slide_path: {"bind": f"/inputs/{slide_filename}", "mode": "ro"}, - output_path: {"bind": "/output_dir", "mode": "rw"}, - }, - nano_cpus=int(num_cores * 1e9), - image=docker_image, - command=command, - detach=True, - ) + volumes_map = { + slide_path: f"/inputs/{slide_filename}", + output_path: "/output_dir", + } - for line in container.logs(stream=True): - print(line.decode(), end="") + runner_config = { + "image": image, + "command": command, + "num_cores": num_cores, + "max_heap_size": max_heap_size, + "volumes_map": volumes_map, + } + runner = runner_provider.get(runner_type, **runner_config) + executor = runner.run() + for line in executor: + print(line) stardist_output = Path(output_path) / "cell_detections.tsv" @@ -158,15 +182,21 @@ def stardist_cell_lymphocyte( output_urlpath: str = ".", num_cores: int = 1, use_gpu: bool = False, + image: str = "mskmind/qupath-stardist:0.4.3", + use_singularity: bool = False, + max_heap_size: str = "64G", storage_options: dict = {}, output_storage_options: dict = {}, ): """Run stardist using qupath CLI Args: - input_slide_image (str): path to slide image (virtual slide formats compatible with openslide, .svs, .tif, .scn, ...) - num_cores (int): Number of cores to use for CPU parallelization + slide_urlpath (str): url/path to slide image (virtual slide formats compatible with openslide, .svs, .tif, .scn, ...) output_urlpath (str): output url/path + num_cores (int): Number of cores to use for CPU parallelization + use_gpu (bool): use GPU + use_singularity (bool): use singularity instead of docker + max_heap_size (str): maximum heap size to pass to java options storage_options (dict): storage options to pass to reading functions output_storage_options (dict): storage options to pass to writing functions @@ -189,18 +219,20 @@ def stardist_cell_lymphocyte( config["output_urlpath"], config["num_cores"], config["use_gpu"], + config["image"], + config["use_singularity"], + config["max_heap_size"], config["storage_options"], config["output_storage_options"], ) - with fs.open(output_header_file, "wb") as of: df.to_parquet(of) logger.info("generated cell data:") logger.info(df) - output_geojson_file = Path(output_path) / f"cell_detections.geojson" + output_geojson_file = Path(output_path) / "cell_detections.geojson" properties = { "cell_objects": str(output_header_file), @@ -223,15 +255,21 @@ def stardist_cell_lymphocyte_main( output_urlpath: str, num_cores: int, use_gpu: bool = False, + image: str = "mskmind/qupath-stardist:0.4.3", + use_singularity: bool = False, + max_heap_size: str = "64G", storage_options: dict = {}, output_storage_options: dict = {}, ) -> pd.DataFrame: """Run stardist using qupath CLI Args: - input_slide_image (str): path to slide image (virtual slide formats compatible with openslide, .svs, .tif, .scn, ...) - num_cores (int): Number of cores to use for CPU parallelization + slide_urlpath (str): url/path to slide image (virtual slide formats compatible with openslide, .svs, .tif, .scn, ...) output_urlpath (str): output url/path + num_cores (int): Number of cores to use for CPU parallelization + use_gpu (bool): use GPU + use_singularity (bool): use singularity instead of docker + max_heap_size (str): maximum heap size to pass to java options storage_options (dict): storage options to pass to reading functions Returns: @@ -241,35 +279,45 @@ def stardist_cell_lymphocyte_main( ofs, output_path = fsspec.core.url_to_fs(output_urlpath, **output_storage_options) + if ofs.protocol == 'file' and not ofs.exists(output_path): + ofs.mkdir(output_path) + qupath_cmd = "QuPath-cpu" if use_gpu: qupath_cmd = "QuPath-gpu" + runner_type = "DOCKER" + if use_singularity: + runner_type = "SINGULARITY" + + slide_filename = Path(slide_path).name - docker_image = "mskmind/qupath-tensorflow:latest" command = f"{qupath_cmd} script --image /inputs/{slide_filename} /scripts/stardist_nuclei_and_lymphocytes.groovy" - logger.info("Launching docker container:") + logger.info(f"Launching {runner_type} container:") logger.info( f"\tvolumes={slide_path}:'/inputs/{slide_filename}', {output_path}:'/output_dir'" ) logger.info(f"\tnano_cpus={int(num_cores * 1e9)}") - logger.info(f"\timage='{docker_image}'") + logger.info(f"\timage='{image}'") logger.info(f"\tcommand={command}") - client = docker.from_env() - container = client.containers.run( - volumes={ - slide_path: {"bind": f"/inputs/{slide_filename}", "mode": "ro"}, - output_path: {"bind": "/output_dir", "mode": "rw"}, - }, - nano_cpus=int(num_cores * 1e9), - image=docker_image, - command=command, - detach=True, - ) + volumes_map = { + slide_path: f"/inputs/{slide_filename}", + output_path: "/output_dir", + } - for line in container.logs(stream=True): - print(line.decode(), end="") + runner_config = { + "image": image, + "command": command, + "num_cores": num_cores, + "max_heap_size": max_heap_size, + "volumes_map": volumes_map, + "use_gpu": use_gpu, + } + runner = runner_provider.get(runner_type, **runner_config) + executor = runner.run() + for line in executor: + print(line) stardist_output = Path(output_path) / "cell_detections.tsv" From 266ddc5be0ab113ab684aa42da6cdfdbc3c1e73c Mon Sep 17 00:00:00 2001 From: Raymond Lim Date: Tue, 29 Aug 2023 16:53:26 -0400 Subject: [PATCH 2/2] fix: detect_tissue: add tile magnification to pass to generate_tiles and rename requested_magnification to thumbnail_magnification --- .../pathology/cli/run_tissue_detection.py | 35 +++++++++++-------- .../cli/test_run_tissue_detection.py | 2 +- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/luna/pathology/cli/run_tissue_detection.py b/src/luna/pathology/cli/run_tissue_detection.py index 6def1a61..b042dfd8 100644 --- a/src/luna/pathology/cli/run_tissue_detection.py +++ b/src/luna/pathology/cli/run_tissue_detection.py @@ -93,7 +93,8 @@ def cli( tiles_urlpath: str = "", filter_query: str = "???", tile_size: Optional[int] = None, - requested_magnification: Optional[int] = None, + thumbnail_magnification: Optional[int] = None, + tile_magnification: Optional[int] = None, batch_size: int = 2000, output_urlpath: str = ".", dask_options: dict = {}, @@ -107,7 +108,7 @@ def cli( tiles_urlpath (str): url/path to tiles manifest (parquet) filter_query (str): pandas query by which to filter tiles based on their various tissue detection scores tile_size (int): size of tiles to use (at the requested magnification) - requested_magnification (Optional[int]): Magnification scale at which to perform computation + thumbnail_magnification (Optional[int]): Magnification scale at which to perform computation output_urlpath (str): Output url/path prefix dask_options (dict): dask options storage_options (dict): storage options to pass to reading functions @@ -137,11 +138,12 @@ def cli( config["slide_urlpath"], config["tiles_urlpath"], config["tile_size"], - config["requested_magnification"], + config["thumbnail_magnification"], + config["tile_magnification"], config["filter_query"], config["batch_size"], config["storage_options"], - config["output_urlpath"] + "/" + slide_id, + config["output_urlpath"], config["output_storage_options"], ) with open(output_header_file, "wb", **config["output_storage_options"]) as of: @@ -159,7 +161,8 @@ def cli( def detect_tissue( slide_manifest: DataFrame, tile_size: int, - requested_magnification: Optional[int] = None, + thumbnail_magnification: Optional[int] = None, + tile_magnification: Optional[int] = None, filter_query: str = "", batch_size: int = 2000, storage_options: dict = {}, @@ -172,7 +175,8 @@ def detect_tissue( df = detect_tissue( row.url, tile_size, - requested_magnification, + thumbnail_magnification, + tile_magnification, filter_query, batch_size, storage_options, @@ -191,7 +195,7 @@ def detect_tissue( # row.id: client.submit(detect_tissue, # row.url, # tile_size, -# requested_magnification, +# thumbnail_magnification, # filter_query, # storage_options, # output_urlpath_prefix, @@ -210,7 +214,8 @@ def detect_tissue( def detect_tissue( slide_urlpaths: Union[str, list[str]], tile_size: int, - requested_magnification: Optional[int] = None, + thumbnail_magnification: Optional[int] = None, + tile_magnification: Optional[int] = None, filter_query: str = "", batch_size: int = 2000, storage_options: dict = {}, @@ -225,7 +230,8 @@ def detect_tissue( slide_urlpath, "", tile_size, - requested_magnification, + thumbnail_magnification, + tile_magnification, filter_query, batch_size, storage_options, @@ -248,7 +254,8 @@ def detect_tissue( slide_urlpath: str, tiles_urlpath: str = "", tile_size: Optional[int] = None, - requested_magnification: Optional[int] = None, + thumbnail_magnification: Optional[int] = None, + tile_magnification: Optional[int] = None, filter_query: str = "", batch_size: int = 2000, storage_options: dict = {}, @@ -259,7 +266,7 @@ def detect_tissue( Args: slide_urlpath (str): slide url/path tile_size (int): size of tiles to use (at the requested magnification) - requested_magnification (Optional[int]): Magnification scale at which to perform computation + thumbnail_magnification (Optional[int]): Magnification scale at which to perform computation filter_query (str): pandas query by which to filter tiles based on their various tissue detection scores storage_options (dict): storage options to pass to reading functions output_urlpath_prefix (str): output url/path prefix @@ -274,14 +281,14 @@ def detect_tissue( with open(tiles_urlpath, **storage_options) as of: tiles_df = pd.read_parquet(of) elif type(tile_size) == int: - tiles_df = generate_tiles(slide_urlpath, tile_size, storage_options) + tiles_df = generate_tiles(slide_urlpath, tile_size, storage_options, tile_magnification) else: raise RuntimeError("Specify tile_size or tile_urlpath") with TiffSlide(slide_urlpath) as slide: logger.info(f"Slide dimensions {slide.dimensions}") to_mag_scale_factor = get_scale_factor_at_magnification( - slide, requested_magnification=requested_magnification + slide, requested_magnification=thumbnail_magnification ) logger.info(f"Thumbnail scale factor: {to_mag_scale_factor}") # Original thumbnail @@ -291,7 +298,7 @@ def detect_tissue( with TiffSlide(slide_urlpath) as slide: logger.info(f"Slide dimensions {slide.dimensions}") to_mag_scale_factor = get_scale_factor_at_magnification( - slide, requested_magnification=requested_magnification + slide, requested_magnification=thumbnail_magnification ) logger.info(f"Thumbnail scale factor: {to_mag_scale_factor}") # Original thumbnail diff --git a/tests/luna/pathology/cli/test_run_tissue_detection.py b/tests/luna/pathology/cli/test_run_tissue_detection.py index b7c1ad5f..acbf101e 100644 --- a/tests/luna/pathology/cli/test_run_tissue_detection.py +++ b/tests/luna/pathology/cli/test_run_tissue_detection.py @@ -34,7 +34,7 @@ def test_stain(tmp_path, dask_client): "tests/testdata/pathology/123.svs", "--output-urlpath", str(tmp_path), - "--requested-magnification", + "--thumbnail-magnification", str(5), "--tile-size", str(256),