constrain number of threads

khanlab · Oct 4, 2024 · edea152 · edea152
1 parent 6079e6d
commit edea152
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 1 deletion.
diff --git a/workflow/rules/import.smk b/workflow/rules/import.smk
@@ -280,7 +280,7 @@ rule tif_to_zarr_gcs:
         ),
     group:
         "preproc"
-    threads: config["cores_per_rule"]
+    threads: 8
     container:
         config["containers"]["spimprep"]
     script:

diff --git a/workflow/scripts/tif_to_zarr_gcs.py b/workflow/scripts/tif_to_zarr_gcs.py
@@ -11,11 +11,13 @@
 from lib.dask_image import imread_pages
 import gcsfs
 import pyvips
+from dask.distributed import Client, LocalCluster
 
 gcsfs_opts={'project': snakemake.params.storage_provider_settings['gcs'].get_settings().project,
                         'token': snakemake.input.creds}
 fs = gcsfs.GCSFileSystem(**gcsfs_opts)
 
+cluster = LocalCluster(n_workers=snakemake.threads, threads_per_worker=1)
 
 def replace_square_brackets(pattern):
     """replace all [ and ] in the string (have to use