add save_work config option

- to save selected intermediate files by copying over to work/ subfolder
in bids dir (for debugging, esp on remote runs)
- so far just saves the imported zarr

config/config.yml

@@ -8,6 +8,8 @@ remote_creds: '~/.config/gcloud/application_default_credentials.json' #this is n
 
 write_ome_zarr_direct: True #use this to skip writing the final zarr output to work first and copying afterwards -- useful when work is not a fast local disk
 
+save_work: False #set to True to copy work files to the bids/work sub-directory
+
 cores_per_rule: 32 
 
 #import wildcards:  tilex, tiley, channel, zslice (and prefix - unused)

workflow/Snakefile

@@ -34,6 +34,7 @@ rule all:
         get_all_targets(),
         get_bids_toplevel_targets(),
         get_qc_targets(),
+        get_save_work_targets(),
     localrule: True
 
 

workflow/rules/common.smk

@@ -158,6 +158,29 @@ def get_qc_targets():
     return targets
 
 
+def get_save_work_targets():
+    targets = []
+    if config["save_work"]:
+        for i in range(len(datasets)):
+            targets.extend(
+                expand_bids(
+                    root="bids/work",
+                    subject="{subject}",
+                    datatype="micr",
+                    sample="{sample}",
+                    acq="{acq}",
+                    desc="raw",
+                    suffix="SPIM.zarr",
+                    expand_kwargs=dict(
+                        subject=datasets.loc[i, "subject"],
+                        sample=datasets.loc[i, "sample"],
+                        acq=datasets.loc[i, "acq"],
+                    ),
+                )
+            )
+    return targets
+
+
 def dataset_is_remote(wildcards):
     return is_remote_gcs(Path(get_dataset_path(wildcards)))
 

workflow/rules/import.smk

@@ -294,3 +294,40 @@ rule tif_to_zarr_gcs:
         config["containers"]["spimprep"]
     script:
         "../scripts/tif_to_zarr_gcs.py"
+
+
+rule save_imported_zarr:
+    """will be run if save_work=True"""
+    input:
+        zarr=lambda wildcards: bids(
+            root=work,
+            subject="{subject}",
+            datatype="micr",
+            sample="{sample}",
+            acq="{acq}",
+            desc="rawfromgcs" if dataset_is_remote(wildcards) else "raw",
+            suffix="SPIM.zarr",
+        ).format(**wildcards),
+    output:
+        zarr=bids(
+            root="bids/work",
+            subject="{subject}",
+            datatype="micr",
+            sample="{sample}",
+            acq="{acq}",
+            desc="raw",
+            suffix="SPIM.zarr",
+        ),
+    log:
+        bids(
+            root="logs",
+            datatype="save_imported_zarr",
+            subject="{subject}",
+            sample="{sample}",
+            acq="{acq}",
+            suffix="log.txt",
+        ),
+    group:
+        "preproc"
+    shell:
+        "cp -Rv {input} {output}"