Got pre_MEDS.py working (for MIMIC-IV, not eICU) for arbitrary suppor…

…ted input file encodings so that unzipping works. Also made the bash arg (I think) work. Also removed rootutils to help address #114

MIMIC-IV_Example/joint_script.sh

@@ -66,7 +66,7 @@ else
 fi
 
 if [ "$DO_UNZIP" == "true" ]; then
-  echo "Unzipping gunzip csv files."
+  echo "Unzipping csv files."
     for file in "${MIMICIV_RAW_DIR}"/*/*.csv.gz; do gzip -d --force "$file"; done
 else
   echo "Skipping unzipping."

MIMIC-IV_Example/pre_MEDS.py

@@ -1,9 +1,6 @@
 #!/usr/bin/env python
 
 """Performs pre-MEDS data wrangling for MIMIC-IV."""
-import rootutils
-
-root = rootutils.setup_root(__file__, dotenv=True, pythonpath=True, cwd=True)
 
 from datetime import datetime
 from functools import partial
@@ -77,7 +74,12 @@ def main(cfg: DictConfig):
     for in_fp in all_fps:
         pfx = get_shard_prefix(raw_cohort_dir, in_fp)
 
-        fp, read_fn = get_supported_fp(raw_cohort_dir, pfx)
+        try:
+            fp, read_fn = get_supported_fp(raw_cohort_dir, pfx)
+        except FileNotFoundError:
+            logger.info(f"Skipping {pfx} @ {str(in_fp.resolve())} as no compatible dataframe file was found.")
+            continue
+
         if fp.suffix in [".csv", ".csv.gz"]:
             read_fn = partial(read_fn, infer_schema_length=100000)
 

eICU_Example/pre_MEDS.py

@@ -4,9 +4,6 @@
 
 See the docstring of `main` for more information.
 """
-import rootutils
-
-root = rootutils.setup_root(__file__, dotenv=True, pythonpath=True, cwd=True)
 
 import gzip
 from collections.abc import Callable

pyproject.toml

@@ -23,7 +23,6 @@ dependencies = [
 [tool.setuptools_scm]
 
 [project.optional-dependencies]
-examples = ["rootutils"]
 dev = ["pre-commit"]
 tests = ["pytest", "pytest-cov", "rootutils"]
 local_parallelism = ["hydra-joblib-launcher"]