refactor: fix all deps into one conda env (#55)

.github/workflows/test.yml

@@ -44,57 +44,35 @@ jobs:
           conda list
           conda clean --all --yes
 
-      - name: Print compiler path (Ubuntu)
+      - name: Print compiler path
         shell: bash -l {0}
         run: which g++
 
-      - name: Compile demon (Ubuntu)
+      - name: Compile demon
         shell: bash -l {0}
         run: >
             g++ resources/demon_model/src/demon.cpp
             -o resources/demon_model/bin/demon
             -I/usr/share/miniconda3/envs/warlock/include
             -lm
 
-      - name: Prepare internal conda envs (Ubuntu)
+      - name: Install demonanalysis into the env
         shell: bash -l {0}
         run: >
-            snakemake
-            --snakefile="workflow/Snakefile"
-            --configfile="tests/test1/config-Ubuntu.yml"
-            --use-conda
-            --conda-create-envs-only
-            --cores 1
-            all
-
-      - name: Install demonanalysis into the env (Ubuntu)
-        shell: bash -l {0}
-        run: |
-            for file in .snakemake/conda/*.yaml; do
-                if [ -f "$file" ]; then
-                    if grep -q "name: warlock-r" "$file"; then
-                        ENVPATH="${file%.yaml}"
-                        break
-                    fi
-                fi
-            done
-            conda deactivate
-            conda activate $ENVPATH
-            Rscript -e "devtools::install('resources/demonanalysis', upgrade=TRUE)" || true
-            Rscript -e "devtools::install('resources/demonanalysis', upgrade=TRUE)" || true
+            Rscript -e "devtools::install('resources/demonanalysis', upgrade=TRUE)"
 
       # Run the tests sequentially;
       # This could be parallelized in the CI but setting up conda
       # is expensive; its faster to run the tests one-by-one.
 
-      - name: Run the pipeline (test1) (Ubuntu)
+      - name: Run the pipeline (test1)
         shell: bash -l {0}
         run: >
             bash warlock.sh
             --configfile tests/test1/config-Ubuntu.yml
             --environment local
 
-      - name: Check outfiles (test1) (Ubuntu)
+      - name: Check outfiles (test1)
         shell: bash -l {0}
         working-directory: ./tests/test1
         run: >
@@ -105,15 +83,15 @@ jobs:
         shell: bash -l {0}
         run: rm -rf tests/test1/output
 
-      - name: Run the pipeline (test2) (Ubuntu)
+      - name: Run the pipeline (test2)
         shell: bash -l {0}
         run: >
             bash warlock.sh
             --configfile tests/test2/config-Ubuntu.yml
             --environment local
             --cores 4
 
-      - name: Check outfiles (test2) (Ubuntu)
+      - name: Check outfiles (test2)
         shell: bash -l {0}
         working-directory: ./tests/test2
         run: |

Dockerfile

@@ -38,7 +38,7 @@ RUN apt-get update && apt-get upgrade -y \
   && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
   && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
 
-##### INSTALL MAMBA + BUILD MAIN CONDA ENV #####
+##### INSTALL MAMBA + BUILD CONDA ENV #####
 RUN \
   conda install mamba -c conda-forge --yes \
   && mamba env create --file environment.yml \
@@ -53,7 +53,6 @@ RUN \
   -I /opt/conda/envs/warlock/include \
   -lm
 
-##### GENERATE INTERNAL ENVS & INSTALL DEPS #####
+##### INSTALL DEMONANALYSIS #####
 RUN \
-  chmod +x prepare-environments.sh \
-  && bash -c 'CONDA_PREFIX="/opt/conda/envs/warlock" ./prepare-environments.sh'
+  bash -c "/opt/conda/envs/warlock/bin/Rscript -e \"devtools::install('resources/demonanalysis', upgrade=FALSE)\""

README.md

@@ -39,9 +39,13 @@ We have prepared a dedicated [conda environment](https://docs.conda.io/projects/
 
     > Note: remember to adjust `miniconda3` (and its path) in the command above, in case you have a different manager installed on your system. All in all, the point is to provide the _include_ directory of your `warlock` environment to the compiler.
 
-4. Create internal environments and install dependencies
+4. [_demonanalysis_](https://github.com/robjohnnoble/demonanalysis) is a custom R package developed specifically for post-processing of simulation results; it needs to be installed manually into the conda environment.
    ```bash
-   bash prepare-environments.sh
+   # In order to use R from conda-forge one needs to set the following env variables, if unset:
+   LANG=en_US.UTF-8
+   LC_ALL=en_US.UTF-8
+   # Install demonanalysis
+   Rscript -e "devtools::install('resources/demonanalysis', upgrade=TRUE)"
    ```
 
 5. Finally, feel free to verify the installation with a small test script

environment.yml

@@ -13,16 +13,35 @@
   name: warlock
 
   channels:
-    - bioconda
     - conda-forge
+    - bioconda
 
   dependencies:
-    - boost-cpp=1.74.0
-    - compilers=1.4.2
-    - graphviz=2.42.3
-    - jinja2=3.1.2
-    - python=3.8.0
-    - pyyaml=6.0
-    - snakemake=7.18.2
+    - bioconductor-graph>=1.76.0
+    - bioconductor-rgraphviz>=2.42.0
+    - boost-cpp>=1.74.0
+    - compilers>=1.7.0
+    - graphviz>=8.0.3
+    - jinja2>=3.1.2
+    - python>=3.8.0
+    - pyyaml>=6.0
+    - r-ape>=5.8
+    - r-base>=4.0.3
+    - r-data.table>=1.15.4
+    - r-devtools>=2.4.5
+    - r-dplyr>=0.7.0
+    - r-ggmuller>=0.5.6
+    - r-ggplot2>=3.5.1
+    - r-gridextra>=2.3
+    - r-lazyeval>=0.2.2
+    - r-moments>=0.14.1
+    - r-optparse>=1.7.5
+    - r-rcolorbrewer>=1.1_3
+    - r-readr>=2.1.5
+    - r-rlang>=1.1.4
+    - r-rvaidememoire>=0.9_83_7
+    - r-scales>=1.3.0
+    - r-teachingdemos>=2.12.1
+    - snakemake>=7.18.2
 
 ...

warlock.sh

@@ -124,7 +124,6 @@ case "$ENV" in
         snakemake \
             --configfile="$CONFIGFILE" \
             --profile="workflow/profiles/local" \
-            --use-conda \
             --cores="$CORES" \
             --nolock \
             all
@@ -133,7 +132,6 @@ case "$ENV" in
         snakemake \
             --configfile="$CONFIGFILE" \
             --profile="workflow/profiles/slurm" \
-            --use-conda \
             --cores="$CORES" \
             --nolock \
             all

workflow/Snakefile

@@ -277,9 +277,6 @@ rule generate_summary_plots:
             "generate_summary_plots.benchmark.log"
         )
 
-    conda:
-        "envs/warlock-r.yml"
-
     shell:
         """
         (mkdir -p {output.DIR_analyses_outdir} \

workflow/profiles/local/config.yaml

@@ -12,6 +12,5 @@
 snakefile: "workflow/Snakefile"
 rerun-incomplete: true
 keep-going: true
-conda-frontend: conda
 
 ...

workflow/profiles/slurm/config.yaml

@@ -17,6 +17,5 @@ jobs: 4096
 rerun-incomplete: true
 keep-going: true
 latency-wait: 60
-conda-frontend: conda
 
 ...