feat: add demonanalysis to the workflow PART 1/2

.github/workflows/test.yml

@@ -14,12 +14,10 @@ on: push
 jobs:
 
   pipeline-testruns:
-    runs-on: ${{ matrix.os }}
-    name: ${{ matrix.os }}
+    runs-on: ubuntu-20.04
+    name: ubuntu-20.04
     strategy:
       fail-fast: false
-      matrix:
-        os: [ubuntu-20.04, macos-11]
 
     steps:
 
@@ -46,139 +44,95 @@ jobs:
           conda list
           conda clean --all --yes
 
-      - name: Install md5sum (macOS)
-        shell: bash -l {0}
-        if: matrix.os == 'macos-11'
-        run: brew install md5sha1sum
-
       - name: Print compiler path (Ubuntu)
         shell: bash -l {0}
-        if: matrix.os == 'ubuntu-20.04'
         run: which g++
 
-      - name: Print compiler path (macOS)
-        shell: bash -l {0}
-        if: matrix.os == 'macos-11'
-        run: which clang++
-
       - name: Compile demon (Ubuntu)
         shell: bash -l {0}
-        if: matrix.os == 'ubuntu-20.04'
         run: >
             g++ resources/demon_model/src/demon.cpp
             -o resources/demon_model/bin/demon
             -I/usr/share/miniconda3/envs/warlock/include
             -lm
 
-      - name: Compile demon (macOS)
+      - name: Prepare internal conda envs (Ubuntu)
         shell: bash -l {0}
-        if: matrix.os == 'macos-11'
         run: >
-            clang++ resources/demon_model/src/demon.cpp
-            -o resources/demon_model/bin/demon
-            -I/Users/runner/miniconda3/envs/warlock/include
-            -lm
+            snakemake
+            --snakefile="workflow/Snakefile"
+            --configfile="tests/test1/config-Ubuntu.yml"
+            --use-conda
+            --conda-create-envs-only
+            --cores 1
+            all
+
+      - name: Install demonanalysis into the env (Ubuntu)
+        shell: bash -l {0}
+        run: |
+            for file in .snakemake/conda/*.yaml; do
+                if [ -f "$file" ]; then
+                    if grep -q "name: warlock-r" "$file"; then
+                        ENVPATH="${file%.yaml}"
+                        break
+                    fi
+                fi
+            done
+            conda deactivate
+            conda activate $ENVPATH
+            Rscript -e "devtools::install('resources/demonanalysis', upgrade=TRUE)"
+            Rscript -e "devtools::install('resources/demonanalysis', upgrade=TRUE)"
 
       # Run the tests sequentially;
       # This could be parallelized in the CI but setting up conda
       # is expensive; its faster to run the tests one-by-one.
 
       - name: Run the pipeline (test1) (Ubuntu)
         shell: bash -l {0}
-        if: matrix.os == 'ubuntu-20.04'
         run: >
             bash warlock.sh
             --configfile tests/test1/config-Ubuntu.yml
             --environment local
 
-      - name: Run the pipeline (test1) (macOS)
-        shell: bash -l {0}
-        if: matrix.os == 'macos-11'
-        run: >
-            bash warlock.sh
-            --configfile tests/test1/config-macOS.yml
-            --environment local
-
       - name: Check md5 (test1) (Ubuntu)
         shell: bash -l {0}
-        if: matrix.os == 'ubuntu-20.04'
         working-directory: ./tests/test1
         run: md5sum -c output-Ubuntu.md5
 
-      - name: Check md5 (test1) (macOS)
-        shell: bash -l {0}
-        if: matrix.os == 'macos-11'
-        working-directory: ./tests/test1
-        run: md5sum -c output-macOS.md5
-
       - name: Cleanup (test1)
         shell: bash -l {0}
         run: rm -rf tests/test1/output
 
       - name: Run the pipeline (test2) (Ubuntu)
         shell: bash -l {0}
-        if: matrix.os == 'ubuntu-20.04'
         run: >
             bash warlock.sh
             --configfile tests/test2/config-Ubuntu.yml
             --environment local
             --cores 4
 
-      - name: Run the pipeline (test2) (macOS)
-        shell: bash -l {0}
-        if: matrix.os == 'macos-11'
-        run: >
-            bash warlock.sh
-            --configfile tests/test2/config-macOS.yml
-            --environment local
-            --cores 4
-
       - name: Check md5 (test2) (Ubuntu)
         shell: bash -l {0}
-        if: matrix.os == 'ubuntu-20.04'
         working-directory: ./tests/test2
         run: md5sum -c output-Ubuntu.md5
 
-      - name: Check md5 (test2) (macOS)
-        shell: bash -l {0}
-        if: matrix.os == 'macos-11'
-        working-directory: ./tests/test2
-        run: md5sum -c output-macOS.md5
-
       - name: Cleanup (test2)
         shell: bash -l {0}
         run: rm -rf tests/test2/output
 
       - name: Run the pipeline (test3) (Ubuntu)
         shell: bash -l {0}
-        if: matrix.os == 'ubuntu-20.04'
         run: >
             bash warlock.sh
             --configfile tests/test3/config-Ubuntu.yml
             --environment local
             --cores 2
 
-      - name: Run the pipeline (test3) (macOS)
-        shell: bash -l {0}
-        if: matrix.os == 'macos-11'
-        run: >
-            bash warlock.sh
-            --configfile tests/test3/config-macOS.yml
-            --environment local
-            --cores 2
-
       - name: Check md5 (test3) (Ubuntu)
         shell: bash -l {0}
-        if: matrix.os == 'ubuntu-20.04'
         working-directory: ./tests/test3
         run: md5sum -c output-Ubuntu.md5
 
-      - name: Check md5 (test3) (macOS)
-        shell: bash -l {0}
-        if: matrix.os == 'macos-11'
-        working-directory: ./tests/test3
-        run: md5sum -c output-macOS.md5
-
       - name: Cleanup (test3)
         shell: bash -l {0}
         run: rm -rf tests/test3/output

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "demon_model"]
 	path = resources/demon_model
 	url = https://github.com/robjohnnoble/demon_model
+[submodule "resources/demonanalysis"]
+	path = resources/demonanalysis
+	url = https://github.com/robjohnnoble/demonanalysis

README.md

@@ -18,7 +18,7 @@ The following repository encapsulates _demon_ into an automated and reproducible
 
 ## Installation
 
-**The workflow is designed to run on Linux and macOS systems.**  
+**The workflow is designed to run on Linux systems.**  
 We have prepared a dedicated [conda environment](https://docs.conda.io/projects/conda/en/latest/user-guide/concepts/environments.html) recipe which will contain all prerequisites required to execute the workflow. Thus Anaconda/Miniconda package manager is a natural dependency (see [Appendix A](#appendix-a-miniconda-installation) for installation instructions.)
 
 1. Clone the repository and navigate inside that directory
@@ -35,14 +35,9 @@ We have prepared a dedicated [conda environment](https://docs.conda.io/projects/
    conda activate warlock
    ```
 3. Compile _demon_
-   * Linux:
-     ```bash
-     g++ resources/demon_model/src/demon.cpp -o resources/demon_model/bin/demon -I$HOME/miniconda3/envs/warlock/include -lm
-     ```
-   * macOS:
-     ```bash
-     clang++ resources/demon_model/src/demon.cpp -o resources/demon_model/bin/demon -I$HOME/miniconda3/envs/warlock/include -lm
-     ```
+   ```bash
+   g++ resources/demon_model/src/demon.cpp -o resources/demon_model/bin/demon -I$HOME/miniconda3/envs/warlock/include -lm
+   ```
 
     > Note: remember to adjust `miniconda3` (and its path) in the command above, in case you have a different manager installed on your system. All in all, the point is to provide the _include_ directory of your `warlock` environment to the compiler.
 

prepare-environments.sh

@@ -0,0 +1,38 @@
+###############################################################################
+#
+#   Create internal conda envs with all dependencies
+#
+#   AUTHOR: Maciej_Bak
+#   CONTACT: wsciekly.maciek@gmail.com
+#
+###############################################################################
+
+# exit at a first command that exits with a !=0 status
+set -eo pipefail
+
+snakemake \
+    --snakefile="workflow/Snakefile" \
+    --configfile="tests/localtest/config-template.yml" \
+    --config workflow_repo_path="$PWD" workflow_analysis_outdir="$PWD/tests/localtest/output" \
+    --use-conda \
+    --conda-create-envs-only \
+    --cores 1 \
+    --nolock \
+    all
+
+for file in .snakemake/conda/*.yaml; do
+    if [ -f "$file" ]; then
+        if grep -q "name: warlock-r" "$file"; then
+            ENVPATH="${file%.yaml}"
+            break
+        fi
+    fi
+done
+
+eval "$(conda shell.bash hook)"
+conda deactivate
+conda activate $ENVPATH
+
+# strange error often appears, installation needs to be called twice
+Rscript -e "devtools::install('resources/demonanalysis', upgrade=TRUE)" || true
+Rscript -e "devtools::install('resources/demonanalysis', upgrade=TRUE)" || true

tests/localtest/config-template.yml

@@ -25,7 +25,7 @@ workflow_analysis_outdir: "{{ WORKFLOW_REPO_PATH }}/tests/localtest/output"
 ### DEMON PARAMETERS ###
 
 # spatial_structure
-demon_log2_deme_carrying_capacity: 8
+demon_log2_deme_carrying_capacity: [8, 9, 10]
 
 # dispersal
 demon_migration_type: 0
@@ -34,13 +34,13 @@ demon_migration_edge_only: 1
 demon_migration_rate_scales_with_K: 1
 
 # mutation_rates
-demon_mu_driver_birth: "1e-05"
-demon_mu_passenger: 0.001
-demon_mu_driver_migration: 0.01
+demon_mu_driver_birth: "5e-06"
+demon_mu_passenger: 0.1
+demon_mu_driver_migration: 0
 demon_passenger_pop_threshold: -1
 
 # fitness_effects
-demon_normal_birth_rate: [0.8, 0.9]
+demon_normal_birth_rate: 0.8
 demon_baseline_death_rate: 0
 demon_s_driver_birth: 0.1
 demon_s_passenger: 0
@@ -51,7 +51,7 @@ demon_max_relative_migration_rate: 10
 # non_biological_parameters
 demon_init_pop: 1
 demon_matrix_max: -1
-demon_max_pop: 1000000
+demon_max_pop: 100000
 demon_max_time: 86400
 demon_max_generations: 2000
 demon_seed: 10