From e3f5b8a0429572c6835e28a93264fec7e4437c08 Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Wed, 31 Jul 2024 10:38:56 +0200 Subject: [PATCH 1/5] Update testing environment to install nextflowpy from pip. --- README.md | 3 +-- conda/testing.yaml | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index bf15d1abc..bc6b8db39 100644 --- a/README.md +++ b/README.md @@ -285,13 +285,12 @@ The changes you make in the web server code will then reflect directly in the we ### Testing #### Nextflow pipelines -The nextflow pipeline is tested using a python integration of nextflow [nextflow.py](https://github.com/goodwright/nextflow.py) and standard [unittests](https://docs.python.org/3/library/unittest.html). You'll therefore need to install [nextflow.py](https://github.com/goodwright/nextflow.py). +The nextflow pipeline is tested using a python integration of nextflow [nextflow.py](https://github.com/goodwright/nextflow.py) and standard [unittests](https://docs.python.org/3/library/unittest.html). To run the tests you need a python environment with the required dependencies: ``` conda env create -f conda/testing.yaml conda activate testing -pip install nextflowpy ``` You can then simply call diff --git a/conda/testing.yaml b/conda/testing.yaml index 783057210..62d924e16 100644 --- a/conda/testing.yaml +++ b/conda/testing.yaml @@ -10,5 +10,7 @@ dependencies: - numpy - pandas - singularity=3.8.4 - - nextflow <=22.10 + - nextflow=22.10 - sqlalchemy + - pip: + - nextflowpy From 461e1aac29d731cb54d70953eaf08c985120d2a6 Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Wed, 31 Jul 2024 10:40:41 +0200 Subject: [PATCH 2/5] Fix checkm conda environment. --- conda/checkm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/checkm.yaml b/conda/checkm.yaml index 8644b6bcf..8a3f5786d 100644 --- a/conda/checkm.yaml +++ b/conda/checkm.yaml @@ -1,6 +1,6 @@ name: checkm channels: + - conda-forge - bioconda - - metagenlab dependencies: - checkm-genome=1.2.3 From d4774169d2e29ef6208934f19ac229d6dad4d770 Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Wed, 31 Jul 2024 15:18:13 +0200 Subject: [PATCH 3/5] Add setuptools to checkm environment. Newer versions of python do not include setuptools anymore, which breaks checkm. I have also opened an issue with checkm as I think they should fix that (https://github.com/Ecogenomics/CheckM/issues/403). --- conda/checkm.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda/checkm.yaml b/conda/checkm.yaml index 8a3f5786d..2aa13468a 100644 --- a/conda/checkm.yaml +++ b/conda/checkm.yaml @@ -4,3 +4,4 @@ channels: - bioconda dependencies: - checkm-genome=1.2.3 + - setuptools From c6ba625297daa9bd5f85579cc8618793522bd15e Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Wed, 31 Jul 2024 15:21:15 +0200 Subject: [PATCH 4/5] Add tests that all conda environments work correctly. --- README.md | 1 + testing/conda/test_environments.py | 47 ++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 testing/conda/test_environments.py diff --git a/README.md b/README.md index bc6b8db39..278c28f92 100644 --- a/README.md +++ b/README.md @@ -327,6 +327,7 @@ If you want to contribute, feel free to open a PR describing your changes and ma ### Making a release - Adapt CHANGELOG.md with release number and date. +- Run the tests (also checking that all conda envs build correctly). - Create docker containers: - Containers are defined in https://github.com/metagenlab/docker-images/ - Update `requirements.txt` and `Dockerfile`s for `zdb` and `annotation-pipeline` diff --git a/testing/conda/test_environments.py b/testing/conda/test_environments.py new file mode 100644 index 000000000..c8f6bdb00 --- /dev/null +++ b/testing/conda/test_environments.py @@ -0,0 +1,47 @@ +import glob +import os +import subprocess +from tempfile import TemporaryDirectory + +from testing.pipelines.base import BaseTestCase + + +class TestCondaEnvironments(BaseTestCase): + + env_commands = { + "amrfinderplus.yaml": "amrfinder --version", + 'annotation.yaml': 'python -c "import Bio"', + "blast.yaml": "blastp -h", + "checkm.yaml": "checkm -h", + "fasttree.yaml": "fasttree -expert", + "kofamscan.yaml": "exec_annotation -h", + "mafft.yaml": "mafft --version", + "main.yaml": "nextflow -v && singularity version", + "orthofinder.yaml": "orthofinder -h", + "pfam_scan.yaml": "pfam_scan.pl -h", + 'testing.yaml': "nextflow -v && singularity version", + 'webapp.yaml': 'python -c "import django"', + 'zdb.yaml': "zdb", + } + + @property + def conda_dir(self): + basedir = os.path.dirname(os.path.dirname(self.test_dir)) + return os.path.join(basedir, "conda") + + def create_env(self, filename, envdir): + filepath = os.path.join(self.conda_dir, filename) + subprocess.check_call(f"mamba env create -f {filepath} -p {envdir}", shell=True) + + def run_with_env(self, command, envdir): + subprocess.check_call(f"conda run -p {envdir} {command}", shell=True) + + def test_all_environments_are_tested(self): + yaml_files = glob.glob("*.yaml", root_dir=self.conda_dir) + self.assertItemsEqual(self.env_commands.keys(), yaml_files) + + def test_all_environments_work(self): + for filename, command in self.env_commands.items(): + with TemporaryDirectory() as tempdir: + self.create_env(filename, tempdir) + self.run_with_env(command, tempdir) From ab309d091dac680d36f7e69f5115520d08e8923d Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Wed, 31 Jul 2024 15:28:04 +0200 Subject: [PATCH 5/5] Add changelog entry. --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 706ab0ae9..6db987234 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ Versions are of the form MAJOR.MINOR.PATCH and this changelog tries to conform to [Common Changelog](https://common-changelog.org) +## Unreleased + +### Fixed + +- Fix checkm conda environment. ([#121](https://github.com/metagenlab/zDB/pull/121)) (Niklaus Johner) + ## 1.3.2 - 2024-07-30