From 8e3036421b7c87f958af2b1428017191d9407228 Mon Sep 17 00:00:00 2001 From: Sebastien Tourbier Date: Tue, 7 Dec 2021 12:08:23 +0100 Subject: [PATCH 1/3] FIX: Creation of commands by docker and singularity wrappers --- cmp/cli/connectomemapper3_docker.py | 45 ++++++++++++++++++------ cmp/cli/connectomemapper3_singularity.py | 26 ++++++++++---- cmp/parser.py | 14 +++++++- 3 files changed, 67 insertions(+), 18 deletions(-) diff --git a/cmp/cli/connectomemapper3_docker.py b/cmp/cli/connectomemapper3_docker.py index 92f2a66e2..c50a44ea7 100644 --- a/cmp/cli/connectomemapper3_docker.py +++ b/cmp/cli/connectomemapper3_docker.py @@ -8,6 +8,7 @@ # General imports import sys +from pathlib import Path # Own imports from cmp.info import __version__ @@ -28,6 +29,16 @@ def create_docker_cmd(args): 'output_dir': "/path/to/output/directory", 'analysis_level': "participant", 'participant_label': ['01', '02', '03'], + 'anat_pipeline_config': "/path/to/ref_anatomical_config.json", + 'dwi_pipeline_config': "/path/to/ref_diffusion_config.json", + 'func_pipeline_config': "/path/to/ref_fMRI_config.json", + ('number_of_threads': 1,) + ('number_of_participants_processed_in_parallel': 1,) + ('mrtrix_random_seed': 1234,) + ('ants_random_seed': 1234,) + ('ants_number_of_threads': 2,) + ('fs_license': "/path/to/license.txt",) + ('notrack': True) } Returns @@ -38,38 +49,50 @@ def create_docker_cmd(args): # Docker run command prelude cmd = 'docker run -t --rm ' cmd += '-u $(id -u):$(id -g) ' - cmd += f'-bind {args.bids_dir}:/bids_dir ' - cmd += f'-bind {args.output_dir}:/output_dir ' - # cmd += f'-v {args.param_file}:/bids_dir/code/participants_params.json ' - # cmd += f'sebastientourbier/mialsuperresolutiontoolkit-bidsapp:v{__version__} ' - cmd += args.docker_image + if args.coverage: + cmd += '--entrypoint /app/run_coverage_cmp3.sh ' + cmd += f'-v {args.bids_dir}:/bids_dir ' + cmd += f'-v {args.output_dir}:/output_dir ' + if args.config_dir: + cmd += f'-v {args.config_dir}:/config ' + else: + cmd += f'-v {args.bids_dir}/code:/config ' + if args.fs_license: + cmd += f'-v {args.fs_license}:/bids_dir/code/license.txt ' + + cmd += f'{args.docker_image} ' # Standard BIDS App inputs cmd += '/bids_dir ' cmd += '/output_dir ' cmd += f'{args.analysis_level} ' - if args.participant_label: cmd += '--participant_label ' for label in args.participant_label: cmd += f'{label} ' - if args.session_label: cmd += '--session_label ' for label in args.session_label: cmd += f'{label} ' - + if args.anat_pipeline_config: + cmd += f'/config/{args.anat_pipeline_config} ' + if args.dwi_pipeline_config: + cmd += f'/config/{args.dwi_pipeline_config} ' + if args.func_pipeline_config: + cmd += f'/config/{args.func_pipeline_config} ' + cmd += f'--fs_license /bids_dir/code/license.txt ' optional_single_args = ( - "anat_pipeline_config", "dwi_pipeline_config", "func_pipeline_config", "number_of_threads", "number_of_participants_processed_in_parallel", "mrtrix_random_seed", "ants_random_seed", "ants_number_of_threads", - "fs_license", "notrack" ) - for arg_name in optional_single_args: argument_value = getattr(args, arg_name) if argument_value: cmd += f'--{arg_name} {argument_value} ' + if args.notrack: + cmd += "--notrack " + if args.coverage: + cmd += "--coverage" return cmd diff --git a/cmp/cli/connectomemapper3_singularity.py b/cmp/cli/connectomemapper3_singularity.py index 786fd353b..1c9e76103 100644 --- a/cmp/cli/connectomemapper3_singularity.py +++ b/cmp/cli/connectomemapper3_singularity.py @@ -8,6 +8,7 @@ # General imports import sys +from pathlib import Path # Own imports from cmp.info import __version__ @@ -50,13 +51,19 @@ def create_singularity_cmd(args): cmd = 'singularity run --containall ' cmd += f'--bind {args.bids_dir}:/bids_dir ' cmd += f'--bind {args.output_dir}:/output_dir ' - cmd += args.singularity_image + if args.config_dir: + cmd += f'--bind {args.config_dir}:/config ' + else: + cmd += f'--bind {args.bids_dir}/code:/config ' + if args.fs_license: + cmd += f'--bind {args.fs_license}:/bids_dir/code/license.txt ' + + cmd += f'{args.singularity_image} ' # Standard BIDS App inputs cmd += '/bids_dir ' cmd += '/output_dir ' cmd += f'{args.analysis_level} ' - if args.participant_label: cmd += '--participant_label ' for label in args.participant_label: @@ -65,18 +72,25 @@ def create_singularity_cmd(args): cmd += '--session_label ' for label in args.session_label: cmd += f'{label} ' - + if args.anat_pipeline_config: + cmd += f'/config/{args.anat_pipeline_config} ' + if args.dwi_pipeline_config: + cmd += f'/config/{args.dwi_pipeline_config} ' + if args.func_pipeline_config: + cmd += f'/config/{args.func_pipeline_config} ' + cmd += f'--fs_license /bids_dir/code/license.txt ' optional_single_args = ( - "anat_pipeline_config", "dwi_pipeline_config", "func_pipeline_config", "number_of_threads", "number_of_participants_processed_in_parallel", "mrtrix_random_seed", "ants_random_seed", "ants_number_of_threads", - "fs_license", "notrack" ) - for arg_name in optional_single_args: argument_value = getattr(args, arg_name) if argument_value: cmd += f'--{arg_name} {argument_value} ' + if args.notrack: + cmd += "--notrack " + if args.coverage: + cmd += "--coverage" return cmd diff --git a/cmp/parser.py b/cmp/parser.py index 5649ece67..252c29d8d 100644 --- a/cmp/parser.py +++ b/cmp/parser.py @@ -108,9 +108,15 @@ def get_docker_wrapper_parser(): p.add_argument( '--docker_image', type=str, - default=f'sebastientourbier/connectomemapper-bidsapp:{__version__} ', + default=f'sebastientourbier/connectomemapper-bidsapp:{__version__}', help="The path to the docker image.", ) + p.add_argument( + '--config_dir', + type=str, + default=None, + help="The path to the directory containing the configuration files.", + ) return p @@ -130,4 +136,10 @@ def get_singularity_wrapper_parser(): default=f'library://connectomicslab/default/connectomemapper-bidsapp:{__version__}', help="The path to the singularity image.", ) + p.add_argument( + '--config_dir', + type=str, + default=None, + help="The path to the directory containing the configuration files.", + ) return p From 954e242fc91523035fd814df833f0d87108c7c66 Mon Sep 17 00:00:00 2001 From: Sebastien Tourbier Date: Tue, 7 Dec 2021 12:09:35 +0100 Subject: [PATCH 2/3] CI: Update the calls to python docker/singularity wrappers (tests 5 and 5 bis) --- .circleci/config.yml | 55 +++++++++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8ba4a45a9..54be86cad 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -533,6 +533,20 @@ jobs: pigz -d --stdout /tmp/cache/docker.tar.gz | docker load docker images fi + - run: + name: Setup miniconda environment and install connectomemapper3 with Python wrappers + command: | + apt update + apt install -y wget + wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh + echo $(sha256sum miniconda.sh) + bash miniconda.sh -b -p $HOME/miniconda + export PATH="$HOME/miniconda/bin:$PATH" + conda update -y conda + conda env create -f environment.yml --name connectomemapper3 + source activate connectomemapper3 + echo $(which pip) + pip install . - run: name: Remove any config file present in ds-sample/code and create ds-sample/test no_output_timeout: 1h @@ -553,19 +567,20 @@ jobs: mkdir -p /tmp/data/${CIRCLE_JOB}/ds-sample cp -R /tmp/data/ds-sample/* /tmp/data/${CIRCLE_JOB}/ds-sample - run: - name: Run anatomical pipeline on ds-sample (test 5 - NativeFreesurfer) + name: Run anatomical pipeline on ds-sample (test 5 - NativeFreesurfer - Python wrapper) no_output_timeout: 5h command: | - # Execute BIDS App - docker run -it --rm \ - --entrypoint /app/run_coverage_cmp3.sh \ - -v /tmp/data/${CIRCLE_JOB}/ds-sample:/bids_dir \ - -v /tmp/data/${CIRCLE_JOB}/ds-sample/derivatives:/output_dir \ - -v /home/circleci/src/connectomemapper3/.circleci/tests/configuration_files:/config \ - sebastientourbier/connectomemapper3 \ - /bids_dir /output_dir participant --participant_label 01 --session_label 01 \ - --anat_pipeline_config /config/ref_anatomical_config_2.ini \ - --fs_license /bids_dir/code/license.txt \ + # Activate the conda environment with the wrapper installed + source activate connectomemapper3 + # Execute BIDS App via the docker python wrapper + bids_dir="/tmp/data/${CIRCLE_JOB}/ds-sample" + output_dir="${bids_dir}/derivatives" + config_dir="/home/circleci/src/connectomemapper3/.circleci/tests/configuration_files" + connectomemapper3_docker \ + "${bids_dir}" "${output_dir}" participant --participant_label 01 --session_label 01 \ + --config_dir "${config_dir}" \ + --anat_pipeline_config "ref_anatomical_config_2.ini" \ + --fs_license "${bids_dir}/code/license.txt" \ --coverage # Rename partial coverage @@ -829,6 +844,12 @@ jobs: echo "Skipping test_singularity_parcellation job" circleci step halt fi + - attach_workspace: + at: /tmp + - singularity/install-go: + go-version: '1.13' + - singularity/debian-install-3: + singularity-version: 3.7.3 - run: name: Setup miniconda environment and install connectomemapper3 with Python wrappers command: | @@ -843,12 +864,6 @@ jobs: source activate connectomemapper3 echo $(which pip) pip install . - - attach_workspace: - at: /tmp - - singularity/install-go: - go-version: '1.13' - - singularity/debian-install-3: - singularity-version: 3.7.3 - run: name: Remove any config file present in ds-sample/code and create ds-sample/test no_output_timeout: 1h @@ -881,8 +896,10 @@ jobs: connectomemapper3_singularity \ "${bids_dir}" "${output_dir}" participant --participant_label 01 --session_label 01 \ --singularity_image "/tmp/cache/connectomemapper3.simg" \ - --anat_pipeline_config "${config_dir}/ref_anatomical_config_2.ini" \ - --fs_license "${bids_dir}/code/license.txt" + --config_dir "${config_dir}" \ + --anat_pipeline_config "ref_anatomical_config_2.ini" \ + --fs_license "${bids_dir}/code/license.txt" \ + --coverage - run: name: Checking outputs of Connectome Mapper run (test 5) command: | From 9527d5034bc087235bef4c5df7bf7ca3f100b170 Mon Sep 17 00:00:00 2001 From: Sebastien Tourbier Date: Tue, 7 Dec 2021 12:10:12 +0100 Subject: [PATCH 3/3] DOC: Update usage examples of python docker/singularity wrappers --- docs/usage.rst | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index ccb4d471c..92a562953 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -37,12 +37,13 @@ When you run ``connectomemapper3_docker``, it will generate a Docker command lin .. code-block:: console $ connectomemapper_docker \ - /home/user/data/ds001 /home/user/data/ds001/derivatives \ + "/home/user/data/ds001" "/home/user/data/ds001/derivatives" \ participant --participant_label 01 --session_label 01 \ - --fs_license /usr/local/freesurfer/license.txt \ - --anat_pipeline_config /home/user/data/ds001/code/ref_anatomical_config.json \ - (--dwi_pipeline_config /home/user/data/ds001/code/ref_diffusion_config.json \) - (--func_pipeline_config /home/user/data/ds001/code/ref_fMRI_config.json \) + --fs_license "/usr/local/freesurfer/license.txt" \ + --config_dir "/home/user/data/ds001/code" \ + --anat_pipeline_config "ref_anatomical_config.json" \ + (--dwi_pipeline_config "ref_diffusion_config.json" \) + (--func_pipeline_config "ref_fMRI_config.json" \) (--number_of_participants_processed_in_parallel 1) When you run ``connectomemapper3_singularity``, it will generate a Singularity command line for you, print it out for reporting purposes, and then execute it without further action needed, e.g.: @@ -50,12 +51,13 @@ When you run ``connectomemapper3_singularity``, it will generate a Singularity c .. code-block:: console $ connectomemapper3_singularity \ - /home/user/data/ds001 /home/user/data/ds001/derivatives \ + "/home/user/data/ds001" "/home/user/data/ds001/derivatives" \ participant --participant_label 01 --session_label 01 \ - --fs_license /usr/local/freesurfer/license.txt \ - --anat_pipeline_config /home/user/data/ds001/code/ref_anatomical_config.json \ - (--dwi_pipeline_config /home/user/data/ds001/code/ref_diffusion_config.json \) - (--func_pipeline_config /home/user/data/ds001/code/ref_fMRI_config.json \) + --fs_license "/usr/local/freesurfer/license.txt" \ + --config_dir "/home/user/data/ds001/code" \ + --anat_pipeline_config "ref_anatomical_config.json" \ + (--dwi_pipeline_config "ref_diffusion_config.json" \) + (--func_pipeline_config "ref_fMRI_config.json" \) (--number_of_participants_processed_in_parallel 1)