diff --git a/.circleci/config.yml b/.circleci/config.yml index b20c4c29f..bc5542780 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -297,43 +297,6 @@ jobs: - store_artifacts: path: /tmp/data/reports - build_docs: - docker: - - image: python:3.7.4 - working_directory: /tmp/src/fmriprep - environment: - - FSLOUTPUTTYPE: 'NIFTI' - steps: - - checkout - - run: - name: Check whether build should be skipped - command: | - if [[ "$( git log --format=oneline -n 1 $CIRCLE_SHA1 | grep -i -E '\[skip[ _]?docs\]' )" != "" ]]; then - echo "Skipping doc building job" - circleci step halt - fi - - run: - name: Check Python version and upgrade pip - command: | - python --version - python -m pip install -U pip - - run: - name: Install graphviz - command: | - apt-get update - apt-get install -y graphviz - - run: - name: Install Requirements (may contain pinned versions) - command: python -m pip install -r docs/requirements.txt - - run: - name: Install fMRIPrep - command: python -m pip install ".[doc]" - - run: - name: Build documentation - command: make SPHINXOPTS="-W" -C docs html - - store_artifacts: - path: /tmp/src/fmriprep/docs/_build/html - ds005: machine: image: circleci/classic:201711-01 @@ -638,7 +601,7 @@ jobs: ${FASTRACK_ARG} \ --fs-no-reconall --sloppy \ --output-spaces MNI152NLin2009cAsym:res-2 anat func \ - --mem_mb 4096 --nthreads 2 -vv + --mem_mb 4096 --nthreads 2 -vv --debug compcor - run: name: Checking outputs of fMRIPrep command: | @@ -653,10 +616,10 @@ jobs: - run: name: Generate report with one artificial error command: | + set -x sudo mv /tmp/${DATASET}/derivatives/fmriprep/sub-100185.html \ /tmp/${DATASET}/derivatives/fmriprep/sub-100185_noerror.html - UUID="$(date '+%Y%m%d-%H%M%S_')$(uuidgen)" - mkdir -p /tmp/${DATASET}/derivatives/fmriprep/sub-100185/log/$UUID/ + UUID=$(grep uuid /tmp/${DATASET}/work/*/config.toml | cut -d\" -f 2) cp /tmp/src/fmriprep/fmriprep/data/tests/crash_files/*.txt \ /tmp/${DATASET}/derivatives/fmriprep/sub-100185/log/$UUID/ set +e @@ -666,7 +629,7 @@ jobs: /tmp/data/${DATASET} /tmp/${DATASET}/derivatives participant \ --fs-no-reconall --sloppy --write-graph \ --output-spaces MNI152NLin2009cAsym:res-2 anat func \ - --reports-only --run-uuid $UUID + --reports-only --config-file /tmp/${DATASET}/work/${UUID}/config.toml -vv RET=$? set -e [[ "$RET" -eq "1" ]] @@ -1048,18 +1011,6 @@ workflows: tags: only: /.*/ - - build_docs: - filters: - branches: - ignore: - - /tests?\/.*/ - - /ds005\/.*/ - - /ds054\/.*/ - - /ds210\/.*/ - - /docker\/.*/ - tags: - only: /.*/ - - test_deploy_pypi: filters: branches: @@ -1140,7 +1091,6 @@ workflows: requires: - test_deploy_pypi - test_pytest - - build_docs - ds005 - ds054 - ds210 diff --git a/.circleci/ds005_fasttrack_outputs.txt b/.circleci/ds005_fasttrack_outputs.txt index 6f006490a..3b5a8f7f4 100644 --- a/.circleci/ds005_fasttrack_outputs.txt +++ b/.circleci/ds005_fasttrack_outputs.txt @@ -1,4 +1,5 @@ fmriprep +fmriprep/.bidsignore fmriprep/dataset_description.json fmriprep/desc-aparcaseg_dseg.tsv fmriprep/desc-aseg_dseg.tsv @@ -10,9 +11,11 @@ fmriprep/logs/CITATION.tex fmriprep/sub-01 fmriprep/sub-01/func fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_AROMAnoiseICs.csv -fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_desc-confounds_regressors.json -fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_desc-confounds_regressors.tsv +fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_desc-confounds_timeseries.json +fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_desc-confounds_timeseries.tsv fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_desc-MELODIC_mixing.tsv +fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_from-scanner_to-T1w_mode-image_xfm.txt +fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_from-T1w_to-scanner_mode-image_xfm.txt fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_space-fsaverage5_hemi-L_bold.func.gii fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_space-fsaverage5_hemi-R_bold.func.gii fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_space-fsLR_den-91k_bold.dtseries.json @@ -42,9 +45,11 @@ fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_space-T1w_desc-brain_mas fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_space-T1w_desc-preproc_bold.json fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_space-T1w_desc-preproc_bold.nii.gz fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_AROMAnoiseICs.csv -fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_desc-confounds_regressors.json -fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_desc-confounds_regressors.tsv +fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_desc-confounds_timeseries.json +fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_desc-confounds_timeseries.tsv fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_desc-MELODIC_mixing.tsv +fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_from-scanner_to-T1w_mode-image_xfm.txt +fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_from-T1w_to-scanner_mode-image_xfm.txt fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_space-fsaverage5_hemi-L_bold.func.gii fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_space-fsaverage5_hemi-R_bold.func.gii fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_space-fsLR_den-91k_bold.dtseries.json diff --git a/.circleci/ds005_outputs.txt b/.circleci/ds005_outputs.txt index 4c80394fa..6380d55be 100644 --- a/.circleci/ds005_outputs.txt +++ b/.circleci/ds005_outputs.txt @@ -1,4 +1,5 @@ fmriprep +fmriprep/.bidsignore fmriprep/dataset_description.json fmriprep/desc-aparcaseg_dseg.tsv fmriprep/desc-aseg_dseg.tsv @@ -51,9 +52,11 @@ fmriprep/sub-01/anat/sub-01_space-MNI152NLin6Asym_label-GM_probseg.nii.gz fmriprep/sub-01/anat/sub-01_space-MNI152NLin6Asym_label-WM_probseg.nii.gz fmriprep/sub-01/func fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_AROMAnoiseICs.csv -fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_desc-confounds_regressors.json -fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_desc-confounds_regressors.tsv +fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_desc-confounds_timeseries.json +fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_desc-confounds_timeseries.tsv fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_desc-MELODIC_mixing.tsv +fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_from-scanner_to-T1w_mode-image_xfm.txt +fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_from-T1w_to-scanner_mode-image_xfm.txt fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_space-fsaverage5_hemi-L_bold.func.gii fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_space-fsaverage5_hemi-R_bold.func.gii fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_space-fsLR_den-91k_bold.dtseries.json @@ -83,9 +86,11 @@ fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_space-T1w_desc-brain_mas fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_space-T1w_desc-preproc_bold.json fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-1_space-T1w_desc-preproc_bold.nii.gz fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_AROMAnoiseICs.csv -fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_desc-confounds_regressors.json -fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_desc-confounds_regressors.tsv +fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_desc-confounds_timeseries.json +fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_desc-confounds_timeseries.tsv fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_desc-MELODIC_mixing.tsv +fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_from-scanner_to-T1w_mode-image_xfm.txt +fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_from-T1w_to-scanner_mode-image_xfm.txt fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_space-fsaverage5_hemi-L_bold.func.gii fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_space-fsaverage5_hemi-R_bold.func.gii fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_space-fsLR_den-91k_bold.dtseries.json diff --git a/.circleci/ds005_partial_fasttrack_outputs.txt b/.circleci/ds005_partial_fasttrack_outputs.txt index 4d7ecd057..2d3ae5dc9 100644 --- a/.circleci/ds005_partial_fasttrack_outputs.txt +++ b/.circleci/ds005_partial_fasttrack_outputs.txt @@ -1,4 +1,5 @@ fmriprep +fmriprep/.bidsignore fmriprep/dataset_description.json fmriprep/desc-aparcaseg_dseg.tsv fmriprep/desc-aseg_dseg.tsv @@ -10,9 +11,11 @@ fmriprep/logs/CITATION.tex fmriprep/sub-01 fmriprep/sub-01/func fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_AROMAnoiseICs.csv -fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_desc-confounds_regressors.json -fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_desc-confounds_regressors.tsv +fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_desc-confounds_timeseries.json +fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_desc-confounds_timeseries.tsv fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_desc-MELODIC_mixing.tsv +fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_from-scanner_to-T1w_mode-image_xfm.txt +fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_from-T1w_to-scanner_mode-image_xfm.txt fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_space-fsaverage5_hemi-L_bold.func.gii fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_space-fsaverage5_hemi-R_bold.func.gii fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-2_space-fsLR_den-91k_bold.dtseries.json diff --git a/.circleci/ds005_partial_outputs.txt b/.circleci/ds005_partial_outputs.txt index b5501a6d3..a8c8bc349 100644 --- a/.circleci/ds005_partial_outputs.txt +++ b/.circleci/ds005_partial_outputs.txt @@ -1,4 +1,5 @@ fmriprep +fmriprep/.bidsignore fmriprep/dataset_description.json fmriprep/desc-aparcaseg_dseg.tsv fmriprep/desc-aseg_dseg.tsv @@ -51,9 +52,11 @@ fmriprep/sub-01/anat/sub-01_space-MNI152NLin6Asym_label-GM_probseg.nii.gz fmriprep/sub-01/anat/sub-01_space-MNI152NLin6Asym_label-WM_probseg.nii.gz fmriprep/sub-01/func fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_AROMAnoiseICs.csv -fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_desc-confounds_regressors.json -fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_desc-confounds_regressors.tsv +fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_desc-confounds_timeseries.json +fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_desc-confounds_timeseries.tsv fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_desc-MELODIC_mixing.tsv +fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_from-scanner_to-T1w_mode-image_xfm.txt +fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_from-T1w_to-scanner_mode-image_xfm.txt fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-fsaverage5_hemi-L.func.gii fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-fsaverage5_hemi-R.func.gii fmriprep/sub-01/func/sub-01_task-mixedgamblestask_run-02_space-fsLR_den-91k_bold.dtseries.json diff --git a/.circleci/ds054_fasttrack_outputs.txt b/.circleci/ds054_fasttrack_outputs.txt index 4764dc8c3..1b5173b06 100644 --- a/.circleci/ds054_fasttrack_outputs.txt +++ b/.circleci/ds054_fasttrack_outputs.txt @@ -1,4 +1,5 @@ fmriprep +fmriprep/.bidsignore fmriprep/dataset_description.json fmriprep/logs fmriprep/logs/CITATION.bib @@ -10,10 +11,16 @@ fmriprep/sub-100185/func fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_boldref.nii.gz fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-brain_mask.json fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-brain_mask.nii.gz -fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-confounds_regressors.json -fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-confounds_regressors.tsv +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-CompCorA_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-CompCorC_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-CompCorT_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-CompCorW_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-confounds_timeseries.json +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-confounds_timeseries.tsv fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-preproc_bold.json fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-preproc_bold.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_from-scanner_to-T1w_mode-image_xfm.txt +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_from-T1w_to-scanner_mode-image_xfm.txt fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_space-MNI152NLin2009cAsym_res-2_boldref.nii.gz fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_space-MNI152NLin2009cAsym_res-2_desc-brain_mask.json fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_space-MNI152NLin2009cAsym_res-2_desc-brain_mask.nii.gz @@ -27,10 +34,16 @@ fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_space-T1w_desc-prepro fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_boldref.nii.gz fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-brain_mask.json fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-brain_mask.nii.gz -fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-confounds_regressors.json -fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-confounds_regressors.tsv +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-CompCorA_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-CompCorC_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-CompCorT_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-CompCorW_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-confounds_timeseries.json +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-confounds_timeseries.tsv fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-preproc_bold.json fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-preproc_bold.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_from-scanner_to-T1w_mode-image_xfm.txt +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_from-T1w_to-scanner_mode-image_xfm.txt fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_space-MNI152NLin2009cAsym_res-2_boldref.nii.gz fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_space-MNI152NLin2009cAsym_res-2_desc-brain_mask.json fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_space-MNI152NLin2009cAsym_res-2_desc-brain_mask.nii.gz diff --git a/.circleci/ds054_outputs.txt b/.circleci/ds054_outputs.txt index 7ce78f299..340a3341b 100644 --- a/.circleci/ds054_outputs.txt +++ b/.circleci/ds054_outputs.txt @@ -1,4 +1,5 @@ fmriprep +fmriprep/.bidsignore fmriprep/dataset_description.json fmriprep/logs fmriprep/logs/CITATION.bib @@ -29,10 +30,16 @@ fmriprep/sub-100185/func fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_boldref.nii.gz fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-brain_mask.json fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-brain_mask.nii.gz -fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-confounds_regressors.json -fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-confounds_regressors.tsv +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-CompCorA_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-CompCorC_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-CompCorT_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-CompCorW_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-confounds_timeseries.json +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-confounds_timeseries.tsv fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-preproc_bold.json fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_desc-preproc_bold.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_from-scanner_to-T1w_mode-image_xfm.txt +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_from-T1w_to-scanner_mode-image_xfm.txt fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_space-MNI152NLin2009cAsym_res-2_boldref.nii.gz fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_space-MNI152NLin2009cAsym_res-2_desc-brain_mask.json fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_space-MNI152NLin2009cAsym_res-2_desc-brain_mask.nii.gz @@ -46,10 +53,16 @@ fmriprep/sub-100185/func/sub-100185_task-machinegame_run-1_space-T1w_desc-prepro fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_boldref.nii.gz fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-brain_mask.json fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-brain_mask.nii.gz -fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-confounds_regressors.json -fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-confounds_regressors.tsv +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-CompCorA_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-CompCorC_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-CompCorT_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-CompCorW_mask.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-confounds_timeseries.json +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-confounds_timeseries.tsv fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-preproc_bold.json fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_desc-preproc_bold.nii.gz +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_from-scanner_to-T1w_mode-image_xfm.txt +fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_from-T1w_to-scanner_mode-image_xfm.txt fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_space-MNI152NLin2009cAsym_res-2_boldref.nii.gz fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_space-MNI152NLin2009cAsym_res-2_desc-brain_mask.json fmriprep/sub-100185/func/sub-100185_task-machinegame_run-2_space-MNI152NLin2009cAsym_res-2_desc-brain_mask.nii.gz diff --git a/.circleci/ds210_fasttrack_outputs.txt b/.circleci/ds210_fasttrack_outputs.txt index 6380a10be..d40943dc2 100644 --- a/.circleci/ds210_fasttrack_outputs.txt +++ b/.circleci/ds210_fasttrack_outputs.txt @@ -1,4 +1,5 @@ fmriprep +fmriprep/.bidsignore fmriprep/dataset_description.json fmriprep/logs fmriprep/logs/CITATION.bib @@ -7,8 +8,10 @@ fmriprep/logs/CITATION.md fmriprep/logs/CITATION.tex fmriprep/sub-02 fmriprep/sub-02/func -fmriprep/sub-02/func/sub-02_task-cuedSGT_run-1_desc-confounds_regressors.json -fmriprep/sub-02/func/sub-02_task-cuedSGT_run-1_desc-confounds_regressors.tsv +fmriprep/sub-02/func/sub-02_task-cuedSGT_run-1_desc-confounds_timeseries.json +fmriprep/sub-02/func/sub-02_task-cuedSGT_run-1_desc-confounds_timeseries.tsv +fmriprep/sub-02/func/sub-02_task-cuedSGT_run-1_from-scanner_to-T1w_mode-image_xfm.txt +fmriprep/sub-02/func/sub-02_task-cuedSGT_run-1_from-T1w_to-scanner_mode-image_xfm.txt fmriprep/sub-02/func/sub-02_task-cuedSGT_run-1_space-MNI152NLin2009cAsym_boldref.nii.gz fmriprep/sub-02/func/sub-02_task-cuedSGT_run-1_space-MNI152NLin2009cAsym_desc-brain_mask.json fmriprep/sub-02/func/sub-02_task-cuedSGT_run-1_space-MNI152NLin2009cAsym_desc-brain_mask.nii.gz diff --git a/.circleci/ds210_outputs.txt b/.circleci/ds210_outputs.txt index 468ab8658..aa9ea6b18 100644 --- a/.circleci/ds210_outputs.txt +++ b/.circleci/ds210_outputs.txt @@ -1,4 +1,5 @@ fmriprep +fmriprep/.bidsignore fmriprep/dataset_description.json fmriprep/logs fmriprep/logs/CITATION.bib @@ -26,8 +27,10 @@ fmriprep/sub-02/anat/sub-02_space-MNI152NLin2009cAsym_label-CSF_probseg.nii.gz fmriprep/sub-02/anat/sub-02_space-MNI152NLin2009cAsym_label-GM_probseg.nii.gz fmriprep/sub-02/anat/sub-02_space-MNI152NLin2009cAsym_label-WM_probseg.nii.gz fmriprep/sub-02/func -fmriprep/sub-02/func/sub-02_task-cuedSGT_run-1_desc-confounds_regressors.json -fmriprep/sub-02/func/sub-02_task-cuedSGT_run-1_desc-confounds_regressors.tsv +fmriprep/sub-02/func/sub-02_task-cuedSGT_run-1_desc-confounds_timeseries.json +fmriprep/sub-02/func/sub-02_task-cuedSGT_run-1_desc-confounds_timeseries.tsv +fmriprep/sub-02/func/sub-02_task-cuedSGT_run-1_from-scanner_to-T1w_mode-image_xfm.txt +fmriprep/sub-02/func/sub-02_task-cuedSGT_run-1_from-T1w_to-scanner_mode-image_xfm.txt fmriprep/sub-02/func/sub-02_task-cuedSGT_run-1_space-MNI152NLin2009cAsym_boldref.nii.gz fmriprep/sub-02/func/sub-02_task-cuedSGT_run-1_space-MNI152NLin2009cAsym_desc-brain_mask.json fmriprep/sub-02/func/sub-02_task-cuedSGT_run-1_space-MNI152NLin2009cAsym_desc-brain_mask.nii.gz diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 415fbaa3f..89ab9953c 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -6,15 +6,35 @@ labels: 'bug' assignees: '' --- - + +### What version of fMRIPrep are you using? + +### What kind of installation are you using? Containers (Singularity, Docker), or "bare-metal"? + +### What is the exact command-line you used? +``` + +``` + +### Have you checked that your inputs are BIDS valid? + +### Did fMRIPrep generate the visual report for this particular subject? If yes, could you share it? + +### Can you find some traces of the error reported in the visual report (at the bottom) or in *crashfiles*? +``` + +``` + +### Are you reusing previously computed results (e.g., FreeSurfer, Anatomical derivatives, work directory of previous run)? + + +### fMRIPrep log +If you have access to the output logged by fMRIPrep, please make sure to attach it as a text file to this issue. + + \ No newline at end of file +--> diff --git a/.github/auto-comment.yml b/.github/auto-comment.yml deleted file mode 100644 index 3eaabdc0e..000000000 --- a/.github/auto-comment.yml +++ /dev/null @@ -1,71 +0,0 @@ -pullRequestOpened: | - Thank your for raising your pull request. - - Some of the *fMRIPRep* maintainers will review your changes as soon as time permits. - I'm attaching below a *Review Checklist* for the reviewers, to check off during the - review. - - ## PR Review - - *Please check off boxes as applicable, and elaborate in comments below. Your review is not limited to these topics, as described in the reviewer guide* - - - [ ] As the reviewer I confirm that there are no conflicts of interest for me to review this work. - - Please check what applies in the following aspects of the PR: - - #### Code documentation - - - [ ] New functions and modules are documented following the guidelines. - - [ ] Existing code required amendments in documentation and has been updated. - - [ ] Sufficient inline comments ensure readability by other contributors in the long term. - - #### Documentation site - - - [ ] The modifications to *fMRIPrep* in this PR **do not require extra documentation**. This is a common situation when a bug fix that does not change the code base substantially is submitted. - - [ ] This PR requires documentation. The corresponding documentation will be submitted as an additional PR in the future. - - [ ] This PR requires extra documentation, and will be included within this PR. Please check the boxes that apply best: - - [ ] An existing feature is substantially modified, changing the interface and/or logic of a module. - - [ ] A new feature is being added, therefore full documentation of it will be included - - [ ] This PR addresses an error in existing documentation. - - [ ] Yes, all expected sections of documentation were generated by the CI build, and look as expected - - #### Tests - - - [ ] The new functionalities in this PR are covered by the existing tests - - [ ] New test batteries have been included with this PR - - #### Data - - - [ ] This PR does not require any additional data to be uploaded to OSF. - - [ ] This PR requires additional data for tests. - - #### Dependencies: smriprep - - - [ ] This PR does not require any update on `smriprep`; or - - [ ] `smriprep` has correctly been pinned. - - #### Dependencies: niworkflows - - - [ ] This PR does not require any update on `niworkflows`; or - - [ ] `niworkflows` has correctly been pinned. - - #### Dependencies: sdcflows - - - [ ] This PR does not require any update on `sdcflows`; or - - [ ] `sdcflows` has correctly been pinned. - - #### Dependencies: Nipype - - - [ ] This PR does not require any update on `nipype`; or - - [ ] `nipype` has correctly been pinned. - - #### Dependencies: other - - - [ ] This PR does not require any update on other dependencies; or - - [ ] other dependencies have correctly been pinned. - - #### Reports generated within CI tests - - - [ ] Yes, I have checked the reports of ds005 and they are not modified or the changes are as expected - - [ ] Yes, I have checked the reports of ds054 and they are not modified or the changes are as expected - - [ ] Yes, I have checked the reports of ds010 and they are not modified or the changes are as expected diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml deleted file mode 100644 index 624aeca7c..000000000 --- a/.github/release-drafter.yml +++ /dev/null @@ -1,5 +0,0 @@ -template: | - ## Release Notes - - ## CHANGES - $CHANGES diff --git a/.maint/contributors.json b/.maint/contributors.json index 9e6a8d243..55644f8a7 100644 --- a/.maint/contributors.json +++ b/.maint/contributors.json @@ -11,8 +11,8 @@ }, { "affiliation": "SIMEXP Lab, CRIUGM, University of Montréal, Montréal, Canada", - "name": "Basile Pinsard", - "orcid": "0000-0002-4391-3075" + "name": "Bellec, Pierre", + "orcid": "0000-0002-9111-0699" }, { "affiliation": "Department of Psychology, New York University", @@ -94,6 +94,11 @@ "name": "Jacoby, Nir", "orcid": "0000-0001-7936-9991" }, + { + "affiliation": "Department of Radiology, Weill Cornell Medicine", + "name": "Jamison, Keith W.", + "orcid": "0000-0001-7139-6661" + }, { "affiliation": "URPP Dynamics of Healthy Aging, University of Zurich", "name": "Liem, Franz", @@ -124,11 +129,6 @@ "name": "Rivera-Dompenciel, Adriana", "orcid": "0000-0002-3339-4857" }, - { - "affiliation": "Department of Psychology, Florida International University", - "name": "Salo, Taylor", - "orcid": "0000-0001-9813-3167" - }, { "affiliation": "Perelman School of Medicine, University of Pennsylvania, PA, USA", "name": "Satterthwaite, Theodore D.", diff --git a/.maint/developers.json b/.maint/developers.json index bf4163dd0..2dc2743ed 100644 --- a/.maint/developers.json +++ b/.maint/developers.json @@ -15,7 +15,7 @@ "orcid": "0000-0003-1358-196X" }, { - "affiliation": "Department of Psychology, Stanford University", + "affiliation": "Department of Radiology, CHUV, Université de Lausanne", "name": "Esteban, Oscar", "orcid": "0000-0001-8435-6191" }, @@ -39,9 +39,19 @@ "name": "Markiewicz, Christopher J.", "orcid": "0000-0002-6533-164X" }, + { + "affiliation": "SIMEXP Lab, CRIUGM, University of Montréal, Montréal, Canada", + "name": "Pinsard, Basile", + "orcid": "0000-0002-4391-3075" + }, { "affiliation": "Department of Psychology, Stanford University", "name": "Poldrack, Russell A.", "orcid": "0000-0001-6755-0259" + }, + { + "affiliation": "Department of Psychology, Florida International University", + "name": "Salo, Taylor", + "orcid": "0000-0001-9813-3167" } ] \ No newline at end of file diff --git a/.maint/paper_author_list.py b/.maint/paper_author_list.py index 1b6950e60..678ddeddc 100644 --- a/.maint/paper_author_list.py +++ b/.maint/paper_author_list.py @@ -40,7 +40,7 @@ def _aslist(inlist): "files: %s." % ', '.join(unmatched), file=sys.stderr) print('Authors (%d):' % len(author_matches)) - print('; '.join([ + print("%s." % '; '.join([ '%s \\ :sup:`%s`\\ ' % (i['name'], idx) for i, idx in zip(author_matches, aff_indexes) ])) diff --git a/.zenodo.json b/.zenodo.json index 7a8714dea..216eefe28 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -3,9 +3,9 @@ "description": "

fMRIPrep is a robust and easy-to-use pipeline for preprocessing of diverse fMRI data. The transparent workflow dispenses of manual intervention, thereby ensuring the reproducibility of the results.

", "contributors": [ { - "affiliation": "Department of Psychology, Florida International University", - "name": "Salo, Taylor", - "orcid": "0000-0001-9813-3167", + "affiliation": "Centre for Modern Interdisciplinary Technologies, Nicolaus Copernicus University in Toru\u0144", + "name": "Finc, Karolina", + "orcid": "0000-0002-0157-030X", "type": "Researcher" }, { @@ -14,12 +14,6 @@ "orcid": "0000-0001-9062-3778", "type": "Researcher" }, - { - "affiliation": "Centre for Modern Interdisciplinary Technologies, Nicolaus Copernicus University in Toru\u0144", - "name": "Finc, Karolina", - "orcid": "0000-0002-0157-030X", - "type": "Researcher" - }, { "affiliation": "Department of Psychology, Stanford University", "name": "Feingold, Franklin", @@ -56,12 +50,6 @@ "orcid": "0000-0001-8012-6399", "type": "Researcher" }, - { - "affiliation": "SIMEXP Lab, CRIUGM, University of Montr\u00e9al, Montr\u00e9al, Canada", - "name": "Basile Pinsard", - "orcid": "0000-0002-4391-3075", - "type": "Researcher" - }, { "affiliation": "Child Mind Institute", "name": "Heinsfeld, Anibal S.", @@ -74,6 +62,12 @@ "orcid": "0000-0001-7936-9991", "type": "Researcher" }, + { + "affiliation": "Department of Radiology, Weill Cornell Medicine", + "name": "Jamison, Keith W.", + "orcid": "0000-0001-7139-6661", + "type": "Researcher" + }, { "affiliation": "McLean Hospital Brain Imaging Center, MA, USA", "name": "Frederick, Blaise B.", @@ -128,18 +122,18 @@ "orcid": "0000-0002-6838-3971", "type": "Researcher" }, - { - "affiliation": "Neuroscience Program, University of Iowa", - "name": "Rivera-Dompenciel, Adriana", - "orcid": "0000-0002-3339-4857", - "type": "Researcher" - }, { "affiliation": "Center for Lifespan Changes in Brain and Cognition, University of Oslo", "name": "Amlien, Inge K.", "orcid": "0000-0002-8508-9088", "type": "Researcher" }, + { + "affiliation": "SIMEXP Lab, CRIUGM, University of Montr\u00e9al, Montr\u00e9al, Canada", + "name": "Bellec, Pierre", + "orcid": "0000-0002-9111-0699", + "type": "Researcher" + }, { "affiliation": "Perelman School of Medicine, University of Pennsylvania, PA, USA", "name": "Cieslak, Matthew", @@ -182,6 +176,12 @@ "orcid": "0000-0001-6948-9068", "type": "Researcher" }, + { + "affiliation": "Neuroscience Program, University of Iowa", + "name": "Rivera-Dompenciel, Adriana", + "orcid": "0000-0002-3339-4857", + "type": "Researcher" + }, { "affiliation": "Perelman School of Medicine, University of Pennsylvania, PA, USA", "name": "Satterthwaite, Theodore D.", @@ -227,7 +227,7 @@ ], "creators": [ { - "affiliation": "Department of Psychology, Stanford University", + "affiliation": "Department of Radiology, CHUV, Universit\u00e9 de Lausanne", "name": "Esteban, Oscar", "orcid": "0000-0001-8435-6191" }, @@ -251,11 +251,21 @@ "name": "Kent, James D.", "orcid": "0000-0002-4892-2659" }, + { + "affiliation": "Department of Psychology, Florida International University", + "name": "Salo, Taylor", + "orcid": "0000-0001-9813-3167" + }, { "affiliation": "Department of Psychology, Stanford University", "name": "Ciric, Rastko", "orcid": "0000-0001-6347-7939" }, + { + "affiliation": "SIMEXP Lab, CRIUGM, University of Montr\u00e9al, Montr\u00e9al, Canada", + "name": "Pinsard, Basile", + "orcid": "0000-0002-4391-3075" + }, { "affiliation": "Department of Psychology, Stanford University", "name": "Blair, Ross W.", diff --git a/CHANGES.rst b/CHANGES.rst index b94834b6d..7676893da 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,116 @@ +20.2.0 (TBD) +============ +With this third minor release series of 2020, +the first *fMRIPrep LTS* (*long-term support*) is finally here! + +This release contains a number of bug-fixes and enhancements mostly +related to easing the maintenance, anticipating patch-release breaking +changes to ensure a longstanding LTS, and addressing some run-to-run +repeatability problems of the CompCor implementation. + +.. admonition:: Long-Term Support (LTS) + + *fMRIPrep* 20.2 LTS introduces the `long-term support program + `__. + This LTS version will be kindly steered and maintained by + the group of Dr. Basile Pinsard and Prof. Pierre Bellec at + `CRIUGM `__, (Université de Montréal). + The LTS is planned for a window of 4 years of support (i.e., until + September 2024). + +.. caution:: + + As with all minor version increments, working directories + from previous versions **should not be reused**. + +Thank you for using *fMRIPrep*! +If you encounter any issues with this release, please let us know +by posting an issue on our GitHub page! + +A full list of changes can be found below. + +* FIX: Revise the reproducibility of CompCor masks (#2130) +* FIX: Simplify transform aggregation in resampling, pass identity transforms for multi-echo cases (#2239) +* FIX: Skip the T1w check if ``--anat-derivatives`` is provided. (#2201) +* FIX: Storing ``--bids-filters`` within config file (#2177) +* FIX: Revise multi-echo reference generation, permitting using SBRefs too (#1803) +* FIX: FreeSurfer license manipulation & canary +* ENH: Output CompCor masks if ``--debug compcor`` is passed (#2248) +* ENH: Conform to BIDS Derivatives as of BIDS 1.4.0 (#2223) +* ENH: Reuse config (#2240) +* ENH: Save BOLD-anatomical transforms to derivatives folder (#2233) +* ENH: Leverage BIDSLayout's ``database_path`` (#2203) +* ENH: Add ``--no-tty`` option to ``fmriprep-docker.py`` (#2204) +* ENH: Report number of echoes in BOLD summary. (#2184) +* ENH: Ensure NiPype telemetry is just pinged once (#2168) +* DOC: Update reference in "Refinement of Brain Mask" description (#2215) +* DOC: List *TemplateFlow* templates that need to be prefetched (#2196) +* DOC: Update references to https://github.com/nipreps (#2191) +* DOC: Pin NiPype with new Sphinx extension syntax (#2092) +* MAINT: Track #2252 from 20.1.x series (#2253) +* MAINT: Silence PyBIDS warning by setting extension mode (#2250) +* MAINT: Drop CircleCI docs build (#2247) +* MAINT: Pin latest *NiPreps* (#2244) +* MAINT: Update ``setup.cfg`` (flake8 and pytest) (#2183) +* MAINT: Delete release-drafter (#2169) +* MAINT: Track bug-fix release on the 20.1.x series (#2165) +* MAINT: Remove auto-comment bot (#2166) +* MAINT: Improve the questions on the bug-report template (#2158) + +.. admonition:: Author list for papers based on *fMRIPrep* 20.2 LTS series + + As described in the `Contributor Guidelines + `__, + anyone listed as developer or contributor may write and submit manuscripts + about *fMRIPrep*. + To do so, please move the author(s) name(s) to the front of the following list: + + Markiewicz, Christopher J. \ :sup:`1`\ ; Goncalves, Mathias \ :sup:`1`\ ; DuPre, Elizabeth \ :sup:`2`\ ; Kent, James D. \ :sup:`3`\ ; Salo, Taylor \ :sup:`4`\ ; Ciric, Rastko \ :sup:`1`\ ; Pinsard, Basile \ :sup:`5`\ ; Finc, Karolina \ :sup:`6`\ ; de la Vega, Alejandro \ :sup:`7`\ ; Feingold, Franklin \ :sup:`1`\ ; Tooley, Ursula A. \ :sup:`8`\ ; Benson, Noah C. \ :sup:`9`\ ; Urchs, Sebastian \ :sup:`2`\ ; Blair, Ross W. \ :sup:`1`\ ; Erramuzpe, Asier \ :sup:`10`\ ; Lurie, Daniel J. \ :sup:`11`\ ; Heinsfeld, Anibal S. \ :sup:`12`\ ; Jacoby, Nir \ :sup:`13`\ ; Jamison, Keith W. \ :sup:`14`\ ; Frederick, Blaise B. \ :sup:`15, 16`\ ; Valabregue, Romain \ :sup:`17`\ ; Sneve, Markus H. \ :sup:`18`\ ; Liem, Franz \ :sup:`19`\ ; Adebimpe, Azeez \ :sup:`20`\ ; Velasco, Pablo \ :sup:`21`\ ; Wexler, Joseph B. \ :sup:`1`\ ; Groen, Iris I. A. \ :sup:`22`\ ; Ma, Feilong \ :sup:`23`\ ; Amlien, Inge K. \ :sup:`18`\ ; Bellec, Pierre \ :sup:`5`\ ; Cieslak, Matthew \ :sup:`20`\ ; Devenyi, Grabriel A. \ :sup:`24`\ ; Ghosh, Satrajit S. \ :sup:`25, 26`\ ; Gomez, Daniel E. P. \ :sup:`27`\ ; Halchenko, Yaroslav O. \ :sup:`23`\ ; Isik, Ayse Ilkay \ :sup:`28`\ ; Moodie, Craig A. \ :sup:`1`\ ; Naveau, Mikaël \ :sup:`29`\ ; Rivera-Dompenciel, Adriana \ :sup:`3`\ ; Satterthwaite, Theodore D. \ :sup:`20`\ ; Sitek, Kevin R. \ :sup:`30`\ ; Stojić, Hrvoje \ :sup:`31`\ ; Thompson, William H. \ :sup:`1`\ ; Wright, Jessey \ :sup:`1`\ ; Ye, Zhifang \ :sup:`32`\ ; Gorgolewski, Krzysztof J. \ :sup:`1`\ ; Poldrack, Russell A. \ :sup:`1`\ ; Esteban, Oscar \ :sup:`33`\ . + + Affiliations: + + 1. Department of Psychology, Stanford University + 2. Montreal Neurological Institute, McGill University + 3. Neuroscience Program, University of Iowa + 4. Department of Psychology, Florida International University + 5. SIMEXP Lab, CRIUGM, University of Montréal, Montréal, Canada + 6. Centre for Modern Interdisciplinary Technologies, Nicolaus Copernicus University in Toruń + 7. University of Texas at Austin + 8. Department of Neuroscience, University of Pennsylvania, PA, USA + 9. Department of Psychology, New York University + 10. Computational Neuroimaging Lab, BioCruces Health Research Institute + 11. Department of Psychology, University of California, Berkeley + 12. Child Mind Institute + 13. Department of Psychology, Columbia University + 14. Department of Radiology, Weill Cornell Medicine + 15. McLean Hospital Brain Imaging Center, MA, USA + 16. Consolidated Department of Psychiatry, Harvard Medical School, MA, USA + 17. CENIR, INSERM U1127, CNRS UMR 7225, UPMC Univ Paris 06 UMR S 1127, Institut du Cerveau et de la Moelle épinière, ICM, F-75013, Paris, France + 18. Center for Lifespan Changes in Brain and Cognition, University of Oslo + 19. URPP Dynamics of Healthy Aging, University of Zurich + 20. Perelman School of Medicine, University of Pennsylvania, PA, USA + 21. Center for Brain Imaging, New York University + 22. Department of Psychology, New York University, NY, USA + 23. Dartmouth College: Hanover, NH, United States + 24. Department of Psychiatry, McGill University + 25. McGovern Institute for Brain Research, MIT, MA, USA + 26. Department of Otolaryngology, Harvard Medical School, MA, USA + 27. Donders Institute for Brain, Cognition and Behaviour, Radboud University Nijmegen + 28. Max Planck Institute for Empirical Aesthetics + 29. Cyceron, UMS 3408 (CNRS - UCBN), France + 30. Speech & Hearing Bioscience & Technology Program, Harvard University + 31. Max Planck UCL Centre for Computational Psychiatry and Ageing Research, University College London + 32. State Key Laboratory of Cognitive Neuroscience and Learning, Beijing Normal University + 33. Department of Radiology, CHUV, Université de Lausanne + +20.1.2 (September 04, 2020) +=========================== +Bug-fix release in the 20.1.x series. + + * FIX: Revise confounds in confounds-correlation plots (#2252) + * FIX: Coerce license path to pathlike (#2180) + * DOC: Update new sMRIPrep location (#2211) + 20.1.1 (June 04, 2020) ====================== Bug-fix release in the 20.1.x series. @@ -357,7 +470,7 @@ Bug-fix release in the 1.5.x series. This release fixes a bug specifically for T1w images with dimensions ≤256 voxels but a field-of-view >256mm. - * FIX: Calculate FoV with shape and zooms (poldracklab/smriprep#161) + * FIX: Calculate FoV with shape and zooms (nipreps/smriprep#161) 1.5.6 (January 22, 2020) ------------------------ @@ -622,7 +735,7 @@ Hotfix release intended for Singularity users. For further detail, please see 1.3.0 (February 7, 2019) ------------------------ We start the 1.3.x series including a few bugfixes, housekeeping duty and a refactors -to leverage `sMRIPrep `__ (which is a fork of +to leverage `sMRIPrep `__ (which is a fork of fMRIPrep's anatomical workflow), pybids>=0.7 for querying dataset, and `TemplateFlow `__ for handling standard spaces. diff --git a/docs/conf.py b/docs/conf.py index fb6fbdb04..2f4106541 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -335,16 +335,16 @@ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = { - "python": ("https://docs.python.org/", None), - "numpy": ("http://docs.scipy.org/doc/numpy", None), - "scipy": ("http://docs.scipy.org/doc/scipy/reference", None), - "matplotlib": ("http://matplotlib.sourceforge.net", None), + "python": ("https://docs.python.org/3/", None), + "numpy": ("https://docs.scipy.org/doc/numpy", None), + "scipy": ("https://docs.scipy.org/doc/scipy/reference", None), + "matplotlib": ("https://matplotlib.org/", None), "bids": ("https://bids-standard.github.io/pybids/", None), "nibabel": ("https://nipy.org/nibabel/", None), "nipype": ("https://nipype.readthedocs.io/en/latest/", None), "niworkflows": ("https://www.nipreps.org/niworkflows/", None), "sdcflows": ("https://www.nipreps.org/sdcflows/", None), - "smriprep": ("https://poldracklab.github.io/smriprep/", None), + "smriprep": ("https://www.nipreps.org/smriprep/", None), "templateflow": ("https://www.templateflow.org/python-client", None), } @@ -352,6 +352,6 @@ def setup(app): - app.add_stylesheet("theme_overrides.css") + app.add_css_file("theme_overrides.css") # We need this for the boilerplate script - app.add_javascript("https://cdn.rawgit.com/chrisfilo/zenodo.js/v0.1/zenodo.js") + app.add_js_file("https://cdn.rawgit.com/chrisfilo/zenodo.js/v0.1/zenodo.js") diff --git a/docs/faq.rst b/docs/faq.rst index a3276741c..49933c54c 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -29,7 +29,7 @@ What if I find some images have undergone some pre-processing already (e.g., my These images imply an unknown level of preprocessing (e.g. was it already bias-field corrected?), which makes it difficult to decide on best-practices for further processing. Hence, supporting such images was considered very low priority for *fMRIPrep*. -For example, see `#707 `_ and an illustration of +For example, see `#707 `_ and an illustration of downstream consequences in `#939 `_. So for OpenFMRI, we've been excluding these subjects, and for user-supplied data, we would recommend @@ -187,11 +187,16 @@ How do you use TemplateFlow in the absence of access to the Internet? --------------------------------------------------------------------- This is a fairly common situation in :abbr:`HPCs (high-performance computing)` systems, where the so-called login nodes have access to the Internet but -compute nodes are isolated, or in PC/laptop enviroments if you are travelling. +compute nodes are isolated, or in PC/laptop environments if you are traveling. *TemplateFlow* will require Internet access the first time it receives a query for a template resource that has not been previously accessed. If you know what are the templates you are planning to use, you could prefetch them using the Python client. +In addition to the ``--output-spaces`` that you specify, *fMRIPrep* will +internally require the ``MNI152NLin2009cAsym`` template. +If the ``--skull-strip-template`` option is not set, then ``OASIS30ANTs`` +will be used. +Finally, both the ``--cifti-output`` and ``--use-aroma`` arguments require ``MNI152NLin6Asym``. To do so, follow the next steps. 1. By default, a mirror of *TemplateFlow* to store the resources will be diff --git a/docs/links.rst b/docs/links.rst index 8f1186900..4bd2bb5db 100644 --- a/docs/links.rst +++ b/docs/links.rst @@ -1,7 +1,8 @@ .. _Nipype: http://nipype.readthedocs.io/en/latest/ .. _BIDS: http://bids.neuroimaging.io -.. _`BIDS Derivatives`: https://docs.google.com/document/d/17ebopupQxuRwp7U7TFvS6BH03ALJOgGHufxK8ToAvyI -.. _`BIDS Derivatives RC1`: https://docs.google.com/document/d/17ebopupQxuRwp7U7TFvS6BH03ALJOgGHufxK8ToAvyI +.. _`BIDS Derivatives`: https://bids-specification.readthedocs.io/en/stable/05-derivatives/01-introduction.html +.. _`BEP 011`: https://bids-specification.readthedocs.io/en/bep011/05-derivatives/04-structural-derivatives.html +.. _`BEP 012`: https://bids-specification.readthedocs.io/en/bep012/05-derivatives/05-functional-derivatives.html .. _Installation: installation.html .. _workflows: workflows.html .. _FSL: https://fsl.fmrib.ox.ac.uk/fsl/fslwiki/ diff --git a/docs/outputs.rst b/docs/outputs.rst index d721bf670..f60e8d6df 100644 --- a/docs/outputs.rst +++ b/docs/outputs.rst @@ -6,7 +6,8 @@ Outputs of *fMRIPrep* --------------------- *fMRIPrep* outputs conform to the :abbr:`BIDS (brain imaging data structure)` -Derivatives specification (see `BIDS Derivatives RC1`_). +Derivatives specification (see `BIDS Derivatives`_, along with the +upcoming `BEP 011`_ and `BEP 012`_). *fMRIPrep* generates three broad classes of outcomes: 1. **Visual QA (quality assessment) reports**: @@ -46,7 +47,7 @@ Derivatives of *fMRIPrep* (preprocessed data) --------------------------------------------- Preprocessed, or derivative, data are written to ``/fmriprep/sub-/``. -The `BIDS Derivatives RC1`_ specification describes the naming and metadata conventions we follow. +The `BIDS Derivatives`_ specification describes the naming and metadata conventions we follow. Anatomical derivatives ~~~~~~~~~~~~~~~~~~~~~~ @@ -126,6 +127,13 @@ these will be indicated with ``[specifiers]``:: sub-_[specifiers]_space-_desc-brain_mask.nii.gz sub-_[specifiers]_space-_desc-preproc_bold.nii.gz +Additionally, the following transforms are saved:: + + sub-/ + func/ + sub-_[specifiers]_from-scanner_to-T1w_mode-image_xfm.txt + sub-_[specifiers]_from-T1w_to-scanner_mode-image_xfm.txt + **Regularly gridded outputs (images)**. Volumetric output spaces labels (```` above, and in the following) include ``T1w`` and ``MNI152NLin2009cAsym`` (default). @@ -139,7 +147,7 @@ mid-thickness surface mesh:: func/ sub-_[specifiers]_space-T1w_desc-aparcaseg_dseg.nii.gz sub-_[specifiers]_space-T1w_desc-aseg_dseg.nii.gz - sub-_[specifiers]_space-_hemi-[LR].func.gii + sub-_[specifiers]_space-_hemi-[LR]_bold.func.gii Surface output spaces include ``fsnative`` (full density subject-specific mesh), ``fsaverage`` and the down-sampled meshes ``fsaverage6`` (41k vertices) and @@ -170,8 +178,8 @@ Confounds_ are saved as a :abbr:`TSV (tab-separated value)` file:: sub-/ func/ - sub-_[specifiers]_desc-confounds_regressors.tsv - sub-_[specifiers]_desc-confounds_regressors.json + sub-_[specifiers]_desc-confounds_timeseries.tsv + sub-_[specifiers]_desc-confounds_timeseries.json These :abbr:`TSV (tab-separated values)` tables look like the example below, where each row of the file corresponds to one time point found in the @@ -226,7 +234,7 @@ session and run in :abbr:`TSV (tab-separated value)` files - one column for each Such tabular files may include over 100 columns of potential confound regressors. .. danger:: - Do not include all columns of ``~_desc-confounds_regressors.tsv`` table + Do not include all columns of ``~_desc-confounds_timeseries.tsv`` table into your design matrix or denoising procedure. Filter the table first, to include only the confounds (or components thereof) you want to remove from your fMRI signal. @@ -379,7 +387,7 @@ For CompCor decompositions, entries include: - ``VarianceExplained``: the fraction of variance explained by the component across the decomposition ROI mask. - ``CumulativeVarianceExplained``: the total fraction of variance explained by this particular component and all preceding components. - - ``Retained``: Indicates whether the component was saved in ``desc-confounds_regressors.tsv`` + - ``Retained``: Indicates whether the component was saved in ``desc-confounds_timeseries.tsv`` for use in denoising. Entries that are not saved in the data file for denoising are still stored in metadata with the ``dropped`` prefix. diff --git a/docs/workflows.rst b/docs/workflows.rst index 26acbe375..d0e5ee47a 100644 --- a/docs/workflows.rst +++ b/docs/workflows.rst @@ -243,7 +243,7 @@ Based on the tissue segmentation of FreeSurfer (located in ``mri/aseg.mgz``) and only when the :ref:`Surface Processing ` step has been executed, *fMRIPrep* replaces the brain mask with a refined one that derives from the ``aseg.mgz`` file as described in -:py:func:`~fmriprep.interfaces.freesurfer.grow_mask`. +:py:class:`~niworkflows.interfaces.freesurfer.RefineBrainMask`. BOLD preprocessing ------------------ diff --git a/fmriprep/__init__.py b/fmriprep/__init__.py index 0d1e70e40..ad00a07a4 100644 --- a/fmriprep/__init__.py +++ b/fmriprep/__init__.py @@ -15,3 +15,13 @@ '__packagename__', '__version__', ] + +# Silence PyBIDS warning for extension entity behavior +# Can be removed once minimum PyBIDS dependency hits 0.14 +try: + import bids + bids.config.set_option('extension_initial_dot', True) +except (ImportError, ValueError): + pass +else: + del bids diff --git a/fmriprep/cli/parser.py b/fmriprep/cli/parser.py index a30eeafc6..6a56fce66 100644 --- a/fmriprep/cli/parser.py +++ b/fmriprep/cli/parser.py @@ -48,8 +48,12 @@ def _drop_sub(value): def _filter_pybids_none_any(dct): import bids - return {k: bids.layout.Query.ANY if v == "*" else v - for k, v in dct.items()} + return { + k: bids.layout.Query.NONE + if v is None + else (bids.layout.Query.ANY if v == "*" else v) + for k, v in dct.items() + } def _bids_filter(value): from json import loads @@ -151,6 +155,13 @@ def _bids_filter(value): help="Reuse the anatomical derivatives from another fMRIPrep run or calculated " "with an alternative processing tool (NOT RECOMMENDED).", ) + g_bids.add_argument( + "--bids-database-dir", + metavar="PATH", + type=PathExists, + help="Path to an existing PyBIDS database folder, for faster indexing " + "(especially useful for large datasets)." + ) g_perfm = parser.add_argument_group("Options to handle performance") g_perfm.add_argument( @@ -307,6 +318,7 @@ def _bids_filter(value): ) g_conf.add_argument( "--random-seed", + dest="_random_seed", action="store", type=int, default=None, @@ -494,12 +506,11 @@ def _bids_filter(value): "aggregation, not reportlet generation for specific nodes.", ) g_other.add_argument( - "--run-uuid", + "--config-file", action="store", - default=None, - help="Specify UUID of previous run, to include error logs in report. " - "No effect without --reports-only.", - ) + metavar="FILE", + help="Use pre-generated configuration file. Values in file will be overridden " + "by command-line arguments.") g_other.add_argument( "--write-graph", action="store_true", @@ -522,9 +533,16 @@ def _bids_filter(value): "improve FMRIPREP and provides an indicator of real " "world usage crucial for obtaining funding.", ) + g_other.add_argument( + "--debug", + action="store", + nargs="+", + choices=config.DEBUG_MODES + ("all",), + help="Debug mode(s) to enable. 'all' is alias for all available modes.", + ) + g_other.add_argument( "--sloppy", - dest="debug", action="store_true", default=False, help="Use low-quality tools for speed - TESTING ONLY", @@ -564,6 +582,12 @@ def parse_args(args=None, namespace=None): parser = _build_parser() opts = parser.parse_args(args, namespace) + + if opts.config_file: + skip = {} if opts.reports_only else {"execution": ("run_uuid",)} + config.load(opts.config_file, skip=skip) + config.loggers.cli.info(f"Loaded previous configuration file {opts.config_file}") + config.execution.log_level = int(max(25 - 5 * opts.verbose_count, logging.DEBUG)) config.from_dict(vars(opts)) diff --git a/fmriprep/cli/run.py b/fmriprep/cli/run.py index af5ccd29a..36fc16259 100755 --- a/fmriprep/cli/run.py +++ b/fmriprep/cli/run.py @@ -12,7 +12,7 @@ def main(): import gc from multiprocessing import Process, Manager from .parser import parse_args - from ..utils.bids import write_derivative_description + from ..utils.bids import write_derivative_description, write_bidsignore parse_args() @@ -26,7 +26,8 @@ def main(): # CRITICAL Save the config to a file. This is necessary because the execution graph # is built as a separate process to keep the memory footprint low. The most # straightforward way to communicate with the child process is via the filesystem. - config_file = config.execution.work_dir / f"config-{config.execution.run_uuid}.toml" + config_file = config.execution.work_dir / config.execution.run_uuid / "config.toml" + config_file.parent.mkdir(exist_ok=True, parents=True) config.to_filename(config_file) # CRITICAL Call build_workflow(config_file, retval) in a subprocess. @@ -165,6 +166,7 @@ def main(): write_derivative_description( config.execution.bids_dir, config.execution.output_dir / "fmriprep" ) + write_bidsignore(config.execution.output_dir / "fmriprep") if failed_reports and not config.execution.notrack: sentry_sdk.capture_message( diff --git a/fmriprep/config.py b/fmriprep/config.py index 93c8fc6a2..6adb67003 100644 --- a/fmriprep/config.py +++ b/fmriprep/config.py @@ -67,36 +67,47 @@ :py:class:`~bids.layout.BIDSLayout`, etc.) """ +import os from multiprocessing import set_start_method +# Disable NiPype etelemetry always +_disable_et = bool( + os.getenv("NO_ET") is not None or os.getenv("NIPYPE_NO_ET") is not None +) +os.environ["NIPYPE_NO_ET"] = "1" +os.environ["NO_ET"] = "1" + +CONFIG_FILENAME = "fmriprep.toml" try: - set_start_method('forkserver') + set_start_method("forkserver") except RuntimeError: pass # context has been already set finally: # Defer all custom import for after initializing the forkserver and # ignoring the most annoying warnings - import os import sys import random - from uuid import uuid4 - from pathlib import Path from time import strftime - from nipype import logging as nlogging, __version__ as _nipype_ver + + from pathlib import Path + from nipype import __version__ as _nipype_ver from templateflow import __version__ as _tf_ver from . import __version__ if not hasattr(sys, "_is_pytest_session"): sys._is_pytest_session = False # Trick to avoid sklearn's FutureWarnings # Disable all warnings in main and children processes only on production versions -if not any(( - "+" in __version__, - __version__.endswith(".dirty"), - os.getenv("FMRIPREP_DEV", "0").lower() in ("1", "on", "true", "y", "yes") -)): +if not any( + ( + "+" in __version__, + __version__.endswith(".dirty"), + os.getenv("FMRIPREP_DEV", "0").lower() in ("1", "on", "true", "y", "yes"), + ) +): from ._warnings import logging + os.environ["PYTHONWARNINGS"] = "ignore" elif os.getenv("FMRIPREP_WARNINGS", "0").lower() in ("1", "on", "true", "y", "yes"): # allow disabling warnings on development versions @@ -105,61 +116,82 @@ else: import logging -logging.addLevelName(25, 'IMPORTANT') # Add a new level between INFO and WARNING -logging.addLevelName(15, 'VERBOSE') # Add a new level between INFO and DEBUG +logging.addLevelName(25, "IMPORTANT") # Add a new level between INFO and WARNING +logging.addLevelName(15, "VERBOSE") # Add a new level between INFO and DEBUG DEFAULT_MEMORY_MIN_GB = 0.01 +# Ping NiPype eTelemetry once if env var was not set +# workers on the pool will have the env variable set from the master process +if not _disable_et: + # Just get so analytics track one hit + from contextlib import suppress + from requests import get as _get_url, ConnectionError, ReadTimeout + + with suppress((ConnectionError, ReadTimeout)): + _get_url("https://rig.mit.edu/et/projects/nipy/nipype", timeout=0.05) + +# Execution environment _exec_env = os.name _docker_ver = None # special variable set in the container -if os.getenv('IS_DOCKER_8395080871'): - _exec_env = 'singularity' - _cgroup = Path('/proc/1/cgroup') - if _cgroup.exists() and 'docker' in _cgroup.read_text(): - _docker_ver = os.getenv('DOCKER_VERSION_8395080871') - _exec_env = 'fmriprep-docker' if _docker_ver else 'docker' +if os.getenv("IS_DOCKER_8395080871"): + _exec_env = "singularity" + _cgroup = Path("/proc/1/cgroup") + if _cgroup.exists() and "docker" in _cgroup.read_text(): + _docker_ver = os.getenv("DOCKER_VERSION_8395080871") + _exec_env = "fmriprep-docker" if _docker_ver else "docker" del _cgroup -_fs_license = os.getenv('FS_LICENSE') -if not _fs_license and os.getenv('FREESURFER_HOME'): - _fs_home = os.getenv('FREESURFER_HOME') +_fs_license = os.getenv("FS_LICENSE") +if not _fs_license and os.getenv("FREESURFER_HOME"): + _fs_home = os.getenv("FREESURFER_HOME") if _fs_home and (Path(_fs_home) / "license.txt").is_file(): _fs_license = str(Path(_fs_home) / "license.txt") del _fs_home -_templateflow_home = Path(os.getenv( - 'TEMPLATEFLOW_HOME', - os.path.join(os.getenv('HOME'), '.cache', 'templateflow')) +_templateflow_home = Path( + os.getenv( + "TEMPLATEFLOW_HOME", os.path.join(os.getenv("HOME"), ".cache", "templateflow") + ) ) try: from psutil import virtual_memory - _free_mem_at_start = round(virtual_memory().free / 1024**3, 1) + + _free_mem_at_start = round(virtual_memory().free / 1024 ** 3, 1) except Exception: _free_mem_at_start = None -_oc_limit = 'n/a' -_oc_policy = 'n/a' +_oc_limit = "n/a" +_oc_policy = "n/a" try: # Memory policy may have a large effect on types of errors experienced - _proc_oc_path = Path('/proc/sys/vm/overcommit_memory') + _proc_oc_path = Path("/proc/sys/vm/overcommit_memory") if _proc_oc_path.exists(): - _oc_policy = { - '0': 'heuristic', '1': 'always', '2': 'never' - }.get(_proc_oc_path.read_text().strip(), 'unknown') - if _oc_policy != 'never': - _proc_oc_kbytes = Path('/proc/sys/vm/overcommit_kbytes') + _oc_policy = {"0": "heuristic", "1": "always", "2": "never"}.get( + _proc_oc_path.read_text().strip(), "unknown" + ) + if _oc_policy != "never": + _proc_oc_kbytes = Path("/proc/sys/vm/overcommit_kbytes") if _proc_oc_kbytes.exists(): _oc_limit = _proc_oc_kbytes.read_text().strip() - if _oc_limit in ('0', 'n/a') and Path('/proc/sys/vm/overcommit_ratio').exists(): - _oc_limit = '{}%'.format( - Path('/proc/sys/vm/overcommit_ratio').read_text().strip() + if ( + _oc_limit in ("0", "n/a") + and Path("/proc/sys/vm/overcommit_ratio").exists() + ): + _oc_limit = "{}%".format( + Path("/proc/sys/vm/overcommit_ratio").read_text().strip() ) except Exception: pass +# Debug modes are names that influence the exposure of internal details to +# the user, either through additional derivatives or increased verbosity +DEBUG_MODES = ("compcor",) + + class _Config: """An abstract class forbidding instantiation.""" @@ -167,18 +199,18 @@ class _Config: def __init__(self): """Avert instantiation.""" - raise RuntimeError('Configuration type is not instantiable.') + raise RuntimeError("Configuration type is not instantiable.") @classmethod - def load(cls, settings, init=True): + def load(cls, settings, init=True, ignore=None): """Store settings from a dictionary.""" + ignore = ignore or {} for k, v in settings.items(): - if v is None: + if k in ignore or v is None: continue if k in cls._paths: setattr(cls, k, Path(v).absolute()) - continue - if hasattr(cls, k): + elif hasattr(cls, k): setattr(cls, k, v) if init: @@ -194,7 +226,7 @@ def get(cls): out = {} for k, v in cls.__dict__.items(): - if k.startswith('_') or v is None: + if k.startswith("_") or v is None: continue if callable(getattr(cls, k)): continue @@ -245,7 +277,7 @@ class environment(_Config): class nipype(_Config): """Nipype settings.""" - crashfile_format = 'txt' + crashfile_format = "txt" """The file format for crashfiles, either text or pickle.""" get_linked_libs = False """Run NiPype's tool to enlist linked libraries for every interface.""" @@ -255,11 +287,11 @@ class nipype(_Config): """Number of processes (compute tasks) that can be run in parallel (multiprocessing only).""" omp_nthreads = None """Number of CPUs a single process can access for multithreaded execution.""" - plugin = 'MultiProc' + plugin = "MultiProc" """NiPype's execution plugin.""" plugin_args = { - 'maxtasksperchild': 1, - 'raise_insufficient': False, + "maxtasksperchild": 1, + "raise_insufficient": False, } """Settings for NiPype's execution plugin.""" resource_monitor = False @@ -271,13 +303,13 @@ class nipype(_Config): def get_plugin(cls): """Format a dictionary for Nipype consumption.""" out = { - 'plugin': cls.plugin, - 'plugin_args': cls.plugin_args, + "plugin": cls.plugin, + "plugin_args": cls.plugin_args, } - if cls.plugin in ('MultiProc', 'LegacyMultiProc'): - out['plugin_args']['n_procs'] = int(cls.nprocs) + if cls.plugin in ("MultiProc", "LegacyMultiProc"): + out["plugin_args"]["n_procs"] = int(cls.nprocs) if cls.memory_gb: - out['plugin_args']['memory_gb'] = float(cls.memory_gb) + out["plugin_args"]["memory_gb"] = float(cls.memory_gb) return out @classmethod @@ -287,27 +319,34 @@ def init(cls): # Configure resource_monitor if cls.resource_monitor: - ncfg.update_config({ - 'monitoring': { - 'enabled': cls.resource_monitor, - 'sample_frequency': '0.5', - 'summary_append': True, + ncfg.update_config( + { + "monitoring": { + "enabled": cls.resource_monitor, + "sample_frequency": "0.5", + "summary_append": True, + } } - }) + ) ncfg.enable_resource_monitor() # Nipype config (logs and execution) - ncfg.update_config({ - 'execution': { - 'crashdump_dir': str(execution.log_dir), - 'crashfile_format': cls.crashfile_format, - 'get_linked_libs': cls.get_linked_libs, - 'stop_on_first_crash': cls.stop_on_first_crash, + ncfg.update_config( + { + "execution": { + "crashdump_dir": str(execution.log_dir), + "crashfile_format": cls.crashfile_format, + "get_linked_libs": cls.get_linked_libs, + "stop_on_first_crash": cls.stop_on_first_crash, + "check_version": False, # disable future telemetry + } } - }) + ) if cls.omp_nthreads is None: - cls.omp_nthreads = min(cls.nprocs - 1 if cls.nprocs > 1 else os.cpu_count(), 8) + cls.omp_nthreads = min( + cls.nprocs - 1 if cls.nprocs > 1 else os.cpu_count(), 8 + ) class execution(_Config): @@ -317,14 +356,18 @@ class execution(_Config): """A path where anatomical derivatives are found to fast-track *sMRIPrep*.""" bids_dir = None """An existing path to the dataset, which must be BIDS-compliant.""" + bids_database_dir = None + """Path to the directory containing SQLite database indices for the input BIDS dataset.""" bids_description_hash = None """Checksum (SHA256) of the ``dataset_description.json`` of the BIDS dataset.""" bids_filters = None """A dictionary of BIDS selection filters.""" boilerplate_only = False """Only generate a boilerplate.""" - debug = False + sloppy = False """Run in sloppy mode (meaning, suboptimal parameters that minimize run-time).""" + debug = [] + """Debug mode(s).""" echo_idx = None """Select a particular echo for multi-echo EPI datasets.""" fs_license_file = _fs_license @@ -350,7 +393,7 @@ class execution(_Config): the command line) as spatial references for outputs.""" reports_only = False """Only build the reports, based on the reportlets found in a cached working directory.""" - run_uuid = '%s_%s' % (strftime('%Y%m%d-%H%M%S'), uuid4()) + run_uuid = f"{strftime('%Y%m%d-%H%M%S')}_{uuid4()}" """Unique identifier of this particular run.""" participant_label = None """List of participant identifiers that are to be preprocessed.""" @@ -358,7 +401,7 @@ class execution(_Config): """Select a particular task from all available in the dataset.""" templateflow_home = _templateflow_home """The root folder of the TemplateFlow client.""" - work_dir = Path('work').absolute() + work_dir = Path("work").absolute() """Path to a working directory where intermediate results will be available.""" write_graph = False """Write out the computational graph corresponding to the planned preprocessing.""" @@ -366,15 +409,16 @@ class execution(_Config): _layout = None _paths = ( - 'anat_derivatives', - 'bids_dir', - 'fs_license_file', - 'fs_subjects_dir', - 'layout', - 'log_dir', - 'output_dir', - 'templateflow_home', - 'work_dir', + "anat_derivatives", + "bids_dir", + "bids_database_dir", + "fs_license_file", + "fs_subjects_dir", + "layout", + "log_dir", + "output_dir", + "templateflow_home", + "work_dir", ) @classmethod @@ -386,15 +430,41 @@ def init(cls): if cls._layout is None: import re from bids.layout import BIDSLayout - work_dir = cls.work_dir / 'bids.db' - work_dir.mkdir(exist_ok=True, parents=True) + + _db_path = cls.bids_database_dir or ( + cls.work_dir / cls.run_uuid / "bids_db" + ) + _db_path.mkdir(exist_ok=True, parents=True) cls._layout = BIDSLayout( str(cls.bids_dir), validate=False, - # database_path=str(work_dir), - ignore=("code", "stimuli", "sourcedata", "models", - "derivatives", re.compile(r'^\.'))) + database_path=_db_path, + reset_database=cls.bids_database_dir is None, + ignore=( + "code", + "stimuli", + "sourcedata", + "models", + "derivatives", + re.compile(r"^\."), + ), + ) + cls.bids_database_dir = _db_path cls.layout = cls._layout + if cls.bids_filters: + from bids.layout import Query + + # unserialize pybids Query enum values + for acq, filters in cls.bids_filters.items(): + cls.bids_filters[acq] = { + k: getattr(Query, v[7:-4]) + if not isinstance(v, Query) and "Query" in v + else v + for k, v in filters.items() + } + + if "all" in cls.debug: + cls.debug = list(DEBUG_MODES) # These variables are not necessary anymore @@ -420,7 +490,7 @@ class workflow(_Config): (positive = exact, negative = maximum).""" bold2t1w_dof = None """Degrees of freedom of the BOLD-to-T1w registration steps.""" - bold2t1w_init = 'register' + bold2t1w_init = "register" """Whether to use standard coregistration ('register') or to initialize coregistration from the BOLD image-header ('header').""" cifti_output = None @@ -439,8 +509,6 @@ class workflow(_Config): """Ignore particular steps for *fMRIPrep*.""" longitudinal = False """Run FreeSurfer ``recon-all`` with the ``-logitudinal`` flag.""" - random_seed = None - """Master random seed to initialize the Pseudorandom Number Generator (PRNG)""" medial_surface_nan = None """Fill medial surface with :abbr:`NaNs (not-a-number)` when sampling.""" regressors_all_comps = None @@ -478,13 +546,13 @@ class loggers: default = logging.getLogger() """The root logger.""" - cli = logging.getLogger('cli') + cli = logging.getLogger("cli") """Command-line interface logging.""" - workflow = nlogging.getLogger('nipype.workflow') + workflow = logging.getLogger("nipype.workflow") """NiPype's workflow logger.""" - interface = nlogging.getLogger('nipype.interface') + interface = logging.getLogger("nipype.interface") """NiPype's interface logger.""" - utils = nlogging.getLogger('nipype.utils') + utils = logging.getLogger("nipype.utils") """NiPype's utils logger.""" @classmethod @@ -498,39 +566,36 @@ def init(cls): """ from nipype import config as ncfg + _handler = logging.StreamHandler(stream=sys.stdout) - _handler.setFormatter( - logging.Formatter(fmt=cls._fmt, datefmt=cls._datefmt) - ) + _handler.setFormatter(logging.Formatter(fmt=cls._fmt, datefmt=cls._datefmt)) cls.cli.addHandler(_handler) cls.default.setLevel(execution.log_level) cls.cli.setLevel(execution.log_level) cls.interface.setLevel(execution.log_level) cls.workflow.setLevel(execution.log_level) cls.utils.setLevel(execution.log_level) - ncfg.update_config({ - 'logging': { - 'log_directory': str(execution.log_dir), - 'log_to_file': True - }, - }) + ncfg.update_config( + {"logging": {"log_directory": str(execution.log_dir), "log_to_file": True}} + ) class seeds(_Config): """Initialize the PRNG and track random seed assignments""" + _random_seed = None master = None - """Master seed used to generate all other tracked seeds""" + """Master random seed to initialize the Pseudorandom Number Generator (PRNG)""" ants = None """Seed used for antsRegistration, antsAI, antsMotionCorr""" @classmethod def init(cls): - cls.master = workflow.random_seed + if cls._random_seed is not None: + cls.master = cls._random_seed if cls.master is None: cls.master = random.randint(1, 65536) random.seed(cls.master) # initialize the PRNG - # functions to set program specific seeds cls.ants = _set_ants_seed() @@ -538,7 +603,7 @@ def init(cls): def _set_ants_seed(): """Fix random seed for antsRegistration, antsAI, antsMotionCorr""" val = random.randint(1, 65536) - os.environ['ANTS_RANDOM_SEED'] = str(val) + os.environ["ANTS_RANDOM_SEED"] = str(val) return val @@ -547,42 +612,48 @@ def from_dict(settings): nipype.load(settings) execution.load(settings) workflow.load(settings) - seeds.init() + seeds.load(settings) loggers.init() -def load(filename): +def load(filename, skip=None): """Load settings from file.""" from toml import loads + + skip = skip or {} filename = Path(filename) settings = loads(filename.read_text()) for sectionname, configs in settings.items(): - if sectionname != 'environment': + if sectionname != "environment": section = getattr(sys.modules[__name__], sectionname) - section.load(configs) + ignore = skip.get(sectionname) + section.load(configs, ignore=ignore) init_spaces() def get(flat=False): """Get config as a dict.""" settings = { - 'environment': environment.get(), - 'execution': execution.get(), - 'workflow': workflow.get(), - 'nipype': nipype.get(), - 'seeds': seeds.get(), + "environment": environment.get(), + "execution": execution.get(), + "workflow": workflow.get(), + "nipype": nipype.get(), + "seeds": seeds.get(), } if not flat: return settings - return {'.'.join((section, k)): v - for section, configs in settings.items() - for k, v in configs.items()} + return { + ".".join((section, k)): v + for section, configs in settings.items() + for k, v in configs.items() + } def dumps(): """Format config into toml.""" from toml import dumps + return dumps(get()) @@ -595,11 +666,11 @@ def to_filename(filename): def init_spaces(checkpoint=True): """Initialize the :attr:`~workflow.spaces` setting.""" from niworkflows.utils.spaces import Reference, SpatialReferences + spaces = execution.output_spaces or SpatialReferences() if not isinstance(spaces, SpatialReferences): spaces = SpatialReferences( - [ref for s in spaces.split(' ') - for ref in Reference.from_string(s)] + [ref for s in spaces.split(" ") for ref in Reference.from_string(s)] ) if checkpoint and not spaces.is_cached(): @@ -607,29 +678,21 @@ def init_spaces(checkpoint=True): # Add the default standard space if not already present (required by several sub-workflows) if "MNI152NLin2009cAsym" not in spaces.get_spaces(nonstandard=False, dim=(3,)): - spaces.add( - Reference("MNI152NLin2009cAsym", {}) - ) + spaces.add(Reference("MNI152NLin2009cAsym", {})) # Ensure user-defined spatial references for outputs are correctly parsed. # Certain options require normalization to a space not explicitly defined by users. # These spaces will not be included in the final outputs. if workflow.use_aroma: # Make sure there's a normalization to FSL for AROMA to use. - spaces.add( - Reference("MNI152NLin6Asym", {"res": "2"}) - ) + spaces.add(Reference("MNI152NLin6Asym", {"res": "2"})) cifti_output = workflow.cifti_output if cifti_output: # CIFTI grayordinates to corresponding FSL-MNI resolutions. - vol_res = '2' if cifti_output == '91k' else '1' - spaces.add( - Reference("fsaverage", {"den": "164k"}) - ) - spaces.add( - Reference("MNI152NLin6Asym", {"res": vol_res}) - ) + vol_res = "2" if cifti_output == "91k" else "1" + spaces.add(Reference("fsaverage", {"den": "164k"})) + spaces.add(Reference("MNI152NLin6Asym", {"res": vol_res})) # Make the SpatialReferences object available workflow.spaces = spaces diff --git a/fmriprep/data/reports-spec.yml b/fmriprep/data/reports-spec.yml index af0b3f698..4552a20e3 100644 --- a/fmriprep/data/reports-spec.yml +++ b/fmriprep/data/reports-spec.yml @@ -85,11 +85,14 @@ sections: subtitle: Alignment of functional and anatomical MRI data (surface driven) - bids: {datatype: figures, desc: rois, suffix: bold} caption: Brain mask calculated on the BOLD signal (red contour), along with the - masks used for a/tCompCor.
The aCompCor mask (magenta contour) is a conservative - CSF and white-matter mask for extracting physiological and movement confounds. -
The fCompCor mask (blue contour) contains the top 5% most variable voxels - within a heavily-eroded brain-mask. - subtitle: Brain mask and (temporal/anatomical) CompCor ROIs + regions of interest (ROIs) used in a/tCompCor for extracting + physiological and movement confounding components.
+ The anatomical CompCor ROI (magenta contour) is a mask combining + CSF and WM (white-matter), where voxels containing a minimal partial volume + of GM have been removed.
+ The temporal CompCor ROI (blue contour) contains the top 2% most + variable voxels within the brain mask. + subtitle: Brain mask and (anatomical/temporal) CompCor ROIs - bids: datatype: figures desc: '[at]compcor' @@ -107,10 +110,11 @@ sections: in the BOLD data. Global signals calculated within the whole-brain (GS), within the white-matter (WM) and within cerebro-spinal fluid (CSF) show the mean BOLD signal in their corresponding masks. DVARS and FD show the standardized DVARS - and framewise-displacement measures for each time point.
A carpet plot - shows the time series for all voxels within the brain mask, or if ``--cifti-output`` - was enabled, all grayordinates. Voxels are grouped into cortical (dark/light blue), - and subcortical (orange) gray matter, cerebellum (green) and white matter and CSF + and framewise-displacement measures for each time point.
+ A carpet plot shows the time series for all voxels within the brain mask, + or if --cifti-output was enabled, all grayordinates. + Voxels are grouped into cortical (dark/light blue), and subcortical (orange) + gray matter, cerebellum (green) and white matter and CSF (red), indicated by the color map on the left-hand side. subtitle: BOLD Summary - bids: {datatype: figures, desc: 'confoundcorr', suffix: bold} diff --git a/fmriprep/data/tests/config.toml b/fmriprep/data/tests/config.toml index d02b31f38..7c525cda6 100644 --- a/fmriprep/data/tests/config.toml +++ b/fmriprep/data/tests/config.toml @@ -61,3 +61,9 @@ stop_on_first_crash = false [nipype.plugin_args] maxtasksperchild = 1 raise_insufficient = false + +[execution.bids_filters.t1w] +reconstruction = "" + +[execution.bids_filters.t2w] +reconstruction = "" diff --git a/fmriprep/interfaces/confounds.py b/fmriprep/interfaces/confounds.py index a4c8b863c..2a8a68cfe 100644 --- a/fmriprep/interfaces/confounds.py +++ b/fmriprep/interfaces/confounds.py @@ -18,12 +18,41 @@ from nipype.utils.filemanip import fname_presuffix from nipype.interfaces.base import ( traits, TraitedSpec, BaseInterfaceInputSpec, File, Directory, isdefined, - SimpleInterface + SimpleInterface, InputMultiObject, OutputMultiObject ) LOGGER = logging.getLogger('nipype.interface') +class _aCompCorMasksInputSpec(BaseInterfaceInputSpec): + in_vfs = InputMultiObject(File(exists=True), desc="Input volume fractions.") + is_aseg = traits.Bool(False, usedefault=True, + desc="Whether the input volume fractions come from FS' aseg.") + bold_zooms = traits.Tuple(traits.Float, traits.Float, traits.Float, mandatory=True, + desc="BOLD series zooms") + + +class _aCompCorMasksOutputSpec(TraitedSpec): + out_masks = OutputMultiObject(File(exists=True), + desc="CSF, WM and combined masks, respectively") + + +class aCompCorMasks(SimpleInterface): + """Generate masks in T1w space for aCompCor.""" + + input_spec = _aCompCorMasksInputSpec + output_spec = _aCompCorMasksOutputSpec + + def _run_interface(self, runtime): + from ..utils.confounds import acompcor_masks + self._results["out_masks"] = acompcor_masks( + self.inputs.in_vfs, + self.inputs.is_aseg, + self.inputs.bold_zooms, + ) + return runtime + + class GatherConfoundsInputSpec(BaseInterfaceInputSpec): signals = File(exists=True, desc='input signals') dvars = File(exists=True, desc='file containing DVARS') diff --git a/fmriprep/interfaces/reports.py b/fmriprep/interfaces/reports.py index 14a5a6d6c..3d5e4780e 100644 --- a/fmriprep/interfaces/reports.py +++ b/fmriprep/interfaces/reports.py @@ -32,6 +32,7 @@ \t\t
    \t\t\t
  • Repetition time (TR): {tr:.03g}s
    • \t\t\t
  • Phase-encoding (PE) direction: {pedir}
    • +\t\t\t
  • {multiecho}
    • \t\t\t
  • Slice timing correction: {stc}
    • \t\t\t
  • Susceptibility distortion correction: {sdc}
    • \t\t\t
  • Registration: {registration}
    • @@ -171,6 +172,7 @@ class FunctionalSummaryInputSpec(BaseInterfaceInputSpec): tr = traits.Float(desc='Repetition time', mandatory=True) dummy_scans = traits.Either(traits.Int(), None, desc='number of dummy scans specified by user') algo_dummy_scans = traits.Int(desc='number of dummy scans determined by algorithm') + echo_idx = traits.List([], usedefault=True, desc="BIDS echo identifiers") class FunctionalSummary(SummaryInterface): @@ -218,10 +220,20 @@ def _generate_segment(self): else: dummy_scan_msg = dummy_scan_tmp.format(n_dum=self.inputs.algo_dummy_scans) + multiecho = "Single-echo EPI sequence." + n_echos = len(self.inputs.echo_idx) + if n_echos == 1: + multiecho = ( + f"Multi-echo EPI sequence: only echo {self.inputs.echo_idx[0]} processed " + "in single-echo mode." + ) + if n_echos > 2: + multiecho = (f"Multi-echo EPI sequence: {n_echos} echoes.") + return FUNCTIONAL_TEMPLATE.format( pedir=pedir, stc=stc, sdc=self.inputs.distortion_correction, registration=reg, confounds=re.sub(r'[\t ]+', ', ', conflist), tr=self.inputs.tr, - dummy_scan_desc=dummy_scan_msg) + dummy_scan_desc=dummy_scan_msg, multiecho=multiecho) class AboutSummaryInputSpec(BaseInterfaceInputSpec): diff --git a/fmriprep/utils/bids.py b/fmriprep/utils/bids.py index 212d48b8b..7bb05a3df 100644 --- a/fmriprep/utils/bids.py +++ b/fmriprep/utils/bids.py @@ -7,20 +7,34 @@ from pathlib import Path +def write_bidsignore(deriv_dir): + bids_ignore = ( + "*.html", "logs/", "figures/", # Reports + "*_xfm.*", # Unspecified transform files + "*.surf.gii", # Unspecified structural outputs + # Unspecified functional outputs + "*_boldref.nii.gz", "*_bold.func.gii", + "*_mixing.tsv", "*_AROMAnoiseICs.csv", "*_timeseries.tsv", + ) + ignore_file = Path(deriv_dir) / ".bidsignore" + + ignore_file.write_text("\n".join(bids_ignore) + "\n") + + def write_derivative_description(bids_dir, deriv_dir): - from ..__about__ import __version__, __url__, DOWNLOAD_URL + from ..__about__ import __version__, DOWNLOAD_URL bids_dir = Path(bids_dir) deriv_dir = Path(deriv_dir) desc = { 'Name': 'fMRIPrep - fMRI PREProcessing workflow', - 'BIDSVersion': '1.1.1', - 'PipelineDescription': { + 'BIDSVersion': '1.4.0', + 'DatasetType': 'derivative', + 'GeneratedBy': [{ 'Name': 'fMRIPrep', 'Version': __version__, 'CodeURL': DOWNLOAD_URL, - }, - 'CodeURL': __url__, + }], 'HowToAcknowledge': 'Please cite our paper (https://doi.org/10.1038/s41592-018-0235-4), ' 'and include the generated citation boilerplate within the Methods ' @@ -29,30 +43,31 @@ def write_derivative_description(bids_dir, deriv_dir): # Keys that can only be set by environment if 'FMRIPREP_DOCKER_TAG' in os.environ: - desc['DockerHubContainerTag'] = os.environ['FMRIPREP_DOCKER_TAG'] + desc['GeneratedBy'][0]['Container'] = { + "Type": "docker", + "Tag": f"poldracklab/fmriprep:{os.environ['FMRIPREP_DOCKER_TAG']}" + } if 'FMRIPREP_SINGULARITY_URL' in os.environ: - singularity_url = os.environ['FMRIPREP_SINGULARITY_URL'] - desc['SingularityContainerURL'] = singularity_url - - singularity_md5 = _get_shub_version(singularity_url) - if singularity_md5 and singularity_md5 is not NotImplemented: - desc['SingularityContainerMD5'] = _get_shub_version(singularity_url) + desc['GeneratedBy'][0]['Container'] = { + "Type": "singularity", + "URI": os.getenv('FMRIPREP_SINGULARITY_URL') + } # Keys deriving from source dataset orig_desc = {} fname = bids_dir / 'dataset_description.json' if fname.exists(): - with fname.open() as fobj: - orig_desc = json.load(fobj) + orig_desc = json.loads(fname.read_text()) if 'DatasetDOI' in orig_desc: - desc['SourceDatasetsURLs'] = ['https://doi.org/{}'.format( - orig_desc['DatasetDOI'])] + desc['SourceDatasets'] = [{ + 'URL': f'https://doi.org/{orig_desc["DatasetDOI"]}', + 'DOI': orig_desc['DatasetDOI'] + }] if 'License' in orig_desc: desc['License'] = orig_desc['License'] - with (deriv_dir / 'dataset_description.json').open('w') as fobj: - json.dump(desc, fobj, indent=4) + Path.write_text(deriv_dir / 'dataset_description.json', json.dumps(desc, indent=4)) def validate_input_dir(exec_env, bids_dir, participant_label): @@ -140,7 +155,3 @@ def validate_input_dir(exec_env, bids_dir, participant_label): subprocess.check_call(['bids-validator', bids_dir, '-c', temp.name]) except FileNotFoundError: print("bids-validator does not appear to be installed", file=sys.stderr) - - -def _get_shub_version(singularity_url): - return NotImplemented diff --git a/fmriprep/utils/confounds.py b/fmriprep/utils/confounds.py new file mode 100644 index 000000000..25a19980c --- /dev/null +++ b/fmriprep/utils/confounds.py @@ -0,0 +1,136 @@ +"""Utilities for confounds manipulation.""" + + +def mask2vf(in_file, zooms=None, out_file=None): + """ + Convert a binary mask on a volume fraction map. + + The algorithm simply applies a Gaussian filter with the kernel size scaled + by the zooms given as argument. + + """ + import numpy as np + import nibabel as nb + from scipy.ndimage import gaussian_filter + + img = nb.load(in_file) + imgzooms = np.array(img.header.get_zooms()[:3], dtype=float) + if zooms is None: + zooms = imgzooms + + zooms = np.array(zooms, dtype=float) + sigma = 0.5 * (zooms / imgzooms) + + data = gaussian_filter(img.get_fdata(dtype=np.float32), sigma=sigma) + + max_data = np.percentile(data[data > 0], 99) + data = np.clip(data / max_data, a_min=0, a_max=1) + + if out_file is None: + return data + + hdr = img.header.copy() + hdr.set_data_dtype(np.float32) + nb.Nifti1Image(data.astype(np.float32), img.affine, hdr).to_filename(out_file) + return out_file + + +def acompcor_masks(in_files, is_aseg=False, zooms=None): + """ + Generate aCompCor masks. + + This function selects the CSF partial volume map from the input, + and generates the WM and combined CSF+WM masks for aCompCor. + + The implementation deviates from Behzadi et al. + Their original implementation thresholded the CSF and the WM partial-volume + masks at 0.99 (i.e., 99% of the voxel volume is filled with a particular tissue), + and then binary eroded that 2 voxels: + + > Anatomical data were segmented into gray matter, white matter, + > and CSF partial volume maps using the FAST algorithm available + > in the FSL software package (Smith et al., 2004). Tissue partial + > volume maps were linearly interpolated to the resolution of the + > functional data series using AFNI (Cox, 1996). In order to form + > white matter ROIs, the white matter partial volume maps were + > thresholded at a partial volume fraction of 0.99 and then eroded by + > two voxels in each direction to further minimize partial voluming + > with gray matter. CSF voxels were determined by first thresholding + > the CSF partial volume maps at 0.99 and then applying a threedimensional + > nearest neighbor criteria to minimize multiple tissue + > partial voluming. Since CSF regions are typically small compared + > to white matter regions mask, erosion was not applied. + + This particular procedure is not generalizable to BOLD data with different voxel zooms + as the mathematical morphology operations will be scaled by those. + Also, from reading the excerpt above and the tCompCor description, I (@oesteban) + believe that they always operated slice-wise given the large slice-thickness of + their functional data. + + Instead, *fMRIPrep*'s implementation deviates from Behzadi's implementation on two + aspects: + + * the masks are prepared in high-resolution, anatomical space and then + projected into BOLD space; and, + * instead of using binary erosion, a dilated GM map is generated -- thresholding + the corresponding PV map at 0.05 (i.e., pixels containing at least 5% of GM tissue) + and then subtracting that map from the CSF, WM and CSF+WM (combined) masks. + This should be equivalent to eroding the masks, except that the erosion + only happens at direct interfaces with GM. + + When the probseg maps provene from FreeSurfer's ``recon-all`` (i.e., they are + discrete), binary maps are *transformed* into some sort of partial volume maps + by means of a Gaussian smoothing filter with sigma adjusted by the size of the + BOLD data. + + """ + from pathlib import Path + import numpy as np + import nibabel as nb + from scipy.ndimage import binary_dilation + from skimage.morphology import ball + + csf_file = in_files[2] # BIDS labeling (CSF=2; last of list) + # Load PV maps (fast) or segments (recon-all) + gm_vf = nb.load(in_files[0]) + wm_vf = nb.load(in_files[1]) + csf_vf = nb.load(csf_file) + + # Prepare target zooms + imgzooms = np.array(gm_vf.header.get_zooms()[:3], dtype=float) + if zooms is None: + zooms = imgzooms + zooms = np.array(zooms, dtype=float) + + if not is_aseg: + gm_data = gm_vf.get_fdata() > 0.05 + wm_data = wm_vf.get_fdata() + csf_data = csf_vf.get_fdata() + else: + csf_file = mask2vf( + csf_file, + zooms=zooms, + out_file=str(Path("acompcor_csf.nii.gz").absolute()), + ) + csf_data = nb.load(csf_file).get_fdata() + wm_data = mask2vf(in_files[1], zooms=zooms) + + # We do not have partial volume maps (recon-all route) + gm_data = np.asanyarray(gm_vf.dataobj, np.uint8) > 0 + + # Dilate the GM mask + gm_data = binary_dilation(gm_data, structure=ball(3)) + + # Output filenames + wm_file = str(Path("acompcor_wm.nii.gz").absolute()) + combined_file = str(Path("acompcor_wmcsf.nii.gz").absolute()) + + # Prepare WM mask + wm_data[gm_data] = 0 # Make sure voxel does not contain GM + nb.Nifti1Image(wm_data, gm_vf.affine, gm_vf.header).to_filename(wm_file) + + # Prepare combined CSF+WM mask + comb_data = csf_data + wm_data + comb_data[gm_data] = 0 # Make sure voxel does not contain GM + nb.Nifti1Image(comb_data, gm_vf.affine, gm_vf.header).to_filename(combined_file) + return [csf_file, wm_file, combined_file] diff --git a/fmriprep/utils/misc.py b/fmriprep/utils/misc.py index d69bf1f43..52eae4b43 100644 --- a/fmriprep/utils/misc.py +++ b/fmriprep/utils/misc.py @@ -9,5 +9,5 @@ def check_deps(workflow): return sorted( (node.interface.__class__.__name__, node.interface._cmd) for node in workflow._get_all_nodes() - if (hasattr(node.interface, '_cmd') and - which(node.interface._cmd.split()[0]) is None)) + if (hasattr(node.interface, '_cmd') + and which(node.interface._cmd.split()[0]) is None)) diff --git a/fmriprep/workflows/base.py b/fmriprep/workflows/base.py index 15bea8386..8a19e074b 100644 --- a/fmriprep/workflows/base.py +++ b/fmriprep/workflows/base.py @@ -138,6 +138,8 @@ def init_single_subject_wf(subject_id): subject_data['t2w'] = [] anat_only = config.workflow.anat_only + anat_derivatives = config.execution.anat_derivatives + spaces = config.workflow.spaces # Make sure we always go through these two checks if not anat_only and not subject_data['bold']: task_id = config.execution.task_id @@ -147,7 +149,23 @@ def init_single_subject_wf(subject_id): subject_id, task_id if task_id else '') ) - if not subject_data['t1w']: + if anat_derivatives: + from smriprep.utils.bids import collect_derivatives + std_spaces = spaces.get_spaces(nonstandard=False, dim=(3,)) + anat_derivatives = collect_derivatives( + anat_derivatives.absolute(), + subject_id, + std_spaces, + config.workflow.run_reconall, + ) + if anat_derivatives is None: + config.loggers.workflow.warning(f"""\ +Attempted to access pre-existing anatomical derivatives at \ +<{config.execution.anat_derivatives}>, however not all expectations of fMRIPrep \ +were met (for participant <{subject_id}>, spaces <{', '.join(std_spaces)}>, \ +reconall <{config.workflow.run_reconall}>).""") + + if not anat_derivatives and not subject_data['t1w']: raise Exception("No T1w images found for participant {}. " "All workflows require T1w images.".format(subject_id)) @@ -184,7 +202,6 @@ def init_single_subject_wf(subject_id): """.format(nilearn_ver=NILEARN_VERSION) - spaces = config.workflow.spaces output_dir = str(config.execution.output_dir) inputnode = pe.Node(niu.IdentityInterface(fields=['subjects_dir']), @@ -192,6 +209,7 @@ def init_single_subject_wf(subject_id): bidssrc = pe.Node(BIDSDataGrabber(subject_data=subject_data, anat_only=anat_only, + anat_derivatives=anat_derivatives, subject_id=subject_id), name='bidssrc') @@ -216,27 +234,10 @@ def init_single_subject_wf(subject_id): dismiss_entities=("echo",)), name='ds_report_about', run_without_submitting=True) - anat_derivatives = config.execution.anat_derivatives - if anat_derivatives: - from smriprep.utils.bids import collect_derivatives - std_spaces = spaces.get_spaces(nonstandard=False, dim=(3,)) - anat_derivatives = collect_derivatives( - anat_derivatives.absolute(), - subject_id, - std_spaces, - config.workflow.run_reconall, - ) - if anat_derivatives is None: - config.loggers.workflow.warning(f"""\ -Attempted to access pre-existing anatomical derivatives at \ -<{config.execution.anat_derivatives}>, however not all expectations of fMRIPrep \ -were met (for participant <{subject_id}>, spaces <{', '.join(std_spaces)}>, \ -reconall <{config.workflow.run_reconall}>).""") - # Preprocessing of T1w (includes registration to MNI) anat_preproc_wf = init_anat_preproc_wf( bids_root=str(config.execution.bids_dir), - debug=config.execution.debug is True, + debug=config.execution.sloppy, existing_derivatives=anat_derivatives, freesurfer=config.workflow.run_reconall, hires=config.workflow.hires, @@ -253,23 +254,34 @@ def init_single_subject_wf(subject_id): workflow.connect([ (inputnode, anat_preproc_wf, [('subjects_dir', 'inputnode.subjects_dir')]), - (bidssrc, bids_info, [(('t1w', fix_multi_T1w_source_name), 'in_file')]), (inputnode, summary, [('subjects_dir', 'subjects_dir')]), - (bidssrc, summary, [('t1w', 't1w'), - ('t2w', 't2w'), - ('bold', 'bold')]), + (bidssrc, summary, [('bold', 'bold')]), (bids_info, summary, [('subject', 'subject_id')]), (bids_info, anat_preproc_wf, [(('subject', _prefix), 'inputnode.subject_id')]), (bidssrc, anat_preproc_wf, [('t1w', 'inputnode.t1w'), ('t2w', 'inputnode.t2w'), ('roi', 'inputnode.roi'), ('flair', 'inputnode.flair')]), - (bidssrc, ds_report_summary, [(('t1w', fix_multi_T1w_source_name), 'source_file')]), (summary, ds_report_summary, [('out_report', 'in_file')]), - (bidssrc, ds_report_about, [(('t1w', fix_multi_T1w_source_name), 'source_file')]), (about, ds_report_about, [('out_report', 'in_file')]), ]) + if not anat_derivatives: + workflow.connect([ + (bidssrc, bids_info, [(('t1w', fix_multi_T1w_source_name), 'in_file')]), + (bidssrc, summary, [('t1w', 't1w'), + ('t2w', 't2w')]), + (bidssrc, ds_report_summary, [(('t1w', fix_multi_T1w_source_name), 'source_file')]), + (bidssrc, ds_report_about, [(('t1w', fix_multi_T1w_source_name), 'source_file')]), + ]) + else: + workflow.connect([ + (bidssrc, bids_info, [(('bold', fix_multi_T1w_source_name), 'in_file')]), + (anat_preproc_wf, summary, [('outputnode.t1w_preproc', 't1w')]), + (anat_preproc_wf, ds_report_summary, [('outputnode.t1w_preproc', 'source_file')]), + (anat_preproc_wf, ds_report_about, [('outputnode.t1w_preproc', 'source_file')]), + ]) + # Overwrite ``out_path_base`` of smriprep's DataSinks for node in workflow.list_node_names(): if node.split('.')[-1].startswith('ds_'): diff --git a/fmriprep/workflows/bold/base.py b/fmriprep/workflows/bold/base.py index 48707913c..86b5bfd9a 100644 --- a/fmriprep/workflows/bold/base.py +++ b/fmriprep/workflows/bold/base.py @@ -17,6 +17,9 @@ from nipype.pipeline import engine as pe from nipype.interfaces import utility as niu +from niworkflows.utils.connections import pop_file, listify + + from ...utils.meepi import combine_meepi_source from ...interfaces import DerivativesDataSink @@ -122,7 +125,7 @@ def init_func_preproc_wf(bold_file): * :py:func:`~fmriprep.workflows.bold.t2s.init_bold_t2s_wf` * :py:func:`~fmriprep.workflows.bold.registration.init_bold_t1_trans_wf` * :py:func:`~fmriprep.workflows.bold.registration.init_bold_reg_wf` - * :py:func:`~fmriprep.workflows.bold.confounds.init_bold_confounds_wf` + * :py:func:`~fmriprep.workflows.bold.confounds.init_bold_confs_wf` * :py:func:`~fmriprep.workflows.bold.confounds.init_ica_aroma_wf` * :py:func:`~fmriprep.workflows.bold.resampling.init_bold_std_trans_wf` * :py:func:`~fmriprep.workflows.bold.resampling.init_bold_preproc_trans_wf` @@ -141,59 +144,66 @@ def init_func_preproc_wf(bold_file): from niworkflows.interfaces.utils import DictMerge from sdcflows.workflows.base import init_sdc_estimate_wf, fieldmap_wrangler - ref_file = bold_file mem_gb = {'filesize': 1, 'resampled': 1, 'largemem': 1} bold_tlen = 10 - multiecho = isinstance(bold_file, list) # Have some options handy - layout = config.execution.layout omp_nthreads = config.nipype.omp_nthreads freesurfer = config.workflow.run_reconall spaces = config.workflow.spaces output_dir = str(config.execution.output_dir) + # Extract BIDS entities and metadata from BOLD file(s) + entities = extract_entities(bold_file) + layout = config.execution.layout + + # Take first file as reference + ref_file = pop_file(bold_file) + metadata = layout.get_metadata(ref_file) + + echo_idxs = listify(entities.get("echo", [])) + multiecho = len(echo_idxs) > 2 + if len(echo_idxs) == 1: + config.loggers.warning( + f"Running a single echo <{ref_file}> from a seemingly multi-echo dataset." + ) + bold_file = ref_file # Just in case - drop the list + + if len(echo_idxs) == 2: + raise RuntimeError( + "Multi-echo processing requires at least three different echos (found two)." + ) + if multiecho: - tes = [layout.get_metadata(echo)['EchoTime'] for echo in bold_file] - ref_file = dict(zip(tes, bold_file))[min(tes)] + # Drop echo entity for future queries, have a boolean shorthand + entities.pop("echo", None) + # reorder echoes from shortest to largest + tes, bold_file = zip(*sorted([ + (layout.get_metadata(bf)["EchoTime"], bf) for bf in bold_file + ])) + ref_file = bold_file[0] # Reset reference to be the shortest TE if os.path.isfile(ref_file): bold_tlen, mem_gb = _create_mem_gb(ref_file) wf_name = _get_wf_name(ref_file) config.loggers.workflow.debug( - 'Creating bold processing workflow for "%s" (%.2f GB / %d TRs). ' + 'Creating bold processing workflow for <%s> (%.2f GB / %d TRs). ' 'Memory resampled/largemem=%.2f/%.2f GB.', ref_file, mem_gb['filesize'], bold_tlen, mem_gb['resampled'], mem_gb['largemem']) - sbref_file = None # Find associated sbref, if possible - # entities = layout.parse_file_entities(ref_file) # entities['suffix'] = 'sbref' - # entities['extension'] = ['nii', 'nii.gz'] # Overwrite extensions - # files = layout.get(return_type='file', **entities) - # refbase = os.path.basename(ref_file) - # if 'sbref' in config.workflow.ignore: - # config.loggers.workflow.info("Single-band reference files ignored.") - # elif files and multiecho: - # config.loggers.workflow.warning( - # "Single-band reference found, but not supported in " - # "multi-echo workflows at this time. Ignoring.") - # elif files: - # sbref_file = files[0] - # sbbase = os.path.basename(sbref_file) - # if len(files) > 1: - # config.loggers.workflow.warning( - # "Multiple single-band reference files found for {}; using " - # "{}".format(refbase, sbbase)) - # else: - # config.loggers.workflow.info("Using single-band reference file %s.", - # sbbase) - # else: - # config.loggers.workflow.info("No single-band-reference found for %s.", - # refbase) + # entities['extension'] = ['.nii', '.nii.gz'] # Overwrite extensions + # sbref_files = layout.get(return_type='file', **entities) - metadata = layout.get_metadata(ref_file) + sbref_msg = f"No single-band-reference found for {os.path.basename(ref_file)}." + if sbref_files and 'sbref' in config.workflow.ignore: + sbref_msg = "Single-band reference file(s) found and ignored." + elif sbref_files: + sbref_msg = "Using single-band reference file(s) {}.".format( + ','.join([os.path.basename(sbf) for sbf in sbref_files])) + config.loggers.workflow.info(sbref_msg) # Find fieldmaps. Options: (phase1|phase2|phasediff|epi|fieldmap|syn) fmaps = None @@ -206,9 +216,11 @@ def init_func_preproc_wf(bold_file): fmaps = {'syn': False} # Short circuits: (True and True and (False or 'TooShort')) == 'TooShort' - run_stc = (bool(metadata.get("SliceTiming")) and - 'slicetiming' not in config.workflow.ignore and - (_get_series_len(ref_file) > 4 or "TooShort")) + run_stc = ( + bool(metadata.get("SliceTiming")) + and 'slicetiming' not in config.workflow.ignore + and (_get_series_len(ref_file) > 4 or "TooShort") + ) # Build workflow workflow = Workflow(name=wf_name) @@ -232,12 +244,10 @@ def init_func_preproc_wf(bold_file): 't1w2fsnative_xfm', 'fsnative2t1w_xfm']), name='inputnode') inputnode.inputs.bold_file = bold_file - if sbref_file is not None: - from niworkflows.interfaces.images import ValidateImage - val_sbref = pe.Node(ValidateImage(in_file=sbref_file), name='val_sbref') outputnode = pe.Node(niu.IdentityInterface( - fields=['bold_t1', 'bold_t1_ref', 'bold_mask_t1', 'bold_aseg_t1', 'bold_aparc_t1', + fields=['bold_t1', 'bold_t1_ref', 'bold2anat_xfm', 'anat2bold_xfm', + 'bold_mask_t1', 'bold_aseg_t1', 'bold_aparc_t1', 'bold_std', 'bold_std_ref', 'bold_mask_std', 'bold_aseg_std', 'bold_aparc_std', 'bold_native', 'bold_cifti', 'cifti_variant', 'cifti_metadata', 'cifti_density', 'surfaces', 'confounds', 'aroma_noise_ics', 'melodic_mix', 'nonaggr_denoised_file', @@ -258,6 +268,7 @@ def init_func_preproc_wf(bold_file): registration_dof=config.workflow.bold2t1w_dof, registration_init=config.workflow.bold2t1w_init, pe_direction=metadata.get("PhaseEncodingDirection"), + echo_idx=echo_idxs, tr=metadata.get("RepetitionTime")), name='summary', mem_gb=config.DEFAULT_MEMORY_MIN_GB, run_without_submitting=True) summary.inputs.dummy_scans = config.workflow.dummy_scans @@ -276,6 +287,8 @@ def init_func_preproc_wf(bold_file): (outputnode, func_derivatives_wf, [ ('bold_t1', 'inputnode.bold_t1'), ('bold_t1_ref', 'inputnode.bold_t1_ref'), + ('bold2anat_xfm', 'inputnode.bold2anat_xfm'), + ('anat2bold_xfm', 'inputnode.anat2bold_xfm'), ('bold_aseg_t1', 'inputnode.bold_aseg_t1'), ('bold_aparc_t1', 'inputnode.bold_aparc_t1'), ('bold_mask_t1', 'inputnode.bold_mask_t1'), @@ -290,16 +303,20 @@ def init_func_preproc_wf(bold_file): ('cifti_metadata', 'inputnode.cifti_metadata'), ('cifti_density', 'inputnode.cifti_density'), ('confounds_metadata', 'inputnode.confounds_metadata'), + ('acompcor_masks', 'inputnode.acompcor_masks'), + ('tcompcor_mask', 'inputnode.tcompcor_mask'), ]), ]) # Generate a tentative boldref - bold_reference_wf = init_bold_reference_wf(omp_nthreads=omp_nthreads) - bold_reference_wf.inputs.inputnode.dummy_scans = config.workflow.dummy_scans - if sbref_file is not None: - workflow.connect([ - (val_sbref, bold_reference_wf, [('out_file', 'inputnode.sbref_file')]), - ]) + initial_boldref_wf = init_bold_reference_wf( + name='initial_boldref_wf', + omp_nthreads=omp_nthreads, + bold_file=bold_file, + sbref_files=sbref_files, + multiecho=multiecho, + ) + initial_boldref_wf.inputs.inputnode.dummy_scans = config.workflow.dummy_scans # Top-level BOLD splitter bold_split = pe.Node(FSLSplit(dimension='t'), name='bold_split', @@ -318,7 +335,7 @@ def init_func_preproc_wf(bold_file): mem_gb=mem_gb['resampled'], name='bold_reg_wf', omp_nthreads=omp_nthreads, - sloppy=config.execution.debug, + sloppy=config.execution.sloppy, use_bbr=config.workflow.use_bbr, use_compression=False, ) @@ -326,8 +343,6 @@ def init_func_preproc_wf(bold_file): # apply BOLD registration to T1w bold_t1_trans_wf = init_bold_t1_trans_wf(name='bold_t1_trans_wf', freesurfer=freesurfer, - use_fieldwarp=bool(fmaps), - multiecho=multiecho, mem_gb=mem_gb['resampled'], omp_nthreads=omp_nthreads, use_compression=False) @@ -336,6 +351,7 @@ def init_func_preproc_wf(bold_file): bold_confounds_wf = init_bold_confs_wf( mem_gb=mem_gb['largemem'], metadata=metadata, + freesurfer=freesurfer, regressors_all_comps=config.workflow.regressors_all_comps, regressors_fd_th=config.workflow.regressors_fd_th, regressors_dvars_th=config.workflow.regressors_dvars_th, @@ -353,17 +369,26 @@ def init_func_preproc_wf(bold_file): ) bold_bold_trans_wf.inputs.inputnode.name_source = ref_file + # Generate a new BOLD reference + # This BOLD references *does not use* single-band reference images. + final_boldref_wf = init_bold_reference_wf( + name='final_boldref_wf', + omp_nthreads=omp_nthreads, + multiecho=multiecho, + ) + final_boldref_wf.__desc__ = None # Unset description to avoid second appearance + # SLICE-TIME CORRECTION (or bypass) ############################################# if run_stc is True: # bool('TooShort') == True, so check True explicitly bold_stc_wf = init_bold_stc_wf(name='bold_stc_wf', metadata=metadata) workflow.connect([ - (bold_reference_wf, bold_stc_wf, [ + (initial_boldref_wf, bold_stc_wf, [ ('outputnode.skip_vols', 'inputnode.skip_vols')]), (bold_stc_wf, boldbuffer, [('outputnode.stc_file', 'bold_file')]), ]) if not multiecho: workflow.connect([ - (bold_reference_wf, bold_stc_wf, [ + (initial_boldref_wf, bold_stc_wf, [ ('outputnode.bold_file', 'inputnode.bold_file')])]) else: # for meepi, iterate through stc_wf for all workflows meepi_echos = boldbuffer.clone(name='meepi_echos') @@ -373,7 +398,7 @@ def init_func_preproc_wf(bold_file): elif not multiecho: # STC is too short or False # bypass STC from original BOLD to the splitter through boldbuffer workflow.connect([ - (bold_reference_wf, boldbuffer, [('outputnode.bold_file', 'bold_file')])]) + (initial_boldref_wf, boldbuffer, [('outputnode.bold_file', 'bold_file')])]) else: # for meepi, iterate over all meepi echos to boldbuffer boldbuffer.iterables = ('bold_file', bold_file) @@ -381,19 +406,23 @@ def init_func_preproc_wf(bold_file): # SDC (SUSCEPTIBILITY DISTORTION CORRECTION) or bypass ########################## bold_sdc_wf = init_sdc_estimate_wf(fmaps, metadata, omp_nthreads=omp_nthreads, - debug=config.execution.debug) + debug=config.execution.sloppy) # MULTI-ECHO EPI DATA ############################################# - if multiecho: + if multiecho: # instantiate relevant interfaces, imports from niworkflows.func.util import init_skullstrip_bold_wf skullstrip_bold_wf = init_skullstrip_bold_wf(name='skullstrip_bold_wf') + split_opt_comb = bold_split.clone(name='split_opt_comb') + inputnode.inputs.bold_file = ref_file # Replace reference w first echo - join_echos = pe.JoinNode(niu.IdentityInterface(fields=['bold_files']), - joinsource=('meepi_echos' if run_stc is True else 'boldbuffer'), - joinfield=['bold_files'], - name='join_echos') + join_echos = pe.JoinNode( + niu.IdentityInterface(fields=['bold_files', 'skullstripped_bold_files']), + joinsource=('meepi_echos' if run_stc is True else 'boldbuffer'), + joinfield=['bold_files', 'skullstripped_bold_files'], + name='join_echos' + ) # create optimal combination, adaptive T2* map bold_t2s_wf = init_bold_t2s_wf(echo_times=tes, @@ -401,26 +430,17 @@ def init_func_preproc_wf(bold_file): omp_nthreads=omp_nthreads, name='bold_t2smap_wf') - workflow.connect([ - (skullstrip_bold_wf, join_echos, [ - ('outputnode.skull_stripped_file', 'bold_files')]), - (join_echos, bold_t2s_wf, [ - ('bold_files', 'inputnode.bold_file')]), - ]) - # MAIN WORKFLOW STRUCTURE ####################################################### workflow.connect([ (inputnode, t1w_brain, [('t1w_preproc', 'in_file'), ('t1w_mask', 'in_mask')]), - # Generate early reference - (inputnode, bold_reference_wf, [('bold_file', 'inputnode.bold_file')]), # BOLD buffer has slice-time corrected if it was run, original otherwise (boldbuffer, bold_split, [('bold_file', 'in_file')]), # HMC - (bold_reference_wf, bold_hmc_wf, [ + (initial_boldref_wf, bold_hmc_wf, [ ('outputnode.raw_ref_image', 'inputnode.raw_ref_image'), ('outputnode.bold_file', 'inputnode.bold_file')]), - (bold_reference_wf, summary, [ + (initial_boldref_wf, summary, [ ('outputnode.algo_dummy_scans', 'algo_dummy_scans')]), # EPI-T1 registration workflow (inputnode, bold_reg_wf, [ @@ -438,8 +458,9 @@ def init_func_preproc_wf(bold_file): ('t1w_aparc', 'inputnode.t1w_aparc')]), (t1w_brain, bold_t1_trans_wf, [ ('out_file', 'inputnode.t1w_brain')]), - # unused if multiecho, but this is safe - (bold_hmc_wf, bold_t1_trans_wf, [('outputnode.xforms', 'inputnode.hmc_xforms')]), + (bold_reg_wf, outputnode, [ + ('outputnode.itk_bold_to_t1', 'bold2anat_xfm'), + ('outputnode.itk_t1_to_bold', 'anat2bold_xfm')]), (bold_reg_wf, bold_t1_trans_wf, [ ('outputnode.itk_bold_to_t1', 'inputnode.itk_bold_to_t1')]), (bold_t1_trans_wf, outputnode, [('outputnode.bold_t1', 'bold_t1'), @@ -450,12 +471,11 @@ def init_func_preproc_wf(bold_file): # SDC (or pass-through workflow) (t1w_brain, bold_sdc_wf, [ ('out_file', 'inputnode.t1w_brain')]), - (bold_reference_wf, bold_sdc_wf, [ + (initial_boldref_wf, bold_sdc_wf, [ ('outputnode.ref_image', 'inputnode.epi_file'), ('outputnode.ref_image_brain', 'inputnode.epi_brain'), ('outputnode.bold_mask', 'inputnode.epi_mask')]), (bold_sdc_wf, bold_t1_trans_wf, [ - ('outputnode.out_warp', 'inputnode.fieldwarp'), ('outputnode.epi_mask', 'inputnode.ref_bold_mask'), ('outputnode.epi_brain', 'inputnode.ref_bold_brain')]), (bold_sdc_wf, bold_bold_trans_wf, [ @@ -472,16 +492,15 @@ def init_func_preproc_wf(bold_file): ('outputnode.rmsd_file', 'inputnode.rmsd_file')]), (bold_reg_wf, bold_confounds_wf, [ ('outputnode.itk_t1_to_bold', 'inputnode.t1_bold_xform')]), - (bold_reference_wf, bold_confounds_wf, [ + (initial_boldref_wf, bold_confounds_wf, [ ('outputnode.skip_vols', 'inputnode.skip_vols')]), - (bold_bold_trans_wf, bold_confounds_wf, [ - ('outputnode.bold_mask', 'inputnode.bold_mask'), - ]), + (final_boldref_wf, bold_confounds_wf, [ + ('outputnode.bold_mask', 'inputnode.bold_mask')]), (bold_confounds_wf, outputnode, [ ('outputnode.confounds_file', 'confounds'), - ]), - (bold_confounds_wf, outputnode, [ ('outputnode.confounds_metadata', 'confounds_metadata'), + ('outputnode.acompcor_masks', 'acompcor_masks'), + ('outputnode.tcompcor_mask', 'tcompcor_mask'), ]), # Connect bold_bold_trans_wf (bold_split, bold_bold_trans_wf, [ @@ -499,22 +518,42 @@ def init_func_preproc_wf(bold_file): ('bold_file', 'inputnode.source_file')]), (bold_bold_trans_wf, bold_confounds_wf, [ ('outputnode.bold', 'inputnode.bold')]), + (bold_bold_trans_wf, final_boldref_wf, [ + ('outputnode.bold', 'inputnode.bold_file')]), (bold_split, bold_t1_trans_wf, [ ('out_files', 'inputnode.bold_split')]), + (bold_hmc_wf, bold_t1_trans_wf, [ + ('outputnode.xforms', 'inputnode.hmc_xforms')]), + (bold_sdc_wf, bold_t1_trans_wf, [ + ('outputnode.out_warp', 'inputnode.fieldwarp')]) ]) - else: # for meepi, create and use optimal combination + else: # for meepi, use optimal combination workflow.connect([ # update name source for optimal combination (inputnode, func_derivatives_wf, [ (('bold_file', combine_meepi_source), 'inputnode.source_file')]), + (bold_bold_trans_wf, join_echos, [ + ('outputnode.bold', 'bold_files')]), + (join_echos, final_boldref_wf, [ + ('bold_files', 'inputnode.bold_file')]), (bold_bold_trans_wf, skullstrip_bold_wf, [ ('outputnode.bold', 'inputnode.in_file')]), + (skullstrip_bold_wf, join_echos, [ + ('outputnode.skull_stripped_file', 'skullstripped_bold_files')]), + (join_echos, bold_t2s_wf, [ + ('skullstripped_bold_files', 'inputnode.bold_file')]), (bold_t2s_wf, bold_confounds_wf, [ ('outputnode.bold', 'inputnode.bold')]), - (bold_t2s_wf, bold_t1_trans_wf, [ - ('outputnode.bold', 'inputnode.bold_split')]), + (bold_t2s_wf, split_opt_comb, [ + ('outputnode.bold', 'in_file')]), + (split_opt_comb, bold_t1_trans_wf, [ + ('out_files', 'inputnode.bold_split')]), ]) + # Already applied in bold_bold_trans_wf, which inputs to bold_t2s_wf + bold_t1_trans_wf.inputs.inputnode.fieldwarp = 'identity' + bold_t1_trans_wf.inputs.inputnode.hmc_xforms = 'identity' + if fmaps: from sdcflows.workflows.outputs import init_sdc_unwarp_report_wf # Report on BOLD correction @@ -522,7 +561,7 @@ def init_func_preproc_wf(bold_file): workflow.connect([ (inputnode, fmap_unwarp_report_wf, [ ('t1w_dseg', 'inputnode.in_seg')]), - (bold_reference_wf, fmap_unwarp_report_wf, [ + (initial_boldref_wf, fmap_unwarp_report_wf, [ ('outputnode.ref_image', 'inputnode.in_pre')]), (bold_reg_wf, fmap_unwarp_report_wf, [ ('outputnode.itk_t1_to_bold', 'inputnode.in_xfm')]), @@ -542,45 +581,45 @@ def init_func_preproc_wf(bold_file): bold_sdc_wf.get_node(node).interface.out_path_base = 'fmriprep' bold_sdc_wf.get_node(node).inputs.dismiss_entities = ("echo",) - # if 'syn' in fmaps: - # sdc_select_std = pe.Node( - # KeySelect(fields=['std2anat_xfm']), - # name='sdc_select_std', run_without_submitting=True) - # sdc_select_std.inputs.key = 'MNI152NLin2009cAsym' - # workflow.connect([ - # (inputnode, sdc_select_std, [('std2anat_xfm', 'std2anat_xfm'), - # ('template', 'keys')]), - # (sdc_select_std, bold_sdc_wf, [('std2anat_xfm', 'inputnode.std2anat_xfm')]), - # ]) - - # if fmaps.get('syn') is True: # SyN forced - # syn_unwarp_report_wf = init_sdc_unwarp_report_wf( - # name='syn_unwarp_report_wf', forcedsyn=True) - # workflow.connect([ - # (inputnode, syn_unwarp_report_wf, [ - # ('t1w_dseg', 'inputnode.in_seg')]), - # (bold_reference_wf, syn_unwarp_report_wf, [ - # ('outputnode.ref_image', 'inputnode.in_pre')]), - # (bold_reg_wf, syn_unwarp_report_wf, [ - # ('outputnode.itk_t1_to_bold', 'inputnode.in_xfm')]), - # (bold_sdc_wf, syn_unwarp_report_wf, [ - # ('outputnode.syn_ref', 'inputnode.in_post')]), - # ]) - # - # # Overwrite ``out_path_base`` of unwarping DataSinks - # # And ensure echo is dropped from report - # for node in syn_unwarp_report_wf.list_node_names(): - # if node.split('.')[-1].startswith('ds_'): - # syn_unwarp_report_wf.get_node(node).interface.out_path_base = 'fmriprep' - # syn_unwarp_report_wf.get_node(node).inputs.dismiss_entities = ("echo",) - - # Map final BOLD mask into T1w space (if required) + # if 'syn' in fmaps: + # sdc_select_std = pe.Node( + # KeySelect(fields=['std2anat_xfm']), + # name='sdc_select_std', run_without_submitting=True) + # sdc_select_std.inputs.key = 'MNI152NLin2009cAsym' + # workflow.connect([ + # (inputnode, sdc_select_std, [('std2anat_xfm', 'std2anat_xfm'), + # ('template', 'keys')]), + # (sdc_select_std, bold_sdc_wf, [('std2anat_xfm', 'inputnode.std2anat_xfm')]), + # ]) + + # if fmaps.get('syn') is True: # SyN forced + # syn_unwarp_report_wf = init_sdc_unwarp_report_wf( + # name='syn_unwarp_report_wf', forcedsyn=True) + # workflow.connect([ + # (inputnode, syn_unwarp_report_wf, [ + # ('t1w_dseg', 'inputnode.in_seg')]), + # (initial_boldref_wf, syn_unwarp_report_wf, [ + # ('outputnode.ref_image', 'inputnode.in_pre')]), + # (bold_reg_wf, syn_unwarp_report_wf, [ + # ('outputnode.itk_t1_to_bold', 'inputnode.in_xfm')]), + # (bold_sdc_wf, syn_unwarp_report_wf, [ + # ('outputnode.syn_ref', 'inputnode.in_post')]), + # ]) + + # # Overwrite ``out_path_base`` of unwarping DataSinks + # # And ensure echo is dropped from report + # for node in syn_unwarp_report_wf.list_node_names(): + # if node.split('.')[-1].startswith('ds_'): + # syn_unwarp_report_wf.get_node(node).interface.out_path_base = 'fmriprep' + # syn_unwarp_report_wf.get_node(node).inputs.dismiss_entities = ("echo",) + + # # Map final BOLD mask into T1w space (if required) # nonstd_spaces = set(spaces.get_nonstandard()) # if nonstd_spaces.intersection(('T1w', 'anat')): # from niworkflows.interfaces.fixes import ( # FixHeaderApplyTransforms as ApplyTransforms # ) - # + # boldmask_to_t1w = pe.Node(ApplyTransforms(interpolation='MultiLabel'), # name='boldmask_to_t1w', mem_gb=0.1) # workflow.connect([ @@ -588,19 +627,19 @@ def init_func_preproc_wf(bold_file): # ('outputnode.itk_bold_to_t1', 'transforms')]), # (bold_t1_trans_wf, boldmask_to_t1w, [ # ('outputnode.bold_mask_t1', 'reference_image')]), - # (bold_bold_trans_wf, boldmask_to_t1w, [ + # (final_boldref_wf, boldmask_to_t1w, [ # ('outputnode.bold_mask', 'input_image')]), # (boldmask_to_t1w, outputnode, [ # ('output_image', 'bold_mask_t1')]), # ]) - # + # if nonstd_spaces.intersection(('func', 'run', 'bold', 'boldref', 'sbref')): # workflow.connect([ - # (bold_bold_trans_wf, outputnode, [ - # ('outputnode.bold', 'bold_native')]), - # (bold_bold_trans_wf, func_derivatives_wf, [ - # ('outputnode.bold_ref', 'inputnode.bold_native_ref'), + # (final_boldref_wf, func_derivatives_wf, [ + # ('outputnode.ref_image', 'inputnode.bold_native_ref'), # ('outputnode.bold_mask', 'inputnode.bold_mask_native')]), + # (bold_bold_trans_wf if not multiecho else bold_t2s_wf, outputnode, [ + # ('outputnode.bold', 'bold_native')]) # ]) if spaces.get_spaces(nonstandard=False, dim=(3,)): @@ -613,7 +652,6 @@ def init_func_preproc_wf(bold_file): spaces=spaces, name='bold_std_trans_wf', use_compression=not config.execution.low_mem, - use_fieldwarp=bool(fmaps), ) workflow.connect([ (inputnode, bold_std_trans_wf, [ @@ -622,14 +660,10 @@ def init_func_preproc_wf(bold_file): ('bold_file', 'inputnode.name_source'), ('t1w_aseg', 'inputnode.bold_aseg'), ('t1w_aparc', 'inputnode.bold_aparc')]), - (bold_hmc_wf, bold_std_trans_wf, [ - ('outputnode.xforms', 'inputnode.hmc_xforms')]), + (final_boldref_wf, bold_std_trans_wf, [ + ('outputnode.bold_mask', 'inputnode.bold_mask')]), (bold_reg_wf, bold_std_trans_wf, [ ('outputnode.itk_bold_to_t1', 'inputnode.itk_bold_to_t1')]), - (bold_bold_trans_wf, bold_std_trans_wf, [ - ('outputnode.bold_mask', 'inputnode.bold_mask')]), - (bold_sdc_wf, bold_std_trans_wf, [ - ('outputnode.out_warp', 'inputnode.fieldwarp')]), (bold_std_trans_wf, outputnode, [('outputnode.bold_std', 'bold_std'), ('outputnode.bold_std_ref', 'bold_std_ref'), ('outputnode.bold_mask_std', 'bold_mask_std')]), @@ -649,18 +683,22 @@ def init_func_preproc_wf(bold_file): if not multiecho: workflow.connect([ (bold_split, bold_std_trans_wf, [ - ('out_files', 'inputnode.bold_split')]) + ('out_files', 'inputnode.bold_split')]), + (bold_sdc_wf, bold_std_trans_wf, [ + ('outputnode.out_warp', 'inputnode.fieldwarp')]), + (bold_hmc_wf, bold_std_trans_wf, [ + ('outputnode.xforms', 'inputnode.hmc_xforms')]), ]) else: - split_opt_comb = bold_split.clone(name='split_opt_comb') workflow.connect([ - (bold_t2s_wf, split_opt_comb, [ - ('outputnode.bold', 'in_file')]), (split_opt_comb, bold_std_trans_wf, [ - ('out_files', 'inputnode.bold_split') - ]) + ('out_files', 'inputnode.bold_split')]) ]) + # Already applied in bold_bold_trans_wf, which inputs to bold_t2s_wf + bold_std_trans_wf.inputs.inputnode.fieldwarp = 'identity' + bold_std_trans_wf.inputs.inputnode.hmc_xforms = 'identity' + # func_derivatives_wf internally parametrizes over snapshotted spaces. workflow.connect([ (bold_std_trans_wf, func_derivatives_wf, [ @@ -678,7 +716,6 @@ def init_func_preproc_wf(bold_file): mem_gb=mem_gb['resampled'], metadata=metadata, omp_nthreads=omp_nthreads, - use_fieldwarp=bool(fmaps), err_on_aroma_warn=config.workflow.aroma_err_on_warn, aroma_melodic_dim=config.workflow.aroma_melodic_dim, name='ica_aroma_wf') @@ -704,7 +741,7 @@ def init_func_preproc_wf(bold_file): ('bold_file', 'inputnode.name_source')]), (bold_hmc_wf, ica_aroma_wf, [ ('outputnode.movpar_file', 'inputnode.movpar_file')]), - (bold_reference_wf, ica_aroma_wf, [ + (initial_boldref_wf, ica_aroma_wf, [ ('outputnode.skip_vols', 'inputnode.skip_vols')]), (bold_confounds_wf, join, [ ('outputnode.confounds_file', 'in_file')]), @@ -798,7 +835,7 @@ def init_func_preproc_wf(bold_file): ('std2anat_xfm', 'inputnode.std2anat_xfm')]), (bold_bold_trans_wf if not multiecho else bold_t2s_wf, carpetplot_wf, [ ('outputnode.bold', 'inputnode.bold')]), - (bold_bold_trans_wf, carpetplot_wf, [ + (final_boldref_wf, carpetplot_wf, [ ('outputnode.bold_mask', 'inputnode.bold_mask')]), (bold_reg_wf, carpetplot_wf, [ ('outputnode.itk_t1_to_bold', 'inputnode.t1_bold_xform')]), @@ -806,7 +843,8 @@ def init_func_preproc_wf(bold_file): workflow.connect([ (bold_confounds_wf, carpetplot_wf, [ - ('outputnode.confounds_file', 'inputnode.confounds_file')]) + ('outputnode.confounds_file', 'inputnode.confounds_file') + ]) ]) # REPORTING ############################################################ @@ -823,7 +861,7 @@ def init_func_preproc_wf(bold_file): workflow.connect([ (summary, ds_report_summary, [('out_report', 'in_file')]), - (bold_reference_wf, ds_report_validation, [ + (initial_boldref_wf, ds_report_validation, [ ('outputnode.validation_report', 'in_file')]), ]) @@ -882,3 +920,40 @@ def _to_join(in_file, join_file): return in_file res = JoinTSVColumns(in_file=in_file, join_file=join_file).run() return res.outputs.out_file + + +def extract_entities(file_list): + """ + Return a dictionary of common entities given a list of files. + + Examples + -------- + >>> extract_entities('sub-01/anat/sub-01_T1w.nii.gz') + {'subject': '01', 'suffix': 'T1w', 'datatype': 'anat', 'extension': '.nii.gz'} + >>> extract_entities(['sub-01/anat/sub-01_T1w.nii.gz'] * 2) + {'subject': '01', 'suffix': 'T1w', 'datatype': 'anat', 'extension': '.nii.gz'} + >>> extract_entities(['sub-01/anat/sub-01_run-1_T1w.nii.gz', + ... 'sub-01/anat/sub-01_run-2_T1w.nii.gz']) + {'subject': '01', 'run': [1, 2], 'suffix': 'T1w', 'datatype': 'anat', + 'extension': '.nii.gz'} + + """ + from collections import defaultdict + from bids.layout import parse_file_entities + + entities = defaultdict(list) + for e, v in [ + ev_pair + for f in listify(file_list) + for ev_pair in parse_file_entities(f).items() + ]: + entities[e].append(v) + + def _unique(inlist): + inlist = sorted(set(inlist)) + if len(inlist) == 1: + return inlist[0] + return inlist + return { + k: _unique(v) for k, v in entities.items() + } diff --git a/fmriprep/workflows/bold/confounds.py b/fmriprep/workflows/bold/confounds.py index 313e74c8a..5672355fe 100644 --- a/fmriprep/workflows/bold/confounds.py +++ b/fmriprep/workflows/bold/confounds.py @@ -27,6 +27,7 @@ def init_bold_confs_wf( regressors_all_comps, regressors_dvars_th, regressors_fd_th, + freesurfer=False, name="bold_confs_wf", ): """ @@ -126,6 +127,7 @@ def init_bold_confs_wf( from niworkflows.interfaces.fixes import FixHeaderApplyTransforms as ApplyTransforms from niworkflows.interfaces.images import SignalExtraction from niworkflows.interfaces.masks import ROIsPlot + from niworkflows.interfaces.nibabel import ApplyMask, Binarize from niworkflows.interfaces.patches import ( RobustACompCor as ACompCor, RobustTCompCor as TCompCor, @@ -134,11 +136,17 @@ def init_bold_confs_wf( CompCorVariancePlot, ConfoundsCorrelationPlot ) from niworkflows.interfaces.utils import ( - TPM2ROI, AddTPMs, AddTSVHeader, TSV2JSON, DictMerge + AddTSVHeader, TSV2JSON, DictMerge + ) + from ...interfaces.confounds import aCompCorMasks + + gm_desc = ( + "dilating a GM mask extracted from the FreeSurfer's *aseg* segmentation" if freesurfer + else "thresholding the corresponding partial volume map at 0.05" ) workflow = Workflow(name=name) - workflow.__desc__ = """\ + workflow.__desc__ = f"""\ Several confounding time-series were calculated based on the *preprocessed BOLD*: framewise displacement (FD), DVARS and three region-wise global signals. @@ -155,15 +163,18 @@ def init_bold_confs_wf( *preprocessed BOLD* time-series (using a discrete cosine filter with 128s cut-off) for the two *CompCor* variants: temporal (tCompCor) and anatomical (aCompCor). -tCompCor components are then calculated from the top 5% variable -voxels within a mask covering the subcortical regions. -This subcortical mask is obtained by heavily eroding the brain mask, -which ensures it does not include cortical GM regions. -For aCompCor, components are calculated within the intersection of -the aforementioned mask and the union of CSF and WM masks calculated -in T1w space, after their projection to the native space of each -functional run (using the inverse BOLD-to-T1w transformation). Components -are also calculated separately within the WM and CSF masks. +tCompCor components are then calculated from the top 2% variable +voxels within the brain mask. +For aCompCor, three probabilistic masks (CSF, WM and combined CSF+WM) +are generated in anatomical space. +The implementation differs from that of Behzadi et al. in that instead +of eroding the masks by 2 pixels on BOLD space, the aCompCor masks are +subtracted a mask of pixels that likely contain a volume fraction of GM. +This mask is obtained by {gm_desc}, and it ensures components are not extracted +from voxels containing a minimal fraction of GM. +Finally, these masks are resampled into BOLD space and binarized by +thresholding at 0.99 (as in the original implementation). +Components are also calculated separately within the WM and CSF masks. For each CompCor decomposition, the *k* components with the largest singular values are retained, such that the retained components' time series are sufficient to explain 50 percent of variance across the nuisance mask (CSF, @@ -174,44 +185,17 @@ def init_bold_confs_wf( The confound time series derived from head motion estimates and global signals were expanded with the inclusion of temporal derivatives and quadratic terms for each [@confounds_satterthwaite_2013]. -Frames that exceeded a threshold of {fd} mm FD or {dv} standardised DVARS -were annotated as motion outliers. -""".format(fd=regressors_fd_th, dv=regressors_dvars_th) +Frames that exceeded a threshold of {regressors_fd_th} mm FD or +{regressors_dvars_th} standardised DVARS were annotated as motion outliers. +""" inputnode = pe.Node(niu.IdentityInterface( fields=['bold', 'bold_mask', 'movpar_file', 'rmsd_file', 'skip_vols', 't1w_mask', 't1w_tpms', 't1_bold_xform']), name='inputnode') outputnode = pe.Node(niu.IdentityInterface( - fields=['confounds_file', 'confounds_metadata']), + fields=['confounds_file', 'confounds_metadata', 'acompcor_masks', 'tcompcor_mask']), name='outputnode') - # Get masks ready in T1w space - acc_tpm = pe.Node(AddTPMs(indices=[1, 2]), # BIDS convention (WM=1, CSF=2) - name='acc_tpm') # acc stands for aCompCor - csf_roi = pe.Node(TPM2ROI(erode_mm=0, mask_erode_mm=30), name='csf_roi') - wm_roi = pe.Node(TPM2ROI( - erode_prop=0.6, mask_erode_prop=0.6**3), # 0.6 = radius; 0.6^3 = volume - name='wm_roi') - acc_roi = pe.Node(TPM2ROI( - erode_prop=0.6, mask_erode_prop=0.6**3), # 0.6 = radius; 0.6^3 = volume - name='acc_roi') - - # Map ROIs in T1w space into BOLD space - csf_tfm = pe.Node(ApplyTransforms(interpolation='NearestNeighbor', float=True), - name='csf_tfm', mem_gb=0.1) - wm_tfm = pe.Node(ApplyTransforms(interpolation='NearestNeighbor', float=True), - name='wm_tfm', mem_gb=0.1) - acc_tfm = pe.Node(ApplyTransforms(interpolation='NearestNeighbor', float=True), - name='acc_tfm', mem_gb=0.1) - tcc_tfm = pe.Node(ApplyTransforms(interpolation='NearestNeighbor', float=True), - name='tcc_tfm', mem_gb=0.1) - - # Ensure ROIs don't go off-limits (reduced FoV) - csf_msk = pe.Node(niu.Function(function=_maskroi), name='csf_msk') - wm_msk = pe.Node(niu.Function(function=_maskroi), name='wm_msk') - acc_msk = pe.Node(niu.Function(function=_maskroi), name='acc_msk') - tcc_msk = pe.Node(niu.Function(function=_maskroi), name='tcc_msk') - # DVARS dvars = pe.Node(nac.ComputeDVARS(save_nstd=True, save_std=True, remove_zerovariance=True), name="dvars", mem_gb=mem_gb) @@ -220,21 +204,29 @@ def init_bold_confs_wf( fdisp = pe.Node(nac.FramewiseDisplacement(parameter_source="SPM"), name="fdisp", mem_gb=mem_gb) - # a/t-CompCor - mrg_lbl_cc = pe.Node(niu.Merge(3), name='merge_rois_cc', run_without_submitting=True) + # Generate aCompCor probseg maps + acc_masks = pe.Node(aCompCorMasks(is_aseg=freesurfer), name="acc_masks") + + # Resample probseg maps in BOLD space via T1w-to-BOLD transform + acc_msk_tfm = pe.MapNode(ApplyTransforms( + interpolation='Gaussian', float=False), iterfield=["input_image"], + name='acc_msk_tfm', mem_gb=0.1) + acc_msk_brain = pe.MapNode(ApplyMask(), name="acc_msk_brain", + iterfield=["in_file"]) + acc_msk_bin = pe.MapNode(Binarize(thresh_low=0.99), name='acc_msk_bin', + iterfield=["in_file"]) + acompcor = pe.Node( + ACompCor(components_file='acompcor.tsv', header_prefix='a_comp_cor_', pre_filter='cosine', + save_pre_filter=True, save_metadata=True, mask_names=['CSF', 'WM', 'combined'], + merge_method='none', failure_mode='NaN'), + name="acompcor", mem_gb=mem_gb) tcompcor = pe.Node( TCompCor(components_file='tcompcor.tsv', header_prefix='t_comp_cor_', pre_filter='cosine', - save_pre_filter=True, save_metadata=True, percentile_threshold=.05, + save_pre_filter=True, save_metadata=True, percentile_threshold=.02, failure_mode='NaN'), name="tcompcor", mem_gb=mem_gb) - acompcor = pe.Node( - ACompCor(components_file='acompcor.tsv', header_prefix='a_comp_cor_', pre_filter='cosine', - save_pre_filter=True, save_metadata=True, mask_names=['combined', 'CSF', 'WM'], - merge_method='none', failure_mode='NaN'), - name="acompcor", mem_gb=mem_gb) - # Set number of components if regressors_all_comps: acompcor.inputs.num_components = 'all' @@ -249,8 +241,11 @@ def init_bold_confs_wf( acompcor.inputs.repetition_time = metadata['RepetitionTime'] # Global and segment regressors - signals_class_labels = ["csf", "white_matter", "global_signal"] - mrg_lbl = pe.Node(niu.Merge(3), name='merge_rois', run_without_submitting=True) + signals_class_labels = [ + "global_signal", "csf", "white_matter", "csf_wm", "tcompcor", + ] + merge_rois = pe.Node(niu.Merge(3, ravel_inputs=True), name='merge_rois', + run_without_submitting=True) signals = pe.Node(SignalExtraction(class_labels=signals_class_labels), name="signals", mem_gb=mem_gb) @@ -297,7 +292,8 @@ def init_bold_confs_wf( name='spike_regressors') # Generate reportlet (ROIs) - mrg_compcor = pe.Node(niu.Merge(2), name='merge_compcor', run_without_submitting=True) + mrg_compcor = pe.Node(niu.Merge(2, ravel_inputs=True), + name='mrg_compcor', run_without_submitting=True) rois_plot = pe.Node(ROIsPlot(colors=['b', 'magenta'], generate_report=True), name='rois_plot', mem_gb=mem_gb) @@ -320,74 +316,52 @@ def init_bold_confs_wf( # Generate reportlet (Confound correlation) conf_corr_plot = pe.Node( - ConfoundsCorrelationPlot(reference_column='global_signal', max_dim=70), + ConfoundsCorrelationPlot(reference_column='global_signal', max_dim=20), name='conf_corr_plot') ds_report_conf_corr = pe.Node( DerivativesDataSink(desc='confoundcorr', datatype="figures", dismiss_entities=("echo",)), name='ds_report_conf_corr', run_without_submitting=True, mem_gb=DEFAULT_MEMORY_MIN_GB) - def _pick_csf(files): - return files[2] # after smriprep#189, this is BIDS-compliant. + def _last(inlist): + return inlist[-1] - def _pick_wm(files): - return files[1] # after smriprep#189, this is BIDS-compliant. + def _select_cols(table): + import pandas as pd + return [ + col for col in pd.read_table(table, nrows=2).columns + if not col.startswith(("a_comp_cor_", "t_comp_cor_", "std_dvars")) + ] workflow.connect([ - # Massage ROIs (in T1w space) - (inputnode, acc_tpm, [('t1w_tpms', 'in_files')]), - (inputnode, csf_roi, [(('t1w_tpms', _pick_csf), 'in_tpm'), - ('t1w_mask', 'in_mask')]), - (inputnode, wm_roi, [(('t1w_tpms', _pick_wm), 'in_tpm'), - ('t1w_mask', 'in_mask')]), - (inputnode, acc_roi, [('t1w_mask', 'in_mask')]), - (acc_tpm, acc_roi, [('out_file', 'in_tpm')]), - # Map ROIs to BOLD - (inputnode, csf_tfm, [('bold_mask', 'reference_image'), - ('t1_bold_xform', 'transforms')]), - (csf_roi, csf_tfm, [('roi_file', 'input_image')]), - (inputnode, wm_tfm, [('bold_mask', 'reference_image'), - ('t1_bold_xform', 'transforms')]), - (wm_roi, wm_tfm, [('roi_file', 'input_image')]), - (inputnode, acc_tfm, [('bold_mask', 'reference_image'), - ('t1_bold_xform', 'transforms')]), - (acc_roi, acc_tfm, [('roi_file', 'input_image')]), - (inputnode, tcc_tfm, [('bold_mask', 'reference_image'), - ('t1_bold_xform', 'transforms')]), - (csf_roi, tcc_tfm, [('eroded_mask', 'input_image')]), - # Mask ROIs with bold_mask - (inputnode, csf_msk, [('bold_mask', 'in_mask')]), - (inputnode, wm_msk, [('bold_mask', 'in_mask')]), - (inputnode, acc_msk, [('bold_mask', 'in_mask')]), - (inputnode, tcc_msk, [('bold_mask', 'in_mask')]), # connect inputnode to each non-anatomical confound node (inputnode, dvars, [('bold', 'in_file'), ('bold_mask', 'in_mask')]), (inputnode, fdisp, [('movpar_file', 'in_file')]), - # tCompCor - (inputnode, tcompcor, [('bold', 'realigned_file')]), - (inputnode, tcompcor, [('skip_vols', 'ignore_initial_volumes')]), - (tcc_tfm, tcc_msk, [('output_image', 'roi_file')]), - (tcc_msk, tcompcor, [('out', 'mask_files')]), - # aCompCor - (inputnode, acompcor, [('bold', 'realigned_file')]), - (inputnode, acompcor, [('skip_vols', 'ignore_initial_volumes')]), - (acc_tfm, acc_msk, [('output_image', 'roi_file')]), - (acc_msk, mrg_lbl_cc, [('out', 'in1')]), - (csf_msk, mrg_lbl_cc, [('out', 'in2')]), - (wm_msk, mrg_lbl_cc, [('out', 'in3')]), - (mrg_lbl_cc, acompcor, [('out', 'mask_files')]), + (inputnode, acompcor, [("bold", "realigned_file"), + ("skip_vols", "ignore_initial_volumes")]), + (inputnode, acc_masks, [("t1w_tpms", "in_vfs"), + (("bold", _get_zooms), "bold_zooms")]), + (inputnode, acc_msk_tfm, [("t1_bold_xform", "transforms"), + ("bold_mask", "reference_image")]), + (inputnode, acc_msk_brain, [("bold_mask", "in_mask")]), + (acc_masks, acc_msk_tfm, [("out_masks", "input_image")]), + (acc_msk_tfm, acc_msk_brain, [("output_image", "in_file")]), + (acc_msk_brain, acc_msk_bin, [("out_file", "in_file")]), + (acc_msk_bin, acompcor, [("out_file", "mask_files")]), + # tCompCor + (inputnode, tcompcor, [("bold", "realigned_file"), + ("skip_vols", "ignore_initial_volumes"), + ("bold_mask", "mask_files")]), # Global signals extraction (constrained by anatomy) (inputnode, signals, [('bold', 'in_file')]), - (csf_tfm, csf_msk, [('output_image', 'roi_file')]), - (csf_msk, mrg_lbl, [('out', 'in1')]), - (wm_tfm, wm_msk, [('output_image', 'roi_file')]), - (wm_msk, mrg_lbl, [('out', 'in2')]), - (inputnode, mrg_lbl, [('bold_mask', 'in3')]), - (mrg_lbl, signals, [('out', 'label_files')]), + (inputnode, merge_rois, [('bold_mask', 'in1')]), + (acc_msk_bin, merge_rois, [('out_file', 'in2')]), + (tcompcor, merge_rois, [('high_variance_masks', 'in3')]), + (merge_rois, signals, [('out', 'label_files')]), # Collate computed confounds together (inputnode, add_motion_headers, [('movpar_file', 'in_file')]), @@ -418,17 +392,20 @@ def _pick_wm(files): # Set outputs (spike_regress, outputnode, [('confounds_file', 'confounds_file')]), (mrg_conf_metadata2, outputnode, [('out_dict', 'confounds_metadata')]), + (tcompcor, outputnode, [("high_variance_masks", "tcompcor_mask")]), + (acc_msk_bin, outputnode, [("out_file", "acompcor_masks")]), (inputnode, rois_plot, [('bold', 'in_file'), ('bold_mask', 'in_mask')]), (tcompcor, mrg_compcor, [('high_variance_masks', 'in1')]), - (acc_msk, mrg_compcor, [('out', 'in2')]), + (acc_msk_bin, mrg_compcor, [(('out_file', _last), 'in2')]), (mrg_compcor, rois_plot, [('out', 'in_rois')]), (rois_plot, ds_report_bold_rois, [('out_report', 'in_file')]), (tcompcor, mrg_cc_metadata, [('metadata_file', 'in1')]), (acompcor, mrg_cc_metadata, [('metadata_file', 'in2')]), (mrg_cc_metadata, compcor_plot, [('out', 'metadata_files')]), (compcor_plot, ds_report_compcor, [('out_file', 'in_file')]), - (concat, conf_corr_plot, [('confounds_file', 'confounds_file')]), + (concat, conf_corr_plot, [('confounds_file', 'confounds_file'), + (('confounds_file', _select_cols), 'columns')]), (conf_corr_plot, ds_report_conf_corr, [('out_file', 'in_file')]), ]) @@ -548,7 +525,6 @@ def init_ica_aroma_wf( err_on_aroma_warn=False, name='ica_aroma_wf', susan_fwhm=6.0, - use_fieldwarp=True, ): """ Build a workflow that runs `ICA-AROMA`_. @@ -601,8 +577,6 @@ def init_ica_aroma_wf( susan_fwhm : :obj:`float` Kernel width (FWHM in mm) for the smoothing step with FSL ``susan`` (default: 6.0mm) - use_fieldwarp : :obj:`bool` - Include SDC warp in single-shot transform from BOLD to MNI err_on_aroma_warn : :obj:`bool` Do not fail on ICA-AROMA errors aroma_melodic_dim : :obj:`int` @@ -628,8 +602,6 @@ def init_ica_aroma_wf( BOLD series mask in template space hmc_xforms List of affine transforms aligning each volume to ``ref_image`` in ITK format - fieldwarp - a :abbr:`DFM (displacements field map)` in ITK format movpar_file SPM-formatted motion parameters file @@ -797,7 +769,6 @@ def _remove_volumes(bold_file, skip_vols): bold_img = nb.load(bold_file) bold_img.__class__(bold_img.dataobj[..., skip_vols:], bold_img.affine, bold_img.header).to_filename(out) - return out @@ -818,21 +789,9 @@ def _add_volumes(bold_file, bold_cut_file, skip_vols): out = fname_presuffix(bold_cut_file, suffix='_addnonsteady') bold_img.__class__(bold_data, bold_img.affine, bold_img.header).to_filename(out) - return out -def _maskroi(in_mask, roi_file): - import numpy as np +def _get_zooms(in_file): import nibabel as nb - from nipype.utils.filemanip import fname_presuffix - - roi = nb.load(roi_file) - roidata = roi.get_data().astype(np.uint8) - msk = nb.load(in_mask).get_data().astype(bool) - roidata[~msk] = 0 - roi.set_data_dtype(np.uint8) - - out = fname_presuffix(roi_file, suffix='_boldmsk') - roi.__class__(roidata, roi.affine, roi.header).to_filename(out) - return out + return tuple(nb.load(in_file).header.get_zooms()[:3]) diff --git a/fmriprep/workflows/bold/outputs.py b/fmriprep/workflows/bold/outputs.py index 7c85034b7..81e3685ca 100644 --- a/fmriprep/workflows/bold/outputs.py +++ b/fmriprep/workflows/bold/outputs.py @@ -4,8 +4,9 @@ from nipype.pipeline import engine as pe from nipype.interfaces import utility as niu -from ...config import DEFAULT_MEMORY_MIN_GB -from ...interfaces import DerivativesDataSink +from fmriprep import config +from fmriprep.config import DEFAULT_MEMORY_MIN_GB +from fmriprep.interfaces import DerivativesDataSink def init_func_derivatives_wf( @@ -61,22 +62,42 @@ def init_func_derivatives_wf( 'bold_std_ref', 'bold_t1', 'bold_t1_ref', 'bold_native', 'bold_native_ref', 'bold_mask_native', 'cifti_variant', 'cifti_metadata', 'cifti_density', 'confounds', 'confounds_metadata', 'melodic_mix', 'nonaggr_denoised_file', - 'source_file', 'surf_files', 'surf_refs', 'template', 'spatial_reference']), + 'source_file', 'surf_files', 'surf_refs', 'template', 'spatial_reference', + 'bold2anat_xfm', 'anat2bold_xfm', 'acompcor_masks', 'tcompcor_mask']), name='inputnode') raw_sources = pe.Node(niu.Function(function=_bids_relative), name='raw_sources') raw_sources.inputs.bids_root = bids_root ds_confounds = pe.Node(DerivativesDataSink( - base_directory=output_dir, desc='confounds', suffix='regressors', + base_directory=output_dir, desc='confounds', suffix='timeseries', dismiss_entities=("echo",)), name="ds_confounds", run_without_submitting=True, mem_gb=DEFAULT_MEMORY_MIN_GB) + ds_ref_t1w_xfm = pe.Node( + DerivativesDataSink(base_directory=output_dir, to='T1w', + mode='image', suffix='xfm', + extension='.txt', + dismiss_entities=('echo',), + **{'from': 'scanner'}), + name='ds_ref_t1w_xfm', run_without_submitting=True) + ds_ref_t1w_inv_xfm = pe.Node( + DerivativesDataSink(base_directory=output_dir, to='scanner', + mode='image', suffix='xfm', + extension='.txt', + dismiss_entities=('echo',), + **{'from': 'T1w'}), + name='ds_t1w_tpl_inv_xfm', run_without_submitting=True) + workflow.connect([ (inputnode, raw_sources, [('source_file', 'in_files')]), (inputnode, ds_confounds, [('source_file', 'source_file'), ('confounds', 'in_file'), ('confounds_metadata', 'meta_dict')]), + (inputnode, ds_ref_t1w_xfm, [('source_file', 'source_file'), + ('bold2anat_xfm', 'in_file')]), + (inputnode, ds_ref_t1w_inv_xfm, [('source_file', 'source_file'), + ('anat2bold_xfm', 'in_file')]), ]) # if nonstd_spaces.intersection(('func', 'run', 'bold', 'boldref', 'sbref')): @@ -122,7 +143,6 @@ def init_func_derivatives_wf( # compress=True, dismiss_entities=("echo",)), # name='ds_bold_t1_ref', run_without_submitting=True, # mem_gb=DEFAULT_MEMORY_MIN_GB) - # # ds_bold_mask_t1 = pe.Node( # DerivativesDataSink(base_directory=output_dir, space='T1w', desc='brain', # suffix='mask', compress=True, dismiss_entities=("echo",)), @@ -171,7 +191,7 @@ def init_func_derivatives_wf( # compress=True), # name='ds_aroma_std', run_without_submitting=True, # mem_gb=DEFAULT_MEMORY_MIN_GB) - # + # workflow.connect([ # (inputnode, ds_aroma_noise_ics, [('source_file', 'source_file'), # ('aroma_noise_ics', 'in_file')]), @@ -319,6 +339,23 @@ def init_func_derivatives_wf( (('cifti_metadata', _read_json), 'meta_dict')]) ]) + if "compcor" in config.execution.debug: + ds_acompcor_masks = pe.Node( + DerivativesDataSink( + base_directory=output_dir, desc=[f"CompCor{_}" for _ in "CWA"], + suffix="mask", compress=True), + name="ds_acompcor_masks", run_without_submitting=True) + ds_tcompcor_mask = pe.Node( + DerivativesDataSink( + base_directory=output_dir, desc="CompCorT", suffix="mask", compress=True), + name="ds_tcompcor_mask", run_without_submitting=True) + workflow.connect([ + (inputnode, ds_acompcor_masks, [("acompcor_masks", "in_file"), + ("source_file", "source_file")]), + (inputnode, ds_tcompcor_mask, [("tcompcor_mask", "in_file"), + ("source_file", "source_file")]), + ]) + return workflow diff --git a/fmriprep/workflows/bold/registration.py b/fmriprep/workflows/bold/registration.py index d972ca47d..fbf54de59 100644 --- a/fmriprep/workflows/bold/registration.py +++ b/fmriprep/workflows/bold/registration.py @@ -175,8 +175,8 @@ def _bold_reg_suffix(fallback, freesurfer): return workflow -def init_bold_t1_trans_wf(freesurfer, mem_gb, omp_nthreads, multiecho=False, use_fieldwarp=False, - use_compression=True, name='bold_t1_trans_wf'): +def init_bold_t1_trans_wf(freesurfer, mem_gb, omp_nthreads, use_compression=True, + name='bold_t1_trans_wf'): """ Co-register the reference BOLD image to T1w-space. @@ -196,10 +196,6 @@ def init_bold_t1_trans_wf(freesurfer, mem_gb, omp_nthreads, multiecho=False, use ---------- freesurfer : :obj:`bool` Enable FreeSurfer functional registration (bbregister) - use_fieldwarp : :obj:`bool` - Include SDC warp in single-shot transform from BOLD to T1 - multiecho : :obj:`bool` - If multiecho data was supplied, HMC already performed mem_gb : :obj:`float` Size of BOLD file in GB omp_nthreads : :obj:`int` @@ -327,38 +323,18 @@ def init_bold_t1_trans_wf(freesurfer, mem_gb, omp_nthreads, multiecho=False, use # Generate a reference on the target T1w space gen_final_ref = init_bold_reference_wf(omp_nthreads, pre_mask=True) - if not multiecho: - # Merge transforms placing the head motion correction last - nforms = 2 + int(use_fieldwarp) - merge_xforms = pe.Node(niu.Merge(nforms), name='merge_xforms', - run_without_submitting=True, mem_gb=DEFAULT_MEMORY_MIN_GB) - if use_fieldwarp: - workflow.connect([ - (inputnode, merge_xforms, [('fieldwarp', 'in2')]) - ]) - - workflow.connect([ - # merge transforms - (inputnode, merge_xforms, [ - ('hmc_xforms', 'in%d' % nforms), - ('itk_bold_to_t1', 'in1')]), - (merge_xforms, bold_to_t1w_transform, [('out', 'transforms')]), - (inputnode, bold_to_t1w_transform, [('bold_split', 'input_image')]), - ]) - - else: - from nipype.interfaces.fsl import Split as FSLSplit - bold_split = pe.Node(FSLSplit(dimension='t'), name='bold_split', - mem_gb=DEFAULT_MEMORY_MIN_GB) - - workflow.connect([ - (inputnode, bold_split, [('bold_split', 'in_file')]), - (bold_split, bold_to_t1w_transform, [('out_files', 'input_image')]), - (inputnode, bold_to_t1w_transform, [('itk_bold_to_t1', 'transforms')]), - ]) + # Merge transforms placing the head motion correction last + merge_xforms = pe.Node(niu.Merge(3), name='merge_xforms', + run_without_submitting=True, mem_gb=DEFAULT_MEMORY_MIN_GB) workflow.connect([ (inputnode, merge, [('name_source', 'header_source')]), + (inputnode, merge_xforms, [ + ('hmc_xforms', 'in3'), # May be 'identity' if HMC already applied + ('fieldwarp', 'in2'), # May be 'identity' if SDC already applied + ('itk_bold_to_t1', 'in1')]), + (inputnode, bold_to_t1w_transform, [('bold_split', 'input_image')]), + (merge_xforms, bold_to_t1w_transform, [('out', 'transforms')]), (gen_ref, bold_to_t1w_transform, [('out_file', 'reference_image')]), (bold_to_t1w_transform, merge, [('out_files', 'in_files')]), (merge, gen_final_ref, [('out_file', 'inputnode.bold_file')]), diff --git a/fmriprep/workflows/bold/resampling.py b/fmriprep/workflows/bold/resampling.py index b7a80ca13..3a81bbd94 100644 --- a/fmriprep/workflows/bold/resampling.py +++ b/fmriprep/workflows/bold/resampling.py @@ -13,7 +13,6 @@ from nipype.pipeline import engine as pe from nipype.interfaces import utility as niu, freesurfer as fs -from nipype.interfaces.fsl import Split as FSLSplit import nipype.interfaces.workbench as wb @@ -164,7 +163,6 @@ def init_bold_std_trans_wf( spaces, name='bold_std_trans_wf', use_compression=True, - use_fieldwarp=False, ): """ Sample fMRI into standard space with a single-step resampling of the original BOLD series. @@ -215,8 +213,6 @@ def init_bold_std_trans_wf( Name of workflow (default: ``bold_std_trans_wf``) use_compression : :obj:`bool` Save registered BOLD series as ``.nii.gz`` - use_fieldwarp : :obj:`bool` - Include SDC warp in single-shot transform from BOLD to MNI Inputs ------ @@ -335,13 +331,8 @@ def init_bold_std_trans_wf( mask_merge_tfms = pe.Node(niu.Merge(2), name='mask_merge_tfms', run_without_submitting=True, mem_gb=DEFAULT_MEMORY_MIN_GB) - nxforms = 3 + use_fieldwarp - merge_xforms = pe.Node(niu.Merge(nxforms), name='merge_xforms', + merge_xforms = pe.Node(niu.Merge(4), name='merge_xforms', run_without_submitting=True, mem_gb=DEFAULT_MEMORY_MIN_GB) - workflow.connect([(inputnode, merge_xforms, [('hmc_xforms', 'in%d' % nxforms)])]) - - if use_fieldwarp: - workflow.connect([(inputnode, merge_xforms, [('fieldwarp', 'in3')])]) bold_to_std_transform = pe.Node( MultiApplyTransforms(interpolation="LanczosWindowedSinc", float=True, copy_dtype=True), @@ -361,8 +352,9 @@ def init_bold_std_trans_wf( ('templates', 'keys')]), (inputnode, mask_std_tfm, [('bold_mask', 'input_image')]), (inputnode, gen_ref, [(('bold_split', _first), 'moving_image')]), - (inputnode, merge_xforms, [ - (('itk_bold_to_t1', _aslist), 'in2')]), + (inputnode, merge_xforms, [('hmc_xforms', 'in4'), + ('fieldwarp', 'in3'), + (('itk_bold_to_t1', _aslist), 'in2')]), (inputnode, merge, [('name_source', 'header_source')]), (inputnode, mask_merge_tfms, [(('itk_bold_to_t1', _aslist), 'in2')]), (inputnode, bold_to_std_transform, [('bold_split', 'input_image')]), @@ -431,7 +423,6 @@ def init_bold_preproc_trans_wf(mem_gb, omp_nthreads, name='bold_preproc_trans_wf', use_compression=True, use_fieldwarp=False, - split_file=False, interpolation='LanczosWindowedSinc'): """ Resample in native (original) space. @@ -459,9 +450,6 @@ def init_bold_preproc_trans_wf(mem_gb, omp_nthreads, Save registered BOLD series as ``.nii.gz`` use_fieldwarp : :obj:`bool` Include SDC warp in single-shot transform from BOLD to MNI - split_file : :obj:`bool` - Whether the input file should be splitted (it is a 4D file) - or it is a list of 3D files (default ``False``, do not split) interpolation : :obj:`str` Interpolation type to be used by ANTs' ``applyTransforms`` (default ``'LanczosWindowedSinc'``) @@ -484,16 +472,9 @@ def init_bold_preproc_trans_wf(mem_gb, omp_nthreads, ------- bold BOLD series, resampled in native space, including all preprocessing - bold_mask - BOLD series mask calculated with the new time-series - bold_ref - BOLD reference image: an average-like 3D image of the time-series - bold_ref_brain - Same as ``bold_ref``, but once the brain mask has been applied """ from niworkflows.engine.workflows import LiterateWorkflow as Workflow - from niworkflows.func.util import init_bold_reference_wf from niworkflows.interfaces.itk import MultiApplyTransforms from niworkflows.interfaces.nilearn import Merge @@ -518,59 +499,26 @@ def init_bold_preproc_trans_wf(mem_gb, omp_nthreads, niu.IdentityInterface(fields=['bold', 'bold_mask', 'bold_ref', 'bold_ref_brain']), name='outputnode') + merge_xforms = pe.Node(niu.Merge(2), name='merge_xforms', + run_without_submitting=True, mem_gb=DEFAULT_MEMORY_MIN_GB) + bold_transform = pe.Node( MultiApplyTransforms(interpolation=interpolation, float=True, copy_dtype=True), name='bold_transform', mem_gb=mem_gb * 3 * omp_nthreads, n_procs=omp_nthreads) - merge = pe.Node(Merge(compress=use_compression), name='merge', - mem_gb=mem_gb * 3) - - # Generate a new BOLD reference - bold_reference_wf = init_bold_reference_wf(omp_nthreads=omp_nthreads) - bold_reference_wf.__desc__ = None # Unset description to avoid second appearance + merge = pe.Node(Merge(compress=use_compression), name='merge', mem_gb=mem_gb * 3) workflow.connect([ + (inputnode, merge_xforms, [('fieldwarp', 'in1'), + ('hmc_xforms', 'in2')]), + (inputnode, bold_transform, [('bold_file', 'input_image'), + (('bold_file', _first), 'reference_image')]), (inputnode, merge, [('name_source', 'header_source')]), + (merge_xforms, bold_transform, [('out', 'transforms')]), (bold_transform, merge, [('out_files', 'in_files')]), - (merge, bold_reference_wf, [('out_file', 'inputnode.bold_file')]), (merge, outputnode, [('out_file', 'bold')]), - (bold_reference_wf, outputnode, [ - ('outputnode.ref_image', 'bold_ref'), - ('outputnode.ref_image_brain', 'bold_ref_brain'), - ('outputnode.bold_mask', 'bold_mask')]), ]) - # Input file is not splitted - if split_file: - bold_split = pe.Node(FSLSplit(dimension='t'), name='bold_split', - mem_gb=mem_gb * 3) - workflow.connect([ - (inputnode, bold_split, [('bold_file', 'in_file')]), - (bold_split, bold_transform, [ - ('out_files', 'input_image'), - (('out_files', _first), 'reference_image'), - ]) - ]) - else: - workflow.connect([ - (inputnode, bold_transform, [('bold_file', 'input_image'), - (('bold_file', _first), 'reference_image')]), - ]) - - if use_fieldwarp: - merge_xforms = pe.Node(niu.Merge(2), name='merge_xforms', - run_without_submitting=True, mem_gb=DEFAULT_MEMORY_MIN_GB) - workflow.connect([ - (inputnode, merge_xforms, [('fieldwarp', 'in1'), - ('hmc_xforms', 'in2')]), - (merge_xforms, bold_transform, [('out', 'transforms')]), - ]) - else: - def _aslist(val): - return [val] - workflow.connect([ - (inputnode, bold_transform, [(('hmc_xforms', _aslist), 'transforms')]), - ]) return workflow diff --git a/fmriprep/workflows/bold/t2s.py b/fmriprep/workflows/bold/t2s.py index fd62a830c..d9648aa85 100644 --- a/fmriprep/workflows/bold/t2s.py +++ b/fmriprep/workflows/bold/t2s.py @@ -36,7 +36,7 @@ def init_bold_t2s_wf(echo_times, mem_gb, omp_nthreads, Parameters ---------- - echo_times : :obj:`list` + echo_times : :obj:`list` or :obj:`tuple` list of TEs associated with each echo mem_gb : :obj:`float` Size of BOLD file in GB @@ -60,12 +60,12 @@ def init_bold_t2s_wf(echo_times, mem_gb, omp_nthreads, workflow = Workflow(name=name) workflow.__desc__ = """\ -A T2* map was estimated from the preprocessed BOLD by fitting to a monoexponential signal -decay model with nonlinear regression, using T2*/S0 estimates from a log-linear +A T2\\* map was estimated from the preprocessed BOLD by fitting to a monoexponential signal +decay model with nonlinear regression, using T2\\*/S0 estimates from a log-linear regression fit as initial values. For each voxel, the maximal number of echoes with reliable signal in that voxel were used to fit the model. -The calculated T2* map was then used to optimally combine preprocessed BOLD across +The calculated T2\\* map was then used to optimally combine preprocessed BOLD across echoes following the method described in [@posse_t2s]. The optimally combined time series was carried forward as the *preprocessed BOLD*. """ @@ -76,7 +76,7 @@ def init_bold_t2s_wf(echo_times, mem_gb, omp_nthreads, LOGGER.log(25, 'Generating T2* map and optimally combined ME-EPI time series.') - t2smap_node = pe.Node(T2SMap(echo_times=echo_times), name='t2smap_node') + t2smap_node = pe.Node(T2SMap(echo_times=list(echo_times)), name='t2smap_node') workflow.connect([ (inputnode, t2smap_node, [('bold_file', 'in_files')]), diff --git a/fmriprep/workflows/bold/tests/test_confounds.py b/fmriprep/workflows/bold/tests/test_confounds.py index a3f6b8074..64ea00f26 100644 --- a/fmriprep/workflows/bold/tests/test_confounds.py +++ b/fmriprep/workflows/bold/tests/test_confounds.py @@ -6,9 +6,11 @@ from ..confounds import _add_volumes, _remove_volumes -skip_pytest = pytest.mark.skipif(not os.getenv('FMRIPREP_REGRESSION_SOURCE') or - not os.getenv('FMRIPREP_REGRESSION_TARGETS'), - reason='FMRIPREP_REGRESSION_{SOURCE,TARGETS} env vars not set') +skip_pytest = pytest.mark.skipif( + not os.getenv('FMRIPREP_REGRESSION_SOURCE') + or not os.getenv('FMRIPREP_REGRESSION_TARGETS'), + reason='FMRIPREP_REGRESSION_{SOURCE,TARGETS} env vars not set' +) @skip_pytest diff --git a/setup.cfg b/setup.cfg index 3dafa35d0..887b84ad4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,17 +24,18 @@ python_requires = >=3.7 install_requires = indexed_gzip >= 0.8.8 nibabel >= 3.0 - nipype >= 1.4 + nipype >= 1.5.1 nitime - nitransforms >= 20.0.0rc3,<20.2 - niworkflows ~= 1.2.5 + nitransforms >= 20.0.0rc3, < 20.2 + niworkflows >= 1.3.0rc3, < 1.4 numpy pandas psutil >= 5.4 pybids >= 0.10.2 pyyaml - sdcflows ~= 1.3.1 - smriprep ~= 0.6.1 + requests + sdcflows >= 1.3.2, < 1.5 + smriprep >= 0.7.0rc1, < 0.8 tedana >= 0.0.9a1, < 0.0.10 templateflow ~= 0.6 toml @@ -54,7 +55,7 @@ doc = packaging pydot >= 1.2.3 pydotplus - sphinx >= 1.5.3, < 3 + sphinx >= 1.8 sphinx-argparse sphinx_rtd_theme sphinxcontrib-napoleon @@ -105,3 +106,18 @@ parentdir_prefix = max-line-length = 99 doctests = True exclude=*build/ +ignore = + W503 +per-file-ignores = + **/__init__.py : F401 + docs/conf.py : E265 + +[tool:pytest] +norecursedirs = .git +addopts = -svx --doctest-modules +doctest_optionflags = ALLOW_UNICODE NORMALIZE_WHITESPACE ELLIPSIS +env = + PYTHONHASHSEED=0 +filterwarnings = + ignore::DeprecationWarning +junit_family=xunit2 diff --git a/wrapper/fmriprep_docker.py b/wrapper/fmriprep_docker.py index ee6b712d4..3958ccf35 100755 --- a/wrapper/fmriprep_docker.py +++ b/wrapper/fmriprep_docker.py @@ -166,8 +166,10 @@ def merge_help(wrapper_help, target_help): overlap = set(w_flags).intersection(t_flags) expected_overlap = { 'anat-derivatives', + 'bids-database-dir', 'fs-license-file', 'fs-subjects-dir', + 'config-file', 'h', 'use-plugin', 'version', @@ -237,6 +239,7 @@ def __call__(self, parser, namespace, values, option_string=None): def _is_file(path, parser): """Ensure a given path exists and it is a file.""" + path = os.path.abspath(path) if not os.path.isfile(path): raise parser.error( "Path should point to a file (or symlink of file): <%s>." % path @@ -298,6 +301,10 @@ def _is_file(path, parser): '--fs-subjects-dir', metavar='PATH', type=os.path.abspath, help='Path to existing FreeSurfer subjects directory to reuse. ' '(default: OUTPUT_DIR/freesurfer)') + g_wrap.add_argument( + '--config-file', metavar='PATH', type=os.path.abspath, + help="Use pre-generated configuration file. Values in file will be overridden " + "by command-line arguments.") g_wrap.add_argument( '--anat-derivatives', metavar='PATH', type=os.path.abspath, help='Path to existing sMRIPrep/fMRIPrep-anatomical derivatives to fasttrack ' @@ -305,6 +312,10 @@ def _is_file(path, parser): g_wrap.add_argument( '--use-plugin', metavar='PATH', action='store', default=None, type=os.path.abspath, help='nipype plugin configuration file') + g_wrap.add_argument( + '--bids-database-dir', metavar='PATH', type=os.path.abspath, + help="Path to an existing PyBIDS database folder, for faster indexing " + "(especially useful for large datasets).") # Developer patch/shell options g_dev = parser.add_argument_group( @@ -324,6 +335,8 @@ def _is_file(path, parser): g_dev.add_argument('--network', action='store', help='Run container with a different network driver ' '("none" to simulate no internet connection)') + g_dev.add_argument('--no-tty', action='store_true', + help='Run docker without TTY flag -it') return parser @@ -392,9 +405,12 @@ def main(): stdout=subprocess.PIPE) docker_version = ret.stdout.decode('ascii').strip() - command = ['docker', 'run', '--rm', '-it', '-e', + command = ['docker', 'run', '--rm', '-e', 'DOCKER_VERSION_8395080871=%s' % docker_version] + if not opts.no_tty: + command.append('-it') + # Patch working repositories into installed package directories if opts.patch: for pkg, repo_path in opts.patch.items(): @@ -430,6 +446,10 @@ def main(): command.extend(['-v', '{}:/opt/subjects'.format(opts.fs_subjects_dir)]) unknown_args.extend(['--fs-subjects-dir', '/opt/subjects']) + if opts.config_file: + command.extend(['-v', '{}:/tmp/config.toml'.format(opts.config_file)]) + unknown_args.extend(['--config-file', '/tmp/config.toml']) + if opts.anat_derivatives: command.extend(['-v', '{}:/opt/smriprep/subjects'.format(opts.anat_derivatives)]) unknown_args.extend(['--anat-derivatives', '/opt/smriprep/subjects']) @@ -455,6 +475,10 @@ def main(): 'ro'))]) unknown_args.extend(['--use-plugin', '/tmp/plugin.yml']) + if opts.bids_database_dir: + command.extend(['-v', ':'.join((opts.bids_database_dir, '/tmp/bids_db'))]) + unknown_args.extend(['--bids-database-dir', '/tmp/bids_db']) + if opts.output_spaces: spaces = [] for space in opts.output_spaces: