✨ Correlation heatmaps

.circleci/config.yml

@@ -0,0 +1,29 @@
+version: 2
+jobs:
+  pytest:
+    docker:
+      - image: circleci/python:3.7.3
+    steps:
+      - checkout
+      - run:
+          name: pytest
+          command: |
+            export PATH=/home/circleci/.local/bin:$PATH
+            mkdir test-results
+            pip install --user -r requirements.txt
+            set +e
+            coverage run -m pytest --junitxml=test-results/junit.xml
+            coverage report --include="./*" --omit="/home/circleci/.local/*"
+            coverage html --include="./*" --omit="/home/circleci/.local/*"
+      - store_test_results:
+          path: test-results
+      - store_artifacts:
+          path: test-results
+      - store_artifacts:
+          path: htmlcov
+
+workflows:
+  version: 2
+  unit_tests:
+    jobs:
+      - pytest

README.md

@@ -1 +1,90 @@
 # CPAC_regtest_pack
+
+```bash
+$ python cpac_correlations_wf.py --help
+usage: cpac_correlations_wf.py [-h] [--old_outputs_path OLD_OUTPUTS_PATH]
+                               [--new_outputs_path NEW_OUTPUTS_PATH]
+                               [--s3_creds S3_CREDS]
+                               [--replacements REPLACEMENTS]
+                               [--corr_map CORR_MAP]
+                               [--working_dir WORKING_DIR]
+                               num_cores run_name
+
+positional arguments:
+  num_cores             number of cores to use - will calculate correlations
+                        in parallel if greater than 1
+  run_name              name for the correlations run
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --old_outputs_path OLD_OUTPUTS_PATH
+                        path to a CPAC outputs directory - the folder
+                        containing the participant-ID labeled directories
+  --new_outputs_path NEW_OUTPUTS_PATH
+                        path to a CPAC outputs directory - the folder
+                        containing the participant-ID labeled directories
+  --s3_creds S3_CREDS   path to your AWS S3 credentials file
+  --replacements REPLACEMENTS
+                        text file containing strings you wish to have removed
+                        from the filepaths if they occur - place one on each
+                        line
+  --corr_map CORR_MAP   YAML file with already-calculated correlations, which
+                        can be provided if you only want to generate the box
+                        plots again
+  --working_dir WORKING_DIR
+                        if you are correlating two working directories of a
+                        single participant to check intermediates
+```
+
+```bash
+$ python correlation_matrix.py --help
+usage: correlation_matrix.py [-h] [--old_outputs_path OLD_OUTPUTS_PATH]
+                             [--old_outputs_software {C-PAC,fmriprep}]
+                             [--new_outputs_path NEW_OUTPUTS_PATH]
+                             [--new_outputs_software {C-PAC,fmriprep}]
+                             [--save] [--no-save]
+                             [--subject_list SUBJECT_LIST] [--session SESSION]
+                             [--feature_list FEATURE_LIST]
+                             num_cores run_name
+
+Create a correlation matrix between two C-PAC output directories.
+
+positional arguments:
+  num_cores             number of cores to use - will calculate correlations
+                        in parallel if greater than 1
+  run_name              name for the correlations run
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --old_outputs_path OLD_OUTPUTS_PATH
+                        path to an outputs directory - the folder containing
+                        the participant-ID labeled directories
+  --old_outputs_software {C-PAC,fmriprep}
+                        (default: fmriprep)
+  --new_outputs_path NEW_OUTPUTS_PATH
+                        path to an outputs directory - the folder containing
+                        the participant-ID labeled directories
+  --new_outputs_software {C-PAC,fmriprep}
+                        (default: C-PAC)
+  --save                save matrices & heatmap (default)
+  --no-save             do not save matrices & heatmap
+  --subject_list SUBJECT_LIST
+                        (default: subjects in OLD_OUTPUTS_PATH sorted by
+                        session, subject ID). TODO: handle path to file
+  --session SESSION     limit to a single given session (integer)
+  --feature_list FEATURE_LIST
+                        TODO: handle path to file (default: ['GS', 'CSF',
+                        'WM', 'tCompCor0', 'aCompCor0', 'aCompCor1',
+                        'aCompCor2', 'aCompCor3', 'aCompCor4', 'FD'])
+
+The following features currently have available definitions to calculate Pearson's r between C-PAC and fmriprep:
+
+key       feature name              documentation link
+--------  ------------------------  ----------------------------------------------------------------------------------
+aCompCor  aCompCor                  https://fcp-indi.github.io/docs/user/nuisance.html#acompcor
+CSF       mean cerebrospinal fluid  https://fcp-indi.github.io/docs/user/nuisance.html#mean-white-matter-csf
+FD        framewise displacement    https://fcp-indi.github.io/docs/user/nuisance.html#regression-of-motion-parameters
+GS        global signal regression  https://fcp-indi.github.io/docs/user/nuisance.html#global-signal-regression
+tCompCor  tCompCor                  https://fcp-indi.github.io/docs/user/nuisance.html#tcompcor
+WM        mean white matter         https://fcp-indi.github.io/docs/user/nuisance.html#mean-white-matter-csf
+```

configs/defaults.py

@@ -0,0 +1,53 @@
+feature_headers = {
+    'GS': {
+        'name': 'global signal regression',
+        'link': 'https://fcp-indi.github.io/docs/user/nuisance.html#'
+                'global-signal-regression',
+        'C-PAC': ['GlobalSignalMean0', 'GlobalSignal_mean'],
+        'fmriprep': 'global_signal'
+    },
+    'CSF': {
+        'name': 'mean cerebrospinal fluid',
+        'link': 'https://fcp-indi.github.io/docs/user/nuisance.html#'
+                'mean-white-matter-csf',
+        'C-PAC': ['CerebrospinalFluidMean0', 'CerebrospinalFluid_mean'],
+        'fmriprep': 'csf'
+    },
+    'WM': {
+        'name': 'mean white matter',
+        'link': 'https://fcp-indi.github.io/docs/user/nuisance.html#'
+                'mean-white-matter-csf',
+        'C-PAC': ['WhiteMatterMean0', 'WhiteMatter_mean'],
+        'fmriprep': 'white_matter'
+    },
+    'aCompCor': {
+        'name': 'aCompCor',
+        'link': 'https://fcp-indi.github.io/docs/user/nuisance.html#acompcor',
+        'C-PAC': ['aCompCorPC', 'aCompCor'],
+        'fmriprep': 'aCompCor_comp_cor_0'
+    },
+    'tCompCor': {
+        'name': 'tCompCor',
+        'link': 'https://fcp-indi.github.io/docs/user/nuisance.html#tcompcor',
+        'C-PAC': ['tCompCorPC', 'tCompCor'],
+        'fmriprep': 'tCompCor_comp_cor_0'
+    },
+    'FD': {
+        'name': 'framewise displacement',
+        'link': 'https://fcp-indi.github.io/docs/user/nuisance.html#'
+                'regression-of-motion-parameters'
+    }
+}
+motion_list = ['FD']
+regressor_list = [
+    'GS',
+    'CSF',
+    'WM',
+    'tCompCor0',
+    'aCompCor0',
+    'aCompCor1',
+    'aCompCor2',
+    'aCompCor3',
+    'aCompCor4'
+]
+software = ["C-PAC", "fmriprep"]

configs/subjects.py

@@ -0,0 +1,162 @@
+import os
+
+from itertools import chain
+from string import ascii_lowercase
+
+
+def cpac_sub(sub):
+    """
+    Function to convert a string from f"sub-{sub_number}{ses_letter}" to
+    f"sub-{sub_number}_ses-{ses_number}"
+
+    Parameter
+    ---------
+    fmriprep_sub: str
+
+    Returns
+    -------
+    sub: str
+
+    Example
+    -------
+    >>> print(cpac_sub("sub-0025427a"))
+    sub-0025427_ses-1
+    """
+    return(f"{sub[:-1]}_ses-{str(ascii_lowercase.find(sub[-1])+1)}")
+
+
+def fmriprep_sub(sub):
+    """
+    Function to convert a string from f"sub-{sub_number}_ses-{ses_number}" to
+    f"sub-{sub_number}{ses_letter}"
+
+    Parameter
+    ---------
+    sub: str
+
+    Returns
+    -------
+    fmriprep_sub: str
+
+    Example
+    -------
+    >>> print(fmriprep_sub("sub-0025427_ses-1"))
+    sub-0025427a
+    """
+    return(f"{sub.split('_')[0]}{ascii_lowercase[int(sub[-1])-1]}")
+
+
+def generate_subject_list_for_directory(path, old_outputs_software="C-PAC"):
+    """
+    Function to take a path and return a subject list.
+
+    Parameter
+    ---------
+    path: str
+
+    old_outputs_software: str, optional, default="C-PAC"
+
+    Returns
+    -------
+    sub_list: list
+    """
+    output = os.path.join(path, "output")
+    sub_ses_list = list(chain.from_iterable([[
+        d for d in os.listdir(
+            os.path.join(output, o)
+        ) if all([
+            os.path.isdir(os.path.join(output, o, d)),
+            d not in ["log", "logs"]
+        ])
+    ] for o in os.listdir(output)]))
+    return(sessions_together([
+        cpac_sub(s) if s[
+            -1
+        ] in ascii_lowercase else s for s in sub_ses_list
+    ]))
+
+
+def generate_subject_list_for_range(
+    subject_start_stop,
+    session_start_stop=None
+):
+    """
+    Function to create a subject list for a given range. All values are
+    inclusive.
+
+    Parameters
+    ----------
+    subject_start_stop: 2-tuple of integers (start, stop) or list of specific
+    values
+
+    session_start_stop: 2-tuple of integers (start, stop) or list of specific
+    values or None
+
+    Returns
+    -------
+    List of strings
+
+    Example
+    -------
+    >>> generate_subject_list_for_range((25427,25428), (1,2))
+    ['sub-0025427_ses-1', 'sub-0025428_ses-1', 'sub-0025427_ses-2', 'sub-0025428_ses-2']
+    """
+    return([
+        f'sub-00{sub}{ses_string}' for ses_string in ([
+            f'_ses-{ses}' for ses in _expand_range(
+                session_start_stop
+            )
+        ] if session_start_stop else [
+            ''
+        ]) for sub in _expand_range(subject_start_stop)
+    ])
+
+
+def sessions_together(sub_list):
+    """
+    Function to sort by session then by subject
+
+    Parameter
+    ---------
+    sub_list: list of str
+
+    Returns
+    -------
+    sub_list: list of str
+
+    Example
+    -------
+    >>> sub_list = [
+    ...    'sub-0025427_ses-1', 'sub-0025427_ses-2', 'sub-0025428_ses-1'
+    ... ]
+    >>> print(sessions_together(sub_list))
+    ['sub-0025427_ses-1', 'sub-0025428_ses-1', 'sub-0025427_ses-2']
+    """
+    sub_list.sort()
+    sub_list.sort(key=lambda x: x.split("ses-")[-1])
+    return(sub_list)
+
+
+def _expand_range(tuple_or_list):
+    """
+    Function to expand an inclusive tuple to a range or return a literal list
+
+    Parameter
+    ---------
+    tuple_or_list: 2-tuple of integers or list
+
+    Returns
+    -------
+    list
+    """
+    return(
+        list(
+            range(
+                tuple_or_list[0],
+                tuple_or_list[1] + 1) if all([
+                isinstance(tuple_or_list, tuple),
+                len(tuple_or_list)==2,
+                *[isinstance(v, int) for v in tuple_or_list]
+            ]) else tuple_or_list
+        )
+    )