👔 Restructure automated integration and regression testing flow

.github/CONTRIBUTING.md

@@ -0,0 +1,34 @@
+# Contributing to "Regression Testing C-PAC on SLURM Clusters"
+
+## Forking this repository
+
+If you're making any changes to secrets or environment variables, please test in a fork of this repository. [`pre-commit`](https://github.com/FCP-INDI/C-PAC/blob/main/CONTRIBUTING.md#pre-commit) is [configured in this repository](https://github.com/childmindresearch/slurm_testing/blob/977061120765151334a2e9d3257102d4a368d791/.pre-commit-config.yaml#L81-L87) to update the [`repository` URL in `pyproject.toml`](https://github.com/childmindresearch/slurm_testing/blob/977061120765151334a2e9d3257102d4a368d791/pyproject.toml#L9) to match the  URL for the local `git remote` named `origin`, and [a GitHub Actions workflow is configured](https://github.com/childmindresearch/slurm_testing/blob/977061120765151334a2e9d3257102d4a368d791/.github/workflows/match_target_repository.yaml) to match that URL to the target repository when a pull request is opened, reopened, or synced.
+
+```mermaid
+flowchart TD
+
+fork[fork repository] --> clone[<code>git clone</code>]
+clone --> changes["{make changes in fork}"]
+changes --> commit
+
+subgraph commit[<code>git commit</code>]
+  subgraph pre-commit
+    update_fork[change <code>repository</code> URL in <code>pyproject.toml</code> to <code>git remote origin</code> URL]
+  end
+end
+
+commit --> push[<code>git push</code>]
+push --> PR
+
+subgraph PR[pull request]
+  subgraph workflow["'Update Repository URL' workflow"]
+    update_PR[change <code>repository</code> URL in <code>pyproject.toml</code> to target repository URL]
+  end
+end
+```
+
+To skip the local update, you can run `SKIP=local-fork-repo-url git commit` where you would normally run `git commit`.
+
+## See Also
+
+Please see [the C-PAC Contributing Guide](https://github.com/FCP-INDI/C-PAC/blob/main/CONTRIBUTING.md), the FCP-INDI [Contributing Guide](https://github.com/FCP-INDI/.github/blob/main/CONTRIBUTING.md) and [Contributor Covenant Code of Conduct](https://github.com/FCP-INDI/.github/blob/main/CODE_OF_CONDUCT.md) for more general contributing guidelines.

.github/scripts/correlate_regtest_lite.SLURM

@@ -2,27 +2,39 @@
 #SBATCH -N 1
 #SBATCH -p RM-shared
 #SBATCH -t 00:05:00
-#SBATCH --ntasks-per-node=1
+#SBATCH --ntasks=1
 
-# Required environment variables: $COMPARISON_PATH, $DATA, $HOME_DIR, $OWNER, $PATH, $PIPELINE, $PUSH_LOGS, $REPO, $SHA
+# Required environment variables: $COMPARISON_PATH, $DATA, $HOME_DIR, $OWNER, $PATH, $PIPELINE, $PUSH_LOGS, $REPO, $SHA, $TOKEN_FILE
 
-if [[ $PUSH_LOGS==true ]]
+set -x
+
+# shellcheck disable=SC1090
+source "${TOKEN_FILE}"
+
+set -x
+
+if [[ $PUSH_LOGS == true ]]
 then
     export _WD=$PWD
     export GIT_REPO_DIR="${HOME_DIR}/logs/${SHA}"
-    cd "${GIT_REPO_DIR}"
-    source "${HOME_DIR}/slurm_testing/.github/scripts/wait_for_git_lock.sh"
+    cd "${GIT_REPO_DIR}" || exit 1
+    # shellcheck disable=SC1090
+    source "${HOME_DIR}/C-PAC_slurm_testing/.github/scripts/wait_for_git_lock.sh"
     wait_for_git_lock
-    git add slurm-${PIPELINE}-${DATA}
+    git add "slurm-${PIPELINE}-${DATA}"
     wait_for_git_lock
     git commit -m "Add logs for ${PIPELINE} ${DATA} ${SHA}"
-    cd $_WD
+    cd "$_WD" || exit 1
 fi
-
+export _CPAC_STATUS_DATA_SOURCE="${DATA}"
+export _CPAC_STATUS_PRECONFIG="${PIPELINE}"
+export _CPAC_STATUS_SUBJECT="${SUBJECT}"
+export _CPAC_STATUS_STATE="${STATE}"
+# "${HOME_DIR}/C-PAC_slurm_testing/.github/scripts/status.py"
 # gh workflow run 'Correlate Regression Test' -F ref="$SHA" -F pipeline1='one' -F pipeline2='two' -F repo="$REPO" -F owner="$OWNER" -F preconfig="$PIPELINE" -F data_source="$DATA"
 
-gh workflow run "Initiate check" -F task="correlate" -F ref="$SHA" -F repo="$REPO" -F owner="$OWNER" -F job="${PIPELINE}-${DATA}-${IMAGE_NAME}" -F preconfig="$PIPELINE" -F data_source="$DATA" || echo "Correlate ${PIPELINE}-${DATA}-${IMAGE_NAME} initiated"
+# gh workflow run "Initiate check" -F task="correlate" -F ref="$SHA" -F repo="$REPO" -F owner="$OWNER" -F job="${PIPELINE}-${DATA}-${IMAGE_NAME}" -F preconfig="$PIPELINE" -F data_source="$DATA" || echo "Correlate ${PIPELINE}-${DATA}-${IMAGE_NAME} initiated"
 
 export GITHUB_WORKSPACE="${HOME_DIR}/${IMAGE_NAME}"
 
-cpac_regsuite_create_yaml --pipeline1 "${COMPARISON_PATH}/${PIPELINE}/${DATA}" --pipeline2 "${GITHUB_WORKSPACE}/lite/${PIPELINE}/${DATA}" --workspace $GITHUB_WORKSPACE --data_source "${DATA}"
+cpac_regsuite_create_yaml --pipeline1 "${COMPARISON_PATH}/${PIPELINE}/${DATA}" --pipeline2 "${GITHUB_WORKSPACE}/lite/${PIPELINE}/${DATA}" --workspace "${GITHUB_WORKSPACE}" --data_source "${DATA}" --branch="${SHA}"

.github/scripts/failed_regtest_lite.SLURM

@@ -2,21 +2,27 @@
 #SBATCH -N 1
 #SBATCH -p RM-shared
 #SBATCH -t 00:05:00
-#SBATCH --ntasks-per-node=1
+#SBATCH --ntasks=1
 
 # Required environment variables: $DATA, $HOME_DIR, $OWNER, $PATH, $PIPELINE, $PUSH_LOGS, $REPO, $SHA
 
-if [[ $PUSH_LOGS==true ]]
+if [[ $PUSH_LOGS == true ]]
 then
+    # shellcheck disable=SC1090
+    source "${TOKEN_FILE}"
     export _WD=$PWD
     export GIT_REPO_DIR="${HOME_DIR}/logs/${SHA}"
-    cd "${GIT_REPO_DIR}"
-    source "${HOME_DIR}/slurm_testing/.github/scripts/wait_for_git_lock.sh"
+    cd "${GIT_REPO_DIR}" || exit 1
+    # shellcheck disable=SC1090
+    source "${HOME_DIR}/C-PAC_slurm_testing/.github/scripts/wait_for_git_lock.sh"
     wait_for_git_lock
-    git add slurm-${PIPELINE}-${DATA}
+    git add "slurm-${PIPELINE}-${DATA}"
     wait_for_git_lock
     git commit -m "Add logs for ${PIPELINE} ${DATA} ${SHA}"
-    cd $_WD
+    cd "$_WD" || exit 1
 fi
-
-gh workflow run "Test run failed" -F ref="$SHA" -F owner="$OWNER" -F repo="$REPO" -F preconfig="$PIPELINE" -F data_source="$DATA"
+export _CPAC_STATUS_DATA_SOURCE="${DATA}"
+export _CPAC_STATUS_PRECONFIG="${PIPELINE}"
+export _CPAC_STATUS_SUBJECT="${SUBJECT}"
+export _CPAC_STATUS_STATE=failed
+# "${HOME_DIR}/C-PAC_slurm_testing/.github/scripts/status.py"

.github/scripts/local_fork_repo_url.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+"""Update local fork's ``repository`` URL."""
+from pathlib import Path
+import re
+
+from git import Repo
+from git.exc import GitCommandError
+
+
+def get_git_remote_origin(repo_dir: Path) -> str:
+    """Get Git remote origin."""
+    try:
+        configured_url = Repo(repo_dir).remotes.origin.url
+        _prefix, suffix = configured_url.split("github.com")
+        return f"https://github.com/{suffix[1:-4]}"
+    except AttributeError as e:
+        if "origin" in str(e):
+            raise GitCommandError(["git", "remote", "get-url", "origin"]) from e
+        raise e
+
+
+def get_repo_dir() -> Path:
+    """Get path to repository directory."""
+    return Path(__file__).parents[2]
+
+
+def main() -> None:
+    """Replace ``repository`` in ``pyproject.toml`` with forked repo."""
+    repo_dir = get_repo_dir()
+    forked_repo_url = get_git_remote_origin(repo_dir)
+    pyproject = repo_dir / "pyproject.toml"
+    repo_pattern = re.compile(r'^repository = ".*"$')
+
+    lines: list[str] = []
+    changed = False
+    with pyproject.open("r", encoding="utf-8") as _pyproject:
+        for line in _pyproject.readlines():
+            if repo_pattern.match(line):
+                newline = f'repository = "{forked_repo_url}"\n'
+                if newline != line:
+                    changed = True
+                    lines.append(newline)
+            else:
+                lines.append(line)
+    if changed:
+        with pyproject.open("w", encoding="utf-8") as _pyproject:
+            _pyproject.write("".join(lines))
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/push_to_github.SLURM

@@ -3,11 +3,12 @@
 #SBATCH -N 1
 #SBATCH -p RM-shared
 #SBATCH -t 00:05:00
-#SBATCH --ntasks-per-node=1
+#SBATCH --ntasks=1
 
 export GIT_REPO_DIR="${HOME_DIR}/logs/${SHA}"
 cd "${GIT_REPO_DIR}"
-source "${HOME_DIR}/slurm_testing/.github/scripts/wait_for_git_lock.sh"
+# shellcheck disable=SC1090
+source "${HOME_DIR}/C-PAC_slurm_testing/.github/scripts/wait_for_git_lock.sh"
 wait_for_git_lock
 git push origin ${REPO}_${SHA} -f
 cd $_WD

.github/scripts/status_regtest_lite.SLURM

@@ -3,19 +3,20 @@
 #SBATCH -N 1
 #SBATCH -p RM-shared
 #SBATCH -t 00:05:00
-#SBATCH --ntasks-per-node=1
+#SBATCH --ntasks=1
 
-if [[ $PUSH_LOGS==true ]]
+if [[ $PUSH_LOGS == true ]]
 then
-    export _WD=$PWD
+    export _WD="$PWD"
     export GIT_REPO_DIR="${HOME_DIR}/logs/${SHA}"
-    cd "${GIT_REPO_DIR}"
-    source "${HOME_DIR}/slurm_testing/.github/scripts/wait_for_git_lock.sh"
+    cd "${GIT_REPO_DIR}" || exit 1
+    # shellcheck disable=SC1090
+    source "${HOME_DIR}/C-PAC_slurm_testing/.github/scripts/wait_for_git_lock.sh"
     wait_for_git_lock
-    git add launch/error.log launch/out.log
+    git add launch/*.err.log launch/*.out.log
     wait_for_git_lock
     git commit -m "Add logs for ${PIPELINE} ${DATA} ${SHA}"
-    cd $_WD
+    cd "$_WD" || exit 1
 fi
 
 gh workflow run 'Run launch status update' -F ref="$SHA" -F repo="$REPO" -F owner="$OWNER" -F status="${STATUS}"

.github/workflows/initiate_check.yml → .github/workflows/initiate_check.yaml

@@ -45,7 +45,7 @@ jobs:
     name: Running ${{ inputs.job }} on ${{ inputs.environment }}
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3.5.3
+        uses: actions/checkout@v4.2.2
       - name: Initiate check
         uses: guibranco/github-status-action-v2@v1.1.7
         with: