Merge pull request #1 from shnizzedy/regression/after_runs

👷 Launch runs for regression tests from CI (1/2)
childmindresearch · Oct 10, 2023 · 3e719f9 · 3e719f9
2 parents befc45e + 2d8040c
commit 3e719f9
Show file tree

Hide file tree

Showing 22 changed files with 673 additions and 119 deletions.
diff --git a/.github/scripts/correlate_regtest_lite.SLURM b/.github/scripts/correlate_regtest_lite.SLURM
@@ -0,0 +1,28 @@
+#!/bin/bash
+#SBATCH -N 1
+#SBATCH -p RM-shared
+#SBATCH -t 00:05:00
+#SBATCH --ntasks-per-node=1
+
+# Required environment variables: $DATA, $HOME_DIR, $OWNER, $PATH, $PIPELINE, $PUSH_LOGS, $REPO, $SHA
+
+if [[ $PUSH_LOGS==true ]]
+then
+    export _WD=$PWD
+    export GIT_REPO_DIR="${HOME_DIR}/logs/${SHA}"
+    cd "${GIT_REPO_DIR}"
+    source "${HOME_DIR}/slurm_testing/.github/scripts/wait_for_git_lock.sh"
+    wait_for_git_lock
+    git add slurm-${PIPELINE}-${DATA}
+    wait_for_git_lock
+    git commit -m "Add logs for ${PIPELINE} ${DATA} ${SHA}"
+    cd $_WD
+fi
+
+# gh workflow run 'Correlate Regression Test' -F ref="$SHA" -F pipeline1='one' -F pipeline2='two' -F repo="$REPO" -F owner="$OWNER" -F preconfig="$PIPELINE" -F data_source="$DATA"
+
+gh workflow run "Initiate check" -F task="correlate" -F ref="$SHA" -F repo="$REPO" -F owner="$OWNER" -F job="${PIPELINE}-${DATA}-${IMAGE_NAME}" -F preconfig="$PIPELINE" -F data_source="$DATA" || echo "Correlate ${PIPELINE}-${DATA}-${IMAGE_NAME} initiated"
+
+export GITHUB_WORKSPACE="${HOME_DIR}/${IMAGE_NAME}"
+
+python3 reg-suite/main.py correlate --pipeline1 ${GITHUB_WORKSPACE}/${PIPELINE}/${DATA} --pipeline2 ${GITHUB_WORKSPACE}/lite/${PIPELINE}/${DATA} --workspace $GITHUB_WORKSPACE
diff --git a/.github/scripts/failed_regtest_lite.SLURM b/.github/scripts/failed_regtest_lite.SLURM
@@ -0,0 +1,22 @@
+#!/bin/bash
+#SBATCH -N 1
+#SBATCH -p RM-shared
+#SBATCH -t 00:05:00
+#SBATCH --ntasks-per-node=1
+
+# Required environment variables: $DATA, $HOME_DIR, $OWNER, $PATH, $PIPELINE, $PUSH_LOGS, $REPO, $SHA
+
+if [[ $PUSH_LOGS==true ]]
+then
+    export _WD=$PWD
+    export GIT_REPO_DIR="${HOME_DIR}/logs/${SHA}"
+    cd "${GIT_REPO_DIR}"
+    source "${HOME_DIR}/slurm_testing/.github/scripts/wait_for_git_lock.sh"
+    wait_for_git_lock
+    git add slurm-${PIPELINE}-${DATA}
+    wait_for_git_lock
+    git commit -m "Add logs for ${PIPELINE} ${DATA} ${SHA}"
+    cd $_WD
+fi
+
+gh workflow run "Test run failed" -F ref="$SHA" -F owner="$OWNER" -F repo="$REPO" -F preconfig="$PIPELINE" -F data_source="$DATA"
diff --git a/.github/scripts/launch_regtest_lite.SLURM b/.github/scripts/launch_regtest_lite.SLURM
@@ -0,0 +1,40 @@
+#!/bin/bash
+# Required environment variables: 
+#SBATCH -N 1
+#SBATCH -p RM-shared
+#SBATCH -t 00:05:00
+#SBATCH --ntasks-per-node=4
+
+export PUSH_LOGS=true
+export _WD=$PWD
+mkdir -p "${HOME_DIR}/logs/${SHA}/launch"
+cd "${HOME_DIR}/logs/${SHA}" || PUSH_LOGS=false
+if [[ $PUSH_LOGS==true ]]
+then
+  # set up logging repo
+  git init
+  git remote add origin git@github.com:${OWNER}/regtest-runlogs.git
+  git switch -c ${REPO}_${SHA} || git checkout -b ${REPO}_${SHA}
+  cd $_WD
+fi
+PATH=$PATH:$PATH_EXTRA
+GH_AVAILABLE=true
+cd "${HOME_DIR}/slurm_testing" || exit 1
+gh repo set-default "$OWNER/slurm_testing" || GH_AVAILABLE=false
+
+BUILD_SLURM_JOB=$(sbatch --export="image=$IMAGE,PATH=$PATH,SHA=$SHA,working_dir=$HOME_DIR" --output="${HOME_DIR}/logs/${SHA}/launch/out.log" --error="${HOME_DIR}/logs/${SHA}/launch/error.log" "${HOME_DIR}/slurm_testing/regression_run_scripts/build_image.sh" | awk '{print $4}')
+
+# only launch if build succeeded
+LAUNCH_SLURM_JOB=$(sbatch --dependency=afterok:"$BUILD_SLURM_JOB" --export="GH_AVAILABLE=$GH_AVAILABLE,HOME_DIR=$HOME_DIR,IMAGE=$IMAGE,OWNER=$OWNER,PATH=$PATH,PUSH_LOGS=$PUSH_LOGS,REPO=$REPO,SHA=$SHA" --output="${HOME_DIR}/logs/${SHA}/launch/out.log" --error="${HOME_DIR}/logs/${SHA}/launch/error.log" "${HOME_DIR}/slurm_testing/regression_run_scripts/regtest_lite.sh" | awk '{print $4}')
+
+# Update run check on GitHub Actions
+if [ "$GH_AVAILABLE" = true ]
+then
+  LAUNCH_SUCCESS_JOB=$(sbatch --dependency=afterok:"$LAUNCH_SLURM_JOB" --output="${HOME_DIR}/logs/${SHA}/launch/out.log" --error="${HOME_DIR}/logs/${SHA}/launch/error.log" -J 'launch_success' --export="HOME_DIR=$HOME_DIR,OWNER=$OWNER,PATH=$PATH,PUSH_LOGS=$PUSH_LOGS,REPO=$REPO,SHA=$SHA,STATUS=success" .github/scripts/status_regtest_lite.SLURM | awk '{print $4}')
+  LAUNCH_FAILURE_JOB=$(sbatch --output="${HOME_DIR}/logs/${SHA}/launch/out.log" --error="${HOME_DIR}/logs/${SHA}/launch/error.log" --dependency=afternotok:"$BUILD_SLURM_JOB"?afternotok:"$LAUNCH_SLURM_JOB" -J 'launch_failure' --export="HOME_DIR=$HOME_DIR,OWNER=$OWNER,PATH=$PATH,PUSH_LOGS=$PUSH_LOGS,REPO=$REPO,SHA=$SHA,STATUS=failure" .github/scripts/status_regtest_lite.SLURM | awk '{print $4}')
+  # Delete local logs after pushing them to GitHub
+  sbatch --dependency=afterok:"$LAUNCH_FAILURE_JOB" -J 'push_logs' --export="HOME_DIR=$HOME_DIR,IMAGE=$IMAGE,REPO=$REPO,SHA=$SHA,_WD=$_WD" .github/scripts/push_to_github.SLURM
+else
+  sbatch --dependency=afterok:"$LAUNCH_SLURM_JOB" --output="${HOME_DIR}/logs/${SHA}/out.log" --error="${HOME_DIR}/logs/${SHA}/launch/error.log" -J 'launch_success' --wrap="echo 'launch succeeded')"
+  sbatch --dependency=afternotok:"$BUILD_SLURM_JOB"?afternotok:"$LAUNCH_SLURM_JOB" --output="${HOME_DIR}/logs/${SHA}/launch/out.log" --error="${HOME_DIR}/logs/${SHA}/launch/error.log" -J 'launch_failure' --wrap=">&2 echo 'launch failed')"
+fi
diff --git a/.github/scripts/push_to_github.SLURM b/.github/scripts/push_to_github.SLURM
@@ -0,0 +1,14 @@
+#!/bin/bash
+# Required environment variables: $HOME_DIR, $IMAGE, $REPO, $SHA, $_WD
+#SBATCH -N 1
+#SBATCH -p RM-shared
+#SBATCH -t 00:05:00
+#SBATCH --ntasks-per-node=1
+
+export GIT_REPO_DIR="${HOME_DIR}/logs/${SHA}"
+cd "${GIT_REPO_DIR}"
+source "${HOME_DIR}/slurm_testing/.github/scripts/wait_for_git_lock.sh"
+wait_for_git_lock
+git push origin ${REPO}_${SHA} -f
+cd $_WD
+rm -rf "${HOME_DIR}/${IMAGE}"
diff --git a/.github/scripts/status_regtest_lite.SLURM b/.github/scripts/status_regtest_lite.SLURM
@@ -0,0 +1,21 @@
+#!/bin/bash
+# Required environment variables: $HOME_DIR, $OWNER, $PATH, $PUSH_LOGS, $REPO, $SHA, $STATUS
+#SBATCH -N 1
+#SBATCH -p RM-shared
+#SBATCH -t 00:05:00
+#SBATCH --ntasks-per-node=1
+
+if [[ $PUSH_LOGS==true ]]
+then
+    export _WD=$PWD
+    export GIT_REPO_DIR="${HOME_DIR}/logs/${SHA}"
+    cd "${GIT_REPO_DIR}"
+    source "${HOME_DIR}/slurm_testing/.github/scripts/wait_for_git_lock.sh"
+    wait_for_git_lock
+    git add launch/error.log launch/out.log
+    wait_for_git_lock
+    git commit -m "Add logs for ${PIPELINE} ${DATA} ${SHA}"
+    cd $_WD
+fi
+
+gh workflow run 'Run launch status update' -F ref="$SHA" -F repo="$REPO" -F owner="$OWNER" -F status="${STATUS}"
diff --git a/.github/scripts/wait_for_git_lock.sh b/.github/scripts/wait_for_git_lock.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+# Required environment variable: $GIT_REPO_DIR 
+
+# Function to check if .git/index.lock exists and wait until it's gone
+function wait_for_git_lock {
+    while [ -f "$GIT_REPO_DIR/.git/index.lock" ]; do
+        >&2 echo "$(date)    A Git lock file (.git/index.lock) is present. Waiting…"
+        sleep 5
+    done
+}
diff --git a/.github/workflows/correlate.yml b/.github/workflows/correlate.yml
@@ -0,0 +1,66 @@
+name: Correlate Regression Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: 'branch, tag, or SHA'
+        type: string
+        required: true
+      owner:
+        description: 'owner on GitHub'
+        type: string
+        required: false
+        default: "FCP-INDI"
+      repo:
+        description: 'repository on GitHub'
+        type: string
+        required: false
+        default: "C-PAC"
+      environment:
+        description: 'Environment in which to run correlations'
+        type: environment
+        required: false
+        default: ACCESS
+      pipeline1:
+        type: string
+        required: true
+      pipeline2:
+        type: string
+        required: true
+      preconfig:
+        type: string
+        required: true
+      data_source:
+        type: string
+        required: true
+
+
+jobs:
+  correlate:
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment}}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3.5.3
+      - name: Finalize run check
+        uses: guibranco/github-status-action-v2@v1.1.7
+        with:
+          authToken: ${{ secrets.GH_PAT }}
+          context: lite run ${{ inputs.preconfig }} ${{ inputs.data_source }}
+          owner: ${{ inputs.owner }}
+          repository: ${{ inputs.repo }}
+          sha: ${{ inputs.ref }}
+          state: success
+          target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}#artifacts
+      - name: Correlating
+        run: |
+          echo "Correlating lite regression runs"
+          echo "(ﾉ◕ヮ◕)ﾉ*:･ﾟ✧"
+          echo "Correlating ${{ inputs.pipeline1 }} × ${{ inputs.pipeline2 }} for ${{ inputs.owner}}/${{ inputs.repo }}@${{ inputs.ref }}"
+          # 
+      # - uses: actions/upload-artifact@v3
+      #   if: always()
+      #   with:
+      #     name: logs
+      #     path: output/*/*/log/
diff --git a/.github/workflows/initiate_check.yml b/.github/workflows/initiate_check.yml
@@ -0,0 +1,58 @@
+name: Initiate check
+run-name: ${{ inputs.job }} ${{ inputs.task }} initiated on ${{ inputs.environment }}
+
+on:
+  workflow_dispatch:
+    inputs:
+      job:
+        description: SLURM job
+        type: string
+        required: true
+      task:
+        description: run or correlate
+        type: string
+        required: true
+      ref:
+        description: 'branch, tag, or SHA'
+        type: string
+        required: true
+      owner:
+        description: 'owner on GitHub'
+        type: string
+        required: false
+        default: "FCP-INDI"
+      repo:
+        description: 'repository on GitHub'
+        type: string
+        required: false
+        default: "C-PAC"
+      environment:
+        description: 'Environment in which to run correlations'
+        type: environment
+        required: false
+        default: ACCESS
+      preconfig:
+        type: string
+        required: true
+      data_source:
+        type: string
+        required: true
+
+jobs:
+  initiate:
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment}}
+    name: Running ${{ inputs.job }} on ${{ inputs.environment }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3.5.3
+      - name: Initiate check
+        uses: guibranco/github-status-action-v2@v1.1.7
+        with:
+          authToken: ${{ secrets.GH_PAT }}
+          context: lite ${{ inputs.task }} ${{ inputs.preconfig }} ${{ inputs.data_source }}
+          description: ${{ inputs.job }} on ${{ inputs.environment }}
+          owner: ${{ inputs.owner }}
+          repository: ${{ inputs.repo }}
+          sha: ${{ inputs.ref }}
+          state: pending
diff --git a/.github/workflows/launch_status.yml b/.github/workflows/launch_status.yml
@@ -0,0 +1,49 @@
+name: Run launch status update
+
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: branch, tag, or SHA
+        type: string
+        required: true
+      owner:
+        description: owner on GitHub
+        type: string
+        required: false
+        default: FCP-INDI
+      repo:
+        description: repository on GitHub
+        type: string
+        required: false
+        default: C-PAC
+      environment:
+        description: Environment in which to run correlations
+        type: environment
+        required: false
+        default: ACCESS
+      status:
+        type: choice
+        options:
+          - success
+          - failure
+          - pending
+
+jobs:
+  status_update:
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment}}
+    name: Update launch status
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3.5.3
+      - name: Finalize run check
+        uses: guibranco/github-status-action-v2@v1.1.7
+        with:
+          authToken: ${{ secrets.GH_PAT }}
+          context: Launch lite regression test
+          owner: ${{ inputs.owner }}
+          repository: ${{ inputs.repo }}
+          sha: ${{ inputs.ref }}
+          state: ${{ inputs.status }}
+          target_url: https://github.com/${{ inputs.owner }}/regtest-runlogs/tree/${{ inputs.repo}}_${{ inputs.ref }}/launch
diff --git a/.github/workflows/run_failed.yml b/.github/workflows/run_failed.yml
@@ -0,0 +1,52 @@
+name: Test run failed
+run-name: ${{ inputs.preconfig }} ${{ inputs.data_source }} run failed on ${{ inputs.environment }}
+
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: 'branch, tag, or SHA'
+        type: string
+        required: true
+      owner:
+        description: 'owner on GitHub'
+        type: string
+        required: false
+        default: "FCP-INDI"
+      repo:
+        description: 'repository on GitHub'
+        type: string
+        required: false
+        default: "C-PAC"
+      environment:
+        description: 'Environment in which to run correlations'
+        type: environment
+        required: false
+        default: ACCESS
+      preconfig:
+        type: string
+        required: true
+      data_source:
+        type: string
+        required: true
+
+jobs:
+  failure:
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment}}
+    name: Update launch status
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3.5.3
+      - name: Finalize run check
+        uses: guibranco/github-status-action-v2@v1.1.7
+        with:
+          authToken: ${{ secrets.GH_PAT }}
+          context: lite run ${{ inputs.preconfig }} ${{ inputs.data_source }}
+          # description: |
+          owner: ${{ inputs.owner }}
+          repository: ${{ inputs.repo }}
+          sha: ${{ inputs.ref }}
+          state: failure
+          # target_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}#artifacts
+          target_url: https://github.com/${{ inputs.owner }}/regtest-runlogs/tree/${{ inputs.repo}}_${{ inputs.ref }}/slurm-${{ inputs.preconfig }}-${{ inputs.data_source }}
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1 @@
+.DS_Store
diff --git a/.markdownlint.yml b/.markdownlint.yml
@@ -0,0 +1,8 @@
+line-length: false
+no-duplicate-header:
+  allow_different_nesting: true
+no-inline-html:
+  allowed_elements:
+    - span
+no-trailing-punctuation:
+  punctuation: .,;:。，；：！