Skip to content

Check Commits and Colab Images #99

Check Commits and Colab Images

Check Commits and Colab Images #99

Workflow file for this run

name: Check Commits and Colab Images
on:
push:
branches: [ "master" ]
pull_request:
branches: [ "master" ]
workflow_dispatch:
schedule:
- cron: 0 12 */1 * *
jobs:
check_commits_and_images:
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.11
- name: Install dependencies
run: pip install requests==2.31.0 pandas==2.1.1 google-cloud-bigquery==3.12.0 pyarrow==13.0.0 nbformat==5.9.2
- name: Authorize Google Cloud
uses: google-github-actions/auth@v1
with:
credentials_json: ${{ secrets.SERVICE_ACCOUNT_KEY }}
create_credentials_file: true
export_environment_variables: true
- name: Run check-commits.py and check-colab-images.py, preprocess notebooks scripts
run: |
python test/src/check-commits.py
python test/src/check-colab-images.py
python test/src/preProcessNotebooks.py
- name: Set result output
id: set-result
run: |
if [[ -f "check_colab_images_result.txt" ]]; then
RESULT=$(cat "check_colab_images_result.txt")
echo "RESULT=$RESULT" >> $GITHUB_ENV
fi
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
with:
tool-cache: false
android: true
dotnet: true
haskell: true
large-packages: true
docker-images: true
swap-storage: true
- name: Docker login
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Pull from GCP and Push Docker image to Docker Hub
if: env.RESULT == 'true'
run: |
docker pull us-docker.pkg.dev/colab-images/public/runtime:latest
docker tag us-docker.pkg.dev/colab-images/public/runtime:latest imagingdatacommons/idc-testing-colab:latest
docker push imagingdatacommons/idc-testing-colab:latest
- name: Pull Docker image from Docker Hub
if: env.RESULT == 'false'
run: |
docker pull imagingdatacommons/idc-testing-colab:latest
- name: Copy Google Cloud credentials to Docker container
run: |
CREDENTIALS_FILE_PATH="${{ env.GOOGLE_APPLICATION_CREDENTIALS }}"
CREDENTIALS_FILE_NAME=$(basename "$CREDENTIALS_FILE_PATH")
GOOGLE_APPLICATION_CREDENTIALS="/content/$CREDENTIALS_FILE_NAME"
echo "GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_APPLICATION_CREDENTIALS" >> $GITHUB_ENV
- name: Run notebook with papermill
run: |
for nb in part1_prerequisites part2_searching_basics part3_exploring_cohorts; do
docker run -d --name colab -v "$(pwd):/content" -e GOOGLE_APPLICATION_CREDENTIALS="${{ env.GOOGLE_APPLICATION_CREDENTIALS }}" imagingdatacommons/idc-testing-colab:latest
docker exec -t colab /bin/bash -c "pip install papermill==2.4.0"
docker exec -t colab /bin/bash -c "set -o xtrace && set -o errexit && set -o pipefail && set -o nounset && set +o errexit && cd content/ && papermill /content/notebooks/getting_started/${nb}.ipynb /content/test/outputs/${nb}_papermill_output.ipynb && set -o errexit && ls -A"
#docker exec -t colab /bin/bash -c "jupyter nbconvert --to html --ExtractOutputPreprocessor.enabled=False /content/test/outputs/output_${nb}.ipynb"
docker stop colab
docker rm colab
done
# skip commit
#- name: Commit changes
# if: ${{ github.event_name != 'pull_request' }}
# uses: stefanzweifel/git-auto-commit-action@v4
# with:
# commit_message: 'Check colab env'
# file_pattern: 'test/*.csv test/outputs/*.ipynb'
# branch: 'master'
#- name: Check output notebooks for errors
# run: |
# for nb in part1_prerequisites part2_searching_basics part3_exploring_cohorts; do
# if grep -q '"name": "stderr"\|"status": "failed"' test/outputs/output_${nb}.ipynb; then
# echo "Error messages found in the ${nb} notebook output:"
# cat test/outputs/output_${nb}.ipynb
# exit 1
# else
# echo "No errors found in the ${nb} notebook output."
# fi
# done
# exit $EXIT_CODE