Build stages #2315
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Requires secret `ACTIONS_WORKFLOW_TOKEN` with `workflow` scope | |
| name: Build stages | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| base_ref: | |
| description: 'Base reference if triggered by a pull request' | |
| required: false | |
| default: "" | |
| jobs: | |
| update-check: | |
| name: Update GitHub check | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Update check's target URL | |
| uses: Sibz/github-status-action@v1.1.6 | |
| with: | |
| authToken: ${{ secrets.ACTIONS_WORKFLOW_TOKEN }} | |
| context: "Build C-PAC images" | |
| state: pending | |
| target_url: ${{ github.server_url }}/${{ github.repository}}/actions/runs/${{ github.run_id }} | |
| Ubuntu: | |
| name: Build C-PAC Ubuntu | |
| needs: | |
| - update-check | |
| strategy: | |
| matrix: | |
| Dockerfile: | |
| - Ubuntu.bionic-non-free | |
| - Ubuntu.xenial-20200114 | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out C-PAC (depth 0) | |
| if: github.event.inputs.base_ref != '' | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 | |
| - name: Check out C-PAC (depth 2) | |
| if: github.event.inputs.base_ref == '' | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 2 | |
| - name: Get changed files since fork point | |
| uses: tj-actions/changed-files@v34.0.0 | |
| if: github.event.inputs.base_ref != '' | |
| id: changed-files-base | |
| with: | |
| use_fork_point: "true" | |
| files: .github/Dockerfiles/* | |
| - name: Get changed files since last commit | |
| uses: tj-actions/changed-files@v34.0.0 | |
| if: github.event.inputs.base_ref == '' | |
| id: changed-files | |
| with: | |
| since_last_remote_commit: "true" | |
| files: .github/Dockerfiles/* | |
| - name: Set tag & see if it exists | |
| continue-on-error: true | |
| run: | | |
| TAG=$(sed 's/\./:/' <(echo ${{ matrix.Dockerfile }})) | |
| DOCKER_TAG=$(echo "ghcr.io/${{ github.repository }}/$TAG" | tr '[:upper:]' '[:lower:]') | |
| echo DOCKER_TAG=$DOCKER_TAG >> $GITHUB_ENV | |
| docker manifest inspect $DOCKER_TAG >/dev/null || echo "not_yet_exists=1" >> $GITHUB_OUTPUT | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2.2.1 | |
| if: contains(steps.changed-files-base.outputs.all_changed_files, matrix.Dockerfile) || contains(steps.changed-files.outputs.all_changed_files, matrix.Dockerfile) | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| if: contains(steps.changed-files-base.outputs.all_changed_files, matrix.Dockerfile) || contains(steps.changed-files.outputs.all_changed_files, matrix.Dockerfile) | |
| - name: Build and push Docker image | |
| uses: docker/build-push-action@v3.2.0 | |
| with: | |
| file: .github/Dockerfiles/${{ matrix.Dockerfile }}.Dockerfile | |
| push: true | |
| tags: | | |
| ${{ env.DOCKER_TAG }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=min | |
| if: contains(steps.changed-files-base.outputs.all_changed_files, matrix.Dockerfile) || contains(steps.changed-files.outputs.all_changed_files, matrix.Dockerfile) | |
| stages: | |
| name: Build Docker stage images | |
| needs: Ubuntu | |
| strategy: | |
| matrix: | |
| Dockerfile: | |
| - AFNI.16.2.07.neurodocker-xenial | |
| - AFNI.21.1.00-bionic | |
| - ANTs.2.2.0.neurodocker-bionic | |
| - c3d.1.0.0-xenial | |
| - connectome-workbench.1.3.2-1.neurodebian-bionic | |
| - connectome-workbench.1.3.2-2.neurodebian-xenial | |
| - FSL.5.0.9-5.neurodebian-xenial | |
| - FSL.5.0.10-bionic | |
| - ICA-AROMA.0.4.5-xenial | |
| - msm.2.0-bionic | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out C-PAC (depth 0) | |
| if: github.event.inputs.base_ref != '' | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 | |
| - name: Check out C-PAC (depth 2) | |
| if: github.event.inputs.base_ref == '' | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 2 | |
| - name: Get changed files since fork point | |
| uses: tj-actions/changed-files@v34.0.0 | |
| if: github.event.inputs.base_ref != '' | |
| id: changed-files-base | |
| with: | |
| use_fork_point: "true" | |
| files: | | |
| dev/docker_data/required_afni_pkgs.txt | |
| .github/Dockerfiles/* | |
| - name: Get changed files since last commit | |
| uses: tj-actions/changed-files@v34.0.0 | |
| if: github.event.inputs.base_ref == '' | |
| id: changed-files | |
| with: | |
| since_last_remote_commit: "true" | |
| files: | | |
| dev/docker_data/required_afni_pkgs.txt | |
| .github/Dockerfiles/* | |
| - name: Clear up some space on runner | |
| if: contains(steps.changed-files-base.outputs.all_changed_files, matrix.Dockerfile) || contains(steps.changed-files.outputs.all_changed_files, matrix.Dockerfile) || (startsWith(matrix.Dockerfile, 'AFNI') && (contains(steps.changed-files.outputs.all_changed_files, 'dev/docker_data/required_afni_pkgs.txt'))) | |
| run: | | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf "/usr/local/share/boost" | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2.2.1 | |
| if: contains(steps.changed-files-base.outputs.all_changed_files, matrix.Dockerfile) || contains(steps.changed-files.outputs.all_changed_files, matrix.Dockerfile) || (startsWith(matrix.Dockerfile, 'AFNI') && (contains(steps.changed-files.outputs.all_changed_files, 'dev/docker_data/required_afni_pkgs.txt'))) | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| if: contains(steps.changed-files-base.outputs.all_changed_files, matrix.Dockerfile) || contains(steps.changed-files.outputs.all_changed_files, matrix.Dockerfile) || (startsWith(matrix.Dockerfile, 'AFNI') && (contains(steps.changed-files.outputs.all_changed_files, 'dev/docker_data/required_afni_pkgs.txt'))) | |
| - name: Set tag & see if it exists | |
| continue-on-error: true | |
| run: | | |
| TAG=$(sed 's/\./:/' <(echo ${{ matrix.Dockerfile }})) | |
| DOCKER_TAG=$(echo "ghcr.io/${{ github.repository }}/$TAG" | tr '[:upper:]' '[:lower:]') | |
| echo DOCKER_TAG=$DOCKER_TAG >> $GITHUB_ENV | |
| docker manifest inspect $DOCKER_TAG >/dev/null || echo "not_yet_exists=1" >> $GITHUB_OUTPUT | |
| if: contains(steps.changed-files-base.outputs.all_changed_files, matrix.Dockerfile) || contains(steps.changed-files.outputs.all_changed_files, matrix.Dockerfile) || (startsWith(matrix.Dockerfile, 'AFNI') && (contains(steps.changed-files.outputs.all_changed_files, 'dev/docker_data/required_afni_pkgs.txt'))) | |
| - name: Prep Dockerfiles for forked repository | |
| if: github.repository_owner != 'FCP-INDI' && contains(steps.changed-files-base.outputs.all_changed_files, matrix.Dockerfile) || contains(steps.changed-files.outputs.all_changed_files, matrix.Dockerfile) || (startsWith(matrix.Dockerfile, 'AFNI') && (contains(steps.changed-files.outputs.all_changed_files, 'dev/docker_data/required_afni_pkgs.txt'))) | |
| run: | | |
| .github/scripts/local_ghcr .github/Dockerfiles/${{ matrix.Dockerfile }}.Dockerfile ${{ github.repository_owner }} $DOCKER_TAG | |
| cat .github/Dockerfiles/${{ matrix.Dockerfile }}.Dockerfile | |
| - name: See Dockerfile | |
| run: cat .github/Dockerfiles/${{ matrix.Dockerfile }}.Dockerfile | |
| if: contains(steps.changed-files-base.outputs.all_changed_files, matrix.Dockerfile) || contains(steps.changed-files.outputs.all_changed_files, matrix.Dockerfile) || (startsWith(matrix.Dockerfile, 'AFNI') && (contains(steps.changed-files.outputs.all_changed_files, 'dev/docker_data/required_afni_pkgs.txt'))) | |
| - name: Build and push Docker image | |
| uses: docker/build-push-action@v3.2.0 | |
| with: | |
| context: . | |
| file: .github/Dockerfiles/${{ matrix.Dockerfile }}.Dockerfile | |
| push: true | |
| tags: | | |
| ${{ env.DOCKER_TAG }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=min | |
| if: contains(steps.changed-files-base.outputs.all_changed_files, matrix.Dockerfile) || contains(steps.changed-files.outputs.all_changed_files, matrix.Dockerfile) || (startsWith(matrix.Dockerfile, 'AFNI') && (contains(steps.changed-files.outputs.all_changed_files, 'dev/docker_data/required_afni_pkgs.txt'))) | |
| build-base: | |
| needs: stages | |
| runs-on: ubuntu-latest | |
| env: | |
| BUILD_CACHE: /home/runner/.docker/buildkit | |
| strategy: | |
| matrix: | |
| variant: | |
| - standard | |
| - ABCD-HCP | |
| - fMRIPrep-LTS | |
| steps: | |
| - name: Maximize build space | |
| uses: easimon/maximize-build-space@v4 | |
| with: | |
| remove-dotnet: 'true' | |
| remove-android: 'true' | |
| remove-haskell: 'true' | |
| overprovision-lvm: 'true' | |
| - name: Check out C-PAC (depth 0) | |
| if: github.event.inputs.base_ref != '' | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 | |
| - name: Check out C-PAC (depth 2) | |
| if: github.event.inputs.base_ref == '' | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 2 | |
| - name: Prep source files | |
| run: | | |
| sed -i -e 's/^/\.github\/Dockerfiles\//' .github/stage_requirements/${{ matrix.variant }}.txt | |
| echo 'dev/docker_data/required_afni_pkgs.txt' >> .github/stage_requirements/${{ matrix.variant }}.txt | |
| echo '.github/workflows/build_stages.yml' >> .github/stage_requirements/${{ matrix.variant }}.txt | |
| echo '.github/stage_requirements/${{ matrix.variant }}.txt' >> .github/stage_requirements/${{ matrix.variant }}.txt | |
| - name: Get changed files since fork point | |
| uses: tj-actions/changed-files@v34.0.0 | |
| if: github.event.inputs.base_ref != '' | |
| id: changed-files-base | |
| with: | |
| use_fork_point: "true" | |
| files_from_source_file: | | |
| .github/stage_requirements/${{ matrix.variant }}.txt | |
| - name: Get changed files since last commit | |
| uses: tj-actions/changed-files@v34.0.0 | |
| if: github.event.inputs.base_ref == '' | |
| id: changed-files | |
| with: | |
| since_last_remote_commit: "true" | |
| files_from_source_file: | | |
| .github/stage_requirements/${{ matrix.variant }}.txt | |
| - name: Set tag & see if it exists | |
| continue-on-error: true | |
| run: | | |
| TAG="stage-base:${{ matrix.variant }}-$(cat version)" | |
| DOCKER_TAG=$(echo "ghcr.io/${{ github.repository }}/$TAG" | tr '[:upper:]' '[:lower:]') | |
| echo DOCKER_TAG=$DOCKER_TAG >> $GITHUB_ENV | |
| docker manifest inspect $DOCKER_TAG >/dev/null || echo "not_yet_exists=1" >> $GITHUB_OUTPUT | |
| id: docker_tag | |
| - name: Clear up some space on runner | |
| if: always() && steps.changed-files-base.outputs.any_changed == 'true' || steps.changed-files.outputs.any_changed == 'true' || steps.docker_tag.not_yet_exists == 1 | |
| run: | | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf "/usr/local/share/boost" | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2.2.1 | |
| if: always() && steps.changed-files-base.outputs.any_changed == 'true' || steps.changed-files.outputs.any_changed == 'true' || steps.docker_tag.not_yet_exists == 1 | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| if: always() && steps.changed-files-base.outputs.any_changed == 'true' || steps.changed-files.outputs.any_changed == 'true' || steps.docker_tag.not_yet_exists == 1 | |
| - name: Prep Dockerfiles for forked repository | |
| if: always() && steps.changed-files-base.outputs.any_changed == 'true' || steps.changed-files.outputs.any_changed == 'true' || steps.docker_tag.not_yet_exists == 1 | |
| run: | | |
| .github/scripts/local_ghcr .github/Dockerfiles/base-${{ matrix.variant }}.Dockerfile ${{ github.repository_owner }} $DOCKER_TAG | |
| cat .github/Dockerfiles/base-${{ matrix.variant }}.Dockerfile | |
| - name: See Dockerfile | |
| run: cat .github/Dockerfiles/base-${{ matrix.variant }}.Dockerfile | |
| if: always() && steps.changed-files-base.outputs.any_changed == 'true' || steps.changed-files.outputs.any_changed == 'true' || steps.docker_tag.not_yet_exists == 1 | |
| - name: Build and push base image | |
| uses: docker/build-push-action@v3.2.0 | |
| with: | |
| context: . | |
| file: .github/Dockerfiles/base-${{ matrix.variant }}.Dockerfile | |
| push: true | |
| tags: | | |
| ${{ env.DOCKER_TAG }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=min | |
| if: always() && steps.changed-files-base.outputs.any_changed == 'true' || steps.changed-files.outputs.any_changed == 'true' || steps.docker_tag.not_yet_exists == 1 | |
| trigger-next-workflow: | |
| needs: build-base | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out C-PAC | |
| uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 2 | |
| - name: Build and test C-PAC | |
| run: gh workflow run build_and_test.yml --ref ${GITHUB_REF_NAME} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.ACTIONS_WORKFLOW_TOKEN }} |