chore: update pre-commit hooks (#164) #712
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # The logic of the file is a bit convoluted so I'll describe what should | |
| # happen. Dockerfiles are expected to be labeled based on how they should be | |
| # released/tagged. See example below to decide how to define your dockerfile. | |
| # Note: one must define version and/or ref.name (enforced by 'check' job.) | |
| # | |
| # * on a PR: | |
| # - changed images | |
| # ✓ build | |
| # ✓ run | |
| # X deploy | |
| # X release (re-tag) | |
| # X unchanged images | |
| # | |
| # * on a push to main: | |
| # - changed images | |
| # ✓ build | |
| # ✓ run | |
| # ✓ deploy image to org.opencontainers.image.version if set (e.g., version "x.y.z") | |
| # ✓ deploy image to org.opencontainers.image.ref.name if set (e.g., "latest") | |
| # X release (re-tag) | |
| # X unchanged images | |
| # | |
| # * on a tag: | |
| # X changed images | |
| # X unchanged images | |
| # | |
| # * on a published release: | |
| # X changed images | |
| # - unchanged images | |
| # ✓ build | |
| # ✓ run | |
| # X deploy | |
| # ✓ release latest images as $RELEASE if org.opencontainers.image.version unset | |
| # ✓ release latest images as stable | |
| # | |
| # Dockerfile/${IMAGE_NAME}-{SUFFIX} | |
| # LABEL org.opencontainers.image.version=${VERSION} | |
| # LABEL org.opencontainers.image.ref.name=${REFNAME} | |
| # | |
| # Containers can have a suffix which will be appropriately added to all above tags accordingly if the suffix is not latest. | |
| # Containers labeled with image.ref.name will have an associated tag on push, but | |
| # Containers labeled with image.version will not have a mapyde-release associated tag, but will have a "-stable" associated tag. | |
| # | |
| # ------------------------------------------------------------------------------------- | |
| # | |
| # Example of above logic for container registry tags for following containers: | |
| # | |
| # - Dockerfiles/pyplotting | |
| # LABEL org.opencontainers.image.ref.name=latest | |
| # | |
| # - Dockerfiles/delphes-snowmass | |
| # LABEL org.opencontainers.image.version=snowmass | |
| # LABEL org.opencontainers.image.ref.name=snowmass | |
| # | |
| # - Dockerfiles/delphes | |
| # LABEL org.opencontainers.image.version=3.5.0 | |
| # LABEL org.opencontainers.image.ref.name=latest | |
| # | |
| # - Dockerfiles/pyplotting-arm | |
| # LABEL org.opencontainers.image.ref.name=latest | |
| # | |
| # - Dockerfile/pyplotting-cuda | |
| # LABEL org.opencontainers.image.ref.name=latest | |
| # LABEL org.opencontainers.image.version=alternate | |
| # | |
| # * on a PR: | |
| # - changed images: nothing | |
| # - unchanged images: nothing | |
| # | |
| # * on a push to main: | |
| # - changed images: pyplotting:latest, pyplotting:latest-arm, pyplotting:latest-cuda, pyplotting:alternate-cuda, delphes:snowmass, delphes:3.5.0, delphes:latest, delphes:latest-snowmass | |
| # X unchanged images | |
| # | |
| # * on a tag: | |
| # X changed images | |
| # X unchanged images | |
| # | |
| # * on a published release (x.y.z): | |
| # X changed images | |
| # - unchanged images: pyplotting:x.y.z, pyplotting:x.y.z-arm, pyplotting:stable, pyplotting:stable-arm, pyplotting:stable-cuda, delphes:latest-stable | |
| # | |
| # ===================================================================================== | |
| name: Docker Images | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| release: | |
| types: [published] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| check: | |
| name: Check Dockerfiles | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Verify labels | |
| run: | | |
| fail=0 | |
| for dockerfile in $(find Dockerfiles/ -type f); do | |
| if grep -q -E 'org\.opencontainers\.image\.(version|ref\.name)' ${dockerfile} | |
| then | |
| if grep -q -E 'org\.opencontainers\.image\.version.*=.*(latest|latest-stable)$' ${dockerfile} | |
| then | |
| echo -e " \033[0;31mX\033[0m ${dockerfile}: version=latest or version=latest-stable" | |
| fail=1 | |
| else | |
| echo -e " \033[0;32m✓\033[0m ${dockerfile}: ok" | |
| fi | |
| else | |
| echo -e " \033[0;31mX\033[0m ${dockerfile}: labels missing" | |
| fail=1 | |
| fi | |
| done | |
| if [[ $fail == 1 ]]; then | |
| exit 1 | |
| fi | |
| changes: | |
| needs: [check] | |
| runs-on: ubuntu-latest | |
| outputs: | |
| changed: ${{ steps.filter.outputs.changed }} | |
| all_images: ${{ steps.set.outputs.all_images }} | |
| changed_images: ${{ steps.set.outputs.changed_images }} | |
| unchanged_images: ${{ steps.set.outputs.unchanged_images }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dorny/paths-filter@v3 | |
| id: filter | |
| with: | |
| list-files: json | |
| filters: | | |
| changed: Dockerfiles/** | |
| - id: set | |
| run: | | |
| ALL_IMAGES=$(find Dockerfiles/ -type f | xargs -n1 basename | jq -R -s -c 'split("\n")[:-1]' | tr '-' ':') | |
| echo "::set-output name=all_images::${ALL_IMAGES}" | |
| printf "Setting all_images=${ALL_IMAGES}\n" | |
| CHANGED_IMAGES=$(printf ${{ toJSON(steps.filter.outputs.changed_files) }} | jq -c '.[] |= split("/")[1]' | tr '-' ':') | |
| echo "::set-output name=changed_images::${CHANGED_IMAGES}" | |
| printf "Setting changed_images=${CHANGED_IMAGES}\n" | |
| UNCHANGED_IMAGES=$(jq -n -c --argjson all_images ${ALL_IMAGES} --argjson changed_images ${CHANGED_IMAGES} '$all_images-$changed_images') | |
| echo "::set-output name=unchanged_images::${UNCHANGED_IMAGES}" | |
| printf "Setting unchanged_images=${UNCHANGED_IMAGES}\n" | |
| build: | |
| name: Build ${{ matrix.image }} | |
| needs: [changes] | |
| if: ${{ needs.changes.outputs.changed == 'true' }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| image: ${{ fromJSON(needs.changes.outputs.changed_images) }} | |
| steps: | |
| - name: Get more space # https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 | |
| run: | | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf "/usr/local/share/boost" | |
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | |
| - uses: actions/checkout@v4 | |
| - name: Prepare | |
| id: prep | |
| run: | | |
| echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ') | |
| echo ::set-output name=dockerfile::$(echo "${{ matrix.image }}" | tr ':' '-') | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build image (amd64) | |
| id: docker_build_test_amd64 | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: Dockerfiles | |
| tags: ghcr.io/${{github.repository}}/${{ matrix.image }} | |
| file: Dockerfiles/${{ steps.prep.outputs.dockerfile }} | |
| labels: | | |
| org.opencontainers.image.source=${{ github.event.repository.html_url }} | |
| org.opencontainers.image.created=${{ steps.prep.outputs.created }} | |
| org.opencontainers.image.revision=${{ github.sha }} | |
| outputs: type=docker,dest=image.tar | |
| platforms: linux/amd64 | |
| - name: Build image (arm64) | |
| if: false | |
| id: docker_build_test_arm64 | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: Dockerfiles | |
| tags: ghcr.io/${{github.repository}}/${{ matrix.image }} | |
| file: Dockerfiles/${{ steps.prep.outputs.dockerfile }} | |
| labels: | | |
| org.opencontainers.image.source=${{ github.event.repository.html_url }} | |
| org.opencontainers.image.created=${{ steps.prep.outputs.created }} | |
| org.opencontainers.image.revision=${{ github.sha }} | |
| outputs: type=docker,dest=image.tar | |
| platforms: linux/arm64 | |
| - name: Image digest | |
| run: | | |
| echo ${{ steps.docker_build_test_amd64.outputs.digest }} | |
| # echo ${{ steps.docker_build_test_arm64.outputs.digest }} | |
| - name: List built images | |
| run: docker images | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ steps.prep.outputs.dockerfile }} | |
| path: image.tar | |
| # run is skipped if build is skipped (no changed images) | |
| run: | |
| needs: [build] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.9 | |
| - name: Download artifact | |
| uses: actions/download-artifact@v4 | |
| - name: Load images | |
| run: | | |
| ls -lavh | |
| for image in */image.tar; do | |
| echo ::group::$(dirname $image) | |
| test -f "$image" && docker load --input $image | | |
| awk '{print $NF}' | xargs docker inspect | |
| echo ::endgroup:: | |
| done | |
| docker image ls -a | |
| - name: Install MaPyDe | |
| run: | | |
| python -m pip install --upgrade pip setuptools wheel | |
| python -m pip install --upgrade . | |
| - name: List installed Python packages | |
| run: python -m pip list | |
| - name: Prepare Inputs for Generation | |
| run: | | |
| python3 -m pip install . | |
| mapyde config parse ci/charginos.toml | |
| mapyde config generate-mg5 ci/charginos.toml | |
| - name: Run Madgraph | |
| # prettier-ignore | |
| run: >- | |
| docker run --rm | |
| -v $PWD/githubci:/data | |
| -v $PWD/cards:/cards | |
| -w /output | |
| ghcr.io/${{ github.repository }}/madgraph:latest | |
| "mg5_aMC /data/run.mg5; rsync -rav PROC_madgraph /data/madgraph" | |
| - name: Run Delphes | |
| # prettier-ignore | |
| run: >- | |
| docker run --rm | |
| -v $PWD/githubci:/data | |
| -v $PWD/cards:/cards | |
| -w /output | |
| ghcr.io/${{ github.repository }}/delphes:latest | |
| 'cp $(find /data/ -name "*hepmc.gz") hepmc.gz; gunzip hepmc.gz; | |
| /usr/local/share/delphes/delphes/DelphesHepMC /cards/delphes/delphes_card_ATLAS.tcl delphes.root hepmc; | |
| rsync -rav --exclude hepmc . /data/delphes' | |
| - name: Run Analysis | |
| # prettier-ignore | |
| run: >- | |
| docker run --rm | |
| -v $PWD/githubci:/data | |
| -v $PWD/cards:/cards | |
| -v $PWD/scripts:/scripts | |
| -w /output | |
| ghcr.io/${{ github.repository }}/delphes:latest | |
| "/scripts/SimpleAna.py --input /data/delphes/delphes.root --output histograms.root; | |
| rsync -rav . /data/analysis" | |
| # deploy skipped if build/run skipped | |
| deploy: | |
| needs: [changes, build, run] | |
| name: Deploy ${{ matrix.image }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| image: ${{ fromJSON(needs.changes.outputs.changed_images) }} | |
| steps: | |
| - name: Prepare | |
| id: prep | |
| run: | | |
| echo ::set-output name=dockerfile::$(echo "${{ matrix.image }}" | tr ':' '-') | |
| - name: Login to GitHub Container Registry | |
| if: github.event_name != 'pull_request' | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Download artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: ${{ steps.prep.outputs.dockerfile }} | |
| path: images | |
| - name: Load images | |
| run: | | |
| docker load --input images/image.tar | |
| - name: List images | |
| run: | | |
| echo ::group::all_images | |
| docker images -a | |
| echo ::endgroup:: | |
| echo ::group::${{ matrix.image }} | |
| docker images -f 'label=org.opencontainers.image.revision=${{ github.sha }}' --format='{{.Repository}}:{{.Tag}}' | |
| echo ::endgroup:: | |
| - name: Check label | |
| id: label | |
| run: | | |
| # split by delimiter ':' into two variables | |
| IFS=: read -r IMAGE_NAME SUFFIX <<< "${{ matrix.image }}" | |
| echo "Parsed IMAGE_NAME=${IMAGE_NAME}, SUFFIX=${SUFFIX:=latest}" | |
| IMAGE="ghcr.io/${{github.repository}}/${IMAGE_NAME}" | |
| VERSION=$(docker image inspect --format '{{ index .Config.Labels "org.opencontainers.image.version"}}' "${IMAGE}:${SUFFIX}") | |
| REFNAME=$(docker image inspect --format '{{ index .Config.Labels "org.opencontainers.image.ref.name"}}' "${IMAGE}:${SUFFIX}") | |
| LATEST="latest" | |
| # if the suffix is not latest, we need to add it to VERSION and REFNAME (if not duplicated) | |
| if [ "${SUFFIX}" != "latest" ] | |
| then | |
| LATEST="latest-${SUFFIX}" | |
| if [ "${VERSION}" ] && [ "${VERSION}" != "${SUFFIX}" ] | |
| then | |
| VERSION="${VERSION}-${SUFFIX}" | |
| fi | |
| if [ "${REFNAME}" ] && [ "${REFNAME}" != "${SUFFIX}" ] | |
| then | |
| REFNAME="${REFNAME}-${SUFFIX}" | |
| fi | |
| fi | |
| if [ "${REFNAME}" == "${VERSION}" ] | |
| then | |
| echo "REFNAME and VERSION are the same. Setting VERSION=''" | |
| VERSION='' | |
| fi | |
| if [ "${LATEST}" == "${REFNAME}" ] | |
| then | |
| echo "LATEST and REFNAME are the same. Setting REFNAME=''" | |
| REFNAME='' | |
| fi | |
| if [ "${LATEST}" == "${VERSION}" ] | |
| then | |
| echo "LATEST and VERSION are the same. Setting VERSION=''" | |
| VERSION='' | |
| fi | |
| [ "${VERSION}" ] && [ "${SUFFIX}" != "${VERSION}" ] && docker image tag "${IMAGE}:${SUFFIX}" "${IMAGE}:${VERSION}" | |
| [ "${VERSION}" ] && [ "${SUFFIX}" != "${VERSION}" ] && echo "Tagging ${IMAGE}:${SUFFIX} as ${IMAGE}:${VERSION}" | |
| [ "${REFNAME}" ] && [ "${SUFFIX}" != "${REFNAME}" ] && docker image tag "${IMAGE}:${SUFFIX}" "${IMAGE}:${REFNAME}" | |
| [ "${REFNAME}" ] && [ "${SUFFIX}" != "${REFNAME}" ] && echo "Tagging ${IMAGE}:${SUFFIX} as ${IMAGE}:${REFNAME}" | |
| [ "${SUFFIX}" != "${LATEST}" ] && docker image tag "${IMAGE}:${SUFFIX}" "${IMAGE}:${LATEST}" | |
| [ "${SUFFIX}" != "${LATEST}" ] && echo "Tagging ${IMAGE}:${SUFFIX} as ${IMAGE}:${LATEST}" | |
| echo "Setting image=${IMAGE}" | |
| echo "Setting latest=${LATEST}" | |
| echo "Setting version=${VERSION}" | |
| echo "Setting refname=${REFNAME}" | |
| echo ::set-output name=image::${IMAGE} | |
| echo ::set-output name=latest::${LATEST} | |
| echo ::set-output name=version::${VERSION} | |
| echo ::set-output name=refname::${REFNAME} | |
| - name: "Deploy as latest: ${{ steps.label.outputs.latest }}" | |
| if: | | |
| github.event_name == 'push' && github.ref == 'refs/heads/main' && | |
| github.repository == 'scipp-atlas/mapyde' | |
| run: | | |
| docker image push "${{ steps.label.outputs.image }}:${{ | |
| steps.label.outputs.latest }}" | |
| - name: "Deploy as version: ${{ steps.label.outputs.version }}" | |
| if: | | |
| github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'scipp-atlas/mapyde' && | |
| steps.label.outputs.version | |
| run: | | |
| docker image push "${{ steps.label.outputs.image }}:${{ | |
| steps.label.outputs.version }}" | |
| - name: "Deploy as refname: ${{ steps.label.outputs.refname }}" | |
| if: | | |
| github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'scipp-atlas/mapyde' && | |
| steps.label.outputs.refname | |
| run: | | |
| docker image push "${{ steps.label.outputs.image }}:${{ | |
| steps.label.outputs.refname }}" | |
| # why such a complicated if statement? see https://github.com/actions/runner/issues/491#issuecomment-850884422 | |
| release: | |
| needs: [changes, build, run] | |
| name: Release ${{ matrix.image }} | |
| if: | | |
| always() && | |
| (needs.build.result == 'success' || needs.build.result == 'skipped') && | |
| (needs.run.result == 'success' || needs.run.result == 'skipped') && | |
| github.event_name == 'release' && github.event.action == 'published' && github.repository == 'scipp-atlas/mapyde' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| image: ${{ fromJSON(needs.changes.outputs.unchanged_images) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Prepare | |
| id: prep | |
| run: | | |
| echo ::set-output name=dockerfile::$(echo "${{ matrix.image }}" | tr ':' '-') | |
| - name: Check label | |
| id: label | |
| run: | | |
| VERSION_UNSET='false' | |
| if grep -q -E 'org\.opencontainers\.image\.version' Dockerfiles/"${{ steps.prep.outputs.dockerfile }}" | |
| then | |
| VERSION_UNSET='true' | |
| fi | |
| echo "Setting version_unset=${VERSION_UNSET}" | |
| echo ::set-output name=version_unset::${VERSION_UNSET} | |
| # split by delimiter ':' into two variables | |
| IFS=: read -r IMAGE_NAME SUFFIX <<< "${{ matrix.image }}" | |
| echo "Parsed IMAGE_NAME=${IMAGE_NAME}, SUFFIX=${SUFFIX:=latest}" | |
| IMAGE="ghcr.io/${{github.repository}}/${IMAGE_NAME}" | |
| LATEST="latest" | |
| STABLE="stable" | |
| TAG=${{ github.event.release.tag_name }} | |
| # if the suffix is not latest, we need to add it to VERSION and REFNAME (if not duplicated) | |
| if [ "${SUFFIX}" != "latest" ] | |
| then | |
| LATEST="latest-${SUFFIX}" | |
| STABLE="stable-${SUFFIX}" | |
| TAG="${TAG}-${SUFFIX}" | |
| fi | |
| echo "Setting image=${IMAGE}" | |
| echo "Setting latest=${LATEST}" | |
| echo "Setting stable=${STABLE}" | |
| echo "Setting tag=${TAG}" | |
| echo ::set-output name=image::${IMAGE} | |
| echo ::set-output name=latest::${LATEST} | |
| echo ::set-output name=stable::${STABLE} | |
| echo ::set-output name=tag::${TAG} | |
| - name: Tag latest as stable | |
| uses: akhilerm/tag-push-action@v2.2.0 | |
| with: | |
| src: | | |
| ${{ steps.label.outputs.image }}:${{ steps.label.outputs.latest }} | |
| dst: | | |
| ${{ steps.label.outputs.image }}:${{ steps.label.outputs.stable }} | |
| - name: Tag latest as ${{ github.event.release.tag_name }} | |
| if: ${{ steps.label.outputs.version_unset == 'true' }} | |
| uses: akhilerm/tag-push-action@v2.2.0 | |
| with: | |
| src: | | |
| ${{ steps.label.outputs.image }}:${{ steps.label.outputs.latest }} | |
| dst: | | |
| ${{ steps.label.outputs.image }}:${{ steps.label.outputs.tag }} |