diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 345ba949..5883ca93 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,11 +13,11 @@ on: types: [published] jobs: - preload_docker: - uses: ./.github/workflows/preload-docker.yml + preload_containers: + uses: ./.github/workflows/preload-containers.yml test: - needs: preload_docker + needs: preload_containers name: Run standard workflow test runs-on: ubuntu-latest env: diff --git a/.github/workflows/preload-containers.yml b/.github/workflows/preload-containers.yml new file mode 100644 index 00000000..8e32eda2 --- /dev/null +++ b/.github/workflows/preload-containers.yml @@ -0,0 +1,78 @@ +name: Preload docker containers + +on: + workflow_call: + inputs: + singularity_version: + required: true + type: string + outputs: + docker-cache-key: + value: docker-${{ github.sha }} + singularity-cache-key: + value: singularity-${{ github.sha }} + +env: + NXF_SINGULARITY_CACHEDIR: "${{ github.workspace }}/singularity" + SINGULARITY_VERSION: "3.8.3" + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - name: Check out pipeline code + uses: actions/checkout@v3 + + - name: Pull and save docker + run: | + git grep 'ext.docker*' ${{ github.workspace }}/conf/modules.config | cut -f 2 -d '=' | xargs -L 2 echo | tr -d ' ' > ${{ runner.temp }}/images.txt + cat ${{ runner.temp }}/images.txt | xargs -I {} sh -c 'docker pull --platform linux/amd64 "$1"' - {} + mkdir -p ${{ runner.temp }}/docker/ + cat ${{ runner.temp }}/images.txt | xargs -I {} sh -c 'docker save "$1" > ${{ runner.temp }}/docker/$(basename "$1").tar' - {} + + - name: Save docker + id: cache-docker + uses: actions/cache@v3 + with: + path: ${{ runner.temp }}/docker + key: docker-${{ github.sha }} + + singularity: + runs-on: ubuntu-latest + steps: + - name: Check out pipeline code + uses: actions/checkout@v3 + + - name: Cache singularity setup + id: cache-singularity-setup + uses: actions/cache@v3 + with: + path: /opt/hostedtoolcache/singularity/${{ env.SINGULARITY_VERSION }}/x64 + key: ${{ runner.os }}-singularity-${{ env.SINGULARITY_VERSION }} + + - name: Set up Singularity + uses: eWaterCycle/setup-singularity@v7 + if: ${{ steps.cache-singularity-setup.outputs.cache-hit != 'true' }} + with: + singularity-version: ${{ env.SINGULARITY_VERSION }} + + - name: Add singularity to path + if: steps.cache-singularity-setup.outputs.cache-hit == 'true' + run: | + echo "/opt/hostedtoolcache/singularity/${{ env.SINGULARITY_VERSION }}/x64/bin" >> $GITHUB_PATH + + - name: Pull and save singularity + run: | + mkdir -p $NXF_SINGULARITY_CACHEDIR + git grep 'ext.singularity*' conf/modules.config | cut -f 2 -d '=' | xargs -L 2 echo | tr -d ' ' > ${{ runner.temp }}/singularity_images.txt + cat ${{ runner.temp }}/singularity_images.txt | sed 's/oras:\/\///;s/https:\/\///;s/\//-/g;s/$/.img/;s/:/-/' > ${{ runner.temp }}/singularity_image_paths.txt + paste -d '\n' ${{ runner.temp }}/singularity_image_paths.txt ${{ runner.temp }}/singularity_images.txt | xargs -L 2 sh -c 'singularity pull --disable-cache --dir $NXF_SINGULARITY_CACHEDIR $0 $1' + + - name: Cache singularity images + id: cache-singularity-pull + uses: actions/cache@v3 + with: + path: ${{ env.NXF_SINGULARITY_CACHEDIR }} + key: singularity-${{ github.sha }} + + diff --git a/.github/workflows/preload-docker.yml b/.github/workflows/preload-docker.yml deleted file mode 100644 index 0f6ce44d..00000000 --- a/.github/workflows/preload-docker.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Preload singularity and docker containers - -on: - workflow_call: - outputs: - docker-cache-key: - description: "Docker cache key" - value: ${{ github.sha }} - -jobs: - docker: - runs-on: ubuntu-latest - steps: - - name: Check out pipeline code - uses: actions/checkout@v3 - - - name: Pull and save docker - run: | - git grep 'ext.docker*' ${{ github.workspace }}/conf/modules.config | cut -f 2 -d '=' | xargs -L 2 echo | tr -d ' ' > ${{ runner.temp }}/images.txt - cat ${{ runner.temp }}/images.txt | xargs -I {} sh -c 'docker pull --platform linux/amd64 "$1"' - {} - mkdir -p ${{ runner.temp }}/docker/ - cat ${{ runner.temp }}/images.txt | xargs -I {} sh -c 'docker save "$1" > ${{ runner.temp }}/docker/$(basename "$1").tar' - {} - - - name: Save docker - id: cache-docker - uses: actions/cache@v3 - with: - path: ${{ runner.temp }}/docker - key: ${{ github.sha }} - - -