Merge branch 'master' of github.com:bioconda/bioconda-recipes

.circleci/check-for-additional-platforms.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+# Check to see if any changed recipes have specified the key
+# extra:additional-platforms, and if so, if they match the platform of the
+# currently-running machine.
+
+# arguments
+git_range=$1
+job_name=$2
+
+# Download ARM version of yq
+yq_platform=$(uname)
+yq_arch=$(uname -m)
+[[ $yq_arch = "aarch64" ]] && yq_arch="arm64"
+mkdir -p ${HOME}/bin
+wget https://github.com/mikefarah/yq/releases/latest/download/yq_${yq_platform}_${yq_arch} -O ${HOME}/bin/yq
+chmod +x ${HOME}/bin/yq
+
+# Find recipes changed from this merge
+files=`git diff --name-only --diff-filter AMR ${git_range} | grep -E 'meta.yaml$' `
+build=0
+
+for file in $files; do
+    echo $file
+    # To create a properly-formatted yaml that yq can parse, comment out jinja2
+    # variable setting with {% %} and remove variable use with {{ }}.
+    additional_platforms=$(cat $file \
+    | sed -E 's/(.*)\{%(.*)%\}(.*)/# \1\2\3/g' \
+    | tr -d '{{' | tr -d '}}' \
+    | ${HOME}/bin/yq '.extra.additional-platforms[]')
+
+    parsing_status=$?
+    if [ $parsing_status -gt 0 ]; then
+        echo "An error occurred while reading/parsing ${file}"
+        exit $parsing_status
+    fi
+
+    # Check if any additional platforms match this job
+    for additional_platform in $additional_platforms; do
+    if [ "${CIRCLE_JOB}" = "${job_name}-${additional_platform}" ]
+    then
+        build=1
+        break
+    fi
+    done
+done
+
+# If no changed recipes apply to this platform, skip remaining steps
+if [[ build -lt 1 ]]
+then
+    echo "No recipes using this platform, skipping rest of job."
+    circleci-agent step halt
+fi

.circleci/config.yml

@@ -0,0 +1,244 @@
+# .circleci/config.yml
+version: 2.1
+
+executors:
+  # osx-arm64:
+  #   macos:
+  #     xcode: 14.2.0 # indicate your selected version of Xcode
+  #   resource_class: macos.m1.large.gen1
+  linux-aarch64:
+    machine:
+      image: ubuntu-2204:current
+    resource_class: arm.medium
+
+jobs: # a basic unit of work in a run
+  build_and_test:
+    parameters:
+      os:
+        type: executor
+    executor: << parameters.os >>
+    steps:
+      - checkout
+
+      - run:
+          name: Check for Additional Platforms
+          command: ./.circleci/check-for-additional-platforms.sh "origin/master...HEAD" "build_and_test"
+
+      - run:
+          name: Fetch bioconda install script
+          command: wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{common,install-and-set-up-conda,configure-conda}.sh
+
+      - run:
+          name: Install bioconda-utils
+          command: |
+            sudo mkdir -p /opt/
+            sudo chmod o+rwx /opt
+            bash install-and-set-up-conda.sh
+
+      - run:
+          name: Setup PATH
+          command:
+            echo 'export PATH=/opt/mambaforge/bin:"$PATH"' >> "$BASH_ENV"
+
+      - run:
+          name: Configure conda
+          command: bash configure-conda.sh
+
+      - run:
+          name: Build and test
+          command: |
+            . common.sh
+            source /opt/mambaforge/etc/profile.d/conda.sh
+            source /opt/mambaforge/etc/profile.d/mamba.sh
+            mamba activate bioconda
+            bioconda-utils build recipes config.yml \
+                --lint --docker --mulled-test \
+                --docker-base-image "quay.io/bioconda/bioconda-utils-build-env-cos7-$(arch):${BIOCONDA_UTILS_TAG#v}" \
+                --git-range origin/master HEAD
+
+      - run:
+          name: Prepare artifacts
+          command: |
+            (
+            mkdir -p /tmp/artifacts/packages
+            cd /opt/mambaforge/envs/bioconda/conda-bld || exit 0
+            find -name .cache | xargs rm -rf || true
+            for n in index.html channeldata.json linux-aarch64 linux-64 osx-64 noarch; do
+              cp -rv $n /tmp/artifacts/packages || true
+            done
+            if command -V docker >/dev/null; then
+              mkdir -p /tmp/artifacts/images
+              cd /tmp/artifacts/images
+              docker image ls --format='{{.Repository}}:{{.Tag}}' | \
+                { grep biocontainers || true ; } | \
+                xargs -n1 -P4 bash -c '
+                  test -n "${1+x}" || exit 0
+                  echo "Start compressing docker image ${1} ..."
+                  docker save "${1}" | gzip -c > "${1##*/}.tar.gz"
+                  echo "Finished compressing docker image ${1} ."
+                  ' --
+            fi
+            ) || true
+
+      - store_artifacts:
+          path: /tmp/artifacts
+
+  build_and_upload:
+    parameters:
+      os:
+        type: executor
+    executor: << parameters.os >>
+    steps:
+      - checkout
+
+      - run:
+          name: Check for Additional Platforms
+          command: ./.circleci/check-for-additional-platforms.sh "${CIRCLE_SHA1}~1 ${CIRCLE_SHA1}" "build_and_upload"
+
+      - run:
+          name: Fetch bioconda install script
+          command: wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/{common,install-and-set-up-conda,configure-conda}.sh
+
+      - run:
+          name: Install bioconda-utils
+          command: |
+            sudo mkdir -p /opt
+            sudo chmod o+rwx /opt
+            bash install-and-set-up-conda.sh
+
+      - run:
+          name: Setup PATH
+          command:
+            echo 'export PATH=/opt/mambaforge/bin:"$PATH"' >> "$BASH_ENV"
+
+      - run:
+          name: Configure conda
+          command: bash configure-conda.sh
+
+      - run:
+          name: Build and push
+          command: |
+            source /opt/mambaforge/etc/profile.d/conda.sh
+            source /opt/mambaforge/etc/profile.d/mamba.sh
+            mamba activate bioconda
+            bioconda-utils handle-merged-pr recipes config.yml \
+              --repo bioconda/bioconda-recipes \
+              --git-range ${CIRCLE_SHA1}~1 ${CIRCLE_SHA1} \
+              --fallback build \
+              --artifact-source circleci
+
+  bulk_build:
+    parameters:
+      os:
+        type: executor
+      runner:
+        type: integer
+    executor: << parameters.os >>
+    steps:
+      - add_ssh_keys:
+          fingerprints:
+            - 1e:85:74:42:35:5f:c5:a2:35:c2:ec:b7:80:c0:7c:40
+
+      - checkout
+
+      - run:
+          name: Check for [ci run]
+          command: |
+            commit_message="$(git log --format=oneline -n 1 $CIRCLE_SHA1)"
+            if [[ $commit_message  =~ "[ci run]" ]]; then
+              echo "[ci run] found, continuing."
+            else
+              echo "[ci run] not found, exiting."
+              circleci-agent step halt
+            fi
+
+      - run:
+          name: set git user
+          command: |
+            git config user.name BiocondaBot
+            git config user.email biocondabot@users.noreply.github.com
+            git branch --set-upstream-to=origin/$CIRCLE_BRANCH $CIRCLE_BRANCH
+
+      - run:
+          name: Fetch bioconda install script
+          command: wget https://raw.githubusercontent.com/bioconda/bioconda-common/bulk/{common,install-and-set-up-conda,configure-conda}.sh
+
+      - run:
+          name: Install bioconda-utils
+          command: |
+            sudo mkdir -p /opt/
+            sudo chmod o+rwx /opt
+            bash install-and-set-up-conda.sh
+
+      - run:
+          name: Setup PATH
+          command:
+            echo 'export PATH=/opt/mambaforge/bin:"$PATH"' >> "$BASH_ENV"
+
+      - run:
+          name: Configure conda
+          command: bash configure-conda.sh
+
+      # For now, do not upload ARM images to biocontainers: --mulled-upload-target biocontainers
+      - run:
+          name: Build and upload
+          command: |
+            set -e
+            . common.sh
+            source /opt/mambaforge/etc/profile.d/conda.sh
+            source /opt/mambaforge/etc/profile.d/mamba.sh
+            mamba activate bioconda
+            echo '============'
+            conda info --all
+            conda config --show-sources
+            python -c 'import bioconda_utils.utils as u ; import pathlib as p ; print(*(f"{f}:\n{p.Path(f).read_text()}" for f in u.load_conda_build_config().exclusive_config_files), sep="\n")'
+            echo '============'
+            bioconda-utils build recipes config.yml \
+              --worker-offset << parameters.runner >> --n-workers 6 \
+              --docker --mulled-test --docker-base-image "quay.io/bioconda/bioconda-utils-build-env-cos7-$(arch):${BIOCONDA_UTILS_TAG#v}" \
+              --anaconda-upload \
+              --record-build-failures \
+              --skiplist-leafs
+            conda clean -y --all
+
+
+workflows:
+  build and test (ARM):
+    jobs:
+      - build_and_test:
+          filters:
+            branches:
+              ignore:
+                - master
+                - bulk
+          matrix:
+            parameters:
+              os:
+                #- osx-arm64
+                - linux-aarch64
+
+  build and upload (ARM):
+    jobs:
+      - build_and_upload:
+          filters:
+            branches:
+              only: master
+          matrix:
+            parameters:
+              os:
+                #- osx-arm64
+                - linux-aarch64
+
+
+  Bulk branch (ARM):
+    jobs:
+      - bulk_build:
+          filters:
+            branches:
+              only: bulk
+          matrix:
+            parameters:
+              os: 
+                #- osx-arm64
+                - linux-aarch64
+              runner: [0, 1, 2, 3, 4, 5]