Singularity demo

conda_smithy/configure_feedstock.py

@@ -1704,6 +1704,11 @@ def _load_forge_config(forge_dir, exclusive_config_file, forge_yml=None):
             "fallback_image": "quay.io/condaforge/linux-anvil-comp7",
             "command": "bash",
         },
+        "singularity": {
+             "executable": "singularity",
+             "fallback_image": "quay.io/condaforge/linux-anvil-comp7",
+             "command": "bash",
+         },
         "templates": {},
         "drone": {},
         "woodpecker": {},

conda_smithy/templates/build_steps.sh.tmpl

@@ -30,6 +30,23 @@ pkgs_dirs:
   - /opt/conda/pkgs
 
 CONDARC
+
+if [ ${SINGULARITY_BUILD:-"False"} == "True" ]; then
+    export PATH="$PATH":/opt/conda/condabin
+    env > envvar
+    # mamba init
+    # source ~/.bashrc
+    eval "$(/opt/conda/bin/conda shell.bash hook)"
+    conda create --prefix=/tmp/building --yes --quiet
+    conda activate /tmp/building
+    # conda clean --packages --yes --quiet
+    sed 's/^/"/;s/$/"/' envvar > envvarmod
+    sed 's/^/export /' envvarmod > envvarmodmod
+    source ./envvarmodmod
+    conda activate /tmp/building
+fi
+
+
 {% if build_with_mambabuild %}
 {%- set CONDA_INSTALL_CMD="mamba" %}
 {%- set BOA="boa " %}

conda_smithy/templates/run_docker_build.sh.tmpl

@@ -21,16 +21,20 @@ if [ -z ${FEEDSTOCK_NAME} ]; then
     export FEEDSTOCK_NAME=$(basename ${FEEDSTOCK_ROOT})
 fi
 
-docker info
-
-# In order for the conda-build process in the container to write to the mounted
-# volumes, we need to run with the same id as the host machine, which is
-# normally the owner of the mounted volumes, or at least has write permission
-export HOST_USER_ID=$(id -u)
-# Check if docker-machine is being used (normally on OSX) and get the uid from
-# the VM
-if hash docker-machine 2> /dev/null && docker-machine active > /dev/null; then
-    export HOST_USER_ID=$(docker-machine ssh $(docker-machine active) id -u)
+if [ ${SINGULARITY_BUILD} == "True" ]; then
+    echo "We are using Singularity!"
+else
+    docker info
+
+    # In order for the conda-build process in the container to write to the mounted
+    # volumes, we need to run with the same id as the host machine, which is
+    # normally the owner of the mounted volumes, or at least has write permission
+    export HOST_USER_ID=$(id -u)
+    # Check if docker-machine is being used (normally on OSX) and get the uid from
+    # the VM
+    if hash docker-machine 2> /dev/null && docker-machine active > /dev/null; then
+        export HOST_USER_ID=$(docker-machine ssh $(docker-machine active) id -u)
+    fi
 fi
 
 ARTIFACTS="$FEEDSTOCK_ROOT/build_artifacts"
@@ -64,39 +68,76 @@ mkdir -p "$ARTIFACTS"
 DONE_CANARY="$ARTIFACTS/conda-forge-build-done-${CONFIG}"
 rm -f "$DONE_CANARY"
 
-# Allow people to specify extra default arguments to `docker run` (e.g. `--rm`)
-DOCKER_RUN_ARGS="${CONDA_FORGE_DOCKER_RUN_ARGS}"
-if [ -z "${CI}" ]; then
-    DOCKER_RUN_ARGS="-it ${DOCKER_RUN_ARGS}"
-fi
 
-( endgroup "Configure Docker" ) 2> /dev/null
-
-( startgroup "Start Docker" ) 2> /dev/null
-
-export UPLOAD_PACKAGES="${UPLOAD_PACKAGES:-True}"
-export IS_PR_BUILD="${IS_PR_BUILD:-False}"
-{{ docker.executable }} pull "${DOCKER_IMAGE}"
-{{ docker.executable }} run ${DOCKER_RUN_ARGS} \
-           -v "${RECIPE_ROOT}":/home/conda/recipe_root:rw,z,delegated \
-           -v "${FEEDSTOCK_ROOT}":/home/conda/feedstock_root:rw,z,delegated \
-           -e CONFIG \
-           -e HOST_USER_ID \
-           -e UPLOAD_PACKAGES \
-           -e IS_PR_BUILD \
-           -e GIT_BRANCH \
-           -e UPLOAD_ON_BRANCH \
-           -e CI \
-           -e FEEDSTOCK_NAME \
-           -e CPU_COUNT \
-           -e BUILD_WITH_CONDA_DEBUG \
-           -e BUILD_OUTPUT_ID \
-{%- for secret in secrets %}
-           -e {{ secret }} \
-{%- endfor %}
-           "${DOCKER_IMAGE}" \
-           {{ docker.command }} \
-           "/home/conda/feedstock_root/${PROVIDER_DIR}/build_steps.sh"
+if [ ${SINGULARITY_BUILD:-"False"} == "True" ]; then
+    DOCKER_RUN_ARGS="${CONDA_FORGE_DOCKER_RUN_ARGS}"
+
+    # ( endgroup "Configure Singularity" ) 2> /dev/null
+
+    # ( startgroup "Start Singularity" ) 2> /dev/null
+
+    export UPLOAD_PACKAGES="${UPLOAD_PACKAGES:-True}"
+    export IS_PR_BUILD="${IS_PR_BUILD:-False}"
+    {{ singularity.executable }} pull -F singularity_image.sif docker://"${DOCKER_IMAGE}"
+    {{ singularity.executable }} exec ${DOCKER_RUN_ARGS} \
+                --writable-tmpfs -C \
+                --bind "${RECIPE_ROOT}":/home/conda/recipe_root \
+                --bind "${FEEDSTOCK_ROOT}":/home/conda/feedstock_root \
+                --bind /tmp:/tmp \
+                --bind /tmp:/home/$USER \
+                --env SINGULARITY_BUILD="${SINGULARITY_BUILD}" \
+                --env CONFIG="${CONFIG}" \
+                --env HOST_USER_ID="${HOST_USER_ID}" \
+                --env UPLOAD_PACKAGES="${UPLOAD_PACKAGES}"  \
+                --env IS_PR_BUILD="${IS_PR_BUILD}"  \
+                --env GIT_BRANCH="${GIT_BRANCH}"  \
+                --env UPLOAD_ON_BRANCH="${UPLOAD_ON_BRANCH}"  \
+                --env CI="${CI}"  \
+                --env FEEDSTOCK_NAME="${FEEDSTOCK_NAME}"  \
+                --env CPU_COUNT="${CPU_COUNT}"  \
+                --env BUILD_WITH_CONDA_DEBUG="${BUILD_WITH_CONDA_DEBUG}"  \
+                --env BUILD_OUTPUT_ID="${BUILD_OUTPUT_ID}"  \
+    {%- for secret in secrets %}
+                --env secret="{{ secret }}" \
+    {%- endfor %}
+            singularity_image.sif \
+            {{ singularity.command }} \
+            "/home/conda/feedstock_root/${PROVIDER_DIR}/build_steps.sh"
+else
+    # Allow people to specify extra default arguments to `docker run` (e.g. `--rm`)
+    DOCKER_RUN_ARGS="${CONDA_FORGE_DOCKER_RUN_ARGS}"
+    if [ -z "${CI}" ]; then
+        DOCKER_RUN_ARGS="-it ${DOCKER_RUN_ARGS}"
+    fi
+
+    ( endgroup "Configure Docker" ) 2> /dev/null
+
+    ( startgroup "Start Docker" ) 2> /dev/null
+
+    export UPLOAD_PACKAGES="${UPLOAD_PACKAGES:-True}"
+    export IS_PR_BUILD="${IS_PR_BUILD:-False}"
+    {{ docker.executable }} pull "${DOCKER_IMAGE}"
+    {{ docker.executable }} run ${DOCKER_RUN_ARGS} \
+            -v "${RECIPE_ROOT}":/home/conda/recipe_root:rw,z,delegated \
+            -v "${FEEDSTOCK_ROOT}":/home/conda/feedstock_root:rw,z,delegated \
+            -e CONFIG \
+            -e HOST_USER_ID \
+            -e UPLOAD_PACKAGES \
+            -e IS_PR_BUILD \
+            -e GIT_BRANCH \
+            -e UPLOAD_ON_BRANCH \
+            -e CI \
+            -e FEEDSTOCK_NAME \
+            -e CPU_COUNT \
+            -e BUILD_WITH_CONDA_DEBUG \
+            -e BUILD_OUTPUT_ID \
+    {%- for secret in secrets %}
+            -e {{ secret }} \
+    {%- endfor %}
+            "${DOCKER_IMAGE}" \
+            {{ docker.command }} \
+            "/home/conda/feedstock_root/${PROVIDER_DIR}/build_steps.sh"
+fi
 
 # verify that the end of the script was reached
 test -f "$DONE_CANARY"