Add RD_USE_GHCR_IMAGES option to BATS to pull images from ghcr.io

This avoids hitting the pull rate limit when running multiple full
BATS runs in sequence, especially if not logged into Docker Hub.

Pull rate limit for docker.io is 100 pulls / 6 hours, or twice that
when authenticated.

Signed-off-by: Jan Dubois <jan.dubois@suse.com>

.github/actions/spelling/allow.txt

@@ -1,5 +1,6 @@
 github
 https
+skopeo
 ssh
 ubuntu
 workarounds

bats/Makefile

@@ -18,8 +18,10 @@ lint:
 	@./scripts/bats-lint.pl $(shell find tests -name '*.bats')
 	find tests -name '*.bash' | xargs shellcheck -s bash -e $(SC_EXCLUDES)
 	find tests -name '*.bats' | xargs shellcheck -s bash -e $(SC_EXCLUDES)
+	find scripts -name '*.sh' | xargs shellcheck -s bash -e $(SC_EXCLUDES)
 	find tests -name '*.bash' | xargs shfmt -s -d
 	find tests -name '*.bats' | xargs shfmt -s -d
+	find scripts -name '*.sh' | xargs shfmt -s -d
 
 DEPS = bin/darwin/jq bin/linux/jq
 

bats/scripts/ghcr-mirror.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Mirror Docker Hub images to ghcr.io to avoid pull limits during testing.
+
+# The script uses skopeo instead of docker pull/push because it needs to
+# copy all images of the repo, and not just the one for the current platform.
+#
+# Log into ghcr.io with a personal access token with write:packages scope:
+#   echo $PAT | skopeo login ghcr.io -u $USER --password-stdin
+#   echo $PASS | skopeo login docker.io -u $USER --password-stdin
+# Remove credentials:
+#   skopeo logout --all
+
+# TODO TODO TODO
+# The package visibility needs to be changed to "public".
+# I've not found any tool/API to do this from the commandline,
+# so I did this manually via the web UI.
+# At the very least we should check that the images are accessible
+# when logged out of ghcr.io.
+# TODO TODO TODO
+
+# TODO TODO TODO
+# Figure out a way to copy only the amd64 and arm64 images, but not the rest.
+# skopeo doesn't seem to support this yet without additional scripting to parse
+# the manifests. And then we would need to test if we can copy a "sparse" manifest
+# to ghcr.io when not all referenced images actually exist.
+# TODO TODO TODO
+
+set -o errexit -o nounset -o pipefail
+set +o xtrace
+
+if ! command -v skopeo >/dev/null; then
+    echo "This script requires the 'skopeo' utility to be installed"
+    exit 1
+fi
+
+source "$(dirname "${BASH_SOURCE[0]}")/../tests/helpers/images.bash"
+
+# IMAGES is setup by ../tests/helpers/images.bash
+# shellcheck disable=SC2153
+for IMAGE in "${IMAGES[@]}"; do
+    echo "===== Copying $IMAGE ====="
+    skopeo copy --all "docker://$IMAGE" "docker://$GHCR_REPO/$IMAGE"
+done

bats/tests/containers/allowed-images.bats

@@ -9,53 +9,53 @@ RD_USE_IMAGE_ALLOW_LIST=true
 }
 
 @test 'update the list of patterns first time' {
-    update_allowed_patterns true '"nginx", "busybox", "python"'
+    update_allowed_patterns true "$IMAGE_NGINX" "$IMAGE_BUSYBOX" "$IMAGE_PYTHON"
     wait_for_container_engine
 }
 
 @test 'verify pull nginx succeeds' {
-    ctrctl pull --quiet nginx
+    ctrctl pull --quiet "$IMAGE_NGINX"
 }
 
 @test 'verify pull busybox succeeds' {
-    ctrctl pull --quiet busybox
+    ctrctl pull --quiet "$IMAGE_BUSYBOX"
 }
 
 @test 'verify pull python succeeds' {
-    ctrctl pull --quiet python
+    ctrctl pull --quiet "$IMAGE_PYTHON"
 }
 
 @test 'verify pull ruby fails' {
-    run ctrctl pull ruby
+    run ctrctl pull "$IMAGE_RUBY"
     assert_failure
 }
 
 @test 'drop python from the allowed-image list, add ruby' {
-    update_allowed_patterns true '"nginx", "busybox", "ruby"'
+    update_allowed_patterns true "$IMAGE_NGINX" "$IMAGE_BUSYBOX" "$IMAGE_RUBY"
 }
 
 @test 'clear images' {
-    for image in nginx busybox python; do
-        ctrctl rmi "$image"
+    for image in IMAGE_NGINX IMAGE_BUSYBOX IMAGE_PYTHON; do
+        ctrctl rmi "${!image}"
     done
 }
 
 @test 'verify pull python fails' {
-    run ctrctl pull --quiet python
+    run ctrctl pull --quiet "$IMAGE_PYTHON"
     assert_failure
 }
 
 @test 'verify pull ruby succeeds' {
-    ctrctl pull --quiet ruby
+    ctrctl pull --quiet "$IMAGE_RUBY"
 }
 
 @test 'clear all patterns' {
-    update_allowed_patterns true ''
+    update_allowed_patterns true
 }
 
 @test 'can run kubectl' {
     wait_for_apiserver
-    kubectl run nginx --image=nginx:latest --port=8080
+    kubectl run nginx --image="${IMAGE_NGINX}:latest" --port=8080
 }
 
 verify_no_nginx() {
@@ -70,12 +70,12 @@ verify_no_nginx() {
 }
 
 @test 'set patterns with the allowed list disabled' {
-    update_allowed_patterns false '"nginx", "busybox", "ruby"'
+    update_allowed_patterns false "$IMAGE_NGINX" "$IMAGE_BUSYBOX" "$IMAGE_RUBY"
     # containerEngine.allowedImages.enabled changed, so wait for a restart
     wait_for_container_engine
     wait_for_apiserver "$RD_KUBERNETES_PREV_VERSION"
 }
 
 @test 'verify pull python succeeds because allowedImages filter is disabled' {
-    ctrctl pull --quiet python
+    ctrctl pull --quiet "$IMAGE_PYTHON"
 }

bats/tests/containers/catch-duplicate-api-patterns.bats

@@ -7,13 +7,13 @@ RD_USE_IMAGE_ALLOW_LIST=true
     wait_for_apiserver
     wait_for_container_engine
 
-    run update_allowed_patterns true '"nginx", "busybox", "ruby", "busybox"'
+    run update_allowed_patterns true "$IMAGE_NGINX" "$IMAGE_BUSYBOX" "$IMAGE_RUBY" "$IMAGE_BUSYBOX"
     assert_failure
-    assert_output --partial $'field \'containerEngine.allowedImages.patterns\' has duplicate entries: "busybox"'
+    assert_output --partial "field 'containerEngine.allowedImages.patterns' has duplicate entries: \"$IMAGE_BUSYBOX\""
 }
 
 @test 'catch attempts to add duplicate patterns via the API with enabled off' {
-    run update_allowed_patterns false '"nginx", "busybox", "ruby", "busybox"'
+    run update_allowed_patterns false "$IMAGE_NGINX" "$IMAGE_BUSYBOX" "$IMAGE_RUBY" "$IMAGE_BUSYBOX"
     assert_failure
-    assert_output --partial $'field \'containerEngine.allowedImages.patterns\' has duplicate entries: "busybox"'
+    assert_output --partial "field 'containerEngine.allowedImages.patterns' has duplicate entries: \"$IMAGE_BUSYBOX\""
 }

bats/tests/containers/platform.bats

@@ -14,13 +14,13 @@ check_uname() {
     local cpu="$2"
 
     # Pull container separately because `ctrctl run` doesn't have a --quiet option
-    ctrctl pull --quiet --platform "$platform" busybox
+    ctrctl pull --quiet --platform "$platform" "$IMAGE_BUSYBOX"
 
     # BUG BUG BUG
     # Adding -i option to work around a bug with the Linux docker CLI in WSL
     # https://github.com/rancher-sandbox/rancher-desktop/issues/3239
     # BUG BUG BUG
-    run ctrctl run -i --platform "$platform" busybox uname -m
+    run ctrctl run -i --platform "$platform" "$IMAGE_BUSYBOX" uname -m
     if is_true "${assert_success:-true}"; then
         assert_success
         assert_output "$cpu"
@@ -42,7 +42,7 @@ check_uname() {
 @test 'uninstall s390x emulator' {
     if is_windows; then
         # On WSL the emulator might still be installed from a previous run
-        ctrctl run --privileged --rm tonistiigi/binfmt --uninstall qemu-s390x
+        ctrctl run --privileged --rm "$IMAGE_TONISTIIGI_BINFMT" --uninstall qemu-s390x
     else
         skip "only required on Windows"
     fi
@@ -55,7 +55,7 @@ check_uname() {
 }
 
 @test 'install s390x emulator' {
-    ctrctl run --privileged --rm tonistiigi/binfmt --install s390x
+    ctrctl run --privileged --rm "$IMAGE_TONISTIIGI_BINFMT" --install s390x
 }
 
 @test 'deploy s390x container' {

bats/tests/containers/switch-engines.bats

@@ -11,11 +11,11 @@ switch_container_engine() {
 }
 
 pull_containers() {
-    ctrctl run -d -p 8085:80 --restart=no nginx
-    ctrctl run -d --restart=always busybox /bin/sh -c "sleep inf"
+    ctrctl run -d -p 8085:80 --restart=no "$IMAGE_NGINX"
+    ctrctl run -d --restart=always "$IMAGE_BUSYBOX" /bin/sh -c "sleep inf"
     run ctrctl ps --format '{{json .Image}}'
-    assert_output --partial nginx
-    assert_output --partial busybox
+    assert_output --partial "$IMAGE_NGINX"
+    assert_output --partial "$IMAGE_BUSYBOX"
 }
 
 @test 'factory reset' {
@@ -35,8 +35,8 @@ pull_containers() {
 
 verify_post_switch_containers() {
     run ctrctl ps --format '{{json .Image}}'
-    assert_output --partial "busybox"
-    refute_output --partial "nginx"
+    assert_output --partial "$IMAGE_BUSYBOX"
+    refute_output --partial "$IMAGE_NGINX"
 }
 
 switch_back_verify_post_switch_containers() {

bats/tests/helpers/defaults.bash

@@ -34,6 +34,17 @@ taking_screenshots() {
     is_true "$RD_TAKE_SCREENSHOTS"
 }
 
+########################################################################
+# When RD_USE_GHCR_IMAGES is true, then all images will be pulled from
+# ghcr.io instead of docker.io, to avoid hitting the docker hub pull
+# rate limit.
+
+: "${RD_USE_GHCR_IMAGES:=false}"
+
+using_ghcr_images() {
+    is_true "$RD_USE_GHCR_IMAGES"
+}
+
 ########################################################################
 : "${RD_USE_IMAGE_ALLOW_LIST:=false}"
 

bats/tests/helpers/images.bash

@@ -0,0 +1,24 @@
+# These images have been mirrored to ghcr.io (using bats/scripts/ghcr-mirror.sh)
+# to avoid hitting Docker Hub pull limits during testing.
+
+# TODO TODO TODO
+# The python image is huge (10GB across all platforms). We should either pin the
+# tag, or replace it with a different image for testing, so we don't have to mirror
+# the images to ghcr.io every time we run the mirror script.
+# TODO TODO TODO
+
+# Any time you add an image here you need to re-run the mirror script!
+IMAGES=(busybox nginx python ruby tonistiigi/binfmt registry:2.8.1)
+
+GHCR_REPO=ghcr.io/rancher-sandbox/bats
+
+# Create IMAGE_FOO_BAR=foo/bar:tag variables
+for IMAGE in "${IMAGES[@]}"; do
+    VAR="IMAGE_$(echo "$IMAGE" | sed 's/:.*//' | tr '[:lower:]' '[:upper:]' | tr / _)"
+    # file may be loaded outside BATS environment
+    if [ "$(type -t using_ghcr_images)" = "function" ] && using_ghcr_images; then
+        eval "$VAR=$GHCR_REPO/$IMAGE"
+    else
+        eval "$VAR=$IMAGE"
+    fi
+done

bats/tests/helpers/info.bash

@@ -37,5 +37,6 @@ show_info() { # @test
         echo "#"
         printf "$format" "Capturing logs:" "$(bool capturing_logs)"
         printf "$format" "Taking screenshots:" "$(bool taking_screenshots)"
+        printf "$format" "Using ghcr.io images:" "$(bool using_ghcr_images)"
     ) >&3
 }

bats/tests/helpers/load.bash

@@ -36,6 +36,9 @@ source "$PATH_BATS_HELPERS/utils.bash"
 # validate_enum() and is_true() from utils.bash.
 source "$PATH_BATS_HELPERS/defaults.bash"
 
+# images.bash uses using_ghcr_images() from defaults.bash
+source "$PATH_BATS_HELPERS/images.bash"
+
 # paths.bash uses RD_LOCATION from defaults.bash
 source "$PATH_BATS_HELPERS/paths.bash"
 

bats/tests/helpers/utils.bash

@@ -84,9 +84,32 @@ try() {
     return "$status"
 }
 
+image_without_tag() {
+    local image=$1
+    # If the tag looks like a port number and follows something that looks
+    # like a domain name, then don't strip the tag (e.g. foo.io:5000).
+    if [[ ${image##*:} =~ ^[0-9]+$ && ${image%:*} =~ \.[a-z]+$ ]]; then
+        echo "$image"
+    else
+        echo "${image%:*}"
+    fi
+}
+
 update_allowed_patterns() {
     local enabled=$1
-    local patterns=$2
+    shift
+
+    local patterns=""
+    local image
+    for image in "$@"; do
+        image=$(image_without_tag "$image")
+        if [ -z "$patterns" ]; then
+            patterns="\"${image}\""
+        else
+            patterns="$patterns, \"${image}\""
+        fi
+    done
+
     # TODO TODO TODO
     # Once https://github.com/rancher-sandbox/rancher-desktop/issues/4939 has been
     # implemented, the `version` field  should be made a constant. Putting in the

bats/tests/helpers/vm.bash

@@ -93,6 +93,10 @@ start_container_engine() {
     # TODO cannot be set from the commandline yet
     image_allow_list="$(bool using_image_allow_list)"
     wsl_integrations="{}"
+    registry="docker.io"
+    if using_ghcr_images; then
+        registry="ghcr.io"
+    fi
     if is_windows; then
         wsl_integrations="{\"$WSL_DISTRO_NAME\":true}"
     fi
@@ -104,7 +108,7 @@ start_container_engine() {
   "containerEngine": {
     "allowedImages": {
       "enabled": $image_allow_list,
-      "patterns": ["docker.io"]
+      "patterns": ["$registry"]
     }
   }
 }

bats/tests/k8s/up-downgrade-k8s.bats

@@ -16,18 +16,18 @@ ARCH_FOR_KUBERLR=amd64
 }
 
 @test 'deploy nginx - always restart' {
-    ctrctl pull nginx
-    run ctrctl run -d -p 8585:80 --restart=always --name nginx-restart nginx
+    ctrctl pull "$IMAGE_NGINX"
+    run ctrctl run -d -p 8585:80 --restart=always --name nginx-restart "$IMAGE_NGINX"
     assert_success
 }
 
 @test 'deploy nginx - no restart' {
-    run ctrctl run -d -p 8686:80 --restart=no --name nginx-no-restart nginx
+    run ctrctl run -d -p 8686:80 --restart=no --name nginx-no-restart "$IMAGE_NGINX"
     assert_success
 }
 
 @test 'deploy busybox' {
-    run kubectl create deploy busybox --image=busybox --replicas=2 -- /bin/sh -c "sleep inf"
+    run kubectl create deploy busybox --image="$IMAGE_BUSYBOX" --replicas=2 -- /bin/sh -c "sleep inf"
     assert_success
 }
 
@@ -57,12 +57,12 @@ verify_busybox() {
 verify_images() {
     if using_docker; then
         run docker images
-        assert_output --partial "nginx" "busybox"
+        assert_output --partial "$IMAGE_NGINX" "$IMAGE_BUSYBOX"
     else
         run nerdctl images --format json
-        assert_output --partial '"Repository":"nginx'
+        assert_output --partial "\"Repository\":\"$IMAGE_NGINX"
         run nerdctl --namespace k8s.io images
-        assert_output --partial "busybox"
+        assert_output --partial "$IMAGE_BUSYBOX"
     fi
 }
 @test 'verify images before upgrade' {