Merge pull request #4917 from jandubois/bats-ghcr

Add RD_USE_GHCR_IMAGES option to BATS to pull images from ghcr.io

.github/actions/spelling/allow.txt

@@ -1,5 +1,6 @@
 github
 https
+skopeo
 ssh
 ubuntu
 workarounds

bats/Makefile

@@ -18,8 +18,10 @@ lint:
 	@./scripts/bats-lint.pl $(shell find tests -name '*.bats')
 	find tests -name '*.bash' | xargs shellcheck -s bash -e $(SC_EXCLUDES)
 	find tests -name '*.bats' | xargs shellcheck -s bash -e $(SC_EXCLUDES)
+	find scripts -name '*.sh' | xargs shellcheck -s bash -e $(SC_EXCLUDES)
 	find tests -name '*.bash' | xargs shfmt -s -d
 	find tests -name '*.bats' | xargs shfmt -s -d
+	find scripts -name '*.sh' | xargs shfmt -s -d
 
 DEPS = bin/darwin/jq bin/linux/jq
 

bats/scripts/ghcr-mirror.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Mirror Docker Hub images to ghcr.io to avoid pull limits during testing.
+
+# The script uses skopeo instead of docker pull/push because it needs to
+# copy all images of the repo, and not just the one for the current platform.
+#
+# Log into ghcr.io with a personal access token with write:packages scope:
+#   echo $PAT | skopeo login ghcr.io -u $USER --password-stdin
+#   echo $PASS | skopeo login docker.io -u $USER --password-stdin
+# Remove credentials:
+#   skopeo logout --all
+
+# TODO TODO TODO
+# The package visibility needs to be changed to "public".
+# I've not found any tool/API to do this from the commandline,
+# so I did this manually via the web UI.
+# At the very least we should check that the images are accessible
+# when logged out of ghcr.io.
+# TODO TODO TODO
+
+# TODO TODO TODO
+# Figure out a way to copy only the amd64 and arm64 images, but not the rest.
+# skopeo doesn't seem to support this yet without additional scripting to parse
+# the manifests. And then we would need to test if we can copy a "sparse" manifest
+# to ghcr.io when not all referenced images actually exist.
+# TODO TODO TODO
+
+set -o errexit -o nounset -o pipefail
+set +o xtrace
+
+if ! command -v skopeo >/dev/null; then
+    echo "This script requires the 'skopeo' utility to be installed"
+    exit 1
+fi
+
+source "$(dirname "${BASH_SOURCE[0]}")/../tests/helpers/images.bash"
+
+# IMAGES is setup by ../tests/helpers/images.bash
+# shellcheck disable=SC2153
+for IMAGE in "${IMAGES[@]}"; do
+    echo "===== Copying $IMAGE ====="
+    skopeo copy --all "docker://$IMAGE" "docker://$GHCR_REPO/$IMAGE"
+done

bats/tests/containers/allowed-images.bats

@@ -9,53 +9,53 @@ RD_USE_IMAGE_ALLOW_LIST=true
 }
 
 @test 'update the list of patterns first time' {
-    update_allowed_patterns true '"nginx", "busybox", "python"'
+    update_allowed_patterns true "$IMAGE_NGINX" "$IMAGE_BUSYBOX" "$IMAGE_PYTHON"
     wait_for_container_engine
 }
 
 @test 'verify pull nginx succeeds' {
-    ctrctl pull --quiet nginx
+    ctrctl pull --quiet "$IMAGE_NGINX"
 }
 
 @test 'verify pull busybox succeeds' {
-    ctrctl pull --quiet busybox
+    ctrctl pull --quiet "$IMAGE_BUSYBOX"
 }
 
 @test 'verify pull python succeeds' {
-    ctrctl pull --quiet python
+    ctrctl pull --quiet "$IMAGE_PYTHON"
 }
 
 @test 'verify pull ruby fails' {
-    run ctrctl pull ruby
+    run ctrctl pull "$IMAGE_RUBY"
     assert_failure
 }
 
 @test 'drop python from the allowed-image list, add ruby' {
-    update_allowed_patterns true '"nginx", "busybox", "ruby"'
+    update_allowed_patterns true "$IMAGE_NGINX" "$IMAGE_BUSYBOX" "$IMAGE_RUBY"
 }
 
 @test 'clear images' {
-    for image in nginx busybox python; do
-        ctrctl rmi "$image"
+    for image in IMAGE_NGINX IMAGE_BUSYBOX IMAGE_PYTHON; do
+        ctrctl rmi "${!image}"
     done
 }
 
 @test 'verify pull python fails' {
-    run ctrctl pull --quiet python
+    run ctrctl pull --quiet "$IMAGE_PYTHON"
     assert_failure
 }
 
 @test 'verify pull ruby succeeds' {
-    ctrctl pull --quiet ruby
+    ctrctl pull --quiet "$IMAGE_RUBY"
 }
 
 @test 'clear all patterns' {
-    update_allowed_patterns true ''
+    update_allowed_patterns true
 }
 
 @test 'can run kubectl' {
     wait_for_apiserver
-    kubectl run nginx --image=nginx:latest --port=8080
+    kubectl run nginx --image="${IMAGE_NGINX}:latest" --port=8080
 }
 
 verify_no_nginx() {
@@ -70,12 +70,12 @@ verify_no_nginx() {
 }
 
 @test 'set patterns with the allowed list disabled' {
-    update_allowed_patterns false '"nginx", "busybox", "ruby"'
+    update_allowed_patterns false "$IMAGE_NGINX" "$IMAGE_BUSYBOX" "$IMAGE_RUBY"
     # containerEngine.allowedImages.enabled changed, so wait for a restart
     wait_for_container_engine
     wait_for_apiserver "$RD_KUBERNETES_PREV_VERSION"
 }
 
 @test 'verify pull python succeeds because allowedImages filter is disabled' {
-    ctrctl pull --quiet python
+    ctrctl pull --quiet "$IMAGE_PYTHON"
 }

bats/tests/containers/catch-duplicate-api-patterns.bats

@@ -7,13 +7,13 @@ RD_USE_IMAGE_ALLOW_LIST=true
     wait_for_apiserver
     wait_for_container_engine
 
-    run update_allowed_patterns true '"nginx", "busybox", "ruby", "busybox"'
+    run update_allowed_patterns true "$IMAGE_NGINX" "$IMAGE_BUSYBOX" "$IMAGE_RUBY" "$IMAGE_BUSYBOX"
     assert_failure
-    assert_output --partial $'field \'containerEngine.allowedImages.patterns\' has duplicate entries: "busybox"'
+    assert_output --partial "field 'containerEngine.allowedImages.patterns' has duplicate entries: \"$IMAGE_BUSYBOX\""
 }
 
 @test 'catch attempts to add duplicate patterns via the API with enabled off' {
-    run update_allowed_patterns false '"nginx", "busybox", "ruby", "busybox"'
+    run update_allowed_patterns false "$IMAGE_NGINX" "$IMAGE_BUSYBOX" "$IMAGE_RUBY" "$IMAGE_BUSYBOX"
     assert_failure
-    assert_output --partial $'field \'containerEngine.allowedImages.patterns\' has duplicate entries: "busybox"'
+    assert_output --partial "field 'containerEngine.allowedImages.patterns' has duplicate entries: \"$IMAGE_BUSYBOX\""
 }

bats/tests/containers/platform.bats

@@ -14,13 +14,13 @@ check_uname() {
     local cpu="$2"
 
     # Pull container separately because `ctrctl run` doesn't have a --quiet option
-    ctrctl pull --quiet --platform "$platform" busybox
+    ctrctl pull --quiet --platform "$platform" "$IMAGE_BUSYBOX"
 
     # BUG BUG BUG
     # Adding -i option to work around a bug with the Linux docker CLI in WSL
     # https://github.com/rancher-sandbox/rancher-desktop/issues/3239
     # BUG BUG BUG
-    run ctrctl run -i --platform "$platform" busybox uname -m
+    run ctrctl run -i --platform "$platform" "$IMAGE_BUSYBOX" uname -m
     if is_true "${assert_success:-true}"; then
         assert_success
         assert_output "$cpu"
@@ -42,7 +42,7 @@ check_uname() {
 @test 'uninstall s390x emulator' {
     if is_windows; then
         # On WSL the emulator might still be installed from a previous run
-        ctrctl run --privileged --rm tonistiigi/binfmt --uninstall qemu-s390x
+        ctrctl run --privileged --rm "$IMAGE_TONISTIIGI_BINFMT" --uninstall qemu-s390x
     else
         skip "only required on Windows"
     fi
@@ -55,7 +55,7 @@ check_uname() {
 }
 
 @test 'install s390x emulator' {
-    ctrctl run --privileged --rm tonistiigi/binfmt --install s390x
+    ctrctl run --privileged --rm "$IMAGE_TONISTIIGI_BINFMT" --install s390x
 }
 
 @test 'deploy s390x container' {

bats/tests/containers/switch-engines.bats

@@ -11,11 +11,11 @@ switch_container_engine() {
 }
 
 pull_containers() {
-    ctrctl run -d -p 8085:80 --restart=no nginx
-    ctrctl run -d --restart=always busybox /bin/sh -c "sleep inf"
+    ctrctl run -d -p 8085:80 --restart=no "$IMAGE_NGINX"
+    ctrctl run -d --restart=always "$IMAGE_BUSYBOX" /bin/sh -c "sleep inf"
     run ctrctl ps --format '{{json .Image}}'
-    assert_output --partial nginx
-    assert_output --partial busybox
+    assert_output --partial "$IMAGE_NGINX"
+    assert_output --partial "$IMAGE_BUSYBOX"
 }
 
 @test 'factory reset' {
@@ -35,8 +35,8 @@ pull_containers() {
 
 verify_post_switch_containers() {
     run ctrctl ps --format '{{json .Image}}'
-    assert_output --partial "busybox"
-    refute_output --partial "nginx"
+    assert_output --partial "$IMAGE_BUSYBOX"
+    refute_output --partial "$IMAGE_NGINX"
 }
 
 switch_back_verify_post_switch_containers() {

bats/tests/helpers/defaults.bash

@@ -34,6 +34,17 @@ taking_screenshots() {
     is_true "$RD_TAKE_SCREENSHOTS"
 }
 
+########################################################################
+# When RD_USE_GHCR_IMAGES is true, then all images will be pulled from
+# ghcr.io instead of docker.io, to avoid hitting the docker hub pull
+# rate limit.
+
+: "${RD_USE_GHCR_IMAGES:=false}"
+
+using_ghcr_images() {
+    is_true "$RD_USE_GHCR_IMAGES"
+}
+
 ########################################################################
 : "${RD_USE_IMAGE_ALLOW_LIST:=false}"
 

bats/tests/helpers/images.bash

@@ -0,0 +1,24 @@
+# These images have been mirrored to ghcr.io (using bats/scripts/ghcr-mirror.sh)
+# to avoid hitting Docker Hub pull limits during testing.
+
+# TODO TODO TODO
+# The python image is huge (10GB across all platforms). We should either pin the
+# tag, or replace it with a different image for testing, so we don't have to mirror
+# the images to ghcr.io every time we run the mirror script.
+# TODO TODO TODO
+
+# Any time you add an image here you need to re-run the mirror script!
+IMAGES=(busybox nginx python ruby tonistiigi/binfmt registry:2.8.1)
+
+GHCR_REPO=ghcr.io/rancher-sandbox/bats
+
+# Create IMAGE_FOO_BAR=foo/bar:tag variables
+for IMAGE in "${IMAGES[@]}"; do
+    VAR="IMAGE_$(echo "$IMAGE" | sed 's/:.*//' | tr '[:lower:]' '[:upper:]' | tr / _)"
+    # file may be loaded outside BATS environment
+    if [ "$(type -t using_ghcr_images)" = "function" ] && using_ghcr_images; then
+        eval "$VAR=$GHCR_REPO/$IMAGE"
+    else
+        eval "$VAR=$IMAGE"
+    fi
+done

bats/tests/helpers/info.bash

@@ -37,5 +37,6 @@ show_info() { # @test
         echo "#"
         printf "$format" "Capturing logs:" "$(bool capturing_logs)"
         printf "$format" "Taking screenshots:" "$(bool taking_screenshots)"
+        printf "$format" "Using ghcr.io images:" "$(bool using_ghcr_images)"
     ) >&3
 }

bats/tests/helpers/load.bash

@@ -36,6 +36,9 @@ source "$PATH_BATS_HELPERS/utils.bash"
 # validate_enum() and is_true() from utils.bash.
 source "$PATH_BATS_HELPERS/defaults.bash"
 
+# images.bash uses using_ghcr_images() from defaults.bash
+source "$PATH_BATS_HELPERS/images.bash"
+
 # paths.bash uses RD_LOCATION from defaults.bash
 source "$PATH_BATS_HELPERS/paths.bash"
 

bats/tests/helpers/utils.bash

@@ -84,9 +84,32 @@ try() {
     return "$status"
 }
 
+image_without_tag() {
+    local image=$1
+    # If the tag looks like a port number and follows something that looks
+    # like a domain name, then don't strip the tag (e.g. foo.io:5000).
+    if [[ ${image##*:} =~ ^[0-9]+$ && ${image%:*} =~ \.[a-z]+$ ]]; then
+        echo "$image"
+    else
+        echo "${image%:*}"
+    fi
+}
+
 update_allowed_patterns() {
     local enabled=$1
-    local patterns=$2
+    shift
+
+    local patterns=""
+    local image
+    for image in "$@"; do
+        image=$(image_without_tag "$image")
+        if [ -z "$patterns" ]; then
+            patterns="\"${image}\""
+        else
+            patterns="$patterns, \"${image}\""
+        fi
+    done
+
     # TODO TODO TODO
     # Once https://github.com/rancher-sandbox/rancher-desktop/issues/4939 has been
     # implemented, the `version` field  should be made a constant. Putting in the

bats/tests/helpers/vm.bash

@@ -93,6 +93,10 @@ start_container_engine() {
     # TODO cannot be set from the commandline yet
     image_allow_list="$(bool using_image_allow_list)"
     wsl_integrations="{}"
+    registry="docker.io"
+    if using_ghcr_images; then
+        registry="ghcr.io"
+    fi
     if is_windows; then
         wsl_integrations="{\"$WSL_DISTRO_NAME\":true}"
     fi
@@ -104,7 +108,7 @@ start_container_engine() {
   "containerEngine": {
     "allowedImages": {
       "enabled": $image_allow_list,
-      "patterns": ["docker.io"]
+      "patterns": ["$registry"]
     }
   }
 }

bats/tests/k8s/up-downgrade-k8s.bats

@@ -16,18 +16,18 @@ ARCH_FOR_KUBERLR=amd64
 }
 
 @test 'deploy nginx - always restart' {
-    ctrctl pull nginx
-    run ctrctl run -d -p 8585:80 --restart=always --name nginx-restart nginx
+    ctrctl pull "$IMAGE_NGINX"
+    run ctrctl run -d -p 8585:80 --restart=always --name nginx-restart "$IMAGE_NGINX"
     assert_success
 }
 
 @test 'deploy nginx - no restart' {
-    run ctrctl run -d -p 8686:80 --restart=no --name nginx-no-restart nginx
+    run ctrctl run -d -p 8686:80 --restart=no --name nginx-no-restart "$IMAGE_NGINX"
     assert_success
 }
 
 @test 'deploy busybox' {
-    run kubectl create deploy busybox --image=busybox --replicas=2 -- /bin/sh -c "sleep inf"
+    run kubectl create deploy busybox --image="$IMAGE_BUSYBOX" --replicas=2 -- /bin/sh -c "sleep inf"
     assert_success
 }
 
@@ -57,12 +57,12 @@ verify_busybox() {
 verify_images() {
     if using_docker; then
         run docker images
-        assert_output --partial "nginx" "busybox"
+        assert_output --partial "$IMAGE_NGINX" "$IMAGE_BUSYBOX"
     else
         run nerdctl images --format json
-        assert_output --partial '"Repository":"nginx'
+        assert_output --partial "\"Repository\":\"$IMAGE_NGINX"
         run nerdctl --namespace k8s.io images
-        assert_output --partial "busybox"
+        assert_output --partial "$IMAGE_BUSYBOX"
     fi
 }
 @test 'verify images before upgrade' {