From 491c041d96711035b69f1e6adc150e6801f0601c Mon Sep 17 00:00:00 2001 From: Jan Dubois Date: Thu, 29 Jun 2023 14:19:08 -0700 Subject: [PATCH] Add RD_USE_GHCR_IMAGES option to BATS to pull images from ghcr.io This avoids hitting the pull rate limit when running multiple full BATS runs in sequence, especially if not logged into Docker Hub. Pull rate limit for docker.io is 100 pulls / 6 hours, or twice that when authenticated. Signed-off-by: Jan Dubois --- .github/actions/spelling/allow.txt | 1 + bats/Makefile | 2 + bats/scripts/ghcr-mirror.sh | 44 +++++++++++++++++++ bats/tests/containers/allowed-images.bats | 28 ++++++------ .../catch-duplicate-api-patterns.bats | 8 ++-- bats/tests/containers/platform.bats | 8 ++-- bats/tests/containers/switch-engines.bats | 12 ++--- bats/tests/helpers/defaults.bash | 11 +++++ bats/tests/helpers/images.bash | 24 ++++++++++ bats/tests/helpers/info.bash | 1 + bats/tests/helpers/load.bash | 3 ++ bats/tests/helpers/utils.bash | 25 ++++++++++- bats/tests/helpers/vm.bash | 6 ++- bats/tests/k8s/up-downgrade-k8s.bats | 14 +++--- bats/tests/registry/creds.bats | 19 ++++---- 15 files changed, 159 insertions(+), 47 deletions(-) create mode 100755 bats/scripts/ghcr-mirror.sh create mode 100644 bats/tests/helpers/images.bash diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index 61567618d82..5ef0d95c792 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -1,5 +1,6 @@ github https +skopeo ssh ubuntu workarounds diff --git a/bats/Makefile b/bats/Makefile index cd2af92afb3..d847947639a 100644 --- a/bats/Makefile +++ b/bats/Makefile @@ -18,8 +18,10 @@ lint: @./scripts/bats-lint.pl $(shell find tests -name '*.bats') find tests -name '*.bash' | xargs shellcheck -s bash -e $(SC_EXCLUDES) find tests -name '*.bats' | xargs shellcheck -s bash -e $(SC_EXCLUDES) + find scripts -name '*.sh' | xargs shellcheck -s bash -e $(SC_EXCLUDES) find tests -name '*.bash' | xargs shfmt -s -d find tests -name '*.bats' | xargs shfmt -s -d + find scripts -name '*.sh' | xargs shfmt -s -d DEPS = bin/darwin/jq bin/linux/jq diff --git a/bats/scripts/ghcr-mirror.sh b/bats/scripts/ghcr-mirror.sh new file mode 100755 index 00000000000..325aeaffd00 --- /dev/null +++ b/bats/scripts/ghcr-mirror.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Mirror Docker Hub images to ghcr.io to avoid pull limits during testing. + +# The script uses skopeo instead of docker pull/push because it needs to +# copy all images of the repo, and not just the one for the current platform. +# +# Log into ghcr.io with a personal access token with write:packages scope: +# echo $PAT | skopeo login ghcr.io -u $USER --password-stdin +# echo $PASS | skopeo login docker.io -u $USER --password-stdin +# Remove credentials: +# skopeo logout --all + +# TODO TODO TODO +# The package visibility needs to be changed to "public". +# I've not found any tool/API to do this from the commandline, +# so I did this manually via the web UI. +# At the very least we should check that the images are accessible +# when logged out of ghcr.io. +# TODO TODO TODO + +# TODO TODO TODO +# Figure out a way to copy only the amd64 and arm64 images, but not the rest. +# skopeo doesn't seem to support this yet without additional scripting to parse +# the manifests. And then we would need to test if we can copy a "sparse" manifest +# to ghcr.io when not all referenced images actually exist. +# TODO TODO TODO + +set -o errexit -o nounset -o pipefail +set +o xtrace + +if ! command -v skopeo >/dev/null; then + echo "This script requires the 'skopeo' utility to be installed" + exit 1 +fi + +source "$(dirname "${BASH_SOURCE[0]}")/../tests/helpers/images.bash" + +# IMAGES is setup by ../tests/helpers/images.bash +# shellcheck disable=SC2153 +for IMAGE in "${IMAGES[@]}"; do + echo "===== Copying $IMAGE =====" + skopeo copy --all "docker://$IMAGE" "docker://$GHCR_REPO/$IMAGE" +done diff --git a/bats/tests/containers/allowed-images.bats b/bats/tests/containers/allowed-images.bats index 06bbd17518f..38fc98ff819 100644 --- a/bats/tests/containers/allowed-images.bats +++ b/bats/tests/containers/allowed-images.bats @@ -9,53 +9,53 @@ RD_USE_IMAGE_ALLOW_LIST=true } @test 'update the list of patterns first time' { - update_allowed_patterns true '"nginx", "busybox", "python"' + update_allowed_patterns true "$IMAGE_NGINX" "$IMAGE_BUSYBOX" "$IMAGE_PYTHON" wait_for_container_engine } @test 'verify pull nginx succeeds' { - ctrctl pull --quiet nginx + ctrctl pull --quiet "$IMAGE_NGINX" } @test 'verify pull busybox succeeds' { - ctrctl pull --quiet busybox + ctrctl pull --quiet "$IMAGE_BUSYBOX" } @test 'verify pull python succeeds' { - ctrctl pull --quiet python + ctrctl pull --quiet "$IMAGE_PYTHON" } @test 'verify pull ruby fails' { - run ctrctl pull ruby + run ctrctl pull "$IMAGE_RUBY" assert_failure } @test 'drop python from the allowed-image list, add ruby' { - update_allowed_patterns true '"nginx", "busybox", "ruby"' + update_allowed_patterns true "$IMAGE_NGINX" "$IMAGE_BUSYBOX" "$IMAGE_RUBY" } @test 'clear images' { - for image in nginx busybox python; do - ctrctl rmi "$image" + for image in IMAGE_NGINX IMAGE_BUSYBOX IMAGE_PYTHON; do + ctrctl rmi "${!image}" done } @test 'verify pull python fails' { - run ctrctl pull --quiet python + run ctrctl pull --quiet "$IMAGE_PYTHON" assert_failure } @test 'verify pull ruby succeeds' { - ctrctl pull --quiet ruby + ctrctl pull --quiet "$IMAGE_RUBY" } @test 'clear all patterns' { - update_allowed_patterns true '' + update_allowed_patterns true } @test 'can run kubectl' { wait_for_apiserver - kubectl run nginx --image=nginx:latest --port=8080 + kubectl run nginx --image="${IMAGE_NGINX}:latest" --port=8080 } verify_no_nginx() { @@ -70,12 +70,12 @@ verify_no_nginx() { } @test 'set patterns with the allowed list disabled' { - update_allowed_patterns false '"nginx", "busybox", "ruby"' + update_allowed_patterns false "$IMAGE_NGINX" "$IMAGE_BUSYBOX" "$IMAGE_RUBY" # containerEngine.allowedImages.enabled changed, so wait for a restart wait_for_container_engine wait_for_apiserver "$RD_KUBERNETES_PREV_VERSION" } @test 'verify pull python succeeds because allowedImages filter is disabled' { - ctrctl pull --quiet python + ctrctl pull --quiet "$IMAGE_PYTHON" } diff --git a/bats/tests/containers/catch-duplicate-api-patterns.bats b/bats/tests/containers/catch-duplicate-api-patterns.bats index e066b30cb70..1afca1b2537 100644 --- a/bats/tests/containers/catch-duplicate-api-patterns.bats +++ b/bats/tests/containers/catch-duplicate-api-patterns.bats @@ -7,13 +7,13 @@ RD_USE_IMAGE_ALLOW_LIST=true wait_for_apiserver wait_for_container_engine - run update_allowed_patterns true '"nginx", "busybox", "ruby", "busybox"' + run update_allowed_patterns true "$IMAGE_NGINX" "$IMAGE_BUSYBOX" "$IMAGE_RUBY" "$IMAGE_BUSYBOX" assert_failure - assert_output --partial $'field \'containerEngine.allowedImages.patterns\' has duplicate entries: "busybox"' + assert_output --partial "field 'containerEngine.allowedImages.patterns' has duplicate entries: \"$IMAGE_BUSYBOX\"" } @test 'catch attempts to add duplicate patterns via the API with enabled off' { - run update_allowed_patterns false '"nginx", "busybox", "ruby", "busybox"' + run update_allowed_patterns false "$IMAGE_NGINX" "$IMAGE_BUSYBOX" "$IMAGE_RUBY" "$IMAGE_BUSYBOX" assert_failure - assert_output --partial $'field \'containerEngine.allowedImages.patterns\' has duplicate entries: "busybox"' + assert_output --partial "field 'containerEngine.allowedImages.patterns' has duplicate entries: \"$IMAGE_BUSYBOX\"" } diff --git a/bats/tests/containers/platform.bats b/bats/tests/containers/platform.bats index 6e08cb5c4f1..24c189e44bb 100644 --- a/bats/tests/containers/platform.bats +++ b/bats/tests/containers/platform.bats @@ -14,13 +14,13 @@ check_uname() { local cpu="$2" # Pull container separately because `ctrctl run` doesn't have a --quiet option - ctrctl pull --quiet --platform "$platform" busybox + ctrctl pull --quiet --platform "$platform" "$IMAGE_BUSYBOX" # BUG BUG BUG # Adding -i option to work around a bug with the Linux docker CLI in WSL # https://github.com/rancher-sandbox/rancher-desktop/issues/3239 # BUG BUG BUG - run ctrctl run -i --platform "$platform" busybox uname -m + run ctrctl run -i --platform "$platform" "$IMAGE_BUSYBOX" uname -m if is_true "${assert_success:-true}"; then assert_success assert_output "$cpu" @@ -42,7 +42,7 @@ check_uname() { @test 'uninstall s390x emulator' { if is_windows; then # On WSL the emulator might still be installed from a previous run - ctrctl run --privileged --rm tonistiigi/binfmt --uninstall qemu-s390x + ctrctl run --privileged --rm "$IMAGE_TONISTIIGI_BINFMT" --uninstall qemu-s390x else skip "only required on Windows" fi @@ -55,7 +55,7 @@ check_uname() { } @test 'install s390x emulator' { - ctrctl run --privileged --rm tonistiigi/binfmt --install s390x + ctrctl run --privileged --rm "$IMAGE_TONISTIIGI_BINFMT" --install s390x } @test 'deploy s390x container' { diff --git a/bats/tests/containers/switch-engines.bats b/bats/tests/containers/switch-engines.bats index f1d0d897d32..27f7c214a0f 100644 --- a/bats/tests/containers/switch-engines.bats +++ b/bats/tests/containers/switch-engines.bats @@ -11,11 +11,11 @@ switch_container_engine() { } pull_containers() { - ctrctl run -d -p 8085:80 --restart=no nginx - ctrctl run -d --restart=always busybox /bin/sh -c "sleep inf" + ctrctl run -d -p 8085:80 --restart=no "$IMAGE_NGINX" + ctrctl run -d --restart=always "$IMAGE_BUSYBOX" /bin/sh -c "sleep inf" run ctrctl ps --format '{{json .Image}}' - assert_output --partial nginx - assert_output --partial busybox + assert_output --partial "$IMAGE_NGINX" + assert_output --partial "$IMAGE_BUSYBOX" } @test 'factory reset' { @@ -35,8 +35,8 @@ pull_containers() { verify_post_switch_containers() { run ctrctl ps --format '{{json .Image}}' - assert_output --partial "busybox" - refute_output --partial "nginx" + assert_output --partial "$IMAGE_BUSYBOX" + refute_output --partial "$IMAGE_NGINX" } switch_back_verify_post_switch_containers() { diff --git a/bats/tests/helpers/defaults.bash b/bats/tests/helpers/defaults.bash index 886e38d9d80..75f04258b09 100644 --- a/bats/tests/helpers/defaults.bash +++ b/bats/tests/helpers/defaults.bash @@ -34,6 +34,17 @@ taking_screenshots() { is_true "$RD_TAKE_SCREENSHOTS" } +######################################################################## +# When RD_USE_GHCR_IMAGES is true, then all images will be pulled from +# ghcr.io instead of docker.io, to avoid hitting the docker hub pull +# rate limit. + +: "${RD_USE_GHCR_IMAGES:=false}" + +using_ghcr_images() { + is_true "$RD_USE_GHCR_IMAGES" +} + ######################################################################## : "${RD_USE_IMAGE_ALLOW_LIST:=false}" diff --git a/bats/tests/helpers/images.bash b/bats/tests/helpers/images.bash new file mode 100644 index 00000000000..b3ca483fd82 --- /dev/null +++ b/bats/tests/helpers/images.bash @@ -0,0 +1,24 @@ +# These images have been mirrored to ghcr.io (using bats/scripts/ghcr-mirror.sh) +# to avoid hitting Docker Hub pull limits during testing. + +# TODO TODO TODO +# The python image is huge (10GB across all platforms). We should either pin the +# tag, or replace it with a different image for testing, so we don't have to mirror +# the images to ghcr.io every time we run the mirror script. +# TODO TODO TODO + +# Any time you add an image here you need to re-run the mirror script! +IMAGES=(busybox nginx python ruby tonistiigi/binfmt registry:2.8.1) + +GHCR_REPO=ghcr.io/rancher-sandbox/bats + +# Create IMAGE_FOO_BAR=foo/bar:tag variables +for IMAGE in "${IMAGES[@]}"; do + VAR="IMAGE_$(echo "$IMAGE" | sed 's/:.*//' | tr '[:lower:]' '[:upper:]' | tr / _)" + # file may be loaded outside BATS environment + if [ "$(type -t using_ghcr_images)" = "function" ] && using_ghcr_images; then + eval "$VAR=$GHCR_REPO/$IMAGE" + else + eval "$VAR=$IMAGE" + fi +done diff --git a/bats/tests/helpers/info.bash b/bats/tests/helpers/info.bash index 5ee77456bfb..0267b398126 100644 --- a/bats/tests/helpers/info.bash +++ b/bats/tests/helpers/info.bash @@ -37,5 +37,6 @@ show_info() { # @test echo "#" printf "$format" "Capturing logs:" "$(bool capturing_logs)" printf "$format" "Taking screenshots:" "$(bool taking_screenshots)" + printf "$format" "Using ghcr.io images:" "$(bool using_ghcr_images)" ) >&3 } diff --git a/bats/tests/helpers/load.bash b/bats/tests/helpers/load.bash index 37ea92bbfdc..99c409e01a8 100644 --- a/bats/tests/helpers/load.bash +++ b/bats/tests/helpers/load.bash @@ -36,6 +36,9 @@ source "$PATH_BATS_HELPERS/utils.bash" # validate_enum() and is_true() from utils.bash. source "$PATH_BATS_HELPERS/defaults.bash" +# images.bash uses using_ghcr_images() from defaults.bash +source "$PATH_BATS_HELPERS/images.bash" + # paths.bash uses RD_LOCATION from defaults.bash source "$PATH_BATS_HELPERS/paths.bash" diff --git a/bats/tests/helpers/utils.bash b/bats/tests/helpers/utils.bash index 228602e72c3..c2c78d6b94a 100644 --- a/bats/tests/helpers/utils.bash +++ b/bats/tests/helpers/utils.bash @@ -84,9 +84,32 @@ try() { return "$status" } +image_without_tag() { + local image=$1 + # If the tag looks like a port number and follows something that looks + # like a domain name, then don't strip the tag (e.g. foo.io:5000). + if [[ ${image##*:} =~ ^[0-9]+$ && ${image%:*} =~ \.[a-z]+$ ]]; then + echo "$image" + else + echo "${image%:*}" + fi +} + update_allowed_patterns() { local enabled=$1 - local patterns=$2 + shift + + local patterns="" + local image + for image in "$@"; do + image=$(image_without_tag "$image") + if [ -z "$patterns" ]; then + patterns="\"${image}\"" + else + patterns="$patterns, \"${image}\"" + fi + done + # TODO TODO TODO # Once https://github.com/rancher-sandbox/rancher-desktop/issues/4939 has been # implemented, the `version` field should be made a constant. Putting in the diff --git a/bats/tests/helpers/vm.bash b/bats/tests/helpers/vm.bash index a4b5b97717d..c13f0ffc93b 100644 --- a/bats/tests/helpers/vm.bash +++ b/bats/tests/helpers/vm.bash @@ -93,6 +93,10 @@ start_container_engine() { # TODO cannot be set from the commandline yet image_allow_list="$(bool using_image_allow_list)" wsl_integrations="{}" + registry="docker.io" + if using_ghcr_images; then + registry="ghcr.io" + fi if is_windows; then wsl_integrations="{\"$WSL_DISTRO_NAME\":true}" fi @@ -104,7 +108,7 @@ start_container_engine() { "containerEngine": { "allowedImages": { "enabled": $image_allow_list, - "patterns": ["docker.io"] + "patterns": ["$registry"] } } } diff --git a/bats/tests/k8s/up-downgrade-k8s.bats b/bats/tests/k8s/up-downgrade-k8s.bats index d806d45f36b..5d4c66842fb 100644 --- a/bats/tests/k8s/up-downgrade-k8s.bats +++ b/bats/tests/k8s/up-downgrade-k8s.bats @@ -16,18 +16,18 @@ ARCH_FOR_KUBERLR=amd64 } @test 'deploy nginx - always restart' { - ctrctl pull nginx - run ctrctl run -d -p 8585:80 --restart=always --name nginx-restart nginx + ctrctl pull "$IMAGE_NGINX" + run ctrctl run -d -p 8585:80 --restart=always --name nginx-restart "$IMAGE_NGINX" assert_success } @test 'deploy nginx - no restart' { - run ctrctl run -d -p 8686:80 --restart=no --name nginx-no-restart nginx + run ctrctl run -d -p 8686:80 --restart=no --name nginx-no-restart "$IMAGE_NGINX" assert_success } @test 'deploy busybox' { - run kubectl create deploy busybox --image=busybox --replicas=2 -- /bin/sh -c "sleep inf" + run kubectl create deploy busybox --image="$IMAGE_BUSYBOX" --replicas=2 -- /bin/sh -c "sleep inf" assert_success } @@ -57,12 +57,12 @@ verify_busybox() { verify_images() { if using_docker; then run docker images - assert_output --partial "nginx" "busybox" + assert_output --partial "$IMAGE_NGINX" "$IMAGE_BUSYBOX" else run nerdctl images --format json - assert_output --partial '"Repository":"nginx' + assert_output --partial "\"Repository\":\"$IMAGE_NGINX" run nerdctl --namespace k8s.io images - assert_output --partial "busybox" + assert_output --partial "$IMAGE_BUSYBOX" fi } @test 'verify images before upgrade' { diff --git a/bats/tests/registry/creds.bats b/bats/tests/registry/creds.bats index 15954d68750..5a58bf86601 100644 --- a/bats/tests/registry/creds.bats +++ b/bats/tests/registry/creds.bats @@ -1,7 +1,6 @@ load '../helpers/load' local_setup() { - REGISTRY_IMAGE="registry:2.8.1" REGISTRY_PORT="5050" DOCKER_CONFIG_FILE="$HOME/.docker/config.json" @@ -61,7 +60,7 @@ create_registry() { -e "REGISTRY_HTTP_TLS_CERTIFICATE=/certs/$REGISTRY_HOST.pem" \ -e "REGISTRY_HTTP_TLS_KEY=/certs/$REGISTRY_HOST-key.pem" \ "$@" \ - "$REGISTRY_IMAGE" + "$IMAGE_REGISTRY" wait_for_registry } @@ -91,7 +90,7 @@ skip_for_insecure_registry() { if using_image_allow_list; then wait_for_shell - update_allowed_patterns true "$(printf '"%s" "docker.io/registry"' "$REGISTRY")" + update_allowed_patterns true "$IMAGE_REGISTRY" "$REGISTRY" fi } @@ -111,7 +110,7 @@ verify_default_credStore() { } @test 'verify allowed-images config' { - run ctrctl pull --quiet busybox + run ctrctl pull --quiet "$IMAGE_BUSYBOX" if using_image_allow_list; then assert_failure assert_output --regexp "(unauthorized|Forbidden)" @@ -128,7 +127,7 @@ verify_default_credStore() { } @test 'pull registry image' { - ctrctl pull --quiet "$REGISTRY_IMAGE" + ctrctl pull --quiet "$IMAGE_REGISTRY" } @test 'create plain registry' { @@ -136,13 +135,13 @@ verify_default_credStore() { } @test 'tag image with registry' { - ctrctl tag "$REGISTRY_IMAGE" "$REGISTRY/$REGISTRY_IMAGE" + ctrctl tag "$IMAGE_REGISTRY" "$REGISTRY/registry" } @test 'expect push image to registry to fail because CA cert has not been installed' { skip_for_insecure_registry - run ctrctl push "$REGISTRY/$REGISTRY_IMAGE" + run ctrctl push "$REGISTRY/registry" assert_failure # we don't get cert errors when going through the proxy; they turn into 502's assert_output --regexp "(certificate signed by unknown authority|502 Bad Gateway)" @@ -168,7 +167,7 @@ verify_default_credStore() { } @test 'expect push image to registry to succeed now' { - ctrctl push "$REGISTRY/$REGISTRY_IMAGE" + ctrctl push "$REGISTRY/registry" } @test 'create registry with basic auth' { @@ -204,7 +203,7 @@ verify_default_credStore() { @test 'verify that pushing fails when not logged in' { run bash -c "echo \"$REGISTRY\" | \"$CRED_HELPER\" erase" assert_nothing - run ctrctl push "$REGISTRY/$REGISTRY_IMAGE" + run ctrctl push "$REGISTRY/registry" assert_failure assert_output --regexp "(401 Unauthorized|no basic auth credentials)" } @@ -214,7 +213,7 @@ verify_default_credStore() { assert_success assert_output --partial "Login Succeeded" - ctrctl push "$REGISTRY/$REGISTRY_IMAGE" + ctrctl push "$REGISTRY/registry" } @test 'verify credentials in host cred store' {