diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index 61567618d82..5ef0d95c792 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -1,5 +1,6 @@ github https +skopeo ssh ubuntu workarounds diff --git a/bats/Makefile b/bats/Makefile index cd2af92afb3..d847947639a 100644 --- a/bats/Makefile +++ b/bats/Makefile @@ -18,8 +18,10 @@ lint: @./scripts/bats-lint.pl $(shell find tests -name '*.bats') find tests -name '*.bash' | xargs shellcheck -s bash -e $(SC_EXCLUDES) find tests -name '*.bats' | xargs shellcheck -s bash -e $(SC_EXCLUDES) + find scripts -name '*.sh' | xargs shellcheck -s bash -e $(SC_EXCLUDES) find tests -name '*.bash' | xargs shfmt -s -d find tests -name '*.bats' | xargs shfmt -s -d + find scripts -name '*.sh' | xargs shfmt -s -d DEPS = bin/darwin/jq bin/linux/jq diff --git a/bats/scripts/ghcr-mirror.sh b/bats/scripts/ghcr-mirror.sh new file mode 100755 index 00000000000..325aeaffd00 --- /dev/null +++ b/bats/scripts/ghcr-mirror.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Mirror Docker Hub images to ghcr.io to avoid pull limits during testing. + +# The script uses skopeo instead of docker pull/push because it needs to +# copy all images of the repo, and not just the one for the current platform. +# +# Log into ghcr.io with a personal access token with write:packages scope: +# echo $PAT | skopeo login ghcr.io -u $USER --password-stdin +# echo $PASS | skopeo login docker.io -u $USER --password-stdin +# Remove credentials: +# skopeo logout --all + +# TODO TODO TODO +# The package visibility needs to be changed to "public". +# I've not found any tool/API to do this from the commandline, +# so I did this manually via the web UI. +# At the very least we should check that the images are accessible +# when logged out of ghcr.io. +# TODO TODO TODO + +# TODO TODO TODO +# Figure out a way to copy only the amd64 and arm64 images, but not the rest. +# skopeo doesn't seem to support this yet without additional scripting to parse +# the manifests. And then we would need to test if we can copy a "sparse" manifest +# to ghcr.io when not all referenced images actually exist. +# TODO TODO TODO + +set -o errexit -o nounset -o pipefail +set +o xtrace + +if ! command -v skopeo >/dev/null; then + echo "This script requires the 'skopeo' utility to be installed" + exit 1 +fi + +source "$(dirname "${BASH_SOURCE[0]}")/../tests/helpers/images.bash" + +# IMAGES is setup by ../tests/helpers/images.bash +# shellcheck disable=SC2153 +for IMAGE in "${IMAGES[@]}"; do + echo "===== Copying $IMAGE =====" + skopeo copy --all "docker://$IMAGE" "docker://$GHCR_REPO/$IMAGE" +done diff --git a/bats/tests/containers/allowed-images.bats b/bats/tests/containers/allowed-images.bats index 06bbd17518f..38fc98ff819 100644 --- a/bats/tests/containers/allowed-images.bats +++ b/bats/tests/containers/allowed-images.bats @@ -9,53 +9,53 @@ RD_USE_IMAGE_ALLOW_LIST=true } @test 'update the list of patterns first time' { - update_allowed_patterns true '"nginx", "busybox", "python"' + update_allowed_patterns true "$IMAGE_NGINX" "$IMAGE_BUSYBOX" "$IMAGE_PYTHON" wait_for_container_engine } @test 'verify pull nginx succeeds' { - ctrctl pull --quiet nginx + ctrctl pull --quiet "$IMAGE_NGINX" } @test 'verify pull busybox succeeds' { - ctrctl pull --quiet busybox + ctrctl pull --quiet "$IMAGE_BUSYBOX" } @test 'verify pull python succeeds' { - ctrctl pull --quiet python + ctrctl pull --quiet "$IMAGE_PYTHON" } @test 'verify pull ruby fails' { - run ctrctl pull ruby + run ctrctl pull "$IMAGE_RUBY" assert_failure } @test 'drop python from the allowed-image list, add ruby' { - update_allowed_patterns true '"nginx", "busybox", "ruby"' + update_allowed_patterns true "$IMAGE_NGINX" "$IMAGE_BUSYBOX" "$IMAGE_RUBY" } @test 'clear images' { - for image in nginx busybox python; do - ctrctl rmi "$image" + for image in IMAGE_NGINX IMAGE_BUSYBOX IMAGE_PYTHON; do + ctrctl rmi "${!image}" done } @test 'verify pull python fails' { - run ctrctl pull --quiet python + run ctrctl pull --quiet "$IMAGE_PYTHON" assert_failure } @test 'verify pull ruby succeeds' { - ctrctl pull --quiet ruby + ctrctl pull --quiet "$IMAGE_RUBY" } @test 'clear all patterns' { - update_allowed_patterns true '' + update_allowed_patterns true } @test 'can run kubectl' { wait_for_apiserver - kubectl run nginx --image=nginx:latest --port=8080 + kubectl run nginx --image="${IMAGE_NGINX}:latest" --port=8080 } verify_no_nginx() { @@ -70,12 +70,12 @@ verify_no_nginx() { } @test 'set patterns with the allowed list disabled' { - update_allowed_patterns false '"nginx", "busybox", "ruby"' + update_allowed_patterns false "$IMAGE_NGINX" "$IMAGE_BUSYBOX" "$IMAGE_RUBY" # containerEngine.allowedImages.enabled changed, so wait for a restart wait_for_container_engine wait_for_apiserver "$RD_KUBERNETES_PREV_VERSION" } @test 'verify pull python succeeds because allowedImages filter is disabled' { - ctrctl pull --quiet python + ctrctl pull --quiet "$IMAGE_PYTHON" } diff --git a/bats/tests/containers/catch-duplicate-api-patterns.bats b/bats/tests/containers/catch-duplicate-api-patterns.bats index e066b30cb70..1afca1b2537 100644 --- a/bats/tests/containers/catch-duplicate-api-patterns.bats +++ b/bats/tests/containers/catch-duplicate-api-patterns.bats @@ -7,13 +7,13 @@ RD_USE_IMAGE_ALLOW_LIST=true wait_for_apiserver wait_for_container_engine - run update_allowed_patterns true '"nginx", "busybox", "ruby", "busybox"' + run update_allowed_patterns true "$IMAGE_NGINX" "$IMAGE_BUSYBOX" "$IMAGE_RUBY" "$IMAGE_BUSYBOX" assert_failure - assert_output --partial $'field \'containerEngine.allowedImages.patterns\' has duplicate entries: "busybox"' + assert_output --partial "field 'containerEngine.allowedImages.patterns' has duplicate entries: \"$IMAGE_BUSYBOX\"" } @test 'catch attempts to add duplicate patterns via the API with enabled off' { - run update_allowed_patterns false '"nginx", "busybox", "ruby", "busybox"' + run update_allowed_patterns false "$IMAGE_NGINX" "$IMAGE_BUSYBOX" "$IMAGE_RUBY" "$IMAGE_BUSYBOX" assert_failure - assert_output --partial $'field \'containerEngine.allowedImages.patterns\' has duplicate entries: "busybox"' + assert_output --partial "field 'containerEngine.allowedImages.patterns' has duplicate entries: \"$IMAGE_BUSYBOX\"" } diff --git a/bats/tests/containers/platform.bats b/bats/tests/containers/platform.bats index 6e08cb5c4f1..24c189e44bb 100644 --- a/bats/tests/containers/platform.bats +++ b/bats/tests/containers/platform.bats @@ -14,13 +14,13 @@ check_uname() { local cpu="$2" # Pull container separately because `ctrctl run` doesn't have a --quiet option - ctrctl pull --quiet --platform "$platform" busybox + ctrctl pull --quiet --platform "$platform" "$IMAGE_BUSYBOX" # BUG BUG BUG # Adding -i option to work around a bug with the Linux docker CLI in WSL # https://github.com/rancher-sandbox/rancher-desktop/issues/3239 # BUG BUG BUG - run ctrctl run -i --platform "$platform" busybox uname -m + run ctrctl run -i --platform "$platform" "$IMAGE_BUSYBOX" uname -m if is_true "${assert_success:-true}"; then assert_success assert_output "$cpu" @@ -42,7 +42,7 @@ check_uname() { @test 'uninstall s390x emulator' { if is_windows; then # On WSL the emulator might still be installed from a previous run - ctrctl run --privileged --rm tonistiigi/binfmt --uninstall qemu-s390x + ctrctl run --privileged --rm "$IMAGE_TONISTIIGI_BINFMT" --uninstall qemu-s390x else skip "only required on Windows" fi @@ -55,7 +55,7 @@ check_uname() { } @test 'install s390x emulator' { - ctrctl run --privileged --rm tonistiigi/binfmt --install s390x + ctrctl run --privileged --rm "$IMAGE_TONISTIIGI_BINFMT" --install s390x } @test 'deploy s390x container' { diff --git a/bats/tests/containers/switch-engines.bats b/bats/tests/containers/switch-engines.bats index f1d0d897d32..27f7c214a0f 100644 --- a/bats/tests/containers/switch-engines.bats +++ b/bats/tests/containers/switch-engines.bats @@ -11,11 +11,11 @@ switch_container_engine() { } pull_containers() { - ctrctl run -d -p 8085:80 --restart=no nginx - ctrctl run -d --restart=always busybox /bin/sh -c "sleep inf" + ctrctl run -d -p 8085:80 --restart=no "$IMAGE_NGINX" + ctrctl run -d --restart=always "$IMAGE_BUSYBOX" /bin/sh -c "sleep inf" run ctrctl ps --format '{{json .Image}}' - assert_output --partial nginx - assert_output --partial busybox + assert_output --partial "$IMAGE_NGINX" + assert_output --partial "$IMAGE_BUSYBOX" } @test 'factory reset' { @@ -35,8 +35,8 @@ pull_containers() { verify_post_switch_containers() { run ctrctl ps --format '{{json .Image}}' - assert_output --partial "busybox" - refute_output --partial "nginx" + assert_output --partial "$IMAGE_BUSYBOX" + refute_output --partial "$IMAGE_NGINX" } switch_back_verify_post_switch_containers() { diff --git a/bats/tests/helpers/defaults.bash b/bats/tests/helpers/defaults.bash index 886e38d9d80..75f04258b09 100644 --- a/bats/tests/helpers/defaults.bash +++ b/bats/tests/helpers/defaults.bash @@ -34,6 +34,17 @@ taking_screenshots() { is_true "$RD_TAKE_SCREENSHOTS" } +######################################################################## +# When RD_USE_GHCR_IMAGES is true, then all images will be pulled from +# ghcr.io instead of docker.io, to avoid hitting the docker hub pull +# rate limit. + +: "${RD_USE_GHCR_IMAGES:=false}" + +using_ghcr_images() { + is_true "$RD_USE_GHCR_IMAGES" +} + ######################################################################## : "${RD_USE_IMAGE_ALLOW_LIST:=false}" diff --git a/bats/tests/helpers/images.bash b/bats/tests/helpers/images.bash new file mode 100644 index 00000000000..b3ca483fd82 --- /dev/null +++ b/bats/tests/helpers/images.bash @@ -0,0 +1,24 @@ +# These images have been mirrored to ghcr.io (using bats/scripts/ghcr-mirror.sh) +# to avoid hitting Docker Hub pull limits during testing. + +# TODO TODO TODO +# The python image is huge (10GB across all platforms). We should either pin the +# tag, or replace it with a different image for testing, so we don't have to mirror +# the images to ghcr.io every time we run the mirror script. +# TODO TODO TODO + +# Any time you add an image here you need to re-run the mirror script! +IMAGES=(busybox nginx python ruby tonistiigi/binfmt registry:2.8.1) + +GHCR_REPO=ghcr.io/rancher-sandbox/bats + +# Create IMAGE_FOO_BAR=foo/bar:tag variables +for IMAGE in "${IMAGES[@]}"; do + VAR="IMAGE_$(echo "$IMAGE" | sed 's/:.*//' | tr '[:lower:]' '[:upper:]' | tr / _)" + # file may be loaded outside BATS environment + if [ "$(type -t using_ghcr_images)" = "function" ] && using_ghcr_images; then + eval "$VAR=$GHCR_REPO/$IMAGE" + else + eval "$VAR=$IMAGE" + fi +done diff --git a/bats/tests/helpers/info.bash b/bats/tests/helpers/info.bash index 5ee77456bfb..0267b398126 100644 --- a/bats/tests/helpers/info.bash +++ b/bats/tests/helpers/info.bash @@ -37,5 +37,6 @@ show_info() { # @test echo "#" printf "$format" "Capturing logs:" "$(bool capturing_logs)" printf "$format" "Taking screenshots:" "$(bool taking_screenshots)" + printf "$format" "Using ghcr.io images:" "$(bool using_ghcr_images)" ) >&3 } diff --git a/bats/tests/helpers/load.bash b/bats/tests/helpers/load.bash index 37ea92bbfdc..99c409e01a8 100644 --- a/bats/tests/helpers/load.bash +++ b/bats/tests/helpers/load.bash @@ -36,6 +36,9 @@ source "$PATH_BATS_HELPERS/utils.bash" # validate_enum() and is_true() from utils.bash. source "$PATH_BATS_HELPERS/defaults.bash" +# images.bash uses using_ghcr_images() from defaults.bash +source "$PATH_BATS_HELPERS/images.bash" + # paths.bash uses RD_LOCATION from defaults.bash source "$PATH_BATS_HELPERS/paths.bash" diff --git a/bats/tests/helpers/utils.bash b/bats/tests/helpers/utils.bash index 228602e72c3..c2c78d6b94a 100644 --- a/bats/tests/helpers/utils.bash +++ b/bats/tests/helpers/utils.bash @@ -84,9 +84,32 @@ try() { return "$status" } +image_without_tag() { + local image=$1 + # If the tag looks like a port number and follows something that looks + # like a domain name, then don't strip the tag (e.g. foo.io:5000). + if [[ ${image##*:} =~ ^[0-9]+$ && ${image%:*} =~ \.[a-z]+$ ]]; then + echo "$image" + else + echo "${image%:*}" + fi +} + update_allowed_patterns() { local enabled=$1 - local patterns=$2 + shift + + local patterns="" + local image + for image in "$@"; do + image=$(image_without_tag "$image") + if [ -z "$patterns" ]; then + patterns="\"${image}\"" + else + patterns="$patterns, \"${image}\"" + fi + done + # TODO TODO TODO # Once https://github.com/rancher-sandbox/rancher-desktop/issues/4939 has been # implemented, the `version` field should be made a constant. Putting in the diff --git a/bats/tests/helpers/vm.bash b/bats/tests/helpers/vm.bash index a4b5b97717d..c13f0ffc93b 100644 --- a/bats/tests/helpers/vm.bash +++ b/bats/tests/helpers/vm.bash @@ -93,6 +93,10 @@ start_container_engine() { # TODO cannot be set from the commandline yet image_allow_list="$(bool using_image_allow_list)" wsl_integrations="{}" + registry="docker.io" + if using_ghcr_images; then + registry="ghcr.io" + fi if is_windows; then wsl_integrations="{\"$WSL_DISTRO_NAME\":true}" fi @@ -104,7 +108,7 @@ start_container_engine() { "containerEngine": { "allowedImages": { "enabled": $image_allow_list, - "patterns": ["docker.io"] + "patterns": ["$registry"] } } } diff --git a/bats/tests/k8s/up-downgrade-k8s.bats b/bats/tests/k8s/up-downgrade-k8s.bats index d806d45f36b..5d4c66842fb 100644 --- a/bats/tests/k8s/up-downgrade-k8s.bats +++ b/bats/tests/k8s/up-downgrade-k8s.bats @@ -16,18 +16,18 @@ ARCH_FOR_KUBERLR=amd64 } @test 'deploy nginx - always restart' { - ctrctl pull nginx - run ctrctl run -d -p 8585:80 --restart=always --name nginx-restart nginx + ctrctl pull "$IMAGE_NGINX" + run ctrctl run -d -p 8585:80 --restart=always --name nginx-restart "$IMAGE_NGINX" assert_success } @test 'deploy nginx - no restart' { - run ctrctl run -d -p 8686:80 --restart=no --name nginx-no-restart nginx + run ctrctl run -d -p 8686:80 --restart=no --name nginx-no-restart "$IMAGE_NGINX" assert_success } @test 'deploy busybox' { - run kubectl create deploy busybox --image=busybox --replicas=2 -- /bin/sh -c "sleep inf" + run kubectl create deploy busybox --image="$IMAGE_BUSYBOX" --replicas=2 -- /bin/sh -c "sleep inf" assert_success } @@ -57,12 +57,12 @@ verify_busybox() { verify_images() { if using_docker; then run docker images - assert_output --partial "nginx" "busybox" + assert_output --partial "$IMAGE_NGINX" "$IMAGE_BUSYBOX" else run nerdctl images --format json - assert_output --partial '"Repository":"nginx' + assert_output --partial "\"Repository\":\"$IMAGE_NGINX" run nerdctl --namespace k8s.io images - assert_output --partial "busybox" + assert_output --partial "$IMAGE_BUSYBOX" fi } @test 'verify images before upgrade' { diff --git a/bats/tests/registry/creds.bats b/bats/tests/registry/creds.bats index 15954d68750..5a58bf86601 100644 --- a/bats/tests/registry/creds.bats +++ b/bats/tests/registry/creds.bats @@ -1,7 +1,6 @@ load '../helpers/load' local_setup() { - REGISTRY_IMAGE="registry:2.8.1" REGISTRY_PORT="5050" DOCKER_CONFIG_FILE="$HOME/.docker/config.json" @@ -61,7 +60,7 @@ create_registry() { -e "REGISTRY_HTTP_TLS_CERTIFICATE=/certs/$REGISTRY_HOST.pem" \ -e "REGISTRY_HTTP_TLS_KEY=/certs/$REGISTRY_HOST-key.pem" \ "$@" \ - "$REGISTRY_IMAGE" + "$IMAGE_REGISTRY" wait_for_registry } @@ -91,7 +90,7 @@ skip_for_insecure_registry() { if using_image_allow_list; then wait_for_shell - update_allowed_patterns true "$(printf '"%s" "docker.io/registry"' "$REGISTRY")" + update_allowed_patterns true "$IMAGE_REGISTRY" "$REGISTRY" fi } @@ -111,7 +110,7 @@ verify_default_credStore() { } @test 'verify allowed-images config' { - run ctrctl pull --quiet busybox + run ctrctl pull --quiet "$IMAGE_BUSYBOX" if using_image_allow_list; then assert_failure assert_output --regexp "(unauthorized|Forbidden)" @@ -128,7 +127,7 @@ verify_default_credStore() { } @test 'pull registry image' { - ctrctl pull --quiet "$REGISTRY_IMAGE" + ctrctl pull --quiet "$IMAGE_REGISTRY" } @test 'create plain registry' { @@ -136,13 +135,13 @@ verify_default_credStore() { } @test 'tag image with registry' { - ctrctl tag "$REGISTRY_IMAGE" "$REGISTRY/$REGISTRY_IMAGE" + ctrctl tag "$IMAGE_REGISTRY" "$REGISTRY/registry" } @test 'expect push image to registry to fail because CA cert has not been installed' { skip_for_insecure_registry - run ctrctl push "$REGISTRY/$REGISTRY_IMAGE" + run ctrctl push "$REGISTRY/registry" assert_failure # we don't get cert errors when going through the proxy; they turn into 502's assert_output --regexp "(certificate signed by unknown authority|502 Bad Gateway)" @@ -168,7 +167,7 @@ verify_default_credStore() { } @test 'expect push image to registry to succeed now' { - ctrctl push "$REGISTRY/$REGISTRY_IMAGE" + ctrctl push "$REGISTRY/registry" } @test 'create registry with basic auth' { @@ -204,7 +203,7 @@ verify_default_credStore() { @test 'verify that pushing fails when not logged in' { run bash -c "echo \"$REGISTRY\" | \"$CRED_HELPER\" erase" assert_nothing - run ctrctl push "$REGISTRY/$REGISTRY_IMAGE" + run ctrctl push "$REGISTRY/registry" assert_failure assert_output --regexp "(401 Unauthorized|no basic auth credentials)" } @@ -214,7 +213,7 @@ verify_default_credStore() { assert_success assert_output --partial "Login Succeeded" - ctrctl push "$REGISTRY/$REGISTRY_IMAGE" + ctrctl push "$REGISTRY/registry" } @test 'verify credentials in host cred store' {