From e9db7c5dfde38ac87732ef3cc1f9e106026ed88f Mon Sep 17 00:00:00 2001 From: Xiaofeng Wang Date: Mon, 18 Mar 2024 14:56:06 +0800 Subject: [PATCH] test: add rollback and bootc upgrade fetch timer test To avoid hitting quay.io race limitation, add retry for podman push and podman build Signed-off-by: Xiaofeng Wang --- tests/integration/install-upgrade.sh | 24 +++++- tests/integration/playbooks/check-system.yaml | 19 +++++ tests/integration/playbooks/rollback.yaml | 75 +++++++++++++++++++ 3 files changed, 114 insertions(+), 4 deletions(-) create mode 100644 tests/integration/playbooks/rollback.yaml diff --git a/tests/integration/install-upgrade.sh b/tests/integration/install-upgrade.sh index 3ec31dc1..7ada4656 100755 --- a/tests/integration/install-upgrade.sh +++ b/tests/integration/install-upgrade.sh @@ -10,6 +10,16 @@ function redprint { echo -e "\033[1;31m[$(date -Isecond)] ${1}\033[0m" } +function retry { + n=0 + until [ "$n" -ge 3 ] + do + "$@" && break + n=$((n+1)) + sleep 10 + done +} + TEMPDIR=$(mktemp -d) trap 'rm -rf -- "$TEMPDIR"' EXIT @@ -92,10 +102,10 @@ greenprint "Login quay.io" podman login -u "${QUAY_USERNAME}" -p "${QUAY_PASSWORD}" quay.io greenprint "Build $TEST_OS installation container image" -podman build --tls-verify=false -t "${TEST_IMAGE_NAME}:${QUAY_REPO_TAG}" -f "$INSTALL_CONTAINERFILE" . +podman build --tls-verify=false --retry=5 --retry-delay=10 -t "${TEST_IMAGE_NAME}:${QUAY_REPO_TAG}" -f "$INSTALL_CONTAINERFILE" . greenprint "Push $TEST_OS installation container image" -podman push "${TEST_IMAGE_NAME}:${QUAY_REPO_TAG}" "$TEST_IMAGE_URL" +retry podman push --tls-verify=false --quiet "${TEST_IMAGE_NAME}:${QUAY_REPO_TAG}" "$TEST_IMAGE_URL" greenprint "Prepare inventory file" tee -a "$INVENTORY_FILE" > /dev/null << EOF @@ -146,9 +156,10 @@ RUN dnf -y install wget && \ EOF greenprint "Build $TEST_OS upgrade container image" -podman build --tls-verify=false -t "${TEST_IMAGE_NAME}:${QUAY_REPO_TAG}" -f "$UPGRADE_CONTAINERFILE" . +podman build --tls-verify=false --retry=5 --retry-delay=10 -t "${TEST_IMAGE_NAME}:${QUAY_REPO_TAG}" -f "$UPGRADE_CONTAINERFILE" . + greenprint "Push $TEST_OS upgrade container image" -podman push "${TEST_IMAGE_NAME}:${QUAY_REPO_TAG}" "$TEST_IMAGE_URL" +retry podman push --tls-verify=false --quiet "${TEST_IMAGE_NAME}:${QUAY_REPO_TAG}" "$TEST_IMAGE_URL" greenprint "Upgrade $TEST_OS system" ansible-playbook -v \ @@ -162,6 +173,11 @@ ansible-playbook -v \ -e upgrade="true" \ playbooks/check-system.yaml +greenprint "Rollback $TEST_OS system" +ansible-playbook -v \ + -i "$INVENTORY_FILE" \ + playbooks/rollback.yaml + greenprint "Remove $PLATFORM instance" ansible-playbook -v \ -i "$INVENTORY_FILE" \ diff --git a/tests/integration/playbooks/check-system.yaml b/tests/integration/playbooks/check-system.yaml index f2c9601d..66854526 100644 --- a/tests/integration/playbooks/check-system.yaml +++ b/tests/integration/playbooks/check-system.yaml @@ -203,6 +203,25 @@ set_fact: failed_counter: "{{ failed_counter | int + 1 }}" + - name: check bootc-fetch-apply-updates.timer left time + shell: systemctl list-timers bootc-fetch-apply-updates.timer --output json | jq -r '.[].left' + register: result_bootc_timer_left + + - name: check bootc-fetch-apply-updates.timer left time greater than 0 + block: + - assert: + that: + - result_bootc_timer_left.stdout | int > 0 + fail_msg: "bootc-fetch-apply-updates.timer won't be triggered" + success_msg: "bootc-fetch-apply-updates.timer is good" + always: + - set_fact: + total_counter: "{{ total_counter | int + 1 }}" + rescue: + - name: failed count + 1 + set_fact: + failed_counter: "{{ failed_counter | int + 1 }}" + - name: check installed package shell: rpm -qa | sort register: result_packages diff --git a/tests/integration/playbooks/rollback.yaml b/tests/integration/playbooks/rollback.yaml new file mode 100644 index 00000000..a801656d --- /dev/null +++ b/tests/integration/playbooks/rollback.yaml @@ -0,0 +1,75 @@ +--- +- hosts: guest + become: false + vars: + total_counter: "0" + failed_counter: "0" + + tasks: + - name: rpm-ostree rollback + command: rpm-ostree rollback + become: true + + - name: Reboot to deploy new system + reboot: + post_reboot_delay: 60 + reboot_timeout: 180 + become: true + ignore_errors: true + + - name: Wait for connection to become reachable/usable + wait_for_connection: + delay: 30 + + - name: rollback checking + block: + - name: get booted cachedupdate imageDigest + shell: bootc status --json | jq -r '.status.booted.cachedUpdate.imageDigest' + register: result_booted_cachedupdate_digest + become: true + + - name: get rollback image digest + shell: bootc status --json | jq -r '.status.rollback.image.imageDigest' + register: result_rollback_image_digest + become: true + + - name: check booted cachedUpdate and rollback image digest + block: + - assert: + that: + - result_booted_cachedupdate_digest.stdout == result_rollback_image_digest.stdout + fail_msg: "rollback failed" + success_msg: "rollback passed" + always: + - set_fact: + total_counter: "{{ total_counter | int + 1 }}" + rescue: + - name: failed count + 1 + set_fact: + failed_counter: "{{ failed_counter | int + 1 }}" + + - name: check installed package + shell: rpm -qa | sort + register: result_packages + + # case: check wget not installed after rollback + - name: check wget not installed + block: + - assert: + that: + - "'wget' not in result_packages.stdout" + fail_msg: "wget installed, ostree rollback might be failed" + success_msg: "wget not installed in ostree rollback" + always: + - set_fact: + total_counter: "{{ total_counter | int + 1 }}" + rescue: + - name: failed count + 1 + set_fact: + failed_counter: "{{ failed_counter | int + 1 }}" + + - assert: + that: + - failed_counter == "0" + fail_msg: "Run {{ total_counter }} tests, but {{ failed_counter }} of them failed" + success_msg: "Totally {{ total_counter }} test passed"