test: add rollback and bootc upgrade fetch timer test

To avoid hitting quay.io race limitation, add retry for podman
push and podman build

Signed-off-by: Xiaofeng Wang <henrywangxf@me.com>

tests/integration/install-upgrade.sh

@@ -10,6 +10,16 @@ function redprint {
     echo -e "\033[1;31m[$(date -Isecond)] ${1}\033[0m"
 }
 
+function retry {
+    n=0
+    until [ "$n" -ge 3 ]
+    do
+       "$@" && break
+       n=$((n+1))
+       sleep 10
+    done
+}
+
 TEMPDIR=$(mktemp -d)
 trap 'rm -rf -- "$TEMPDIR"' EXIT
 
@@ -92,10 +102,10 @@ greenprint "Login quay.io"
 podman login -u "${QUAY_USERNAME}" -p "${QUAY_PASSWORD}" quay.io
 
 greenprint "Build $TEST_OS installation container image"
-podman build --tls-verify=false -t "${TEST_IMAGE_NAME}:${QUAY_REPO_TAG}" -f "$INSTALL_CONTAINERFILE" .
+podman build --tls-verify=false --retry=5 --retry-delay=10 -t "${TEST_IMAGE_NAME}:${QUAY_REPO_TAG}" -f "$INSTALL_CONTAINERFILE" .
 
 greenprint "Push $TEST_OS installation container image"
-podman push "${TEST_IMAGE_NAME}:${QUAY_REPO_TAG}" "$TEST_IMAGE_URL"
+retry podman push --tls-verify=false --quiet "${TEST_IMAGE_NAME}:${QUAY_REPO_TAG}" "$TEST_IMAGE_URL"
 
 greenprint "Prepare inventory file"
 tee -a "$INVENTORY_FILE" > /dev/null << EOF
@@ -146,9 +156,10 @@ RUN dnf -y install wget && \
 EOF
 
 greenprint "Build $TEST_OS upgrade container image"
-podman build --tls-verify=false -t "${TEST_IMAGE_NAME}:${QUAY_REPO_TAG}" -f "$UPGRADE_CONTAINERFILE" .
+podman build --tls-verify=false --retry=5 --retry-delay=10 -t "${TEST_IMAGE_NAME}:${QUAY_REPO_TAG}" -f "$UPGRADE_CONTAINERFILE" .
+
 greenprint "Push $TEST_OS upgrade container image"
-podman push "${TEST_IMAGE_NAME}:${QUAY_REPO_TAG}" "$TEST_IMAGE_URL"
+retry podman push --tls-verify=false --quiet "${TEST_IMAGE_NAME}:${QUAY_REPO_TAG}" "$TEST_IMAGE_URL"
 
 greenprint "Upgrade $TEST_OS system"
 ansible-playbook -v \
@@ -162,6 +173,11 @@ ansible-playbook -v \
     -e upgrade="true" \
     playbooks/check-system.yaml
 
+greenprint "Rollback $TEST_OS system"
+ansible-playbook -v \
+    -i "$INVENTORY_FILE" \
+    playbooks/rollback.yaml
+
 greenprint "Remove $PLATFORM instance"
 ansible-playbook -v \
     -i "$INVENTORY_FILE" \

tests/integration/playbooks/check-system.yaml

@@ -203,6 +203,25 @@
           set_fact:
             failed_counter: "{{ failed_counter | int + 1 }}"
 
+    - name: check bootc-fetch-apply-updates.timer left time
+      shell: systemctl list-timers bootc-fetch-apply-updates.timer --output json | jq -r '.[].left'
+      register: result_bootc_timer_left
+
+    - name: check bootc-fetch-apply-updates.timer left time greater than 0
+      block:
+        - assert:
+            that:
+              - result_bootc_timer_left.stdout | int > 0
+            fail_msg: "bootc-fetch-apply-updates.timer won't be triggered"
+            success_msg: "bootc-fetch-apply-updates.timer is good"
+      always:
+        - set_fact:
+            total_counter: "{{ total_counter | int + 1 }}"
+      rescue:
+        - name: failed count + 1
+          set_fact:
+            failed_counter: "{{ failed_counter | int + 1 }}"
+
     - name: check installed package
       shell: rpm -qa | sort
       register: result_packages

tests/integration/playbooks/rollback.yaml

@@ -0,0 +1,75 @@
+---
+- hosts: guest
+  become: false
+  vars:
+    total_counter: "0"
+    failed_counter: "0"
+
+  tasks:
+    - name: rpm-ostree rollback
+      command: rpm-ostree rollback
+      become: true
+
+    - name: Reboot to deploy new system
+      reboot:
+        post_reboot_delay: 60
+        reboot_timeout: 180
+      become: true
+      ignore_errors: true
+
+    - name: Wait for connection to become reachable/usable
+      wait_for_connection:
+        delay: 30
+
+    - name: rollback checking
+      block:
+        - name: get booted cachedupdate imageDigest
+          shell: bootc status --json | jq -r '.status.booted.cachedUpdate.imageDigest'
+          register: result_booted_cachedupdate_digest
+          become: true
+
+        - name: get rollback image digest
+          shell: bootc status --json | jq -r '.status.rollback.image.imageDigest'
+          register: result_rollback_image_digest
+          become: true
+
+        - name: check booted cachedUpdate and rollback image digest
+          block:
+            - assert:
+                that:
+                  - result_booted_cachedupdate_digest.stdout == result_rollback_image_digest.stdout
+                fail_msg: "rollback failed"
+                success_msg: "rollback passed"
+          always:
+            - set_fact:
+                total_counter: "{{ total_counter | int + 1 }}"
+          rescue:
+            - name: failed count + 1
+              set_fact:
+                failed_counter: "{{ failed_counter | int + 1 }}"
+
+        - name: check installed package
+          shell: rpm -qa | sort
+          register: result_packages
+
+        # case: check wget not installed after rollback
+        - name: check wget not installed
+          block:
+            - assert:
+                that:
+                  - "'wget' not in result_packages.stdout"
+                fail_msg: "wget installed, ostree rollback might be failed"
+                success_msg: "wget not installed in ostree rollback"
+          always:
+            - set_fact:
+                total_counter: "{{ total_counter | int + 1 }}"
+          rescue:
+            - name: failed count + 1
+              set_fact:
+                failed_counter: "{{ failed_counter | int + 1 }}"
+
+    - assert:
+        that:
+          - failed_counter == "0"
+        fail_msg: "Run {{ total_counter }} tests, but {{ failed_counter }} of them failed"
+        success_msg: "Totally {{ total_counter }} test passed"