spread.yaml

project: snapd

environment:
    GOHOME: /home/gopath
    GOPATH: $GOHOME
    # TODO: some distros started shipping Go 1.16 which defaults to having
    # modules enabled, drop this when snapd starts supporting Go modules
    GO111MODULE: "off"
    REUSE_PROJECT: '$(HOST: echo "$REUSE_PROJECT")'
    PROJECT_PATH: $GOHOME/src/github.com/snapcore/snapd
    # /usr/lib/go-1.6/bin for trusty (needs to be last as we use
    # a different go in gccgo tests)
    PATH: $GOHOME/bin:/snap/bin:$PATH:/usr/lib/go-1.6/bin:/var/lib/snapd/snap/bin:$PROJECT_PATH/tests/bin
    TESTSLIB: $PROJECT_PATH/tests/lib
    TESTSTOOLS: $PROJECT_PATH/tests/lib/tools
    SNAPPY_TESTING: 1
    # we run the entire suite with re-exec on (the default) and modify
    # the core snap so that it contains our new code.  So we run new
    # snapd from the deb that re-execs into new snapd in core.  To
    # test purely from the deb, set "export SPREAD_SNAP_REEXEC=0"
    SNAP_REEXEC: '$(HOST: echo "${SPREAD_SNAP_REEXEC:-}")'
    MODIFY_CORE_SNAP_FOR_REEXEC: '$(HOST: echo "${SPREAD_MODIFY_CORE_SNAP_FOR_REEXEC:-1}")'
    SPREAD_STORE_USER: '$(HOST: echo "$SPREAD_STORE_USER")'
    SPREAD_STORE_PASSWORD: '$(HOST: echo "$SPREAD_STORE_PASSWORD")'
    SPREAD_STORE_EXPIRED_MACAROON: '$(HOST: echo "$SPREAD_STORE_EXPIRED_MACAROON")'
    SPREAD_STORE_EXPIRED_DISCHARGE: '$(HOST: echo "$SPREAD_STORE_EXPIRED_DISCHARGE")'
    SPREAD_DEBUG_EACH: '$(HOST: echo "${SPREAD_DEBUG_EACH:-1}")'
    LANG: "C.UTF-8"
    LANGUAGE: "en"
    # important to ensure adhoc and linode/qemu behave the same
    SUDO_USER: ""
    SUDO_UID: ""
    TRUST_TEST_KEYS: '$(HOST: echo "${SPREAD_TRUST_TEST_KEYS:-true}")'
    MANAGED_DEVICE: "false"
    # a global setting for LXD channel to use in the tests
    LXD_SNAP_CHANNEL: "latest/candidate"
    CORE_CHANNEL: '$(HOST: echo "${SPREAD_CORE_CHANNEL:-edge}")'
    BASE_CHANNEL: '$(HOST: echo "${SPREAD_BASE_CHANNEL:-edge}")'
    KERNEL_CHANNEL: '$(HOST: echo "${SPREAD_KERNEL_CHANNEL:-edge}")'
    GADGET_CHANNEL: '$(HOST: echo "${SPREAD_GADGET_CHANNEL:-edge}")'
    SNAPD_CHANNEL: '$(HOST: echo "${SPREAD_SNAPD_CHANNEL:-edge}")'
    REMOTE_STORE: '$(HOST: echo "${SPREAD_REMOTE_STORE:-production}")'
    SNAPPY_USE_STAGING_STORE: '$(HOST: if [ "$SPREAD_REMOTE_STORE" = staging ]; then echo 1; else echo 0; fi)'
    DELTA_REF: 2.50
    DELTA_PREFIX: snapd-$DELTA_REF/
    REPACK_KEEP_VENDOR: '$(HOST: echo "${REPACK_KEEP_VENDOR:-n}")'
    SNAPD_PUBLISHED_VERSION: '$(HOST: echo "$SPREAD_SNAPD_PUBLISHED_VERSION")'
    HTTP_PROXY: '$(HOST: echo "$SPREAD_HTTP_PROXY")'
    HTTPS_PROXY: '$(HOST: echo "$SPREAD_HTTPS_PROXY")'
    NO_PROXY: "127.0.0.1"
    NEW_CORE_CHANNEL: '$(HOST: echo "$SPREAD_NEW_CORE_CHANNEL")'
    SRU_VALIDATION: '$(HOST: echo "${SPREAD_SRU_VALIDATION:-0}")'
    # use the ppa_validation_name to install snapd from that ppa
    PPA_VALIDATION_NAME: '$(HOST: echo "${SPREAD_PPA_VALIDATION_NAME:-}")'
    PRE_CACHE_SNAPS: test-snapd-tools test-snapd-sh jq
    # always skip removing the rsync snap
    SKIP_REMOVE_SNAPS: '$(HOST: echo "${SPREAD_SKIP_REMOVE_SNAPS:-}") test-snapd-rsync test-snapd-rsync-core18 test-snapd-rsync-core20'
    # Use the installed snapd and reset the systems without removing snapd
    REUSE_SNAPD: '$(HOST: echo "${SPREAD_REUSE_SNAPD:-0}")'

    # Directory where the nested images and test assets are stored
    NESTED_WORK_DIR: '$(HOST: echo "${NESTED_WORK_DIR:-/tmp/work-dir}")'
    # Channel used to create the nested vm
    NESTED_CORE_CHANNEL: '$(HOST: echo "${NESTED_CORE_CHANNEL:-edge}")'
    # Use cloud init to make initial system configuration instead of user assertion
    NESTED_CORE_REFRESH_CHANNEL: '$(HOST: echo "${NESTED_CORE_REFRESH_CHANNEL:-edge}")'
    # Use cloud init to make initial system configuration instead of user assertion
    NESTED_USE_CLOUD_INIT: '$(HOST: echo "${NESTED_USE_CLOUD_INIT:-true}")'
    # Build and use snapd from current branch
    NESTED_BUILD_SNAPD_FROM_CURRENT: '$(HOST: echo "${NESTED_BUILD_SNAPD_FROM_CURRENT:-true}")'
    # Download and use an custom image from this url
    NESTED_CUSTOM_IMAGE_URL: '$(HOST: echo "${NESTED_CUSTOM_IMAGE_URL:-}")'
    # Configure nested images to be reused on the following tests
    NESTED_CONFIGURE_IMAGES: '$(HOST: echo "${NESTED_CONFIGURE_IMAGES:-false}")'

backends:
    google:
        key: '$(HOST: echo "$SPREAD_GOOGLE_KEY")'
        location: snapd-spread/us-east1-b
        halt-timeout: 2h
        systems:
            - ubuntu-14.04-64:
                  workers: 6
            - ubuntu-16.04-64:
                  workers: 8
            - ubuntu-18.04-32:
                  workers: 6
            - ubuntu-18.04-64:
                  workers: 8
            - ubuntu-20.04-64:
                  workers: 8
            - ubuntu-core-16-64:
                  image: ubuntu-16.04-64
                  workers: 6
            - ubuntu-core-18-64:
                  image: ubuntu-16.04-64
                  workers: 6
            - ubuntu-core-20-64:
                  image: ubuntu-20.04-64
                  workers: 6
                  storage: 20G
            - ubuntu-secboot-20.04-64:
                  image: ubuntu-20.04-64
                  workers: 1
                  secure-boot: true
            - ubuntu-21.04-64:
                  workers: 8
            - ubuntu-21.10-64:
                  image: ubuntu-os-cloud-devel/ubuntu-2110
                  storage: 12G
                  workers: 8

            - debian-9-64:
                  workers: 6
                  manual: true
            - debian-10-64:
                  workers: 6
            - debian-sid-64:
                  workers: 6

            - fedora-33-64:
                  workers: 6
            - fedora-34-64:
                  workers: 6

            - arch-linux-64:
                  workers: 6
            - amazon-linux-2-64:
                  workers: 6
                  storage: preserve-size

            - centos-7-64:
                  workers: 6
                  storage: preserve-size
                  image: centos-7-64
            # unstable systems below
            - opensuse-15.1-64:
                  workers: 6
            - opensuse-15.2-64:
                  workers: 6
            - opensuse-tumbleweed-64:
                  workers: 6
                  manual: true
            - centos-8-64:
                  workers: 6
                  storage: preserve-size
                  image: centos-8-64

    google-sru:
        type: google
        key: '$(HOST: echo "$SPREAD_GOOGLE_KEY")'
        location: snapd-spread/us-east1-b
        halt-timeout: 2h
        systems:
            - ubuntu-18.04-64:
                  workers: 6
            - ubuntu-20.04-64:
                  workers: 6
            - ubuntu-21.04-64:
                  workers: 6

    google-nested:
        type: google
        key: '$(HOST: echo "$SPREAD_GOOGLE_KEY")'
        location: snapd-spread/us-east1-b
        plan: n2-standard-2
        halt-timeout: 2h
        systems:
            - ubuntu-16.04-64:
                  image: ubuntu-1604-64-virt-enabled
                  storage: 20G
                  workers: 3
            - ubuntu-18.04-64:
                  image: ubuntu-1804-64-virt-enabled
                  storage: 20G
                  workers: 3
            - ubuntu-20.04-64:
                  image: ubuntu-2004-64-virt-enabled
                  storage: 20G
                  workers: 7
            - ubuntu-21.04-64:
                  image: ubuntu-2104-64-virt-enabled
                  storage: 20G
                  workers: 3

    qemu-nested:
        memory: 4G
        type: qemu
        systems:
            - ubuntu-16.04-64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-18.04-64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-20.04-64:
                  username: ubuntu
                  password: ubuntu

    qemu:
        systems:
            - ubuntu-14.04-32:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-14.04-64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-16.04-32:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-16.04-64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-core-16-64:
                  image: ubuntu-16.04-64
                  username: ubuntu
                  password: ubuntu
            - ubuntu-core-18-64:
                  image: ubuntu-16.04-64
                  username: ubuntu
                  password: ubuntu
            - ubuntu-core-20-64:
                  image: ubuntu-20.04-64
                  username: ubuntu
                  password: ubuntu
                  flags: [virtio]
            - ubuntu-18.04-64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-18.04-32:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-20.04-64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-20.04-32:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-20.10-64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-21.04-64:
                  username: ubuntu
                  password: ubuntu
            - debian-sid-64:
                  username: debian
                  password: debian
            - debian-9-64:
                  username: debian
                  password: debian
            - debian-sid-64:
                  username: debian
                  password: debian
            - centos-7-64:
                  username: centos
                  password: centos
            - amazon-linux-2-64:
                  username: ec2-user
                  password: ec2-user
            - opensuse-15.2-64:
                  username: opensuse
                  password: opensuse
            - opensuse-tumbleweed-64:
                  username: opensuse
                  password: opensuse
    autopkgtest:
        type: adhoc
        allocate: |
            echo "Allocating ad-hoc $SPREAD_SYSTEM"
            if [ -z "${ADT_ARTIFACTS}" ]; then
                FATAL "adhoc only works inside autopkgtest"
                exit 1
            fi
            echo 'ubuntu ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/99-spread-users
            ADDRESS localhost:22
        discard: |
            echo "Discarding ad-hoc $SPREAD_SYSTEM"
        systems:
            # Trusty
            - ubuntu-14.04-amd64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-14.04-i386:
                  username: ubuntu
                  password: ubuntu
            # Xenial
            - ubuntu-16.04-amd64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-16.04-i386:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-16.04-ppc64el:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-16.04-armhf:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-16.04-s390x:
                  username: ubuntu
                  password: ubuntu
            # Artful
            - ubuntu-17.10-amd64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-17.10-i386:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-17.10-ppc64el:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-17.10-armhf:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-17.10-s390x:
                  username: ubuntu
                  password: ubuntu
            # Bionic
            - ubuntu-18.04-amd64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-18.04-i386:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-18.04-ppc64el:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-18.04-armhf:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-18.04-s390x:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-18.04-arm64:
                  username: ubuntu
                  password: ubuntu
            # Focal
            - ubuntu-20.04-amd64:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-20.04-i386:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-20.04-ppc64el:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-20.04-armhf:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-20.04-s390x:
                  username: ubuntu
                  password: ubuntu
            - ubuntu-20.04-arm64:
                  username: ubuntu
                  password: ubuntu

    external:
        type: adhoc
        environment:
            SPREAD_EXTERNAL_ADDRESS: '$(HOST: echo "${SPREAD_EXTERNAL_ADDRESS:-localhost:8022}")'
            MANAGED_DEVICE: "true"
            TRUST_TEST_KEYS: "false"
        allocate: |
            ADDRESS $SPREAD_EXTERNAL_ADDRESS
        systems:
            - ubuntu-core-16-64:
                  username: external
                  password: ubuntu
            - ubuntu-core-16-32:
                  username: external
                  password: ubuntu
            - ubuntu-core-16-arm-64:
                  username: external
                  password: ubuntu
            - ubuntu-core-16-arm-32:
                  username: external
                  password: ubuntu
            - ubuntu-core-18-64:
                  username: external
                  password: ubuntu
            - ubuntu-core-18-32:
                  username: external
                  password: ubuntu
            - ubuntu-core-18-arm-64:
                  username: external
                  password: ubuntu
            - ubuntu-core-18-arm-32:
                  username: external
                  password: ubuntu
            - ubuntu-core-20-64:
                  username: external
                  password: ubuntu
            - ubuntu-core-20-32:
                  username: external
                  password: ubuntu
            - ubuntu-core-20-arm-64:
                  username: external
                  password: ubuntu
            - ubuntu-core-20-arm-32:
                  username: external
                  password: ubuntu

path: /home/gopath/src/github.com/snapcore/snapd

exclude:
    - .git
    - cmd/snap/snap
    - cmd/snapd/snapd
    - cmd/snapctl/snapctl
    - cmd/snap-exec/snap-exec
    - cmd/autom4te.cache
    - "*.o"
    - "*.a"
    - ./vendor
    - "*.snap"
    - "*.img"

debug-each: |
    if [ "$SPREAD_DEBUG_EACH" = 1 ]; then
        #shellcheck source=tests/lib/state.sh
        . "$TESTSLIB/state.sh"
        #shellcheck source=tests/lib/nested.sh
        . "$TESTSLIB/nested.sh"

        echo '# System information'
        cat /etc/os-release || true

        echo '# Kernel information'
        uname -a

        echo '# Go information'
        go version || true

        if nested_is_nested_system; then
            echo '# nested VM status'
            systemctl status nested-vm || true
            nested_print_serial_log
            # add another echo in case the serial log is missing a newline
            echo

            tests.nested exec "sudo journalctl --no-pager -u snapd" || true
        fi

        echo '# journal messages for snapd'
        "$TESTSTOOLS"/journal-state get-log -u snapd
        case "$SPREAD_SYSTEM" in
            fedora-*|centos-*|amazon-*)
                if [ -e "$RUNTIME_STATE_PATH/audit-stamp" ]; then
                    ausearch -i -m AVC --checkpoint "$RUNTIME_STATE_PATH/audit-stamp" --start checkpoint || true
                else
                    ausearch -i -m AVC || true
                fi
                (
                    find /root/snap -printf '%Z\t%H/%P\n' || true
                    find /home -regex '/home/[^/]*/snap\(/.*\)?' -printf '%Z\t%H/%P\n' || true
                ) | grep -v snappy_home_t || true
                find /var/snap -printf '%Z\t%H/%P\n' | grep -v snappy_var_t || true
                ;;
            opensuse-*)
                echo '# apparmor denials logged by auditd'
                ausearch -m AVC | grep DENIED || true
                ;;
            *)
                echo '# apparmor denials '
                dmesg --ctime | grep DENIED || true
                ;;
        esac
        echo '# seccomp denials (kills) '
        dmesg --ctime | grep type=1326 || true
        echo '# snap connections --all'
        snap connections --all || true
        echo '# tasks executed on system'
        cat "$RUNTIME_STATE_PATH/runs" || true
        echo '# free space'
        df -h || true
        echo '# mounts'
        # use ascii output to prevent travis from messing up the encoding
        findmnt --ascii -o+PROPAGATION || true
        echo "# processes"
        ps axl
        echo "# /var/lib/snapd"
        find /var/lib/snapd/ -not -path '/var/lib/snapd/snap/*' -ls || true
    fi

rename:
    # Move content into a directory, so that deltas computed by repack benefit
    # from the content looking similar to codeload.github.com.
    - s,^,$DELTA_PREFIX,S

repack: |
    # For Linode, compute a delta based on a known git reference that can be
    # obtained directly from GitHub. There's nothing special about that reference,
    # other than it will often be in the local repository's history already.
    # The more recent the reference, the smaller the delta.
    if ! echo "$SPREAD_BACKENDS" | grep -e linode -e google; then
        cat <&3 >&4
    elif ! git show-ref "$DELTA_REF" > /dev/null; then
        cat <&3 >&4
    else
        tmpdir="$(mktemp -d)"
        #shellcheck disable=SC2064
        trap "rm -rf delta-ref.tar current.delta repacked-current.tar $tmpdir" EXIT
        if [ "$REPACK_KEEP_VENDOR" = "n" ]; then
            tar -C "$tmpdir" -xvf - <&3
            find "$tmpdir/$DELTA_PREFIX/vendor/" ! -wholename "$tmpdir/$DELTA_PREFIX/vendor/" \
                ! -wholename "$tmpdir/$DELTA_PREFIX/vendor/vendor.json" \
                -delete
            tar -C "$tmpdir" -c "$DELTA_PREFIX" --sort=name > repacked-current.tar
        else
            cat <&3 > repacked-current.tar
        fi
        git archive -o delta-ref.tar --format=tar --prefix="$DELTA_PREFIX" "$DELTA_REF"
        xdelta3 -S none -s delta-ref.tar repacked-current.tar > current.delta
        tar c current.delta >&4
    fi

kill-timeout: 30m

prepare: |
    # NOTE: This part of the code needs to be in spread.yaml as it runs before
    # the rest of the source code (including the tests/lib directory) is
    # around. The purpose of this code is to fix some connectivity issues and
    # then apply the delta of the git repository.

    # apt update is hanging on security.ubuntu.com with IPv6, prefer IPv4 over IPv6
    cat <<EOF > gai.conf
    precedence  ::1/128       50
    precedence  ::/0          40
    precedence  2002::/16     30
    precedence ::/96          20
    precedence ::ffff:0:0/96 100
    EOF
    if ! mv gai.conf /etc/gai.conf; then
        echo "/etc/gai.conf is not writable, ubuntu-core system? apt update won't be affected in that case"
        rm -f gai.conf
    fi
    if command -v restorecon ; then
        # restore proper context otherwise SELinux may complain
        restorecon -v /etc/gai.conf
    fi

    # Note that os.query or any other tool cannot be used here before the current.delta file is unpacked
    if [[ "$SPREAD_SYSTEM" == fedora-* ]]; then
        # The Fedora archive mirror seems to be unreliable.
        # Switch to the main archive by commenting out metalink and uncommenting
        # baseurl with a tweak to go to dl.fedoraproject.org which doens't redirect
        # to mirrors again.
        #
        # https://forum.snapcraft.io/t/issues-with-the-fedora-mirror-network/3489/
        sed -i -s -E -e 's@^#?baseurl=http://download.fedoraproject.org/@baseurl=http://dl.fedoraproject.org/@g' -e 's@^metalink=@#metalink@g' /etc/yum.repos.d/fedora*.repo

        dnf --refresh -y makecache

        # enable audit daemon
        systemctl enable --now auditd.service
    fi

    if [[ "$SPREAD_SYSTEM" == opensuse-* ]]; then
        # refresh metadatadata
        # Auto import gpg keys needed for could repository added to support google backend
        zypper --gpg-auto-import-keys ref
        # We seem to be hitting a flaky openSUSE mirror from time to time,
        # increase the number of download attempts libzypp will try to
        # workaround that.
        cat <<-EOF >> /etc/zypp/zypp.conf
    # added by spread tests
    download.max_silent_tries = 20
    EOF

        # Make sure docs are installed with the packages
        sed 's/rpm.install.excludedocs = yes/rpm.install.excludedocs = no/g' -i /etc/zypp/zypp.conf
    fi

    if [[ "$SPREAD_SYSTEM" == arch-* ]]; then
        # Possible that AppArmor was not started and is not enabled in the
        # image, do both now
        if systemctl show -p LoadState apparmor.service | MATCH 'LoadState=loaded' ; then
            if ! systemctl is-enabled apparmor.service; then
                systemctl enable apparmor.service
            fi
            systemctl start apparmor.service
        else
            exit 1
        fi
    fi

    if [[ "$SPREAD_SYSTEM" == debian-* ]]; then
        apt-get update && apt-get install -y eatmydata
    fi

    if [[ "$SPREAD_SYSTEM" == centos-* ]]; then
        # make sure EPEL is enabled
        yum install -y epel-release
    fi

    case "$SPREAD_SYSTEM" in
        ubuntu-*|debian-*)
            # make sure unattended-upgrades does not get in the way
            if systemctl is-enabled unattended-upgrades.service; then
                systemctl stop unattended-upgrades.service
                systemctl mask unattended-upgrades.service
            fi
            ;;
    esac

    # Unpack delta, or move content out of the prefixed directory (see rename and repack above).
    # (needs to be in spread.yaml directly because there's nothing else on the filesystem yet)
    if [ -f current.delta ]; then
        tf=$(mktemp)
        # NOTE: We can't use tests/lib/pkgdb.sh here as it doesn't exist at
        # this time when none of the test files is yet in place.
        case "$SPREAD_SYSTEM" in
            ubuntu-*|debian-*)
                apt-get update >& "$tf" || ( cat "$tf"; exit 1 )
                apt-get install -y xdelta3 curl eatmydata >& "$tf" || ( cat "$tf"; exit 1 )
                ;;
            amazon-*|centos-7-*)
                yum install -y xdelta curl &> "$tf" || (cat "$tf"; exit 1)
                ;;
            fedora-*|centos-*)
                dnf install --refresh -y xdelta curl &> "$tf" || (cat "$tf"; exit 1)
                ;;
            opensuse-*)
                zypper -q --gpg-auto-import-keys refresh
                zypper -q install -y xdelta3 curl &> "$tf" || (cat "$tf"; exit 1)
                ;;
            arch-*)
                # there may be a libc upgrade which only -Syu handles;
                # ignore linux kernel as we would fail to detect it and handle
                # reboot; actual distro upgrade is done later in prepare.
                pacman -Syu --noconfirm xdelta3 curl --ignore linux &> "$tf" || (cat "$tf"; exit 1)
                ;;
        esac
        rm -f "$tf"
        curl -sS -o - "https://codeload.github.com/snapcore/snapd/tar.gz/$DELTA_REF" | gunzip > delta-ref.tar
        xdelta3 -q -c -d -s delta-ref.tar current.delta | tar x --strip-components=1
        rm -f delta-ref.tar current.delta
    elif [ -d "$DELTA_PREFIX" ]; then
        find "$DELTA_PREFIX" -mindepth 1 -maxdepth 1 -exec mv {} . \;
        rmdir "$DELTA_PREFIX"
    fi

    # TODO: drop once 21.10 images are fixed
    if [[ "$SPREAD_SYSTEM" == ubuntu-21.10-* ]] && [[ -e /home/ubuntu/.ssh ]]; then
        chown -R ubuntu:ubuntu /home/ubuntu/.ssh
    fi

    # Take the MATCH and REBOOT functions from spread and allow our shell
    # scripts to use them as shell commands. The replacements are real
    # executables in tests/lib/bin (which is on PATH) but they source
    # spread-funcs.sh written here, base on the definitions provided by SPREAD.
    # This ensures that 1) spread functions define the code 2) both MATCH and
    # REBOOT are executables and not functions, and can be called from any
    # context.
    type MATCH | tail -n +2 > "$TESTSLIB"/spread-funcs.sh
    unset MATCH
    type NOMATCH | tail -n +2 >> "$TESTSLIB"/spread-funcs.sh
    unset NOMATCH
    type REBOOT | tail -n +2 >> "$TESTSLIB"/spread-funcs.sh
    unset REBOOT

    # NOTE: At this stage the source tree is available and no more special
    # considerations apply.
    "$TESTSLIB"/prepare-restore.sh --prepare-project
prepare-each: |
    "$TESTSLIB"/prepare-restore.sh --prepare-project-each
restore: |
    "$TESTSLIB"/prepare-restore.sh --restore-project
restore-each: |
    "$TESTSLIB"/prepare-restore.sh --restore-project-each
suites:
    tests/lib/tools/suite/:
        summary: Tests for tests/lib/tools tools
        prepare-each: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite-each-minimal-no-snaps
        restore-each: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite-each-minimal-no-snaps
    # The essential tests designed to run inside the autopkgtest
    # environment on each platform. On autopkgtest we cannot run all tests
    # as this is very slow and we run into timeouts.
    #
    # These tests are executed on all other plattforms as they
    # are designed to run on pristine systems
    tests/smoke/:
        summary: Essential system level tests for snapd
        prepare: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite
        prepare-each: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite-each
        restore-each: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite-each
        restore: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite
    # All other tests run now and will heavily modify the system.
    tests/main/:
        summary: Full-system tests for snapd
        systems: [-ubuntu-secboot-*]
        prepare: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite
        prepare-each: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite-each
        restore-each: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite-each
        restore: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite
        debug: |
            if [ "$SPREAD_DEBUG_EACH" = 1 ]; then
                systemctl status snapd.socket || true
                journalctl -e
            fi
    tests/core/:
        summary: Subset of Ubuntu Core specific tests
        systems: [ubuntu-core-*]
        prepare: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite
        prepare-each: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite-each
        restore-each: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite-each
        restore: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite
    tests/completion/:
        summary: completion tests
        # ppc64el disabled because of https://bugs.launchpad.net/snappy/+bug/1655594
        systems: [-ubuntu-core-*, -ubuntu-*-ppc64el, -ubuntu-secboot-*]
        prepare: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite
        prepare-each: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite-each
        restore-each: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite-each
        restore: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite
        environment:
            _/plain: _
            _/plain_plusdirs: _
            _/funky: _
            _/files: _
            # dirs fails on indirection because of (mis)handling of trailing
            # slashes. This might be configuration-dependent.
            # _/dirs: _
            _/hosts: _
            _/hosts_n_dirs: _
            # twisted fails in travis (but not regular spread).
            # _/twisted: _
            _/func: _
            _/funkyfunc: _
            _/funcarg: _

    tests/regression/:
        summary: Regression tests for snapd
        systems: [-ubuntu-secboot-*]
        prepare: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite
        prepare-each: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite-each
        restore-each: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite-each
        restore: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite

    tests/upgrade/:
        summary: Tests for snapd upgrade
        # Test cases are not yet ported to openSUSE that is why we keep
        # it disabled. A later PR will enable most tests and
        # drop this blacklist.
        systems: [-ubuntu-core-*, -opensuse-*, -ubuntu-secboot-*]
        prepare-each: |
            # FIXME: this should really use prepare-restore.sh --prepare-suite-each
            # like other suites, needs more investigation

            # shellcheck source=tests/lib/state.sh
            . "$TESTSLIB"/state.sh
            mkdir -p "$RUNTIME_STATE_PATH"
            # save the job which is going to be executed in the system
            echo -n "$SPREAD_JOB " >> "$RUNTIME_STATE_PATH/runs"
        restore: |
            if [ "$REMOTE_STORE" = staging ]; then
                echo "skip upgrade tests while talking to the staging store"
                exit 0
            fi
        restore-each: |
            if [ "$REMOTE_STORE" = staging ]; then
                echo "skip upgrade tests while talking to the staging store"
                exit 0
            fi
            #shellcheck source=tests/lib/pkgdb.sh
            . "$TESTSLIB"/pkgdb.sh
            distro_purge_package snapd
            distro_purge_package snapd-xdg-open || true
    tests/cross/:
        summary: Cross-compile tests
        systems: [ubuntu-16.04-64, ubuntu-18.04-64]

    tests/unit/:
        summary: Suite to run unit tests (non-go and different go runtimes)
        # Test cases are not yet ported to Fedora/openSUSE/Arch that is why
        # we keep them disabled. A later PR will enable most tests and
        # drop this blacklist.
        systems:
            [
                -ubuntu-core-*,
                -fedora-*,
                -opensuse-*,
                -arch-*,
                -amazon-*,
                -centos-*,
                -ubuntu-secboot-*,
            ]
        # unittests are run as part of the autopkgtest build already
        backends: [-autopkgtest]
        environment:
            # env vars required for coverage reporting from a spread task
            TRAVIS_BUILD_NUMBER: '$(HOST: echo "$TRAVIS_BUILD_NUMBER")'
            TRAVIS_BRANCH: '$(HOST: echo "$TRAVIS_BRANCH")'
            TRAVIS_COMMIT: '$(HOST: echo "$TRAVIS_COMMIT")'
            TRAVIS_JOB_NUMBER: '$(HOST: echo "$TRAVIS_JOB_NUMBER")'
            TRAVIS_PULL_REQUEST: '$(HOST: echo "$TRAVIS_PULL_REQUEST")'
            TRAVIS_JOB_ID: '$(HOST: echo "$TRAVIS_JOB_ID")'
            TRAVIS_REPO_SLUG: '$(HOST: echo "$TRAVIS_REPO_SLUG")'
            TRAVIS_TAG: '$(HOST: echo "$TRAVIS_TAG")'
            COVERMODE: '$(HOST: echo "$COVERMODE")'
        prepare: |
            #shellcheck source=tests/lib/prepare.sh
            . "$TESTSLIB"/prepare.sh
            prepare_classic
        prepare-each: |
            "$TESTSLIB"/reset.sh --reuse-core
            #shellcheck source=tests/lib/prepare.sh
            . "$TESTSLIB"/prepare.sh
            prepare_each_classic
        restore: |
            "$TESTSLIB"/reset.sh --store
            #shellcheck source=tests/lib/pkgdb.sh
            . "$TESTSLIB"/pkgdb.sh

            distro_purge_package snapd
            case "$SPREAD_SYSTEM" in
                arch-*)
                    # there is no snap-confine and ubuntu-core-launcher
                    # in Arch
                    ;;
                *)
                    distro_purge_package snap-confine ubuntu-core-launcher
                    ;;
            esac

    tests/nightly/:
        summary: Suite for nightly, expensive, tests
        manual: true
        # Test cases are not yet ported to Fedora/openSUSE/Arch/AMZN2 that is why
        # we keep them disabled. A later PR will enable most tests and
        # drop this blacklist.
        prepare: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite
        prepare-each: |
            "$TESTSLIB"/prepare-restore.sh --prepare-suite-each
        restore-each: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite-each
        restore: |
            "$TESTSLIB"/prepare-restore.sh --restore-suite

    tests/nested/manual/:
        summary: Tests for nested images controlled manually from the tests
        backends: [google-nested, qemu-nested]
        systems:
            - ubuntu-16.04-64
            - ubuntu-18.04-64
            - ubuntu-20.04-64
            - ubuntu-20.10-64
        environment:
            NESTED_TYPE: "classic"
            # Enable kvm in the qemu command line
            NESTED_ENABLE_KVM: '$(HOST: echo "${NESTED_ENABLE_KVM:-true}")'
            # Enable tpm in the nested vm in case it is supported
            NESTED_ENABLE_TPM: '$(HOST: echo "${NESTED_ENABLE_TPM:-}")'
            # Enable secure boot in the nested vm in case it is supported
            NESTED_ENABLE_SECURE_BOOT: '$(HOST: echo "${NESTED_ENABLE_SECURE_BOOT:-}")'
        manual: true
        warn-timeout: 10m
        kill-timeout: 60m
        prepare: |
            #shellcheck source=tests/lib/pkgdb.sh
            . "$TESTSLIB"/pkgdb.sh
            distro_update_package_db
            distro_install_package snapd qemu qemu-utils genisoimage sshpass qemu-kvm cloud-image-utils ovmf kpartx xz-utils
            snap install ubuntu-image --classic

            # Install the snapd built
            dpkg -i "$SPREAD_PATH"/../snapd_*.deb
        prepare-each: |
            tests.backup prepare
            tests.nested prepare
        restore-each: |
            tests.nested remove-vm
            tests.nested restore
            tests.backup restore
        restore: |
            #shellcheck source=tests/lib/pkgdb.sh
            . "$TESTSLIB"/pkgdb.sh
            distro_purge_package qemu genisoimage sshpass qemu-kvm cloud-image-utils xz-utils

    tests/nested/classic/:
        summary: Tests for nested images
        backends: [google-nested, qemu-nested]
        systems: [ubuntu-16.04-64, ubuntu-18.04-64, ubuntu-20*]
        environment:
            NESTED_TYPE: "classic"
            # Channel used to create the nested vm
            NESTED_ENABLE_KVM: '$(HOST: echo "${NESTED_ENABLE_KVM:-true}")'
            # Enable tpm in the nested vm in case it is supported
            NESTED_ENABLE_TPM: '$(HOST: echo "${NESTED_ENABLE_TPM:-false}")'
            # Enable secure boot in the nested vm in case it is supported
            NESTED_ENABLE_SECURE_BOOT: '$(HOST: echo "${NESTED_ENABLE_SECURE_BOOT:-false}")'
        manual: true
        prepare: |
            #shellcheck source=tests/lib/pkgdb.sh
            . "$TESTSLIB"/pkgdb.sh
            distro_update_package_db
            distro_install_package snapd qemu qemu-utils genisoimage sshpass qemu-kvm cloud-image-utils ovmf kpartx xz-utils
            snap install ubuntu-image --classic

            # Install the snapd built
            dpkg -i "$SPREAD_PATH"/../snapd_*.deb

            tests.nested prepare
            tests.nested build-image classic
        prepare-each: |
            tests.backup prepare
            tests.nested create-vm classic
        restore-each: |
            tests.nested remove-vm
            tests.backup restore
        restore: |
            tests.nested restore

            #shellcheck source=tests/lib/pkgdb.sh
            . "$TESTSLIB"/pkgdb.sh
            distro_purge_package qemu genisoimage sshpass qemu-kvm cloud-image-utils xz-utils

    tests/nested/core/:
        summary: Tests for nested images
        backends: [google-nested, qemu-nested]
        systems: [ubuntu-16.04-64, ubuntu-18.04-64, ubuntu-20.04-64]
        environment:
            NESTED_TYPE: "core"
            # Enable kvm in the qemu command line
            NESTED_ENABLE_KVM: '$(HOST: echo "${NESTED_ENABLE_KVM:-}")'
            # Enable tpm in the nested vm in case it is supported
            NESTED_ENABLE_TPM: '$(HOST: echo "${NESTED_ENABLE_TPM:-}")'
            # Enable secure boot in the nested vm in case it is supported
            NESTED_ENABLE_SECURE_BOOT: '$(HOST: echo "${NESTED_ENABLE_SECURE_BOOT:-}")'
        manual: true
        prepare: |
            #shellcheck source=tests/lib/pkgdb.sh
            . "$TESTSLIB"/pkgdb.sh
            distro_update_package_db
            distro_install_package snapd qemu qemu-utils genisoimage sshpass qemu-kvm cloud-image-utils ovmf kpartx xz-utils
            snap install ubuntu-image --classic

            # Install the snapd built
            dpkg -i "$SPREAD_PATH"/../snapd_*.deb

            tests.nested prepare
            tests.nested build-image core
        prepare-each: |
            tests.backup prepare
            tests.nested create-vm core
        restore-each: |
            tests.nested remove-vm
            tests.backup restore
        restore: |
            tests.nested restore

            #shellcheck source=tests/lib/pkgdb.sh
            . "$TESTSLIB"/pkgdb.sh
            distro_purge_package qemu genisoimage sshpass qemu-kvm cloud-image-utils xz-utils

# vim:ts=4:sw=4:et