From 2ff555eb05d8aa71225614cf5fb4b4cf4437c7ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Doktor?= Date: Wed, 31 Jan 2024 09:06:16 +0100 Subject: [PATCH 1/4] tests.e2e.ansible: Add retries to steps using external sources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit the network in CI environment tends to break from time to time, let's allow up to 3 retries for tasks that support it and that use external sources. Fixes: #339 Signed-off-by: Lukáš Doktor --- tests/e2e/ansible/install_docker.yaml | 18 ++++++++++++++++++ tests/e2e/ansible/install_kubeadm.yaml | 2 ++ tests/e2e/ansible/install_test_deps.yaml | 6 ++++++ tests/e2e/ansible/start_docker_registry.yaml | 6 ++++++ 4 files changed, 32 insertions(+) diff --git a/tests/e2e/ansible/install_docker.yaml b/tests/e2e/ansible/install_docker.yaml index 8577e781..066edb85 100644 --- a/tests/e2e/ansible/install_docker.yaml +++ b/tests/e2e/ansible/install_docker.yaml @@ -22,17 +22,23 @@ - gnupg - lsb-release state: present + retries: 3 + delay: 10 - name: Add docker repo GPG key apt_key: url: https://download.docker.com/linux/ubuntu/gpg keyring: /etc/apt/trusted.gpg.d/docker.gpg state: present + retries: 3 + delay: 10 - name: Add docker repo apt_repository: repo: "deb [arch={{ target_arch }} signed-by=/etc/apt/trusted.gpg.d/docker.gpg] https://download.docker.com/linux/ubuntu {{ ansible_lsb.codename }} stable" filename: docker update_cache: yes state: present + retries: 3 + delay: 10 - name: Install docker packages package: name: @@ -40,6 +46,8 @@ - docker-ce - docker-ce-cli state: present + retries: 3 + delay: 10 - name: Create the docker group group: name: docker @@ -54,10 +62,14 @@ - docker.io - docker-buildx state: present + retries: 3 + delay: 10 - name: Create the docker group group: name: docker state: present + retries: 3 + delay: 10 when: docker_exist.rc != 0 and ansible_distribution == "Ubuntu" and ansible_distribution_version == "22.04" - name: Handle docker installation on CentOS. block: @@ -65,10 +77,14 @@ dnf: name: yum-utils state: present + retries: 3 + delay: 10 - name: Add docker yum repo shell: yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo args: creates: /etc/yum.repos.d/docker-ce.repo + retries: 3 + delay: 10 - name: Install docker packages dnf: name: @@ -76,6 +92,8 @@ - docker-ce - docker-ce-cli state: present + retries: 3 + delay: 10 when: docker_exist.rc != 0 and ansible_distribution == "CentOS" # # In order to prevent "systemd: docker.service Start request repeated too quickly" diff --git a/tests/e2e/ansible/install_kubeadm.yaml b/tests/e2e/ansible/install_kubeadm.yaml index 97ef5a5b..90787fdb 100644 --- a/tests/e2e/ansible/install_kubeadm.yaml +++ b/tests/e2e/ansible/install_kubeadm.yaml @@ -46,6 +46,8 @@ url: https://storage.googleapis.com/kubernetes-release/release/{{ k8s_version }}/bin/linux/{{ target_arch }}/{{ item }} dest: /usr/local/bin mode: '+x' + retries: 3 + delay: 10 with_items: - kubeadm - kubelet diff --git a/tests/e2e/ansible/install_test_deps.yaml b/tests/e2e/ansible/install_test_deps.yaml index 527bc413..8410d171 100644 --- a/tests/e2e/ansible/install_test_deps.yaml +++ b/tests/e2e/ansible/install_test_deps.yaml @@ -8,6 +8,8 @@ package: name: "{{ test_pkgs[ansible_distribution | lower] }}" state: present + retries: 3 + delay: 10 # Install bats from source so that we get the latest features. - name: Install bats from sources block: @@ -20,6 +22,8 @@ repo: https://github.com/bats-core/bats-core.git dest: bats-core force: yes + retries: 3 + delay: 10 - name: Install bats shell: cmd: ./install.sh /usr/local @@ -39,6 +43,8 @@ cp -f ./kustomize /usr/local/bin args: creates: /usr/local/bin/kustomize + retries: 3 + delay: 10 when: kustomize_exist.rc != 0 - block: - name: Download and extract Go tarball diff --git a/tests/e2e/ansible/start_docker_registry.yaml b/tests/e2e/ansible/start_docker_registry.yaml index a8eead8e..fad63ac9 100644 --- a/tests/e2e/ansible/start_docker_registry.yaml +++ b/tests/e2e/ansible/start_docker_registry.yaml @@ -17,6 +17,8 @@ # TODO: this is ubuntu specific... name: python3-pip state: present + retries: 3 + delay: 10 # The docker pip is needed by the docker_container ansible module itself. - name: Install docker pip pip: @@ -30,6 +32,8 @@ if [ "$version_ge" = "2.29.0" ]; then pip3 install requests==2.28.1 fi + retries: 3 + delay: 10 - name: Start a docker registry docker_container: name: "{{ local_registry_name }}" @@ -38,6 +42,8 @@ ports: - "{{ local_registry_port }}:{{ local_registry_port }}" state: started + retries: 3 + delay: 10 # # Remove the docker registry container. # From d8eefcb983473347bbd64d94140d578f46e7a056 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Doktor?= Date: Wed, 31 Jan 2024 11:37:47 +0100 Subject: [PATCH 2/4] tests.e2e.ansible: Fetch external tarballs with retries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Network tends to be flaky, let's use get_url module with retries to ensure external resources are fetched and unpack them afterwards (unfortunatelly the unarchive module does not supports retries). Signed-off-by: Lukáš Doktor --- tests/e2e/ansible/install_build_deps.yaml | 10 ++++++++-- tests/e2e/ansible/install_kubeadm.yaml | 16 ++++++++++++++-- tests/e2e/ansible/install_test_deps.yaml | 14 ++++++++++---- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/tests/e2e/ansible/install_build_deps.yaml b/tests/e2e/ansible/install_build_deps.yaml index 3d81225a..b605dab0 100644 --- a/tests/e2e/ansible/install_build_deps.yaml +++ b/tests/e2e/ansible/install_build_deps.yaml @@ -13,10 +13,16 @@ name: "{{ build_pkgs[ansible_distribution | lower] }}" state: present - block: - - name: Download and extract Go tarball + - name: Download Go tarball + get_url: + url: https://go.dev/dl/go{{ go_version }}.linux-{{ target_arch }}.tar.gz + dest: /tmp/go{{ go_version }}.linux-{{ target_arch }}.tar.gz + retries: 3 + delay: 10 + - name: Extract Go tarball unarchive: # TODO: use facts - src: https://go.dev/dl/go{{ go_version }}.linux-{{ target_arch }}.tar.gz + src: /tmp/go{{ go_version }}.linux-{{ target_arch }}.tar.gz creates: /usr/local/go dest: /usr/local remote_src: yes diff --git a/tests/e2e/ansible/install_kubeadm.yaml b/tests/e2e/ansible/install_kubeadm.yaml index 90787fdb..b8dca6e8 100644 --- a/tests/e2e/ansible/install_kubeadm.yaml +++ b/tests/e2e/ansible/install_kubeadm.yaml @@ -29,15 +29,27 @@ file: path: "{{ cni_home }}/bin" state: directory + - name: Download CNI plugins + get_url: + url: "https://github.com/containernetworking/plugins/releases/download/{{ cni_version }}/cni-plugins-linux-{{ target_arch }}-{{ cni_version }}.tgz" + dest: /tmp/cni-plugins.tgz + retries: 3 + delay: 10 - name: Install CNI plugins unarchive: # TODO: use facts - src: "https://github.com/containernetworking/plugins/releases/download/{{ cni_version }}/cni-plugins-linux-{{ target_arch }}-{{ cni_version }}.tgz" + src: /tmp/cni-plugins.tgz dest: "{{ cni_home }}/bin" remote_src: yes + - name: Download crictl + get_url: + url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/{{ k8s_version }}/crictl-{{ k8s_version }}-linux-{{ target_arch }}.tar.gz" + dest: /tmp/crictl.tar.gz + retries: 3 + delay: 10 - name: Install crictl unarchive: - src: "https://github.com/kubernetes-sigs/cri-tools/releases/download/{{ k8s_version }}/crictl-{{ k8s_version }}-linux-{{ target_arch }}.tar.gz" + src: /tmp/crictl.tar.gz creates: /usr/local/bin/crictl dest: /usr/local/bin remote_src: yes diff --git a/tests/e2e/ansible/install_test_deps.yaml b/tests/e2e/ansible/install_test_deps.yaml index 8410d171..70427180 100644 --- a/tests/e2e/ansible/install_test_deps.yaml +++ b/tests/e2e/ansible/install_test_deps.yaml @@ -22,8 +22,8 @@ repo: https://github.com/bats-core/bats-core.git dest: bats-core force: yes - retries: 3 - delay: 10 + retries: 3 + delay: 10 - name: Install bats shell: cmd: ./install.sh /usr/local @@ -47,10 +47,16 @@ delay: 10 when: kustomize_exist.rc != 0 - block: - - name: Download and extract Go tarball + - name: Download Go tarball + get_url: + url: https://go.dev/dl/go{{ go_version }}.linux-{{ target_arch }}.tar.gz + dest: /tmp/go{{ go_version }}.linux-{{ target_arch }}.tar.gz + retries: 3 + delay: 10 + - name: Extract Go tarball unarchive: # TODO: use facts - src: https://go.dev/dl/go{{ go_version }}.linux-{{ target_arch }}.tar.gz + src: /tmp/go{{ go_version }}.linux-{{ target_arch }}.tar.gz creates: /usr/local/go dest: /usr/local remote_src: yes From 9c9c0802854587375dc19c346346cdf81a89d88c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Doktor?= Date: Wed, 31 Jan 2024 11:41:33 +0100 Subject: [PATCH 3/4] tests.e2e.ansible: Allow curl to retry download MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit as network tends to be flaky, allow curl to retry in case it fails to fetch the kustomize. Signed-off-by: Lukáš Doktor --- tests/e2e/ansible/install_test_deps.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/ansible/install_test_deps.yaml b/tests/e2e/ansible/install_test_deps.yaml index 70427180..06d3d3e0 100644 --- a/tests/e2e/ansible/install_test_deps.yaml +++ b/tests/e2e/ansible/install_test_deps.yaml @@ -39,7 +39,7 @@ ignore_errors: yes - name: Install kustomize shell: | - curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash + curl -s --retry 3 --retry-delay 10 "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash cp -f ./kustomize /usr/local/bin args: creates: /usr/local/bin/kustomize From 8ea89f9cfa4b369851fc558a680d027652e2fe4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Doktor?= Date: Wed, 31 Jan 2024 11:44:10 +0100 Subject: [PATCH 4/4] tests.e2e.ansible: Disable warning for custom shell scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit we do want to do multiple steps and doing that in shell makes sense in those cases, ignore the ansible warning explicitly. Signed-off-by: Lukáš Doktor --- tests/e2e/ansible/install_containerd.yaml | 1 + tests/e2e/ansible/install_test_deps.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/e2e/ansible/install_containerd.yaml b/tests/e2e/ansible/install_containerd.yaml index 4f6d1021..2dcd4116 100644 --- a/tests/e2e/ansible/install_containerd.yaml +++ b/tests/e2e/ansible/install_containerd.yaml @@ -31,6 +31,7 @@ containerd config default > /etc/containerd/config.toml args: executable: /bin/bash + warn: false - name: Restart containerd service service: name: containerd diff --git a/tests/e2e/ansible/install_test_deps.yaml b/tests/e2e/ansible/install_test_deps.yaml index 06d3d3e0..337a9b42 100644 --- a/tests/e2e/ansible/install_test_deps.yaml +++ b/tests/e2e/ansible/install_test_deps.yaml @@ -43,6 +43,7 @@ cp -f ./kustomize /usr/local/bin args: creates: /usr/local/bin/kustomize + warn: false retries: 3 delay: 10 when: kustomize_exist.rc != 0