[ci] Add retries to docker push (#12773)

This should mitigate failures like in https://ci.tlcpack.ai/blue/organizations/jenkins/tvm/detail/main/4274/pipeline. This also moves the `retry` function to a script now that we have PR #12604. Co-authored-by: driazati <driazati@users.noreply.github.com>
apache · Sep 15, 2022 · c00ce57 · c00ce57
1 parent c900250
commit c00ce57
Show file tree

Hide file tree

Showing 8 changed files with 223 additions and 1,443 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
diff --git a/ci/jenkins/Build.groovy.j2 b/ci/jenkins/Build.groovy.j2
@@ -84,8 +84,8 @@ stage('Build') {
     if (!skip_ci) {
       node('CPU-SMALL') {
         ws({{ m.per_exec_ws('tvm/build-gpu') }}) {
-          docker_init(ci_gpu)
           init_git()
+          docker_init(ci_gpu)
           sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build"
           make("${ci_gpu} --no-gpu", 'build', '-j2')
           {{ m.upload_artifacts(tag='gpu', filenames=tvm_multilib, folders=microtvm_template_projects) }}
@@ -102,8 +102,8 @@ stage('Build') {
     if (!skip_ci && is_docs_only_build != 1) {
       node('CPU-SMALL') {
         ws({{ m.per_exec_ws('tvm/build-cpu') }}) {
-          docker_init(ci_cpu)
           init_git()
+          docker_init(ci_cpu)
           sh (
             script: "${docker_run} ${ci_cpu} ./tests/scripts/task_config_build_cpu.sh build",
             label: 'Create CPU cmake config',
@@ -126,8 +126,8 @@ stage('Build') {
     if (!skip_ci && is_docs_only_build != 1) {
       node('CPU-SMALL') {
         ws({{ m.per_exec_ws('tvm/build-cpu-minimal') }}) {
-          docker_init(ci_minimal)
           init_git()
+          docker_init(ci_minimal)
           sh (
             script: "${docker_run} ${ci_minimal} ./tests/scripts/task_config_build_minimal.sh build",
             label: 'Create CPU minimal cmake config',
@@ -144,8 +144,8 @@ stage('Build') {
     if (!skip_ci && is_docs_only_build != 1) {
       node('CPU-SMALL') {
         ws({{ m.per_exec_ws('tvm/build-wasm') }}) {
-          docker_init(ci_wasm)
           init_git()
+          docker_init(ci_wasm)
           sh (
             script: "${docker_run} ${ci_wasm} ./tests/scripts/task_config_build_wasm.sh build",
             label: 'Create WASM cmake config',
@@ -169,8 +169,8 @@ stage('Build') {
     if (!skip_ci && is_docs_only_build != 1) {
       node('CPU-SMALL') {
         ws({{ m.per_exec_ws('tvm/build-i386') }}) {
-          docker_init(ci_i386)
           init_git()
+          docker_init(ci_i386)
           sh (
             script: "${docker_run} ${ci_i386} ./tests/scripts/task_config_build_i386.sh build",
             label: 'Create i386 cmake config',
@@ -187,8 +187,8 @@ stage('Build') {
     if (!skip_ci && is_docs_only_build != 1) {
       node('ARM-SMALL') {
         ws({{ m.per_exec_ws('tvm/build-arm') }}) {
-          docker_init(ci_arm)
           init_git()
+          docker_init(ci_arm)
           sh (
             script: "${docker_run} ${ci_arm} ./tests/scripts/task_config_build_arm.sh build",
             label: 'Create ARM cmake config',
@@ -205,8 +205,8 @@ stage('Build') {
     if (!skip_ci && is_docs_only_build != 1) {
       node('CPU-SMALL') {
         ws({{ m.per_exec_ws('tvm/build-cortexm') }}) {
-          docker_init(ci_cortexm)
           init_git()
+          docker_init(ci_cortexm)
           sh (
             script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_config_build_cortexm.sh build",
             label: 'Create Cortex-M cmake config',
@@ -223,8 +223,8 @@ stage('Build') {
     if (!skip_ci && is_docs_only_build != 1) {
       node('CPU-SMALL') {
         ws({{ m.per_exec_ws('tvm/build-hexagon') }}) {
-          docker_init(ci_hexagon)
           init_git()
+          docker_init(ci_hexagon)
           sh (
             script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_config_build_hexagon.sh build",
             label: 'Create Hexagon cmake config',
@@ -245,8 +245,8 @@ stage('Build') {
     if (!skip_ci && is_docs_only_build != 1) {
       node('CPU-SMALL') {
         ws({{ m.per_exec_ws('tvm/build-riscv') }}) {
-          docker_init(ci_riscv)
           init_git()
+          docker_init(ci_riscv)
           sh (
             script: "${docker_run} ${ci_riscv} ./tests/scripts/task_config_build_riscv.sh build",
             label: 'Create RISC-V cmake config',

diff --git a/ci/jenkins/Deploy.groovy.j2 b/ci/jenkins/Deploy.groovy.j2
@@ -30,10 +30,11 @@ def update_docker(ecr_image, hub_image) {
   sh(
     script: """
     set -eux
+    . ci/scripts/retry.sh
     docker tag \
       ${ecr_image} \
       ${hub_image}
-    docker push ${hub_image}
+    retry 5 docker push ${hub_image}
     """,
     label: "Update ${hub_image} on Docker Hub",
   )
@@ -144,9 +145,10 @@ def deploy() {
                 sh(
                   script: """
                     set -eux
+                    . ci/scripts/retry.sh
                     docker pull tlcpackstaging/{{ image.name }}:${tag}
                     docker tag tlcpackstaging/{{ image.name }}:${tag} tlcpack/{{ image.name.replace("_", "-") }}:${tag}
-                    docker push tlcpack/{{ image.name.replace("_", "-") }}:${tag}
+                    retry 5 docker push tlcpack/{{ image.name.replace("_", "-") }}:${tag}
                   """,
                   label: 'Tag tlcpackstaging/{{ image.name }} image to tlcpack',
                 )

diff --git a/ci/jenkins/DockerBuild.groovy.j2 b/ci/jenkins/DockerBuild.groovy.j2
@@ -21,8 +21,9 @@ def ecr_push(full_name) {
       sh(
         script: """
           set -x
+          . ci/scripts/retry.sh
           docker tag ${full_name} \$AWS_ECR_REPO/${full_name}
-          docker push \$AWS_ECR_REPO/${full_name}
+          retry 5 docker push \$AWS_ECR_REPO/${full_name}
         """,
         label: 'Upload image to ECR'
       )
@@ -63,7 +64,8 @@ def ecr_pull(full_name) {
       sh(
         script: """
           set -eux
-          docker pull ${full_name}
+          . ci/scripts/retry.sh
+          retry 5 docker pull ${full_name}
         """,
         label: 'Pull image from ECR'
       )

diff --git a/ci/jenkins/Prepare.groovy.j2 b/ci/jenkins/Prepare.groovy.j2
@@ -33,7 +33,7 @@ def init_git() {
   sh(
     script: """
       set -eux
-      {{ m.bash_retry() }}
+      . ci/scripts/retry.sh
       retry 3 timeout 5m git submodule update --init -f --jobs 0
     """,
     label: 'Update git submodules',
@@ -65,8 +65,8 @@ def docker_init(image) {
     sh(
       script: """
       set -eux
-      {{ m.bash_retry() }}
-      retry 3 docker pull ${image}
+      . ci/scripts/retry.sh
+      retry 5 docker pull ${image}
       """,
       label: 'Pull docker image',
     )

diff --git a/ci/jenkins/Test.groovy.j2 b/ci/jenkins/Test.groovy.j2
@@ -294,8 +294,8 @@ stage('Test') {
     if (!skip_ci) {
       node('GPU') {
         ws({{ m.per_exec_ws('tvm/docs-python-gpu') }}) {
-          docker_init(ci_gpu)
           init_git()
+          docker_init(ci_gpu)
           {{ m.download_artifacts(tag='gpu', filenames=tvm_multilib, folders=microtvm_template_projects) }}
           add_microtvm_permissions()
           timeout(time: 180, unit: 'MINUTES') {

diff --git a/ci/jenkins/macros.j2 b/ci/jenkins/macros.j2
@@ -39,8 +39,8 @@ def {{ method_name }}() {
     node('{{ node }}') {
       ws({{ per_exec_ws(ws) }}) {
         try {
-          docker_init({{ docker_image }})
           init_git()
+          docker_init({{ docker_image }})
           timeout(time: max_time, unit: 'MINUTES') {
             withEnv([
               'PLATFORM={{ platform }}',
@@ -71,8 +71,8 @@ def {{ method_name }}() {
   '{{ name }} {{ shard_index }} of {{ num_shards }}': {
     node('{{ node }}') {
       ws({{ per_exec_ws(ws) }}) {
-        docker_init({{ docker_image }})
         init_git()
+        docker_init({{ docker_image }})
         timeout(time: max_time, unit: 'MINUTES') {
           withEnv([
             'TVM_NUM_SHARDS={{ num_shards }}',
@@ -95,8 +95,8 @@ def {{ method_name }}() {
       ws({{ per_exec_ws(ws) }}) {
         timeout(time: max_time, unit: 'MINUTES') {
           try {
-            docker_init({{ docker_image }})
             init_git()
+            docker_init({{ docker_image }})
             withEnv(['PLATFORM={{ platform }}'], {
               {{ caller() | indent(width=8) | trim }}
             })
@@ -120,8 +120,8 @@ def {{ method_name }}() {
         ws({{ per_exec_ws(ws) }}) {
           timeout(time: max_time, unit: 'MINUTES') {
             try {
-              docker_init({{ docker_image }})
               init_git()
+              docker_init({{ docker_image }})
               withEnv(['PLATFORM={{ platform }}',
               'TEST_STEP_NAME={{ name }}',
               "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
@@ -140,28 +140,6 @@ def {{ method_name }}() {
   },
 {% endmacro %}
 
-{% macro bash_retry() %}
-retry() {
-  local max_retries=\$1
-  shift
-  local n=0
-  local backoff_max=30
-  until [ "\$n" -ge \$max_retries ]
-  do
-      "\$@" && break
-      n=\$((n+1))
-      if [ "\$n" -eq \$max_retries ]; then
-          echo "failed to update after attempt \$n / \$max_retries, giving up"
-          exit 1
-      fi
-
-      WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))')
-      echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again"
-      sleep \$WAIT
-  done
-}
-{% endmacro %}
-
 {% macro deploy_step(name, feature_flag, ws) %}
   '{{ name }}': {
     if ({{ feature_flag }}) {
@@ -182,7 +160,7 @@ retry() {
 sh(
             script: """
               set -eux
-              {{ bash_retry() | indent(width=14) }}
+              . ci/scripts/retry.sh
               {% for filename in filenames %}
               md5sum {{ filename }}
               retry 3 aws s3 cp --no-progress {{ filename }} s3://${s3_prefix}/{{ tag }}/{{ filename }}
@@ -199,7 +177,7 @@ sh(
 sh(
             script: """
               set -eux
-              {{ bash_retry() | indent(width=14) }}
+              . ci/scripts/retry.sh
               {% for filename in filenames %}
               retry 3 aws s3 cp --no-progress s3://${s3_prefix}/{{ tag }}/{{ filename }} {{ filename }}
               md5sum {{ filename }}

diff --git a/ci/scripts/retry.sh b/ci/scripts/retry.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -eux
+
+retry() {
+  local max_retries=$1
+  shift
+  local n=0
+  until [ "$n" -ge "$max_retries" ]
+  do
+      "$@" && break
+      n=$((n+1))
+      if [ "$n" -eq "$max_retries" ]; then
+          echo "failed to update after attempt $n / $max_retries, giving up"
+          exit 1
+      fi
+
+      WAIT=$(python3 -c 'import random; print(random.randint(10, 30))')
+      echo "failed to update $n / $max_retries, waiting $WAIT to try again"
+      sleep "$WAIT"
+  done
+}