diff --git a/Jenkinsfile b/Jenkinsfile index 874d4be71aa63..efee1dc6287d5 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,7 +45,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-08-05T03:11:14.413104 +// Generated at 2022-08-05T11:02:36.235062 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // NOTE: these lines are scanned by docker/dev_common.sh. Please update the regex as needed. --> @@ -137,25 +137,30 @@ def init_git() { ) sh( - script: ''' + script: """ set -eux - n=0 - max_retries=3 - backoff_max=30 - until [ "$n" -ge $max_retries ] - do - timeout 5m git submodule update --init -f --jobs 0 && break - n=$((n+1)) - if [ "$n" -eq $max_retries ]; then - echo "failed to update $n / $max_retries, giving up" - exit 1 - fi - - WAIT=$((RANDOM % "$backoff_max")) - echo "failed to update $n / $max_retries, waiting $WAIT to try again" - sleep $WAIT - done - ''', + retry() { + local max_retries=\$1 + shift + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT + done +} + + retry 3 timeout 5m git submodule update --init -f --jobs 0 + """, label: 'Update git submodules', ) } @@ -185,26 +190,26 @@ def docker_init(image) { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } - retry 3 docker pull ${image} + retry 3 docker pull fake${image} """, label: 'Pull docker image', ) @@ -271,6 +276,32 @@ def prepare() { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/prepare") { init_git() + sh( + script: """ + retry() { + local max_retries=\$1 + shift + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT + done +} + + retry 3 docker pull abciosdjfoij12 + """ + ) + if (env.DETERMINE_DOCKER_IMAGES == 'yes') { sh( script: "./tests/scripts/determine_docker_images.py ci_arm=${ci_arm} ci_cpu=${ci_cpu} ci_gpu=${ci_gpu} ci_hexagon=${ci_hexagon} ci_i386=${ci_i386} ci_lint=${ci_lint} ci_qemu=${ci_qemu} ci_wasm=${ci_wasm} ", @@ -685,23 +716,23 @@ stage('Build') { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } md5sum build/libtvm.so @@ -725,23 +756,23 @@ stage('Build') { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } md5sum build/libtvm.so @@ -775,23 +806,23 @@ stage('Build') { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } md5sum build/libvta_tsim.so @@ -860,23 +891,23 @@ stage('Build') { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } md5sum build/libvta_tsim.so @@ -914,23 +945,23 @@ stage('Build') { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } md5sum build/libtvm.so @@ -966,23 +997,23 @@ stage('Build') { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } md5sum build/libtvm.so @@ -1021,23 +1052,23 @@ stage('Build') { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } md5sum build/libtvm.so @@ -1082,23 +1113,23 @@ def shard_run_unittest_GPU_1_of_3() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu2/build/libtvm.so build/libtvm.so @@ -1119,23 +1150,23 @@ def shard_run_unittest_GPU_1_of_3() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so @@ -1196,23 +1227,23 @@ def shard_run_unittest_GPU_2_of_3() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so @@ -1276,23 +1307,23 @@ def shard_run_unittest_GPU_3_of_3() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so @@ -1353,23 +1384,23 @@ def shard_run_integration_CPU_1_of_10() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so @@ -1427,23 +1458,23 @@ def shard_run_integration_CPU_2_of_10() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so @@ -1501,23 +1532,23 @@ def shard_run_integration_CPU_3_of_10() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so @@ -1575,23 +1606,23 @@ def shard_run_integration_CPU_4_of_10() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so @@ -1649,23 +1680,23 @@ def shard_run_integration_CPU_5_of_10() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so @@ -1723,23 +1754,23 @@ def shard_run_integration_CPU_6_of_10() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so @@ -1797,23 +1828,23 @@ def shard_run_integration_CPU_7_of_10() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so @@ -1871,23 +1902,23 @@ def shard_run_integration_CPU_8_of_10() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so @@ -1945,23 +1976,23 @@ def shard_run_integration_CPU_9_of_10() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so @@ -2019,23 +2050,23 @@ def shard_run_integration_CPU_10_of_10() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so @@ -2094,23 +2125,23 @@ def shard_run_python_i386_1_of_5() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so @@ -2168,23 +2199,23 @@ def shard_run_python_i386_2_of_5() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so @@ -2242,23 +2273,23 @@ def shard_run_python_i386_3_of_5() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so @@ -2315,23 +2346,23 @@ def shard_run_python_i386_4_of_5() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so @@ -2388,23 +2419,23 @@ def shard_run_python_i386_5_of_5() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so @@ -2462,23 +2493,23 @@ def shard_run_test_Hexagon_1_of_7() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so @@ -2535,23 +2566,23 @@ def shard_run_test_Hexagon_2_of_7() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so @@ -2607,23 +2638,23 @@ def shard_run_test_Hexagon_3_of_7() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so @@ -2679,23 +2710,23 @@ def shard_run_test_Hexagon_4_of_7() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so @@ -2751,23 +2782,23 @@ def shard_run_test_Hexagon_5_of_7() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so @@ -2823,23 +2854,23 @@ def shard_run_test_Hexagon_6_of_7() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so @@ -2895,23 +2926,23 @@ def shard_run_test_Hexagon_7_of_7() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so @@ -2968,23 +2999,23 @@ def shard_run_integration_aarch64_1_of_4() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so @@ -3041,23 +3072,23 @@ def shard_run_integration_aarch64_2_of_4() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so @@ -3114,23 +3145,23 @@ def shard_run_integration_aarch64_3_of_4() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so @@ -3187,23 +3218,23 @@ def shard_run_integration_aarch64_4_of_4() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so @@ -3261,23 +3292,23 @@ def shard_run_topi_GPU_1_of_4() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so @@ -3333,23 +3364,23 @@ def shard_run_topi_GPU_2_of_4() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so @@ -3405,23 +3436,23 @@ def shard_run_topi_GPU_3_of_4() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so @@ -3477,23 +3508,23 @@ def shard_run_topi_GPU_4_of_4() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so @@ -3550,23 +3581,23 @@ def shard_run_frontend_GPU_1_of_6() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so @@ -3622,23 +3653,23 @@ def shard_run_frontend_GPU_2_of_6() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so @@ -3694,23 +3725,23 @@ def shard_run_frontend_GPU_3_of_6() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so @@ -3766,23 +3797,23 @@ def shard_run_frontend_GPU_4_of_6() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so @@ -3838,23 +3869,23 @@ def shard_run_frontend_GPU_5_of_6() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so @@ -3910,23 +3941,23 @@ def shard_run_frontend_GPU_6_of_6() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so @@ -3983,23 +4014,23 @@ def shard_run_topi_aarch64_1_of_2() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so @@ -4060,23 +4091,23 @@ def shard_run_topi_aarch64_2_of_2() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so @@ -4137,23 +4168,23 @@ def shard_run_frontend_aarch64_1_of_2() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so @@ -4209,23 +4240,23 @@ def shard_run_frontend_aarch64_2_of_2() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so @@ -4282,23 +4313,23 @@ def shard_run_test_Qemu_1_of_4() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/qemu/build/libtvm.so build/libtvm.so @@ -4359,23 +4390,23 @@ def shard_run_test_Qemu_2_of_4() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/qemu/build/libtvm.so build/libtvm.so @@ -4431,23 +4462,23 @@ def shard_run_test_Qemu_3_of_4() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/qemu/build/libtvm.so build/libtvm.so @@ -4503,23 +4534,23 @@ def shard_run_test_Qemu_4_of_4() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/qemu/build/libtvm.so build/libtvm.so @@ -4721,23 +4752,23 @@ stage('Test') { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so @@ -4794,23 +4825,23 @@ stage('Test') { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so @@ -4859,23 +4890,23 @@ stage('Test') { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so @@ -4903,23 +4934,23 @@ stage('Test') { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } md5sum docs.tgz @@ -5029,23 +5060,23 @@ def deploy() { script: """ set -eux retry() { - local retries=\$1 + local max_retries=\$1 shift - - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi + + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } retry 3 aws s3 cp --no-progress s3://${s3_prefix}/docs/docs.tgz docs.tgz diff --git a/ci/jenkins/Prepare.groovy.j2 b/ci/jenkins/Prepare.groovy.j2 index 8504c0dfb3f55..aec79aab757f0 100644 --- a/ci/jenkins/Prepare.groovy.j2 +++ b/ci/jenkins/Prepare.groovy.j2 @@ -31,25 +31,11 @@ def init_git() { ) sh( - script: ''' + script: """ set -eux - n=0 - max_retries=3 - backoff_max=30 - until [ "$n" -ge $max_retries ] - do - timeout 5m git submodule update --init -f --jobs 0 && break - n=$((n+1)) - if [ "$n" -eq $max_retries ]; then - echo "failed to update $n / $max_retries, giving up" - exit 1 - fi - - WAIT=$((RANDOM % "$backoff_max")) - echo "failed to update $n / $max_retries, waiting $WAIT to try again" - sleep $WAIT - done - ''', + {{ m.bash_retry() }} + retry 3 timeout 5m git submodule update --init -f --jobs 0 + """, label: 'Update git submodules', ) } @@ -79,7 +65,7 @@ def docker_init(image) { script: """ set -eux {{ m.bash_retry() }} - retry 3 docker pull ${image} + retry 3 docker pull fake${image} """, label: 'Pull docker image', ) @@ -146,6 +132,13 @@ def prepare() { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/prepare") { init_git() + sh( + script: """ + {{ m.bash_retry() }} + retry 3 docker pull abciosdjfoij12 + """ + ) + if (env.DETERMINE_DOCKER_IMAGES == 'yes') { sh( script: "./tests/scripts/determine_docker_images.py {% for image in images %}{{ image.name }}={% raw %}${{% endraw %}{{ image.name }}{% raw %}}{% endraw %} {% endfor %}", diff --git a/ci/jenkins/macros.j2 b/ci/jenkins/macros.j2 index 9f4e1cd8748b3..b719203db21bf 100644 --- a/ci/jenkins/macros.j2 +++ b/ci/jenkins/macros.j2 @@ -111,23 +111,23 @@ def {{ method_name }}() { {% macro bash_retry() %} retry() { - local retries=\$1 + local max_retries=\$1 shift + local n=0 + local backoff_max=30 + until [ "\$n" -ge \$max_retries ] + do + "\$@" && break + n=\$((n+1)) + if [ "\$n" -eq \$max_retries ]; then + echo "failed to update after attempt \$n / \$max_retries, giving up" + exit 1 + fi - local count=0 - until "\$@"; do - exit=\$? - wait=\$((2 ** \$count)) - count=\$((\$count + 1)) - if [ \$count -lt \$retries ]; then - echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." - sleep \$wait - else - echo "Retry \$count/\$retries exited \$exit, no more retries left." - return \$exit - fi + WAIT=\$(python3 -c 'import random; print(random.randint(10, 30))') + echo "failed to update \$n / \$max_retries, waiting \$WAIT to try again" + sleep \$WAIT done - return 0 } {% endmacro %}