From 07362baebc89135f5613517bd6bba494e844c290 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 28 Jul 2022 11:27:44 -0700 Subject: [PATCH] [ci] Add retries to S3 uploads/downloads `aws s3 cp` sometimes segfaults, so retry it when the command fails --- Jenkinsfile | 1496 ++++++++++++++++++++++++++++++++++++------ ci/jenkins/macros.j2 | 32 +- 2 files changed, 1306 insertions(+), 222 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c2f6407333389..8446155ca8f83 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,7 +45,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-07-15T13:35:24.676914 +// Generated at 2022-07-29T14:57:47.758536 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // NOTE: these lines are scanned by docker/dev_common.sh. Please update the regex as needed. --> @@ -649,15 +649,35 @@ stage('Build') { sh( script: """ set -eux + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + md5sum build/libtvm.so - aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/gpu/build/libtvm.so + retry 3 aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/gpu/build/libtvm.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress build/libvta_fsim.so s3://${s3_prefix}/gpu/build/libvta_fsim.so + retry 3 aws s3 cp --no-progress build/libvta_fsim.so s3://${s3_prefix}/gpu/build/libvta_fsim.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress build/libtvm_runtime.so s3://${s3_prefix}/gpu/build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress build/libtvm_runtime.so s3://${s3_prefix}/gpu/build/libtvm_runtime.so md5sum build/config.cmake - aws s3 cp --no-progress build/config.cmake s3://${s3_prefix}/gpu/build/config.cmake - aws s3 cp --no-progress build/microtvm_template_projects s3://${s3_prefix}/gpu/build/microtvm_template_projects --recursive + retry 3 aws s3 cp --no-progress build/config.cmake s3://${s3_prefix}/gpu/build/config.cmake + retry 3 aws s3 cp --no-progress build/microtvm_template_projects s3://${s3_prefix}/gpu/build/microtvm_template_projects --recursive """, label: 'Upload artifacts to S3', ) @@ -669,14 +689,34 @@ stage('Build') { sh( script: """ set -eux + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + md5sum build/libtvm.so - aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/gpu2/build/libtvm.so + retry 3 aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/gpu2/build/libtvm.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress build/libvta_fsim.so s3://${s3_prefix}/gpu2/build/libvta_fsim.so + retry 3 aws s3 cp --no-progress build/libvta_fsim.so s3://${s3_prefix}/gpu2/build/libvta_fsim.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress build/libtvm_runtime.so s3://${s3_prefix}/gpu2/build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress build/libtvm_runtime.so s3://${s3_prefix}/gpu2/build/libtvm_runtime.so md5sum build/config.cmake - aws s3 cp --no-progress build/config.cmake s3://${s3_prefix}/gpu2/build/config.cmake + retry 3 aws s3 cp --no-progress build/config.cmake s3://${s3_prefix}/gpu2/build/config.cmake """, label: 'Upload artifacts to S3', ) @@ -699,16 +739,36 @@ stage('Build') { sh( script: """ set -eux + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + md5sum build/libvta_tsim.so - aws s3 cp --no-progress build/libvta_tsim.so s3://${s3_prefix}/cpu/build/libvta_tsim.so + retry 3 aws s3 cp --no-progress build/libvta_tsim.so s3://${s3_prefix}/cpu/build/libvta_tsim.so md5sum build/libtvm.so - aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/cpu/build/libtvm.so + retry 3 aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/cpu/build/libtvm.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress build/libvta_fsim.so s3://${s3_prefix}/cpu/build/libvta_fsim.so + retry 3 aws s3 cp --no-progress build/libvta_fsim.so s3://${s3_prefix}/cpu/build/libvta_fsim.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress build/libtvm_runtime.so s3://${s3_prefix}/cpu/build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress build/libtvm_runtime.so s3://${s3_prefix}/cpu/build/libtvm_runtime.so md5sum build/config.cmake - aws s3 cp --no-progress build/config.cmake s3://${s3_prefix}/cpu/build/config.cmake + retry 3 aws s3 cp --no-progress build/config.cmake s3://${s3_prefix}/cpu/build/config.cmake """, label: 'Upload artifacts to S3', ) @@ -764,16 +824,36 @@ stage('Build') { sh( script: """ set -eux + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + md5sum build/libvta_tsim.so - aws s3 cp --no-progress build/libvta_tsim.so s3://${s3_prefix}/i386/build/libvta_tsim.so + retry 3 aws s3 cp --no-progress build/libvta_tsim.so s3://${s3_prefix}/i386/build/libvta_tsim.so md5sum build/libtvm.so - aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/i386/build/libtvm.so + retry 3 aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/i386/build/libtvm.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress build/libvta_fsim.so s3://${s3_prefix}/i386/build/libvta_fsim.so + retry 3 aws s3 cp --no-progress build/libvta_fsim.so s3://${s3_prefix}/i386/build/libvta_fsim.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress build/libtvm_runtime.so s3://${s3_prefix}/i386/build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress build/libtvm_runtime.so s3://${s3_prefix}/i386/build/libtvm_runtime.so md5sum build/config.cmake - aws s3 cp --no-progress build/config.cmake s3://${s3_prefix}/i386/build/config.cmake + retry 3 aws s3 cp --no-progress build/config.cmake s3://${s3_prefix}/i386/build/config.cmake """, label: 'Upload artifacts to S3', ) @@ -798,14 +878,34 @@ stage('Build') { sh( script: """ set -eux + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + md5sum build/libtvm.so - aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/arm/build/libtvm.so + retry 3 aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/arm/build/libtvm.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress build/libvta_fsim.so s3://${s3_prefix}/arm/build/libvta_fsim.so + retry 3 aws s3 cp --no-progress build/libvta_fsim.so s3://${s3_prefix}/arm/build/libvta_fsim.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress build/libtvm_runtime.so s3://${s3_prefix}/arm/build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress build/libtvm_runtime.so s3://${s3_prefix}/arm/build/libtvm_runtime.so md5sum build/config.cmake - aws s3 cp --no-progress build/config.cmake s3://${s3_prefix}/arm/build/config.cmake + retry 3 aws s3 cp --no-progress build/config.cmake s3://${s3_prefix}/arm/build/config.cmake """, label: 'Upload artifacts to S3', ) @@ -830,13 +930,33 @@ stage('Build') { sh( script: """ set -eux + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + md5sum build/libtvm.so - aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/qemu/build/libtvm.so + retry 3 aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/qemu/build/libtvm.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress build/libtvm_runtime.so s3://${s3_prefix}/qemu/build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress build/libtvm_runtime.so s3://${s3_prefix}/qemu/build/libtvm_runtime.so md5sum build/config.cmake - aws s3 cp --no-progress build/config.cmake s3://${s3_prefix}/qemu/build/config.cmake - aws s3 cp --no-progress build/microtvm_template_projects s3://${s3_prefix}/qemu/build/microtvm_template_projects --recursive + retry 3 aws s3 cp --no-progress build/config.cmake s3://${s3_prefix}/qemu/build/config.cmake + retry 3 aws s3 cp --no-progress build/microtvm_template_projects s3://${s3_prefix}/qemu/build/microtvm_template_projects --recursive """, label: 'Upload artifacts to S3', ) @@ -865,13 +985,33 @@ stage('Build') { sh( script: """ set -eux + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + md5sum build/libtvm.so - aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/hexagon/build/libtvm.so + retry 3 aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/hexagon/build/libtvm.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress build/libtvm_runtime.so s3://${s3_prefix}/hexagon/build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress build/libtvm_runtime.so s3://${s3_prefix}/hexagon/build/libtvm_runtime.so md5sum build/config.cmake - aws s3 cp --no-progress build/config.cmake s3://${s3_prefix}/hexagon/build/config.cmake - aws s3 cp --no-progress build/hexagon_api_output s3://${s3_prefix}/hexagon/build/hexagon_api_output --recursive + retry 3 aws s3 cp --no-progress build/config.cmake s3://${s3_prefix}/hexagon/build/config.cmake + retry 3 aws s3 cp --no-progress build/hexagon_api_output s3://${s3_prefix}/hexagon/build/hexagon_api_output --recursive """, label: 'Upload artifacts to S3', ) @@ -906,13 +1046,33 @@ def shard_run_unittest_GPU_1_of_3() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/gpu2/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu2/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu2/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu2/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu2/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu2/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu2/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu2/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -923,13 +1083,33 @@ def shard_run_unittest_GPU_1_of_3() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -980,13 +1160,33 @@ def shard_run_unittest_GPU_2_of_3() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -1040,13 +1240,33 @@ def shard_run_unittest_GPU_3_of_3() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -1097,15 +1317,35 @@ def shard_run_integration_CPU_1_of_6() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so md5sum build/libvta_tsim.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -1151,15 +1391,35 @@ def shard_run_integration_CPU_2_of_6() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so md5sum build/libvta_tsim.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -1205,15 +1465,35 @@ def shard_run_integration_CPU_3_of_6() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so md5sum build/libvta_tsim.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -1259,15 +1539,35 @@ def shard_run_integration_CPU_4_of_6() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so md5sum build/libvta_tsim.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -1313,15 +1613,35 @@ def shard_run_integration_CPU_5_of_6() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so md5sum build/libvta_tsim.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -1367,15 +1687,35 @@ def shard_run_integration_CPU_6_of_6() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so md5sum build/libvta_tsim.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -1422,13 +1762,33 @@ def shard_run_python_i386_1_of_5() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -1476,13 +1836,33 @@ def shard_run_python_i386_2_of_5() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -1530,13 +1910,33 @@ def shard_run_python_i386_3_of_5() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -1583,13 +1983,33 @@ def shard_run_python_i386_4_of_5() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -1636,13 +2056,33 @@ def shard_run_python_i386_5_of_5() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -1690,13 +2130,33 @@ def shard_run_test_Hexagon_1_of_7() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/config.cmake build/config.cmake md5sum build/config.cmake - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/hexagon_api_output build/hexagon_api_output --recursive + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/hexagon_api_output build/hexagon_api_output --recursive """, label: 'Download artifacts from S3', ) @@ -1743,13 +2203,33 @@ def shard_run_test_Hexagon_2_of_7() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/config.cmake build/config.cmake md5sum build/config.cmake - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/hexagon_api_output build/hexagon_api_output --recursive + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/hexagon_api_output build/hexagon_api_output --recursive """, label: 'Download artifacts from S3', ) @@ -1795,13 +2275,33 @@ def shard_run_test_Hexagon_3_of_7() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/config.cmake build/config.cmake md5sum build/config.cmake - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/hexagon_api_output build/hexagon_api_output --recursive + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/hexagon_api_output build/hexagon_api_output --recursive """, label: 'Download artifacts from S3', ) @@ -1847,13 +2347,33 @@ def shard_run_test_Hexagon_4_of_7() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/config.cmake build/config.cmake md5sum build/config.cmake - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/hexagon_api_output build/hexagon_api_output --recursive + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/hexagon_api_output build/hexagon_api_output --recursive """, label: 'Download artifacts from S3', ) @@ -1899,13 +2419,33 @@ def shard_run_test_Hexagon_5_of_7() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/config.cmake build/config.cmake md5sum build/config.cmake - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/hexagon_api_output build/hexagon_api_output --recursive + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/hexagon_api_output build/hexagon_api_output --recursive """, label: 'Download artifacts from S3', ) @@ -1951,13 +2491,33 @@ def shard_run_test_Hexagon_6_of_7() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/config.cmake build/config.cmake md5sum build/config.cmake - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/hexagon_api_output build/hexagon_api_output --recursive + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/hexagon_api_output build/hexagon_api_output --recursive """, label: 'Download artifacts from S3', ) @@ -2003,13 +2563,33 @@ def shard_run_test_Hexagon_7_of_7() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/config.cmake build/config.cmake md5sum build/config.cmake - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/hexagon_api_output build/hexagon_api_output --recursive + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/hexagon_api_output build/hexagon_api_output --recursive """, label: 'Download artifacts from S3', ) @@ -2056,13 +2636,33 @@ def shard_run_integration_aarch64_1_of_4() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -2109,13 +2709,33 @@ def shard_run_integration_aarch64_2_of_4() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -2162,13 +2782,33 @@ def shard_run_integration_aarch64_3_of_4() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -2215,13 +2855,33 @@ def shard_run_integration_aarch64_4_of_4() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -2269,13 +2929,33 @@ def shard_run_topi_GPU_1_of_4() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -2321,13 +3001,33 @@ def shard_run_topi_GPU_2_of_4() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -2373,13 +3073,33 @@ def shard_run_topi_GPU_3_of_4() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -2425,13 +3145,33 @@ def shard_run_topi_GPU_4_of_4() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -2478,13 +3218,33 @@ def shard_run_frontend_GPU_1_of_6() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -2530,13 +3290,33 @@ def shard_run_frontend_GPU_2_of_6() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -2582,13 +3362,33 @@ def shard_run_frontend_GPU_3_of_6() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -2634,13 +3434,33 @@ def shard_run_frontend_GPU_4_of_6() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -2686,13 +3506,33 @@ def shard_run_frontend_GPU_5_of_6() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -2738,13 +3578,33 @@ def shard_run_frontend_GPU_6_of_6() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -2791,13 +3651,33 @@ def shard_run_topi_aarch64_1_of_2() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -2848,13 +3728,33 @@ def shard_run_topi_aarch64_2_of_2() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -2905,13 +3805,33 @@ def shard_run_frontend_aarch64_1_of_2() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -2957,13 +3877,33 @@ def shard_run_frontend_aarch64_2_of_2() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -3131,15 +4071,35 @@ stage('Test') { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so md5sum build/libvta_tsim.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -3184,13 +4144,33 @@ stage('Test') { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/qemu/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/qemu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/qemu/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/qemu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/qemu/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/qemu/build/config.cmake build/config.cmake md5sum build/config.cmake - aws s3 cp --no-progress s3://${s3_prefix}/qemu/build/microtvm_template_projects build/microtvm_template_projects --recursive + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/qemu/build/microtvm_template_projects build/microtvm_template_projects --recursive """, label: 'Download artifacts from S3', ) @@ -3237,13 +4217,33 @@ stage('Test') { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', @@ -3282,15 +4282,35 @@ stage('Test') { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake md5sum build/config.cmake - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/microtvm_template_projects build/microtvm_template_projects --recursive + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/microtvm_template_projects build/microtvm_template_projects --recursive """, label: 'Download artifacts from S3', ) @@ -3306,8 +4326,28 @@ stage('Test') { sh( script: """ set -eux + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + md5sum docs.tgz - aws s3 cp --no-progress docs.tgz s3://${s3_prefix}/docs/docs.tgz + retry 3 aws s3 cp --no-progress docs.tgz s3://${s3_prefix}/docs/docs.tgz """, label: 'Upload artifacts to S3', ) @@ -3408,7 +4448,27 @@ def deploy() { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/docs/docs.tgz docs.tgz + retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 + } + + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/docs/docs.tgz docs.tgz md5sum docs.tgz """, label: 'Download artifacts from S3', diff --git a/ci/jenkins/macros.j2 b/ci/jenkins/macros.j2 index 5d996ce19a559..9f4e1cd8748b3 100644 --- a/ci/jenkins/macros.j2 +++ b/ci/jenkins/macros.j2 @@ -109,16 +109,39 @@ def {{ method_name }}() { }, {% endmacro %} +{% macro bash_retry() %} +retry() { + local retries=\$1 + shift + + local count=0 + until "\$@"; do + exit=\$? + wait=\$((2 ** \$count)) + count=\$((\$count + 1)) + if [ \$count -lt \$retries ]; then + echo "Retry \$count/\$retries exited \$exit, retrying in \$wait seconds..." + sleep \$wait + else + echo "Retry \$count/\$retries exited \$exit, no more retries left." + return \$exit + fi + done + return 0 +} +{% endmacro %} + {% macro upload_artifacts(tag, filenames, folders=None) %} sh( script: """ set -eux + {{ bash_retry() | indent(width=14) }} {% for filename in filenames %} md5sum {{ filename }} - aws s3 cp --no-progress {{ filename }} s3://${s3_prefix}/{{ tag }}/{{ filename }} + retry 3 aws s3 cp --no-progress {{ filename }} s3://${s3_prefix}/{{ tag }}/{{ filename }} {% endfor %} {% for folder in (folders or []) %} - aws s3 cp --no-progress {{ folder }} s3://${s3_prefix}/{{ tag }}/{{ folder }} --recursive + retry 3 aws s3 cp --no-progress {{ folder }} s3://${s3_prefix}/{{ tag }}/{{ folder }} --recursive {% endfor %} """, label: 'Upload artifacts to S3', @@ -129,12 +152,13 @@ sh( sh( script: """ set -eux + {{ bash_retry() | indent(width=14) }} {% for filename in filenames %} - aws s3 cp --no-progress s3://${s3_prefix}/{{ tag }}/{{ filename }} {{ filename }} + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/{{ tag }}/{{ filename }} {{ filename }} md5sum {{ filename }} {% endfor %} {% for folder in (folders or []) %} - aws s3 cp --no-progress s3://${s3_prefix}/{{ tag }}/{{ folder }} {{ folder }} --recursive + retry 3 aws s3 cp --no-progress s3://${s3_prefix}/{{ tag }}/{{ folder }} {{ folder }} --recursive {% endfor %} """, label: 'Download artifacts from S3',