diff --git a/ci/build.py b/ci/build.py index 0a1ad4cf5751..a9d6a63537f2 100755 --- a/ci/build.py +++ b/ci/build.py @@ -43,43 +43,6 @@ CCACHE_MAXSIZE = '500G' - - -def retry(ExceptionToCheck, tries=4, delay_s=1, backoff=2): - """Retry calling the decorated function using an exponential backoff. - - http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/ - original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry - - :param ExceptionToCheck: the exception to check. may be a tuple of - exceptions to check - :type ExceptionToCheck: Exception or tuple - :param tries: number of times to try (not retry) before giving up - :type tries: int - :param delay_s: initial delay between retries in seconds - :type delay_s: int - :param backoff: backoff multiplier e.g. value of 2 will double the delay - each retry - :type backoff: int - """ - import time - from functools import wraps - def decorated_retry(f): - @wraps(f) - def f_retry(*args, **kwargs): - mtries, mdelay = tries, delay_s - while mtries > 1: - try: - return f(*args, **kwargs) - except ExceptionToCheck as e: - logging.warning("Exception: %s, Retrying in %d seconds...", str(e), mdelay) - time.sleep(mdelay) - mtries -= 1 - mdelay *= backoff - return f(*args, **kwargs) - return f_retry # true decorator - return decorated_retry - def under_ci() -> bool: """:return: True if we run in Jenkins.""" return 'JOB_NAME' in os.environ @@ -114,8 +77,9 @@ def build_docker(platform: str, docker_binary: str, registry: str, num_retries: :param num_retries: Number of retries to build the docker image :return: Id of the top level image """ + tag = get_docker_tag(platform=platform, registry=registry) - logging.info("Building docker container tagged '%s' with %s", tag, docker_binary) + logging.info("Building container tagged '%s' with %s", tag, docker_binary) # # We add a user with the same group as the executing non-root user so files created in the # container match permissions of the local user. Same for the group. @@ -127,24 +91,40 @@ def build_docker(platform: str, docker_binary: str, registry: str, num_retries: # docker pull see: docker_cache.load_docker_cache # # This doesn't work with multi head docker files. - # - cmd = [docker_binary, "build", - "-f", get_dockerfile(platform), - "--build-arg", "USER_ID={}".format(os.getuid()), - "--build-arg", "GROUP_ID={}".format(os.getgid()), - "--cache-from", tag, - "-t", tag, - "docker"] - - @retry(subprocess.CalledProcessError, tries=num_retries) - def run_cmd(): + # + + for i in range(num_retries): + logging.info('%d out of %d tries to build the docker image.', i + 1, num_retries) + + cmd = [docker_binary, "build", + "-f", get_dockerfile(platform), + "--build-arg", "USER_ID={}".format(os.getuid()), + "--build-arg", "GROUP_ID={}".format(os.getgid()), + "--cache-from", tag, + "-t", tag, + "docker"] logging.info("Running command: '%s'", ' '.join(cmd)) - check_call(cmd) + try: + check_call(cmd) + # Docker build was successful. Call break to break out of the retry mechanism + break + except subprocess.CalledProcessError as e: + saved_exception = e + logging.error('Failed to build docker image') + # Building the docker image failed. Call continue to trigger the retry mechanism + continue + else: + # Num retries exceeded + logging.exception('Exception during build of docker image', saved_exception) + logging.fatal('Failed to build the docker image, aborting...') + sys.exit(1) - run_cmd() # Get image id by reading the tag. It's guaranteed (except race condition) that the tag exists. Otherwise, the # check_call would have failed - return _get_local_image_id(docker_binary=docker_binary, docker_tag=tag) + image_id = _get_local_image_id(docker_binary=docker_binary, docker_tag=tag) + if not image_id: + raise FileNotFoundError('Unable to find docker image id matching with {}'.format(tag)) + return image_id def _get_local_image_id(docker_binary, docker_tag): @@ -156,8 +136,6 @@ def _get_local_image_id(docker_binary, docker_tag): cmd = [docker_binary, "images", "-q", docker_tag] image_id_b = subprocess.check_output(cmd) image_id = image_id_b.decode('utf-8').strip() - if not image_id: - raise RuntimeError('Unable to find docker image id matching with tag {}'.format(tag)) return image_id @@ -208,7 +186,7 @@ def container_run(platform: str, '-e', "CCACHE_LOGFILE=/tmp/ccache.log", # a container-scoped log, useful for ccache verification. tag] runlist.extend(command) - cmd = ' \\\n\t'.join(runlist) + cmd = '\\\n\t'.join(runlist) ret = 0 if not dry_run and not interactive: logging.info("Running %s in container %s", command, tag) @@ -221,14 +199,14 @@ def container_run(platform: str, # -ti can't be after the tag, as is interpreted as a command so hook it up after the -u argument idx = into_cmd.index('-u') + 2 into_cmd[idx:idx] = ['-ti'] - cmd = ' \\\n\t'.join(into_cmd) + cmd = '\\\n\t'.join(into_cmd) logging.info("Executing:\n%s\n", cmd) docker_run_cmd = ' '.join(into_cmd) ret = call(into_cmd) if not dry_run and not interactive and ret != 0: logging.error("Running of command in container failed (%s):\n%s\n", ret, cmd) - logging.error("You can get into the container by adding the -i option to this script") + logging.error("You can get into the container by adding the -i option") raise subprocess.CalledProcessError(ret, cmd) return docker_run_cmd @@ -325,6 +303,7 @@ def use_cache(): command = list(chain(*args.command)) docker_binary = get_docker_binary(args.nvidiadocker) shared_memory_size = args.shared_memory_size + num_docker_build_retires = args.docker_build_retries if args.list: list_platforms() @@ -333,7 +312,7 @@ def use_cache(): tag = get_docker_tag(platform=platform, registry=args.docker_registry) if use_cache(): load_docker_cache(tag=tag, docker_registry=args.docker_registry) - build_docker(platform, docker_binary, registry=args.docker_registry, num_retries=args.docker_build_retries) + build_docker(platform, docker_binary, registry=args.docker_registry, num_retries=num_docker_build_retires) if args.build_only: logging.warning("Container was just built. Exiting due to build-only.") return 0 @@ -367,7 +346,7 @@ def use_cache(): tag = get_docker_tag(platform=platform, registry=args.docker_registry) if use_cache(): load_docker_cache(tag=tag, docker_registry=args.docker_registry) - build_docker(platform, docker_binary, args.docker_registry, num_retries=args.docker_build_retries) + build_docker(platform, docker_binary, args.docker_registry, num_retries=num_docker_build_retires) if args.build_only: continue build_platform = "build_{}".format(platform) diff --git a/ci/docker/Dockerfile.build.android_armv7 b/ci/docker/Dockerfile.build.android_armv7 old mode 100644 new mode 100755 diff --git a/ci/docker/Dockerfile.build.android_armv8 b/ci/docker/Dockerfile.build.android_armv8 old mode 100644 new mode 100755 diff --git a/ci/docker/Dockerfile.build.armv6 b/ci/docker/Dockerfile.build.armv6 old mode 100644 new mode 100755 diff --git a/ci/docker/Dockerfile.build.armv7 b/ci/docker/Dockerfile.build.armv7 old mode 100644 new mode 100755 diff --git a/ci/docker/Dockerfile.build.armv8 b/ci/docker/Dockerfile.build.armv8 old mode 100644 new mode 100755 diff --git a/ci/docker/Dockerfile.build.centos7_cpu b/ci/docker/Dockerfile.build.centos7_cpu old mode 100644 new mode 100755 diff --git a/ci/docker/Dockerfile.build.centos7_gpu b/ci/docker/Dockerfile.build.centos7_gpu old mode 100644 new mode 100755 diff --git a/ci/docker/Dockerfile.build.jetson b/ci/docker/Dockerfile.build.jetson old mode 100644 new mode 100755 diff --git a/ci/docker/Dockerfile.build.ubuntu_base_cpu b/ci/docker/Dockerfile.build.ubuntu_base_cpu old mode 100644 new mode 100755 diff --git a/ci/docker/Dockerfile.build.ubuntu_base_gpu b/ci/docker/Dockerfile.build.ubuntu_base_gpu old mode 100644 new mode 100755 diff --git a/ci/docker/Dockerfile.build.ubuntu_blc b/ci/docker/Dockerfile.build.ubuntu_blc old mode 100644 new mode 100755 index 208cba2111f6..294740ce1392 --- a/ci/docker/Dockerfile.build.ubuntu_blc +++ b/ci/docker/Dockerfile.build.ubuntu_blc @@ -24,13 +24,8 @@ WORKDIR /work/deps COPY install/ubuntu_core.sh /work/ RUN /work/ubuntu_core.sh - -COPY install/ubuntu_python2.sh /work/ -RUN /work/ubuntu_python2.sh - -COPY install/ubuntu_python3.sh /work/ -RUN /work/ubuntu_python3.sh - +COPY install/ubuntu_python.sh /work/ +RUN /work/ubuntu_python.sh COPY install/ubuntu_npm_blc.sh /work/ RUN /work/ubuntu_npm_blc.sh diff --git a/ci/docker/Dockerfile.build.ubuntu_build_cuda b/ci/docker/Dockerfile.build.ubuntu_build_cuda old mode 100644 new mode 100755 index 19e9265f88d0..9ed0cbbe3e52 --- a/ci/docker/Dockerfile.build.ubuntu_build_cuda +++ b/ci/docker/Dockerfile.build.ubuntu_build_cuda @@ -27,30 +27,20 @@ WORKDIR /work/deps COPY install/ubuntu_core.sh /work/ RUN /work/ubuntu_core.sh - COPY install/deb_ubuntu_ccache.sh /work/ RUN /work/deb_ubuntu_ccache.sh - -COPY install/ubuntu_python2.sh /work/ -RUN /work/ubuntu_python2.sh - -COPY install/ubuntu_python3.sh /work/ -RUN /work/ubuntu_python3.sh - +COPY install/ubuntu_python.sh /work/ +RUN /work/ubuntu_python.sh COPY install/ubuntu_scala.sh /work/ COPY install/sbt.gpg /work/ RUN /work/ubuntu_scala.sh - COPY install/ubuntu_r.sh /work/ COPY install/r.gpg /work/ RUN /work/ubuntu_r.sh - COPY install/ubuntu_perl.sh /work/ RUN /work/ubuntu_perl.sh - COPY install/ubuntu_clang.sh /work/ RUN /work/ubuntu_clang.sh - COPY install/ubuntu_mklml.sh /work/ RUN /work/ubuntu_mklml.sh diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/docker/Dockerfile.build.ubuntu_cpu old mode 100644 new mode 100755 index 08fb04df03e2..6751465758c8 --- a/ci/docker/Dockerfile.build.ubuntu_cpu +++ b/ci/docker/Dockerfile.build.ubuntu_cpu @@ -28,11 +28,8 @@ RUN /work/ubuntu_core.sh COPY install/deb_ubuntu_ccache.sh /work/ RUN /work/deb_ubuntu_ccache.sh -COPY install/ubuntu_python2.sh /work/ -RUN /work/ubuntu_python2.sh - -COPY install/ubuntu_python3.sh /work/ -RUN /work/ubuntu_python3.sh +COPY install/ubuntu_python.sh /work/ +RUN /work/ubuntu_python.sh COPY install/ubuntu_scala.sh /work/ COPY install/sbt.gpg /work/ diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/docker/Dockerfile.build.ubuntu_gpu old mode 100644 new mode 100755 index d99dafb0bd0c..8fcbcbbb9674 --- a/ci/docker/Dockerfile.build.ubuntu_gpu +++ b/ci/docker/Dockerfile.build.ubuntu_gpu @@ -28,11 +28,8 @@ RUN /work/ubuntu_core.sh COPY install/deb_ubuntu_ccache.sh /work/ RUN /work/deb_ubuntu_ccache.sh -COPY install/ubuntu_python2.sh /work/ -RUN /work/ubuntu_python2.sh - -COPY install/ubuntu_python3.sh /work/ -RUN /work/ubuntu_python3.sh +COPY install/ubuntu_python.sh /work/ +RUN /work/ubuntu_python.sh COPY install/ubuntu_scala.sh /work/ COPY install/sbt.gpg /work/ diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt b/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt old mode 100644 new mode 100755 index 3f0bbc666f25..255da316041f --- a/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt +++ b/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt @@ -24,16 +24,10 @@ WORKDIR /work/deps COPY install/ubuntu_core.sh /work/ RUN /work/ubuntu_core.sh - COPY install/deb_ubuntu_ccache.sh /work/ RUN /work/deb_ubuntu_ccache.sh - -COPY install/ubuntu_python2.sh /work/ -RUN /work/ubuntu_python2.sh - -COPY install/ubuntu_python3.sh /work/ -RUN /work/ubuntu_python3.sh - +COPY install/ubuntu_python.sh /work/ +RUN /work/ubuntu_python.sh COPY install/tensorrt.sh /work RUN /work/tensorrt.sh diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_cpu b/ci/docker/Dockerfile.build.ubuntu_nightly_cpu old mode 100644 new mode 100755 index 834710c4ceb6..1b126c18be47 --- a/ci/docker/Dockerfile.build.ubuntu_nightly_cpu +++ b/ci/docker/Dockerfile.build.ubuntu_nightly_cpu @@ -28,11 +28,8 @@ RUN /work/ubuntu_core.sh COPY install/deb_ubuntu_ccache.sh /work/ RUN /work/deb_ubuntu_ccache.sh -COPY install/ubuntu_python2.sh /work/ -RUN /work/ubuntu_python2.sh - -COPY install/ubuntu_python3.sh /work/ -RUN /work/ubuntu_python3.sh +COPY install/ubuntu_python.sh /work/ +RUN /work/ubuntu_python.sh COPY install/ubuntu_scala.sh /work/ COPY install/sbt.gpg /work/ diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu b/ci/docker/Dockerfile.build.ubuntu_nightly_gpu old mode 100644 new mode 100755 index fb34307063a0..deeed8b0d52a --- a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu +++ b/ci/docker/Dockerfile.build.ubuntu_nightly_gpu @@ -28,11 +28,8 @@ RUN /work/ubuntu_core.sh COPY install/deb_ubuntu_ccache.sh /work/ RUN /work/deb_ubuntu_ccache.sh -COPY install/ubuntu_python2.sh /work/ -RUN /work/ubuntu_python2.sh - -COPY install/ubuntu_python3.sh /work/ -RUN /work/ubuntu_python3.sh +COPY install/ubuntu_python.sh /work/ +RUN /work/ubuntu_python.sh COPY install/ubuntu_scala.sh /work/ COPY install/sbt.gpg /work/ diff --git a/ci/docker/Dockerfile.build.ubuntu_rat b/ci/docker/Dockerfile.build.ubuntu_rat old mode 100644 new mode 100755 diff --git a/ci/docker/install/ubuntu_python2.sh b/ci/docker/install/ubuntu_python.sh similarity index 84% rename from ci/docker/install/ubuntu_python2.sh rename to ci/docker/install/ubuntu_python.sh index f0526e2d8300..e71cac8a3898 100755 --- a/ci/docker/install/ubuntu_python2.sh +++ b/ci/docker/install/ubuntu_python.sh @@ -22,10 +22,12 @@ set -ex # install libraries for mxnet's python package on ubuntu -apt-get install -y python-dev virtualenv wget +apt-get install -y python-dev python3-dev virtualenv # the version of the pip shipped with ubuntu may be too lower, install a recent version here wget -nv https://bootstrap.pypa.io/get-pip.py +python3 get-pip.py python2 get-pip.py -pip2 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 mock +pip2 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 +pip3 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 diff --git a/ci/docker/install/ubuntu_python3.sh b/ci/docker/install/ubuntu_python3.sh deleted file mode 100755 index 1dad5a7aa207..000000000000 --- a/ci/docker/install/ubuntu_python3.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# build and install are separated so changes to build don't invalidate -# the whole docker cache for the image - -set -ex -# install libraries for mxnet's python package on ubuntu -apt-get install -y python3-dev virtualenv wget - -# the version of the pip shipped with ubuntu may be too lower, install a recent version here -wget -nv https://bootstrap.pypa.io/get-pip.py -python3 get-pip.py - -pip3 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 mock diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index e4aac8b18bdd..a0c4723353d0 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -994,7 +994,6 @@ broken_link_checker() { ./tests/nightly/broken_link_checker_test/broken_link_checker.sh } - ############################################################## # MAIN # diff --git a/ci/test_docker_cache.py b/ci/test_docker_cache.py old mode 100755 new mode 100644