From a171bb9e547cd7cfbcbc68ee6eacc50b4cc3d9be Mon Sep 17 00:00:00 2001 From: Jeff Reback Date: Wed, 22 Mar 2017 09:56:58 -0400 Subject: [PATCH] travis deduping on prs closes #12438 Author: Jeff Reback Closes #15780 from jreback/dedupe and squashes the following commits: 64f217e [Jeff Reback] replace . by space b6f2a62 [Jeff Reback] formatting 0c33d9b [Jeff Reback] tests commit 24f6ae6 [Jeff Reback] CI: fast finish travis builds for the same PR --- .travis.yml | 5 ++- ci/install_travis.sh | 41 +++++++++++---------- ci/travis_fast_finish.py | 77 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 21 deletions(-) create mode 100755 ci/travis_fast_finish.py diff --git a/.travis.yml b/.travis.yml index 67b37f1d58931..270f8c2fc76c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -177,15 +177,14 @@ matrix: - USE_CACHE=true before_install: + - echo "Checking to see if this build is outdated" + - ci/travis_fast_finish.py || { echo "Failing outdated build to end it."; exit 1; } - echo "before_install" - source ci/travis_process_gbq_encryption.sh - - echo $VIRTUAL_ENV - export PATH="$HOME/miniconda3/bin:$PATH" - df -h - - date - pwd - uname -a - - python -V - git --version - git tag - ci/before_install_travis.sh diff --git a/ci/install_travis.sh b/ci/install_travis.sh index c940083f5ae9e..66633c0592748 100755 --- a/ci/install_travis.sh +++ b/ci/install_travis.sh @@ -1,18 +1,6 @@ #!/bin/bash -# There are 2 distinct pieces that get zipped and cached -# - The venv site-packages dir including the installed dependencies -# - The pandas build artifacts, using the build cache support via -# scripts/use_build_cache.py -# -# if the user opted in to use the cache and we're on a whitelisted fork -# - if the server doesn't hold a cached version of venv/pandas build, -# do things the slow way, and put the results on the cache server -# for the next time. -# - if the cache files are available, instal some necessaries via apt -# (no compiling needed), then directly goto script and collect 200$. -# - +# edit the locale file if needed function edit_init() { if [ -n "$LOCALE_OVERRIDE" ]; then @@ -26,15 +14,18 @@ function edit_init() fi } +echo echo "[install_travis]" edit_init home_dir=$(pwd) -echo "[home_dir: $home_dir]" +echo +echo "[home_dir]: $home_dir" # install miniconda MINICONDA_DIR="$HOME/miniconda3" +echo echo "[Using clean Miniconda install]" if [ -d "$MINICONDA_DIR" ]; then @@ -49,14 +40,17 @@ else fi time bash miniconda.sh -b -p "$MINICONDA_DIR" || exit 1 +echo echo "[show conda]" which conda +echo echo "[update conda]" conda config --set ssl_verify false || exit 1 conda config --set always_yes true --set changeps1 false || exit 1 conda update -q conda +echo echo "[add channels]" # add the pandas channel to take priority # to add extra packages @@ -73,26 +67,28 @@ fi conda info -a || exit 1 # set the compiler cache to work +echo if [ "$USE_CACHE" ] && [ "${TRAVIS_OS_NAME}" == "linux" ]; then echo "[Using ccache]" export PATH=/usr/lib/ccache:/usr/lib64/ccache:$PATH gcc=$(which gcc) - echo "[gcc: $gcc]" + echo "[gcc]: $gcc" ccache=$(which ccache) - echo "[ccache: $ccache]" + echo "[ccache]: $ccache" export CC='ccache gcc' elif [ "$USE_CACHE" ] && [ "${TRAVIS_OS_NAME}" == "osx" ]; then echo "[Using ccache]" time brew install ccache export PATH=/usr/local/opt/ccache/libexec:$PATH gcc=$(which gcc) - echo "[gcc: $gcc]" + echo "[gcc]: $gcc" ccache=$(which ccache) - echo "[ccache: $ccache]" + echo "[ccache]: $ccache" else echo "[Not using ccache]" fi +echo echo "[create env]" # may have installation instructions for this build @@ -106,6 +102,7 @@ else fi # build deps +echo echo "[build installs]" REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.build" if [ -e ${REQ} ]; then @@ -113,6 +110,7 @@ if [ -e ${REQ} ]; then fi # may have addtl installation instructions for this build +echo echo "[build addtl installs]" REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.build.sh" if [ -e ${REQ} ]; then @@ -132,6 +130,7 @@ if [ "$COVERAGE" ]; then pip install coverage pytest-cov fi +echo if [ "$BUILD_TEST" ]; then # build & install testing @@ -151,6 +150,7 @@ else fi # we may have run installations +echo echo "[conda installs]" REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.run" if [ -e ${REQ} ]; then @@ -158,6 +158,7 @@ if [ -e ${REQ} ]; then fi # we may have additional pip installs +echo echo "[pip installs]" REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.pip" if [ -e ${REQ} ]; then @@ -165,6 +166,7 @@ if [ -e ${REQ} ]; then fi # may have addtl installation instructions for this build +echo echo "[addtl installs]" REQ="ci/requirements-${PYTHON_VERSION}${JOB_TAG}.sh" if [ -e ${REQ} ]; then @@ -176,14 +178,17 @@ if [ -z "$BUILD_TEST" ]; then # remove any installed pandas package # w/o removing anything else + echo echo "[removing installed pandas]" conda remove pandas --force # install our pandas + echo echo "[running setup.py develop]" python setup.py develop || exit 1 fi +echo echo "[done]" exit 0 diff --git a/ci/travis_fast_finish.py b/ci/travis_fast_finish.py new file mode 100755 index 0000000000000..c2e2a9159918b --- /dev/null +++ b/ci/travis_fast_finish.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python + +# script to cancel previous travis builds for the same PR +# originally from +# https://github.com/conda-forge/staged-recipes/pull/2257 + +try: + from future_builtins import ( + map, + filter, + ) +except ImportError: + pass + +import codecs +import contextlib +import json +import os + +try: + from urllib.request import ( + Request, + urlopen, + ) +except ImportError: + from urllib2 import ( + Request, + urlopen, + ) + + +def check_latest_pr_build(repo, pr, build_num): + # Not a PR so it is latest. + if pr is None: + return True + + headers = { + "Accept": "application/vnd.travis-ci.2+json", + } + url = "https://api.travis-ci.org/repos/{repo}/builds?event_type=pull_request" + + request = Request(url.format(repo=repo), headers=headers) + with contextlib.closing(urlopen(request)) as response: + reader = codecs.getreader("utf-8") + data = json.load(reader(response)) + + # Parse the response to get a list of build numbers for this PR. + builds = data["builds"] + pr_builds = filter(lambda b: b["pull_request_number"] == pr, builds) + pr_build_nums = sorted(map(lambda b: int(b["number"]), pr_builds)) + + print("build_num: {}".format(build_num)) + print("pr_build_nums: {}".format(','.join([str(n) for n in pr_build_nums]))) + + # Check if our build number is the latest (largest) + # out of all of the builds for this PR. + if build_num < max(pr_build_nums): + return False + else: + return True + + +def main(): + repo = os.environ["TRAVIS_REPO_SLUG"] + + pr = os.environ["TRAVIS_PULL_REQUEST"] + pr = None if pr == "false" else int(pr) + build_num = int(os.environ["TRAVIS_BUILD_NUMBER"]) + + print("checking for fast_finish: {}-{}-{}".format(repo, pr, build_num)) + + return int(check_latest_pr_build(repo, pr, build_num) is False) + + +if __name__ == "__main__": + import sys + sys.exit(main())