diff --git a/cli/travis/setup.sh b/cli/travis/setup.sh deleted file mode 100755 index 5f6c76fe22..0000000000 --- a/cli/travis/setup.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -cli_dir="$(dirname "$( dirname "${BASH_SOURCE[0]}" )" )" -cd "$cli_dir" - -# Don't use --user in virtualenv -if [[ "$(pip -V)" != *${HOME}* ]]; then - pip_flags=--user -else - pip_flags= -fi - -# Parse dependencies from setup.py -dependencies="$(sed -nE "s/^\\s+'([^']+)',\$/\\1/p" < setup.py)" - -pip install $pip_flags $dependencies diff --git a/executor/travis/run_tests.sh b/executor/travis/run_tests.sh deleted file mode 100755 index 97df935211..0000000000 --- a/executor/travis/run_tests.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -# Runs the Cook Executor tests - -export PROJECT_DIR=`pwd` -cd ${PROJECT_DIR} - -python --version -pytest --version - -pytest -n4 diff --git a/executor/travis/setup.sh b/executor/travis/setup.sh deleted file mode 100755 index 165739f8eb..0000000000 --- a/executor/travis/setup.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -# Sets up the travis worker to be able to run executor tests. - -export PROJECT_DIR=`pwd` -cd ${PROJECT_DIR} - -python --version -pip install -e '.[test]' diff --git a/integration/travis/prepare_integration.sh b/integration/travis/prepare_integration.sh deleted file mode 100755 index 93cfb1e522..0000000000 --- a/integration/travis/prepare_integration.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -set -ev - -PROJECT_DIR=`pwd` ../travis/prepare.sh -python --version - -# Explicitly uninstall cli -if [[ $(pip list --format=columns | grep cook-client) ]]; -then - pip uninstall -y cook-client -fi - -pip install -r requirements.txt diff --git a/integration/travis/run_integration.sh b/integration/travis/run_integration.sh deleted file mode 100755 index 481f745c5a..0000000000 --- a/integration/travis/run_integration.sh +++ /dev/null @@ -1,185 +0,0 @@ -#!/bin/bash - -# Usage: ./run_integration [OPTIONS...] -# --auth={http-basic,one-user} Use the specified authentication scheme. Default is one-user. -# --executor={cook,mesos} Use the specified job executor. Default is mesos. -# --pools={on,off} Use or don't use pools in the Cook under test. Default is on. - -set -ev - -export PROJECT_DIR=`pwd` - -COOK_AUTH=one-user -COOK_EXECUTOR=mesos -COOK_POOLS=on -CONFIG_FILE=scheduler_travis_config.edn -COOK_TEST_DOCKER_IMAGE="" - -while (( $# > 0 )); do - case "$1" in - --auth=*) - COOK_AUTH="${1#--auth=}" - shift - ;; - --executor=*) - COOK_EXECUTOR="${1#--executor=}" - shift - ;; - --pools=*) - COOK_POOLS="${1#--pools=}" - shift - ;; - --image=*) - COOK_TEST_DOCKER_IMAGE="${1#--image=}" - shift - ;; - *) - echo "Unrecognized option: $1" - exit 1 - esac -done - -case "$COOK_AUTH" in - http-basic) - export COOK_HTTP_BASIC_AUTH=true - ;; - one-user) - export COOK_EXECUTOR_PORTION=1 - # One user auth is configured to use root - export COOK_DOCKER_UID=0 - export COOK_DOCKER_GID=0 - ;; - *) - echo "Unrecognized auth scheme: $COOK_AUTH" - exit 1 -esac - -case "$COOK_EXECUTOR" in - cook) - echo "$(date +%H:%M:%S) Cook executor has been enabled" - COOK_EXECUTOR_COMMAND="${GITHUB_WORKSPACE}/travis/cook-executor-local/cook-executor-local" - # Build cook-executor - ${GITHUB_WORKSPACE}/travis/build_cook_executor.sh - # Run with docker - ;; - mesos) - COOK_EXECUTOR_COMMAND="" - ;; - *) - echo "Unrecognized executor: $EXECUTOR" - exit 1 -esac - -function wait_for_cook { - COOK_PORT=${1:-12321} - while ! curl -s localhost:${COOK_PORT} >/dev/null; - do - echo "$(date +%H:%M:%S) Cook is not listening on ${COOK_PORT} yet" - sleep 2.0 - done - echo "$(date +%H:%M:%S) Connected to Cook on ${COOK_PORT}!" - curl -s localhost:${COOK_PORT}/info - echo -} -export -f wait_for_cook - -# Start minimesos -cd ${GITHUB_WORKSPACE}/travis -./minimesos up -$(./minimesos info | grep MINIMESOS) -export COOK_ZOOKEEPER="${MINIMESOS_ZOOKEEPER_IP}:2181" -export MINIMESOS_ZOOKEEPER=${MINIMESOS_ZOOKEEPER%;} -export MINIMESOS_MASTER=${MINIMESOS_MASTER%;} - -SCHEDULER_DIR=${GITHUB_WORKSPACE}/scheduler -./datomic-free-0.9.5394/bin/transactor ${SCHEDULER_DIR}/datomic/datomic_transactor.properties & -COOK_DATOMIC_URI_1=datomic:free://localhost:4334/cook-jobs -COOK_DATOMIC_URI_2=datomic:mem://cook-jobs - -# Generate SSL certificate -COOK_KEYSTORE_PATH=${SCHEDULER_DIR}/cook.p12 -keytool -genkeypair -keystore ${COOK_KEYSTORE_PATH} -storetype PKCS12 -storepass cookstore -dname "CN=cook, OU=Cook Developers, O=Two Sigma Investments, L=New York, ST=New York, C=US" -keyalg RSA -keysize 2048 -export COOK_KEYSTORE_PATH=${COOK_KEYSTORE_PATH} - -case "$COOK_POOLS" in - on) - echo "Pools are turned on for this run" - DEFAULT_POOL=mesos-gamma - cd ${SCHEDULER_DIR} - lein exec -p datomic/data/seed_pools.clj ${COOK_DATOMIC_URI_1} - ;; - off) - echo "Pools are turned off for this run" - ;; - *) - echo "Unrecognized pools toggle: $COOK_POOLS" - exit 1 -esac - -mkdir ${SCHEDULER_DIR}/log - -# Seed running jobs, which are used to test the task reconciler -cd ${SCHEDULER_DIR} -COOK_FRAMEWORK_ID=cook-framework-1 lein exec -p datomic/data/seed_running_jobs.clj ${COOK_DATOMIC_URI_1} - -# Start three cook schedulers. -# We want one cluster with two cooks to run MasterSlaveTest, and a second cluster to run MultiClusterTest. -# The basic tests will run against cook-framework-1 -## on travis, ports on 172.17.0.1 are bindable from the host OS, and are also -## available for processes inside minimesos containers to connect to -export COOK_EXECUTOR_COMMAND=${COOK_EXECUTOR_COMMAND} -if [[ ! -z "${COOK_TEST_DOCKER_IMAGE}" ]]; then - export COOK_TEST_DOCKER_IMAGE=${COOK_TEST_DOCKER_IMAGE} -fi - -# Start one cook listening on port 12321, this will be the master of the "cook-framework-1" framework -LIBPROCESS_IP=172.17.0.1 COOK_DATOMIC="${COOK_DATOMIC_URI_1}" COOK_PORT=12321 COOK_SSL_PORT=12322 COOK_FRAMEWORK_ID=cook-framework-1 COOK_LOGFILE="log/cook-12321.log" COOK_DEFAULT_POOL=${DEFAULT_POOL} lein run ${PROJECT_DIR}/travis/${CONFIG_FILE} & -# Start a second cook listening on port 22321, this will be the master of the "cook-framework-2" framework -LIBPROCESS_IP=172.17.0.1 COOK_DATOMIC="${COOK_DATOMIC_URI_2}" COOK_PORT=22321 COOK_SSL_PORT=22322 COOK_ZOOKEEPER_LOCAL=true COOK_ZOOKEEPER_LOCAL_PORT=4291 COOK_FRAMEWORK_ID=cook-framework-2 COOK_LOGFILE="log/cook-22321.log" lein run ${PROJECT_DIR}/travis/${CONFIG_FILE} & - -# Wait for the cooks to be listening -timeout 180s bash -c "wait_for_cook 12321" || curl_error=true -if [ "$curl_error" = true ]; then - echo "$(date +%H:%M:%S) Timed out waiting for cook to start listening" - ${GITHUB_WORKSPACE}/travis/upload_logs.sh - exit 1 -fi - -# Start a third cook listening on port 12323, this will be a slave on the "cook-framework-1" framework -LIBPROCESS_IP=172.17.0.1 COOK_DATOMIC="${COOK_DATOMIC_URI_1}" COOK_PORT=12323 COOK_SSL_PORT=12324 COOK_FRAMEWORK_ID=cook-framework-1 COOK_LOGFILE="log/cook-12323.log" COOK_DEFAULT_POOL=${DEFAULT_POOL} lein run ${PROJECT_DIR}/travis/${CONFIG_FILE} & - -timeout 180s bash -c "wait_for_cook 12323" || curl_error=true -if [ "$curl_error" = true ]; then - echo "$(date +%H:%M:%S) Timed out waiting for cook to start listening" - ${GITHUB_WORKSPACE}/travis/upload_logs.sh - exit 1 -fi -timeout 180s bash -c "wait_for_cook 22321" || curl_error=true -if [ "$curl_error" = true ]; then - echo "$(date +%H:%M:%S) Timed out waiting for cook to start listening" - ${GITHUB_WORKSPACE}/travis/upload_logs.sh - exit 1 -fi - -# Ensure the Cook Scheduler CLI is available -command -v cs - -# Run the integration tests -cd ${PROJECT_DIR} -export COOK_MULTI_CLUSTER= -export COOK_MASTER_SLAVE= -export COOK_SLAVE_URL=http://localhost:12323 -export COOK_MESOS_LEADER_URL=${MINIMESOS_MASTER} -{ - echo "Using Mesos leader URL: ${COOK_MESOS_LEADER_URL}" - pytest -n4 -v --color=no --timeout-method=thread --boxed -m "not serial and not travis_skip" || test_failures=true - pytest -n0 -v --color=no --timeout-method=thread --boxed -m "serial and not travis_skip" || test_failures=true -} &> >(tee ./log/pytest.log) - - -# If there were failures, then we should save the logs -if [ "$test_failures" = true ]; then - echo "Uploading logs..." - ${GITHUB_WORKSPACE}/travis/upload_logs.sh - exit 1 -fi diff --git a/integration/travis/scheduler_travis_config.edn b/integration/travis/scheduler_travis_config.edn deleted file mode 100644 index adcbf8e6c3..0000000000 --- a/integration/travis/scheduler_travis_config.edn +++ /dev/null @@ -1,79 +0,0 @@ -{:port #config/env-int "COOK_PORT" - :ssl {:port #config/env-int "COOK_SSL_PORT" - :keystore-path #config/env "COOK_KEYSTORE_PATH" - :keystore-type "pkcs12" - :keystore-pass "cookstore"} - :hostname "172.17.0.1" - :authorization {;; Note that internally, Cook will select :http-basic if it's set to true, - ;; and fall back to :one-user only if :http-basic is false. - :http-basic #config/env-bool "COOK_HTTP_BASIC_AUTH" - :one-user "root"} - :authorization-config {;; What function should be used to perform user authorization? - ;; See the docstring in cook.rest.authorization for details. - :authorization-fn cook.rest.authorization/configfile-admins-auth-open-gets - ;; These users have admin privileges when using configfile-admins-auth; - ;; e.g., they can view and modify other users' jobs. - :admins #{"root" "travis"} - ;; users that are allowed to do things on behalf of others - :impersonators #{"poser" "travis"}} - :compute-clusters [{:factory-fn cook.mesos.mesos-compute-cluster/factory-fn - :config {:framework-id #config/env "COOK_FRAMEWORK_ID" - :compute-cluster-name "local-mesos" - :master #config/env "MINIMESOS_ZOOKEEPER" - :failover-timeout-ms nil - :role "cook"}}] - :container-defaults {:volumes [{:host-path "/tmp/cook-integration-mount" - :container-path "/mnt/cook-integration" - :mode "RW"}]} - :cors-origins ["https?://cors.example.com"] - :database {:datomic-uri #config/env "COOK_DATOMIC"} - :zookeeper {:connection #config/env "COOK_ZOOKEEPER" - :local? #config/env-bool "COOK_ZOOKEEPER_LOCAL" - :local-port #config/env-int-default ["COOK_ZOOKEEPER_LOCAL_PORT" -1]} - :scheduler {:offer-incubate-ms 15000 - :task-constraints {:timeout-hours 1 - :timeout-interval-minutes 1 - :memory-gb 48 - :retry-limit 200 - :cpus 10 - :command-length-limit 5000}} - :rate-limit {:expire-minutes 120 ; Expire unused rate limit entries after 2 hours. - ; Keep these job-launch and job-submission values as they are for integration tests. Making them smaller can cause - ; spurious failures, and making them larger will cause the rate-limit integration test to skip itself. - :job-launch {:bucket-size #config/env-int-default ["JOB_LAUNCH_RATE_LIMIT_BUCKET_SIZE" 10000] - :enforce? true - :tokens-replenished-per-minute #config/env-int-default ["JOB_LAUNCH_RATE_LIMIT_REPLENISHED_PER_MINUTE" 10000]} - :job-submission {:bucket-size 1000 - :enforce? true - :tokens-replenished-per-minute 600} - :user-limit-per-m 1000000} - :plugins {:job-submission-validator {:batch-timeout-seconds 40 - :factory-fns ["cook.plugins.demo-plugin/submission-factory" - "cook.plugins.demo-plugin/submission-factory2"]} - :job-launch-filter {:age-out-last-seen-deadline-minutes 10 - :age-out-first-seen-deadline-minutes 600 - :age-out-seen-count 10 - :factory-fn "cook.plugins.demo-plugin/launch-factory"}} - :rebalancer {:dru-scale 1 - :interval-seconds 30 - :max-preemption 500.0 - :min-dru-diff 1.0 - :safe-dru-threshold 1.0} - :mesos {:leader-path "/cook-scheduler"} - :executor {:command #config/env "COOK_EXECUTOR_COMMAND" - :environment {"EXECUTOR_DEFAULT_PROGRESS_OUTPUT_NAME" "stdout"} - :portion #config/env-int-default ["COOK_EXECUTOR_PORTION" 0]} - :agent-query-cache {:ttl-ms 1000} - :sandbox-syncer {:sync-interval-ms 1000} - :unhandled-exceptions {:log-level :error} - :metrics {:jmx true} - :nrepl {:enabled? false} - :log {:file #config/env "COOK_LOGFILE" - :levels {"datomic.db" :warn - "datomic.peer" :warn - "datomic.kv-cluster" :warn - "cook.mesos.fenzo-utils" :debug - "cook.mesos.rebalancer" :debug - "cook.mesos.scheduler" :debug - :default :info}} - :pools {:default #config/env "COOK_DEFAULT_POOL"}} diff --git a/scheduler/travis/setup.sh b/scheduler/travis/setup.sh deleted file mode 100755 index 414ae943e2..0000000000 --- a/scheduler/travis/setup.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -set -e - -# Install the current version of the jobclient -pushd ${GITHUB_WORKSPACE}/jobclient/java -mvn install -popd - -# Install lein dependencies -lein with-profiles +test deps - diff --git a/simulator/travis/prepare_simulation.sh b/simulator/travis/prepare_simulation.sh deleted file mode 100755 index 9d57d357c4..0000000000 --- a/simulator/travis/prepare_simulation.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -set -ev - -export PROJECT_DIR=`pwd` - -lein deps - -../travis/prepare.sh - -docker pull python:3 diff --git a/simulator/travis/run_simulation.sh b/simulator/travis/run_simulation.sh deleted file mode 100755 index b2dee33a0c..0000000000 --- a/simulator/travis/run_simulation.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -set -ev - -export PROJECT_DIR=`pwd` -${GITHUB_WORKSPACE}/travis/start_scheduler.sh - -cd ${PROJECT_DIR} -lein run -c config/settings.edn setup-database -c travis/simulator_config.edn - -set +e -lein run -c config/settings.edn travis -c travis/simulator_config.edn -SIM_EXIT_CODE=$? - -if [ ${SIM_EXIT_CODE} -ne 0 ]; then - echo "Displaying executor logs" - ${GITHUB_WORKSPACE}/travis/show_executor_logs.sh -fi - -exit ${SIM_EXIT_CODE} diff --git a/simulator/travis/scheduler_config.edn b/simulator/travis/scheduler_config.edn deleted file mode 100644 index 917b494b9c..0000000000 --- a/simulator/travis/scheduler_config.edn +++ /dev/null @@ -1,41 +0,0 @@ -{:port 12321 - :hostname "172.17.0.1" - :authorization {:http-basic true} - :database {:datomic-uri "datomic:free://localhost:4334/cook-jobs"} - :zookeeper {:local? true} - :scheduler {:offer-incubate-ms 15000 - :task-constraints {:timeout-hours 1 - :timeout-interval-minutes 1 - :memory-gb 48 - :retry-limit 200 - :cpus 6}} - :rebalancer {:interval-seconds 20 - :safe-dru-threshold 0.0 - ;; virtually ANY improvement in DRU balance should provoke preemption: - :min-dru-diff 1E-309 - :max-preemption 64.0 - :dru-scale 1} - - :mesos {:master "zk://172.17.0.3:2181/mesos" ; minimesos zookeeper - :failover-timeout-ms nil - :leader-path "/cook-scheduler" - :role "cook" - :framework-id "cook-framework"} - :compute-clusters [{:factory-fn cook.mesos.mesos-compute-cluster/factory-fn - :config {:compute-cluster-name "default-compute-cluster-from-config-defaulting" - :framework-id "cook-framework" - :master "zk://172.17.0.3:2181/mesos" - :failover-timeout nil - :principal nil - :role "cook" - :framework-name nil}}] - :unhandled-exceptions {:log-level :error} - :metrics {:jmx true} - :nrepl {:enabled? true - :port 8888} - :log {:file "log/cook.log" - :levels {"datomic.db" :warn - "datomic.peer" :warn - "datomic.kv-cluster" :warn - "cook.mesos.rebalancer" :debug - :default :info}}} diff --git a/simulator/travis/simulator_config.edn b/simulator/travis/simulator_config.edn deleted file mode 100644 index b608fda275..0000000000 --- a/simulator/travis/simulator_config.edn +++ /dev/null @@ -1,79 +0,0 @@ -{:sim-db-uri "datomic:free://localhost:4334/cook-sim" - :cook-db-uri "datomic:free://localhost:4334/cook-jobs" - :cook-api-uri "http://localhost:12321" - :process-count 10 - :sim-model {:label "Travis sim run" - :duration-seconds 30 - :user-profiles - [{:description "Heavy Users" - :usernames ["heavy1" "heavy2" "heavy3" "heavy4" "heavy5"] - :docker-tendency 0.4 - :group-tendency 0.4 - :group-size {:mean 5 - :std-dev 2 - :floor 2 - :ceiling 10} - :seconds-between-jobs {:mean 2 - :std-dev 1 - :floor 1 - :ceiling 120} - :job-duration {:mean 10 - :std-dev 2 - :floor 1 - :ceiling 120} - :job-memory {:mean 512 - :std-dev 200 - :floor 256 - :ceiling 2048} - :job-cpu {:mean 3.0 - :std-dev 1.0 - :floor 1.0 - :ceiling 6.0}} - {:description "Light Users" - :usernames ["light1" "light2" "light3" "light4" "light5"] - :docker-tendency 0.6 - :group-tendency 0.6 - :group-size {:mean 3 - :std-dev 1 - :floor 2 - :ceiling 6} - :seconds-between-jobs {:mean 6 - :std-dev 1 - :floor 1 - :ceiling 120} - :job-duration {:mean 15 - :std-dev 2 - :floor 1 - :ceiling 120} - :job-memory {:mean 1024 - :std-dev 1000 - :floor 256 - :ceiling 4000} - :job-cpu {:mean 4.0 - :std-dev 2.0 - :floor 1.0 - :ceiling 6.0}} - {:description "Unschedulable Users" - :usernames ["unsched1"] - :group-tendency 0.5 - :group-size {:mean 4 - :std-dev 1 - :floor 2 - :ceiling 6} - :seconds-between-jobs {:mean 10 - :std-dev 3 - :floor 1 - :ceiling 120} - :docker-tendency 0.5 - :job-duration {:mean 15 - :std-dev 2 - :floor 1 - :ceiling 120} - :job-memory {:mean 10240 - :std-dev 1000 - :floor 9000 - :ceiling 20000} - :job-cpu {:mean 4.0 - :std-dev 2.0 - :floor 1.0 - :ceiling 6.0}}]}} diff --git a/travis/build_cook_executor.sh b/travis/build_cook_executor.sh deleted file mode 100755 index a8c3ecb236..0000000000 --- a/travis/build_cook_executor.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -ev - -cd ${GITHUB_WORKSPACE}/executor -pip install -r requirements.txt -./bin/prepare-executor.sh local ${GITHUB_WORKSPACE}/scheduler/resources/public -tar -C ${GITHUB_WORKSPACE}/travis -xzf ./dist/cook-executor-local.tar.gz diff --git a/travis/gdrive_upload b/travis/gdrive_upload deleted file mode 100755 index c078cb247a..0000000000 --- a/travis/gdrive_upload +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 - -import base64 -import os -import requests -import sys -import warnings - -if len(sys.argv) != 3: - print('USAGE: {} JOB-ID XZ-FILE', sys.argv[0]) - print('Upload an xz-compressed file to our Google Drive stash') - sys.exit(1) - -tarball_path = sys.argv[2] - -# upload to google drive -app_url = os.environ.get('GDRIVE_LOG_POST_URL') - -if not app_url: - print('Missing application url. Please set GDRIVE_LOG_POST_URL in the environment.') - sys.exit(1) - -with open(tarball_path, 'rb') as tarball: - post_data = { - 'job_id': sys.argv[1], - 'tarball': base64.b64encode(tarball.read()) - } - -with warnings.catch_warnings(): - warnings.simplefilter('ignore') - response = requests.post(app_url, data=post_data, timeout=10) - -print() -print('==============================') -print('== UPLOAD RESPONSE:') -print('==============================') -print(response.text) -print('==============================') -print() - -if not response.text.strip().endswith('successfully'): - print('UPLOAD FAILED!') - sys.exit(1) diff --git a/travis/install_mesos.sh b/travis/install_mesos.sh deleted file mode 100755 index 6d29ff4d76..0000000000 --- a/travis/install_mesos.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -PACKAGE_CACHE_DIR=$HOME/.apt-cache -DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]') -CODENAME=$(lsb_release -cs) - -if [ -d "$PACKAGE_CACHE_DIR" ] && [ -n "$(find $PACKAGE_CACHE_DIR -name 'mesos_*.deb')" ]; then - echo 'Using cached Mesos library...' - cp -f $PACKAGE_CACHE_DIR/*.deb /var/cache/apt/archives/ -else - echo 'Downloading Mesos library...' - apt-key adv --keyserver keyserver.ubuntu.com --recv E56151BF - echo "deb http://repos.mesosphere.io/${DISTRO} ${CODENAME} main" | sudo tee /etc/apt/sources.list.d/mesosphere.list - apt-get update -qq - apt-get install mesos -y --download-only - mkdir -p $PACKAGE_CACHE_DIR/ - cp -f /var/cache/apt/archives/*.deb $PACKAGE_CACHE_DIR/ -fi - -set -x - -apt-get install --allow-downgrades --fix-broken --no-download --yes $PACKAGE_CACHE_DIR/*.deb -APT_EXIT_CODE=$? - -if [ $APT_EXIT_CODE -ne 0 ] || ! [ -f $MESOS_NATIVE_JAVA_LIBRARY ]; then - echo 'Mesos installation error!' - exit $APT_EXIT_CODE -fi diff --git a/travis/minimesos b/travis/minimesos deleted file mode 100755 index f609359197..0000000000 --- a/travis/minimesos +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash - -set -e - -MINIMESOS_TAG="0.14.20180710" -PARAMS="--debug $@" -MINIMESOS_CLI_IMAGE="twosigma/minimesos-cli" - -command_exists() { - command -v "$@" > /dev/null 2>&1 -} - -DOCKER_VERSION=$(docker version --format "{{.Server.Version}}") -SMALLEST_VERSION=$(printf "%s\n1.11.0\n" $DOCKER_VERSION | sort -t '.' -k 1,1 -k 2,2 -k 3,3 -k 4,4 -g | head -n 1) - -if ! command_exists docker || [ $SMALLEST_VERSION != "1.11.0" ]; then - echo "Minimesos requires Docker 1.11.0 or higher" - exit 1 -fi - -if [ "$DOCKER_HOST" != "" ] && [[ $DOCKER_HOST == tcp* ]]; then - DOCKER_HOST_IP=$(echo "$DOCKER_HOST" | grep -o '[0-9]\+[.][0-9]\+[.][0-9]\+[.][0-9]\+') -elif command_exists docker-machine && [ "$DOCKER_MACHINE_NAME" != "" ]; then - DOCKER_HOST_IP=$(docker-machine ip ${DOCKER_MACHINE_NAME}) -elif [ $(uname) != "Darwin" ]; then - DOCKER_HOST_IP=$(ip addr show dev docker0 | grep inet | sed -r "s/.*inet\s([0-9\.]+)\/.*/\1/" | head -n 1) -else - DOCKER_HOST_IP="" -fi - -pullImage() { - if [ "$(docker images $1 | grep $2 2> /dev/null)" = "" ]; then - echo "Pulling $1:$2" - docker pull "$1:$2" - fi -} - -if [ "$#" -gt 0 -a "$1" = up ]; then - pullImage ${MINIMESOS_CLI_IMAGE} ${MINIMESOS_TAG} -fi - -if [ $(uname) == "Darwin" ]; then - MINIMESOS_OS="Mac OS X" -else - MINIMESOS_OS="Linux" -fi - -MINIMESOS_HOST_DIR="$(pwd)" -MINIMESOS_DIR="$(pwd)/.minimesos" -if [ ! -d "${MINIMESOS_DIR}" ]; then - mkdir -p "${MINIMESOS_DIR}" - echo "# Created minimesos directory at ${MINIMESOS_DIR}." -fi - -docker run --rm -v "${MINIMESOS_HOST_DIR}":"${MINIMESOS_HOST_DIR}" \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /sys/fs/cgroup:/sys/fs/cgroup \ - -i \ - --env DOCKER_HOST_IP=${DOCKER_HOST_IP} \ - --env MINIMESOS_OS="${MINIMESOS_OS}" \ - --entrypoint java \ - ${MINIMESOS_CLI_IMAGE}:${MINIMESOS_TAG} \ - -Dminimesos.host.dir="${MINIMESOS_HOST_DIR}" \ - -jar /usr/local/share/minimesos/minimesos-cli.jar ${PARAMS} diff --git a/travis/minimesosFile b/travis/minimesosFile deleted file mode 100644 index 0a71cfb5dc..0000000000 --- a/travis/minimesosFile +++ /dev/null @@ -1,205 +0,0 @@ -minimesos { - clusterName = "Cook Simulation Cluster" - loggingLevel = "INFO" - mapAgentSandboxVolume = true - mapPortsToHost = false - mesosVersion = "1.0.0" - timeout = 60 - - agent { - imageName = "containersol/mesos-agent" - imageTag = "1.0.0-0.1.0" - loggingLevel = "# INHERIT FROM CLUSTER" - portNumber = 5051 - attributes = "cook-pool:mesos-gamma" - - resources { - cpu { - role = "*" - value = 4 - } - cpu { - role = "cook" - value = 4 - } - - disk { - role = "*" - value = 10000 - } - disk { - role = "cook" - value = 10000 - } - - mem { - role = "*" - value = 4096 - } - mem { - role = "cook" - value = 4096 - } - - ports { - role = "*" - value = "[31000-32000]" - } - ports { - role = "cook" - value = "[32000-33000]" - } - } - } - - agent { - imageName = "containersol/mesos-agent" - imageTag = "1.0.0-0.1.0" - loggingLevel = "# INHERIT FROM CLUSTER" - portNumber = 5051 - attributes = "cook-pool:mesos-gamma" - - resources { - cpu { - role = "*" - value = 4 - } - cpu { - role = "cook" - value = 4 - } - - disk { - role = "*" - value = 10000 - } - disk { - role = "cook" - value = 10000 - } - - mem { - role = "*" - value = 4096 - } - mem { - role = "cook" - value = 4096 - } - - ports { - role = "*" - value = "[31000-32000]" - } - ports { - role = "cook" - value = "[32000-33000]" - } - } - } - - agent { - imageName = "containersol/mesos-agent" - imageTag = "1.0.0-0.1.0" - loggingLevel = "# INHERIT FROM CLUSTER" - portNumber = 5051 - attributes = "cook-pool:mesos-alpha" - - resources { - cpu { - role = "*" - value = 4 - } - cpu { - role = "cook" - value = 4 - } - - disk { - role = "*" - value = 10000 - } - disk { - role = "cook" - value = 10000 - } - - mem { - role = "*" - value = 4096 - } - mem { - role = "cook" - value = 4096 - } - - ports { - role = "*" - value = "[31000-32000]" - } - ports { - role = "cook" - value = "[32000-33000]" - } - } - } - - agent { - imageName = "containersol/mesos-agent" - imageTag = "1.0.0-0.1.0" - loggingLevel = "# INHERIT FROM CLUSTER" - portNumber = 5051 - attributes = "cook-pool:mesos-alpha" - - resources { - cpu { - role = "*" - value = 4 - } - cpu { - role = "cook" - value = 4 - } - - disk { - role = "*" - value = 10000 - } - disk { - role = "cook" - value = 10000 - } - - mem { - role = "*" - value = 4096 - } - mem { - role = "cook" - value = 4096 - } - - ports { - role = "*" - value = "[31000-32000]" - } - ports { - role = "cook" - value = "[32000-33000]" - } - } - } - - master { - aclJson = null - authenticate = false - imageName = "containersol/mesos-master" - imageTag = "1.0.0-0.1.0" - loggingLevel = "# INHERIT FROM CLUSTER" - } - - zookeeper { - imageName = "jplock/zookeeper" - imageTag = "3.4.6" - } -} \ No newline at end of file diff --git a/travis/prepare.sh b/travis/prepare.sh deleted file mode 100755 index c7bab11dee..0000000000 --- a/travis/prepare.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -set -ev - -cd ${GITHUB_WORKSPACE}/scheduler -lein deps -lein uberjar -VERSION=$(lein print :version | tr -d '"') - -cd ${GITHUB_WORKSPACE}/travis -unzip ${GITHUB_WORKSPACE}/scheduler/datomic/datomic-free-0.9.5394.zip -cp "${GITHUB_WORKSPACE}/scheduler/target/cook-${VERSION}.jar" datomic-free-0.9.5394/lib/ diff --git a/travis/show_executor_logs.sh b/travis/show_executor_logs.sh deleted file mode 100755 index 083368373c..0000000000 --- a/travis/show_executor_logs.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -set -v - -echo "Printing out all executor logs..." -while read path; do - echo "Contents of ${path}"; - cat "${path}"; - echo "------------------------------------" -done <<< "$(find ${GITHUB_WORKSPACE}/travis/.minimesos -name 'stdout' -o -name 'stderr' -o -name 'executor.log')" - -${GITHUB_WORKSPACE}/travis/show_scheduler_logs.sh diff --git a/travis/show_scheduler_logs.sh b/travis/show_scheduler_logs.sh deleted file mode 100755 index baa491659b..0000000000 --- a/travis/show_scheduler_logs.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -for log in ${GITHUB_WORKSPACE}/scheduler/log/cook*.log; -do - echo "Contents of ${log}" - cat "${log}"; - echo "------------------------------------" -done diff --git a/travis/start_scheduler.sh b/travis/start_scheduler.sh deleted file mode 100755 index a80ca90b90..0000000000 --- a/travis/start_scheduler.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -ev - -cd ${GITHUB_WORKSPACE}/travis - -./build_cook_executor.sh -./datomic-free-0.9.5394/bin/transactor ${GITHUB_WORKSPACE}/scheduler/datomic/datomic_transactor.properties & -./minimesos up - -cd ${GITHUB_WORKSPACE}/scheduler -# on travis, ports on 172.17.0.1 are bindable from the host OS, and are also -# available for processes inside minimesos containers to connect to -LIBPROCESS_IP=172.17.0.1 lein run ${PROJECT_DIR}/travis/scheduler_config.edn & diff --git a/travis/upload_logs.sh b/travis/upload_logs.sh deleted file mode 100755 index e5be21d851..0000000000 --- a/travis/upload_logs.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash - -set -e - -cd ${GITHUB_WORKSPACE} - -# Create dump name -repo=${GITHUB_REPOSITORY} -pr_number=$(jq -r ".pull_request.number" "$GITHUB_EVENT_PATH") -dump_name="${repo//\//-}-PR${pr_number}-${GITHUB_WORKFLOW// /-}-$GITHUB_RUN_ID" - -# List the last 10 containers -docker ps --all --last 10 - -# Grab the Mesos master logs -mkdir -p ./mesos/master-logs -mesos_master_container=$(docker ps --all --latest --filter "name=minimesos-master-" --format "{{.ID}}") -docker cp --follow-link $mesos_master_container:/var/log/mesos-master.INFO ./mesos/master-logs/ -docker cp --follow-link $mesos_master_container:/var/log/mesos-master.WARNING ./mesos/master-logs/ - -# Grab the Mesos agent logs -mesos_agent_containers=$(docker ps --all --last 6 --filter "name=minimesos-agent-" --format "{{.ID}}") -for container in $mesos_agent_containers; -do - destination=./mesos/agent-logs/$container - mkdir -p $destination - docker cp --follow-link $container:/var/log/mesos-slave.INFO $destination - docker cp --follow-link $container:/var/log/mesos-slave.WARNING $destination - docker cp --follow-link $container:/var/log/mesos-slave.ERROR $destination - docker cp --follow-link $container:/var/log/mesos-fetcher.INFO $destination || echo "Container $container does not have mesos-fetcher.INFO" -done - -tarball=./dump.txz -tar -cJf $tarball --transform="s|\./[^/]*/\.*|${dump_name}/|" --warning=no-file-changed ./scheduler/log ./travis/.minimesos ./mesos/master-logs ./mesos/agent-logs || exitcode=$? -# GNU tar always exits with 0, 1 or 2 (https://www.gnu.org/software/tar/manual/html_section/tar_19.html) -# 0 = Successful termination -# 1 = Some files differ (we're OK with this) -# 2 = Fatal error -if [ "$exitcode" == "2" ]; then - echo "The tar command exited with exit code $exitcode, exiting..." - exit $exitcode -fi -./travis/gdrive_upload "travis-${dump_name}" $tarball