[develop] Travis CI + Buildkite 3.0

.cicd/build.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+set -eo pipefail
+. ./.cicd/helpers/general.sh
+mkdir -p $BUILD_DIR
+CMAKE_EXTRAS="-DBUILD_MONGO_DB_PLUGIN=true"
+if [[ $(uname) == 'Darwin' ]]; then
+    # You can't use chained commands in execute
+    [[ $TRAVIS == true ]] && export PINNED=false && ccache -s && CMAKE_EXTRAS="-DCMAKE_CXX_COMPILER_LAUNCHER=ccache" && ./$CICD_DIR/platforms/macos-10.14.sh
+    ( [[ ! $PINNED == false || $UNPINNED == true ]] ) && CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_TOOLCHAIN_FILE=$HELPERS_DIR/clang.make"
+    cd $BUILD_DIR
+    cmake $CMAKE_EXTRAS ..
+    make -j$JOBS
+else # Linux
+    ARGS=${ARGS:-"--rm --init -v $(pwd):$MOUNTED_DIR"}
+    . $HELPERS_DIR/file-hash.sh $CICD_DIR/platforms/$IMAGE_TAG.dockerfile
+    PRE_COMMANDS="cd $MOUNTED_DIR/build"
+    # PRE_COMMANDS: Executed pre-cmake
+    # CMAKE_EXTRAS: Executed within and right before the cmake path (cmake CMAKE_EXTRAS ..)
+    [[ ! $IMAGE_TAG =~ 'unpinned' ]] && CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_TOOLCHAIN_FILE=$MOUNTED_DIR/.cicd/helpers/clang.make -DCMAKE_CXX_COMPILER_LAUNCHER=ccache"
+    if [[ $IMAGE_TAG == 'amazon_linux-2' ]]; then
+        PRE_COMMANDS="$PRE_COMMANDS && export PATH=/usr/lib64/ccache:\\\$PATH"
+    elif [[ $IMAGE_TAG == 'centos-7.6' ]]; then
+        PRE_COMMANDS="$PRE_COMMANDS && export PATH=/usr/lib64/ccache:\\\$PATH"
+    elif [[ $IMAGE_TAG == 'ubuntu-16.04' ]]; then
+        PRE_COMMANDS="$PRE_COMMANDS && export PATH=/usr/lib/ccache:\\\$PATH"
+    elif [[ $IMAGE_TAG == 'ubuntu-18.04' ]]; then
+        PRE_COMMANDS="$PRE_COMMANDS && export PATH=/usr/lib/ccache:\\\$PATH"
+    elif [[ $IMAGE_TAG == 'amazon_linux-2-unpinned' ]]; then
+        PRE_COMMANDS="$PRE_COMMANDS && export PATH=/usr/lib64/ccache:\\\$PATH"
+        CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_CXX_COMPILER='clang++' -DCMAKE_C_COMPILER='clang'"
+    elif [[ $IMAGE_TAG == 'centos-7.6-unpinned' ]]; then
+        PRE_COMMANDS="$PRE_COMMANDS && source /opt/rh/devtoolset-8/enable && source /opt/rh/rh-python36/enable && export PATH=/usr/lib64/ccache:\\\$PATH"
+    elif [[ $IMAGE_TAG == 'ubuntu-18.04-unpinned' ]]; then
+        PRE_COMMANDS="$PRE_COMMANDS && export PATH=/usr/lib/ccache:\\\$PATH"
+        CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_CXX_COMPILER='clang++' -DCMAKE_C_COMPILER='clang'"
+    fi
+    BUILD_COMMANDS="cmake $CMAKE_EXTRAS -DCMAKE_BUILD_TYPE='Release' .. && make -j$JOBS"
+    # Docker Commands
+    if [[ $BUILDKITE == true ]]; then
+        # Generate Base Images
+        $CICD_DIR/generate-base-images.sh
+        [[ $ENABLE_INSTALL == true ]] && COMMANDS="cp -r $MOUNTED_DIR /root/eosio && cd /root/eosio/build &&"
+        COMMANDS="$COMMANDS $BUILD_COMMANDS"
+        [[ $ENABLE_INSTALL == true ]] && COMMANDS="$COMMANDS && make install"
+    elif [[ $TRAVIS == true ]]; then
+        ARGS="$ARGS -v /usr/lib/ccache -v $HOME/.ccache:/opt/.ccache -e JOBS -e TRAVIS -e CCACHE_DIR=/opt/.ccache"
+        COMMANDS="ccache -s && $BUILD_COMMANDS"
+    fi
+    COMMANDS="$PRE_COMMANDS && $COMMANDS"
+    echo "$ docker run $ARGS $(buildkite-intrinsics) $FULL_TAG bash -c \"$COMMANDS\""
+    eval docker run $ARGS $(buildkite-intrinsics) $FULL_TAG bash -c \"$COMMANDS\"
+fi

.cicd/generate-base-images.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -eo pipefail
+. ./.cicd/helpers/general.sh
+. $HELPERS_DIR/file-hash.sh $CICD_DIR/platforms/${IMAGE_TAG}.dockerfile
+# look for Docker image
+echo "+++ :mag_right: Looking for $FULL_TAG"
+ORG_REPO=$(echo $FULL_TAG | cut -d: -f1)
+TAG=$(echo $FULL_TAG | cut -d: -f2)
+EXISTS=$(curl -s -H "Authorization: Bearer $(curl -sSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${ORG_REPO}:pull" | jq --raw-output .token)" "https://registry.hub.docker.com/v2/${ORG_REPO}/manifests/$TAG")
+# build, if neccessary
+if [[ $EXISTS =~ '404 page not found' || $EXISTS =~ 'manifest unknown' ]]; then # if we cannot pull the image, we build and push it first
+    docker build -t $FULL_TAG -f $CICD_DIR/platforms/${IMAGE_TAG}.dockerfile .
+    docker push $FULL_TAG
+else
+    echo "$FULL_TAG already exists."
+fi

.cicd/helpers/clang.make

@@ -0,0 +1,10 @@
+set(CMAKE_C_COMPILER_WORKS 1)
+set(CMAKE_CXX_COMPILER_WORKS 1)
+set(CMAKE_C_COMPILER /usr/local/bin/clang)
+set(CMAKE_CXX_COMPILER /usr/local/bin/clang++)
+set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES /usr/local/include/c++/v1 /usr/local/include /usr/include)
+set(CMAKE_CXX_FLAGS_INIT "-nostdinc++")
+set(CMAKE_EXE_LINKER_FLAGS_INIT "-stdlib=libc++ -nostdlib++")
+set(CMAKE_SHARED_LINKER_FLAGS_INIT "-stdlib=libc++ -nostdlib++")
+set(CMAKE_MODULE_LINKER_FLAGS_INIT "-stdlib=libc++ -nostdlib++")
+set(CMAKE_CXX_STANDARD_LIBRARIES "/usr/local/lib/libc++.a /usr/local/lib/libc++abi.a")

.cicd/helpers/file-hash.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -eo pipefail
+[[ -z $1 ]] && echo "Please provide the file to be hashed" && exit 1
+FILE_NAME=$(basename $1 | awk '{split($0,a,/\.(d|s)/); print a[1] }')
+export DETERMINED_HASH=$(sha1sum $1 | awk '{ print $1 }')
+( [[ $FILE_NAME =~ 'macos' ]] && [[ $PINNED == false || $UNPINNED == true ]] ) && FILE_NAME="${FILE_NAME}-unpinned"
+export HASHED_IMAGE_TAG="eos-${FILE_NAME}-${DETERMINED_HASH}"
+export FULL_TAG="eosio/producer:$HASHED_IMAGE_TAG"

.cicd/helpers/general.sh

@@ -0,0 +1,28 @@
+export ROOT_DIR=$( dirname "${BASH_SOURCE[0]}" )/../..
+export BUILD_DIR=$ROOT_DIR/build
+export CICD_DIR=$ROOT_DIR/.cicd
+export HELPERS_DIR=$CICD_DIR/helpers
+export JOBS=${JOBS:-"$(getconf _NPROCESSORS_ONLN)"}
+export MOUNTED_DIR='/workdir'
+
+# capitalize each word in a string
+function capitalize()
+{
+    if [[ ! $1 =~ 'mac' ]]; then # Don't capitalize mac
+        echo $1 | awk '{$1=toupper(substr($1,1,1))substr($1,2)}1'
+    else
+        echo $1
+    fi
+}
+
+# load buildkite intrinsic environment variables for use in docker run
+function buildkite-intrinsics()
+{
+    BK_ENV=''
+    if [[ -f $BUILDKITE_ENV_FILE ]]; then
+        while read -r var; do
+            BK_ENV="$BK_ENV --env ${var%%=*}"
+        done < "$BUILDKITE_ENV_FILE"
+    fi
+    echo "$BK_ENV"
+}

.cicd/installation-build.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -eo pipefail
+. ./.cicd/helpers/general.sh
+export ENABLE_INSTALL=true
+if [[ $BUILDKITE_BRANCH =~ ^release/[0-9].[0-9]+.x$ || $BUILDKITE_BRANCH =~ ^master$ || $BUILDKITE_BRANCH =~ ^develop$ || $FORCE_BINARIES_BUILD == true ]]; then
+    export BRANCH=$(echo $BUILDKITE_BRANCH | sed 's/\//\_/')
+    export CONTRACTS_BUILDER_TAG="eosio/ci-contracts-builder:base-ubuntu-18.04"
+    export ARGS="--name ci-contracts-builder-$BUILDKITE_COMMIT --init -v $(pwd):$MOUNTED_DIR"
+    $CICD_DIR/build.sh
+    docker commit ci-contracts-builder-$BUILDKITE_COMMIT $CONTRACTS_BUILDER_TAG-latest
+    docker commit ci-contracts-builder-$BUILDKITE_COMMIT $CONTRACTS_BUILDER_TAG-$BUILDKITE_COMMIT
+    docker commit ci-contracts-builder-$BUILDKITE_COMMIT $CONTRACTS_BUILDER_TAG-$BRANCH
+    docker commit ci-contracts-builder-$BUILDKITE_COMMIT $CONTRACTS_BUILDER_TAG-$BRANCH-$BUILDKITE_COMMIT
+    docker push $CONTRACTS_BUILDER_TAG-latest
+    docker push $CONTRACTS_BUILDER_TAG-$BUILDKITE_COMMIT
+    docker push $CONTRACTS_BUILDER_TAG-$BRANCH
+    docker push $CONTRACTS_BUILDER_TAG-$BRANCH-$BUILDKITE_COMMIT
+    docker stop ci-contracts-builder-$BUILDKITE_COMMIT && docker rm ci-contracts-builder-$BUILDKITE_COMMIT
+else
+    echo "This pipeline will only generate images against master, develop, and release branches. Set FORCE_BINARIES_BUILD=true to force a build on another branch. Exiting..."
+    exit 0
+fi