diff --git a/.gitignore b/.gitignore index e2d18c3..7ff4b33 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ cluster/.vagrant cluster/export cluster/*.log release +artifact_staging/ diff --git a/Makefile b/Makefile index 3e4f3ef..82a79bf 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,45 @@ +# backwards compatibility name for CONTIV_INSTALLER_VERSION +export BUILD_VERSION ?= devbuild +# sets the version for the installer output artifacts +export CONTIV_INSTALLER_VERSION ?= $(BUILD_VERSION) +# downloaded and built assets intended to go in installer by build.sh +export CONTIV_ARTIFACT_STAGING := $(PWD)/artifact_staging +# some assets are retrieved from GitHub, this is the default version to fetch +export DEFAULT_DOWNLOAD_CONTIV_VERSION := 1.1.5 +export NETPLUGIN_OWNER ?= contiv +# setting NETPLUGIN_BRANCH compiles that commit on demand, +# setting CONTIV_NETPLUGIN_VERSION will download that released version +ifeq ($(NETPLUGIN_BRANCH),) +export CONTIV_NETPLUGIN_VERSION ?= $(DEFAULT_DOWNLOAD_CONTIV_VERSION) +else +export CONTIV_NETPLUGIN_VERSION := $(NETPLUGIN_OWNER)-$(NETPLUGIN_BRANCH) +endif +export CONTIV_V2PLUGIN_VERSION ?= $(DEFAULT_DOWNLOAD_CONTIV_VERSION) +export CONTIV_NETPLUGIN_TARBALL_NAME := netplugin-$(CONTIV_NETPLUGIN_VERSION).tar.bz2 +export CONTIV_ANSIBLE_COMMIT ?= 4e67f54a8042debfc3d8b504046d0a1d4ea38c37 +export CONTIV_ANSIBLE_OWNER ?= contiv + # this is the classic first makefile target, and it's also the default target # run when `make` is invoked with no specific target. all: build rel_ver = $(shell ./scripts/get_latest_release.sh) +# accepts CONTIV_ANSIBLE_COMMIT and CONTIV_ANSIBLE_OWNER environment vars +download-ansible-repo: + @scripts/download_ansible_repo.sh + +# set NETPLUGIN_OWNER (default contiv) and NETPLUGIN_BRANCH make variables +# to compile locally +# e.g. make NETPLUGIN_OWNER=contiv NETPLUGIN_BRANCH=master +prepare-netplugin-tarball: + @scripts/prepare_netplugin_tarball.sh + +assemble-build: + @bash ./scripts/build.sh + # build creates a release package for contiv. # It uses a pre-built image specified by BUILD_VERSION. -build: - rm -rf release/ - @bash ./scripts/build.sh +build: download-ansible-repo prepare-netplugin-tarball assemble-build # ansible-image creates the docker image for ansible container # It uses the version specified by BUILD_VERSION or creates an image with the latest tag. diff --git a/scripts/build.sh b/scripts/build.sh index 588c578..57a2545 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -1,5 +1,10 @@ #!/bin/bash +# Required environment variables: +# * CONTIV_INSTALLER_VERSION - sets the tarball artifact filenames +# * CONTIV_NETPLUGIN_VERSION - updates config files to locate contiv tarball +# * CONTIV_V2PLUGIN_VERSION - which v2plugin version to download during install + set -xeuo pipefail # ensure this script wasn't called from the directory where this script @@ -10,106 +15,44 @@ if [ "$script_dir" == "." ]; then exit 1 fi -DEV_IMAGE_NAME="devbuild" -VERSION=${BUILD_VERSION-$DEV_IMAGE_NAME} - -contiv_version=${CONTIV_VERSION:-"1.0.3"} pull_images=${CONTIV_CI_HOST:-"false"} aci_gw_version=${CONTIV_ACI_GW_VERSION:-"latest"} -ansible_image_version=${CONTIV_ANSIBLE_IMAGE_VERSION:-$contiv_version} -auth_proxy_version=${CONTIV_API_PROXY_VERSION:-$contiv_version} +ansible_image_version=${CONTIV_ANSIBLE_IMAGE_VERSION:-$DEFAULT_DOWNLOAD_CONTIV_VERSION} +auth_proxy_version=${CONTIV_API_PROXY_VERSION:-$DEFAULT_DOWNLOAD_CONTIV_VERSION} docker_version=${CONTIV_DOCKER_VERSION:-1.12.6} etcd_version=${CONTIV_ETCD_VERSION:-v2.3.8} -contiv_ansible_commit=${CONTIV_ANSIBLE_COMMIT:-4e67f54a8042debfc3d8b504046d0a1d4ea38c37} -contiv_ansible_owner=${CONTIV_ANSIBLE_OWNER:-contiv} - -# the installer currently pulls the v2plugin image directly from Docker Hub, but -# this will change to being downloaded from the Docker Store in the future. -# because of this, the default value for this variable will become the latest -# version that is available in the Docker Store and should be considered -# independent of $contiv_version above. -v2plugin_version=${CONTIV_V2PLUGIN_VERSION:-"1.0.3"} - -function usage() { - echo "Usage:" - echo "./release.sh -a -c -e -p " - exit 1 -} +v2plugin_version=${CONTIV_V2PLUGIN_VERSION} -function error_ret() { - echo "" - echo $1 - exit 1 -} - -while getopts ":a:p:c:e:v:" opt; do - case $opt in - a) - aci_gw_version=$OPTARG - ;; - c) - contiv_version=$OPTARG - ;; - e) - etcd_version=$OPTARG - ;; - p) - auth_proxy_version=$OPTARG - ;; - v) - v2plugin_version=$OPTARG - ;; - :) - echo "An argument required for $OPTARG was not passed" - usage - ;; - ?) - usage - ;; - esac -done - -release_dir="release" -output_dir="$release_dir/contiv-$VERSION/" -output_file="$release_dir/contiv-$VERSION.tgz" -tmp_output_file="contiv-$VERSION.tgz" -full_output_file="$release_dir/contiv-full-$VERSION.tgz" -tmp_full_output_file="contiv-full-$VERSION.tgz" - -# Clean older dist folders and release binaries -rm -rf $output_dir -rm -rf $output_file +# where everything is assembled, always start with a clean dir and clean it up +output_tmp_dir="$(mktemp -d)" +output_dir="${output_tmp_dir}/contiv-${CONTIV_INSTALLER_VERSION}" +mkdir -p ${output_dir} +trap 'rm -rf ${output_tmp_dir}' EXIT + +release_dir=release +mkdir -p $release_dir +output_file="${release_dir}/contiv-${CONTIV_INSTALLER_VERSION}.tgz" +full_output_file="$release_dir/contiv-full-${CONTIV_INSTALLER_VERSION}.tgz" # Release files # k8s - install.sh to take the args and construct contiv.yaml as required and to launch kubectl # swarm - install.sh launches the container to do the actual installation # Top level install.sh which will either take k8s/swarm install params and do the required. -mkdir -p $output_dir -cp -rf install $output_dir -cp README.md $output_dir +cp -rf install README.md $output_dir cp -rf scripts/generate-certificate.sh $output_dir/install # Get the ansible support files chmod +x $output_dir/install/genInventoryFile.py chmod +x $output_dir/install/generate-certificate.sh -# This is maybe optional - but assume we need it for -curl -sSL https://github.com/contiv/netplugin/releases/download/$contiv_version/netplugin-$contiv_version.tar.bz2 -o $output_dir/netplugin-$contiv_version.tar.bz2 -pushd $output_dir -tar oxf netplugin-$contiv_version.tar.bz2 netctl -rm -f netplugin-$contiv_version.tar.bz2 -popd -# add ansible repo contents where final tarball will include -mkdir $output_dir/ansible -curl -sL https://api.github.com/repos/${contiv_ansible_owner}/ansible/tarball/$contiv_ansible_commit | - tar --strip-components 1 -C $output_dir/ansible -z -x +cp -a ${CONTIV_ARTIFACT_STAGING}/ansible ${output_dir}/ # Replace versions files=$(find $output_dir -type f -name "*.yaml" -or -name "*.sh" -or -name "*.json") sed -i.bak 's/__ACI_GW_VERSION__/'"$aci_gw_version"'/g' $files sed -i.bak 's/__API_PROXY_VERSION__/'"$auth_proxy_version"'/g' $files sed -i.bak 's/__CONTIV_INSTALL_VERSION__/'"$ansible_image_version"'/g' $files -sed -i.bak 's/__CONTIV_VERSION__/'"$contiv_version"'/g' $files +sed -i.bak 's/__CONTIV_VERSION__/'"$CONTIV_NETPLUGIN_VERSION"'/g' $files sed -i.bak 's/__DOCKER_VERSION__/'"$docker_version"'/g' $files sed -i.bak 's/__ETCD_VERSION__/'"$etcd_version"'/g' $files sed -i.bak 's/__CONTIV_V2PLUGIN_VERSION__/'"$v2plugin_version"'/g' $files @@ -117,10 +60,6 @@ sed -i.bak 's/__CONTIV_V2PLUGIN_VERSION__/'"$v2plugin_version"'/g' $files # Make all shell script files executable chmod +x $(find $output_dir -type f -name "*.sh") -# Cleanup the backup files -rm -rf $output_dir/scripts -rm -rf $(find $output_dir -type f -name "*.bak") - # Clean up the Dockerfile, it is not part of the release bits. rm -f $output_dir/install/ansible/Dockerfile @@ -128,8 +67,15 @@ rm -f $output_dir/install/ansible/Dockerfile binary_cache=$output_dir/contiv_cache mkdir -p $binary_cache -# Create the minimal tar bundle -tar czf $tmp_output_file -C $release_dir contiv-$VERSION +# only build installer that pulls artifacts over internet if not building +# a specific commit of netplugin +if [ -z "${NETPLUGIN_BRANCH:-}" ]; then + # Create the minimal tar bundle + tar czf $output_file -C $output_tmp_dir contiv-${CONTIV_INSTALLER_VERSION} + echo -n "Contiv Installer version '$CONTIV_INSTALLER_VERSION' with " + echo "netplugin version '$CONTIV_NETPLUGIN_VERSION' is available " + echo "at '$output_file'" +fi # Save the auth proxy & aci-gw images for packaging the full docker images with contiv install binaries if [[ "$(docker images -q contiv/auth_proxy:$auth_proxy_version 2>/dev/null)" == "" || "$pull_images" == "true" ]]; then @@ -147,6 +93,18 @@ curl --fail -sL -o $binary_cache/openvswitch-2.5.0-2.el7.x86_64.rpm http://cbs.c curl --fail -sL -o $binary_cache/ovs-common.deb http://mirrors.kernel.org/ubuntu/pool/main/o/openvswitch/openvswitch-common_2.5.2-0ubuntu0.16.04.3_amd64.deb curl --fail -sL -o $binary_cache/ovs-switch.deb http://mirrors.kernel.org/ubuntu/pool/main/o/openvswitch/openvswitch-switch_2.5.2-0ubuntu0.16.04.3_amd64.deb +# Copy the netplugin release into the binary cache for "full" installer +# Netplugin releases built locally based on a branch are named by their SHA, +# but there is a symlink to point to the SHA named tarball by it's branch name +plugin_tball=${CONTIV_ARTIFACT_STAGING}/$CONTIV_NETPLUGIN_TARBALL_NAME +if [[ -L "${plugin_tball}" ]]; then + # copy the link (so other processes can find the tarball) and the tarball + target_plugin_tball=$(readlink ${plugin_tball}) + cp -a ${plugin_tball} ${binary_cache}/ + plugin_tball=${CONTIV_ARTIFACT_STAGING}/${target_plugin_tball} +fi +cp ${plugin_tball} ${binary_cache}/ + env_file=$output_dir/install/ansible/env.json sed -i.bak 's#__AUTH_PROXY_LOCAL_INSTALL__#true#g' "$env_file" sed -i.bak 's#__CONTIV_NETWORK_LOCAL_INSTALL__#true#g' "$env_file" @@ -154,10 +112,10 @@ sed -i.bak 's#__CONTIV_NETWORK_LOCAL_INSTALL__#true#g' "$env_file" echo "Ansible extra vars from env.json:" cat $env_file # Create the full tar bundle -tar czf $tmp_full_output_file -C $release_dir contiv-$VERSION - -mv $tmp_output_file $output_file -mv $tmp_full_output_file $full_output_file -rm -rf $output_dir - -echo "Success: Contiv Installer version $VERSION is available at $output_file" +tar czf $full_output_file -C $output_tmp_dir contiv-${CONTIV_INSTALLER_VERSION} +echo -n "Contiv Installer version '$CONTIV_INSTALLER_VERSION' with " +echo "netplugin version '$CONTIV_NETPLUGIN_VERSION' is available " +echo "at '$full_output_file', it includes all contiv assets " +echo "required for installation" +echo +echo -e "\nSuccess" diff --git a/scripts/download_ansible_repo.sh b/scripts/download_ansible_repo.sh new file mode 100755 index 0000000..11eae15 --- /dev/null +++ b/scripts/download_ansible_repo.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -euo pipefail + +ANSIBLE_REPO_DIR=${CONTIV_ARTIFACT_STAGING}/ansible + +rm -rf $ANSIBLE_REPO_DIR + +mkdir -p $ANSIBLE_REPO_DIR $CONTIV_ARTIFACT_STAGING + +echo downloading ${CONTIV_ANSIBLE_OWNER}/ansible commit: $CONTIV_ANSIBLE_COMMIT +curl --fail -sL https://api.github.com/repos/${CONTIV_ANSIBLE_OWNER}/ansible/tarball/$CONTIV_ANSIBLE_COMMIT \ + | tar --strip-components 1 -C $ANSIBLE_REPO_DIR -z -x diff --git a/scripts/legacy_swarm_test.sh b/scripts/legacy_swarm_test.sh index c08697c..1625ebe 100644 --- a/scripts/legacy_swarm_test.sh +++ b/scripts/legacy_swarm_test.sh @@ -24,16 +24,12 @@ if [ "$ssh_key" == "" ]; then ssh_key=$(vagrant ssh-config legacy-swarm-master | grep IdentityFile | awk '{print $2}' | xargs) fi popd + +./scripts/unpack-installer.sh + # Extract and launch the installer -mkdir -p release -cd release -if [ ! -f "${install_version}.tgz" ]; then - # For release builds, get the build from github releases - curl -L -O https://github.com/contiv/install/releases/download/${BUILD_VERSION}/${install_version}.tgz -fi -tar oxf $install_version.tgz -cd $install_version +cd release/$install_version ./install/ansible/install_swarm.sh -f ../../cluster/.cfg_legacy-swarm.yaml -e $ssh_key -u $user -i # Wait for CONTIV to start for up to 10 minutes @@ -46,7 +42,7 @@ for i in {0..20}; do cat <