From 329d49fcd5d1a22cf449ac755e29ebdf84ac5dc5 Mon Sep 17 00:00:00 2001 From: Brandon Grohmann <140200346+brandongrohmann@users.noreply.github.com> Date: Fri, 3 Nov 2023 13:59:05 -0700 Subject: [PATCH] [ci] Updates ci based on the vault changes (#223) This includes the following PRs from vault: * genesis-community/vault-genesis-kit#28 * genesis-community/vault-genesis-kit#29 * genesis-community/vault-genesis-kit#30 * genesis-community/vault-genesis-kit#31 * genesis-community/vault-genesis-kit#32 --- ci/pipeline/resources/build.yml | 12 ++-- ci/pipeline/resources/github.yml | 2 +- ci/pipeline/resources/spec-check.yml | 12 ++-- ci/pipeline/resources/version.yml | 16 +++-- ci/repipe | 8 +-- ci/scripts/build-kit | 4 +- ci/scripts/build-upstream-jobs | 98 ++++++++++++++++++++++++++++ ci/scripts/compare-release-specs | 9 ++- ci/scripts/smoketests | 1 + ci/scripts/test-addons | 2 + ci/scripts/test-deployment | 53 +++++++-------- 11 files changed, 163 insertions(+), 54 deletions(-) create mode 100755 ci/scripts/build-upstream-jobs create mode 100644 ci/scripts/smoketests create mode 100644 ci/scripts/test-addons diff --git a/ci/pipeline/resources/build.yml b/ci/pipeline/resources/build.yml index 4c206928..8817eae2 100644 --- a/ci/pipeline/resources/build.yml +++ b/ci/pipeline/resources/build.yml @@ -2,8 +2,10 @@ resources: - name: build type: s3 source: - bucket: (( grab meta.aws.bucket )) - region_name: (( grab meta.aws.region_name )) - regexp: (( concat meta.name "/build/(.*)\.tar.gz" )) - access_key_id: (( grab meta.aws.access_key )) - secret_access_key: (( grab meta.aws.secret_key )) + endpoint: (( grab meta.aws.endpoint || ~ )) + skip_ssl_verification: (( grab meta.aws.insecure || false )) + access_key_id: (( grab meta.aws.access_key )) + secret_access_key: (( grab meta.aws.secret_key )) + region_name: (( grab meta.aws.region_name )) + bucket: (( grab meta.aws.bucket )) + regexp: (( concat meta.name "/build/(.*)\.tar.gz" )) diff --git a/ci/pipeline/resources/github.yml b/ci/pipeline/resources/github.yml index ab5c11e7..2feeee95 100644 --- a/ci/pipeline/resources/github.yml +++ b/ci/pipeline/resources/github.yml @@ -2,6 +2,6 @@ resources: - name: github type: github-release source: - user: (( grab meta.github.owner )) + owner: (( grab meta.github.owner )) repository: (( grab meta.github.repo )) access_token: (( grab meta.github.access_token )) diff --git a/ci/pipeline/resources/spec-check.yml b/ci/pipeline/resources/spec-check.yml index 1704c611..45b34dd1 100644 --- a/ci/pipeline/resources/spec-check.yml +++ b/ci/pipeline/resources/spec-check.yml @@ -2,8 +2,10 @@ resources: - name: spec-check type: s3 source: - bucket: (( grab meta.aws.bucket )) - region_name: (( grab meta.aws.region_name )) - regexp: (( concat meta.name "/spec-check/diff-(.*)" )) - access_key_id: (( grab meta.aws.access_key )) - secret_access_key: (( grab meta.aws.secret_key )) + endpoint: (( grab meta.aws.endpoint || ~ )) + skip_ssl_verification: (( grab meta.aws.insecure || false )) + access_key_id: (( grab meta.aws.access_key )) + secret_access_key: (( grab meta.aws.secret_key )) + bucket: (( grab meta.aws.bucket )) + region_name: (( grab meta.aws.region_name )) + regexp: :(( concat meta.name "/spec-check/diff-(.*)" )) diff --git a/ci/pipeline/resources/version.yml b/ci/pipeline/resources/version.yml index f0a136fd..76a9634f 100644 --- a/ci/pipeline/resources/version.yml +++ b/ci/pipeline/resources/version.yml @@ -2,10 +2,12 @@ resources: - name: version type: semver source : - driver: s3 - bucket: (( grab meta.aws.bucket )) - region_name: (( grab meta.aws.region_name )) - key: (( concat meta.name "/version" )) - access_key_id: (( grab meta.aws.access_key )) - secret_access_key: (( grab meta.aws.secret_key )) - initial_version: (( grab meta.initial_version || "0.0.1" )) + driver: s3 + endpoint: (( grab meta.aws.endpoint || ~ )) + skip_ssl_verification: (( grab meta.aws.insecure || false )) + access_key_id: (( grab meta.aws.access_key )) + secret_access_key: (( grab meta.aws.secret_key )) + bucket: (( grab meta.aws.bucket )) + region_name: (( grab meta.aws.region_name )) + key: (( concat meta.name "/version" )) + initial_version: (( grab meta.initial_version || "0.0.1" )) diff --git a/ci/repipe b/ci/repipe index 8f5c034e..777dbffe 100755 --- a/ci/repipe +++ b/ci/repipe @@ -16,7 +16,7 @@ need_command() { local cmd=${1:?need_command() - no command name given} local url=${2:-} - if [[ ! -x "$(type -p "$cmd")" ]]; then + if [[ ! -x "$(type -P "$cmd")" ]]; then echo >&2 "${cmd} is not installed." if [[ -n "$url" ]]; then echo >&2 "Please download it from ${url}" @@ -96,7 +96,7 @@ OPTIONS: whatever is set in 'meta.exposed' in the settings.yml file) -o Open pipeline in browser if os supports it (mac only currently) after applying changes. Specify twice to not do anything else. - --fly Path to fly command, otherwise will use $(type -p fly) + --fly Path to fly command, otherwise will use $(type -P fly) EOF exit $rc @@ -167,7 +167,7 @@ need_command jq if [[ -z "$fly" ]] ; then need_command fly; - fly="$(type -p fly)" + fly="$(type -P fly)" fi # -- Get settings file -------------------------------------------------------- @@ -220,7 +220,7 @@ if (( DRYRUN > 0 )) ; then if [[ -n "${persistent_file}" ]] ; then if [[ "${persistent_file}" =~ '->0x' ]] ; then persistent_file='' - elif [[ -n "$(type -p realpath || true)" ]] ; then + elif [[ -n "$(type -P realpath || true)" ]] ; then persistent_file="$(realpath --relative-to="$call_dir" "$persistent_file")" fi fi diff --git a/ci/scripts/build-kit b/ci/scripts/build-kit index 1bdea3ab..be68f86d 100755 --- a/ci/scripts/build-kit +++ b/ci/scripts/build-kit @@ -40,8 +40,8 @@ if [[ ${#check_dirs[@]} -gt 0 ]] ; then header "Checking SHA1s of specified components (not including bosh-deployment) ..." out="$(eval "spruce merge --skip-eval $( \ grep -rl '^releases:' "${check_dirs[@]}" \ - | sed -e "s/\\(.*\\)/<(spruce json \\1 | jq -r '{releases: .releases}')/" |tr "\n" " " \ - ) | spruce json | jq -r ." )" + | sed -e "s/\\(.*\\)/<(spruce json \\1 | jq -r '{releases: [ \"(( merge on sha1 ))\", .releases[] ]}')/" |tr "\n" " " \ + ) | spruce json | jq -r ." )" echo "$out" | spruce merge | spruce json | "${CI_ROOT}/ci/scripts/check-sha1s" fi diff --git a/ci/scripts/build-upstream-jobs b/ci/scripts/build-upstream-jobs new file mode 100755 index 00000000..b8666152 --- /dev/null +++ b/ci/scripts/build-upstream-jobs @@ -0,0 +1,98 @@ +#!/bin/bash - +set -ue +base_dir="$(cd "$(dirname "$0")/.." && pwd)" +mkdir -p "${base_dir}/pipeline/upstream/" + +_lookup() { + echo "${1}" | base64 --decode | jq -r "${2}" +} + +# Read upstream.yml +update_group=() +upstream_details="$(spruce json "${base_dir}/settings.yml" | jq -r '.meta.upstream.bosh_releases//[] | .[] | @base64')" + +# For each release in upstream.yml, +for release in $upstream_details ; do + name="$(_lookup "$release" .name)" + type="$(_lookup "$release" '.type//"bosh-io-release"')" + path="$(_lookup "$release" '.path//"manifests/releases/'"$name"'.yml"')" + repo="$(_lookup "$release" '.repository')" + if [[ $type == 'bosh-io-release' ]] ; then + source=$'\n'" repository: $repo"; + elif [[ $type == 'github-release' ]] ; then + owner="$(_lookup "$release" '.owner//""')" + if [[ -z "$owner" && "$repo" =~ / ]] ; then + owner="${repo%%/*}" + repo="${repo#*/}" + fi + source=$'\n'" repository: $repo"$'\n'" owner: $owner"; + + token="$(_lookup "$release" '.access_token//""')" + if [[ -n "$token" ]] ; then + source="$source"$'\n access_token: "'"$token"'"' + fi + else + echo >&2 "Unknown resource type for $name upstream release: $type" + echo >&2 "Expecting one of: bosh-io-release, github-release" + echo >&2 "Update upstream.bosh-releases configuration in ci/settings.yml" + exit 1 + fi + job="update-${name}-release" + release="${name}-release" + + update_group+=( "$job" ) + + cat <> "$base_dir/pipeline/upstream/update_${name}_release.yml" +jobs: +- (( append )) +- name: $job + public: false + serial: true + serial_groups: [upstream-releases] + plan: + - do: + - in_parallel: + - { get: git, trigger: false, passed: [spec-tests] } + - { get: git-ci, trigger: false } + - get: $release + trigger: true + params: + tarball: false + - task: $job + file: git-ci/ci/tasks/update-release.yml + input_mapping: {bosh-release: $release} + params: + RELEASE_NAME: $name + RELEASE_PATH: $path + BRANCH: (( grab meta.github.branch )) + GIT_EMAIL: (( grab meta.git.email )) + GIT_NAME: (( grab meta.git.name )) + - put: git + params: + merge: true + repository: git + +resources: + - (( append )) + - name: $release + type: $type + check_every: 24h + source: $source +EOF + +done +group_file="$base_dir/pipeline/upstream/update_group.yml" +if [[ "${#update_group[@]}" -gt 0 ]] ; then + ( + echo "groups:" + echo "- (( merge on name ))" + echo "- name: upstream" + echo " jobs:" + echo " - (( append ))" + for job in ${update_group[@]+"${update_group[@]}"} ; do + echo " - $job" + done + ) >> "$group_file" +elif [[ -f "$group_file" ]] ; then + rm -f "$group_file" +fi diff --git a/ci/scripts/compare-release-specs b/ci/scripts/compare-release-specs index 6fe7f366..b8976514 100755 --- a/ci/scripts/compare-release-specs +++ b/ci/scripts/compare-release-specs @@ -3,7 +3,7 @@ set -ue # What branch is the comparison of the current working branch being compared against compare_branch="${1:-origin/}" -check_dirs="spec manifests manifests/releases" +check_dirs="spec/results manifests" orig_dir="$(pwd)" # needed because of when running locally or in ci @@ -31,6 +31,11 @@ workdir="$(mktemp -d)" mkdir "$workdir/compare" cp -R "$(pwd)/.git" "$workdir/compare/" pushd "$workdir/compare" > /dev/null +if ! git show -q "$compare_branch" &>/dev/null ; then + echo "The latest release ($compare_branch) is NOT an ancestor to this commit." + echo "This should never happen -- Cannot continue!" + exit 1 +fi git checkout -qf --detach "$compare_branch" prev_releases="$(releases)" popd > /dev/null @@ -113,7 +118,7 @@ if [[ "${#changed[@]}" -gt 0 && -n "${changed[0]}" ]] ; then echo "$curr_releases" \ | jq --argjson gitrepos "$upstreamrepo" -r 'reduce .releases[] as {$name, $url, $sha1, $version} ({repos: []}; ($url - | if ($url | test("https?://s3.amazonaws.com")) then + | if ($url | test("https?://s3(-.*)?.amazonaws.com")) then ($gitrepos.repos | map(select(.name == $name))[0].repo) elif ($url | test("https?://storage.googleapis.com")) then ($gitrepos.repos | map(select(.name == $name))[0].repo) diff --git a/ci/scripts/smoketests b/ci/scripts/smoketests new file mode 100644 index 00000000..0c39aa27 --- /dev/null +++ b/ci/scripts/smoketests @@ -0,0 +1 @@ + genesis "do" "${DEPLOY_ENV}" -- smoketest diff --git a/ci/scripts/test-addons b/ci/scripts/test-addons new file mode 100644 index 00000000..0e1f7b14 --- /dev/null +++ b/ci/scripts/test-addons @@ -0,0 +1,2 @@ + genesis "do" "${DEPLOY_ENV}" -- setup-cli -f + genesis "do" "${DEPLOY_ENV}" -- login diff --git a/ci/scripts/test-deployment b/ci/scripts/test-deployment index fe815f47..413e5a3f 100755 --- a/ci/scripts/test-deployment +++ b/ci/scripts/test-deployment @@ -17,7 +17,7 @@ header() { } has_feature() { - genesis lookup "$1" kit.features 2>/dev/null | jq -e --arg feature "$2" '. | index($feature)' >/dev/null + genesis "$1" lookup kit.features 2>/dev/null | jq -e --arg feature "$2" '. | index($feature)' >/dev/null } is_proto() { @@ -29,9 +29,9 @@ cleanup_environment() { if [[ -f .genesis/manifests/$env-state.yml ]] ; then header "Preparing to delete proto environment $env" echo "Generating reference manifest..." - genesis manifest "$env" --no-redact > manifest.yml 2>/dev/null + genesis "$env" manifest --no-redact > manifest.yml 2>/dev/null echo $'\n'"Building BOSH variables file..." - genesis lookup --merged "${env}" bosh-variables > vars.yml 2>/dev/null + genesis "${env}" lookup --merged bosh-variables > vars.yml 2>/dev/null echo $'\n'"$env state file:" echo "----------------->8------------------" cat ".genesis/manifests/$env-state.yml" @@ -68,16 +68,8 @@ cleanup() { done } -# Replace this with genesis lookup --env once its available (2.8.5) -cat <<'EOF' > get-env.pl -#!/usr/bin/env perl -use lib "$ENV{HOME}/.geese/lib"; -use JSON::PP qw/encode_json/; -print encode_json(\%ENV); -EOF -chmod +x "get-env.pl" -vault_path="$(genesis sh "${DEPLOY_ENV}" -s "$(pwd)/get-env.pl" 2>/dev/null | grep '^{' | jq -r '.GENESIS_SECRETS_BASE')" -exodus_path="$(genesis sh "${DEPLOY_ENV}" -s "$(pwd)/get-env.pl" 2>/dev/null | grep '^{' | jq -r '.GENESIS_EXODUS_BASE')" +vault_path="$(genesis "$DEPLOY_ENV" lookup --env GENESIS_SECRETS_BASE)" +exodus_path="$(genesis "$DEPLOY_ENV" lookup --env GENESIS_EXODUS_BASE)" vault_path="${vault_path%/}" # trim any trailing slash # ----- @@ -105,10 +97,10 @@ if [[ "$SKIP_REPLACE_SECRETS" == "false" ]] ; then # Remove credhub values if ! is_proto "$DEPLOY_ENV" ; then ( - bosh_env="$(genesis lookup "$DEPLOY_ENV" genesis 2>/dev/null | jq -r '.bosh_env // .env')" + bosh_env="$(genesis "$DEPLOY_ENV" lookup genesis 2>/dev/null | jq -r '.bosh_env // .env')" [[ "$bosh_env" =~ / ]] || bosh_env="${bosh_env}/bosh" - bosh_exodus="$(genesis lookup --exodus-for "$bosh_env" "$DEPLOY_ENV" . "{}" 2>/dev/null)" + bosh_exodus="$(genesis "$DEPLOY_ENV" lookup --exodus-for "$bosh_env" . "{}" 2>/dev/null)" CREDHUB_SERVER="$(jq -r '.credhub_url // ""' <<<"$bosh_exodus")" if [[ -n "$CREDHUB_SERVER" ]] ; then echo @@ -157,8 +149,8 @@ fi if [[ "$SKIP_DEPLOY" == "false" ]]; then header "Deploying ${DEPLOY_ENV} environment to verify functionality..." - genesis "do" "${DEPLOY_ENV}" -- list - genesis add-secrets "${DEPLOY_ENV}" + genesis "${DEPLOY_ENV}" "do" -- list + genesis "${DEPLOY_ENV}" add-secrets # get and upload stemcell version if needed (handled by bosh cli if version and name are supplied) stemcell_iaas= @@ -173,7 +165,7 @@ if [[ "$SKIP_DEPLOY" == "false" ]]; then esac if [[ -n "$stemcell_iaas" ]] ; then - stemcell_data="$(genesis lookup --merged "${DEPLOY_ENV}" stemcells)" + stemcell_data="$(genesis "${DEPLOY_ENV}" lookup --merged stemcells)" stemcell_os="$(jq -r '.[0].os' <<<"$stemcell_data")" stemcell_version="$(jq -r '.[0].version' <<<"$stemcell_data")" stemcell_name="bosh-${stemcell_iaas}-${stemcell_os}-go_agent" @@ -184,15 +176,15 @@ if [[ "$SKIP_DEPLOY" == "false" ]]; then upload_options=() upload_params="" fi - if ! genesis bosh "$DEPLOY_ENV" stemcells 2>/dev/null \ + if ! genesis "${DEPLOY_ENV}" bosh stemcells 2>/dev/null \ | grep "^${stemcell_name}" \ | awk '{print $2}' | sed -e 's/\*//' \ | grep "^${stemcell_version}\$" ; then - genesis bosh "$DEPLOY_ENV" upload-stemcell "https://bosh.io/d/stemcells/$stemcell_name${upload_params}" ${upload_options[@]+"${upload_options[@]}"} + genesis "${DEPLOY_ENV}" bosh upload-stemcell "https://bosh.io/d/stemcells/$stemcell_name${upload_params}" ${upload_options[@]+"${upload_options[@]}"} fi fi - genesis deploy -y "${DEPLOY_ENV}" + genesis "${DEPLOY_ENV}" deploy -y if [[ -f .genesis/manifests/${DEPLOY_ENV}-state.yml ]] ; then echo $'\n'"${DEPLOY_ENV} state file:" @@ -201,20 +193,25 @@ if [[ "$SKIP_DEPLOY" == "false" ]]; then echo "----------------->8------------------" fi - genesis info "${DEPLOY_ENV}" + genesis "${DEPLOY_ENV}" info if ! is_proto "$DEPLOY_ENV" ; then - genesis bosh "${DEPLOY_ENV}" instances --ps + genesis "${DEPLOY_ENV}" bosh instances --ps fi fi if [[ "$SKIP_SMOKE_TESTS" == "false" ]]; then - header "Validating addons..." - genesis "do" "${DEPLOY_ENV}" -- setup-cli -f - genesis "do" "${DEPLOY_ENV}" -- login + if [[ -f "$0/test-addons" ]] ; then + header "Validating addons..." + # shellcheck source=/dev/null + source "$0/test-addons" + fi - header "Running smoke tests..." - genesis "do" "${DEPLOY_ENV}" -- smoketest + if [[ -f "$0/smoketests" ]] ; then + header "Running smoke tests..." + # shellcheck source=/dev/null + source "$0/smoketests" + fi else echo "Skipping smoke_tests" fi