From 84fdfef4b603b8ca91ed9a10f7eed3f1c8a3bc55 Mon Sep 17 00:00:00 2001 From: Brandon Grohmann <140200346+brandongrohmann@users.noreply.github.com> Date: Fri, 3 Nov 2023 14:02:42 -0700 Subject: [PATCH] [ci] Updates based on the vault changes (#26) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit removes cat file as it's not need This includes the following PRs from vault: • genesis-community/vault-genesis-kit#28 • genesis-community/vault-genesis-kit#29 • genesis-community/vault-genesis-kit#30 • genesis-community/vault-genesis-kit#31 • genesis-community/vault-genesis-kit#32 --- ci/pipeline/resources/build.yml | 12 +-- ci/pipeline/resources/github.yml | 2 +- ci/pipeline/resources/spec-check.yml | 12 +-- ci/pipeline/resources/version.yml | 16 ++-- ci/repipe | 8 +- ci/scripts/build-kit | 4 +- ci/scripts/build-upstream-jobs | 29 +++++-- ci/scripts/cats | 114 --------------------------- ci/scripts/compare-release-specs | 9 ++- ci/settings.yml | 1 + 10 files changed, 59 insertions(+), 148 deletions(-) delete mode 100755 ci/scripts/cats diff --git a/ci/pipeline/resources/build.yml b/ci/pipeline/resources/build.yml index 4c20692..8817eae 100644 --- a/ci/pipeline/resources/build.yml +++ b/ci/pipeline/resources/build.yml @@ -2,8 +2,10 @@ resources: - name: build type: s3 source: - bucket: (( grab meta.aws.bucket )) - region_name: (( grab meta.aws.region_name )) - regexp: (( concat meta.name "/build/(.*)\.tar.gz" )) - access_key_id: (( grab meta.aws.access_key )) - secret_access_key: (( grab meta.aws.secret_key )) + endpoint: (( grab meta.aws.endpoint || ~ )) + skip_ssl_verification: (( grab meta.aws.insecure || false )) + access_key_id: (( grab meta.aws.access_key )) + secret_access_key: (( grab meta.aws.secret_key )) + region_name: (( grab meta.aws.region_name )) + bucket: (( grab meta.aws.bucket )) + regexp: (( concat meta.name "/build/(.*)\.tar.gz" )) diff --git a/ci/pipeline/resources/github.yml b/ci/pipeline/resources/github.yml index ab5c11e..2feeee9 100644 --- a/ci/pipeline/resources/github.yml +++ b/ci/pipeline/resources/github.yml @@ -2,6 +2,6 @@ resources: - name: github type: github-release source: - user: (( grab meta.github.owner )) + owner: (( grab meta.github.owner )) repository: (( grab meta.github.repo )) access_token: (( grab meta.github.access_token )) diff --git a/ci/pipeline/resources/spec-check.yml b/ci/pipeline/resources/spec-check.yml index 1704c61..c000546 100644 --- a/ci/pipeline/resources/spec-check.yml +++ b/ci/pipeline/resources/spec-check.yml @@ -2,8 +2,10 @@ resources: - name: spec-check type: s3 source: - bucket: (( grab meta.aws.bucket )) - region_name: (( grab meta.aws.region_name )) - regexp: (( concat meta.name "/spec-check/diff-(.*)" )) - access_key_id: (( grab meta.aws.access_key )) - secret_access_key: (( grab meta.aws.secret_key )) + endpoint: (( grab meta.aws.endpoint || ~ )) + skip_ssl_verification: (( grab meta.aws.insecure || false )) + access_key_id: (( grab meta.aws.access_key )) + secret_access_key: (( grab meta.aws.secret_key )) + bucket: (( grab meta.aws.bucket )) + region_name: (( grab meta.aws.region_name )) + regexp: (( concat meta.name "/spec-check/diff-(.*)" )) diff --git a/ci/pipeline/resources/version.yml b/ci/pipeline/resources/version.yml index f0a136f..76a9634 100644 --- a/ci/pipeline/resources/version.yml +++ b/ci/pipeline/resources/version.yml @@ -2,10 +2,12 @@ resources: - name: version type: semver source : - driver: s3 - bucket: (( grab meta.aws.bucket )) - region_name: (( grab meta.aws.region_name )) - key: (( concat meta.name "/version" )) - access_key_id: (( grab meta.aws.access_key )) - secret_access_key: (( grab meta.aws.secret_key )) - initial_version: (( grab meta.initial_version || "0.0.1" )) + driver: s3 + endpoint: (( grab meta.aws.endpoint || ~ )) + skip_ssl_verification: (( grab meta.aws.insecure || false )) + access_key_id: (( grab meta.aws.access_key )) + secret_access_key: (( grab meta.aws.secret_key )) + bucket: (( grab meta.aws.bucket )) + region_name: (( grab meta.aws.region_name )) + key: (( concat meta.name "/version" )) + initial_version: (( grab meta.initial_version || "0.0.1" )) diff --git a/ci/repipe b/ci/repipe index 8f5c034..777dbff 100755 --- a/ci/repipe +++ b/ci/repipe @@ -16,7 +16,7 @@ need_command() { local cmd=${1:?need_command() - no command name given} local url=${2:-} - if [[ ! -x "$(type -p "$cmd")" ]]; then + if [[ ! -x "$(type -P "$cmd")" ]]; then echo >&2 "${cmd} is not installed." if [[ -n "$url" ]]; then echo >&2 "Please download it from ${url}" @@ -96,7 +96,7 @@ OPTIONS: whatever is set in 'meta.exposed' in the settings.yml file) -o Open pipeline in browser if os supports it (mac only currently) after applying changes. Specify twice to not do anything else. - --fly Path to fly command, otherwise will use $(type -p fly) + --fly Path to fly command, otherwise will use $(type -P fly) EOF exit $rc @@ -167,7 +167,7 @@ need_command jq if [[ -z "$fly" ]] ; then need_command fly; - fly="$(type -p fly)" + fly="$(type -P fly)" fi # -- Get settings file -------------------------------------------------------- @@ -220,7 +220,7 @@ if (( DRYRUN > 0 )) ; then if [[ -n "${persistent_file}" ]] ; then if [[ "${persistent_file}" =~ '->0x' ]] ; then persistent_file='' - elif [[ -n "$(type -p realpath || true)" ]] ; then + elif [[ -n "$(type -P realpath || true)" ]] ; then persistent_file="$(realpath --relative-to="$call_dir" "$persistent_file")" fi fi diff --git a/ci/scripts/build-kit b/ci/scripts/build-kit index 1bdea3a..be68f86 100755 --- a/ci/scripts/build-kit +++ b/ci/scripts/build-kit @@ -40,8 +40,8 @@ if [[ ${#check_dirs[@]} -gt 0 ]] ; then header "Checking SHA1s of specified components (not including bosh-deployment) ..." out="$(eval "spruce merge --skip-eval $( \ grep -rl '^releases:' "${check_dirs[@]}" \ - | sed -e "s/\\(.*\\)/<(spruce json \\1 | jq -r '{releases: .releases}')/" |tr "\n" " " \ - ) | spruce json | jq -r ." )" + | sed -e "s/\\(.*\\)/<(spruce json \\1 | jq -r '{releases: [ \"(( merge on sha1 ))\", .releases[] ]}')/" |tr "\n" " " \ + ) | spruce json | jq -r ." )" echo "$out" | spruce merge | spruce json | "${CI_ROOT}/ci/scripts/check-sha1s" fi diff --git a/ci/scripts/build-upstream-jobs b/ci/scripts/build-upstream-jobs index 31fcf68..b866615 100755 --- a/ci/scripts/build-upstream-jobs +++ b/ci/scripts/build-upstream-jobs @@ -14,15 +14,29 @@ upstream_details="$(spruce json "${base_dir}/settings.yml" | jq -r '.meta.upstre # For each release in upstream.yml, for release in $upstream_details ; do name="$(_lookup "$release" .name)" - type="$(_lookup "$release" '.type//"bosh-io-release"')" # Other valid value is github-release + type="$(_lookup "$release" '.type//"bosh-io-release"')" path="$(_lookup "$release" '.path//"manifests/releases/'"$name"'.yml"')" repo="$(_lookup "$release" '.repository')" - owner="" - if [[ $type == "github-release" ]] ; then - owner=$'\n'" owner: ${repo%/*}" - repo="${repo##*/}" - fi + if [[ $type == 'bosh-io-release' ]] ; then + source=$'\n'" repository: $repo"; + elif [[ $type == 'github-release' ]] ; then + owner="$(_lookup "$release" '.owner//""')" + if [[ -z "$owner" && "$repo" =~ / ]] ; then + owner="${repo%%/*}" + repo="${repo#*/}" + fi + source=$'\n'" repository: $repo"$'\n'" owner: $owner"; + token="$(_lookup "$release" '.access_token//""')" + if [[ -n "$token" ]] ; then + source="$source"$'\n access_token: "'"$token"'"' + fi + else + echo >&2 "Unknown resource type for $name upstream release: $type" + echo >&2 "Expecting one of: bosh-io-release, github-release" + echo >&2 "Update upstream.bosh-releases configuration in ci/settings.yml" + exit 1 + fi job="update-${name}-release" release="${name}-release" @@ -63,8 +77,7 @@ resources: - name: $release type: $type check_every: 24h - source: - repository: $repo$owner + source: $source EOF done diff --git a/ci/scripts/cats b/ci/scripts/cats deleted file mode 100755 index 8a9c5d2..0000000 --- a/ci/scripts/cats +++ /dev/null @@ -1,114 +0,0 @@ -#!/bin/bash -set -e - -header() { - echo - echo "================================================================================" - echo "$1" - echo "--------------------------------------------------------------------------------" - echo -} - -bail() { - echo >&2 "$* Did you misconfigure Concourse?" - exit 2 -} - -lookup() { - genesis -C "$WORK_DIR" lookup "$DEPLOY_ENV" "$@" 2>/dev/null -} - -run_cats() { - echo "Running CATS from deployment dir $2" - - base=$(lookup params.base_domain) - system_domain="$(lookup params.system_domain "system.$base")" - ADMIN_PASSWORD="$(credhub get -q -n "$credhub_path/cf_admin_password")" - export API_URL=api.$system_domain - export ADMIN_PASSWORD - export APP_DOMAINS_0=run.$base - case $RUN_CATS in - "include_container_networking") - sed -i "s#include_security_groups:.*#include_security_groups: true#g" "$CI_DIR/cats.yml" - ;; - "include_deployments") - sed -i "s#include_v3:.*#include_v3: true#g" "$CI_DIR/cats.yml" - ;; - "include_service_instance_sharing") - sed -i "s#include_services:.*#include_services: true#g" "$CI_DIR/cats.yml" - ;; - "include_sso") - sed -i "s#include_services:.*#include_services: true#g" "$CI_DIR/cats.yml" - ;; - "include_tasks") - sed -i "s#include_v3:.*#include_v3: true#g" "$CI_DIR/cats.yml" - ;; - "include_zipkin") - sed -i "s#include_routing:.*#include_routing: true#g" "$CI_DIR/cats.yml" - ;; - esac - - sed -i "s#${RUN_CATS}:.*#${RUN_CATS}: true#g" "$CI_DIR/cats.yml" - tmpdir="$ROOT_DIR/cats" - export CONFIG=${tmpdir}/config.json - spruce merge "$CI_DIR/cats.yml" | spruce json | jq '.' > "${CONFIG}" - pushd "${tmpdir}" &>/dev/null - FAILFAST_FLAG="" - VERBOSE_FLAG="" - if [[ ${FAILFAST} == "true" ]] ; then - FAILFAST_FLAG="--failFast" - fi - if [[ ${VERBOSE} == "true" ]] ; then - VERBOSE_FLAG="-v" - fi - echo "RUNNING CATS WITH NODES: ${CATS_NODES:-4} ${VERBOSE_FLAG} ${FAILFAST_FLAG}" - ./bin/test -nodes="${CATS_NODES:-4}" ${VERBOSE_FLAG} ${FAILFAST_FLAG} - popd &>/dev/null -} - -ROOT_DIR="$(pwd)" -WORK_DIR="${ROOT_DIR}/work/cf-deployments" -CI_DIR="${ROOT_DIR}/git-ci/ci" - -test -n "${DEPLOY_ENV:-}" || bail "DEPLOY_ENV must be set to the deployment environment name." -test -n "${KIT_SHORTNAME:-}" || bail "KIT_SHORTNAME must be set to the short name of this kit." - -# Attach to Vault -test -n "${VAULT_URI:-}" || bail "VAULT_URI must be set to the Vault connection address - suggest using ((vault.url))" -test -n "${VAULT_TOKEN:-}" || bail "VAULT_TOKEN must be set to a valid token to connect to the Vault - suggest using ((vault.token))" -safe target da-vault "$VAULT_URI" -k -echo "$VAULT_TOKEN" | safe auth token - -# Get BOSH/CREDHUB envs -bosh_env="$(lookup genesis 2>/dev/null | jq -r '.bosh_env // .env')" -[[ "$bosh_env" =~ / ]] || bosh_env="${bosh_env}/bosh" - -bosh_exodus="$(lookup --exodus-for "$bosh_env" . "{}" 2>/dev/null)" -BOSH_ENVIRONMENT="$( jq -r '.url // ""' <<<"$bosh_exodus")" -BOSH_CLIENT="$( jq -r '.admin_username // ""' <<<"$bosh_exodus")" -BOSH_CLIENT_SECRET="$(jq -r '.admin_password // ""' <<<"$bosh_exodus")" -BOSH_CA_CERT="$( jq -r '.ca_cert // ""' <<<"$bosh_exodus")" -export BOSH_ENVIRONMENT BOSH_CLIENT BOSH_CLIENT_SECRET BOSH_CA_CERT - -CREDHUB_SERVER="$( jq -r '.credhub_url // ""' <<<"$bosh_exodus")" -if [[ -n "$CREDHUB_SERVER" ]] ; then - echo - CREDHUB_CLIENT="$( jq -r '.credhub_username // ""' <<<"$bosh_exodus")" - CREDHUB_SECRET="$( jq -r '.credhub_password // ""' <<<"$bosh_exodus")" - CREDHUB_CA_CERT="$(jq -r '"\(.credhub_ca_cert)\(.ca_cert)"' <<<"$bosh_exodus")" - export CREDHUB_SERVER CREDHUB_CLIENT CREDHUB_SECRET CREDHUB_CA_CERT - credhub_path="/${bosh_env/\//-}/${DEPLOY_ENV}-${KIT_SHORTNAME}" - echo -fi - -test -n "${BOSH_ENVIRONMENT:-}" || bail "BOSH_ENVIRONMENT must be set to BOSH directory URL - missing from $bosh_env deployment exodus data" -test -n "${BOSH_CA_CERT:-}" || bail "BOSH_CA_CERT must be set to the BOSH CA Cert - missing from $bosh_env deployment exodus data" -test -n "${BOSH_CLIENT:-}" || bail "BOSH_CLIENT must be set to BOSH Client user - missing from $bosh_env deployment exodus data" -test -n "${BOSH_CLIENT_SECRET:-}" || bail "BOSH_CLIENT_SECRET must be set to BOSH Client secret - missing from $bosh_env deployment exodus data" -test -n "${CREDHUB_SERVER:-}" || bail "CREDHUB_SERVER must be set to Credhub URL - missing from $bosh_env deployment exodus data" -test -n "${CREDHUB_CLIENT:-}" || bail "CREDHUB_CLIENT must be set to Credhub username - missing from $bosh_env deployment exodus data" -test -n "${CREDHUB_SECRET:-}" || bail "CREDHUB_SECRET must be set to Credhub password - missing from $bosh_env deployment exodus data" -test -n "${CREDHUB_CA_CERT:-}" || bail "CREDHUB_CA_CERT must be set to Credhub ca_cert and UAA ca_cert password - missing from $bosh_env deployment exodus data" - -cd git -run_cats --deployment-dir "$ROOT_DIR" diff --git a/ci/scripts/compare-release-specs b/ci/scripts/compare-release-specs index 6fe7f36..b897651 100755 --- a/ci/scripts/compare-release-specs +++ b/ci/scripts/compare-release-specs @@ -3,7 +3,7 @@ set -ue # What branch is the comparison of the current working branch being compared against compare_branch="${1:-origin/}" -check_dirs="spec manifests manifests/releases" +check_dirs="spec/results manifests" orig_dir="$(pwd)" # needed because of when running locally or in ci @@ -31,6 +31,11 @@ workdir="$(mktemp -d)" mkdir "$workdir/compare" cp -R "$(pwd)/.git" "$workdir/compare/" pushd "$workdir/compare" > /dev/null +if ! git show -q "$compare_branch" &>/dev/null ; then + echo "The latest release ($compare_branch) is NOT an ancestor to this commit." + echo "This should never happen -- Cannot continue!" + exit 1 +fi git checkout -qf --detach "$compare_branch" prev_releases="$(releases)" popd > /dev/null @@ -113,7 +118,7 @@ if [[ "${#changed[@]}" -gt 0 && -n "${changed[0]}" ]] ; then echo "$curr_releases" \ | jq --argjson gitrepos "$upstreamrepo" -r 'reduce .releases[] as {$name, $url, $sha1, $version} ({repos: []}; ($url - | if ($url | test("https?://s3.amazonaws.com")) then + | if ($url | test("https?://s3(-.*)?.amazonaws.com")) then ($gitrepos.repos | map(select(.name == $name))[0].repo) elif ($url | test("https?://storage.googleapis.com")) then ($gitrepos.repos | map(select(.name == $name))[0].repo) diff --git a/ci/settings.yml b/ci/settings.yml index 4e64991..3c46e07 100644 --- a/ci/settings.yml +++ b/ci/settings.yml @@ -18,6 +18,7 @@ meta: type: github-release repository: cloudfoundry/app-autoscaler-release path: overlay/releases/app-autoscaler.yml + access_token: ((github.access_token)) bosh: uri: ((bosh.uri))