Skip to content

Commit

Permalink
[ci] Updates ci based on the vault changes (#223)
Browse files Browse the repository at this point in the history
  • Loading branch information
brandongrohmann authored Nov 3, 2023
1 parent 2049e10 commit 329d49f
Show file tree
Hide file tree
Showing 11 changed files with 163 additions and 54 deletions.
12 changes: 7 additions & 5 deletions ci/pipeline/resources/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@ resources:
- name: build
type: s3
source:
bucket: (( grab meta.aws.bucket ))
region_name: (( grab meta.aws.region_name ))
regexp: (( concat meta.name "/build/(.*)\.tar.gz" ))
access_key_id: (( grab meta.aws.access_key ))
secret_access_key: (( grab meta.aws.secret_key ))
endpoint: (( grab meta.aws.endpoint || ~ ))
skip_ssl_verification: (( grab meta.aws.insecure || false ))
access_key_id: (( grab meta.aws.access_key ))
secret_access_key: (( grab meta.aws.secret_key ))
region_name: (( grab meta.aws.region_name ))
bucket: (( grab meta.aws.bucket ))
regexp: (( concat meta.name "/build/(.*)\.tar.gz" ))
2 changes: 1 addition & 1 deletion ci/pipeline/resources/github.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@ resources:
- name: github
type: github-release
source:
user: (( grab meta.github.owner ))
owner: (( grab meta.github.owner ))
repository: (( grab meta.github.repo ))
access_token: (( grab meta.github.access_token ))
12 changes: 7 additions & 5 deletions ci/pipeline/resources/spec-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@ resources:
- name: spec-check
type: s3
source:
bucket: (( grab meta.aws.bucket ))
region_name: (( grab meta.aws.region_name ))
regexp: (( concat meta.name "/spec-check/diff-(.*)" ))
access_key_id: (( grab meta.aws.access_key ))
secret_access_key: (( grab meta.aws.secret_key ))
endpoint: (( grab meta.aws.endpoint || ~ ))
skip_ssl_verification: (( grab meta.aws.insecure || false ))
access_key_id: (( grab meta.aws.access_key ))
secret_access_key: (( grab meta.aws.secret_key ))
bucket: (( grab meta.aws.bucket ))
region_name: (( grab meta.aws.region_name ))
regexp: :(( concat meta.name "/spec-check/diff-(.*)" ))
16 changes: 9 additions & 7 deletions ci/pipeline/resources/version.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@ resources:
- name: version
type: semver
source :
driver: s3
bucket: (( grab meta.aws.bucket ))
region_name: (( grab meta.aws.region_name ))
key: (( concat meta.name "/version" ))
access_key_id: (( grab meta.aws.access_key ))
secret_access_key: (( grab meta.aws.secret_key ))
initial_version: (( grab meta.initial_version || "0.0.1" ))
driver: s3
endpoint: (( grab meta.aws.endpoint || ~ ))
skip_ssl_verification: (( grab meta.aws.insecure || false ))
access_key_id: (( grab meta.aws.access_key ))
secret_access_key: (( grab meta.aws.secret_key ))
bucket: (( grab meta.aws.bucket ))
region_name: (( grab meta.aws.region_name ))
key: (( concat meta.name "/version" ))
initial_version: (( grab meta.initial_version || "0.0.1" ))
8 changes: 4 additions & 4 deletions ci/repipe
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ need_command() {
local cmd=${1:?need_command() - no command name given}
local url=${2:-}

if [[ ! -x "$(type -p "$cmd")" ]]; then
if [[ ! -x "$(type -P "$cmd")" ]]; then
echo >&2 "${cmd} is not installed."
if [[ -n "$url" ]]; then
echo >&2 "Please download it from ${url}"
Expand Down Expand Up @@ -96,7 +96,7 @@ OPTIONS:
whatever is set in 'meta.exposed' in the settings.yml file)
-o Open pipeline in browser if os supports it (mac only currently)
after applying changes. Specify twice to not do anything else.
--fly <x> Path to fly command, otherwise will use $(type -p fly)
--fly <x> Path to fly command, otherwise will use $(type -P fly)
EOF
exit $rc
Expand Down Expand Up @@ -167,7 +167,7 @@ need_command jq

if [[ -z "$fly" ]] ; then
need_command fly;
fly="$(type -p fly)"
fly="$(type -P fly)"
fi

# -- Get settings file --------------------------------------------------------
Expand Down Expand Up @@ -220,7 +220,7 @@ if (( DRYRUN > 0 )) ; then
if [[ -n "${persistent_file}" ]] ; then
if [[ "${persistent_file}" =~ '->0x' ]] ; then
persistent_file=''
elif [[ -n "$(type -p realpath || true)" ]] ; then
elif [[ -n "$(type -P realpath || true)" ]] ; then
persistent_file="$(realpath --relative-to="$call_dir" "$persistent_file")"
fi
fi
Expand Down
4 changes: 2 additions & 2 deletions ci/scripts/build-kit
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,8 @@ if [[ ${#check_dirs[@]} -gt 0 ]] ; then
header "Checking SHA1s of specified components (not including bosh-deployment) ..."
out="$(eval "spruce merge --skip-eval $( \
grep -rl '^releases:' "${check_dirs[@]}" \
| sed -e "s/\\(.*\\)/<(spruce json \\1 | jq -r '{releases: .releases}')/" |tr "\n" " " \
) | spruce json | jq -r ." )"
| sed -e "s/\\(.*\\)/<(spruce json \\1 | jq -r '{releases: [ \"(( merge on sha1 ))\", .releases[] ]}')/" |tr "\n" " " \
) | spruce json | jq -r ." )"
echo "$out" | spruce merge | spruce json | "${CI_ROOT}/ci/scripts/check-sha1s"
fi

Expand Down
98 changes: 98 additions & 0 deletions ci/scripts/build-upstream-jobs
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
#!/bin/bash -
set -ue
base_dir="$(cd "$(dirname "$0")/.." && pwd)"
mkdir -p "${base_dir}/pipeline/upstream/"

_lookup() {
echo "${1}" | base64 --decode | jq -r "${2}"
}

# Read upstream.yml
update_group=()
upstream_details="$(spruce json "${base_dir}/settings.yml" | jq -r '.meta.upstream.bosh_releases//[] | .[] | @base64')"

# For each release in upstream.yml,
for release in $upstream_details ; do
name="$(_lookup "$release" .name)"
type="$(_lookup "$release" '.type//"bosh-io-release"')"
path="$(_lookup "$release" '.path//"manifests/releases/'"$name"'.yml"')"
repo="$(_lookup "$release" '.repository')"
if [[ $type == 'bosh-io-release' ]] ; then
source=$'\n'" repository: $repo";
elif [[ $type == 'github-release' ]] ; then
owner="$(_lookup "$release" '.owner//""')"
if [[ -z "$owner" && "$repo" =~ / ]] ; then
owner="${repo%%/*}"
repo="${repo#*/}"
fi
source=$'\n'" repository: $repo"$'\n'" owner: $owner";

token="$(_lookup "$release" '.access_token//""')"
if [[ -n "$token" ]] ; then
source="$source"$'\n access_token: "'"$token"'"'
fi
else
echo >&2 "Unknown resource type for $name upstream release: $type"
echo >&2 "Expecting one of: bosh-io-release, github-release"
echo >&2 "Update upstream.bosh-releases configuration in ci/settings.yml"
exit 1
fi
job="update-${name}-release"
release="${name}-release"

update_group+=( "$job" )

cat <<EOF >> "$base_dir/pipeline/upstream/update_${name}_release.yml"
jobs:
- (( append ))
- name: $job
public: false
serial: true
serial_groups: [upstream-releases]
plan:
- do:
- in_parallel:
- { get: git, trigger: false, passed: [spec-tests] }
- { get: git-ci, trigger: false }
- get: $release
trigger: true
params:
tarball: false
- task: $job
file: git-ci/ci/tasks/update-release.yml
input_mapping: {bosh-release: $release}
params:
RELEASE_NAME: $name
RELEASE_PATH: $path
BRANCH: (( grab meta.github.branch ))
GIT_EMAIL: (( grab meta.git.email ))
GIT_NAME: (( grab meta.git.name ))
- put: git
params:
merge: true
repository: git
resources:
- (( append ))
- name: $release
type: $type
check_every: 24h
source: $source
EOF

done
group_file="$base_dir/pipeline/upstream/update_group.yml"
if [[ "${#update_group[@]}" -gt 0 ]] ; then
(
echo "groups:"
echo "- (( merge on name ))"
echo "- name: upstream"
echo " jobs:"
echo " - (( append ))"
for job in ${update_group[@]+"${update_group[@]}"} ; do
echo " - $job"
done
) >> "$group_file"
elif [[ -f "$group_file" ]] ; then
rm -f "$group_file"
fi
9 changes: 7 additions & 2 deletions ci/scripts/compare-release-specs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ set -ue

# What branch is the comparison of the current working branch being compared against
compare_branch="${1:-origin/}"
check_dirs="spec manifests manifests/releases"
check_dirs="spec/results manifests"

orig_dir="$(pwd)"
# needed because of when running locally or in ci
Expand Down Expand Up @@ -31,6 +31,11 @@ workdir="$(mktemp -d)"
mkdir "$workdir/compare"
cp -R "$(pwd)/.git" "$workdir/compare/"
pushd "$workdir/compare" > /dev/null
if ! git show -q "$compare_branch" &>/dev/null ; then
echo "The latest release ($compare_branch) is NOT an ancestor to this commit."
echo "This should never happen -- Cannot continue!"
exit 1
fi
git checkout -qf --detach "$compare_branch"
prev_releases="$(releases)"
popd > /dev/null
Expand Down Expand Up @@ -113,7 +118,7 @@ if [[ "${#changed[@]}" -gt 0 && -n "${changed[0]}" ]] ; then
echo "$curr_releases" \
| jq --argjson gitrepos "$upstreamrepo" -r 'reduce .releases[] as {$name, $url, $sha1, $version} ({repos: []};
($url
| if ($url | test("https?://s3.amazonaws.com")) then
| if ($url | test("https?://s3(-.*)?.amazonaws.com")) then
($gitrepos.repos | map(select(.name == $name))[0].repo)
elif ($url | test("https?://storage.googleapis.com")) then
($gitrepos.repos | map(select(.name == $name))[0].repo)
Expand Down
1 change: 1 addition & 0 deletions ci/scripts/smoketests
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
genesis "do" "${DEPLOY_ENV}" -- smoketest
2 changes: 2 additions & 0 deletions ci/scripts/test-addons
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
genesis "do" "${DEPLOY_ENV}" -- setup-cli -f
genesis "do" "${DEPLOY_ENV}" -- login
53 changes: 25 additions & 28 deletions ci/scripts/test-deployment
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ header() {
}

has_feature() {
genesis lookup "$1" kit.features 2>/dev/null | jq -e --arg feature "$2" '. | index($feature)' >/dev/null
genesis "$1" lookup kit.features 2>/dev/null | jq -e --arg feature "$2" '. | index($feature)' >/dev/null
}

is_proto() {
Expand All @@ -29,9 +29,9 @@ cleanup_environment() {
if [[ -f .genesis/manifests/$env-state.yml ]] ; then
header "Preparing to delete proto environment $env"
echo "Generating reference manifest..."
genesis manifest "$env" --no-redact > manifest.yml 2>/dev/null
genesis "$env" manifest --no-redact > manifest.yml 2>/dev/null
echo $'\n'"Building BOSH variables file..."
genesis lookup --merged "${env}" bosh-variables > vars.yml 2>/dev/null
genesis "${env}" lookup --merged bosh-variables > vars.yml 2>/dev/null
echo $'\n'"$env state file:"
echo "----------------->8------------------"
cat ".genesis/manifests/$env-state.yml"
Expand Down Expand Up @@ -68,16 +68,8 @@ cleanup() {
done
}

# Replace this with genesis lookup --env once its available (2.8.5)
cat <<'EOF' > get-env.pl
#!/usr/bin/env perl
use lib "$ENV{HOME}/.geese/lib";
use JSON::PP qw/encode_json/;
print encode_json(\%ENV);
EOF
chmod +x "get-env.pl"
vault_path="$(genesis sh "${DEPLOY_ENV}" -s "$(pwd)/get-env.pl" 2>/dev/null | grep '^{' | jq -r '.GENESIS_SECRETS_BASE')"
exodus_path="$(genesis sh "${DEPLOY_ENV}" -s "$(pwd)/get-env.pl" 2>/dev/null | grep '^{' | jq -r '.GENESIS_EXODUS_BASE')"
vault_path="$(genesis "$DEPLOY_ENV" lookup --env GENESIS_SECRETS_BASE)"
exodus_path="$(genesis "$DEPLOY_ENV" lookup --env GENESIS_EXODUS_BASE)"
vault_path="${vault_path%/}" # trim any trailing slash
# -----

Expand Down Expand Up @@ -105,10 +97,10 @@ if [[ "$SKIP_REPLACE_SECRETS" == "false" ]] ; then

# Remove credhub values
if ! is_proto "$DEPLOY_ENV" ; then (
bosh_env="$(genesis lookup "$DEPLOY_ENV" genesis 2>/dev/null | jq -r '.bosh_env // .env')"
bosh_env="$(genesis "$DEPLOY_ENV" lookup genesis 2>/dev/null | jq -r '.bosh_env // .env')"
[[ "$bosh_env" =~ / ]] || bosh_env="${bosh_env}/bosh"

bosh_exodus="$(genesis lookup --exodus-for "$bosh_env" "$DEPLOY_ENV" . "{}" 2>/dev/null)"
bosh_exodus="$(genesis "$DEPLOY_ENV" lookup --exodus-for "$bosh_env" . "{}" 2>/dev/null)"
CREDHUB_SERVER="$(jq -r '.credhub_url // ""' <<<"$bosh_exodus")"
if [[ -n "$CREDHUB_SERVER" ]] ; then
echo
Expand Down Expand Up @@ -157,8 +149,8 @@ fi

if [[ "$SKIP_DEPLOY" == "false" ]]; then
header "Deploying ${DEPLOY_ENV} environment to verify functionality..."
genesis "do" "${DEPLOY_ENV}" -- list
genesis add-secrets "${DEPLOY_ENV}"
genesis "${DEPLOY_ENV}" "do" -- list
genesis "${DEPLOY_ENV}" add-secrets

# get and upload stemcell version if needed (handled by bosh cli if version and name are supplied)
stemcell_iaas=
Expand All @@ -173,7 +165,7 @@ if [[ "$SKIP_DEPLOY" == "false" ]]; then
esac

if [[ -n "$stemcell_iaas" ]] ; then
stemcell_data="$(genesis lookup --merged "${DEPLOY_ENV}" stemcells)"
stemcell_data="$(genesis "${DEPLOY_ENV}" lookup --merged stemcells)"
stemcell_os="$(jq -r '.[0].os' <<<"$stemcell_data")"
stemcell_version="$(jq -r '.[0].version' <<<"$stemcell_data")"
stemcell_name="bosh-${stemcell_iaas}-${stemcell_os}-go_agent"
Expand All @@ -184,15 +176,15 @@ if [[ "$SKIP_DEPLOY" == "false" ]]; then
upload_options=()
upload_params=""
fi
if ! genesis bosh "$DEPLOY_ENV" stemcells 2>/dev/null \
if ! genesis "${DEPLOY_ENV}" bosh stemcells 2>/dev/null \
| grep "^${stemcell_name}" \
| awk '{print $2}' | sed -e 's/\*//' \
| grep "^${stemcell_version}\$" ; then
genesis bosh "$DEPLOY_ENV" upload-stemcell "https://bosh.io/d/stemcells/$stemcell_name${upload_params}" ${upload_options[@]+"${upload_options[@]}"}
genesis "${DEPLOY_ENV}" bosh upload-stemcell "https://bosh.io/d/stemcells/$stemcell_name${upload_params}" ${upload_options[@]+"${upload_options[@]}"}
fi
fi

genesis deploy -y "${DEPLOY_ENV}"
genesis "${DEPLOY_ENV}" deploy -y

if [[ -f .genesis/manifests/${DEPLOY_ENV}-state.yml ]] ; then
echo $'\n'"${DEPLOY_ENV} state file:"
Expand All @@ -201,20 +193,25 @@ if [[ "$SKIP_DEPLOY" == "false" ]]; then
echo "----------------->8------------------"
fi

genesis info "${DEPLOY_ENV}"
genesis "${DEPLOY_ENV}" info
if ! is_proto "$DEPLOY_ENV" ; then
genesis bosh "${DEPLOY_ENV}" instances --ps
genesis "${DEPLOY_ENV}" bosh instances --ps
fi

fi

if [[ "$SKIP_SMOKE_TESTS" == "false" ]]; then
header "Validating addons..."
genesis "do" "${DEPLOY_ENV}" -- setup-cli -f
genesis "do" "${DEPLOY_ENV}" -- login
if [[ -f "$0/test-addons" ]] ; then
header "Validating addons..."
# shellcheck source=/dev/null
source "$0/test-addons"
fi

header "Running smoke tests..."
genesis "do" "${DEPLOY_ENV}" -- smoketest
if [[ -f "$0/smoketests" ]] ; then
header "Running smoke tests..."
# shellcheck source=/dev/null
source "$0/smoketests"
fi
else
echo "Skipping smoke_tests"
fi
Expand Down

0 comments on commit 329d49f

Please sign in to comment.