[ci] ci updates base on the vault

ci/pipeline/base.yml

@@ -20,23 +20,24 @@ meta:
   pipeline: (( grab meta.name ))
 
   upstream:
-    package:  example # dummy value for things that don't use upstream
-    path:     (( grab meta.upstream.package ))
-    org:      cloudfoundry
-    repo:     (( concat meta.upstream.org "/" meta.upstream.package ))
-    url:      (( concat "https://github.com/" meta.upstream.repo ".git" ))
+    manifest:
+      package:  example # dummy value for things that don't use upstream
+      path:     (( grab meta.upstream.manifest.package ))
+      org:      cloudfoundry
+      repo:     (( concat meta.upstream.manifest.org "/" meta.upstream.manifest.package ))
+      url:      (( concat "https://github.com/" meta.upstream.manifest.repo ".git" ))
 
   vault:
     url:   (( param "Please provide the address of your Vault" ))
     token: (( param "Please provide a Vault Token" ))
 
   git:
-    email: genesis-ci@rubidiumstudios.com
-    name:  Genesis CI Bot
+    email: ((github.email))
+    name:  ((github.name))
 
   image:
-    repository: registry.ops.scalecf.net/genesis-community/genesis
-    tag: latest
+    name: ((image/genesis.url))
+    tag:  ((image/genesis.tag))
 
   aws:
     bucket:      genesis-pipeline
@@ -98,9 +99,11 @@ groups:
     - acceptance-tests
     - prepare
     - ship-release
+
   - name: upstream
     jobs:
-    - upstream-sync
+    - upstream-manifest
+
   - name: versions
     jobs:
     - major

ci/pipeline/jobs/upstream-sync.yml → ci/pipeline/jobs/upstream-manifest.yml

@@ -1,20 +1,20 @@
 jobs:
-- name: upstream-sync
+- name: upstream-manifest
   public: true
   serial: true
   plan:
   - do:
     - in_parallel:
       - { get: git, trigger: false, passed: [spec-tests] }
       - { get: git-ci, trigger: false}
-      - { get: upstream, trigger: true }
-    - task: upstream-sync
-      file: git-ci/ci/tasks/get-latest-upstream.yml
+      - { get: upstream-manifest, trigger: true }
+    - task: upstream-manifest
+      file: git-ci/ci/tasks/get-latest-upstream-manifest.yml
       params:
         GIT_EMAIL:        (( grab meta.git.email ))
         GIT_NAME:         (( grab meta.git.name ))
-        UPSTREAM_REPO:    (( grab meta.upstream.repo ))
-        UPSTREAM_PATH:    (( grab meta.upstream.path || meta.upstream.repo ))
+        UPSTREAM_PATH:    (( grab meta.upstream.manifest.path ))
+        UPSTREAM_REPO:    (( grab meta.upstream.manifest.repo ))
     - put: git
       params:
         merge: true

ci/pipeline/resources/git-latest-tag.yml

@@ -6,5 +6,5 @@ resources:
       uri:         (( grab meta.github.uri ))
       branch:      (( grab meta.github.branch ))
       private_key: (( grab meta.github.private_key ))
-      tag_regex:   '^v[1-9][0-9\.]*$'
+      tag_regex:   '^v[0-9\.]*$'
       disable_ci_skip: true

ci/pipeline/resources/upstream-manifest.yml

@@ -0,0 +1,8 @@
+resources:
+  - name: upstream-manifest
+    type: git
+    check_every: 168h
+    source:
+      uri:    (( grab meta.upstream.manifest.url ))
+      branch: (( grab meta.upstream.manifest.branch || "master" ))
+      ignore_paths: [ci/*]

ci/repipe

@@ -16,7 +16,7 @@ need_command() {
   local cmd=${1:?need_command() - no command name given}
   local url=${2:-}
 
-  if [[ ! -x "$(type -p "$cmd")" ]]; then
+  if [[ ! -x "$(type -P "$cmd")" ]]; then
     echo >&2 "${cmd} is not installed."
     if [[ -n "$url" ]]; then
       echo >&2 "Please download it from ${url}"
@@ -96,7 +96,7 @@ OPTIONS:
              whatever is set in 'meta.exposed' in the settings.yml file)
   -o         Open pipeline in browser if os supports it (mac only currently)
              after applying changes. Specify twice to not do anything else.
-  --fly <x>  Path to fly command, otherwise will use $(type -p fly)
+  --fly <x>  Path to fly command, otherwise will use $(type -P fly)
 
 EOF
   exit $rc
@@ -167,7 +167,7 @@ need_command jq
 
 if [[ -z "$fly" ]] ; then
   need_command fly;
-  fly="$(type -p fly)"
+  fly="$(type -P fly)"
 fi
 
 # -- Get settings file --------------------------------------------------------
@@ -217,10 +217,10 @@ if (( DRYRUN > 0 )) ; then
   if [ ! -t 1 ] ; then
     pid="$$"
     persistent_file="$(/usr/sbin/lsof -p "$pid" -a -d 1 | tail -n 1 | awk '{print $NF}')"
-    if [[ -n "${persistent_file}" ]] ; then 
-      if [[ "${persistent_file}" =~ '->0x' ]] ; then 
+    if [[ -n "${persistent_file}" ]] ; then
+      if [[ "${persistent_file}" =~ '->0x' ]] ; then
         persistent_file=''
-      elif [[ -n "$(type -p realpath || true)" ]] ; then
+      elif [[ -n "$(type -P realpath || true)" ]] ; then
         persistent_file="$(realpath --relative-to="$call_dir" "$persistent_file")"
       fi
     fi

ci/scripts/build-kit

@@ -32,14 +32,18 @@ safe target da-vault "$VAULT_URI" -k
 echo "$VAULT_TOKEN" | safe auth token
 safe read secret/handshake
 
-header "Checking SHA1s of specified components (not including bosh-deployment) ..."
-pushd "$REPO_ROOT" > /dev/null
-out="$(eval "spruce merge --skip-eval $( \
-  grep -rl '^releases:' overlay/\
-  | sed -e "s/\\(.*\\)/<(spruce json \\1 | jq -r '{releases: .releases}')/" |tr "\n" " " \
+check_dirs=()
+for dir in overlay manifests spec/results; do
+  [[ -d "$REPO_ROOT/$dir" ]] && check_dirs+=( "$REPO_ROOT/$dir/" )
+done
+if [[ ${#check_dirs[@]} -gt 0 ]] ; then
+  header "Checking SHA1s of specified components (not including bosh-deployment) ..."
+  out="$(eval "spruce merge --skip-eval $( \
+    grep -rl '^releases:' "${check_dirs[@]}" \
+    | sed -e "s/\\(.*\\)/<(spruce json \\1 | jq -r '{releases: [ \"(( merge on sha1 ))\", .releases[] ]}')/" |tr "\n" " " \
   ) | spruce json | jq -r ." )"
-popd > /dev/null
-echo "$out" | spruce merge | spruce json | "${CI_ROOT}/ci/scripts/check-sha1s"
+  echo "$out" | spruce merge | spruce json | "${CI_ROOT}/ci/scripts/check-sha1s"
+fi
 
 header "Building $KIT_SHORTNAME kit v$VERSION"
 genesis -C "$REPO_ROOT" compile-kit -v "$VERSION" -n "$KIT_SHORTNAME"

ci/scripts/build-upstream-jobs

@@ -0,0 +1,98 @@
+#!/bin/bash -
+set -ue
+base_dir="$(cd "$(dirname "$0")/.." && pwd)"
+mkdir -p "${base_dir}/pipeline/upstream/"
+
+_lookup() {
+ echo "${1}" | base64 --decode | jq -r "${2}"
+}
+
+# Read upstream.yml
+update_group=()
+upstream_details="$(spruce json "${base_dir}/settings.yml" | jq -r '.meta.upstream.bosh_releases//[] | .[] | @base64')"
+
+# For each release in upstream.yml,
+for release in $upstream_details ; do
+  name="$(_lookup "$release" .name)"
+  type="$(_lookup "$release" '.type//"bosh-io-release"')"
+  path="$(_lookup "$release" '.path//"manifests/releases/'"$name"'.yml"')"
+  repo="$(_lookup "$release" '.repository')"
+  if [[ $type == 'bosh-io-release' ]] ; then
+    source=$'\n'"      repository: $repo";
+  elif [[ $type == 'github-release' ]] ; then
+    owner="$(_lookup "$release" '.owner//""')"
+    if [[ -z "$owner" && "$repo" =~ / ]] ; then
+      owner="${repo%%/*}"
+      repo="${repo#*/}"
+    fi
+    source=$'\n'"      repository: $repo"$'\n'"      owner: $owner";
+
+    token="$(_lookup "$release" '.access_token//""')"
+    if [[ -n "$token" ]] ; then
+      source="$source"$'\n      access_token: "'"$token"'"'
+    fi
+  else
+    echo >&2 "Unknown resource type for $name upstream release: $type"
+    echo >&2 "Expecting one of: bosh-io-release, github-release"
+    echo >&2 "Update upstream.bosh-releases configuration in ci/settings.yml"
+    exit 1
+  fi
+  job="update-${name}-release"
+  release="${name}-release"
+
+  update_group+=( "$job" )
+
+  cat <<EOF >> "$base_dir/pipeline/upstream/update_${name}_release.yml"
+jobs:
+- (( append ))
+- name: $job
+  public: false
+  serial: true
+  serial_groups: [upstream-releases]
+  plan:
+  - do:
+    - in_parallel:
+      - { get: git,    trigger: false, passed: [spec-tests] }
+      - { get: git-ci, trigger: false  }
+      - get: $release
+        trigger: true
+        params:
+          tarball: false
+    - task: $job
+      file: git-ci/ci/tasks/update-release.yml
+      input_mapping: {bosh-release: $release}
+      params:
+        RELEASE_NAME:  $name
+        RELEASE_PATH:  $path
+        BRANCH:        (( grab meta.github.branch ))
+        GIT_EMAIL:     (( grab meta.git.email ))
+        GIT_NAME:      (( grab meta.git.name ))
+    - put: git
+      params:
+        merge: true
+        repository: git
+
+resources:
+  - (( append ))
+  - name: $release
+    type: $type
+    check_every: 24h
+    source: $source
+EOF
+
+done
+group_file="$base_dir/pipeline/upstream/update_group.yml"
+if [[ "${#update_group[@]}" -gt 0 ]] ; then
+  (
+  echo "groups:"
+  echo "- (( merge on name ))"
+  echo "- name: upstream"
+  echo "  jobs:"
+  echo "  - (( append ))"
+  for job in ${update_group[@]+"${update_group[@]}"} ; do
+    echo "  - $job"
+  done
+  ) >> "$group_file"
+elif [[ -f "$group_file" ]] ; then
+  rm -f "$group_file"
+fi

ci/scripts/compare-release-specs

@@ -3,7 +3,7 @@ set -ue
 
 # What branch is the comparison of the current working branch being compared against
 compare_branch="${1:-origin/}"
-check_dirs="spec manifests manifests/releases"
+check_dirs="spec/results manifests"
 
 orig_dir="$(pwd)"
 # needed because of when running locally or in ci
@@ -23,24 +23,28 @@ release_files() {
 releases() {
   eval "spruce merge --skip-eval $( \
   release_files \
-  | sed -e "s/\\(.*\\)/\<(spruce json \\1 | jq -r '{releases: .releases}')/" |tr "\n" " " \
+  | sed -e "s/\\(.*\\)/\<(spruce json \\1 | jq -r '{releases: [ \"(( merge on sha1 ))\", .releases[] ]}')/" |tr "\n" " " \
   ) | spruce merge | spruce json | jq -r ."
 }
 
 workdir="$(mktemp -d)"
 mkdir "$workdir/compare"
 cp -R "$(pwd)/.git" "$workdir/compare/"
 pushd "$workdir/compare" > /dev/null
+if ! git show -q "$compare_branch" &>/dev/null ; then
+  echo "The latest release ($compare_branch) is NOT an ancestor to this commit."
+  echo "This should never happen -- Cannot continue!"
+ exit 1
+fi
 git checkout -qf --detach "$compare_branch"
 prev_releases="$(releases)"
 popd > /dev/null
 rm -rf "$workdir/compare"
 
 curr_releases="$(releases)"
 
-
-prev_rel_names="$(echo "$prev_releases"| jq -r '.releases[] | .name' | sort)"
-curr_rel_names="$(echo "$curr_releases"| jq -r '.releases[] | .name' | sort)"
+prev_rel_names="$(echo "$prev_releases"| jq -r '.releases[] | .name' | sort | uniq)"
+curr_rel_names="$(echo "$curr_releases"| jq -r '.releases[] | .name' | sort | uniq)"
 
 removed=()
 while IFS='' read -r rel ; do
@@ -54,9 +58,11 @@ done <<<"$(diff -p <(echo "$prev_rel_names") <(echo "$curr_rel_names") | grep '^
 unchanged=()
 changed=()
 while IFS='' read -r rel; do
-  prev_ver="$(echo "$prev_releases" | jq -r --arg r "$rel" '(.releases[] | select(.name == $r) | .version ) // "--none--" ' )"
+  prev_ver="$(echo "$prev_releases" | jq -r --arg r "$rel" \
+    '.releases | map(select(.name == $r) | .version) | sort | unique | if(.|length>0) then .|join(",") else "--none--" end' )"
   if [[ "$prev_ver" == "--none--" ]] ; then continue ; fi
-  curr_ver="$(echo "$curr_releases" | jq -r --arg r "$rel" '.releases[] | select(.name == $r) | .version' )"
+  curr_ver="$(echo "$curr_releases" | jq -r --arg r "$rel" \
+    '.releases | map(select(.name == $r) | .version) | sort | unique | join(",")' )"
   if [[ "$prev_ver" == "$curr_ver" ]] ; then
     unchanged+=( "$rel $curr_ver" )
   else
@@ -102,28 +108,38 @@ if [[ "${#changed[@]}" -gt 0 && -n "${changed[0]}" ]] ; then
   if [ -f "${ci_dir}/ci/upstreamrepo.yml" ]; then
     upstreamrepo=$(spruce json "${ci_dir}/ci/upstreamrepo.yml")
   else
-    upstreamrepo="[]"
+    upstreamrepo='{"repos": []}'
   fi
+  # TODO: do this in two phases -- first phase pull out all the non-compiled
+  #       versions, then run through with the compiled versions, picking up the
+  #       non-compiled version's git repo.  Also indicate if they are compiled
+  #       or not, and if so, what os is the target. (because that may change)
   repos="$(
     echo "$curr_releases" \
     | jq --argjson gitrepos "$upstreamrepo" -r 'reduce .releases[] as {$name, $url, $sha1, $version} ({repos: []};
         ($url
         | if ($url | test("https?://s3(-.*)?.amazonaws.com")) then
             ($gitrepos.repos | map(select(.name == $name))[0].repo)
-            elif ($url | test("https?://bosh.io")) then
+          elif ($url | test("https?://storage.googleapis.com")) then
+            ($gitrepos.repos | map(select(.name == $name))[0].repo)
+          elif ($url | test("https?://bosh.io")) then
             ($url | sub("^.*/d/";"https://") | sub("\\?v=.*$";""))
-            elif ($url | test("https?://github.com")) then
+          elif ($url | test("https?://github.com")) then
             ($url | sub("^.*http";"http") | sub("/releases/download/.*$";""))
-            else
+          else
             $url
-            end
+          end
         ) as $repo |
         (.repos += [{$name,$repo}])
     )')"
 
   for info in "${changed[@]}" ; do
     read -r rel prev_ver curr_ver <<<"$info"
-    repo="$(echo "$repos" | jq -r --arg r "$rel" '.repos[] | select(.name == $r) | .repo' )"
+
+    #TODO: handle multiple versions (comma separated) -- right now we're just taking the first one with a repo.
+    #TODO: handle compiled releases better -- right now just skipping.
+    repo="$(echo "$repos" | jq -r --arg r "$rel" '.repos | map(select(.name == $r and .repo != null)) | .[0].repo//""' )"
+    [[ -n "$repo" ]] || continue
     rel_dir="$workdir/releases/$rel"
     mkdir -p "$rel_dir"
     git -C "$rel_dir" init >/dev/null 2>&1 && \

ci/scripts/smoketests

@@ -0,0 +1,20 @@
+	header "Validating ${DEPLOY_ENV} BOSH director admin / concourse UAA credentials..."
+	(
+		BOSH_CA_CERT="$(genesis lookup --exodus "$DEPLOY_ENV" ca_cert 2>/dev/null)"
+		BOSH_ENVIRONMENT="$(genesis lookup --exodus "$DEPLOY_ENV" url 2>/dev/null)"
+		export BOSH_CA_CERT
+		export BOSH_ENVIRONMENT
+
+		BOSH_CLIENT="admin" \
+		BOSH_CLIENT_SECRET="$(safe read "$vault_path/users/admin:password")" \
+		$BOSH env
+
+		BOSH_CLIENT="concourse" \
+		BOSH_CLIENT_SECRET="$(safe read "$vault_path/users/concourse:password")" \
+		$BOSH env
+	)
+
+	# TODO: need an actual smoke-test, but this doesn't exist yet...
+	#if ! is_proto "$DEPLOY_ENV" ; then
+	#	genesis "do" "${DEPLOY_ENV}" -- smoketests
+	#fi

ci/scripts/test-addons

@@ -0,0 +1,6 @@
+  genesis "do" "${DEPLOY_ENV}" -- login
+  genesis "do" "${DEPLOY_ENV}" -- credhub-login
+  if has_feature "${DEPLOY_ENV}" "vault-credhub-proxy" ; then
+    safe_target_orig="$(safe target --json | jq -r .name)"
+    genesis "do" "${DEPLOY_ENV}" -- vault-proxy-login
+    safe target "$safe_target_orig"