[ci] Updates based on the vault changes (#26)

removes cat file as it's not need

This includes the following PRs from vault:
• genesis-community/vault-genesis-kit#28
• genesis-community/vault-genesis-kit#29
• genesis-community/vault-genesis-kit#30
• genesis-community/vault-genesis-kit#31
• genesis-community/vault-genesis-kit#32

ci/pipeline/resources/build.yml

@@ -2,8 +2,10 @@ resources:
   - name: build
     type: s3
     source:
-      bucket:            (( grab meta.aws.bucket ))
-      region_name:       (( grab meta.aws.region_name ))
-      regexp:            (( concat meta.name "/build/(.*)\.tar.gz" ))
-      access_key_id:     (( grab meta.aws.access_key ))
-      secret_access_key: (( grab meta.aws.secret_key ))
+      endpoint:              (( grab meta.aws.endpoint || ~ ))
+      skip_ssl_verification: (( grab meta.aws.insecure || false ))
+      access_key_id:         (( grab meta.aws.access_key ))
+      secret_access_key:     (( grab meta.aws.secret_key ))
+      region_name:           (( grab meta.aws.region_name ))
+      bucket:                (( grab meta.aws.bucket ))
+      regexp:                (( concat meta.name "/build/(.*)\.tar.gz" ))

ci/pipeline/resources/github.yml

@@ -2,6 +2,6 @@ resources:
   - name: github
     type: github-release
     source:
-      user:         (( grab meta.github.owner ))
+      owner:        (( grab meta.github.owner ))
       repository:   (( grab meta.github.repo ))
       access_token: (( grab meta.github.access_token ))

ci/pipeline/resources/spec-check.yml

@@ -2,8 +2,10 @@ resources:
   - name: spec-check
     type: s3
     source:
-      bucket:            (( grab meta.aws.bucket ))
-      region_name:       (( grab meta.aws.region_name ))
-      regexp:            (( concat meta.name "/spec-check/diff-(.*)" ))
-      access_key_id:     (( grab meta.aws.access_key ))
-      secret_access_key: (( grab meta.aws.secret_key ))
+      endpoint:              (( grab meta.aws.endpoint || ~ ))
+      skip_ssl_verification: (( grab meta.aws.insecure || false ))
+      access_key_id:         (( grab meta.aws.access_key ))
+      secret_access_key:     (( grab meta.aws.secret_key ))
+      bucket:                (( grab meta.aws.bucket ))
+      region_name:           (( grab meta.aws.region_name ))
+      regexp:                (( concat meta.name "/spec-check/diff-(.*)" ))

ci/pipeline/resources/version.yml

@@ -2,10 +2,12 @@ resources:
   - name: version
     type: semver
     source :
-      driver:            s3
-      bucket:            (( grab meta.aws.bucket ))
-      region_name:       (( grab meta.aws.region_name ))
-      key:               (( concat meta.name "/version" ))
-      access_key_id:     (( grab meta.aws.access_key ))
-      secret_access_key: (( grab meta.aws.secret_key ))
-      initial_version:   (( grab meta.initial_version || "0.0.1" ))
+      driver:                s3
+      endpoint:              (( grab meta.aws.endpoint || ~ ))
+      skip_ssl_verification: (( grab meta.aws.insecure || false ))
+      access_key_id:         (( grab meta.aws.access_key ))
+      secret_access_key:     (( grab meta.aws.secret_key ))
+      bucket:                (( grab meta.aws.bucket ))
+      region_name:           (( grab meta.aws.region_name ))
+      key:                   (( concat meta.name "/version" ))
+      initial_version:       (( grab meta.initial_version || "0.0.1" ))

ci/repipe

@@ -16,7 +16,7 @@ need_command() {
   local cmd=${1:?need_command() - no command name given}
   local url=${2:-}
 
-  if [[ ! -x "$(type -p "$cmd")" ]]; then
+  if [[ ! -x "$(type -P "$cmd")" ]]; then
     echo >&2 "${cmd} is not installed."
     if [[ -n "$url" ]]; then
       echo >&2 "Please download it from ${url}"
@@ -96,7 +96,7 @@ OPTIONS:
              whatever is set in 'meta.exposed' in the settings.yml file)
   -o         Open pipeline in browser if os supports it (mac only currently)
              after applying changes. Specify twice to not do anything else.
-  --fly <x>  Path to fly command, otherwise will use $(type -p fly)
+  --fly <x>  Path to fly command, otherwise will use $(type -P fly)
 
 EOF
   exit $rc
@@ -167,7 +167,7 @@ need_command jq
 
 if [[ -z "$fly" ]] ; then
   need_command fly;
-  fly="$(type -p fly)"
+  fly="$(type -P fly)"
 fi
 
 # -- Get settings file --------------------------------------------------------
@@ -220,7 +220,7 @@ if (( DRYRUN > 0 )) ; then
     if [[ -n "${persistent_file}" ]] ; then
       if [[ "${persistent_file}" =~ '->0x' ]] ; then
         persistent_file=''
-      elif [[ -n "$(type -p realpath || true)" ]] ; then
+      elif [[ -n "$(type -P realpath || true)" ]] ; then
         persistent_file="$(realpath --relative-to="$call_dir" "$persistent_file")"
       fi
     fi

ci/scripts/build-kit

@@ -40,8 +40,8 @@ if [[ ${#check_dirs[@]} -gt 0 ]] ; then
   header "Checking SHA1s of specified components (not including bosh-deployment) ..."
   out="$(eval "spruce merge --skip-eval $( \
     grep -rl '^releases:' "${check_dirs[@]}" \
-    | sed -e "s/\\(.*\\)/<(spruce json \\1 | jq -r '{releases: .releases}')/" |tr "\n" " " \
-    ) | spruce json | jq -r ." )"
+    | sed -e "s/\\(.*\\)/<(spruce json \\1 | jq -r '{releases: [ \"(( merge on sha1 ))\", .releases[] ]}')/" |tr "\n" " " \
+  ) | spruce json | jq -r ." )"
   echo "$out" | spruce merge | spruce json | "${CI_ROOT}/ci/scripts/check-sha1s"
 fi
 

ci/scripts/build-upstream-jobs

@@ -14,15 +14,29 @@ upstream_details="$(spruce json "${base_dir}/settings.yml" | jq -r '.meta.upstre
 # For each release in upstream.yml,
 for release in $upstream_details ; do
   name="$(_lookup "$release" .name)"
-  type="$(_lookup "$release" '.type//"bosh-io-release"')" # Other valid value is github-release
+  type="$(_lookup "$release" '.type//"bosh-io-release"')"
   path="$(_lookup "$release" '.path//"manifests/releases/'"$name"'.yml"')"
   repo="$(_lookup "$release" '.repository')"
-  owner=""
-  if [[ $type == "github-release" ]] ; then
-    owner=$'\n'"      owner: ${repo%/*}"
-    repo="${repo##*/}"
-  fi
+  if [[ $type == 'bosh-io-release' ]] ; then
+    source=$'\n'"      repository: $repo";
+  elif [[ $type == 'github-release' ]] ; then
+    owner="$(_lookup "$release" '.owner//""')"
+    if [[ -z "$owner" && "$repo" =~ / ]] ; then
+      owner="${repo%%/*}"
+      repo="${repo#*/}"
+    fi
+    source=$'\n'"      repository: $repo"$'\n'"      owner: $owner";
 
+    token="$(_lookup "$release" '.access_token//""')"
+    if [[ -n "$token" ]] ; then
+      source="$source"$'\n      access_token: "'"$token"'"'
+    fi
+  else
+    echo >&2 "Unknown resource type for $name upstream release: $type"
+    echo >&2 "Expecting one of: bosh-io-release, github-release"
+    echo >&2 "Update upstream.bosh-releases configuration in ci/settings.yml"
+    exit 1
+  fi
   job="update-${name}-release"
   release="${name}-release"
 
@@ -63,8 +77,7 @@ resources:
   - name: $release
     type: $type
     check_every: 24h
-    source:
-      repository: $repo$owner
+    source: $source
 EOF
 
 done

ci/scripts/compare-release-specs

@@ -3,7 +3,7 @@ set -ue
 
 # What branch is the comparison of the current working branch being compared against
 compare_branch="${1:-origin/}"
-check_dirs="spec manifests manifests/releases"
+check_dirs="spec/results manifests"
 
 orig_dir="$(pwd)"
 # needed because of when running locally or in ci
@@ -31,6 +31,11 @@ workdir="$(mktemp -d)"
 mkdir "$workdir/compare"
 cp -R "$(pwd)/.git" "$workdir/compare/"
 pushd "$workdir/compare" > /dev/null
+if ! git show -q "$compare_branch" &>/dev/null ; then
+  echo "The latest release ($compare_branch) is NOT an ancestor to this commit."
+  echo "This should never happen -- Cannot continue!"
+ exit 1
+fi
 git checkout -qf --detach "$compare_branch"
 prev_releases="$(releases)"
 popd > /dev/null
@@ -113,7 +118,7 @@ if [[ "${#changed[@]}" -gt 0 && -n "${changed[0]}" ]] ; then
     echo "$curr_releases" \
     | jq --argjson gitrepos "$upstreamrepo" -r 'reduce .releases[] as {$name, $url, $sha1, $version} ({repos: []};
         ($url
-        | if ($url | test("https?://s3.amazonaws.com")) then
+        | if ($url | test("https?://s3(-.*)?.amazonaws.com")) then
             ($gitrepos.repos | map(select(.name == $name))[0].repo)
           elif ($url | test("https?://storage.googleapis.com")) then
             ($gitrepos.repos | map(select(.name == $name))[0].repo)

ci/settings.yml

@@ -18,6 +18,7 @@ meta:
       type:       github-release
       repository: cloudfoundry/app-autoscaler-release
       path:       overlay/releases/app-autoscaler.yml
+      access_token: ((github.access_token))
 
   bosh:
     uri:      ((bosh.uri))