Skip to content

Commit

Permalink
[CI] Buildkite support with Baseline pipeline (#100492)
Browse files Browse the repository at this point in the history
  • Loading branch information
brianseeders authored Jun 1, 2021
1 parent 0ecca23 commit 54c3ca1
Show file tree
Hide file tree
Showing 37 changed files with 729 additions and 3 deletions.
10 changes: 10 additions & 0 deletions .buildkite/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# Kibana / Buildkite

## Directory Structure

- `hooks` - special directory used by Buildkite agents for [hooks](https://buildkite.com/docs/agent/v3/hooks)
- `pipelines` - contains pipeline definitions
- `scripts/common` - scripts that get `source`d by other scripts to set environment variables or import shared functions
- `scripts/lifecycle` - general scripts for tasks that run before or after individual steps or the entire build
- `scripts/steps` - scripts that define something that will run for a step defined in a pipeline
- `scripts/*` - all other scripts are building blocks that make up the tasks in pipelines. They may be run by other scripts, but should not be `source`d
79 changes: 79 additions & 0 deletions .buildkite/agents.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
{
"gcp": {
"project": "elastic-kibana-ci",
"zones": ["us-central1-a", "us-central1-b", "us-central1-c", "us-central1-f"],
"serviceAccount": "elastic-buildkite-agent@elastic-kibana-ci.iam.gserviceaccount.com",
"imageFamily": "kb-ubuntu",
"subnetwork": "buildkite",
"disableExternalIp": true,
"diskType": "pd-ssd",
"diskSizeGb": 75,
"overprovision": 0,
"minimumAgents": 0,
"maximumAgents": 50,
"gracefulStopAfterMins": 360,
"hardStopAfterMins": 540,
"idleTimeoutMins": 10,
"exitAfterOneJob": false,

"agents": [
{
"queue": "default",
"name": "kb-default",
"minimumAgents": 1,
"maximumAgents": 100,
"idleTimeoutMins": 60,
"machineType": "e2-small"
},
{
"queue": "c2-8",
"name": "kb-c2-8",
"machineType": "c2-standard-8",
"localSsds": 1
},
{
"queue": "c2-4",
"name": "kb-c2-4",
"machineType": "c2-standard-4",
"localSsds": 1
},
{
"queue": "jest",
"name": "kb-jest",
"machineType": "n2-standard-2",
"diskSizeGb": 128
},
{
"queue": "ci-group",
"name": "kb-cigroup",
"machineType": "n2-standard-8",
"diskSizeGb": 256
},
{
"queue": "ci-group-4",
"name": "kb-cigroup-4",
"machineType": "n2-standard-4",
"diskSizeGb": 128
},
{
"queue": "ci-group-4d",
"name": "kb-cigroup-4d",
"machineType": "n2d-standard-4",
"diskSizeGb": 128
},
{
"queue": "ci-group-6",
"name": "kb-cigroup-6",
"machineType": "n2-custom-6-16384",
"diskSizeGb": 128
},
{
"queue": "packer",
"name": "kb-packer",
"serviceAccount": "buildkite-packer-agent@elastic-kibana-ci.iam.gserviceaccount.com",
"maximumAgents": 10,
"machineType": "e2-small"
}
]
}
}
3 changes: 3 additions & 0 deletions .buildkite/hooks/post-command
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
#!/usr/bin/env bash

source .buildkite/scripts/lifecycle/post_command.sh
3 changes: 3 additions & 0 deletions .buildkite/hooks/pre-command
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
#!/usr/bin/env bash

source .buildkite/scripts/lifecycle/pre_command.sh
24 changes: 24 additions & 0 deletions .buildkite/pipelines/on_merge.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
env:
GITHUB_COMMIT_STATUS_ENABLED: 'true'
GITHUB_COMMIT_STATUS_CONTEXT: 'buildkite/on-merge'
steps:
- command: .buildkite/scripts/lifecycle/pre_build.sh
label: Pre-Build

- wait

- command: .buildkite/scripts/steps/on_merge_build_and_metrics.sh
label: Default Build and Metrics
agents:
queue: c2-8

- command: .buildkite/scripts/steps/on_merge_ts_refs_api_docs.sh
label: Build TS Refs and Check Public API Docs
agents:
queue: c2-4

- wait: ~
continue_on_failure: true

- command: .buildkite/scripts/lifecycle/post_build.sh
label: Post-Build
22 changes: 22 additions & 0 deletions .buildkite/scripts/bootstrap.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
#!/usr/bin/env bash

set -euo pipefail

source .buildkite/scripts/common/util.sh

echo "--- yarn install and bootstrap"
yarn kbn bootstrap --verbose

###
### upload ts-refs-cache artifacts as quickly as possible so they are available for download
###
if [[ "${BUILD_TS_REFS_CACHE_CAPTURE:-}" == "true" ]]; then
echo "--- Upload ts-refs-cache"
cd "$KIBANA_DIR/target/ts_refs_cache"
gsutil cp "*.zip" 'gs://kibana-ci-ts-refs-cache/'
cd "$KIBANA_DIR"
fi

if [[ "$DISABLE_BOOTSTRAP_VALIDATION" != "true" ]]; then
verify_no_git_changes 'yarn kbn bootstrap'
fi
16 changes: 16 additions & 0 deletions .buildkite/scripts/build_kibana.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#!/usr/bin/env bash

set -euo pipefail

export KBN_NP_PLUGINS_BUILT=true

echo "--- Build Kibana Distribution"
node scripts/build --debug --no-oss

echo "--- Archive Kibana Distribution"
linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
installDir="$KIBANA_DIR/install/kibana"
mkdir -p "$installDir"
tar -xzf "$linuxBuild" -C "$installDir" --strip=1
mkdir -p "$KIBANA_BUILD_LOCATION"
cp -pR install/kibana/. "$KIBANA_BUILD_LOCATION/"
32 changes: 32 additions & 0 deletions .buildkite/scripts/build_kibana_plugins.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
#!/usr/bin/env bash

set -euo pipefail

echo "--- Build Platform Plugins"
node scripts/build_kibana_platform_plugins \
--scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \
--scan-dir "$KIBANA_DIR/test/interpreter_functional/plugins" \
--scan-dir "$KIBANA_DIR/test/common/fixtures/plugins" \
--scan-dir "$KIBANA_DIR/examples" \
--scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \
--scan-dir "$KIBANA_DIR/test/common/fixtures/plugins" \
--scan-dir "$XPACK_DIR/test/plugin_functional/plugins" \
--scan-dir "$XPACK_DIR/test/functional_with_es_ssl/fixtures/plugins" \
--scan-dir "$XPACK_DIR/test/alerting_api_integration/plugins" \
--scan-dir "$XPACK_DIR/test/plugin_api_integration/plugins" \
--scan-dir "$XPACK_DIR/test/plugin_api_perf/plugins" \
--scan-dir "$XPACK_DIR/test/licensing_plugin/plugins" \
--scan-dir "$XPACK_DIR/test/usage_collection/plugins" \
--scan-dir "$XPACK_DIR/test/security_functional/fixtures/common" \
--scan-dir "$XPACK_DIR/examples" \
--verbose

echo "--- Archive built plugins"
shopt -s globstar
tar -zcf \
target/kibana-default-plugins.tar.gz \
x-pack/plugins/**/target/public \
x-pack/test/**/target/public \
examples/**/target/public \
x-pack/examples/**/target/public \
test/**/target/public
72 changes: 72 additions & 0 deletions .buildkite/scripts/common/env.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
#!/usr/bin/env bash

export CI=true

KIBANA_DIR=$(pwd)
export KIBANA_DIR
export XPACK_DIR="$KIBANA_DIR/x-pack"

export CACHE_DIR="$HOME/.kibana"
PARENT_DIR="$(cd "$KIBANA_DIR/.."; pwd)"
export PARENT_DIR
export WORKSPACE="${WORKSPACE:-$PARENT_DIR}"

KIBANA_PKG_BRANCH="$(jq -r .branch "$KIBANA_DIR/package.json")"
export KIBANA_PKG_BRANCH
export KIBANA_BASE_BRANCH="$KIBANA_PKG_BRANCH"

export GECKODRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
export CHROMEDRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
export RE2_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
export CYPRESS_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/cypress"

export NODE_OPTIONS="--max-old-space-size=4096"

export FORCE_COLOR=1
export TEST_BROWSER_HEADLESS=1

export ELASTIC_APM_ENVIRONMENT=ci
export ELASTIC_APM_TRANSACTION_SAMPLE_RATE=0.1

CI_REPORTING_ENABLED=false # TODO enable when ready, only controls checks reporter and APM

if is_pr; then
export ELASTIC_APM_ACTIVE=false
export CHECKS_REPORTER_ACTIVE="${CI_REPORTING_ENABLED-}"

# These can be removed once we're not supporting Jenkins and Buildkite at the same time
# These are primarily used by github checks reporter and can be configured via /github_checks_api.json
export ghprbGhRepository="elastic/kibana"
export ghprbActualCommit="$BUILDKITE_COMMIT"
export BUILD_URL="$BUILDKITE_BUILD_URL"

# set_git_merge_base # TODO for PRs
else
export ELASTIC_APM_ACTIVE="${CI_REPORTING_ENABLED-}"
export CHECKS_REPORTER_ACTIVE=false
fi

export FLEET_PACKAGE_REGISTRY_PORT=6104
export TEST_CORS_SERVER_PORT=6105

export DETECT_CHROMEDRIVER_VERSION=true
export CHROMEDRIVER_FORCE_DOWNLOAD=true

export GCS_UPLOAD_PREFIX=FAKE_UPLOAD_PREFIX # TODO remove the need for this

export KIBANA_BUILD_LOCATION="$WORKSPACE/kibana-build-xpack"

if [[ "${BUILD_TS_REFS_CACHE_ENABLE:-}" != "true" ]]; then
export BUILD_TS_REFS_CACHE_ENABLE=false
fi

export BUILD_TS_REFS_DISABLE=true
export DISABLE_BOOTSTRAP_VALIDATION=true

export TEST_KIBANA_HOST=localhost
export TEST_KIBANA_PORT=6101
export TEST_KIBANA_URL="http://elastic:changeme@localhost:6101"
export TEST_ES_URL="http://elastic:changeme@localhost:6102"
export TEST_ES_TRANSPORT_PORT=6103
export TEST_CORS_SERVER_PORT=6106
export ALERTING_PROXY_PORT=6105
22 changes: 22 additions & 0 deletions .buildkite/scripts/common/setup_bazel.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
#!/usr/bin/env bash

KIBANA_BUILDBUDDY_CI_API_KEY=$(vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key)
export KIBANA_BUILDBUDDY_CI_API_KEY

cp "$KIBANA_DIR/src/dev/ci_setup/.bazelrc-ci" "$HOME/.bazelrc"

###
### append auth token to buildbuddy into "$HOME/.bazelrc";
###
echo "# Appended by .buildkite/scripts/setup_bazel.sh" >> "$HOME/.bazelrc"
echo "build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY" >> "$HOME/.bazelrc"

###
### remove write permissions on buildbuddy remote cache for prs
###
if [[ "${BUILDKITE_PULL_REQUEST:-}" && "$BUILDKITE_PULL_REQUEST" != "false" ]] ; then
{
echo "# Uploads logs & artifacts without writing to cache"
echo "build --noremote_upload_local_results"
} >> "$HOME/.bazelrc"
fi
46 changes: 46 additions & 0 deletions .buildkite/scripts/common/setup_node.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
#!/usr/bin/env bash

echo "--- Setup Node"

NODE_VERSION="$(cat "$KIBANA_DIR/.node-version")"
export NODE_VERSION
export NODE_DIR="$CACHE_DIR/node/$NODE_VERSION"
export NODE_BIN_DIR="$NODE_DIR/bin"
export YARN_OFFLINE_CACHE="$CACHE_DIR/yarn-offline-cache"

if [[ ! -d "$NODE_DIR" ]]; then
hostArch="$(command uname -m)"
case "${hostArch}" in
x86_64 | amd64) nodeArch="x64" ;;
aarch64) nodeArch="arm64" ;;
*) nodeArch="${hostArch}" ;;
esac

nodeUrl="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$nodeArch.tar.gz"

echo "node.js v$NODE_VERSION not found at $NODE_DIR, downloading from $nodeUrl"

mkdir -p "$NODE_DIR"
curl --silent -L "$nodeUrl" | tar -xz -C "$NODE_DIR" --strip-components=1
else
echo "node.js v$NODE_VERSION already installed to $NODE_DIR, re-using"
ls -alh "$NODE_BIN_DIR"
fi

export PATH="$NODE_BIN_DIR:$PATH"


echo "--- Setup Yarn"

YARN_VERSION=$(node -e "console.log(String(require('./package.json').engines.yarn || '').replace(/^[^\d]+/,''))")
export YARN_VERSION

if [[ ! $(which yarn) || $(yarn --version) != "$YARN_VERSION" ]]; then
npm install -g "yarn@^${YARN_VERSION}"
fi

yarn config set yarn-offline-mirror "$YARN_OFFLINE_CACHE"

YARN_GLOBAL_BIN=$(yarn global bin)
export YARN_GLOBAL_BIN
export PATH="$PATH:$YARN_GLOBAL_BIN"
51 changes: 51 additions & 0 deletions .buildkite/scripts/common/util.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
#!/usr/bin/env bash

checks-reporter-with-killswitch() {
if [ "$CHECKS_REPORTER_ACTIVE" == "true" ] ; then
yarn run github-checks-reporter "$@"
else
arguments=("$@");
"${arguments[@]:1}";
fi
}

is_pr() {
[[ "${GITHUB_PR_NUMBER-}" ]] && return
false
}

verify_no_git_changes() {
RED='\033[0;31m'
C_RESET='\033[0m' # Reset color

GIT_CHANGES="$(git ls-files --modified)"
if [ "$GIT_CHANGES" ]; then
echo -e "\n${RED}ERROR: '$1' caused changes to the following files:${C_RESET}\n"
echo -e "$GIT_CHANGES\n"
exit 1
fi
}

# docker_run can be used in place of `docker run`
# it automatically passes along all of Buildkite's tracked environment variables, and mounts the buildkite-agent in the running container
docker_run() {
args=()

if [[ -n "${BUILDKITE_ENV_FILE:-}" ]] ; then
# Read in the env file and convert to --env params for docker
# This is because --env-file doesn't support newlines or quotes per https://docs.docker.com/compose/env-file/#syntax-rules
while read -r var; do
args+=( --env "${var%%=*}" )
done < "$BUILDKITE_ENV_FILE"
fi

BUILDKITE_AGENT_BINARY_PATH=$(command -v buildkite-agent)
args+=(
"--env" "BUILDKITE_JOB_ID"
"--env" "BUILDKITE_BUILD_ID"
"--env" "BUILDKITE_AGENT_ACCESS_TOKEN"
"--volume" "$BUILDKITE_AGENT_BINARY_PATH:/usr/bin/buildkite-agent"
)

docker run "${args[@]}" "$@"
}
Loading

0 comments on commit 54c3ca1

Please sign in to comment.