-
Notifications
You must be signed in to change notification settings - Fork 8.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[CI] Buildkite support with Baseline pipeline (#100492)
- Loading branch information
1 parent
0ecca23
commit 54c3ca1
Showing
37 changed files
with
729 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
# Kibana / Buildkite | ||
|
||
## Directory Structure | ||
|
||
- `hooks` - special directory used by Buildkite agents for [hooks](https://buildkite.com/docs/agent/v3/hooks) | ||
- `pipelines` - contains pipeline definitions | ||
- `scripts/common` - scripts that get `source`d by other scripts to set environment variables or import shared functions | ||
- `scripts/lifecycle` - general scripts for tasks that run before or after individual steps or the entire build | ||
- `scripts/steps` - scripts that define something that will run for a step defined in a pipeline | ||
- `scripts/*` - all other scripts are building blocks that make up the tasks in pipelines. They may be run by other scripts, but should not be `source`d |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,79 @@ | ||
{ | ||
"gcp": { | ||
"project": "elastic-kibana-ci", | ||
"zones": ["us-central1-a", "us-central1-b", "us-central1-c", "us-central1-f"], | ||
"serviceAccount": "elastic-buildkite-agent@elastic-kibana-ci.iam.gserviceaccount.com", | ||
"imageFamily": "kb-ubuntu", | ||
"subnetwork": "buildkite", | ||
"disableExternalIp": true, | ||
"diskType": "pd-ssd", | ||
"diskSizeGb": 75, | ||
"overprovision": 0, | ||
"minimumAgents": 0, | ||
"maximumAgents": 50, | ||
"gracefulStopAfterMins": 360, | ||
"hardStopAfterMins": 540, | ||
"idleTimeoutMins": 10, | ||
"exitAfterOneJob": false, | ||
|
||
"agents": [ | ||
{ | ||
"queue": "default", | ||
"name": "kb-default", | ||
"minimumAgents": 1, | ||
"maximumAgents": 100, | ||
"idleTimeoutMins": 60, | ||
"machineType": "e2-small" | ||
}, | ||
{ | ||
"queue": "c2-8", | ||
"name": "kb-c2-8", | ||
"machineType": "c2-standard-8", | ||
"localSsds": 1 | ||
}, | ||
{ | ||
"queue": "c2-4", | ||
"name": "kb-c2-4", | ||
"machineType": "c2-standard-4", | ||
"localSsds": 1 | ||
}, | ||
{ | ||
"queue": "jest", | ||
"name": "kb-jest", | ||
"machineType": "n2-standard-2", | ||
"diskSizeGb": 128 | ||
}, | ||
{ | ||
"queue": "ci-group", | ||
"name": "kb-cigroup", | ||
"machineType": "n2-standard-8", | ||
"diskSizeGb": 256 | ||
}, | ||
{ | ||
"queue": "ci-group-4", | ||
"name": "kb-cigroup-4", | ||
"machineType": "n2-standard-4", | ||
"diskSizeGb": 128 | ||
}, | ||
{ | ||
"queue": "ci-group-4d", | ||
"name": "kb-cigroup-4d", | ||
"machineType": "n2d-standard-4", | ||
"diskSizeGb": 128 | ||
}, | ||
{ | ||
"queue": "ci-group-6", | ||
"name": "kb-cigroup-6", | ||
"machineType": "n2-custom-6-16384", | ||
"diskSizeGb": 128 | ||
}, | ||
{ | ||
"queue": "packer", | ||
"name": "kb-packer", | ||
"serviceAccount": "buildkite-packer-agent@elastic-kibana-ci.iam.gserviceaccount.com", | ||
"maximumAgents": 10, | ||
"machineType": "e2-small" | ||
} | ||
] | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
#!/usr/bin/env bash | ||
|
||
source .buildkite/scripts/lifecycle/post_command.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
#!/usr/bin/env bash | ||
|
||
source .buildkite/scripts/lifecycle/pre_command.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
env: | ||
GITHUB_COMMIT_STATUS_ENABLED: 'true' | ||
GITHUB_COMMIT_STATUS_CONTEXT: 'buildkite/on-merge' | ||
steps: | ||
- command: .buildkite/scripts/lifecycle/pre_build.sh | ||
label: Pre-Build | ||
|
||
- wait | ||
|
||
- command: .buildkite/scripts/steps/on_merge_build_and_metrics.sh | ||
label: Default Build and Metrics | ||
agents: | ||
queue: c2-8 | ||
|
||
- command: .buildkite/scripts/steps/on_merge_ts_refs_api_docs.sh | ||
label: Build TS Refs and Check Public API Docs | ||
agents: | ||
queue: c2-4 | ||
|
||
- wait: ~ | ||
continue_on_failure: true | ||
|
||
- command: .buildkite/scripts/lifecycle/post_build.sh | ||
label: Post-Build |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
#!/usr/bin/env bash | ||
|
||
set -euo pipefail | ||
|
||
source .buildkite/scripts/common/util.sh | ||
|
||
echo "--- yarn install and bootstrap" | ||
yarn kbn bootstrap --verbose | ||
|
||
### | ||
### upload ts-refs-cache artifacts as quickly as possible so they are available for download | ||
### | ||
if [[ "${BUILD_TS_REFS_CACHE_CAPTURE:-}" == "true" ]]; then | ||
echo "--- Upload ts-refs-cache" | ||
cd "$KIBANA_DIR/target/ts_refs_cache" | ||
gsutil cp "*.zip" 'gs://kibana-ci-ts-refs-cache/' | ||
cd "$KIBANA_DIR" | ||
fi | ||
|
||
if [[ "$DISABLE_BOOTSTRAP_VALIDATION" != "true" ]]; then | ||
verify_no_git_changes 'yarn kbn bootstrap' | ||
fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
#!/usr/bin/env bash | ||
|
||
set -euo pipefail | ||
|
||
export KBN_NP_PLUGINS_BUILT=true | ||
|
||
echo "--- Build Kibana Distribution" | ||
node scripts/build --debug --no-oss | ||
|
||
echo "--- Archive Kibana Distribution" | ||
linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')" | ||
installDir="$KIBANA_DIR/install/kibana" | ||
mkdir -p "$installDir" | ||
tar -xzf "$linuxBuild" -C "$installDir" --strip=1 | ||
mkdir -p "$KIBANA_BUILD_LOCATION" | ||
cp -pR install/kibana/. "$KIBANA_BUILD_LOCATION/" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,32 @@ | ||
#!/usr/bin/env bash | ||
|
||
set -euo pipefail | ||
|
||
echo "--- Build Platform Plugins" | ||
node scripts/build_kibana_platform_plugins \ | ||
--scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \ | ||
--scan-dir "$KIBANA_DIR/test/interpreter_functional/plugins" \ | ||
--scan-dir "$KIBANA_DIR/test/common/fixtures/plugins" \ | ||
--scan-dir "$KIBANA_DIR/examples" \ | ||
--scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \ | ||
--scan-dir "$KIBANA_DIR/test/common/fixtures/plugins" \ | ||
--scan-dir "$XPACK_DIR/test/plugin_functional/plugins" \ | ||
--scan-dir "$XPACK_DIR/test/functional_with_es_ssl/fixtures/plugins" \ | ||
--scan-dir "$XPACK_DIR/test/alerting_api_integration/plugins" \ | ||
--scan-dir "$XPACK_DIR/test/plugin_api_integration/plugins" \ | ||
--scan-dir "$XPACK_DIR/test/plugin_api_perf/plugins" \ | ||
--scan-dir "$XPACK_DIR/test/licensing_plugin/plugins" \ | ||
--scan-dir "$XPACK_DIR/test/usage_collection/plugins" \ | ||
--scan-dir "$XPACK_DIR/test/security_functional/fixtures/common" \ | ||
--scan-dir "$XPACK_DIR/examples" \ | ||
--verbose | ||
|
||
echo "--- Archive built plugins" | ||
shopt -s globstar | ||
tar -zcf \ | ||
target/kibana-default-plugins.tar.gz \ | ||
x-pack/plugins/**/target/public \ | ||
x-pack/test/**/target/public \ | ||
examples/**/target/public \ | ||
x-pack/examples/**/target/public \ | ||
test/**/target/public |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,72 @@ | ||
#!/usr/bin/env bash | ||
|
||
export CI=true | ||
|
||
KIBANA_DIR=$(pwd) | ||
export KIBANA_DIR | ||
export XPACK_DIR="$KIBANA_DIR/x-pack" | ||
|
||
export CACHE_DIR="$HOME/.kibana" | ||
PARENT_DIR="$(cd "$KIBANA_DIR/.."; pwd)" | ||
export PARENT_DIR | ||
export WORKSPACE="${WORKSPACE:-$PARENT_DIR}" | ||
|
||
KIBANA_PKG_BRANCH="$(jq -r .branch "$KIBANA_DIR/package.json")" | ||
export KIBANA_PKG_BRANCH | ||
export KIBANA_BASE_BRANCH="$KIBANA_PKG_BRANCH" | ||
|
||
export GECKODRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache" | ||
export CHROMEDRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache" | ||
export RE2_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache" | ||
export CYPRESS_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/cypress" | ||
|
||
export NODE_OPTIONS="--max-old-space-size=4096" | ||
|
||
export FORCE_COLOR=1 | ||
export TEST_BROWSER_HEADLESS=1 | ||
|
||
export ELASTIC_APM_ENVIRONMENT=ci | ||
export ELASTIC_APM_TRANSACTION_SAMPLE_RATE=0.1 | ||
|
||
CI_REPORTING_ENABLED=false # TODO enable when ready, only controls checks reporter and APM | ||
|
||
if is_pr; then | ||
export ELASTIC_APM_ACTIVE=false | ||
export CHECKS_REPORTER_ACTIVE="${CI_REPORTING_ENABLED-}" | ||
|
||
# These can be removed once we're not supporting Jenkins and Buildkite at the same time | ||
# These are primarily used by github checks reporter and can be configured via /github_checks_api.json | ||
export ghprbGhRepository="elastic/kibana" | ||
export ghprbActualCommit="$BUILDKITE_COMMIT" | ||
export BUILD_URL="$BUILDKITE_BUILD_URL" | ||
|
||
# set_git_merge_base # TODO for PRs | ||
else | ||
export ELASTIC_APM_ACTIVE="${CI_REPORTING_ENABLED-}" | ||
export CHECKS_REPORTER_ACTIVE=false | ||
fi | ||
|
||
export FLEET_PACKAGE_REGISTRY_PORT=6104 | ||
export TEST_CORS_SERVER_PORT=6105 | ||
|
||
export DETECT_CHROMEDRIVER_VERSION=true | ||
export CHROMEDRIVER_FORCE_DOWNLOAD=true | ||
|
||
export GCS_UPLOAD_PREFIX=FAKE_UPLOAD_PREFIX # TODO remove the need for this | ||
|
||
export KIBANA_BUILD_LOCATION="$WORKSPACE/kibana-build-xpack" | ||
|
||
if [[ "${BUILD_TS_REFS_CACHE_ENABLE:-}" != "true" ]]; then | ||
export BUILD_TS_REFS_CACHE_ENABLE=false | ||
fi | ||
|
||
export BUILD_TS_REFS_DISABLE=true | ||
export DISABLE_BOOTSTRAP_VALIDATION=true | ||
|
||
export TEST_KIBANA_HOST=localhost | ||
export TEST_KIBANA_PORT=6101 | ||
export TEST_KIBANA_URL="http://elastic:changeme@localhost:6101" | ||
export TEST_ES_URL="http://elastic:changeme@localhost:6102" | ||
export TEST_ES_TRANSPORT_PORT=6103 | ||
export TEST_CORS_SERVER_PORT=6106 | ||
export ALERTING_PROXY_PORT=6105 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
#!/usr/bin/env bash | ||
|
||
KIBANA_BUILDBUDDY_CI_API_KEY=$(vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key) | ||
export KIBANA_BUILDBUDDY_CI_API_KEY | ||
|
||
cp "$KIBANA_DIR/src/dev/ci_setup/.bazelrc-ci" "$HOME/.bazelrc" | ||
|
||
### | ||
### append auth token to buildbuddy into "$HOME/.bazelrc"; | ||
### | ||
echo "# Appended by .buildkite/scripts/setup_bazel.sh" >> "$HOME/.bazelrc" | ||
echo "build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY" >> "$HOME/.bazelrc" | ||
|
||
### | ||
### remove write permissions on buildbuddy remote cache for prs | ||
### | ||
if [[ "${BUILDKITE_PULL_REQUEST:-}" && "$BUILDKITE_PULL_REQUEST" != "false" ]] ; then | ||
{ | ||
echo "# Uploads logs & artifacts without writing to cache" | ||
echo "build --noremote_upload_local_results" | ||
} >> "$HOME/.bazelrc" | ||
fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,46 @@ | ||
#!/usr/bin/env bash | ||
|
||
echo "--- Setup Node" | ||
|
||
NODE_VERSION="$(cat "$KIBANA_DIR/.node-version")" | ||
export NODE_VERSION | ||
export NODE_DIR="$CACHE_DIR/node/$NODE_VERSION" | ||
export NODE_BIN_DIR="$NODE_DIR/bin" | ||
export YARN_OFFLINE_CACHE="$CACHE_DIR/yarn-offline-cache" | ||
|
||
if [[ ! -d "$NODE_DIR" ]]; then | ||
hostArch="$(command uname -m)" | ||
case "${hostArch}" in | ||
x86_64 | amd64) nodeArch="x64" ;; | ||
aarch64) nodeArch="arm64" ;; | ||
*) nodeArch="${hostArch}" ;; | ||
esac | ||
|
||
nodeUrl="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$nodeArch.tar.gz" | ||
|
||
echo "node.js v$NODE_VERSION not found at $NODE_DIR, downloading from $nodeUrl" | ||
|
||
mkdir -p "$NODE_DIR" | ||
curl --silent -L "$nodeUrl" | tar -xz -C "$NODE_DIR" --strip-components=1 | ||
else | ||
echo "node.js v$NODE_VERSION already installed to $NODE_DIR, re-using" | ||
ls -alh "$NODE_BIN_DIR" | ||
fi | ||
|
||
export PATH="$NODE_BIN_DIR:$PATH" | ||
|
||
|
||
echo "--- Setup Yarn" | ||
|
||
YARN_VERSION=$(node -e "console.log(String(require('./package.json').engines.yarn || '').replace(/^[^\d]+/,''))") | ||
export YARN_VERSION | ||
|
||
if [[ ! $(which yarn) || $(yarn --version) != "$YARN_VERSION" ]]; then | ||
npm install -g "yarn@^${YARN_VERSION}" | ||
fi | ||
|
||
yarn config set yarn-offline-mirror "$YARN_OFFLINE_CACHE" | ||
|
||
YARN_GLOBAL_BIN=$(yarn global bin) | ||
export YARN_GLOBAL_BIN | ||
export PATH="$PATH:$YARN_GLOBAL_BIN" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,51 @@ | ||
#!/usr/bin/env bash | ||
|
||
checks-reporter-with-killswitch() { | ||
if [ "$CHECKS_REPORTER_ACTIVE" == "true" ] ; then | ||
yarn run github-checks-reporter "$@" | ||
else | ||
arguments=("$@"); | ||
"${arguments[@]:1}"; | ||
fi | ||
} | ||
|
||
is_pr() { | ||
[[ "${GITHUB_PR_NUMBER-}" ]] && return | ||
false | ||
} | ||
|
||
verify_no_git_changes() { | ||
RED='\033[0;31m' | ||
C_RESET='\033[0m' # Reset color | ||
|
||
GIT_CHANGES="$(git ls-files --modified)" | ||
if [ "$GIT_CHANGES" ]; then | ||
echo -e "\n${RED}ERROR: '$1' caused changes to the following files:${C_RESET}\n" | ||
echo -e "$GIT_CHANGES\n" | ||
exit 1 | ||
fi | ||
} | ||
|
||
# docker_run can be used in place of `docker run` | ||
# it automatically passes along all of Buildkite's tracked environment variables, and mounts the buildkite-agent in the running container | ||
docker_run() { | ||
args=() | ||
|
||
if [[ -n "${BUILDKITE_ENV_FILE:-}" ]] ; then | ||
# Read in the env file and convert to --env params for docker | ||
# This is because --env-file doesn't support newlines or quotes per https://docs.docker.com/compose/env-file/#syntax-rules | ||
while read -r var; do | ||
args+=( --env "${var%%=*}" ) | ||
done < "$BUILDKITE_ENV_FILE" | ||
fi | ||
|
||
BUILDKITE_AGENT_BINARY_PATH=$(command -v buildkite-agent) | ||
args+=( | ||
"--env" "BUILDKITE_JOB_ID" | ||
"--env" "BUILDKITE_BUILD_ID" | ||
"--env" "BUILDKITE_AGENT_ACCESS_TOKEN" | ||
"--volume" "$BUILDKITE_AGENT_BINARY_PATH:/usr/bin/buildkite-agent" | ||
) | ||
|
||
docker run "${args[@]}" "$@" | ||
} |
Oops, something went wrong.