Skip to content

Commit

Permalink
Merge branch 'master' into build/rm-oss
Browse files Browse the repository at this point in the history
  • Loading branch information
kibanamachine authored Jun 23, 2021
2 parents b982549 + 5244019 commit cd82cb1
Showing 7,963 changed files with 304,779 additions and 151,842 deletions.
The diff you're trying to view is too large. We only load the first 3000 changed files.
10 changes: 10 additions & 0 deletions .buildkite/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# Kibana / Buildkite

## Directory Structure

- `hooks` - special directory used by Buildkite agents for [hooks](https://buildkite.com/docs/agent/v3/hooks)
- `pipelines` - contains pipeline definitions
- `scripts/common` - scripts that get `source`d by other scripts to set environment variables or import shared functions
- `scripts/lifecycle` - general scripts for tasks that run before or after individual steps or the entire build
- `scripts/steps` - scripts that define something that will run for a step defined in a pipeline
- `scripts/*` - all other scripts are building blocks that make up the tasks in pipelines. They may be run by other scripts, but should not be `source`d
79 changes: 79 additions & 0 deletions .buildkite/agents.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
{
"gcp": {
"project": "elastic-kibana-ci",
"zones": ["us-central1-a", "us-central1-b", "us-central1-c", "us-central1-f"],
"serviceAccount": "elastic-buildkite-agent@elastic-kibana-ci.iam.gserviceaccount.com",
"imageFamily": "kb-ubuntu",
"subnetwork": "buildkite",
"disableExternalIp": true,
"diskType": "pd-ssd",
"diskSizeGb": 75,
"overprovision": 0,
"minimumAgents": 0,
"maximumAgents": 50,
"gracefulStopAfterMins": 360,
"hardStopAfterMins": 540,
"idleTimeoutMins": 10,
"exitAfterOneJob": false,

"agents": [
{
"queue": "default",
"name": "kb-default",
"minimumAgents": 1,
"maximumAgents": 100,
"idleTimeoutMins": 60,
"machineType": "e2-small"
},
{
"queue": "c2-8",
"name": "kb-c2-8",
"machineType": "c2-standard-8",
"localSsds": 1
},
{
"queue": "c2-4",
"name": "kb-c2-4",
"machineType": "c2-standard-4",
"localSsds": 1
},
{
"queue": "jest",
"name": "kb-jest",
"machineType": "n2-standard-2",
"diskSizeGb": 128
},
{
"queue": "ci-group",
"name": "kb-cigroup",
"machineType": "n2-standard-8",
"diskSizeGb": 256
},
{
"queue": "ci-group-4",
"name": "kb-cigroup-4",
"machineType": "n2-standard-4",
"diskSizeGb": 128
},
{
"queue": "ci-group-4d",
"name": "kb-cigroup-4d",
"machineType": "n2d-standard-4",
"diskSizeGb": 128
},
{
"queue": "ci-group-6",
"name": "kb-cigroup-6",
"machineType": "n2-custom-6-16384",
"diskSizeGb": 128
},
{
"queue": "packer",
"name": "kb-packer",
"serviceAccount": "buildkite-packer-agent@elastic-kibana-ci.iam.gserviceaccount.com",
"maximumAgents": 10,
"machineType": "e2-small"
}
]
}
}
3 changes: 3 additions & 0 deletions .buildkite/hooks/post-command
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
#!/usr/bin/env bash

source .buildkite/scripts/lifecycle/post_command.sh
3 changes: 3 additions & 0 deletions .buildkite/hooks/pre-command
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
#!/usr/bin/env bash

source .buildkite/scripts/lifecycle/pre_command.sh
24 changes: 24 additions & 0 deletions .buildkite/pipelines/on_merge.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
env:
GITHUB_COMMIT_STATUS_ENABLED: 'true'
GITHUB_COMMIT_STATUS_CONTEXT: 'buildkite/on-merge'
steps:
- command: .buildkite/scripts/lifecycle/pre_build.sh
label: Pre-Build

- wait

- command: .buildkite/scripts/steps/on_merge_build_and_metrics.sh
label: Default Build and Metrics
agents:
queue: c2-8

- command: .buildkite/scripts/steps/on_merge_ts_refs_api_docs.sh
label: Build TS Refs and Check Public API Docs
agents:
queue: c2-4

- wait: ~
continue_on_failure: true

- command: .buildkite/scripts/lifecycle/post_build.sh
label: Post-Build
22 changes: 22 additions & 0 deletions .buildkite/scripts/bootstrap.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
#!/usr/bin/env bash

set -euo pipefail

source .buildkite/scripts/common/util.sh

echo "--- yarn install and bootstrap"
yarn kbn bootstrap --verbose

###
### upload ts-refs-cache artifacts as quickly as possible so they are available for download
###
if [[ "${BUILD_TS_REFS_CACHE_CAPTURE:-}" == "true" ]]; then
echo "--- Upload ts-refs-cache"
cd "$KIBANA_DIR/target/ts_refs_cache"
gsutil cp "*.zip" 'gs://kibana-ci-ts-refs-cache/'
cd "$KIBANA_DIR"
fi

if [[ "$DISABLE_BOOTSTRAP_VALIDATION" != "true" ]]; then
verify_no_git_changes 'yarn kbn bootstrap'
fi
16 changes: 16 additions & 0 deletions .buildkite/scripts/build_kibana.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
#!/usr/bin/env bash

set -euo pipefail

export KBN_NP_PLUGINS_BUILT=true

echo "--- Build Kibana Distribution"
node scripts/build --debug --no-oss

echo "--- Archive Kibana Distribution"
linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
installDir="$KIBANA_DIR/install/kibana"
mkdir -p "$installDir"
tar -xzf "$linuxBuild" -C "$installDir" --strip=1
mkdir -p "$KIBANA_BUILD_LOCATION"
cp -pR install/kibana/. "$KIBANA_BUILD_LOCATION/"
32 changes: 32 additions & 0 deletions .buildkite/scripts/build_kibana_plugins.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
#!/usr/bin/env bash

set -euo pipefail

echo "--- Build Platform Plugins"
node scripts/build_kibana_platform_plugins \
--scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \
--scan-dir "$KIBANA_DIR/test/interpreter_functional/plugins" \
--scan-dir "$KIBANA_DIR/test/common/fixtures/plugins" \
--scan-dir "$KIBANA_DIR/examples" \
--scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \
--scan-dir "$KIBANA_DIR/test/common/fixtures/plugins" \
--scan-dir "$XPACK_DIR/test/plugin_functional/plugins" \
--scan-dir "$XPACK_DIR/test/functional_with_es_ssl/fixtures/plugins" \
--scan-dir "$XPACK_DIR/test/alerting_api_integration/plugins" \
--scan-dir "$XPACK_DIR/test/plugin_api_integration/plugins" \
--scan-dir "$XPACK_DIR/test/plugin_api_perf/plugins" \
--scan-dir "$XPACK_DIR/test/licensing_plugin/plugins" \
--scan-dir "$XPACK_DIR/test/usage_collection/plugins" \
--scan-dir "$XPACK_DIR/test/security_functional/fixtures/common" \
--scan-dir "$XPACK_DIR/examples" \
--verbose

echo "--- Archive built plugins"
shopt -s globstar
tar -zcf \
target/kibana-default-plugins.tar.gz \
x-pack/plugins/**/target/public \
x-pack/test/**/target/public \
examples/**/target/public \
x-pack/examples/**/target/public \
test/**/target/public
72 changes: 72 additions & 0 deletions .buildkite/scripts/common/env.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
#!/usr/bin/env bash

export CI=true

KIBANA_DIR=$(pwd)
export KIBANA_DIR
export XPACK_DIR="$KIBANA_DIR/x-pack"

export CACHE_DIR="$HOME/.kibana"
PARENT_DIR="$(cd "$KIBANA_DIR/.."; pwd)"
export PARENT_DIR
export WORKSPACE="${WORKSPACE:-$PARENT_DIR}"

KIBANA_PKG_BRANCH="$(jq -r .branch "$KIBANA_DIR/package.json")"
export KIBANA_PKG_BRANCH
export KIBANA_BASE_BRANCH="$KIBANA_PKG_BRANCH"

export GECKODRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
export CHROMEDRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
export RE2_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
export CYPRESS_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/cypress"

export NODE_OPTIONS="--max-old-space-size=4096"

export FORCE_COLOR=1
export TEST_BROWSER_HEADLESS=1

export ELASTIC_APM_ENVIRONMENT=ci
export ELASTIC_APM_TRANSACTION_SAMPLE_RATE=0.1

CI_REPORTING_ENABLED=false # TODO enable when ready, only controls checks reporter and APM

if is_pr; then
export ELASTIC_APM_ACTIVE=false
export CHECKS_REPORTER_ACTIVE="${CI_REPORTING_ENABLED-}"

# These can be removed once we're not supporting Jenkins and Buildkite at the same time
# These are primarily used by github checks reporter and can be configured via /github_checks_api.json
export ghprbGhRepository="elastic/kibana"
export ghprbActualCommit="$BUILDKITE_COMMIT"
export BUILD_URL="$BUILDKITE_BUILD_URL"

# set_git_merge_base # TODO for PRs
else
export ELASTIC_APM_ACTIVE="${CI_REPORTING_ENABLED-}"
export CHECKS_REPORTER_ACTIVE=false
fi

export FLEET_PACKAGE_REGISTRY_PORT=6104
export TEST_CORS_SERVER_PORT=6105

export DETECT_CHROMEDRIVER_VERSION=true
export CHROMEDRIVER_FORCE_DOWNLOAD=true

export GCS_UPLOAD_PREFIX=FAKE_UPLOAD_PREFIX # TODO remove the need for this

export KIBANA_BUILD_LOCATION="$WORKSPACE/kibana-build-xpack"

if [[ "${BUILD_TS_REFS_CACHE_ENABLE:-}" != "true" ]]; then
export BUILD_TS_REFS_CACHE_ENABLE=false
fi

export BUILD_TS_REFS_DISABLE=true
export DISABLE_BOOTSTRAP_VALIDATION=true

export TEST_KIBANA_HOST=localhost
export TEST_KIBANA_PORT=6101
export TEST_KIBANA_URL="http://elastic:changeme@localhost:6101"
export TEST_ES_URL="http://elastic:changeme@localhost:6102"
export TEST_ES_TRANSPORT_PORT=6103
export TEST_CORS_SERVER_PORT=6106
export ALERTING_PROXY_PORT=6105
22 changes: 22 additions & 0 deletions .buildkite/scripts/common/setup_bazel.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
#!/usr/bin/env bash

KIBANA_BUILDBUDDY_CI_API_KEY=$(vault read -field=value secret/kibana-issues/dev/kibana-buildbuddy-ci-api-key)
export KIBANA_BUILDBUDDY_CI_API_KEY

cp "$KIBANA_DIR/src/dev/ci_setup/.bazelrc-ci" "$HOME/.bazelrc"

###
### append auth token to buildbuddy into "$HOME/.bazelrc";
###
echo "# Appended by .buildkite/scripts/setup_bazel.sh" >> "$HOME/.bazelrc"
echo "build --remote_header=x-buildbuddy-api-key=$KIBANA_BUILDBUDDY_CI_API_KEY" >> "$HOME/.bazelrc"

###
### remove write permissions on buildbuddy remote cache for prs
###
if [[ "${BUILDKITE_PULL_REQUEST:-}" && "$BUILDKITE_PULL_REQUEST" != "false" ]] ; then
{
echo "# Uploads logs & artifacts without writing to cache"
echo "build --noremote_upload_local_results"
} >> "$HOME/.bazelrc"
fi
46 changes: 46 additions & 0 deletions .buildkite/scripts/common/setup_node.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
#!/usr/bin/env bash

echo "--- Setup Node"

NODE_VERSION="$(cat "$KIBANA_DIR/.node-version")"
export NODE_VERSION
export NODE_DIR="$CACHE_DIR/node/$NODE_VERSION"
export NODE_BIN_DIR="$NODE_DIR/bin"
export YARN_OFFLINE_CACHE="$CACHE_DIR/yarn-offline-cache"

if [[ ! -d "$NODE_DIR" ]]; then
hostArch="$(command uname -m)"
case "${hostArch}" in
x86_64 | amd64) nodeArch="x64" ;;
aarch64) nodeArch="arm64" ;;
*) nodeArch="${hostArch}" ;;
esac

nodeUrl="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$nodeArch.tar.gz"

echo "node.js v$NODE_VERSION not found at $NODE_DIR, downloading from $nodeUrl"

mkdir -p "$NODE_DIR"
curl --silent -L "$nodeUrl" | tar -xz -C "$NODE_DIR" --strip-components=1
else
echo "node.js v$NODE_VERSION already installed to $NODE_DIR, re-using"
ls -alh "$NODE_BIN_DIR"
fi

export PATH="$NODE_BIN_DIR:$PATH"


echo "--- Setup Yarn"

YARN_VERSION=$(node -e "console.log(String(require('./package.json').engines.yarn || '').replace(/^[^\d]+/,''))")
export YARN_VERSION

if [[ ! $(which yarn) || $(yarn --version) != "$YARN_VERSION" ]]; then
npm install -g "yarn@^${YARN_VERSION}"
fi

yarn config set yarn-offline-mirror "$YARN_OFFLINE_CACHE"

YARN_GLOBAL_BIN=$(yarn global bin)
export YARN_GLOBAL_BIN
export PATH="$PATH:$YARN_GLOBAL_BIN"
51 changes: 51 additions & 0 deletions .buildkite/scripts/common/util.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
#!/usr/bin/env bash

checks-reporter-with-killswitch() {
if [ "$CHECKS_REPORTER_ACTIVE" == "true" ] ; then
yarn run github-checks-reporter "$@"
else
arguments=("$@");
"${arguments[@]:1}";
fi
}

is_pr() {
[[ "${GITHUB_PR_NUMBER-}" ]] && return
false
}

verify_no_git_changes() {
RED='\033[0;31m'
C_RESET='\033[0m' # Reset color

GIT_CHANGES="$(git ls-files --modified)"
if [ "$GIT_CHANGES" ]; then
echo -e "\n${RED}ERROR: '$1' caused changes to the following files:${C_RESET}\n"
echo -e "$GIT_CHANGES\n"
exit 1
fi
}

# docker_run can be used in place of `docker run`
# it automatically passes along all of Buildkite's tracked environment variables, and mounts the buildkite-agent in the running container
docker_run() {
args=()

if [[ -n "${BUILDKITE_ENV_FILE:-}" ]] ; then
# Read in the env file and convert to --env params for docker
# This is because --env-file doesn't support newlines or quotes per https://docs.docker.com/compose/env-file/#syntax-rules
while read -r var; do
args+=( --env "${var%%=*}" )
done < "$BUILDKITE_ENV_FILE"
fi

BUILDKITE_AGENT_BINARY_PATH=$(command -v buildkite-agent)
args+=(
"--env" "BUILDKITE_JOB_ID"
"--env" "BUILDKITE_BUILD_ID"
"--env" "BUILDKITE_AGENT_ACCESS_TOKEN"
"--volume" "$BUILDKITE_AGENT_BINARY_PATH:/usr/bin/buildkite-agent"
)

docker run "${args[@]}" "$@"
}
Loading

0 comments on commit cd82cb1

Please sign in to comment.