diff --git a/.buildkite/pipelines/pull_request/base.yml b/.buildkite/pipelines/pull_request/base.yml index 158c22c0bb0c5..9c2527fcdd413 100644 --- a/.buildkite/pipelines/pull_request/base.yml +++ b/.buildkite/pipelines/pull_request/base.yml @@ -136,12 +136,13 @@ steps: - exit_status: '-1' limit: 3 - - command: .buildkite/scripts/steps/check_types.sh - label: 'Check Types' - agents: - queue: n2-16-spot - timeout_in_minutes: 60 - retry: - automatic: - - exit_status: '-1' - limit: 3 +# TODO: Enable in #166813 after fixing types +# - command: .buildkite/scripts/steps/check_types.sh +# label: 'Check Types' +# agents: +# queue: n2-16-spot +# timeout_in_minutes: 60 +# retry: +# automatic: +# - exit_status: '-1' +# limit: 3 diff --git a/.buildkite/pipelines/pull_request/type_check.yml b/.buildkite/pipelines/pull_request/type_check.yml new file mode 100644 index 0000000000000..dc6f8471ff1b2 --- /dev/null +++ b/.buildkite/pipelines/pull_request/type_check.yml @@ -0,0 +1,10 @@ +steps: + - command: .buildkite/scripts/steps/check_types.sh + label: 'Check Types' + agents: + queue: n2-16-spot + timeout_in_minutes: 60 + retry: + automatic: + - exit_status: '-1' + limit: 3 diff --git a/.buildkite/pipelines/pull_request/type_check_selective.yml b/.buildkite/pipelines/pull_request/type_check_selective.yml new file mode 100644 index 0000000000000..7d01f128aac3c --- /dev/null +++ b/.buildkite/pipelines/pull_request/type_check_selective.yml @@ -0,0 +1,10 @@ +steps: + - command: .buildkite/scripts/steps/check_types_commits.sh + label: 'Check Types Commit Diff' + agents: + queue: n2-16-spot + timeout_in_minutes: 60 + retry: + automatic: + - exit_status: '-1' + limit: 3 diff --git a/.buildkite/scripts/pipelines/pull_request/pipeline.ts b/.buildkite/scripts/pipelines/pull_request/pipeline.ts index 80d1312af6e64..37bd89f2a75a7 100644 --- a/.buildkite/scripts/pipelines/pull_request/pipeline.ts +++ b/.buildkite/scripts/pipelines/pull_request/pipeline.ts @@ -59,6 +59,12 @@ const uploadPipeline = (pipelineContent: string | object) => { pipeline.push(getPipeline('.buildkite/pipelines/pull_request/kbn_handlebars.yml')); } + if (GITHUB_PR_LABELS.includes('ci:hard-typecheck')) { + pipeline.push(getPipeline('.buildkite/pipelines/pull_request/type_check.yml')); + } else { + pipeline.push(getPipeline('.buildkite/pipelines/pull_request/type_check_selective.yml')); + } + if ( (await doAnyChangesMatch([ /^src\/plugins\/controls/, diff --git a/.buildkite/scripts/steps/build_api_docs.sh b/.buildkite/scripts/steps/build_api_docs.sh index f86032c902d1a..ba1c2ec7ec94d 100755 --- a/.buildkite/scripts/steps/build_api_docs.sh +++ b/.buildkite/scripts/steps/build_api_docs.sh @@ -4,8 +4,9 @@ set -euo pipefail .buildkite/scripts/bootstrap.sh -echo "--- Run scripts/type_check to ensure that all build available" -node scripts/type_check +# TODO: Enable in #166813 after fixing types +# echo "--- Run scripts/type_check to ensure that all build available" +# node scripts/type_check echo "--- Build API Docs" node --max-old-space-size=12000 scripts/build_api_docs diff --git a/.buildkite/scripts/steps/check_types_commits.sh b/.buildkite/scripts/steps/check_types_commits.sh new file mode 100644 index 0000000000000..0b76d3d8e089a --- /dev/null +++ b/.buildkite/scripts/steps/check_types_commits.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash + +set -euo pipefail + + +if [[ "${CI-}" == "true" ]]; then + .buildkite/scripts/bootstrap.sh + + sha1="${GITHUB_PR_TARGET_BRANCH-}" + sha2="${GITHUB_PR_TRIGGERED_SHA-}" +else + # Script take between 0 and 2 arguments representing two commit SHA's: + # If 0, it will diff HEAD and HEAD^ + # If 1 (SHA1), it will diff SHA1 and SHA1^ + # If 2 (SHA1, SHA2), it will diff SHA1 and SHA2 + sha1="${1-HEAD}" + sha2="${2-$sha1^}" +fi + +uniq_dirs=() +uniq_tsconfigs=() + +echo "Detecting files changed between $sha1 and $sha2..." + +files=($(git diff --name-only $sha1 $sha2)) + +add_dir () { + new_dir=$1 + + if [ ${#uniq_dirs[@]} -gt 0 ]; then + for dir in "${uniq_dirs[@]}" + do + if [[ "$new_dir" == "$dir" ]]; then + return + fi + done + fi + + uniq_dirs+=($new_dir) +} + +add_tsconfig () { + new_tsconfig=$1 + + if [ ${#uniq_tsconfigs[@]} -gt 0 ]; then + for tsconfig in "${uniq_tsconfigs[@]}" + do + if [[ "$new_tsconfig" == "$tsconfig" ]]; then + return + fi + done + fi + + echo " $new_tsconfig" + uniq_tsconfigs+=($new_tsconfig) +} + +contains_tsconfig () { + dir=$1 + tsconfig="$dir/tsconfig.json" + if [ -f "$tsconfig" ]; then + true + else + false + fi +} + +find_tsconfig () { + dir=$1 + + if [[ "$dir" == "." ]]; then + return + fi + + if contains_tsconfig $dir; then + add_tsconfig "$dir/tsconfig.json" + else + find_tsconfig $(dirname -- "$dir") + fi +} + +if [ ${#files[@]} -eq 0 ]; then + echo "No files found!" + exit +fi + +for file in "${files[@]}" +do + dir=$(dirname -- "$file") + + # Ignore buildkite dir because it traverses many kbn packages and emits incorrect results + if [[ "$dir" != .buildkite* ]]; then + add_dir $dir + fi +done + +echo "Looking for related tsconfig.json files..." + +for dir in "${uniq_dirs[@]}" +do + find_tsconfig $dir +done + +if [ ${#uniq_tsconfigs[@]} -eq 0 ]; then + echo "No tsconfig.json files found for changes in $sha1 $sha2" + exit +fi + +echo "Running scripts/type_check for each found tsconfig.json file..." + +for tsconfig in "${uniq_tsconfigs[@]}" +do + node scripts/type_check --project $tsconfig +done \ No newline at end of file diff --git a/package.json b/package.json index 54123db6f2eb7..2749e9b2143c4 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,8 @@ "test:ftr:runner": "node scripts/functional_test_runner", "test:ftr:server": "node scripts/functional_tests_server", "test:jest": "node scripts/jest", - "test:jest_integration": "node scripts/jest_integration" + "test:jest_integration": "node scripts/jest_integration", + "test:type_check": "node scripts/type_check" }, "repository": { "type": "git", diff --git a/packages/kbn-dev-proc-runner/src/proc.ts b/packages/kbn-dev-proc-runner/src/proc.ts index d30a893ae4c75..472d43d66e70d 100644 --- a/packages/kbn-dev-proc-runner/src/proc.ts +++ b/packages/kbn-dev-proc-runner/src/proc.ts @@ -13,7 +13,7 @@ import stripAnsi from 'strip-ansi'; import execa from 'execa'; import * as Rx from 'rxjs'; -import { tap, share, take, mergeMap, map, ignoreElements } from 'rxjs/operators'; +import { tap, share, take, mergeMap, map, ignoreElements, filter } from 'rxjs/operators'; import chalk from 'chalk'; import treeKill from 'tree-kill'; import { ToolingLog } from '@kbn/tooling-log'; @@ -87,13 +87,23 @@ export function startProc(name: string, options: ProcOptions, log: ToolingLog) { const outcome$: Rx.Observable = Rx.race( // observe first exit event - Rx.fromEvent<[number]>(childProcess, 'exit').pipe( + Rx.fromEvent<[number, string]>(childProcess, 'exit').pipe( + filter(([code]) => { + if (stopCalled) { + // when stop was already called, that's a graceful exit, let those events pass. + return true; + } else { + // filtering out further interruption events to prevent `take()` from closing the stream. + return code !== null; + } + }), take(1), map(([code]) => { if (stopCalled) { return null; } - // JVM exits with 143 on SIGTERM and 130 on SIGINT, dont' treat then as errors + + // JVM exits with 143 on SIGTERM and 130 on SIGINT, don't treat them as errors if (code > 0 && !(code === 143 || code === 130)) { throw createFailError(`[${name}] exited with code ${code}`, { exitCode: code, @@ -108,6 +118,12 @@ export function startProc(name: string, options: ProcOptions, log: ToolingLog) { Rx.fromEvent(childProcess, 'error').pipe( take(1), mergeMap((err) => Rx.throwError(err)) + ), + + // observe a promise rejection + Rx.from(childProcess).pipe( + take(1), + mergeMap((err) => Rx.throwError(err)) ) ).pipe(share()); @@ -132,7 +148,7 @@ export function startProc(name: string, options: ProcOptions, log: ToolingLog) { share() ); - const outcomePromise = Rx.merge(lines$.pipe(ignoreElements()), outcome$).toPromise(); + const outcomePromise = Rx.firstValueFrom(Rx.merge(lines$.pipe(ignoreElements()), outcome$)); async function stop(signal: NodeJS.Signals) { if (stopCalled) { diff --git a/packages/kbn-dev-proc-runner/src/proc_runner.ts b/packages/kbn-dev-proc-runner/src/proc_runner.ts index 1226cbeb3eef1..510a96dc043be 100644 --- a/packages/kbn-dev-proc-runner/src/proc_runner.ts +++ b/packages/kbn-dev-proc-runner/src/proc_runner.ts @@ -135,6 +135,9 @@ export class ProcRunner { // wait for process to complete await proc.outcomePromise; } + } catch (e) { + this.log.error(e); + throw e; } finally { // while the procRunner closes promises will resolve/reject because // processes and stopping, but consumers of run() shouldn't have to diff --git a/packages/kbn-ts-type-check-cli/run_type_check_cli.ts b/packages/kbn-ts-type-check-cli/run_type_check_cli.ts index 0295525578952..3f4385118c9c6 100644 --- a/packages/kbn-ts-type-check-cli/run_type_check_cli.ts +++ b/packages/kbn-ts-type-check-cli/run_type_check_cli.ts @@ -124,6 +124,9 @@ run( '--pretty', ...(flagsReader.boolean('verbose') ? ['--verbose'] : []), ], + env: { + NODE_OPTIONS: '--max-old-space-size=8192', + }, cwd: REPO_ROOT, wait: true, });