[CI] Add pipeline task queue framework and merge workers into one

.ci/Dockerfile

@@ -0,0 +1,35 @@
+ARG NODE_VERSION=10.21.0
+
+FROM node:${NODE_VERSION} AS base
+
+RUN apt-get update && \
+    apt-get -y install xvfb gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 \
+      libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 \
+      libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 \
+      libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 \
+      libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget openjdk-8-jre && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN curl -sSL https://dl.google.com/linux/linux_signing_key.pub | apt-key add - \
+  && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \
+  && apt-get update \
+  && apt-get install -y rsync jq bsdtar google-chrome-stable \
+  --no-install-recommends \
+  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+RUN LATEST_VAULT_RELEASE=$(curl -s https://api.github.com/repos/hashicorp/vault/tags | jq --raw-output .[0].name[1:]) \
+  && curl -L https://releases.hashicorp.com/vault/${LATEST_VAULT_RELEASE}/vault_${LATEST_VAULT_RELEASE}_linux_amd64.zip -o vault.zip \
+  && unzip vault.zip \
+  && rm vault.zip \
+  && chmod +x vault \
+  && mv vault /usr/local/bin/vault
+
+RUN groupadd -r kibana && useradd -r -g kibana kibana && mkdir /home/kibana && chown kibana:kibana /home/kibana
+
+COPY ./bash_standard_lib.sh /usr/local/bin/bash_standard_lib.sh
+RUN chmod +x /usr/local/bin/bash_standard_lib.sh
+
+COPY ./runbld /usr/local/bin/runbld
+RUN chmod +x /usr/local/bin/runbld
+
+USER kibana

.ci/bash_standard_lib.sh

@@ -0,0 +1,128 @@
+#! /usr/bin/env bash
+
+# Readonly vars
+readonly CACHE_DIR="${HOME}/.git-references"
+
+# Global vars
+declare -a cloned_git_repos
+
+#==============================================================================
+#
+# What: A library of standard Bash functions
+# Why: Simplify and improve the quality of bash scripts being produced.
+#
+#==============================================================================
+
+# cache_maven_deps function
+# -------------------------------------
+# Run a `mvnw compile` in order to populate the local
+# m2 repository.
+#
+# Expects a single param which is a project directory to change into
+# -------------------------------------
+cache_maven_deps() {
+  local project=$1
+
+  # Check that 'project' dir exists
+  if [[ ! -d $project ]]; then
+    printf 'Project dir %s doesn''t exist. Exiting...\n' $project >&2
+    exit 1
+  fi
+
+  # Switch directory, and run mvnw compile
+  pushd $project
+  ./mvnw compile
+  popd
+}
+
+# clean_git_repo_clones function
+# -------------------------------------
+# Clean up any Git repo clones performed by the 'clone_git_repo' function.
+# Should be called by the 'EXIT' trap.
+# -------------------------------------
+clean_git_repo_clones() {
+  printf 'Cleaning cloned git repos...\n'
+
+  # Loop over cloned repos and remove them...
+  for cloned_repo in "${cloned_git_repos[@]}"
+  do
+    printf 'Removing repo %s\n' "$cloned_repo"
+    rm -rf $cloned_repo
+  done
+}
+
+# clone_git_repo function
+# -------------------------------------
+# Clone a Git repo, using either a Git SSH source, or a repo short-name.
+# If a matching local reference repo is found, then that will be used as the `--reference` for the Git clone,
+# otherwise a full clone is undertaken.
+#
+# Expects a single param which is the repo name or repo SSH source
+# -------------------------------------
+clone_git_repo() {
+  local repo=$1
+  local repo_name
+  local repo_url
+
+  # Calculate the "humanish" part of the source repo
+  repo_name=$(echo "$repo" | sed -e 's|/$||' -e 's|:*/*\.git$||' -e 's|.*[/:]||g')
+	local mirror_dir="${CACHE_DIR}/${repo_name}.git"
+
+  # If $repo is a SSH repo source, use it as the repo_url.
+  # Otherwise, attempt to get the origin URL from the matching reference repo, failing if the reference repo is not present.
+  if [[ $repo == git* ]]; then
+    repo_url=$repo
+  else
+    if [[ -d $mirror_dir ]]; then
+      repo_url=$(cd "$mirror_dir" && git remote get-url origin)
+    else
+      printf 'Attempting to clone %s using the short-name, however no matching reference repo found. Exiting...\n' "$repo" >&2
+      exit 1
+    fi
+  fi
+
+  # Check if we have a reference repo clone for this repo,
+  # and use it for the clone if we do.
+  # Otherwise clone the repo directly.
+  if [[ -d $mirror_dir ]]; then
+    printf 'Cloning repo %s using reference repo...\n' "$repo"
+    git clone -q --reference "$mirror_dir" "$repo_url" || exit 1
+  else
+    printf 'Cloning repo %s directly...' "$repo"
+    git clone -q "$repo_url" || exit 1
+  fi
+
+  # Add repo to cloned repo array
+  cloned_git_repos+=("$repo")
+
+  # Clean up clone on exit
+  trap clean_git_repo_clones EXIT
+}
+
+# retry function
+# -------------------------------------
+# Retry a command for a specified number of times until the command exits successfully.
+# Retry wait period backs off exponentially after each retry.
+#
+# The first argument should be the number of retries.
+# Remainder is treated as the command to execute.
+# -------------------------------------
+retry() {
+  local retries=$1
+  shift
+
+  local count=0
+  until "$@"; do
+    exit=$?
+    wait=$((2 ** $count))
+    count=$(($count + 1))
+    if [ $count -lt $retries ]; then
+      printf "Retry %s/%s exited %s, retrying in %s seconds...\n" "$count" "$retries" "$exit" "$wait" >&2
+      sleep $wait
+    else
+      printf "Retry %s/%s exited %s, no more retries left.\n" "$count" "$retries" "$exit" >&2
+      return $exit
+    fi
+  done
+  return 0
+}

.ci/runbld_no_junit.yml

@@ -3,4 +3,4 @@
 profiles:
 - ".*": # Match any job
     tests:
-      junit-filename-pattern: "8d8bd494-d909-4e67-a052-7e8b5aaeb5e4" # A bogus path that should never exist
+      junit-filename-pattern: false

.gitignore

@@ -47,6 +47,7 @@ npm-debug.log*
 .tern-project
 .nyc_output
 .ci/pipeline-library/build/
+.ci/runbld
 .gradle
 
 # apm plugin

Jenkinsfile

@@ -8,49 +8,7 @@ kibanaPipeline(timeoutMinutes: 155, checkPrChanges: true, setCommitStatus: true)
     ciStats.trackBuild {
       catchError {
         retryable.enable()
-        parallel([
-          'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),
-          'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'),
-          'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
-            'oss-firefoxSmoke': kibanaPipeline.functionalTestProcess('kibana-firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh'),
-            'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1),
-            'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2),
-            'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3),
-            'oss-ciGroup4': kibanaPipeline.ossCiGroupProcess(4),
-            'oss-ciGroup5': kibanaPipeline.ossCiGroupProcess(5),
-            'oss-ciGroup6': kibanaPipeline.ossCiGroupProcess(6),
-            'oss-ciGroup7': kibanaPipeline.ossCiGroupProcess(7),
-            'oss-ciGroup8': kibanaPipeline.ossCiGroupProcess(8),
-            'oss-ciGroup9': kibanaPipeline.ossCiGroupProcess(9),
-            'oss-ciGroup10': kibanaPipeline.ossCiGroupProcess(10),
-            'oss-ciGroup11': kibanaPipeline.ossCiGroupProcess(11),
-            'oss-ciGroup12': kibanaPipeline.ossCiGroupProcess(12),
-            'oss-accessibility': kibanaPipeline.functionalTestProcess('kibana-accessibility', './test/scripts/jenkins_accessibility.sh'),
-            // 'oss-visualRegression': kibanaPipeline.functionalTestProcess('visualRegression', './test/scripts/jenkins_visual_regression.sh'),
-          ]),
-          'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
-            'xpack-firefoxSmoke': kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh'),
-            'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1),
-            'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2),
-            'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3),
-            'xpack-ciGroup4': kibanaPipeline.xpackCiGroupProcess(4),
-            'xpack-ciGroup5': kibanaPipeline.xpackCiGroupProcess(5),
-            'xpack-ciGroup6': kibanaPipeline.xpackCiGroupProcess(6),
-            'xpack-ciGroup7': kibanaPipeline.xpackCiGroupProcess(7),
-            'xpack-ciGroup8': kibanaPipeline.xpackCiGroupProcess(8),
-            'xpack-ciGroup9': kibanaPipeline.xpackCiGroupProcess(9),
-            'xpack-ciGroup10': kibanaPipeline.xpackCiGroupProcess(10),
-            'xpack-accessibility': kibanaPipeline.functionalTestProcess('xpack-accessibility', './test/scripts/jenkins_xpack_accessibility.sh'),
-            // 'xpack-pageLoadMetrics': kibanaPipeline.functionalTestProcess('xpack-pageLoadMetrics', './test/scripts/jenkins_xpack_page_load_metrics.sh'),
-            'xpack-securitySolutionCypress': { processNumber ->
-              whenChanged(['x-pack/plugins/security_solution/', 'x-pack/test/security_solution_cypress/']) {
-                kibanaPipeline.functionalTestProcess('xpack-securitySolutionCypress', './test/scripts/jenkins_security_solution_cypress.sh')(processNumber)
-              }
-            },
-
-            // 'xpack-visualRegression': kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh'),
-          ]),
-        ])
+        kibanaPipeline.allCiTasks()
       }
     }
   }

src/dev/ci_setup/checkout_sibling_es.sh

@@ -7,10 +7,14 @@ function checkout_sibling {
   targetDir=$2
   useExistingParamName=$3
   useExisting="$(eval "echo "\$$useExistingParamName"")"
+  repoAddress="git@github.com:"
+  if [[ "$USE_HTTPS_CLONE" ]]; then
+    repoAddress="https://github.com/"
+  fi
 
   if [ -z ${useExisting:+x} ]; then
     if [ -d "$targetDir" ]; then
-      echo "I expected a clean workspace but an '${project}' sibling directory already exists in [$PARENT_DIR]!"
+      echo "I expected a clean workspace but an '${project}' sibling directory already exists in [$WORKSPACE]!"
       echo
       echo "Either define '${useExistingParamName}' or remove the existing '${project}' sibling."
       exit 1
@@ -21,8 +25,9 @@ function checkout_sibling {
     cloneBranch=""
 
     function clone_target_is_valid {
+
       echo " -> checking for '${cloneBranch}' branch at ${cloneAuthor}/${project}"
-      if [[ -n "$(git ls-remote --heads "git@github.com:${cloneAuthor}/${project}.git" ${cloneBranch} 2>/dev/null)" ]]; then
+      if [[ -n "$(git ls-remote --heads "${repoAddress}${cloneAuthor}/${project}.git" ${cloneBranch} 2>/dev/null)" ]]; then
         return 0
       else
         return 1
@@ -71,7 +76,7 @@ function checkout_sibling {
       fi
 
       echo " -> checking out '${cloneBranch}' branch from ${cloneAuthor}/${project}..."
-      git clone -b "$cloneBranch" "git@github.com:${cloneAuthor}/${project}.git" "$targetDir" --depth=1
+      git clone -b "$cloneBranch" "${repoAddress}${cloneAuthor}/${project}.git" "$targetDir" --depth=1
       echo " -> checked out ${project} revision: $(git -C "${targetDir}" rev-parse HEAD)"
       echo
     }
@@ -87,12 +92,12 @@ function checkout_sibling {
   fi
 }
 
-checkout_sibling "elasticsearch" "${PARENT_DIR}/elasticsearch" "USE_EXISTING_ES"
+checkout_sibling "elasticsearch" "${WORKSPACE}/elasticsearch" "USE_EXISTING_ES"
 export TEST_ES_FROM=${TEST_ES_FROM:-snapshot}
 
 # Set the JAVA_HOME based on the Java property file in the ES repo
 # This assumes the naming convention used on CI (ex: ~/.java/java10)
-ES_DIR="$PARENT_DIR/elasticsearch"
+ES_DIR="$WORKSPACE/elasticsearch"
 ES_JAVA_PROP_PATH=$ES_DIR/.ci/java-versions.properties
 
 

src/dev/ci_setup/setup_env.sh

@@ -53,6 +53,8 @@ export PARENT_DIR="$parentDir"
 kbnBranch="$(jq -r .branch "$KIBANA_DIR/package.json")"
 export KIBANA_PKG_BRANCH="$kbnBranch"
 
+export WORKSPACE="${WORKSPACE:-$PARENT_DIR}"
+
 ###
 ### download node
 ###
@@ -161,7 +163,7 @@ export -f checks-reporter-with-killswitch
 
 source "$KIBANA_DIR/src/dev/ci_setup/load_env_keys.sh"
 
-ES_DIR="$PARENT_DIR/elasticsearch"
+ES_DIR="$WORKSPACE/elasticsearch"
 ES_JAVA_PROP_PATH=$ES_DIR/.ci/java-versions.properties
 
 if [[ -d "$ES_DIR" && -f "$ES_JAVA_PROP_PATH" ]]; then

src/dev/notice/generate_notice_from_source.ts

@@ -49,8 +49,10 @@ export async function generateNoticeFromSource({ productName, directory, log }:
     ignore: [
       '{node_modules,build,target,dist,data,built_assets}/**',
       'packages/*/{node_modules,build,target,dist}/**',
+      'src/plugins/*/{node_modules,build,target,dist}/**',
       'x-pack/{node_modules,build,target,dist,data}/**',
       'x-pack/packages/*/{node_modules,build,target,dist}/**',
+      'x-pack/plugins/*/{node_modules,build,target,dist}/**',
     ],
   };
 

tasks/config/karma.js

@@ -110,7 +110,7 @@ module.exports = function (grunt) {
       customLaunchers: {
         Chrome_Headless: {
           base: 'Chrome',
-          flags: ['--headless', '--disable-gpu', '--remote-debugging-port=9222'],
+          flags: ['--headless', '--disable-gpu', '--remote-debugging-port=9222', '--no-sandbox'],
         },
       },
 

tasks/test_jest.js

@@ -22,18 +22,18 @@ const { resolve } = require('path');
 module.exports = function (grunt) {
   grunt.registerTask('test:jest', function () {
     const done = this.async();
-    runJest(resolve(__dirname, '../scripts/jest.js')).then(done, done);
+    runJest(resolve(__dirname, '../scripts/jest.js'), ['--maxWorkers=10']).then(done, done);
   });
 
   grunt.registerTask('test:jest_integration', function () {
     const done = this.async();
     runJest(resolve(__dirname, '../scripts/jest_integration.js')).then(done, done);
   });
 
-  function runJest(jestScript) {
+  function runJest(jestScript, args = []) {
     const serverCmd = {
       cmd: 'node',
-      args: [jestScript, '--ci'],
+      args: [jestScript, '--ci', ...args],
       opts: { stdio: 'inherit' },
     };
 

test/scripts/checks/doc_api_changes.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+source src/dev/ci_setup/setup_env.sh
+
+yarn run grunt run:checkDocApiChanges

test/scripts/checks/file_casing.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+source src/dev/ci_setup/setup_env.sh
+
+yarn run grunt run:checkFileCasing

test/scripts/checks/i18n.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+source src/dev/ci_setup/setup_env.sh
+
+yarn run grunt run:i18nCheck

test/scripts/checks/licenses.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+source src/dev/ci_setup/setup_env.sh
+
+yarn run grunt run:licenses

test/scripts/checks/lock_file_symlinks.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+source src/dev/ci_setup/setup_env.sh
+
+yarn run grunt run:checkLockfileSymlinks

test/scripts/checks/test_hardening.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+source src/dev/ci_setup/setup_env.sh
+
+yarn run grunt run:test_hardening

test/scripts/checks/test_projects.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+source src/dev/ci_setup/setup_env.sh
+
+yarn run grunt run:test_projects

test/scripts/checks/ts_projects.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+source src/dev/ci_setup/setup_env.sh
+
+yarn run grunt run:checkTsProjects

test/scripts/checks/type_check.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+source src/dev/ci_setup/setup_env.sh
+
+yarn run grunt run:typeCheck