Merge branch 'main' into fix-typecheck-foundations

elastic · Sep 25, 2023 · 57ed7b6 · 57ed7b6
2 parents 7b06b81 + c20d177
commit 57ed7b6
Show file tree

Hide file tree

Showing 84 changed files with 1,966 additions and 661 deletions.
diff --git a/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml b/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml
@@ -0,0 +1,58 @@
+# https://buildkite.com/elastic/kibana-elasticsearch-serverless-verify-and-promote/
+agents:
+  queue: kibana-default
+
+steps:
+  - label: "Annotate runtime parameters"
+    command: |
+      buildkite-agent annotate --context es-serverless-image --style info "ES Serverless image: $ES_SERVERLESS_IMAGE"
+      buildkite-agent annotate --context kibana-commit --style info "Kibana build hash: $BUILDKITE_BRANCH / $BUILDKITE_COMMIT"
+
+  - group: "(:kibana: x :elastic:) Trigger Kibana Serverless suite"
+    if: "build.env('SKIP_VERIFICATION') != '1' && build.env('SKIP_VERIFICATION') != 'true'"
+    steps:
+      - label: "Pre-Build"
+        command: .buildkite/scripts/lifecycle/pre_build.sh
+        key: pre-build
+        timeout_in_minutes: 10
+        agents:
+          queue: kibana-default
+
+      - label: "Build Kibana Distribution and Plugins"
+        command: .buildkite/scripts/steps/build_kibana.sh
+        agents:
+          queue: n2-16-spot
+        key: build
+        depends_on: pre-build
+        if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
+        timeout_in_minutes: 60
+        retry:
+          automatic:
+            - exit_status: '-1'
+              limit: 3
+
+      - label: "Pick Test Group Run Order"
+        command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
+        agents:
+          queue: kibana-default
+        env:
+          FTR_CONFIGS_SCRIPT: 'TEST_ES_SERVERLESS_IMAGE=$ES_SERVERLESS_IMAGE .buildkite/scripts/steps/test/ftr_configs.sh'
+          FTR_CONFIG_PATTERNS: '**/test_serverless/**'
+          LIMIT_CONFIG_TYPE: 'functional'
+        retry:
+          automatic:
+            - exit_status: '*'
+              limit: 1
+
+  - wait: ~
+
+  - label: ":arrow_up::elastic::arrow_up: Promote docker image"
+    command: .buildkite/scripts/steps/es_serverless/promote_es_serverless_image.sh $ES_SERVERLESS_IMAGE
+
+  - wait: ~
+
+  - label: 'Post-Build'
+    command: .buildkite/scripts/lifecycle/post_build.sh
+    timeout_in_minutes: 10
+    agents:
+      queue: kibana-default
diff --git a/.buildkite/scripts/steps/es_serverless/promote_es_serverless_image.sh b/.buildkite/scripts/steps/es_serverless/promote_es_serverless_image.sh
@@ -0,0 +1,75 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+BASE_ES_SERVERLESS_REPO=docker.elastic.co/elasticsearch-ci/elasticsearch-serverless
+TARGET_IMAGE=docker.elastic.co/kibana-ci/elasticsearch-serverless:latest-verified
+
+ES_SERVERLESS_BUCKET=kibana-ci-es-serverless-images
+MANIFEST_FILE_NAME=latest-verified.json
+
+SOURCE_IMAGE_OR_TAG=$1
+if [[ $SOURCE_IMAGE_OR_TAG =~ :[a-zA-Z_-]+$ ]]; then
+  # $SOURCE_IMAGE_OR_TAG was a full image
+  SOURCE_IMAGE=$SOURCE_IMAGE_OR_TAG
+else
+  # $SOURCE_IMAGE_OR_TAG was an image tag
+  SOURCE_IMAGE="$BASE_ES_SERVERLESS_REPO:$SOURCE_IMAGE_OR_TAG"
+fi
+
+echo "--- Promoting ${SOURCE_IMAGE_OR_TAG} to ':latest-verified'"
+
+echo "Re-tagging $SOURCE_IMAGE -> $TARGET_IMAGE"
+
+echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
+docker pull "$SOURCE_IMAGE"
+docker tag "$SOURCE_IMAGE" "$TARGET_IMAGE"
+docker push "$TARGET_IMAGE"
+
+ORIG_IMG_DATA=$(docker inspect "$SOURCE_IMAGE")
+ELASTIC_COMMIT_HASH=$(echo $ORIG_IMG_DATA | jq -r '.[].Config.Labels["org.opencontainers.image.revision"]')
+
+docker logout docker.elastic.co
+
+echo "Image push to $TARGET_IMAGE successful."
+echo "Promotion successful! Henceforth, thou shall be named Sir $TARGET_IMAGE"
+
+MANIFEST_UPLOAD_PATH="Skipped"
+if [[ "$UPLOAD_MANIFEST" =~ ^(1|true)$ && "$SOURCE_IMAGE_OR_TAG" =~ ^git-[0-9a-fA-F]{12}$ ]]; then
+  echo "--- Uploading latest-verified manifest to GCS"
+  cat << EOT >> $MANIFEST_FILE_NAME
+{
+  "build_url": "$BUILDKITE_BUILD_URL",
+  "kibana_commit": "$BUILDKITE_COMMIT",
+  "kibana_branch": "$BUILDKITE_BRANCH",
+  "elasticsearch_serverless_tag": "$SOURCE_IMAGE_OR_TAG",
+  "elasticsearch_serverless_image_url: "$SOURCE_IMAGE",
+  "elasticsearch_serverless_commit": "TODO: this currently can't be decided",
+  "elasticsearch_commit": "$ELASTIC_COMMIT_HASH",
+  "created_at": "`date`",
+  "timestamp": "`FORCE_COLOR=0 node -p 'Date.now()'`"
+}
+EOT
+
+  gsutil -h "Cache-Control:no-cache, max-age=0, no-transform" \
+    cp $MANIFEST_FILE_NAME "gs://$ES_SERVERLESS_BUCKET/$MANIFEST_FILE_NAME"
+  gsutil acl ch -u AllUsers:R "gs://$ES_SERVERLESS_BUCKET/$MANIFEST_FILE_NAME"
+  MANIFEST_UPLOAD_PATH="<a href=\"https://storage.googleapis.com/$ES_SERVERLESS_BUCKET/$MANIFEST_FILE_NAME\">$MANIFEST_FILE_NAME</a>"
+
+elif [[ "$UPLOAD_MANIFEST" =~ ^(1|true)$ ]]; then
+  echo "--- Skipping upload of latest-verified manifest to GCS, ES Serverless build tag is not pointing to a hash"
+elif [[ "$SOURCE_IMAGE_OR_TAG" =~ ^git-[0-9a-fA-F]{12}$ ]]; then
+  echo "--- Skipping upload of latest-verified manifest to GCS, flag was not provided"
+fi
+
+echo "--- Annotating build with info"
+cat << EOT | buildkite-agent annotate --style "success"
+  <h2>Promotion successful!</h2>
+  <br/>New image: $TARGET_IMAGE
+  <br/>Source image: $SOURCE_IMAGE
+  <br/>Kibana commit: <a href="https://github.com/elastic/kibana/commit/$BUILDKITE_COMMIT">$BUILDKITE_COMMIT</a>
+  <br/>Elasticsearch commit: <a href="https://github.com/elastic/elasticsearch/commit/$ELASTIC_COMMIT_HASH">$ELASTIC_COMMIT_HASH</a>
+  <br/>Manifest file: $MANIFEST_UPLOAD_PATH
+EOT
diff --git a/packages/kbn-dev-cli-runner/src/flags.ts b/packages/kbn-dev-cli-runner/src/flags.ts
@@ -27,6 +27,7 @@ export interface FlagOptions {
   allowUnexpected?: boolean;
   guessTypesForUnexpectedFlags?: boolean;
   help?: string;
+  examples?: string;
   alias?: { [key: string]: string | string[] };
   boolean?: string[];
   string?: string[];
@@ -47,6 +48,7 @@ export function mergeFlagOptions(global: FlagOptions = {}, local: FlagOptions =
     },
 
     help: local.help,
+    examples: local.examples,
 
     allowUnexpected: !!(global.allowUnexpected || local.allowUnexpected),
     guessTypesForUnexpectedFlags: !!(global.allowUnexpected || local.allowUnexpected),

diff --git a/packages/kbn-dev-cli-runner/src/help.ts b/packages/kbn-dev-cli-runner/src/help.ts
@@ -36,25 +36,31 @@ export function getHelp({
   usage,
   flagHelp,
   defaultLogLevel,
+  examples,
 }: {
   description?: string;
   usage?: string;
   flagHelp?: string;
   defaultLogLevel?: string;
+  examples?: string;
 }) {
   const optionHelp = joinAndTrimLines(
     dedent(flagHelp || ''),
     getLogLevelFlagsHelp(defaultLogLevel),
     GLOBAL_FLAGS
   );
 
+  const examplesHelp = examples ? joinAndTrimLines('Examples:', examples) : '';
+
   return `
   ${dedent(usage || '') || DEFAULT_GLOBAL_USAGE}
 
   ${indent(dedent(description || 'Runs a dev task'), 2)}
 
   Options:
-    ${indent(optionHelp, 4)}\n\n`;
+    ${indent(optionHelp, 4)}
+${examplesHelp ? `\n  ${indent(examplesHelp, 4)}` : ''}
+`;
 }
 
 export function getCommandLevelHelp({

diff --git a/packages/kbn-dev-cli-runner/src/run.ts b/packages/kbn-dev-cli-runner/src/run.ts
@@ -50,6 +50,7 @@ export async function run(fn: RunFn, options: RunOptions = {}) {
     usage: options.usage,
     flagHelp: options.flags?.help,
     defaultLogLevel: options.log?.defaultLevel,
+    examples: options.flags?.examples,
   });
 
   if (flags.help) {

diff --git a/packages/kbn-es/src/cli_commands/serverless.ts b/packages/kbn-es/src/cli_commands/serverless.ts
@@ -13,9 +13,8 @@ import { getTimeReporter } from '@kbn/ci-stats-reporter';
 
 import { Cluster } from '../cluster';
 import {
-  SERVERLESS_REPO,
-  SERVERLESS_TAG,
-  SERVERLESS_IMG,
+  ES_SERVERLESS_REPO_ELASTICSEARCH,
+  ES_SERVERLESS_DEFAULT_IMAGE,
   DEFAULT_PORT,
   ServerlessOptions,
 } from '../utils';
@@ -28,9 +27,8 @@ export const serverless: Command = {
     return dedent`
     Options:
 
-      --tag               Image tag of ES serverless to run from ${SERVERLESS_REPO} [default: ${SERVERLESS_TAG}]
-      --image             Full path of ES serverless image to run, has precedence over tag. [default: ${SERVERLESS_IMG}]
-      
+      --tag               Image tag of ES serverless to run from ${ES_SERVERLESS_REPO_ELASTICSEARCH}
+      --image             Full path of ES serverless image to run, has precedence over tag. [default: ${ES_SERVERLESS_DEFAULT_IMAGE}]
       --background        Start ES serverless without attaching to the first node's logs
       --basePath          Path to the directory where the ES cluster will store data
       --clean             Remove existing file system object store before running
@@ -39,14 +37,14 @@ export const serverless: Command = {
       --ssl               Enable HTTP SSL on the ES cluster
       --skipTeardown      If this process exits, leave the ES cluster running in the background
       --waitForReady      Wait for the ES cluster to be ready to serve requests
-      
+
       -E                  Additional key=value settings to pass to ES
       -F                  Absolute paths for files to mount into containers
 
     Examples:
 
-      es serverless --tag git-fec36430fba2-x86_64
-      es serverless --image docker.elastic.co/repo:tag
+      es serverless --tag git-fec36430fba2-x86_64 # loads ${ES_SERVERLESS_REPO_ELASTICSEARCH}:git-fec36430fba2-x86_64
+      es serverless --image docker.elastic.co/kibana-ci/elasticsearch-serverless:latest-verified
     `;
   },
   run: async (defaults = {}) => {

diff --git a/packages/kbn-es/src/utils/docker.test.ts b/packages/kbn-es/src/utils/docker.test.ts
@@ -23,7 +23,7 @@ import {
   runDockerContainer,
   runServerlessCluster,
   runServerlessEsNode,
-  SERVERLESS_IMG,
+  ES_SERVERLESS_DEFAULT_IMAGE,
   setupServerlessVolumes,
   stopServerlessCluster,
   teardownServerlessClusterSync,
@@ -451,7 +451,7 @@ describe('runServerlessEsNode()', () => {
   const node = {
     params: ['--env', 'foo=bar', '--volume', 'foo/bar'],
     name: 'es01',
-    image: SERVERLESS_IMG,
+    image: ES_SERVERLESS_DEFAULT_IMAGE,
   };
 
   test('should call the correct Docker command', async () => {
@@ -462,7 +462,7 @@ describe('runServerlessEsNode()', () => {
     expect(execa.mock.calls[0][0]).toEqual('docker');
     expect(execa.mock.calls[0][1]).toEqual(
       expect.arrayContaining([
-        SERVERLESS_IMG,
+        ES_SERVERLESS_DEFAULT_IMAGE,
         ...node.params,
         '--name',
         node.name,
@@ -530,7 +530,9 @@ describe('teardownServerlessClusterSync()', () => {
     teardownServerlessClusterSync(log, defaultOptions);
 
     expect(execa.commandSync.mock.calls).toHaveLength(2);
-    expect(execa.commandSync.mock.calls[0][0]).toEqual(expect.stringContaining(SERVERLESS_IMG));
+    expect(execa.commandSync.mock.calls[0][0]).toEqual(
+      expect.stringContaining(ES_SERVERLESS_DEFAULT_IMAGE)
+    );
     expect(execa.commandSync.mock.calls[1][0]).toEqual(`docker kill ${nodes.join(' ')}`);
   });
 

diff --git a/packages/kbn-es/src/utils/docker.ts b/packages/kbn-es/src/utils/docker.ts
@@ -38,9 +38,12 @@ import {
 import { SYSTEM_INDICES_SUPERUSER } from './native_realm';
 import { waitUntilClusterReady } from './wait_until_cluster_ready';
 
-interface BaseOptions {
-  tag?: string;
+interface ImageOptions {
   image?: string;
+  tag?: string;
+}
+
+interface BaseOptions extends ImageOptions {
   port?: number;
   ssl?: boolean;
   /** Kill running cluster before starting a new cluster  */
@@ -106,9 +109,10 @@ export const DOCKER_REPO = `${DOCKER_REGISTRY}/elasticsearch/elasticsearch`;
 export const DOCKER_TAG = `${pkg.version}-SNAPSHOT`;
 export const DOCKER_IMG = `${DOCKER_REPO}:${DOCKER_TAG}`;
 
-export const SERVERLESS_REPO = `${DOCKER_REGISTRY}/elasticsearch-ci/elasticsearch-serverless`;
-export const SERVERLESS_TAG = 'latest';
-export const SERVERLESS_IMG = `${SERVERLESS_REPO}:${SERVERLESS_TAG}`;
+export const ES_SERVERLESS_REPO_KIBANA = `${DOCKER_REGISTRY}/kibana-ci/elasticsearch-serverless`;
+export const ES_SERVERLESS_REPO_ELASTICSEARCH = `${DOCKER_REGISTRY}/elasticsearch-ci/elasticsearch-serverless`;
+export const ES_SERVERLESS_LATEST_VERIFIED_TAG = 'latest-verified';
+export const ES_SERVERLESS_DEFAULT_IMAGE = `${ES_SERVERLESS_REPO_KIBANA}:${ES_SERVERLESS_LATEST_VERIFIED_TAG}`;
 
 // See for default cluster settings
 // https://github.com/elastic/elasticsearch-serverless/blob/main/serverless-build-tools/src/main/kotlin/elasticsearch.serverless-run.gradle.kts
@@ -275,7 +279,12 @@ export function resolveDockerImage({
   image,
   repo,
   defaultImg,
-}: (ServerlessOptions | DockerOptions) & { repo: string; defaultImg: string }) {
+}: {
+  tag?: string;
+  image?: string;
+  repo: string;
+  defaultImg: string;
+}) {
   if (image) {
     if (!image.includes(DOCKER_REGISTRY)) {
       throw createCliError(
@@ -525,11 +534,12 @@ export async function setupServerlessVolumes(log: ToolingLog, options: Serverles
 /**
  * Resolve the Serverless ES image based on defaults and CLI options
  */
-function getServerlessImage(options: ServerlessOptions) {
+function getServerlessImage({ image, tag }: ImageOptions) {
   return resolveDockerImage({
-    ...options,
-    repo: SERVERLESS_REPO,
-    defaultImg: SERVERLESS_IMG,
+    image,
+    tag,
+    repo: ES_SERVERLESS_REPO_ELASTICSEARCH,
+    defaultImg: ES_SERVERLESS_DEFAULT_IMAGE,
   });
 }
 
@@ -573,7 +583,10 @@ function getESClient(clientOptions: ClientOptions): Client {
  * Runs an ES Serverless Cluster through Docker
  */
 export async function runServerlessCluster(log: ToolingLog, options: ServerlessOptions) {
-  const image = getServerlessImage(options);
+  const image = getServerlessImage({
+    image: options.image,
+    tag: options.tag,
+  });
   await setupDocker({ log, image, options });
 
   const volumeCmd = await setupServerlessVolumes(log, options);
@@ -686,8 +699,13 @@ export function teardownServerlessClusterSync(log: ToolingLog, options: Serverle
 /**
  * Resolve the Elasticsearch image based on defaults and CLI options
  */
-function getDockerImage(options: DockerOptions) {
-  return resolveDockerImage({ ...options, repo: DOCKER_REPO, defaultImg: DOCKER_IMG });
+function getDockerImage({ image, tag }: ImageOptions) {
+  return resolveDockerImage({
+    image,
+    tag,
+    repo: DOCKER_REPO,
+    defaultImg: DOCKER_IMG,
+  });
 }
 
 /**
@@ -713,7 +731,10 @@ export async function runDockerContainer(log: ToolingLog, options: DockerOptions
   let image;
 
   if (!options.dockerCmd) {
-    image = getDockerImage(options);
+    image = getDockerImage({
+      image: options.image,
+      tag: options.tag,
+    });
     await setupDocker({ log, image, options });
   }
 

diff --git a/packages/kbn-test/src/es/es_test_config.ts b/packages/kbn-test/src/es/es_test_config.ts
@@ -27,6 +27,10 @@ class EsTestConfig {
     return process.env.TEST_ES_FROM || 'snapshot';
   }
 
+  getESServerlessImage() {
+    return process.env.TEST_ES_SERVERLESS_IMAGE;
+  }
+
   getTransportPort() {
     return process.env.TEST_ES_TRANSPORT_PORT || '9300-9400';
   }