Merge branch 'main' into fix/treemap-suggestions-2

.backportrc.json

@@ -3,6 +3,7 @@
   "repoName": "kibana",
   "targetBranchChoices": [
     "main",
+    "8.12",
     "8.11",
     "8.10",
     "8.9",
@@ -48,7 +49,7 @@
     "backport"
   ],
   "branchLabelMapping": {
-    "^v8.12.0$": "main",
+    "^v8.13.0$": "main",
     "^v(\\d+).(\\d+).\\d+$": "$1.$2"
   },
   "autoMerge": true,

.buildkite/pipeline-utils/buildkite/client.ts

@@ -7,24 +7,31 @@
  */
 
 import axios, { AxiosInstance } from 'axios';
-import { execSync } from 'child_process';
+import { execSync, ExecSyncOptions } from 'child_process';
 import { dump } from 'js-yaml';
 import { parseLinkHeader } from './parse_link_header';
 import { Artifact } from './types/artifact';
 import { Build, BuildStatus } from './types/build';
 import { Job, JobState } from './types/job';
 
+type ExecType =
+  | ((command: string, execOpts: ExecSyncOptions) => Buffer | null)
+  | ((command: string, execOpts: ExecSyncOptions) => string | null);
+
 export interface BuildkiteClientConfig {
   baseUrl?: string;
   token?: string;
+  exec?: ExecType;
 }
 
 export interface BuildkiteGroup {
   group: string;
   steps: BuildkiteStep[];
 }
 
-export interface BuildkiteStep {
+export type BuildkiteStep = BuildkiteCommandStep | BuildkiteInputStep;
+
+export interface BuildkiteCommandStep {
   command: string;
   label: string;
   parallelism?: number;
@@ -43,6 +50,50 @@ export interface BuildkiteStep {
   env?: { [key: string]: string };
 }
 
+interface BuildkiteInputTextField {
+  text: string;
+  key: string;
+  hint?: string;
+  required?: boolean;
+  default?: string;
+}
+
+interface BuildkiteInputSelectField {
+  select: string;
+  key: string;
+  hint?: string;
+  required?: boolean;
+  default?: string;
+  multiple?: boolean;
+  options: Array<{
+    label: string;
+    value: string;
+  }>;
+}
+
+export interface BuildkiteInputStep {
+  input: string;
+  prompt?: string;
+  fields: Array<BuildkiteInputTextField | BuildkiteInputSelectField>;
+  if?: string;
+  allow_dependency_failure?: boolean;
+  branches?: string;
+  parallelism?: number;
+  agents?: {
+    queue: string;
+  };
+  timeout_in_minutes?: number;
+  key?: string;
+  depends_on?: string | string[];
+  retry?: {
+    automatic: Array<{
+      exit_status: string;
+      limit: number;
+    }>;
+  };
+  env?: { [key: string]: string };
+}
+
 export interface BuildkiteTriggerBuildParams {
   commit: string;
   branch: string;
@@ -61,6 +112,7 @@ export interface BuildkiteTriggerBuildParams {
 
 export class BuildkiteClient {
   http: AxiosInstance;
+  exec: ExecType;
 
   constructor(config: BuildkiteClientConfig = {}) {
     const BUILDKITE_BASE_URL =
@@ -78,6 +130,8 @@ export class BuildkiteClient {
       },
     });
 
+    this.exec = config.exec ?? execSync;
+
     // this.agentHttp = axios.create({
     //   baseURL: BUILDKITE_AGENT_BASE_URL,
     //   headers: {
@@ -97,6 +151,32 @@ export class BuildkiteClient {
     return resp.data as Build;
   };
 
+  getBuildsAfterDate = async (
+    pipelineSlug: string,
+    date: string,
+    numberOfBuilds: number
+  ): Promise<Build[]> => {
+    const response = await this.http.get(
+      `v2/organizations/elastic/pipelines/${pipelineSlug}/builds?created_from=${date}&per_page=${numberOfBuilds}`
+    );
+    return response.data as Build[];
+  };
+
+  getBuildForCommit = async (pipelineSlug: string, commit: string): Promise<Build | null> => {
+    if (commit.length !== 40) {
+      throw new Error(`Invalid commit hash: ${commit}, this endpoint works with full SHAs only`);
+    }
+
+    const response = await this.http.get(
+      `v2/organizations/elastic/pipelines/${pipelineSlug}/builds?commit=${commit}`
+    );
+    const builds = response.data as Build[];
+    if (builds.length === 0) {
+      return null;
+    }
+    return builds[0];
+  };
+
   getCurrentBuild = (includeRetriedJobs = false) => {
     if (!process.env.BUILDKITE_PIPELINE_SLUG || !process.env.BUILDKITE_BUILD_NUMBER) {
       throw new Error(
@@ -235,31 +315,52 @@ export class BuildkiteClient {
   };
 
   setMetadata = (key: string, value: string) => {
-    execSync(`buildkite-agent meta-data set '${key}'`, {
+    this.exec(`buildkite-agent meta-data set '${key}'`, {
       input: value,
       stdio: ['pipe', 'inherit', 'inherit'],
     });
   };
 
+  getMetadata(key: string, defaultValue: string | null = null): string | null {
+    try {
+      const stdout = this.exec(`buildkite-agent meta-data get '${key}'`, {
+        stdio: ['pipe'],
+      });
+      return stdout?.toString().trim() || defaultValue;
+    } catch (e) {
+      if (e.message.includes('404 Not Found')) {
+        return defaultValue;
+      } else {
+        throw e;
+      }
+    }
+  }
+
   setAnnotation = (
     context: string,
     style: 'info' | 'success' | 'warning' | 'error',
-    value: string
+    value: string,
+    append: boolean = false
   ) => {
-    execSync(`buildkite-agent annotate --context '${context}' --style '${style}'`, {
-      input: value,
-      stdio: ['pipe', 'inherit', 'inherit'],
-    });
+    this.exec(
+      `buildkite-agent annotate --context '${context}' --style '${style}' ${
+        append ? '--append' : ''
+      }`,
+      {
+        input: value,
+        stdio: ['pipe', 'inherit', 'inherit'],
+      }
+    );
   };
 
   uploadArtifacts = (pattern: string) => {
-    execSync(`buildkite-agent artifact upload '${pattern}'`, {
+    this.exec(`buildkite-agent artifact upload '${pattern}'`, {
       stdio: ['ignore', 'inherit', 'inherit'],
     });
   };
 
   uploadSteps = (steps: Array<BuildkiteStep | BuildkiteGroup>) => {
-    execSync(`buildkite-agent pipeline upload`, {
+    this.exec(`buildkite-agent pipeline upload`, {
       input: dump({ steps }),
       stdio: ['pipe', 'inherit', 'inherit'],
     });

.buildkite/pipeline-utils/ci-stats/client.ts

@@ -201,7 +201,7 @@ export class CiStatsClient {
           headers: this.defaultHeaders,
         });
       } catch (error) {
-        console.error('CI Stats request error:', error);
+        console.error('CI Stats request error:', error?.response?.data?.message);
 
         if (attempt < maxAttempts) {
           const sec = attempt * 3;
@@ -210,7 +210,7 @@ export class CiStatsClient {
           continue;
         }
 
-        throw error;
+        throw new Error('Failed to connect to CI Stats.');
       }
     }
   }

.buildkite/pipeline-utils/ci-stats/on_complete.ts

@@ -28,7 +28,15 @@ export async function onComplete() {
 
   const report = await ciStats.getPrReport(process.env.CI_STATS_BUILD_ID);
   if (report?.md) {
-    buildkite.setMetadata('pr_comment:ci_stats_report:body', report.md);
+    // buildkite has a metadata size limit of 100kb, so we only add this, if it's small enough
+    if (new Blob([report.md]).size < 100000) {
+      buildkite.setMetadata('pr_comment:ci_stats_report:body', report.md);
+    } else {
+      buildkite.setMetadata(
+        'pr_comment:ci_stats_report:body',
+        'The CI Stats report is too large to be displayed here, check out the CI build annotation for this information.'
+      );
+    }
 
     const annotationType = report?.success ? 'info' : 'error';
     buildkite.setAnnotation('ci-stats-report', annotationType, report.md);

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -273,7 +273,12 @@ export async function pickTestGroupRunOrder() {
           ]
         : []),
       // if we are running on a external job, like kibana-code-coverage-main, try finding times that are specific to that job
-      ...(!prNumber && pipelineSlug !== 'kibana-on-merge'
+      // kibana-elasticsearch-serverless-verify-and-promote is not necessarily run in commit order -
+      // using kibana-on-merge groups will provide a closer approximation, with a failure mode -
+      // of too many ftr groups instead of potential timeouts.
+      ...(!prNumber &&
+      pipelineSlug !== 'kibana-on-merge' &&
+      pipelineSlug !== 'kibana-elasticsearch-serverless-verify-and-promote'
         ? [
             {
               branch: ownBranch,

.buildkite/pipeline-utils/github/github.ts

@@ -91,3 +91,7 @@ export const doAnyChangesMatch = async (
 
   return anyFilesMatchRequired;
 };
+
+export function getGithubClient() {
+  return github;
+}

.buildkite/pipeline-utils/index.ts

@@ -10,3 +10,4 @@ export * from './buildkite';
 export * as CiStats from './ci-stats';
 export * from './github';
 export * as TestFailures from './test-failures';
+export * from './utils';

.buildkite/pipeline-utils/utils.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { execSync } from 'child_process';
+
+const getKibanaDir = (() => {
+  let kibanaDir: string | undefined;
+  return () => {
+    if (!kibanaDir) {
+      kibanaDir = execSync('git rev-parse --show-toplevel', { encoding: 'utf-8' })
+        .toString()
+        .trim();
+    }
+
+    return kibanaDir;
+  };
+})();
+
+export { getKibanaDir };

.buildkite/pipelines/artifacts.yml

@@ -61,7 +61,17 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: KIBANA_DOCKER_CONTEXT=ubi9 .buildkite/scripts/steps/artifacts/docker_context.sh
+  - command: KIBANA_DOCKER_CONTEXT=ubi .buildkite/scripts/steps/artifacts/docker_context.sh
+    label: 'Docker Context Verification'
+    agents:
+      queue: n2-2
+    timeout_in_minutes: 30
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: KIBANA_DOCKER_CONTEXT=ironbank .buildkite/scripts/steps/artifacts/docker_context.sh
     label: 'Docker Context Verification'
     agents:
       queue: n2-2

.buildkite/pipelines/es_serverless/emergency_relelease_branch_testing.yml

@@ -0,0 +1,17 @@
+# https://buildkite.com/elastic/kibana-serverless-emergency-release-branch-testing
+
+## Triggers the artifacts container image build for emergency releases
+agents:
+  queue: kibana-default
+
+notify:
+  - slack: "#kibana-mission-control"
+    if: "build.state == 'passed' || build.state == 'failed' || build.state == 'scheduled'"
+
+steps:
+  - trigger: "kibana-artifacts-container-image"
+    label: ":docker: Build Kibana Artifacts Container Image"
+    build:
+      branch: $BUILDKITE_BRANCH
+      commit: $BUILDKITE_COMMIT
+      message: Running PR build for $BUILDKITE_BRANCH