Merge branch 'main' into issue-180551-version-find-rules-api

.backportrc.json

@@ -3,6 +3,7 @@
   "repoName": "kibana",
   "targetBranchChoices": [
     "main",
+    "8.14",
     "8.13",
     "8.12",
     "8.11",
@@ -50,7 +51,7 @@
     "backport"
   ],
   "branchLabelMapping": {
-    "^v8.14.0$": "main",
+    "^v8.15.0$": "main",
     "^v(\\d+).(\\d+).\\d+$": "$1.$2"
   },
   "autoMerge": true,

.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml

@@ -22,7 +22,7 @@ spec:
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
         ES_SERVERLESS_IMAGE: latest
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
-      allow_rebuilds: false
+      allow_rebuilds: true
       branch_configuration: main
       default_branch: main
       repository: elastic/kibana
@@ -35,7 +35,7 @@ spec:
         trigger_mode: none
         build_tags: false
         prefix_pull_request_fork_branch_names: false
-        skip_pull_request_builds_for_existing_commits: true
+        skip_pull_request_builds_for_existing_commits: false
       teams:
         everyone:
           access_level: BUILD_AND_READ

.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml

@@ -21,8 +21,8 @@ spec:
       env:
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
-      allow_rebuilds: false
-      branch_configuration: main 8.13 7.17
+      allow_rebuilds: true
+      branch_configuration: main 8.14 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/es_snapshots/build.yml
@@ -34,7 +34,7 @@ spec:
         trigger_mode: none
         build_tags: false
         prefix_pull_request_fork_branch_names: false
-        skip_pull_request_builds_for_existing_commits: true
+        skip_pull_request_builds_for_existing_commits: false
       teams:
         everyone:
           access_level: BUILD_AND_READ
@@ -49,10 +49,10 @@ spec:
           cronline: 0 9 * * * America/New_York
           message: Daily build
           branch: main
-        Daily build (8.13):
+        Daily build (8.14):
           cronline: 0 9 * * * America/New_York
           message: Daily build
-          branch: '8.13'
+          branch: '8.14'
         Daily build (7.17):
           cronline: 0 9 * * * America/New_York
           message: Daily build
@@ -81,8 +81,8 @@ spec:
       env:
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
-      allow_rebuilds: false
-      branch_configuration: main 8.13 7.17
+      allow_rebuilds: true
+      branch_configuration: main 8.14 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/es_snapshots/promote.yml
@@ -94,7 +94,7 @@ spec:
         trigger_mode: none
         build_tags: false
         prefix_pull_request_fork_branch_names: false
-        skip_pull_request_builds_for_existing_commits: true
+        skip_pull_request_builds_for_existing_commits: false
       teams:
         everyone:
           access_level: BUILD_AND_READ
@@ -128,8 +128,8 @@ spec:
       env:
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
-      allow_rebuilds: false
-      branch_configuration: main 8.13 7.17
+      allow_rebuilds: true
+      branch_configuration: main 8.14 7.17
       default_branch: main
       repository: elastic/kibana
       pipeline_file: .buildkite/pipelines/es_snapshots/verify.yml
@@ -141,7 +141,7 @@ spec:
         trigger_mode: none
         build_tags: false
         prefix_pull_request_fork_branch_names: false
-        skip_pull_request_builds_for_existing_commits: true
+        skip_pull_request_builds_for_existing_commits: false
       teams:
         everyone:
           access_level: BUILD_AND_READ

.buildkite/pipeline-resource-definitions/kibana-flaky.yml

@@ -0,0 +1,41 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-flaky-test-suite-runner
+  description: ':warning: Trigger a new build here: https://ci-stats.kibana.dev/trigger_flaky_test_runner :warning:'
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-flaky-test-suite-runner'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: 'group:kibana-operations'
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / flaky-test-suite-runner
+      description: ':warning: Trigger a new build here: https://ci-stats.kibana.dev/trigger_flaky_test_runner :warning:'
+    spec:
+      allow_rebuilds: true
+      default_branch: refs/pull/INSERT_PR_NUMBER/head
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/flaky_tests/pipeline.sh
+      skip_intermediate_builds: false
+      provider_settings:
+        build_branches: true
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: false
+      teams:
+        everyone:
+          access_level: BUILD_AND_READ
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ

.buildkite/pipeline-resource-definitions/kibana-on-merge-unsupported-ftrs.yml

@@ -0,0 +1,46 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: bk-kibana-on-merge-unsupported-ftrs
+  description: Runs unsupported ftr tests for each commit of Kibana
+  links:
+    - url: 'https://buildkite.com/elastic/kibana-on-merge-unsupported-ftrs'
+      title: Pipeline link
+spec:
+  type: buildkite-pipeline
+  owner: 'group:kibana-operations'
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / on merge unsupported ftrs
+      description: Runs unsupported ftr tests for each commit of Kibana
+    spec:
+      env:
+        SLACK_NOTIFICATIONS_CHANNEL: '#kibana-unsupported-ftrs-alerts'
+        ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+      allow_rebuilds: true
+      branch_configuration: main 8.14 7.17
+      default_branch: main
+      repository: elastic/kibana
+      pipeline_file: .buildkite/pipelines/on_merge_unsupported_ftrs.yml
+      skip_intermediate_builds: false
+      provider_settings:
+        build_branches: false
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+        build_tags: false
+        prefix_pull_request_fork_branch_names: false
+        skip_pull_request_builds_for_existing_commits: false
+      teams:
+        everyone:
+          access_level: BUILD_AND_READ
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        appex-qa:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-tech-leads:
+          access_level: MANAGE_BUILD_AND_READ

.buildkite/pipeline-resource-definitions/locations.yml

@@ -1,3 +1,4 @@
+# * This file is auto-updated by running fix-location-collection.ts * #
 apiVersion: backstage.io/v1alpha1
 kind: Location
 metadata:
@@ -10,7 +11,9 @@ spec:
     - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml'
     - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml'
     - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-esql-grammar-sync.yml'
+    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-flaky.yml'
     - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-migration-staging.yml'
+    - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-on-merge-unsupported-ftrs.yml'
     - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-purge-cloud-deployments.yml'
     - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release.yml'
     - 'https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-ess/security-solution-ess.yml'

.buildkite/pipelines/flaky_tests/pipeline.sh

@@ -5,4 +5,4 @@ set -euo pipefail
 UUID="$(cat /proc/sys/kernel/random/uuid)"
 export UUID
 
-ts-node .buildkite/pipelines/flaky_tests/pipeline.ts | buildkite-agent pipeline upload
+ts-node .buildkite/pipelines/flaky_tests/pipeline.ts

.buildkite/pipelines/flaky_tests/pipeline.ts

@@ -31,6 +31,34 @@ if (Number.isNaN(concurrency)) {
 const BASE_JOBS = 1;
 const MAX_JOBS = 500;
 
+// TODO: remove this after https://github.com/elastic/kibana-operations/issues/15 is finalized
+/** This function bridges the agent targeting between gobld and kibana-buildkite agent targeting */
+const getAgentRule = (queueName: string = 'n2-4-spot') => {
+  if (
+    process.env.BUILDKITE_AGENT_META_DATA_QUEUE === 'gobld' ||
+    process.env.BUILDKITE_AGENT_META_DATA_PROVIDER === 'k8s'
+  ) {
+    const [kind, cores, addition] = queueName.split('-');
+    const additionalProps =
+      {
+        spot: { preemptible: true },
+        virt: { localSsdInterface: 'nvme', enableNestedVirtualization: true, localSsds: 1 },
+      }[addition] || {};
+
+    return {
+      provider: 'gcp',
+      image: 'family/kibana-ubuntu-2004',
+      imageProject: 'elastic-images-qa',
+      machineType: `${kind}-standard-${cores}`,
+      ...additionalProps,
+    };
+  } else {
+    return {
+      queue: queueName,
+    };
+  }
+};
+
 function getTestSuitesFromJson(json: string) {
   const fail = (errorMsg: string) => {
     console.error('+++ Invalid test config provided');
@@ -49,8 +77,10 @@ function getTestSuitesFromJson(json: string) {
     fail(`JSON test config must be an array`);
   }
 
-  /** @type {Array<{ type: 'group', key: string; count: number } | { type: 'ftrConfig', ftrConfig: string; count: number }>} */
-  const testSuites = [];
+  const testSuites: Array<
+    | { type: 'group'; key: string; count: number }
+    | { type: 'ftrConfig'; ftrConfig: string; count: number }
+  > = [];
   for (const item of parsed) {
     if (typeof item !== 'object' || item === null) {
       fail(`testSuites must be objects`);
@@ -73,6 +103,7 @@ function getTestSuitesFromJson(json: string) {
       }
 
       testSuites.push({
+        type: 'ftrConfig',
         ftrConfig,
         count,
       });
@@ -84,6 +115,7 @@ function getTestSuitesFromJson(json: string) {
       fail(`testSuite.key must be a string`);
     }
     testSuites.push({
+      type: 'group',
       key,
       count,
     });
@@ -117,7 +149,7 @@ const pipeline = {
 steps.push({
   command: '.buildkite/scripts/steps/build_kibana.sh',
   label: 'Build Kibana Distribution and Plugins',
-  agents: { queue: 'c2-8' },
+  agents: getAgentRule('c2-8'),
   key: 'build',
   if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''",
 });
@@ -127,7 +159,7 @@ for (const testSuite of testSuites) {
     continue;
   }
 
-  if (testSuite.ftrConfig) {
+  if (testSuite.type === 'ftrConfig') {
     steps.push({
       command: `.buildkite/scripts/steps/test/ftr_configs.sh`,
       env: {
@@ -138,9 +170,7 @@ for (const testSuite of testSuites) {
       concurrency,
       concurrency_group: process.env.UUID,
       concurrency_method: 'eager',
-      agents: {
-        queue: 'n2-4-spot-2',
-      },
+      agents: getAgentRule('n2-4-spot'),
       depends_on: 'build',
       timeout_in_minutes: 150,
       cancel_on_build_failing: true,
@@ -164,7 +194,7 @@ for (const testSuite of testSuites) {
       steps.push({
         command: `.buildkite/scripts/steps/functional/${suiteName}.sh`,
         label: group.name,
-        agents: { queue: agentQueue },
+        agents: getAgentRule(agentQueue),
         depends_on: 'build',
         timeout_in_minutes: 150,
         parallelism: testSuite.count,