Merge branch 'main' into remove_version_from_mlconfigversion

.backportrc.json

@@ -1,9 +1,9 @@
 {
   "upstream" : "elastic/elasticsearch",
-  "targetBranchChoices" : [ "main", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
+  "targetBranchChoices" : [ "main", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
-    "^v8.11.0$" : "main",
+    "^v8.12.0$" : "main",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
 }

.buildkite/check-es-serverless.yml

@@ -1,5 +1,5 @@
 steps:
-  - trigger: elasticsearch-serverless-update-submodule
+  - trigger: elasticsearch-serverless-validate-submodule
     label: ":elasticsearch: Check elasticsearch changes against serverless"
     build:
       message: "Validate latest elasticsearch changes"

.buildkite/hooks/pre-command

@@ -92,7 +92,7 @@ fi
 
 # Initialize the build scan and gobld annotations with empty/open <details> tags
 # This ensures that they are collapsible when they get appended to
-if [[ "${BUILDKITE_LABEL:-}" == *"Pipeline upload"* ]]; then
+if [[ "${BUILDKITE_LABEL:-}" == *"Pipeline upload"* || "${BUILDKITE_LABEL:-}" == *"Upload Pipeline"* ]]; then
   cat << EOF | buildkite-agent annotate --context "gradle-build-scans" --style "info"
 <details>
 

.buildkite/pipelines/intake.yml

@@ -15,23 +15,23 @@ steps:
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004
-      machineType: custom-32-98304
+      machineType: n1-standard-32
       buildDirectory: /dev/shm/bk
   - label: part2
     command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2
     timeout_in_minutes: 300
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004
-      machineType: custom-32-98304
+      machineType: n1-standard-32
       buildDirectory: /dev/shm/bk
   - label: part3
     command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3
     timeout_in_minutes: 300
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004
-      machineType: custom-32-98304
+      machineType: n1-standard-32
       buildDirectory: /dev/shm/bk
   - group: bwc-snapshots
     steps:
@@ -40,7 +40,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.14", "8.10.3", "8.11.0"]
+            BWC_VERSION: ["7.17.15", "8.10.5", "8.11.0", "8.12.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -1056,6 +1056,22 @@ steps:
         env:
           BWC_VERSION: 7.17.14
 
+      - label: "{{matrix.image}} / 7.17.15 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.15
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 7.17.15
+
       - label: "{{matrix.image}} / 8.0.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.0.0
         timeout_in_minutes: 300
@@ -1648,6 +1664,38 @@ steps:
         env:
           BWC_VERSION: 8.10.3
 
+      - label: "{{matrix.image}} / 8.10.4 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.10.4
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.10.4
+
+      - label: "{{matrix.image}} / 8.10.5 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.10.5
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.10.5
+
       - label: "{{matrix.image}} / 8.11.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.0
         timeout_in_minutes: 300
@@ -1664,6 +1712,22 @@ steps:
         env:
           BWC_VERSION: 8.11.0
 
+      - label: "{{matrix.image}} / 8.12.0 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.12.0
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.12.0
+
   - group: packaging-tests-windows
     steps:
       - label: "{{matrix.image}} / packaging-tests-windows"

.buildkite/pipelines/periodic-platform-support.yml

@@ -2,7 +2,7 @@ steps:
   - group: platform-support-unix
     steps:
       - label: "{{matrix.image}} / platform-support-unix"
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true check
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true functionalTests
         timeout_in_minutes: 420
         matrix:
           setup:
@@ -46,7 +46,6 @@ steps:
               - checkPart1
               - checkPart2
               - checkPart3
-              - bwcTestSnapshots
               - checkRestCompat
         agents:
           provider: gcp
@@ -70,7 +69,6 @@ steps:
               - checkPart1
               - checkPart2
               - checkPart3
-              - bwcTestSnapshots
               - checkRestCompat
         agents:
           provider: aws

.buildkite/pipelines/periodic.template.yml

@@ -89,7 +89,7 @@ steps:
           GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
   - label: release-tests
     command: .buildkite/scripts/release-tests.sh
-    timeout_in_minutes: 300
+    timeout_in_minutes: 360
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic.yml

@@ -642,6 +642,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 7.17.14
+      - label: 7.17.15 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.15#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 7.17.15
       - label: 8.0.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.0.0#bwcTest
         timeout_in_minutes: 300
@@ -1012,6 +1022,26 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.10.3
+      - label: 8.10.4 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.4#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.10.4
+      - label: 8.10.5 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.5#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.10.5
       - label: 8.11.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.0#bwcTest
         timeout_in_minutes: 300
@@ -1022,6 +1052,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.11.0
+      - label: 8.12.0 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.12.0#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.12.0
   - label: concurrent-search-tests
     command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true check
     timeout_in_minutes: 420
@@ -1075,7 +1115,7 @@ steps:
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
-          machineType: custom-32-98304
+          machineType: n1-standard-32
           buildDirectory: /dev/shm/bk
         env:
           ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
@@ -1103,14 +1143,14 @@ steps:
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
-          machineType: custom-32-98304
+          machineType: n1-standard-32
           buildDirectory: /dev/shm/bk
         env:
           ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
           GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
   - label: release-tests
     command: .buildkite/scripts/release-tests.sh
-    timeout_in_minutes: 300
+    timeout_in_minutes: 360
     agents:
       provider: gcp
       image: family/elasticsearch-ubuntu-2004
@@ -1176,6 +1216,7 @@ steps:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
           machineType: n2-standard-8
+          buildDirectory: /dev/shm/bk
       - label: third-party / s3
         command: |
           export amazon_s3_bucket=elasticsearch-ci.us-west-2

.buildkite/pipelines/pull-request/validate-changelogs.yml

@@ -0,0 +1,9 @@
+steps:
+  - label: validate-changelogs
+    command: .ci/scripts/run-gradle.sh -Dignore.tests.seed validateChangelogs
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

.buildkite/pull-requests.json

@@ -11,7 +11,7 @@
       "set_commit_status": false,
       "build_on_commit": true,
       "build_on_comment": true,
-      "trigger_comment_regex": "buildkite\\W+elasticsearch-ci.+",
+      "trigger_comment_regex": "run\\W+elasticsearch-ci.+",
       "labels": [
         "buildkite-opt-in"
       ],

.buildkite/scripts/dra-workflow.sh

@@ -71,8 +71,6 @@ find "$WORKSPACE" -type d -path "*/build/distributions" -exec chmod a+w {} \;
 
 echo --- Running release-manager
 
-exit 0
-
 # Artifacts should be generated
 docker run --rm \
   --name release-manager \

.buildkite/scripts/periodic.trigger.sh

@@ -12,33 +12,23 @@ for BRANCH in "${BRANCHES[@]}"; do
   LAST_GOOD_COMMIT=$(echo "${BUILD_JSON}" | jq -r '.commit')
 
   cat <<EOF
+  - trigger: elasticsearch-periodic
+    label: Trigger periodic pipeline for $BRANCH
+    async: true
+    build:
+      branch: "$BRANCH"
+      commit: "$LAST_GOOD_COMMIT"
+  - trigger: elasticsearch-periodic-packaging
+    label: Trigger periodic-packaging pipeline for $BRANCH
+    async: true
+    build:
+      branch: "$BRANCH"
+      commit: "$LAST_GOOD_COMMIT"
   - trigger: elasticsearch-periodic-platform-support
     label: Trigger periodic-platform-support pipeline for $BRANCH
     async: true
     build:
       branch: "$BRANCH"
       commit: "$LAST_GOOD_COMMIT"
 EOF
-
-### Only platform-support enabled for right now
-#   cat <<EOF
-#   - trigger: elasticsearch-periodic
-#     label: Trigger periodic pipeline for $BRANCH
-#     async: true
-#     build:
-#       branch: "$BRANCH"
-#       commit: "$LAST_GOOD_COMMIT"
-#   - trigger: elasticsearch-periodic-packaging
-#     label: Trigger periodic-packaging pipeline for $BRANCH
-#     async: true
-#     build:
-#       branch: "$BRANCH"
-#       commit: "$LAST_GOOD_COMMIT"
-#   - trigger: elasticsearch-periodic-platform-support
-#     label: Trigger periodic-platform-support pipeline for $BRANCH
-#     async: true
-#     build:
-#       branch: "$BRANCH"
-#       commit: "$LAST_GOOD_COMMIT"
-# EOF
 done