Merge branch 'main' into do-not-delete-slm-history-indices-in-rest-tests

masseyke · Oct 11, 2023 · cd0b34c · cd0b34c
2 parents 2fd0fa6 + 8a6df32
commit cd0b34c
Show file tree

Hide file tree

Showing 157 changed files with 4,336 additions and 2,118 deletions.
diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml
@@ -40,7 +40,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.14", "8.10.3", "8.11.0", "8.12.0"]
+            BWC_VERSION: ["7.17.15", "8.10.4", "8.11.0", "8.12.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml
@@ -1056,6 +1056,22 @@ steps:
         env:
           BWC_VERSION: 7.17.14
 
+      - label: "{{matrix.image}} / 7.17.15 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.15
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 7.17.15
+
       - label: "{{matrix.image}} / 8.0.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.0.0
         timeout_in_minutes: 300
@@ -1648,6 +1664,22 @@ steps:
         env:
           BWC_VERSION: 8.10.3
 
+      - label: "{{matrix.image}} / 8.10.4 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.10.4
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.10.4
+
       - label: "{{matrix.image}} / 8.11.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.0
         timeout_in_minutes: 300

diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml
@@ -642,6 +642,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 7.17.14
+      - label: 7.17.15 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.15#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 7.17.15
       - label: 8.0.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.0.0#bwcTest
         timeout_in_minutes: 300
@@ -1012,6 +1022,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.10.3
+      - label: 8.10.4 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.4#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.10.4
       - label: 8.11.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.0#bwcTest
         timeout_in_minutes: 300

diff --git a/.buildkite/scripts/periodic.trigger.sh b/.buildkite/scripts/periodic.trigger.sh
@@ -12,33 +12,23 @@ for BRANCH in "${BRANCHES[@]}"; do
   LAST_GOOD_COMMIT=$(echo "${BUILD_JSON}" | jq -r '.commit')
 
   cat <<EOF
+  - trigger: elasticsearch-periodic
+    label: Trigger periodic pipeline for $BRANCH
+    async: true
+    build:
+      branch: "$BRANCH"
+      commit: "$LAST_GOOD_COMMIT"
+  - trigger: elasticsearch-periodic-packaging
+    label: Trigger periodic-packaging pipeline for $BRANCH
+    async: true
+    build:
+      branch: "$BRANCH"
+      commit: "$LAST_GOOD_COMMIT"
   - trigger: elasticsearch-periodic-platform-support
     label: Trigger periodic-platform-support pipeline for $BRANCH
     async: true
     build:
       branch: "$BRANCH"
       commit: "$LAST_GOOD_COMMIT"
 EOF
-
-### Only platform-support enabled for right now
-#   cat <<EOF
-#   - trigger: elasticsearch-periodic
-#     label: Trigger periodic pipeline for $BRANCH
-#     async: true
-#     build:
-#       branch: "$BRANCH"
-#       commit: "$LAST_GOOD_COMMIT"
-#   - trigger: elasticsearch-periodic-packaging
-#     label: Trigger periodic-packaging pipeline for $BRANCH
-#     async: true
-#     build:
-#       branch: "$BRANCH"
-#       commit: "$LAST_GOOD_COMMIT"
-#   - trigger: elasticsearch-periodic-platform-support
-#     label: Trigger periodic-platform-support pipeline for $BRANCH
-#     async: true
-#     build:
-#       branch: "$BRANCH"
-#       commit: "$LAST_GOOD_COMMIT"
-# EOF
 done
diff --git a/.ci/bwcVersions b/.ci/bwcVersions
@@ -63,6 +63,7 @@ BWC_VERSION:
   - "7.17.12"
   - "7.17.13"
   - "7.17.14"
+  - "7.17.15"
   - "8.0.0"
   - "8.0.1"
   - "8.1.0"
@@ -100,5 +101,6 @@ BWC_VERSION:
   - "8.10.1"
   - "8.10.2"
   - "8.10.3"
+  - "8.10.4"
   - "8.11.0"
   - "8.12.0"
diff --git a/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-arm.yml b/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-arm.yml
@@ -2,7 +2,8 @@
 - job:
     name: elastic+elasticsearch+%BRANCH%+multijob+platform-support-arm
     display-name: "elastic / elasticsearch # %BRANCH% - arm compatibility"
-    description: "Elasticsearch %BRANCH% ARM (aarch64) compatibility testing.\n"
+    description: "This job has been migrated to Buildkite.\n"
+    disabled: true
     child-workspace: "/dev/shm/elastic+elasticsearch+%BRANCH%+multijob+platform-support-arm"
     project-type: matrix
     node: master
@@ -20,14 +21,14 @@
           type: user-defined
           name: GRADLE_TASK
           values:
-            - 'checkPart1'
-            - 'checkPart2'
-            - 'checkPart3'
-            - 'bwcTestSnapshots'
-            - 'checkRestCompat'
+            - "checkPart1"
+            - "checkPart2"
+            - "checkPart3"
+            - "bwcTestSnapshots"
+            - "checkRestCompat"
     builders:
       - inject:
-          properties-file: '.ci/java-versions-aarch64.properties'
+          properties-file: ".ci/java-versions-aarch64.properties"
           properties-content: |
             COMPOSE_HTTP_TIMEOUT=120
             JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA

diff --git a/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-unix.yml b/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-unix.yml
@@ -2,7 +2,8 @@
 - job:
     name: elastic+elasticsearch+%BRANCH%+multijob+platform-support-unix
     display-name: "elastic / elasticsearch # %BRANCH% - unix compatibility"
-    description: "Elasticsearch %BRANCH% unix compatibility testing.\n"
+    description: "This job has been migrated to Buildkite.\n"
+    disabled: true
     project-type: matrix
     node: master
     child-workspace: "/var/lib/jenkins/workspace/elastic+elasticsearch+%BRANCH%+multijob+platform-support-unix"
@@ -34,7 +35,7 @@
             - "almalinux-8&&immutable"
     builders:
       - inject:
-          properties-file: '.ci/java-versions.properties'
+          properties-file: ".ci/java-versions.properties"
           properties-content: |
             JAVA_HOME=$HOME/.java/$ES_BUILD_JAVA
             JAVA11_HOME=$HOME/.java/java11

diff --git a/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-windows.yml b/.ci/jobs.t/elastic+elasticsearch+multijob+platform-support-windows.yml
@@ -2,7 +2,8 @@
 - job:
     name: elastic+elasticsearch+%BRANCH%+multijob+platform-support-windows
     display-name: "elastic / elasticsearch # %BRANCH% - windows compatibility"
-    description: "Elasticsearch %BRANCH% Windows compatibility testing.\n"
+    description: "This job has been migrated to Buildkite.\n"
+    disabled: true
     project-type: matrix
     node: master
     # Use a hard-coded workspace directory to avoid hitting file path limits with auto-generated workspace path
@@ -25,14 +26,14 @@
           type: user-defined
           name: GRADLE_TASK
           values:
-            - 'checkPart1'
-            - 'checkPart2'
-            - 'checkPart3'
-            - 'bwcTestSnapshots'
-            - 'checkRestCompat'
+            - "checkPart1"
+            - "checkPart2"
+            - "checkPart3"
+            - "bwcTestSnapshots"
+            - "checkRestCompat"
     builders:
       - inject:
-          properties-file: '.ci/java-versions.properties'
+          properties-file: ".ci/java-versions.properties"
           properties-content: |
             JAVA_HOME=$USERPROFILE\\.java\\$ES_BUILD_JAVA
             JAVA11_HOME=$USERPROFILE\\.java\\java11

diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+platform-support-trigger.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+platform-support-trigger.yml
diff --git a/.ci/jobs.t/elastic+elasticsearch+periodic+platform-support.yml b/.ci/jobs.t/elastic+elasticsearch+periodic+platform-support.yml
@@ -2,7 +2,8 @@
 - job:
     name: elastic+elasticsearch+%BRANCH%+periodic+platform-support
     display-name: "elastic / elasticsearch # %BRANCH% - platform support"
-    description: "Testing of the Elasticsearch %BRANCH% branch platform support tests.\n"
+    description: "This job has been migrated to Buildkite.\n"
+    disabled: true
     project-type: multijob
     node: master
     vault: []

diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions
@@ -1,5 +1,5 @@
 BWC_VERSION:
-  - "7.17.14"
-  - "8.10.3"
+  - "7.17.15"
+  - "8.10.4"
   - "8.11.0"
   - "8.12.0"
diff --git a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties
@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=bb09982fdf52718e4c7b25023d10df6d35a5fff969860bdf5a5bd27a3ab27a9e
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.3-all.zip
+distributionSha256Sum=f2b9ed0faf8472cbe469255ae6c86eddb77076c75191741b4a462f33128dd419
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

diff --git a/build-tools-internal/src/main/groovy/elasticsearch.build-complete.gradle b/build-tools-internal/src/main/groovy/elasticsearch.build-complete.gradle
@@ -28,6 +28,7 @@ if (buildNumber && performanceTest == null && GradleUtils.isIncludedBuild(projec
             include("**/build/test-results/**/*.xml")
             include("**/build/testclusters/**")
             include("**/build/testrun/*/temp/**")
+            include("**/build/**/hs_err_pid*.log")
             exclude("**/build/testclusters/**/data/**")
             exclude("**/build/testclusters/**/distro/**")
             exclude("**/build/testclusters/**/repo/**")

diff --git a/build-tools-internal/src/main/resources/minimumGradleVersion b/build-tools-internal/src/main/resources/minimumGradleVersion
@@ -1 +1 @@
-8.3
+8.4
diff --git a/...ion/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java b/...ion/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java
@@ -23,7 +23,6 @@
 import org.bouncycastle.openpgp.operator.jcajce.JcaKeyFingerprintCalculator;
 import org.bouncycastle.openpgp.operator.jcajce.JcaPGPContentVerifierBuilderProvider;
 import org.elasticsearch.Build;
-import org.elasticsearch.Version;
 import org.elasticsearch.bootstrap.PluginPolicyInfo;
 import org.elasticsearch.bootstrap.PolicyUtil;
 import org.elasticsearch.cli.ExitCodes;
@@ -84,6 +83,8 @@
 import java.util.Set;
 import java.util.Timer;
 import java.util.TimerTask;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import java.util.zip.ZipEntry;
@@ -303,7 +304,7 @@ private Path download(InstallablePlugin plugin, Path tmpDir) throws Exception {
                 // else carry on to regular download
             }
 
-            final String url = getElasticUrl(getStagingHash(), Version.CURRENT, isSnapshot(), pluginId, Platforms.PLATFORM_NAME);
+            final String url = getElasticUrl(getStagingHash(), isSnapshot(), pluginId, Platforms.PLATFORM_NAME);
             terminal.println(logPrefix + "Downloading " + pluginId + " from elastic");
             return downloadAndValidate(url, tmpDir, true);
         }
@@ -341,7 +342,7 @@ private Path getPluginArchivePath(String pluginId, String pluginArchiveDir) thro
         if (Files.isDirectory(path) == false) {
             throw new UserException(ExitCodes.CONFIG, "Location in ES_PLUGIN_ARCHIVE_DIR is not a directory");
         }
-        return PathUtils.get(pluginArchiveDir, pluginId + "-" + Version.CURRENT + (isSnapshot() ? "-SNAPSHOT" : "") + ".zip");
+        return PathUtils.get(pluginArchiveDir, pluginId + "-" + Build.current().qualifiedVersion() + ".zip");
     }
 
     // pkg private so tests can override
@@ -356,25 +357,30 @@ boolean isSnapshot() {
     /**
      * Returns the url for an official elasticsearch plugin.
      */
-    private String getElasticUrl(
-        final String stagingHash,
-        final Version version,
-        final boolean isSnapshot,
-        final String pluginId,
-        final String platform
-    ) throws IOException, UserException {
+    private String getElasticUrl(final String stagingHash, final boolean isSnapshot, final String pluginId, final String platform)
+        throws IOException, UserException {
         final String baseUrl;
         if (isSnapshot && stagingHash == null) {
             throw new UserException(
                 ExitCodes.CONFIG,
                 "attempted to install release build of official plugin on snapshot build of Elasticsearch"
             );
         }
+        // assumption: we will only be publishing plugins to snapshot or staging when they're versioned
+        String semanticVersion = getSemanticVersion(Build.current().version());
+        if (semanticVersion == null) {
+            throw new UserException(
+                ExitCodes.CONFIG,
+                "attempted to download a plugin for a non-semantically-versioned build of Elasticsearch: ["
+                    + Build.current().version()
+                    + "]"
+            );
+        }
         if (stagingHash != null) {
             if (isSnapshot) {
-                baseUrl = nonReleaseUrl("snapshots", version, stagingHash, pluginId);
+                baseUrl = nonReleaseUrl("snapshots", semanticVersion, stagingHash, pluginId);
             } else {
-                baseUrl = nonReleaseUrl("staging", version, stagingHash, pluginId);
+                baseUrl = nonReleaseUrl("staging", semanticVersion, stagingHash, pluginId);
             }
         } else {
             baseUrl = String.format(Locale.ROOT, "https://artifacts.elastic.co/downloads/elasticsearch-plugins/%s", pluginId);
@@ -393,7 +399,7 @@ private String getElasticUrl(
         return String.format(Locale.ROOT, "%s/%s-%s.zip", baseUrl, pluginId, Build.current().qualifiedVersion());
     }
 
-    private static String nonReleaseUrl(final String hostname, final Version version, final String stagingHash, final String pluginId) {
+    private static String nonReleaseUrl(final String hostname, final String version, final String stagingHash, final String pluginId) {
         return String.format(
             Locale.ROOT,
             "https://%s.elastic.co/%s-%s/downloads/elasticsearch-plugins/%s",
@@ -1088,4 +1094,9 @@ private static void setFileAttributes(final Path path, final Set<PosixFilePermis
     public void close() throws IOException {
         IOUtils.rm(pathsToDeleteOnShutdown.toArray(new Path[0]));
     }
+
+    static String getSemanticVersion(String version) {
+        Matcher matcher = Pattern.compile("^(\\d+\\.\\d+\\.\\d+)\\D?.*").matcher(version);
+        return matcher.matches() ? matcher.group(1) : null;
+    }
 }