Merge branch 'making-BulkRequest-implement-RefCounted' into making-Bu…

…lkShardRequest-implement-RefCounted

.buildkite/packer_cache.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+ROOT_DIR=$(cd "$(dirname "$0")/.." && pwd)
+
+branches=($(cat "$ROOT_DIR/branches.json" | jq -r '.branches[].branch'))
+for branch in "${branches[@]}"; do
+  echo "Resolving dependencies for ${branch} branch"
+  rm -rf "checkout/$branch"
+  git clone /opt/git-mirrors/elastic-elasticsearch --branch "$branch" --single-branch "checkout/$branch"
+
+  CHECKOUT_DIR=$(cd "./checkout/${branch}" && pwd)
+  CI_DIR="$CHECKOUT_DIR/.ci"
+
+  if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then
+    ## On ARM we use a different properties file for setting java home
+    ## Also, we don't bother attempting to resolve dependencies for the 6.8 branch
+    source "$CI_DIR/java-versions-aarch64.properties"
+    export JAVA16_HOME="$HOME/.java/jdk16"
+  else
+    source "$CI_DIR/java-versions.properties"
+    ## We are caching BWC versions too, need these so we can build those
+    export JAVA8_HOME="$HOME/.java/java8"
+    export JAVA11_HOME="$HOME/.java/java11"
+    export JAVA12_HOME="$HOME/.java/openjdk12"
+    export JAVA13_HOME="$HOME/.java/openjdk13"
+    export JAVA14_HOME="$HOME/.java/openjdk14"
+    export JAVA15_HOME="$HOME/.java/openjdk15"
+    export JAVA16_HOME="$HOME/.java/openjdk16"
+  fi
+
+  export JAVA_HOME="$HOME/.java/$ES_BUILD_JAVA"
+  "checkout/${branch}/gradlew" --project-dir "$CHECKOUT_DIR" --parallel -s resolveAllDependencies -Dorg.gradle.warning.mode=none -DisCI
+  rm -rf "checkout/${branch}"
+done

.buildkite/pipelines/intake.yml

@@ -48,7 +48,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.16", "8.11.4", "8.12.0", "8.13.0"]
+            BWC_VERSION: ["7.17.17", "8.11.4", "8.12.0", "8.13.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -1089,6 +1089,22 @@ steps:
         env:
           BWC_VERSION: 7.17.16
 
+      - label: "{{matrix.image}} / 7.17.17 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.17
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 7.17.17
+
       - label: "{{matrix.image}} / 8.0.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.0.0
         timeout_in_minutes: 300

.buildkite/pipelines/periodic-platform-support.yml

@@ -80,3 +80,19 @@ steps:
           diskName: /dev/sda1
         env:
           GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
+  - group: platform-support-unix-aws
+    steps:
+      - label: "{{matrix.image}} / platform-support-aws"
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true functionalTests
+        timeout_in_minutes: 420
+        matrix:
+          setup:
+            image:
+              - amazonlinux-2023
+        agents:
+          provider: aws
+          imagePrefix: elasticsearch-{{matrix.image}}
+          instanceType: m6a.8xlarge
+          diskSizeGb: 350
+          diskType: gp3
+          diskName: /dev/sda1

.buildkite/pipelines/periodic.yml

@@ -662,6 +662,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 7.17.16
+      - label: 7.17.17 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.17#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 7.17.17
       - label: 8.0.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.0.0#bwcTest
         timeout_in_minutes: 300

.buildkite/scripts/periodic.trigger.sh

@@ -6,11 +6,26 @@ echo "steps:"
 
 source .buildkite/scripts/branches.sh
 
+IS_FIRST=true
+SKIP_DELAY="${SKIP_DELAY:-false}"
+
 for BRANCH in "${BRANCHES[@]}"; do
   INTAKE_PIPELINE_SLUG="elasticsearch-intake"
   BUILD_JSON=$(curl -sH "Authorization: Bearer ${BUILDKITE_API_TOKEN}" "https://api.buildkite.com/v2/organizations/elastic/pipelines/${INTAKE_PIPELINE_SLUG}/builds?branch=${BRANCH}&state=passed&per_page=1" | jq '.[0] | {commit: .commit, url: .web_url}')
   LAST_GOOD_COMMIT=$(echo "${BUILD_JSON}" | jq -r '.commit')
 
+  # Put a delay between each branch's set of pipelines by prepending each non-first branch with a sleep
+  # This is to smooth out the spike in agent requests
+  if [[ "$IS_FIRST" != "true" && "$SKIP_DELAY" != "true" ]]; then
+      cat <<EOF
+  - command: sleep 540
+    soft_fail: true
+  - wait: ~
+    continue_on_failure: true
+EOF
+  fi
+  IS_FIRST=false
+
   cat <<EOF
   - trigger: elasticsearch-periodic
     label: Trigger periodic pipeline for $BRANCH

.ci/bwcVersions

@@ -65,6 +65,7 @@ BWC_VERSION:
   - "7.17.14"
   - "7.17.15"
   - "7.17.16"
+  - "7.17.17"
   - "8.0.0"
   - "8.0.1"
   - "8.1.0"

.ci/snapshotBwcVersions

@@ -1,5 +1,5 @@
 BWC_VERSION:
-  - "7.17.16"
+  - "7.17.17"
   - "8.11.4"
   - "8.12.0"
   - "8.13.0"

.github/CODEOWNERS

@@ -23,5 +23,6 @@ x-pack/plugin/core/src/main/resources/fleet-* @elastic/fleet
 # Kibana Security
 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @elastic/kibana-security
 
-# APM
-x-pack/plugin/apm-data @elastic/apm-server
+# APM Data index templates, etc.
+x-pack/plugin/apm-data/src/main/resources @elastic/apm-server
+x-pack/plugin/apm-data/src/yamlRestTest/resources @elastic/apm-server

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java

@@ -109,6 +109,10 @@ private TaskProvider<LoggedExec> createRunBwcGradleTask(
                 loggedExec.args("-Dorg.elasticsearch.build.cache.url=" + buildCacheUrl);
             }
 
+            if (System.getProperty("isCI") != null) {
+                loggedExec.args("-DisCI");
+            }
+
             loggedExec.args("-Dbuild.snapshot=true", "-Dscan.tag.NESTED");
             final LogLevel logLevel = project.getGradle().getStartParameter().getLogLevel();
             List<LogLevel> nonDefaultLogLevels = Arrays.asList(LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/packer/CacheCacheableTestFixtures.java

@@ -25,7 +25,9 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.lang.reflect.Constructor;
 import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLClassLoader;
@@ -74,9 +76,14 @@ public void execute() {
                 Set<Class<?>> classes = (Set<Class<?>>) reflections.getSubTypesOf(ifClass);
 
                 for (Class<?> cacheableTestFixtureClazz : classes) {
-                    Object o = cacheableTestFixtureClazz.getDeclaredConstructor().newInstance();
-                    Method cacheMethod = cacheableTestFixtureClazz.getMethod("cache");
-                    cacheMethod.invoke(o);
+                    if (Modifier.isAbstract(cacheableTestFixtureClazz.getModifiers()) == false) {
+                        Constructor<?> declaredConstructor = cacheableTestFixtureClazz.getDeclaredConstructor();
+                        declaredConstructor.setAccessible(true);
+                        Object o = declaredConstructor.newInstance();
+                        Method cacheMethod = cacheableTestFixtureClazz.getMethod("cache");
+                        System.out.println("Caching resources from " + cacheableTestFixtureClazz.getName());
+                        cacheMethod.invoke(o);
+                    }
                 }
             } catch (Exception e) {
                 throw new RuntimeException(e);

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/JvmOptionsParser.java

@@ -10,6 +10,7 @@
 
 import org.elasticsearch.bootstrap.ServerArgs;
 import org.elasticsearch.cli.ExitCodes;
+import org.elasticsearch.cli.ProcessInfo;
 import org.elasticsearch.cli.UserException;
 
 import java.io.BufferedReader;
@@ -39,7 +40,7 @@
 /**
  * Parses JVM options from a file and prints a single line with all JVM options to standard output.
  */
-final class JvmOptionsParser {
+public final class JvmOptionsParser {
 
     static class JvmOptionsFileParserException extends Exception {
 
@@ -59,7 +60,6 @@ SortedMap<Integer, String> invalidLines() {
             this.jvmOptionsFile = jvmOptionsFile;
             this.invalidLines = invalidLines;
         }
-
     }
 
     /**
@@ -70,25 +70,27 @@ SortedMap<Integer, String> invalidLines() {
      * variable.
      *
      * @param args            the start-up arguments
-     * @param configDir       the ES config dir
+     * @param processInfo     information about the CLI process.
      * @param tmpDir          the directory that should be passed to {@code -Djava.io.tmpdir}
-     * @param envOptions      the options passed through the ES_JAVA_OPTS env var
      * @return the list of options to put on the Java command line
      * @throws InterruptedException if the java subprocess is interrupted
      * @throws IOException          if there is a problem reading any of the files
      * @throws UserException        if there is a problem parsing the `jvm.options` file or `jvm.options.d` files
      */
-    static List<String> determineJvmOptions(ServerArgs args, Path configDir, Path tmpDir, String envOptions) throws InterruptedException,
+    public static List<String> determineJvmOptions(ServerArgs args, ProcessInfo processInfo, Path tmpDir) throws InterruptedException,
         IOException, UserException {
-
         final JvmOptionsParser parser = new JvmOptionsParser();
 
         final Map<String, String> substitutions = new HashMap<>();
         substitutions.put("ES_TMPDIR", tmpDir.toString());
-        substitutions.put("ES_PATH_CONF", configDir.toString());
+        substitutions.put("ES_PATH_CONF", args.configDir().toString());
+
+        final String envOptions = processInfo.envVars().get("ES_JAVA_OPTS");
 
         try {
-            return parser.jvmOptions(args, configDir, tmpDir, envOptions, substitutions);
+            return Collections.unmodifiableList(
+                parser.jvmOptions(args, args.configDir(), tmpDir, envOptions, substitutions, processInfo.sysprops())
+            );
         } catch (final JvmOptionsFileParserException e) {
             final String errorMessage = String.format(
                 Locale.ROOT,
@@ -122,7 +124,8 @@ private List<String> jvmOptions(
         final Path config,
         Path tmpDir,
         final String esJavaOpts,
-        final Map<String, String> substitutions
+        final Map<String, String> substitutions,
+        final Map<String, String> cliSysprops
     ) throws InterruptedException, IOException, JvmOptionsFileParserException, UserException {
 
         final List<String> jvmOptions = readJvmOptionsFiles(config);
@@ -137,7 +140,7 @@ private List<String> jvmOptions(
         );
         substitutedJvmOptions.addAll(machineDependentHeap.determineHeapSettings(config, substitutedJvmOptions));
         final List<String> ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions);
-        final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions(args.nodeSettings());
+        final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions(args.nodeSettings(), cliSysprops);
 
         final List<String> apmOptions = APMJvmOptions.apmJvmOptions(args.nodeSettings(), args.secrets(), args.logsDir(), tmpDir);
 

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/ServerCli.java

@@ -243,8 +243,15 @@ protected Command loadTool(String toolname, String libs) {
     }
 
     // protected to allow tests to override
-    protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo, ServerArgs args) throws UserException {
-        return ServerProcess.start(terminal, processInfo, args);
+    protected ServerProcess startServer(Terminal terminal, ProcessInfo processInfo, ServerArgs args) throws Exception {
+        var tempDir = ServerProcessUtils.setupTempDir(processInfo);
+        var jvmOptions = JvmOptionsParser.determineJvmOptions(args, processInfo, tempDir);
+        var serverProcessBuilder = new ServerProcessBuilder().withTerminal(terminal)
+            .withProcessInfo(processInfo)
+            .withServerArgs(args)
+            .withTempDir(tempDir)
+            .withJvmOptions(jvmOptions);
+        return serverProcessBuilder.start();
     }
 
     // protected to allow tests to override