Merge branch 'main' into adding-simulate-ingest-api

.buildkite/pipelines/intake.yml

@@ -40,7 +40,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.15", "8.11.1", "8.12.0"]
+            BWC_VERSION: ["7.17.15", "8.11.2", "8.12.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -1713,6 +1713,22 @@ steps:
         env:
           BWC_VERSION: 8.11.1
 
+      - label: "{{matrix.image}} / 8.11.2 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.11.2
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.11.2
+
       - label: "{{matrix.image}} / 8.12.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.12.0
         timeout_in_minutes: 300

.buildkite/pipelines/periodic.yml

@@ -1052,6 +1052,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.11.1
+      - label: 8.11.2 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.11.2#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.11.2
       - label: 8.12.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.12.0#bwcTest
         timeout_in_minutes: 300

.ci/bwcVersions

@@ -104,4 +104,5 @@ BWC_VERSION:
   - "8.10.4"
   - "8.11.0"
   - "8.11.1"
+  - "8.11.2"
   - "8.12.0"

.ci/snapshotBwcVersions

@@ -1,4 +1,4 @@
 BWC_VERSION:
   - "7.17.15"
-  - "8.11.1"
+  - "8.11.2"
   - "8.12.0"

build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle

@@ -142,18 +142,17 @@ buildScan {
           // Add a build annotation
           // See: https://buildkite.com/docs/agent/v3/cli-annotate
           def body = """<div class="mb3"><span class="p1 border rounded">${System.getenv('BUILDKITE_LABEL')}</span> :gradle: ${result.failure ? 'failed' : 'successful'} build: <a href="${scan.buildScanUri}"><code>gradle ${gradle.startParameter.taskNames.join(' ')}</code></a></div>"""
-          new ProcessBuilder(
+          def process = [
             'buildkite-agent',
             'annotate',
             '--context',
             result.failure ? 'gradle-build-scans-failed' : 'gradle-build-scans',
             '--append',
             '--style',
-            result.failure ? 'error' : 'info',
-            body
-          )
-            .start()
-            .waitFor()
+            result.failure ? 'error' : 'info'
+          ].execute()
+          process.withWriter { it.write(body) } // passing the body in as an argument has issues on Windows, so let's use stdin of the process instead
+          process.waitFor()
         }
       }
     } else {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java

@@ -286,11 +286,12 @@ static void createBuildBwcTask(
             if (project.getGradle().getStartParameter().isBuildCacheEnabled()) {
                 c.getArgs().add("--build-cache");
             }
+            File rootDir = project.getRootDir();
             c.doLast(new Action<Task>() {
                 @Override
                 public void execute(Task task) {
                     if (expectedOutputFile.exists() == false) {
-                        Path relativeOutputPath = project.getRootDir().toPath().relativize(expectedOutputFile.toPath());
+                        Path relativeOutputPath = rootDir.toPath().relativize(expectedOutputFile.toPath());
                         final String message = "Building %s didn't generate expected artifact [%s]. The working branch may be "
                             + "out-of-date - try merging in the latest upstream changes to the branch.";
                         throw new InvalidUserDataException(message.formatted(bwcVersion.get(), relativeOutputPath));

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionDownloadPlugin.java

@@ -23,16 +23,17 @@
 import org.elasticsearch.gradle.internal.info.GlobalBuildInfoPlugin;
 import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.GradleException;
-import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.artifacts.Dependency;
+import org.gradle.api.artifacts.dsl.DependencyHandler;
 import org.gradle.api.provider.Provider;
 
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.function.Function;
 
-import static org.elasticsearch.gradle.util.GradleUtils.projectDependency;
-
 /**
  * An internal elasticsearch build plugin that registers additional
  * distribution resolution strategies to the 'elasticsearch.download-distribution' plugin
@@ -64,18 +65,18 @@ public void apply(Project project) {
      * <p>
      * BWC versions are resolved as project to projects under `:distribution:bwc`.
      */
-    private void registerInternalDistributionResolutions(NamedDomainObjectContainer<DistributionResolution> resolutions) {
-        resolutions.register("localBuild", distributionResolution -> distributionResolution.setResolver((project, distribution) -> {
+    private void registerInternalDistributionResolutions(List<DistributionResolution> resolutions) {
+        resolutions.add(new DistributionResolution("local-build", (project, distribution) -> {
             if (isCurrentVersion(distribution)) {
                 // non-external project, so depend on local build
                 return new ProjectBasedDistributionDependency(
-                    config -> projectDependency(project, distributionProjectPath(distribution), config)
+                    config -> projectDependency(project.getDependencies(), distributionProjectPath(distribution), config)
                 );
             }
             return null;
         }));
 
-        resolutions.register("bwc", distributionResolution -> distributionResolution.setResolver((project, distribution) -> {
+        resolutions.add(new DistributionResolution("bwc", (project, distribution) -> {
             BwcVersions.UnreleasedVersionInfo unreleasedInfo = BuildParams.getBwcVersions()
                 .unreleasedInfo(Version.fromString(distribution.getVersion()));
             if (unreleasedInfo != null) {
@@ -89,7 +90,7 @@ private void registerInternalDistributionResolutions(NamedDomainObjectContainer<
                 }
                 String projectConfig = getProjectConfig(distribution, unreleasedInfo);
                 return new ProjectBasedDistributionDependency(
-                    (config) -> projectDependency(project, unreleasedInfo.gradleProjectPath(), projectConfig)
+                    (config) -> projectDependency(project.getDependencies(), unreleasedInfo.gradleProjectPath(), projectConfig)
                 );
             }
             return null;
@@ -116,6 +117,13 @@ private static String getProjectConfig(ElasticsearchDistribution distribution, B
         }
     }
 
+    private static Dependency projectDependency(DependencyHandler dependencyHandler, String projectPath, String projectConfig) {
+        Map<String, Object> depConfig = new HashMap<>();
+        depConfig.put("path", projectPath);
+        depConfig.put("configuration", projectConfig);
+        return dependencyHandler.project(depConfig);
+    }
+
     private static String distributionProjectPath(ElasticsearchDistribution distribution) {
         String projectPath = ":distribution";
         if (distribution.getType() == ElasticsearchDistributionTypes.INTEG_TEST_ZIP) {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/LegacyRestTestBasePlugin.java

@@ -22,12 +22,18 @@
 import org.gradle.api.NamedDomainObjectContainer;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
+import org.gradle.api.Task;
 import org.gradle.api.plugins.JavaBasePlugin;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
+import org.gradle.api.specs.NotSpec;
+import org.gradle.api.specs.Spec;
 import org.gradle.api.tasks.Sync;
+import org.gradle.api.tasks.TaskContainer;
 import org.gradle.api.tasks.bundling.Zip;
 
+import java.util.Collections;
+
 import javax.inject.Inject;
 
 import static org.elasticsearch.gradle.internal.RestrictedBuildApiService.BUILD_API_RESTRICTIONS_SYS_PROPERTY;
@@ -47,6 +53,7 @@ public class LegacyRestTestBasePlugin implements Plugin<Project> {
     private static final String TESTS_CLUSTER_REMOTE_ACCESS = "tests.cluster.remote_access";
 
     private ProviderFactory providerFactory;
+    private Project project;
 
     @Inject
     public LegacyRestTestBasePlugin(ProviderFactory providerFactory) {
@@ -55,6 +62,7 @@ public LegacyRestTestBasePlugin(ProviderFactory providerFactory) {
 
     @Override
     public void apply(Project project) {
+        this.project = project;
         Provider<RestrictedBuildApiService> serviceProvider = project.getGradle()
             .getSharedServices()
             .registerIfAbsent("restrictedBuildAPI", RestrictedBuildApiService.class, spec -> {
@@ -118,9 +126,30 @@ public void apply(Project project) {
                     t.getClusters().forEach(c -> c.plugin(bundle));
                 }
             });
+            configureCacheability(t);
         });
     }
 
+    private void configureCacheability(StandaloneRestIntegTestTask testTask) {
+        TaskContainer tasks = project.getTasks();
+        Spec<Task> taskSpec = t -> tasks.withType(StandaloneRestIntegTestTask.class)
+            .stream()
+            .filter(task -> task != testTask)
+            .anyMatch(task -> Collections.disjoint(task.getClusters(), testTask.getClusters()) == false);
+        testTask.getOutputs()
+            .doNotCacheIf(
+                "Caching disabled for this task since it uses a cluster shared by other tasks",
+                /*
+                 * Look for any other tasks which use the same cluster as this task. Since tests often have side effects for the cluster
+                 * they execute against, this state can cause issues when trying to cache tests results of tasks that share a cluster. To
+                 * avoid any undesired behavior we simply disable the cache if we detect that this task uses a cluster shared between
+                 * multiple tasks.
+                 */
+                taskSpec
+            );
+        testTask.getOutputs().upToDateWhen(new NotSpec(taskSpec));
+    }
+
     private String systemProperty(String propName) {
         return providerFactory.systemProperty(propName).getOrNull();
     }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/RestTestBasePlugin.java

@@ -38,6 +38,8 @@
 import org.gradle.api.artifacts.ProjectDependency;
 import org.gradle.api.artifacts.type.ArtifactTypeDefinition;
 import org.gradle.api.attributes.Attribute;
+import org.gradle.api.file.ConfigurableFileCollection;
+import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.FileTree;
 import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.tasks.ClasspathNormalizer;
@@ -134,16 +136,20 @@ public void apply(Project project) {
             task.systemProperty("tests.system_call_filter", "false");
 
             // Register plugins and modules as task inputs and pass paths as system properties to tests
-            nonInputSystemProperties.systemProperty(TESTS_CLUSTER_MODULES_PATH_SYSPROP, modulesConfiguration::getAsPath);
-            registerConfigurationInputs(task, modulesConfiguration);
-            nonInputSystemProperties.systemProperty(TESTS_CLUSTER_PLUGINS_PATH_SYSPROP, pluginsConfiguration::getAsPath);
-            registerConfigurationInputs(task, extractedPluginsConfiguration);
+            var modulePath = project.getObjects().fileCollection().from(modulesConfiguration);
+            nonInputSystemProperties.systemProperty(TESTS_CLUSTER_MODULES_PATH_SYSPROP, modulePath::getAsPath);
+            registerConfigurationInputs(task, modulesConfiguration.getName(), modulePath);
+            var pluginPath = project.getObjects().fileCollection().from(pluginsConfiguration);
+            nonInputSystemProperties.systemProperty(TESTS_CLUSTER_PLUGINS_PATH_SYSPROP, pluginPath::getAsPath);
+            registerConfigurationInputs(
+                task,
+                extractedPluginsConfiguration.getName(),
+                project.getObjects().fileCollection().from(extractedPluginsConfiguration)
+            );
 
             // Wire up integ-test distribution by default for all test tasks
-            nonInputSystemProperties.systemProperty(
-                INTEG_TEST_DISTRIBUTION_SYSPROP,
-                () -> integTestDistro.getExtracted().getSingleFile().getPath()
-            );
+            FileCollection extracted = integTestDistro.getExtracted();
+            nonInputSystemProperties.systemProperty(INTEG_TEST_DISTRIBUTION_SYSPROP, () -> extracted.getSingleFile().getPath());
             nonInputSystemProperties.systemProperty(TESTS_RUNTIME_JAVA_SYSPROP, BuildParams.getRuntimeJavaHome());
 
             // Add `usesDefaultDistribution()` extension method to test tasks to indicate they require the default distro
@@ -216,15 +222,15 @@ private FileTree getDistributionFiles(ElasticsearchDistribution distribution, Ac
         return distribution.getExtracted().getAsFileTree().matching(patternFilter);
     }
 
-    private void registerConfigurationInputs(Task task, Configuration configuration) {
+    private void registerConfigurationInputs(Task task, String configurationName, ConfigurableFileCollection configuration) {
         task.getInputs()
             .files(providerFactory.provider(() -> configuration.getAsFileTree().filter(f -> f.getName().endsWith(".jar") == false)))
-            .withPropertyName(configuration.getName() + "-files")
+            .withPropertyName(configurationName + "-files")
             .withPathSensitivity(PathSensitivity.RELATIVE);
 
         task.getInputs()
             .files(providerFactory.provider(() -> configuration.getAsFileTree().filter(f -> f.getName().endsWith(".jar"))))
-            .withPropertyName(configuration.getName() + "-classpath")
+            .withPropertyName(configurationName + "-classpath")
             .withNormalizer(ClasspathNormalizer.class);
     }
 

build-tools/src/integTest/groovy/org/elasticsearch/gradle/TestClustersPluginFuncTest.groovy

@@ -34,7 +34,7 @@ class TestClustersPluginFuncTest extends AbstractGradleFuncTest {
                 id 'elasticsearch.testclusters'
             }
 
-            class SomeClusterAwareTask extends DefaultTask implements TestClustersAware {
+            abstract class SomeClusterAwareTask extends DefaultTask implements TestClustersAware {
 
                 private Collection<ElasticsearchCluster> clusters = new HashSet<>();