From ba52a225dd9d04e8adfde7998fe4698151cbb7b9 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Sun, 15 Dec 2024 10:49:25 +0100 Subject: [PATCH] [Build] Make checkPart4 configuration cache compatible (#117250) This makes checkPart4 tasks in general configuration cache compliant. We do not enable it by default in our pipelines yet. That will eventually be done in separate PRs. To track any regressions we cover the checkPart4 task in the weekly gradle cache validation build job --- .../gradle-configuration-cache-validation.sh | 6 ++-- .../gradle/internal/BwcSetupExtension.java | 31 ++++++++++++------- .../plugin/sql/qa/jdbc/security/build.gradle | 11 ++++--- .../sql/qa/server/security/build.gradle | 25 +++++++++++++-- 4 files changed, 50 insertions(+), 23 deletions(-) diff --git a/.buildkite/scripts/gradle-configuration-cache-validation.sh b/.buildkite/scripts/gradle-configuration-cache-validation.sh index 55a4b18a1e887..2257b4cfb17a5 100755 --- a/.buildkite/scripts/gradle-configuration-cache-validation.sh +++ b/.buildkite/scripts/gradle-configuration-cache-validation.sh @@ -3,16 +3,16 @@ set -euo pipefail # This is a workaround for https://github.com/gradle/gradle/issues/28159 -.ci/scripts/run-gradle.sh --no-daemon precommit +.ci/scripts/run-gradle.sh --no-daemon $@ -.ci/scripts/run-gradle.sh --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 +.ci/scripts/run-gradle.sh --configuration-cache -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 $@ # Create a temporary file tmpOutputFile=$(mktemp) trap "rm $tmpOutputFile" EXIT echo "2nd run" -.ci/scripts/run-gradle.sh --configuration-cache precommit -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 | tee $tmpOutputFile +.ci/scripts/run-gradle.sh --configuration-cache -Dorg.gradle.configuration-cache.inputs.unsafe.ignore.file-system-checks=build/*.tar.bz2 $@ | tee $tmpOutputFile # Check if the command was successful if grep -q "Configuration cache entry reused." $tmpOutputFile; then diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java index 5992a40275b46..6050e5c9a60d4 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BwcSetupExtension.java @@ -16,6 +16,7 @@ import org.gradle.api.Action; import org.gradle.api.GradleException; import org.gradle.api.Project; +import org.gradle.api.Task; import org.gradle.api.logging.LogLevel; import org.gradle.api.model.ObjectFactory; import org.gradle.api.provider.Property; @@ -103,10 +104,16 @@ private static TaskProvider createRunBwcGradleTask( loggedExec.dependsOn("checkoutBwcBranch"); loggedExec.getWorkingDir().set(checkoutDir.get()); - loggedExec.getNonTrackedEnvironment().put("JAVA_HOME", providerFactory.of(JavaHomeValueSource.class, spec -> { - spec.getParameters().getVersion().set(unreleasedVersionInfo.map(it -> it.version())); - spec.getParameters().getCheckoutDir().set(checkoutDir); - }).flatMap(s -> getJavaHome(objectFactory, toolChainService, Integer.parseInt(s)))); + loggedExec.doFirst(new Action() { + @Override + public void execute(Task task) { + Provider minimumCompilerVersionValueSource = providerFactory.of(JavaHomeValueSource.class, spec -> { + spec.getParameters().getVersion().set(unreleasedVersionInfo.map(it -> it.version())); + spec.getParameters().getCheckoutDir().set(checkoutDir); + }).flatMap(s -> getJavaHome(objectFactory, toolChainService, Integer.parseInt(s))); + loggedExec.getNonTrackedEnvironment().put("JAVA_HOME", minimumCompilerVersionValueSource.get()); + } + }); if (isCi && OS.current() != OS.WINDOWS) { // TODO: Disabled for now until we can figure out why files are getting corrupted @@ -169,14 +176,6 @@ private static Provider getJavaHome(ObjectFactory objectFactory, JavaToo .map(launcher -> launcher.getMetadata().getInstallationPath().getAsFile().getAbsolutePath()); } - private static String readFromFile(File file) { - try { - return FileUtils.readFileToString(file).trim(); - } catch (IOException ioException) { - throw new GradleException("Cannot read java properties file.", ioException); - } - } - public abstract static class JavaHomeValueSource implements ValueSource { private String minimumCompilerVersionPath(Version bwcVersion) { @@ -185,6 +184,14 @@ private String minimumCompilerVersionPath(Version bwcVersion) { : "buildSrc/" + MINIMUM_COMPILER_VERSION_PATH; } + private static String readFromFile(File file) { + try { + return FileUtils.readFileToString(file).trim(); + } catch (IOException ioException) { + throw new GradleException("Cannot read java properties file.", ioException); + } + } + @Override public String obtain() { return readFromFile( diff --git a/x-pack/plugin/sql/qa/jdbc/security/build.gradle b/x-pack/plugin/sql/qa/jdbc/security/build.gradle index 82510285cb996..b40a8aaaa5580 100644 --- a/x-pack/plugin/sql/qa/jdbc/security/build.gradle +++ b/x-pack/plugin/sql/qa/jdbc/security/build.gradle @@ -50,6 +50,7 @@ subprojects { tasks.withType(RestIntegTestTask).configureEach { + def taskName = name dependsOn copyTestClasses classpath += configurations.testArtifacts testClassesDirs = project.files(testArtifactsDir) @@ -58,15 +59,15 @@ subprojects { project.gradle.sharedServices, TestClustersPlugin.REGISTRY_SERVICE_NAME ) - project.getProviders().of(TestClusterValueSource.class) { + + def clusterInfo = project.getProviders().of(TestClusterValueSource.class) { it.parameters.path.set(clusterPath) - it.parameters.clusterName.set("javaRestTest") + it.parameters.clusterName.set(taskName) it.parameters.service = serviceProvider } - nonInputProperties.systemProperty 'tests.audit.logfile', - "${-> testClusters.javaRestTest.singleNode().getAuditLog()}" + nonInputProperties.systemProperty 'tests.audit.logfile', clusterInfo.map { it.auditLogs.get(0) } nonInputProperties.systemProperty 'tests.audit.yesterday.logfile', - "${-> testClusters.javaRestTest.singleNode().getAuditLog().getParentFile()}/javaRestTest_audit-${new Date().format('yyyy-MM-dd')}.json" + clusterInfo.map { it.auditLogs.get(0).getParentFile().toString() + "/javaRestTest_audit-${new Date().format('yyyy-MM-dd')}-1.json" } } } diff --git a/x-pack/plugin/sql/qa/server/security/build.gradle b/x-pack/plugin/sql/qa/server/security/build.gradle index 2d9f7b563d073..ef81ebda5cb89 100644 --- a/x-pack/plugin/sql/qa/server/security/build.gradle +++ b/x-pack/plugin/sql/qa/server/security/build.gradle @@ -1,3 +1,9 @@ +import org.elasticsearch.gradle.internal.test.RestIntegTestTask +import org.elasticsearch.gradle.testclusters.TestClusterValueSource +import org.elasticsearch.gradle.testclusters.TestClustersPlugin +import org.elasticsearch.gradle.testclusters.TestClustersRegistry +import org.elasticsearch.gradle.util.GradleUtils + apply plugin: 'elasticsearch.internal-test-artifact' dependencies { @@ -16,6 +22,8 @@ subprojects { // Use tests from the root security qa project in subprojects configurations.create('testArtifacts').transitive(false) + def clusterPath = getPath() + dependencies { javaRestTestImplementation project(":x-pack:plugin:core") javaRestTestImplementation(testArtifact(project(xpackModule('core')))) @@ -50,12 +58,23 @@ subprojects { tasks.named("javaRestTest").configure { dependsOn copyTestClasses + + Provider serviceProvider = GradleUtils.getBuildService( + project.gradle.sharedServices, + TestClustersPlugin.REGISTRY_SERVICE_NAME + ) + + def clusterInfo = project.getProviders().of(TestClusterValueSource.class) { + it.parameters.path.set(clusterPath) + it.parameters.clusterName.set("javaRestTest") + it.parameters.service = serviceProvider + } + testClassesDirs += project.files(testArtifactsDir) classpath += configurations.testArtifacts - nonInputProperties.systemProperty 'tests.audit.logfile', - "${-> testClusters.javaRestTest.singleNode().getAuditLog()}" + nonInputProperties.systemProperty 'tests.audit.logfile', clusterInfo.map { it.auditLogs.get(0) } nonInputProperties.systemProperty 'tests.audit.yesterday.logfile', - "${-> testClusters.javaRestTest.singleNode().getAuditLog().getParentFile()}/javaRestTest_audit-${new Date().format('yyyy-MM-dd')}-1.json.gz" + clusterInfo.map { it.auditLogs.get(0).getParentFile().toString() + "/javaRestTest_audit-${new Date().format('yyyy-MM-dd')}-1.json.gz"} } }