From 51e229e0b2a780c311c6a25e22b39cd0c8a00ed3 Mon Sep 17 00:00:00 2001 From: Rene Groeschke Date: Sat, 14 Dec 2024 22:30:21 +0100 Subject: [PATCH] [Build]Make thirdparty audit tasks uptodate more effective Filtering out the project dependencies allows way better uptodate and caching behaviour. We are only interested in thirdparty libs anyhow in this context. --- .../ThirdPartyAuditPrecommitPlugin.java | 8 +++- .../precommit/ThirdPartyAuditTask.java | 15 ++----- .../internal/util/DependenciesUtils.java | 7 ++++ x-pack/plugin/identity-provider/build.gradle | 8 +++- x-pack/plugin/security/build.gradle | 42 ++++++++++--------- 5 files changed, 45 insertions(+), 35 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java index e45a1d3dd25b1..95778b672f353 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java @@ -16,12 +16,14 @@ import org.gradle.api.Task; import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.component.ModuleComponentIdentifier; +import org.gradle.api.file.FileCollection; import org.gradle.api.tasks.TaskProvider; import java.io.File; import java.nio.file.Path; import static org.elasticsearch.gradle.internal.util.DependenciesUtils.createFileCollectionFromNonTransitiveArtifactsView; +import static org.elasticsearch.gradle.internal.util.DependenciesUtils.projectedDependenciesFilteredView; import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams; public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin { @@ -59,9 +61,11 @@ public TaskProvider createTask(Project project) { // usually only one task is created. but this construct makes our integTests easier to setup project.getTasks().withType(ThirdPartyAuditTask.class).configureEach(t -> { Configuration runtimeConfiguration = project.getConfigurations().getByName("runtimeClasspath"); + FileCollection runtimeThirdParty = projectedDependenciesFilteredView(runtimeConfiguration); Configuration compileOnly = project.getConfigurations() .getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME); - t.setClasspath(runtimeConfiguration.plus(compileOnly)); + FileCollection compileOnlyThirdParty = projectedDependenciesFilteredView(compileOnly); + t.getThirdPartyClasspath().from(runtimeThirdParty, compileOnlyThirdParty); t.getJarsToScan() .from( createFileCollectionFromNonTransitiveArtifactsView( @@ -78,7 +82,7 @@ public TaskProvider createTask(Project project) { t.getJavaHome().set(buildParams.flatMap(params -> params.getRuntimeJavaHome()).map(File::getPath)); t.setSignatureFile(resourcesDir.resolve("forbidden/third-party-audit.txt").toFile()); t.getJdkJarHellClasspath().from(jdkJarHellConfig); - t.getForbiddenAPIsClasspath().from(project.getConfigurations().getByName("forbiddenApisCliJar").plus(compileOnly)); + t.getForbiddenAPIsClasspath().from(project.getConfigurations().getByName("forbiddenApisCliJar").plus(compileOnlyThirdParty)); }); return audit; } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java index 442797775de2f..59ba9bae0a57d 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java @@ -17,7 +17,6 @@ import org.gradle.api.JavaVersion; import org.gradle.api.file.ArchiveOperations; import org.gradle.api.file.ConfigurableFileCollection; -import org.gradle.api.file.FileCollection; import org.gradle.api.file.FileSystemOperations; import org.gradle.api.file.FileTree; import org.gradle.api.file.ProjectLayout; @@ -96,8 +95,6 @@ public abstract class ThirdPartyAuditTask extends DefaultTask { private final ProjectLayout projectLayout; - private FileCollection classpath; - @Inject public ThirdPartyAuditTask( ArchiveOperations archiveOperations, @@ -198,9 +195,7 @@ public Set getMissingClassExcludes() { public abstract Property getRuntimeJavaVersion(); @Classpath - public FileCollection getClasspath() { - return classpath; - } + public abstract ConfigurableFileCollection getThirdPartyClasspath(); @TaskAction public void runThirdPartyAudit() throws IOException { @@ -345,7 +340,7 @@ private String runForbiddenAPIsCli() throws IOException { if (javaHome.isPresent()) { spec.setExecutable(javaHome.get() + "/bin/java"); } - spec.classpath(getForbiddenAPIsClasspath(), classpath); + spec.classpath(getForbiddenAPIsClasspath(), getThirdPartyClasspath()); // Enable explicitly for each release as appropriate. Just JDK 20/21/22/23 for now, and just the vector module. if (isJavaVersion(VERSION_20) || isJavaVersion(VERSION_21) || isJavaVersion(VERSION_22) || isJavaVersion(VERSION_23)) { spec.jvmArgs("--add-modules", "jdk.incubator.vector"); @@ -383,7 +378,7 @@ private boolean isJavaVersion(JavaVersion version) { private Set runJdkJarHellCheck() throws IOException { ByteArrayOutputStream standardOut = new ByteArrayOutputStream(); ExecResult execResult = execOperations.javaexec(spec -> { - spec.classpath(getJdkJarHellClasspath(), classpath); + spec.classpath(getJdkJarHellClasspath(), getThirdPartyClasspath()); spec.getMainClass().set(JDK_JAR_HELL_MAIN_CLASS); spec.args(getJarExpandDir()); spec.setIgnoreExitValue(true); @@ -402,8 +397,4 @@ private Set runJdkJarHellCheck() throws IOException { return new TreeSet<>(Arrays.asList(jdkJarHellCheckList.split("\\r?\\n"))); } - public void setClasspath(FileCollection classpath) { - this.classpath = classpath; - } - } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java index 9080f62f19937..a8b0478a237bb 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java @@ -12,11 +12,13 @@ import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.ResolvableDependencies; import org.gradle.api.artifacts.component.ComponentIdentifier; +import org.gradle.api.artifacts.component.ProjectComponentIdentifier; import org.gradle.api.artifacts.result.ResolvedComponentResult; import org.gradle.api.artifacts.result.ResolvedDependencyResult; import org.gradle.api.file.FileCollection; import org.gradle.api.specs.AndSpec; import org.gradle.api.specs.Spec; +import org.jetbrains.annotations.NotNull; import java.util.Set; import java.util.stream.Collectors; @@ -47,4 +49,9 @@ public static FileCollection createFileCollectionFromNonTransitiveArtifactsView( }).getFiles(); } + public static @NotNull FileCollection projectedDependenciesFilteredView(Configuration configuration) { + return configuration.getIncoming() + .artifactView(v -> v.componentFilter(i -> (i instanceof ProjectComponentIdentifier == false))) + .getFiles(); + } } diff --git a/x-pack/plugin/identity-provider/build.gradle b/x-pack/plugin/identity-provider/build.gradle index f9c121da0f550..13dd97499ed29 100644 --- a/x-pack/plugin/identity-provider/build.gradle +++ b/x-pack/plugin/identity-provider/build.gradle @@ -8,6 +8,7 @@ apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.publish' apply plugin: 'elasticsearch.internal-cluster-test' + esplugin { name 'x-pack-identity-provider' description 'Elasticsearch Expanded Pack Plugin - Identity Provider' @@ -19,6 +20,10 @@ base { archivesName = 'x-pack-identity-provider' } +configurations { + shadowedDeps +} + dependencies { compileOnly project(path: xpackModule('core')) @@ -29,6 +34,7 @@ dependencies { api "org.opensaml:opensaml-messaging-api:${versions.opensaml}" api "org.opensaml:opensaml-messaging-impl:${versions.opensaml}" api project(path: ':x-pack:libs:es-opensaml-security-api', configuration: 'shadow') + shadowedDeps project(path: ':x-pack:libs:es-opensaml-security-api', configuration: 'shadow') api "org.opensaml:opensaml-security-impl:${versions.opensaml}" api "org.opensaml:opensaml-profile-api:${versions.opensaml}" api "org.opensaml:opensaml-profile-impl:${versions.opensaml}" @@ -64,7 +70,6 @@ dependencies { testImplementation(testArtifact(project(xpackModule('security')))) testImplementation project(':modules:lang-mustache') internalClusterTestImplementation project(":modules:analysis-common") - } tasks.named("dependencyLicenses").configure { @@ -87,6 +92,7 @@ tasks.named('forbiddenApisMain').configure { // classes are missing, e.g. com.ibm.icu.lang.UCharacter tasks.named("thirdPartyAudit").configure { + thirdPartyClasspath.from(configurations.shadowedDeps) ignoreMissingClasses( // SAML dependencies // [missing classes] Some cli utilities that we don't use depend on these missing JCommander classes diff --git a/x-pack/plugin/security/build.gradle b/x-pack/plugin/security/build.gradle index ebd435bf653a2..d5090f5469d4b 100644 --- a/x-pack/plugin/security/build.gradle +++ b/x-pack/plugin/security/build.gradle @@ -15,6 +15,10 @@ base { archivesName = 'x-pack-security' } +configurations { + shadowedDeps +} + dependencies { compileOnly project(path: xpackModule('core')) api project(path: ':modules:transport-netty4') @@ -47,6 +51,7 @@ dependencies { api "org.opensaml:opensaml-messaging-api:${versions.opensaml}" api "org.opensaml:opensaml-messaging-impl:${versions.opensaml}" api project(path: ':x-pack:libs:es-opensaml-security-api', configuration: 'shadow') + shadowedDeps project(path: ':x-pack:libs:es-opensaml-security-api', configuration: 'shadow') // api "org.opensaml:opensaml-security-api:${versions.opensaml}" api "org.opensaml:opensaml-security-impl:${versions.opensaml}" api "org.opensaml:opensaml-profile-api:${versions.opensaml}" @@ -81,6 +86,7 @@ dependencies { // Dependencies for oidc api "com.nimbusds:oauth2-oidc-sdk:11.10.1" api project(path: xpackModule('security:lib:nimbus-jose-jwt-modified'), configuration: 'shadow') + shadowedDeps project(path: xpackModule('security:lib:nimbus-jose-jwt-modified'), configuration: 'shadow') if (isEclipse) { /* * Eclipse can't pick up the shadow dependency so we point it at the unmodified version of the library @@ -212,6 +218,7 @@ tasks.named('forbiddenApisTest').configure { // classes are missing, e.g. com.ibm.icu.lang.UCharacter tasks.named("thirdPartyAudit").configure { + thirdPartyClasspath.from(configurations.shadowedDeps) ignoreMissingClasses( // SAML dependencies // [missing classes] Some cli utilities that we don't use depend on these missing JCommander classes @@ -385,6 +392,21 @@ tasks.named("thirdPartyAudit").configure { 'org.bouncycastle.util.Arrays', 'org.bouncycastle.util.io.Streams', 'org.bouncycastle.cert.X509CertificateHolder', + 'javax.xml.bind.JAXBContext', + 'javax.xml.bind.JAXBElement', + 'javax.xml.bind.JAXBException', + 'javax.xml.bind.Unmarshaller', + 'javax.xml.bind.UnmarshallerHandler', + // Optional dependency of oauth2-oidc-sdk that we don't need since we do not support AES-SIV for JWE + 'org.cryptomator.siv.SivMode', + 'com.nimbusds.common.contenttype.ContentType', + 'com.nimbusds.common.contenttype.ContentType$Parameter', + 'javax.activation.ActivationDataFlavor', + 'javax.activation.DataContentHandler', + 'javax.activation.DataHandler', + 'javax.activation.DataSource', + 'javax.activation.FileDataSource', + 'javax.activation.FileTypeMap' ) ignoreViolations( @@ -405,26 +427,6 @@ tasks.named("thirdPartyAudit").configure { ) } -tasks.named("thirdPartyAudit").configure { - ignoreMissingClasses( - 'javax.xml.bind.JAXBContext', - 'javax.xml.bind.JAXBElement', - 'javax.xml.bind.JAXBException', - 'javax.xml.bind.Unmarshaller', - 'javax.xml.bind.UnmarshallerHandler', - // Optional dependency of oauth2-oidc-sdk that we don't need since we do not support AES-SIV for JWE - 'org.cryptomator.siv.SivMode', - 'com.nimbusds.common.contenttype.ContentType', - 'com.nimbusds.common.contenttype.ContentType$Parameter', - 'javax.activation.ActivationDataFlavor', - 'javax.activation.DataContentHandler', - 'javax.activation.DataHandler', - 'javax.activation.DataSource', - 'javax.activation.FileDataSource', - 'javax.activation.FileTypeMap' - ) -} - tasks.named("internalClusterTest").configure { /* * Some tests in this module set up a lot of transport threads so we reduce the buffer size per transport thread from the 1M default