[Build]Make thirdparty audit tasks uptodate more effective

Filtering out the project dependencies allows way better uptodate and caching behaviour. We are only interested in thirdparty libs anyhow in this context.
breskeby · Dec 14, 2024 · 51e229e · 51e229e
1 parent d695020
commit 51e229e
Show file tree

Hide file tree

Showing 5 changed files with 45 additions and 35 deletions.
diff --git a/...main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java b/...main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java
@@ -16,12 +16,14 @@
 import org.gradle.api.Task;
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.component.ModuleComponentIdentifier;
+import org.gradle.api.file.FileCollection;
 import org.gradle.api.tasks.TaskProvider;
 
 import java.io.File;
 import java.nio.file.Path;
 
 import static org.elasticsearch.gradle.internal.util.DependenciesUtils.createFileCollectionFromNonTransitiveArtifactsView;
+import static org.elasticsearch.gradle.internal.util.DependenciesUtils.projectedDependenciesFilteredView;
 import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams;
 
 public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin {
@@ -59,9 +61,11 @@ public TaskProvider<? extends Task> createTask(Project project) {
         // usually only one task is created. but this construct makes our integTests easier to setup
         project.getTasks().withType(ThirdPartyAuditTask.class).configureEach(t -> {
             Configuration runtimeConfiguration = project.getConfigurations().getByName("runtimeClasspath");
+            FileCollection runtimeThirdParty = projectedDependenciesFilteredView(runtimeConfiguration);
             Configuration compileOnly = project.getConfigurations()
                 .getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME);
-            t.setClasspath(runtimeConfiguration.plus(compileOnly));
+            FileCollection compileOnlyThirdParty = projectedDependenciesFilteredView(compileOnly);
+            t.getThirdPartyClasspath().from(runtimeThirdParty, compileOnlyThirdParty);
             t.getJarsToScan()
                 .from(
                     createFileCollectionFromNonTransitiveArtifactsView(
@@ -78,7 +82,7 @@ public TaskProvider<? extends Task> createTask(Project project) {
             t.getJavaHome().set(buildParams.flatMap(params -> params.getRuntimeJavaHome()).map(File::getPath));
             t.setSignatureFile(resourcesDir.resolve("forbidden/third-party-audit.txt").toFile());
             t.getJdkJarHellClasspath().from(jdkJarHellConfig);
-            t.getForbiddenAPIsClasspath().from(project.getConfigurations().getByName("forbiddenApisCliJar").plus(compileOnly));
+            t.getForbiddenAPIsClasspath().from(project.getConfigurations().getByName("forbiddenApisCliJar").plus(compileOnlyThirdParty));
         });
         return audit;
     }

diff --git a/...ternal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java b/...ternal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java
@@ -17,7 +17,6 @@
 import org.gradle.api.JavaVersion;
 import org.gradle.api.file.ArchiveOperations;
 import org.gradle.api.file.ConfigurableFileCollection;
-import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.FileSystemOperations;
 import org.gradle.api.file.FileTree;
 import org.gradle.api.file.ProjectLayout;
@@ -96,8 +95,6 @@ public abstract class ThirdPartyAuditTask extends DefaultTask {
 
     private final ProjectLayout projectLayout;
 
-    private FileCollection classpath;
-
     @Inject
     public ThirdPartyAuditTask(
         ArchiveOperations archiveOperations,
@@ -198,9 +195,7 @@ public Set<String> getMissingClassExcludes() {
     public abstract Property<JavaVersion> getRuntimeJavaVersion();
 
     @Classpath
-    public FileCollection getClasspath() {
-        return classpath;
-    }
+    public abstract ConfigurableFileCollection getThirdPartyClasspath();
 
     @TaskAction
     public void runThirdPartyAudit() throws IOException {
@@ -345,7 +340,7 @@ private String runForbiddenAPIsCli() throws IOException {
             if (javaHome.isPresent()) {
                 spec.setExecutable(javaHome.get() + "/bin/java");
             }
-            spec.classpath(getForbiddenAPIsClasspath(), classpath);
+            spec.classpath(getForbiddenAPIsClasspath(), getThirdPartyClasspath());
             // Enable explicitly for each release as appropriate. Just JDK 20/21/22/23 for now, and just the vector module.
             if (isJavaVersion(VERSION_20) || isJavaVersion(VERSION_21) || isJavaVersion(VERSION_22) || isJavaVersion(VERSION_23)) {
                 spec.jvmArgs("--add-modules", "jdk.incubator.vector");
@@ -383,7 +378,7 @@ private boolean isJavaVersion(JavaVersion version) {
     private Set<String> runJdkJarHellCheck() throws IOException {
         ByteArrayOutputStream standardOut = new ByteArrayOutputStream();
         ExecResult execResult = execOperations.javaexec(spec -> {
-            spec.classpath(getJdkJarHellClasspath(), classpath);
+            spec.classpath(getJdkJarHellClasspath(), getThirdPartyClasspath());
             spec.getMainClass().set(JDK_JAR_HELL_MAIN_CLASS);
             spec.args(getJarExpandDir());
             spec.setIgnoreExitValue(true);
@@ -402,8 +397,4 @@ private Set<String> runJdkJarHellCheck() throws IOException {
         return new TreeSet<>(Arrays.asList(jdkJarHellCheckList.split("\\r?\\n")));
     }
 
-    public void setClasspath(FileCollection classpath) {
-        this.classpath = classpath;
-    }
-
 }
diff --git a/...ools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java b/...ools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java
@@ -12,11 +12,13 @@
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.ResolvableDependencies;
 import org.gradle.api.artifacts.component.ComponentIdentifier;
+import org.gradle.api.artifacts.component.ProjectComponentIdentifier;
 import org.gradle.api.artifacts.result.ResolvedComponentResult;
 import org.gradle.api.artifacts.result.ResolvedDependencyResult;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.specs.AndSpec;
 import org.gradle.api.specs.Spec;
+import org.jetbrains.annotations.NotNull;
 
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -47,4 +49,9 @@ public static FileCollection createFileCollectionFromNonTransitiveArtifactsView(
         }).getFiles();
     }
 
+    public static @NotNull FileCollection projectedDependenciesFilteredView(Configuration configuration) {
+        return configuration.getIncoming()
+            .artifactView(v -> v.componentFilter(i -> (i instanceof ProjectComponentIdentifier == false)))
+            .getFiles();
+    }
 }
diff --git a/x-pack/plugin/identity-provider/build.gradle b/x-pack/plugin/identity-provider/build.gradle
@@ -8,6 +8,7 @@
 apply plugin: 'elasticsearch.internal-es-plugin'
 apply plugin: 'elasticsearch.publish'
 apply plugin: 'elasticsearch.internal-cluster-test'
+
 esplugin {
   name 'x-pack-identity-provider'
   description 'Elasticsearch Expanded Pack Plugin - Identity Provider'
@@ -19,6 +20,10 @@ base {
   archivesName = 'x-pack-identity-provider'
 }
 
+configurations {
+  shadowedDeps
+}
+
 dependencies {
   compileOnly project(path: xpackModule('core'))
 
@@ -29,6 +34,7 @@ dependencies {
   api "org.opensaml:opensaml-messaging-api:${versions.opensaml}"
   api "org.opensaml:opensaml-messaging-impl:${versions.opensaml}"
   api project(path: ':x-pack:libs:es-opensaml-security-api', configuration: 'shadow')
+  shadowedDeps project(path: ':x-pack:libs:es-opensaml-security-api', configuration: 'shadow')
   api "org.opensaml:opensaml-security-impl:${versions.opensaml}"
   api "org.opensaml:opensaml-profile-api:${versions.opensaml}"
   api "org.opensaml:opensaml-profile-impl:${versions.opensaml}"
@@ -64,7 +70,6 @@ dependencies {
   testImplementation(testArtifact(project(xpackModule('security'))))
   testImplementation project(':modules:lang-mustache')
   internalClusterTestImplementation project(":modules:analysis-common")
-
 }
 
 tasks.named("dependencyLicenses").configure {
@@ -87,6 +92,7 @@ tasks.named('forbiddenApisMain').configure {
 
 // classes are missing, e.g. com.ibm.icu.lang.UCharacter
 tasks.named("thirdPartyAudit").configure {
+    thirdPartyClasspath.from(configurations.shadowedDeps)
     ignoreMissingClasses(
         // SAML dependencies
         // [missing classes] Some cli utilities that we don't use depend on these missing JCommander classes

diff --git a/x-pack/plugin/security/build.gradle b/x-pack/plugin/security/build.gradle
@@ -15,6 +15,10 @@ base {
   archivesName = 'x-pack-security'
 }
 
+configurations {
+  shadowedDeps
+}
+
 dependencies {
   compileOnly project(path: xpackModule('core'))
   api project(path: ':modules:transport-netty4')
@@ -47,6 +51,7 @@ dependencies {
   api "org.opensaml:opensaml-messaging-api:${versions.opensaml}"
   api "org.opensaml:opensaml-messaging-impl:${versions.opensaml}"
   api project(path: ':x-pack:libs:es-opensaml-security-api', configuration: 'shadow')
+  shadowedDeps project(path: ':x-pack:libs:es-opensaml-security-api', configuration: 'shadow')
 //  api "org.opensaml:opensaml-security-api:${versions.opensaml}"
   api "org.opensaml:opensaml-security-impl:${versions.opensaml}"
   api "org.opensaml:opensaml-profile-api:${versions.opensaml}"
@@ -81,6 +86,7 @@ dependencies {
   // Dependencies for oidc
   api "com.nimbusds:oauth2-oidc-sdk:11.10.1"
   api project(path: xpackModule('security:lib:nimbus-jose-jwt-modified'), configuration: 'shadow')
+  shadowedDeps project(path: xpackModule('security:lib:nimbus-jose-jwt-modified'), configuration: 'shadow')
   if (isEclipse) {
     /*
      * Eclipse can't pick up the shadow dependency so we point it at the unmodified version of the library
@@ -212,6 +218,7 @@ tasks.named('forbiddenApisTest').configure {
 
 // classes are missing, e.g. com.ibm.icu.lang.UCharacter
 tasks.named("thirdPartyAudit").configure {
+  thirdPartyClasspath.from(configurations.shadowedDeps)
   ignoreMissingClasses(
     // SAML dependencies
     // [missing classes] Some cli utilities that we don't use depend on these missing JCommander classes
@@ -385,6 +392,21 @@ tasks.named("thirdPartyAudit").configure {
     'org.bouncycastle.util.Arrays',
     'org.bouncycastle.util.io.Streams',
     'org.bouncycastle.cert.X509CertificateHolder',
+    'javax.xml.bind.JAXBContext',
+    'javax.xml.bind.JAXBElement',
+    'javax.xml.bind.JAXBException',
+    'javax.xml.bind.Unmarshaller',
+    'javax.xml.bind.UnmarshallerHandler',
+    // Optional dependency of oauth2-oidc-sdk that we don't need since we do not support AES-SIV for JWE
+    'org.cryptomator.siv.SivMode',
+    'com.nimbusds.common.contenttype.ContentType',
+    'com.nimbusds.common.contenttype.ContentType$Parameter',
+    'javax.activation.ActivationDataFlavor',
+    'javax.activation.DataContentHandler',
+    'javax.activation.DataHandler',
+    'javax.activation.DataSource',
+    'javax.activation.FileDataSource',
+    'javax.activation.FileTypeMap'
   )
 
   ignoreViolations(
@@ -405,26 +427,6 @@ tasks.named("thirdPartyAudit").configure {
   )
 }
 
-tasks.named("thirdPartyAudit").configure {
-  ignoreMissingClasses(
-    'javax.xml.bind.JAXBContext',
-    'javax.xml.bind.JAXBElement',
-    'javax.xml.bind.JAXBException',
-    'javax.xml.bind.Unmarshaller',
-    'javax.xml.bind.UnmarshallerHandler',
-    // Optional dependency of oauth2-oidc-sdk that we don't need since we do not support AES-SIV for JWE
-    'org.cryptomator.siv.SivMode',
-    'com.nimbusds.common.contenttype.ContentType',
-    'com.nimbusds.common.contenttype.ContentType$Parameter',
-    'javax.activation.ActivationDataFlavor',
-    'javax.activation.DataContentHandler',
-    'javax.activation.DataHandler',
-    'javax.activation.DataSource',
-    'javax.activation.FileDataSource',
-    'javax.activation.FileTypeMap'
-  )
-}
-
 tasks.named("internalClusterTest").configure {
   /*
    * Some tests in this module set up a lot of transport threads so we reduce the buffer size per transport thread from the 1M default