Fix fastutil shadowing (#3476)

* support running CLI tests with K2
* explicitly control dependencies in CLI integration tests

---------

Co-authored-by: Ignat Beresnev <ignat.beresnev@jetbrains.com>

dokka-integration-tests/cli/build.gradle.kts

@@ -15,50 +15,65 @@ dependencies {
     implementation(projects.utilities)
 }
 
-/* Create a fat base plugin jar for cli tests */
-val basePluginShadow: Configuration by configurations.creating {
+val cliPluginsClasspath: Configuration by configurations.creating {
+    description = "plugins/dependencies required to run CLI with base plugin"
     attributes {
-        attribute(Usage.USAGE_ATTRIBUTE, project.objects.named(Usage::class.java, "java-runtime"))
+        attribute(Usage.USAGE_ATTRIBUTE, project.objects.named(Usage.JAVA_RUNTIME))
     }
+
+    // we don't fetch transitive dependencies here to be able to control external dependencies explicitly
+    isTransitive = false
 }
 
-val cliConfiguration: Configuration by configurations.creating {
+val cliClasspath: Configuration by configurations.creating {
+    description = "dependency on CLI JAR"
     attributes {
-        attribute(Usage.USAGE_ATTRIBUTE, project.objects.named(Usage::class.java, Usage.JAVA_RUNTIME))
-        attribute(Bundling.BUNDLING_ATTRIBUTE, project.objects.named(Bundling::class.java, Bundling.SHADOWED))
+        attribute(Usage.USAGE_ATTRIBUTE, project.objects.named(Usage.JAVA_RUNTIME))
+        attribute(Bundling.BUNDLING_ATTRIBUTE, project.objects.named(Bundling.SHADOWED))
     }
     // we should have single artifact here
     isTransitive = false
 }
 
 dependencies {
-    cliConfiguration("org.jetbrains.dokka:runner-cli")
+    cliClasspath("org.jetbrains.dokka:runner-cli")
+
+    cliPluginsClasspath("org.jetbrains.dokka:plugin-base")
+    // required dependencies of `plugin-base`
+    cliPluginsClasspath(libs.freemarker)
+    cliPluginsClasspath(libs.kotlinx.html)
 
-    basePluginShadow("org.jetbrains.dokka:plugin-base")
+    val tryK2 = project.providers
+        .gradleProperty("org.jetbrains.dokka.experimental.tryK2")
+        .map(String::toBoolean)
+        .orNull ?: false
+
+    val analysisDependency = when {
+        tryK2 -> "org.jetbrains.dokka:analysis-kotlin-symbols"
+        else -> "org.jetbrains.dokka:analysis-kotlin-descriptors"
+    }
 
-    // TODO [beresnev] analysis switcher
-    basePluginShadow("org.jetbrains.dokka:analysis-kotlin-descriptors") {
+    cliPluginsClasspath(analysisDependency) {
         attributes {
-            attribute(Bundling.BUNDLING_ATTRIBUTE, project.objects.named(Bundling::class.java, Bundling.SHADOWED))
+            attribute(Bundling.BUNDLING_ATTRIBUTE, project.objects.named(Bundling.SHADOWED))
         }
     }
 }
 
-val basePluginShadowJar by tasks.register("basePluginShadowJar", ShadowJar::class) {
-    configurations = listOf(basePluginShadow)
-    archiveFileName.set("fat-base-plugin-${project.version}.jar")
-    archiveClassifier.set("")
+val cliPluginsShadowJar by tasks.registering(ShadowJar::class) {
+    archiveFileName.set("cli-plugins-${project.version}.jar")
+    configurations = listOf(cliPluginsClasspath)
 
     // service files are merged to make sure all Dokka plugins
     // from the dependencies are loaded, and not just a single one.
     mergeServiceFiles()
 }
 
 tasks.integrationTest {
-    dependsOn(cliConfiguration)
-    dependsOn(basePluginShadowJar)
+    dependsOn(cliClasspath)
+    dependsOn(cliPluginsShadowJar)
 
     inputs.dir(file("projects"))
-    environment("CLI_JAR_PATH", cliConfiguration.singleFile)
-    environment("BASE_PLUGIN_JAR_PATH", basePluginShadowJar.archiveFile.get())
+    environment("CLI_JAR_PATH", cliClasspath.singleFile)
+    environment("BASE_PLUGIN_JAR_PATH", cliPluginsShadowJar.get().archiveFile.get())
 }

dokka-integration-tests/cli/projects/it-cli/src/main/kotlin/RootPackageClass.kt

@@ -0,0 +1,10 @@
+@file:Suppress("unused")
+
+/**
+ * A class that lives inside the root package
+ *
+ * <me@mail.com> - checks markdown parsing of emails and non-trivial cases like #3329, should compile
+ */
+class RootPackageClass {
+    val description = "I do live in the root package!"
+}

dokka-subprojects/analysis-kotlin-descriptors/build.gradle.kts

@@ -2,6 +2,7 @@
  * Copyright 2014-2024 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
  */
 
+import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
 import dokkabuild.overridePublicationArtifactId
 
 plugins {
@@ -22,7 +23,7 @@ dependencies {
     implementation(projects.dokkaSubprojects.analysisKotlinDescriptorsIde)
 }
 
-tasks.shadowJar {
+tasks.withType<ShadowJar>().configureEach {
     // service files are merged to make sure all Dokka plugins
     // from the dependencies are loaded, and not just a single one.
     mergeServiceFiles()
@@ -34,9 +35,8 @@ tasks.shadowJar {
  * KT issue: https://youtrack.jetbrains.com/issue/KT-47150
  *
  * what is happening here?
- *   fastutil is removed from shadow-jar completely,
- *   instead we declare a maven RUNTIME dependency on fastutil;
- *   this dependency will be fetched by Gradle at build time (as any other dependency from maven-central)
+ * 1. we create intermediate `shadowDependenciesJar` with dependencies but without fastutil classes in it
+ * 2. then we create final `shadowJar` with full fastutil from maven and dependencies from `shadowDependenciesJar` instead of original dependencies
  *
  * why do we need this?
  *   because `kotlin-compiler` artifact includes unshaded (not-relocated) STRIPPED `fastutil` dependency,
@@ -47,5 +47,34 @@ tasks.shadowJar {
  *   and so such classes are not replaced afterward by `shadowJar` task - it visits every class once
  *
  */
-dependencies.shadow(libs.fastutil)
-tasks.shadowJar { exclude("it/unimi/dsi/fastutil/**") }
+
+val shadowOverride: Configuration by configurations.creating {
+    description = "dependencies which we need to replace with original ones because `kotlin-compiler` minimizes them"
+    attributes {
+        attribute(Usage.USAGE_ATTRIBUTE, project.objects.named(Usage.JAVA_RUNTIME))
+    }
+}
+
+dependencies {
+    shadowOverride(libs.fastutil)
+}
+
+val shadowDependenciesJar by tasks.registering(ShadowJar::class) {
+    group = "shadow"
+    description = "Create a shadow jar from dependencies without fastutil"
+
+    archiveClassifier.set("dependencies")
+    destinationDirectory.set(project.layout.buildDirectory.dir("shadowDependenciesLibs"))
+
+    // we need to create JAR with dependencies, but without fastutil,
+    // so we include `runtimeClasspath` configuration (the same configuration which is used by default `shadowJar` task)
+    // and include `fastutil` from the result
+    configurations = listOf(project.configurations.runtimeClasspath.get())
+    exclude("it/unimi/dsi/fastutil/**")
+}
+
+tasks.shadowJar {
+    // override configurations to remove dependencies handled in `shadowJarDependencies`
+    configurations = emptyList()
+    from(shadowOverride, shadowDependenciesJar)
+}

dokka-subprojects/analysis-kotlin-symbols/build.gradle.kts

@@ -2,6 +2,7 @@
  * Copyright 2014-2024 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
  */
 
+import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
 import dokkabuild.overridePublicationArtifactId
 
 plugins {
@@ -84,22 +85,20 @@ dependencies {
     compileOnly(libs.kotlinx.coroutines.core)
 }
 
-tasks.shadowJar {
+tasks.withType<ShadowJar>().configureEach {
     // service files are merged to make sure all Dokka plugins
     // from the dependencies are loaded, and not just a single one.
     mergeServiceFiles()
 }
 
-
 /**
  * hack for shadow jar and fastutil because of kotlin-compiler
  *
  * KT issue: https://youtrack.jetbrains.com/issue/KT-47150
  *
  * what is happening here?
- *   fastutil is removed from shadow-jar completely,
- *   instead we declare a maven RUNTIME dependency on fastutil;
- *   this dependency will be fetched by Gradle at build time (as any other dependency from maven-central)
+ * 1. we create intermediate `shadowDependenciesJar` with dependencies but without fastutil classes in it
+ * 2. then we create final `shadowJar` with full fastutil from maven and dependencies from `shadowDependenciesJar` instead of original dependencies
  *
  * why do we need this?
  *   because `kotlin-compiler` artifact includes unshaded (not-relocated) STRIPPED `fastutil` dependency,
@@ -110,6 +109,34 @@ tasks.shadowJar {
  *   and so such classes are not replaced afterward by `shadowJar` task - it visits every class once
  *
  */
-dependencies.shadow(libs.fastutil)
-tasks.shadowJar { exclude("it/unimi/dsi/fastutil/**") }
 
+val shadowOverride: Configuration by configurations.creating {
+    description = "dependencies which we need to replace with original ones because `kotlin-compiler` minimizes them"
+    attributes {
+        attribute(Usage.USAGE_ATTRIBUTE, project.objects.named(Usage.JAVA_RUNTIME))
+    }
+}
+
+dependencies {
+    shadowOverride(libs.fastutil)
+}
+
+val shadowDependenciesJar by tasks.registering(ShadowJar::class) {
+    group = "shadow"
+    description = "Create a shadow jar from dependencies without fastutil"
+
+    archiveClassifier.set("dependencies")
+    destinationDirectory.set(project.layout.buildDirectory.dir("shadowDependenciesLibs"))
+
+    // we need to create JAR with dependencies, but without fastutil,
+    // so we include `runtimeClasspath` configuration (the same configuration which is used by default `shadowJar` task)
+    // and include `fastutil` from the result
+    configurations = listOf(project.configurations.runtimeClasspath.get())
+    exclude("it/unimi/dsi/fastutil/**")
+}
+
+tasks.shadowJar {
+    // override configurations to remove dependencies handled in `shadowJarDependencies`
+    configurations = emptyList()
+    from(shadowOverride, shadowDependenciesJar)
+}