[fix] Handle dependencies defined in jdeps files | #BAZEL-560 Done

Merge-request: BAZEL-MR-461
Merged-by: Tomasz Pasternak <Tomasz.Pasternak@jetbrains.com>

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 
 ## [Unreleased]
+- Include libraries defined in `jdeps` files during sync
 
 ### Features 🎉
 

aspects/core.bzl

@@ -32,6 +32,7 @@ COMPILE_DEPS = [
     "deps",
     "jars",
     "exports",
+    "associates",
 ]
 
 PRIVATE_COMPILE_DEPS = [

aspects/rules/jvm/jvm_info.bzl

@@ -111,6 +111,9 @@ def extract_jvm_info(target, ctx, output_groups, **kwargs):
     else:
         return None, None
 
+    # I don't know why, but it seems that the "java_outputs" variable can have a different type, depending on language
+    jdeps = get_jdeps(target)
+
     resolve_files = []
 
     jars, resolve_files_jars = map_with_resolve_files(to_jvm_outputs, java_outputs)
@@ -148,6 +151,12 @@ def extract_jvm_info(target, ctx, output_groups, **kwargs):
         jvm_flags = jvm_flags,
         main_class = main_class,
         args = args,
+        jdeps = [file_location(j) for j in jdeps],
     )
 
     return create_proto(target, ctx, info, "jvm_target_info"), None
+
+def get_jdeps(target):
+    if JavaInfo in target:
+        return [jo.jdeps for jo in target[JavaInfo].java_outputs if jo.jdeps != None]
+    return []

server/src/main/java/org/jetbrains/bsp/bazel/server/sync/BUILD

@@ -20,6 +20,7 @@ kt_jvm_library(
         "//server/src/main/java/org/jetbrains/bsp/bazel/server/sync/dependencytree",
         "//server/src/main/java/org/jetbrains/bsp/bazel/server/sync/proto:bsp_target_info_java_proto",
         "@com_google_protobuf//:protobuf_java",
+        "@io_bazel//src/main/protobuf:deps_java_proto",
         "@maven//:ch_epfl_scala_bsp4j_2_13",
         "@maven//:com_fasterxml_jackson_core_jackson_annotations",
         "@maven//:com_fasterxml_jackson_core_jackson_databind",

server/src/main/java/org/jetbrains/bsp/bazel/server/sync/BazelPathsResolver.kt

@@ -45,14 +45,24 @@ class BazelPathsResolver(private val bazelInfo: BazelInfo) {
     private fun resolveAbsolute(fileLocation: FileLocation): Path =
         Paths.get(fileLocation.relativePath)
 
-    private fun resolveExternal(fileLocation: FileLocation): Path =
-        bazelInfo
+    private fun resolveExternal(fileLocation: FileLocation): Path {
+        val outputBaseRelativePath = Paths.get(fileLocation.rootExecutionPathFragment, fileLocation.relativePath)
+        return resolveExternal(outputBaseRelativePath)
+    }
+
+    private fun resolveExternal(outputBaseRelativePath: Path): Path = bazelInfo
             .outputBase
-            .resolve(fileLocation.rootExecutionPathFragment)
-            .resolve(fileLocation.relativePath)
+            .resolve(outputBaseRelativePath)
 
-    private fun resolveOutput(fileLocation: FileLocation): Path =
-        Paths.get(bazelInfo.execRoot, fileLocation.rootExecutionPathFragment, fileLocation.relativePath)
+    private fun resolveOutput(fileLocation: FileLocation): Path {
+        val execRootRelativePath = Paths.get(fileLocation.rootExecutionPathFragment, fileLocation.relativePath)
+        return resolveOutput(execRootRelativePath)
+    }
+
+    fun resolveOutput(execRootRelativePath: Path): Path = when {
+        execRootRelativePath.startsWith("external") -> resolveExternal(execRootRelativePath)
+        else -> Paths.get(bazelInfo.execRoot).resolve(execRootRelativePath)
+    }
 
     private fun resolveSource(fileLocation: FileLocation): Path =
         bazelInfo.workspaceRoot.resolve(fileLocation.relativePath)

server/src/main/java/org/jetbrains/bsp/bazel/server/sync/BazelProjectMapper.kt

@@ -1,5 +1,7 @@
 package org.jetbrains.bsp.bazel.server.sync
 
+import com.google.common.hash.Hashing
+import com.google.devtools.build.lib.view.proto.Deps
 import kotlinx.coroutines.flow.asFlow
 import kotlinx.coroutines.flow.filterNot
 import kotlinx.coroutines.flow.map
@@ -20,6 +22,10 @@ import org.jetbrains.bsp.bazel.server.sync.model.SourceSet
 import org.jetbrains.bsp.bazel.server.sync.model.Tag
 import org.jetbrains.bsp.bazel.workspacecontext.WorkspaceContext
 import java.net.URI
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.Paths
 
 const val KOTLIN_STDLIB_ROOT_EXECUTION = "external/com_github_jetbrains_kotlin"
 const val KOTLIN_STDLIB_RELATIVE_PATH_PREFIX = "lib/"
@@ -41,13 +47,14 @@ class BazelProjectMapper(
         val targetsAsLibraries = targets - targetsToImport.map { it.id }.toSet()
         val annotationProcessorLibraries = annotationProcessorLibraries(targetsToImport)
         val kotlinStdlibsMapper = calculateKotlinStdlibsMapper(targetsToImport)
-        val allLibraries = concatenateMaps(annotationProcessorLibraries, kotlinStdlibsMapper)
-        val modulesFromBazel = createModules(targetsToImport, dependencyTree, allLibraries)
-        val librariesToImport = createLibraries(targetsAsLibraries) +
-            allLibraries.values.flatten().associateBy { it.label }
+        val librariesFromDeps = concatenateMaps(annotationProcessorLibraries, kotlinStdlibsMapper)
+        val librariesFromDepsAndTargets = createLibraries(targetsAsLibraries) + librariesFromDeps.values.flatten().associateBy { it.label }
+        val extraLibrariesFromJdeps = jdepsLibraries(targetsToImport.associateBy { it.id }, librariesFromDeps, librariesFromDepsAndTargets)
         val workspaceRoot = bazelPathsResolver.workspaceRoot()
+        val modulesFromBazel = createModules(targetsToImport, dependencyTree, concatenateMaps(librariesFromDeps, extraLibrariesFromJdeps))
         val modifiedModules = modifyModules(modulesFromBazel, workspaceRoot, workspaceContext)
         val sourceToTarget = buildReverseSourceMapping(modifiedModules)
+        val librariesToImport = librariesFromDepsAndTargets + extraLibrariesFromJdeps.values.flatten().associateBy { it.label }
         return Project(workspaceRoot, modifiedModules.toList(), sourceToTarget, librariesToImport)
     }
 
@@ -69,6 +76,7 @@ class BazelProjectMapper(
                             .toSet(),
                         sources = emptySet(),
                         dependencies = emptyList(),
+                        interfaceJars = emptySet(),
                     )
             }
             .map { it.key to listOf(it.value) }
@@ -84,6 +92,98 @@ class BazelProjectMapper(
         return rulesKotlinTargets.associateWith { listOf(projectLevelKotlinStdlibs) }
     }
 
+    /**
+     * In some cases, the jar dependencies of a target might be injected by bazel or rules and not are not
+     * available via `deps` field of a target. For this reason, we read JavaOutputInfo's jdeps file and
+     * filter out jars that have not been included in the target's `deps` list.
+     *
+     * The old Bazel Plugin performs similar step here
+     * https://github.com/bazelbuild/intellij/blob/b68ec8b33aa54ead6d84dd94daf4822089b3b013/java/src/com/google/idea/blaze/java/sync/importer/BlazeJavaWorkspaceImporter.java#L256
+     */
+    private fun jdepsLibraries(targetsToImport: Map<String, TargetInfo>, libraryDependencies: Map<String, List<Library>>, librariesToImport: Map<String, Library>):
+            Map<String, List<Library>> {
+        val targetsToJdepsJars = getAllJdepsDependencies(targetsToImport, libraryDependencies, librariesToImport)
+        val libraryNameToLibraryValueMap = HashMap<String, Library>()
+        return targetsToJdepsJars.mapValues {
+            it.value.map { lib ->
+                val label = syntheticLabel(lib.toString())
+                libraryNameToLibraryValueMap.computeIfAbsent(label) { _ ->
+                    Library(
+                            label = label,
+                            dependencies = emptyList(),
+                            interfaceJars = emptySet(),
+                            outputs = setOf(bazelPathsResolver.resolveUri(lib)),
+                            sources = emptySet())
+                }
+            }
+        }
+    }
+
+    private fun getAllJdepsDependencies(targetsToImport: Map<String, TargetInfo>,
+                                        libraryDependencies: Map<String, List<Library>>,
+                                        librariesToImport: Map<String, Library>): Map<String, Set<Path>> =
+            targetsToImport
+                    .filter { targetSupportsJdeps(it.value) }
+                    .mapValues { targetInfo ->
+                        val jarsFromDirectDependencies = getAllOutputJarsFromTransitiveDeps(targetInfo,
+                                targetsToImport,
+                                libraryDependencies,
+                                librariesToImport)
+                        val jarsFromJdeps = dependencyJarsFromJdepsFiles(targetInfo.value)
+                        jarsFromJdeps - jarsFromDirectDependencies
+                    }
+                    .filterValues { it.isNotEmpty() }
+
+    private fun getAllOutputJarsFromTransitiveDeps(
+            targetInfo: Map.Entry<String, TargetInfo>,
+            targetsToImport: Map<String, TargetInfo>,
+            libraryDependencies: Map<String, List<Library>>,
+            librariesToImport: Map<String, Library>): Set<Path> {
+        return getAllTransitiveDependencies(targetInfo.value, targetsToImport, libraryDependencies, librariesToImport)
+                .flatMap { dep ->
+                    val jarsFromTargets = targetsToImport[dep]?.let { getTargetOutputJars(it) + getTargetInterfaceJars(it) }.orEmpty()
+                    val jarsFromLibraries = librariesToImport[dep]?.let { it.outputs + it.interfaceJars }.orEmpty().map { Paths.get(it.path) }
+                    jarsFromTargets + jarsFromLibraries
+                }.toSet()
+    }
+
+    private fun getAllTransitiveDependencies(target: TargetInfo,
+                                             targetsToImport: Map<String, TargetInfo>,
+                                             libraryDependencies: Map<String, List<Library>>,
+                                             allLibraries: Map<String, Library>): HashSet<String> {
+        var toVisit = target.dependenciesList.map { it.id } + libraryDependencies[target.id].orEmpty().map { it.label }
+        val visited = HashSet<String>()
+        while (toVisit.isNotEmpty()) {
+            val current = toVisit.first()
+            val dependencyLabels = targetsToImport[current]?.dependenciesList.orEmpty().map { it.id } + allLibraries[current]?.dependencies.orEmpty()
+            visited += current
+            toVisit = toVisit + dependencyLabels - current - visited
+        }
+        return visited
+    }
+    private fun dependencyJarsFromJdepsFiles(targetInfo: TargetInfo): Set<Path> =
+            targetInfo.jvmTargetInfo.jdepsList.flatMap {
+                val path = bazelPathsResolver.resolve(it)
+                if (path.toFile().exists()) {
+                    val bytes = Files.readAllBytes(path)
+                    Deps.Dependencies.parseFrom(bytes).dependencyList.map { dependency ->
+                        bazelPathsResolver.resolveOutput(Paths.get(dependency.path))
+                    }
+                } else {
+                    emptySet()
+                }
+            }.toSet()
+
+    private fun targetSupportsJdeps(targetInfo: TargetInfo): Boolean {
+        val languages = inferLanguages(targetInfo)
+        return setOf(Language.JAVA, Language.KOTLIN).containsAll(languages)
+    }
+
+    private fun syntheticLabel(lib: String): String {
+        val shaOfPath = Hashing.sha256().hashString(lib, StandardCharsets.UTF_8) // just in case of a conflict in filename
+        return Paths.get(lib).fileName.toString().replace("[^0-9a-zA-Z]".toRegex(), "-") + "-" + shaOfPath
+    }
+
     private fun calculateProjectLevelKotlinStdlibs(targets: Sequence<TargetInfo>) =
         Library(
             label = "rules_kotlin_kotlin-stdlibs",
@@ -110,6 +210,7 @@ class BazelProjectMapper(
                 outputs = getTargetJarUris(targetInfo),
                 sources = getSourceJarUris(targetInfo),
                 dependencies = targetInfo.dependenciesList.map { it.id },
+                interfaceJars = getTargetInterfaceJars(targetInfo).map { it.toUri() }.toSet(),
             )
         }
     }
@@ -127,6 +228,18 @@ class BazelProjectMapper(
             .flatMap { it.sourceJarsList }
             .resolveUris()
 
+    private fun getTargetOutputJars(targetInfo: TargetInfo) =
+            targetInfo.jvmTargetInfo.jarsList
+                    .flatMap { it.binaryJarsList }
+                    .map { bazelPathsResolver.resolve(it) }
+                    .toSet()
+
+    private fun getTargetInterfaceJars(targetInfo: TargetInfo) =
+            targetInfo.jvmTargetInfo.jarsList
+                    .flatMap { it.interfaceJarsList }
+                    .map { bazelPathsResolver.resolve(it) }
+                    .toSet()
+
     private fun selectTargetsToImport(
         workspaceContext: WorkspaceContext, rootTargets: Set<String>, tree: DependencyTree
     ): Sequence<TargetInfo> = tree.allTargetsAtDepth(

server/src/main/java/org/jetbrains/bsp/bazel/server/sync/model/Library.kt

@@ -7,4 +7,5 @@ data class Library(
         val outputs: Set<URI>,
         val sources: Set<URI>,
         val dependencies: List<String>,
+        val interfaceJars: Set<URI> = emptySet(),
 )

server/src/main/java/org/jetbrains/bsp/bazel/server/sync/proto/bsp_target_info.proto

@@ -37,6 +37,7 @@ message JvmTargetInfo {
   repeated string jvm_flags = 7;
   string main_class = 8;
   repeated string args = 9;
+  repeated FileLocation jdeps = 10;
 }
 
 message JavaToolchainInfo {