[feature] generating a custom file with extension imports based on pr…

…oject's external rules | #BAZEL-556 Done

just to rerun the build

edit changelog, add some aspect documentation

Merge remote-tracking branch 'origin/master' into kasia/language-extension-file

triple quote strings

cr fixes

Merge remote-tracking branch 'origin/master' into kasia/language-extension-file

# Conflicts:
#	aspects/core.bzl
#	aspects/utils/utils.bzl

Revert "asdf"

This reverts commit fed1d73.

asdf

moved language enum to top level

add a comment to the generated file

rename of the file that invokes the query

fetching external rules and creating an extension file based on the result

rename LanguageExtensionManager to BazelBspEnvironmentManager

test reformatting

add test

cleanup

moved extensions to extension file that is replaced on each project reload

concurrent builder map

grouping by target id and choosing one info per target id + comment

pr fixes

fixed target info matching

better matching of proto files by file name

some logging

adding '-general' to TargetInfo file name, TargetInfoReader with bugs

generating files in extensions

slightly formatted code

yay tests work!!
still need to prettify the code

unified interface for extensions


Merge-request: BAZEL-MR-442
Merged-by: Katarzyna Mielnik <katarzyna.anna.mielnik@jetbrains.com>

CHANGELOG.md

@@ -5,6 +5,9 @@
 
 ## [Unreleased]
 
+### Features 🎉
+- The server generates `extensions.bzl` based on languages (external rules) relevant to the project. 
+
 ## [3.0.0] - 09.08.2023
 
 ### BREAKING CHANGES 🚨

aspects/core.bzl

@@ -1,20 +1,6 @@
-load("//aspects:rules/python/python_info.bzl", "extract_python_info")
-load("//aspects:rules/kt/kt_info.bzl", "extract_kotlin_info")
-load("//aspects:rules/cpp/cpp_info.bzl", "extract_cpp_info")
-load("//aspects:rules/scala/scala_info.bzl", "extract_scala_info", "extract_scala_toolchain_info")
-load("//aspects:rules/java/java_info.bzl", "JAVA_RUNTIME_TOOLCHAIN_TYPE", "extract_java_info", "extract_java_runtime", "extract_java_toolchain")
 load("//aspects:utils/utils.bzl", "abs", "create_struct", "file_location", "get_aspect_ids", "update_sync_output_groups")
-
-EXTENSIONS = [
-    extract_java_info,
-    extract_kotlin_info,
-    extract_java_toolchain,
-    extract_java_runtime,
-    extract_scala_info,
-    extract_scala_toolchain_info,
-    extract_python_info,
-    extract_cpp_info,
-]
+load("//aspects:rules/java/java_info.bzl", "JAVA_RUNTIME_TOOLCHAIN_TYPE")
+load("//aspects:extensions.bzl", "EXTENSIONS")
 
 def create_all_extension_info(target, ctx, output_groups, dep_targets):
     info = [create_extension_info(target = target, ctx = ctx, output_groups = output_groups, dep_targets = dep_targets) for create_extension_info in EXTENSIONS]

aspects/extensions.bzl

@@ -0,0 +1,2 @@
+# This file is supposed to be overwritten with extensions relevant to the project.
+EXTENSIONS = []

docs/dev/ASPECTS.md

@@ -0,0 +1,9 @@
+# Aspects
+
+## Extensions
+
+### The `extensions.bzl` file
+The server generates the `extensions.bzl` file on each sync according to a list of external rules relevant to the project.
+See `server/src/main/java/org/jetbrains/bsp/bazel/server/bsp/managers/BazelBspEnvironmentManager.kt` for the implementation.
+
+The file itself contains a declaration of functions invoked in `core.bzl`. Each function is supposed to provide language-specific data.

server/src/main/java/org/jetbrains/bsp/bazel/server/bsp/managers/BUILD

@@ -1,8 +1,11 @@
-load("@rules_java//java:defs.bzl", "java_library")
+load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library")
 
-java_library(
+kt_jvm_library(
     name = "managers",
-    srcs = glob(["*.java"]),
+    srcs = glob([
+        "*.java",
+        "*.kt",
+    ]),
     visibility = ["//server:__subpackages__"],
     deps = [
         "//bazelrunner",

server/src/main/java/org/jetbrains/bsp/bazel/server/bsp/managers/BazelBspAspectsManager.java

@@ -13,23 +13,31 @@
 import org.jetbrains.bsp.bazel.server.bsp.utils.InternalAspectsResolver;
 import org.jetbrains.bsp.bazel.workspacecontext.TargetsSpec;
 
+import java.nio.file.Paths;
+import java.util.List;
+
 public class BazelBspAspectsManager {
 
   private final BazelBspCompilationManager bazelBspCompilationManager;
   private final InternalAspectsResolver aspectsResolver;
+  private final BazelBspEnvironmentManager bazelBspEnvironmentManager;
 
   public BazelBspAspectsManager(
       BazelBspCompilationManager bazelBspCompilationManager,
-      InternalAspectsResolver aspectResolver) {
+      InternalAspectsResolver aspectResolver,
+      BazelBspEnvironmentManager bazelBspEnvironmentManager) {
     this.bazelBspCompilationManager = bazelBspCompilationManager;
     this.aspectsResolver = aspectResolver;
+    this.bazelBspEnvironmentManager = bazelBspEnvironmentManager;
   }
 
   public BepOutput fetchFilesFromOutputGroups(
       CancelChecker cancelChecker,
       TargetsSpec targetSpecs,
       String aspect,
       List<String> outputGroups) {
+    bazelBspEnvironmentManager.generateLanguageExtensions(cancelChecker);
+
     if (targetSpecs.getValues().isEmpty()) return new BepOutput(new HashMap<>(), new HashMap<>(), new HashSet<>());
     return
         bazelBspCompilationManager

server/src/main/java/org/jetbrains/bsp/bazel/server/bsp/managers/BazelBspEnvironmentManager.kt

@@ -0,0 +1,87 @@
+package org.jetbrains.bsp.bazel.server.bsp.managers
+
+import org.eclipse.lsp4j.jsonrpc.CancelChecker
+import org.jetbrains.bsp.bazel.commons.Constants
+import org.jetbrains.bsp.bazel.server.bsp.utils.InternalAspectsResolver
+import java.nio.file.Paths
+import kotlin.io.path.writeText
+
+enum class Language(private val fileName: String, val functions: List<String>) {
+    Java(
+        "//aspects:rules/java/java_info.bzl",
+        listOf("extract_java_info", "extract_java_toolchain", "extract_java_runtime")
+    ),
+    Scala("//aspects:rules/scala/scala_info.bzl", listOf("extract_scala_info", "extract_scala_toolchain_info")),
+    Cpp("//aspects:rules/cpp/cpp_info.bzl", listOf("extract_cpp_info")),
+    Kotlin("//aspects:rules/kt/kt_info.bzl", listOf("extract_kotlin_info")),
+    Python("//aspects:rules/python/python_info.bzl", listOf("extract_python_info"));
+
+    fun toLoadStatement(): String =
+        this.functions.joinToString(
+            prefix = """load("${this.fileName}", """,
+            separator = ", ",
+            postfix = ")"
+        ) { "\"$it\"" }
+}
+
+class BazelBspEnvironmentManager(
+    private val internalAspectsResolver: InternalAspectsResolver,
+    private val bazelExternalRulesQuery: BazelExternalRulesQuery,
+) {
+
+    fun generateLanguageExtensions(cancelChecker: CancelChecker) {
+        val ruleNames = bazelExternalRulesQuery.fetchExternalRuleNames(cancelChecker)
+        val languages = getProjectLanguages(ruleNames)
+        val fileContent = prepareFileContent(languages)
+
+        createNewExtensionsFile(fileContent)
+    }
+
+    private fun getProjectLanguages(allRuleNames: List<String>): List<Language> {
+        fun checkForLanguage(languageRuleNames: List<String>, language: Language) =
+            checkForLanguageRules(allRuleNames, languageRuleNames, language)
+
+        return listOfNotNull(
+            checkForLanguage(listOf("rules_python"), Language.Python),
+            checkForLanguage(listOf("rules_cc"), Language.Cpp),
+            checkForLanguage(listOf("io_bazel_rules_kotlin"), Language.Kotlin),
+            checkForLanguage(listOf("io_bazel_rules_scala"), Language.Scala),
+            checkForLanguage(listOf("rules_java"), Language.Java)
+        )
+    }
+
+    private fun checkForLanguageRules(
+        allRuleNames: List<String>,
+        languageRuleNames: List<String>,
+        language: Language
+    ): Language? =
+        if (languageRuleNames.any { allRuleNames.contains(it) })
+            language
+        else null
+
+    private fun prepareFileContent(languages: List<Language>) =
+        listOf(
+            "# This is a generated file, do not edit it",
+            createLoadStatementsString(languages),
+            createExtensionListString(languages)
+        ).joinToString(
+            separator = "\n",
+            postfix = "\n"
+        )
+
+    private fun createLoadStatementsString(languages: List<Language>): String {
+        val loadStatements = languages.map { it.toLoadStatement() }
+        return loadStatements.joinToString(postfix = "\n", separator = "\n")
+    }
+
+    private fun createExtensionListString(languages: List<Language>): String {
+        val functionNames = languages.flatMap { it.functions }
+        return functionNames.joinToString(prefix = "EXTENSIONS = [\n", postfix = "\n]", separator = ",\n ") { "\t$it" }
+    }
+
+    private fun createNewExtensionsFile(fileContent: String) {
+        val aspectsPath = Paths.get(internalAspectsResolver.bazelBspRoot, Constants.ASPECTS_ROOT)
+        val file = aspectsPath.resolve("extensions.bzl")
+        file.writeText(fileContent)
+    }
+}

server/src/main/java/org/jetbrains/bsp/bazel/server/bsp/managers/BazelExternalRulesQuery.kt

@@ -0,0 +1,17 @@
+package org.jetbrains.bsp.bazel.server.bsp.managers
+
+import org.eclipse.lsp4j.jsonrpc.CancelChecker
+import org.jetbrains.bsp.bazel.bazelrunner.BazelRunner
+
+interface BazelExternalRulesQuery {
+    fun fetchExternalRuleNames(cancelChecker: CancelChecker): List<String>
+}
+
+class BazelExternalRulesQueryImpl(private val bazelRunner: BazelRunner) : BazelExternalRulesQuery {
+
+    override fun fetchExternalRuleNames(cancelChecker: CancelChecker): List<String> =
+        bazelRunner.commandBuilder().query().withArgument("//external:*").executeBazelCommand()
+            .waitAndGetResult(cancelChecker, ensureAllOutputRead = true)
+            .stdoutLines
+            .mapNotNull { it.split(':').getOrNull(1) }
+}

server/src/main/java/org/jetbrains/bsp/bazel/server/common/ServerContainer.kt

@@ -10,6 +10,8 @@ import org.jetbrains.bsp.bazel.server.sync.MetricsLogger
 import org.jetbrains.bsp.bazel.server.bsp.info.BspInfo
 import org.jetbrains.bsp.bazel.server.bsp.managers.BazelBspAspectsManager
 import org.jetbrains.bsp.bazel.server.bsp.managers.BazelBspCompilationManager
+import org.jetbrains.bsp.bazel.server.bsp.managers.BazelBspEnvironmentManager
+import org.jetbrains.bsp.bazel.server.bsp.managers.BazelExternalRulesQueryImpl
 import org.jetbrains.bsp.bazel.server.bsp.utils.InternalAspectsResolver
 import org.jetbrains.bsp.bazel.server.sync.*
 import org.jetbrains.bsp.bazel.server.sync.languages.LanguagePluginsService
@@ -53,7 +55,8 @@ class ServerContainer internal constructor(
             val bazelInfo = bazelDataResolver.resolveBazelInfo { }
 
             val aspectsResolver = InternalAspectsResolver(bspInfo)
-            val bazelBspAspectsManager = BazelBspAspectsManager(compilationManager, aspectsResolver)
+            val bazelBspEnvironmentManager = BazelBspEnvironmentManager(aspectsResolver, BazelExternalRulesQueryImpl(bazelRunner))
+            val bazelBspAspectsManager = BazelBspAspectsManager(compilationManager, aspectsResolver, bazelBspEnvironmentManager)
             val bazelPathsResolver = BazelPathsResolver(bazelInfo)
             val jdkResolver = JdkResolver(bazelPathsResolver, JdkVersionResolver())
             val javaLanguagePlugin = JavaLanguagePlugin(bazelPathsResolver, jdkResolver, bazelInfo)

server/src/main/java/org/jetbrains/bsp/bazel/server/sync/ProjectResolver.kt

@@ -6,6 +6,7 @@ import org.jetbrains.bsp.bazel.commons.Stopwatch
 import org.jetbrains.bsp.bazel.logger.BspClientLogger
 import org.jetbrains.bsp.bazel.server.bep.BepOutput
 import org.jetbrains.bsp.bazel.server.bsp.managers.BazelBspAspectsManager
+import org.jetbrains.bsp.bazel.server.bsp.managers.BazelBspEnvironmentManager
 import org.jetbrains.bsp.bazel.server.sync.model.Project
 import org.jetbrains.bsp.bazel.workspacecontext.WorkspaceContext
 import org.jetbrains.bsp.bazel.workspacecontext.WorkspaceContextProvider

server/src/test/java/org/jetbrains/bsp/bazel/server/bsp/managers/BUILD

@@ -0,0 +1,12 @@
+load("@//rules/kotlin:junit5.bzl", "kt_test")
+
+kt_test(
+    name = "BazelBspEnvironmentManagerTest",
+    size = "small",
+    src = "BazelBspEnvironmentManagerTest.kt",
+    deps = [
+        "//commons",
+        "//server/src/main/java/org/jetbrains/bsp/bazel/server/bsp/managers",
+        "@maven//:org_eclipse_lsp4j_org_eclipse_lsp4j_jsonrpc",
+    ],
+)