From e6c65c9024cbe648ac04d194e6fe88ba92646aef Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Wed, 24 Jan 2024 21:38:12 -0800 Subject: [PATCH] Add `bazel mod dump_repo_mapping` `bazel mod dump_repo_mapping` with no arguments is explicitly made an error so that a new mode that dumps all repository mappings with a single Bazel invocation can be added later if needed, e.g. to support IntelliJ's "sync" workflow. RELNOTES: `bazel mod dump_repo_mapping ...` returns the repository mappings of the given repositories in NDJSON. This information can be used by IDEs and Starlark language servers to resolve labels with `--enable_bzlmod`. Work towards #20631 Closes #20686. PiperOrigin-RevId: 601332180 Change-Id: I828d7c88637bea175e11eccc52c6202f6da4c32c --- .../bazel/bzlmod/modcommand/ModOptions.java | 3 +- .../devtools/build/lib/bazel/commands/BUILD | 1 + .../build/lib/bazel/commands/ModCommand.java | 128 ++++++++++++++---- .../devtools/build/lib/bazel/commands/mod.txt | 3 + src/test/py/bazel/bzlmod/mod_command_test.py | 67 +++++++++ 5 files changed, 178 insertions(+), 24 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ModOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ModOptions.java index ab45297ae423fc..bac6b42d6b9c48 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ModOptions.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ModOptions.java @@ -172,7 +172,8 @@ public enum ModSubcommand { PATH(true), EXPLAIN(true), SHOW_REPO(false), - SHOW_EXTENSION(false); + SHOW_EXTENSION(false), + DUMP_REPO_MAPPING(false); /** Whether this subcommand produces a graph output. */ private final boolean isGraph; diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD index d4bd139ed31e78..2ce21351e194f2 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD @@ -65,6 +65,7 @@ java_library( "//src/main/java/com/google/devtools/common/options", "//src/main/java/net/starlark/java/eval", "//src/main/protobuf:failure_details_java_proto", + "//third_party:gson", "//third_party:guava", ], ) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/ModCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/ModCommand.java index 77c0be124615c8..ec6d9d3caa8158 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/ModCommand.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/ModCommand.java @@ -13,6 +13,7 @@ // limitations under the License. package com.google.devtools.build.lib.bazel.commands; +import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModOptions.Charset.UTF8; @@ -44,6 +45,7 @@ import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModOptions.ModSubcommandConverter; import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModuleArg; import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModuleArg.ModuleArgConverter; +import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.devtools.build.lib.cmdline.RepositoryMapping; import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.events.Event; @@ -57,6 +59,7 @@ import com.google.devtools.build.lib.server.FailureDetails; import com.google.devtools.build.lib.server.FailureDetails.FailureDetail; import com.google.devtools.build.lib.server.FailureDetails.ModCommand.Code; +import com.google.devtools.build.lib.skyframe.RepositoryMappingValue; import com.google.devtools.build.lib.skyframe.SkyframeExecutor; import com.google.devtools.build.lib.util.AbruptExitException; import com.google.devtools.build.lib.util.DetailedExitCode; @@ -70,11 +73,17 @@ import com.google.devtools.common.options.OptionsParser; import com.google.devtools.common.options.OptionsParsingException; import com.google.devtools.common.options.OptionsParsingResult; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.stream.JsonWriter; +import java.io.IOException; import java.io.OutputStreamWriter; +import java.io.Writer; import java.util.List; import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; +import java.util.stream.IntStream; /** Queries the Bzlmod external dependency graph. */ @Command( @@ -125,8 +134,51 @@ public BlazeCommandResult exec(CommandEnvironment env, OptionsParsingResult opti } private BlazeCommandResult execInternal(CommandEnvironment env, OptionsParsingResult options) { + ModOptions modOptions = options.getOptions(ModOptions.class); + Preconditions.checkArgument(modOptions != null); + + if (options.getResidue().isEmpty()) { + String errorMessage = + String.format( + "No subcommand specified, choose one of : %s.", ModSubcommand.printValues()); + return reportAndCreateFailureResult(env, errorMessage, Code.MOD_COMMAND_UNKNOWN); + } + + // The first element in the residue must be the subcommand, and then comes a list of arguments. + String subcommandStr = options.getResidue().get(0); + ModSubcommand subcommand; + try { + subcommand = new ModSubcommandConverter().convert(subcommandStr); + } catch (OptionsParsingException e) { + String errorMessage = + String.format("Invalid subcommand, choose one from : %s.", ModSubcommand.printValues()); + return reportAndCreateFailureResult(env, errorMessage, Code.MOD_COMMAND_UNKNOWN); + } + List args = options.getResidue().subList(1, options.getResidue().size()); + + ImmutableList.Builder repositoryMappingKeysBuilder = + ImmutableList.builder(); + if (subcommand.equals(ModSubcommand.DUMP_REPO_MAPPING)) { + if (args.isEmpty()) { + // Make this case an error so that we are free to add a mode that emits all mappings in a + // single JSON object later. + return reportAndCreateFailureResult( + env, "No repository name(s) specified", Code.INVALID_ARGUMENTS); + } + for (String arg : args) { + try { + repositoryMappingKeysBuilder.add(RepositoryMappingValue.key(RepositoryName.create(arg))); + } catch (LabelSyntaxException e) { + return reportAndCreateFailureResult(env, e.getMessage(), Code.INVALID_ARGUMENTS); + } + } + } + ImmutableList repoMappingKeys = + repositoryMappingKeysBuilder.build(); + BazelDepGraphValue depGraphValue; BazelModuleInspectorValue moduleInspector; + ImmutableList repoMappingValues; SkyframeExecutor skyframeExecutor = env.getSkyframeExecutor(); LoadingPhaseThreadsOption threadsOption = options.getOptions(LoadingPhaseThreadsOption.class); @@ -140,10 +192,14 @@ private BlazeCommandResult execInternal(CommandEnvironment env, OptionsParsingRe try { env.syncPackageLoading(options); + ImmutableSet.Builder keys = ImmutableSet.builder(); + if (subcommand.equals(ModSubcommand.DUMP_REPO_MAPPING)) { + keys.addAll(repoMappingKeys); + } else { + keys.add(BazelDepGraphValue.KEY, BazelModuleInspectorValue.KEY); + } EvaluationResult evaluationResult = - skyframeExecutor.prepareAndGet( - ImmutableSet.of(BazelDepGraphValue.KEY, BazelModuleInspectorValue.KEY), - evaluationContext); + skyframeExecutor.prepareAndGet(keys.build(), evaluationContext); if (evaluationResult.hasError()) { Exception e = evaluationResult.getError().getException(); @@ -159,6 +215,11 @@ private BlazeCommandResult execInternal(CommandEnvironment env, OptionsParsingRe moduleInspector = (BazelModuleInspectorValue) evaluationResult.get(BazelModuleInspectorValue.KEY); + repoMappingValues = + repoMappingKeys.stream() + .map(evaluationResult::get) + .map(RepositoryMappingValue.class::cast) + .collect(toImmutableList()); } catch (InterruptedException e) { String errorMessage = "mod command interrupted: " + e.getMessage(); env.getReporter().handle(Event.error(errorMessage)); @@ -169,27 +230,29 @@ private BlazeCommandResult execInternal(CommandEnvironment env, OptionsParsingRe return BlazeCommandResult.detailedExitCode(e.getDetailedExitCode()); } - ModOptions modOptions = options.getOptions(ModOptions.class); - Preconditions.checkArgument(modOptions != null); - - if (options.getResidue().isEmpty()) { - String errorMessage = - String.format( - "No subcommand specified, choose one of : %s.", ModSubcommand.printValues()); - return reportAndCreateFailureResult(env, errorMessage, Code.MOD_COMMAND_UNKNOWN); - } - - // The first element in the residue must be the subcommand, and then comes a list of arguments. - String subcommandStr = options.getResidue().get(0); - ModSubcommand subcommand; - try { - subcommand = new ModSubcommandConverter().convert(subcommandStr); - } catch (OptionsParsingException e) { - String errorMessage = - String.format("Invalid subcommand, choose one from : %s.", ModSubcommand.printValues()); - return reportAndCreateFailureResult(env, errorMessage, Code.MOD_COMMAND_UNKNOWN); + if (subcommand.equals(ModSubcommand.DUMP_REPO_MAPPING)) { + String missingRepos = + IntStream.range(0, repoMappingKeys.size()) + .filter(i -> repoMappingValues.get(i) == RepositoryMappingValue.NOT_FOUND_VALUE) + .mapToObj(repoMappingKeys::get) + .map(RepositoryMappingValue.Key::repoName) + .map(RepositoryName::getName) + .collect(joining(", ")); + if (!missingRepos.isEmpty()) { + return reportAndCreateFailureResult( + env, "Repositories not found: " + missingRepos, Code.INVALID_ARGUMENTS); + } + try { + dumpRepoMappings( + repoMappingValues, + new OutputStreamWriter( + env.getReporter().getOutErr().getOutputStream(), + modOptions.charset == UTF8 ? UTF_8 : US_ASCII)); + } catch (IOException e) { + throw new IllegalStateException(e); + } + return BlazeCommandResult.success(); } - List args = options.getResidue().subList(1, options.getResidue().size()); // Extract and check the --base_module argument first to use it when parsing the other args. // Can only be a TargetModule or a repoName relative to the ROOT. @@ -453,6 +516,8 @@ private BlazeCommandResult execInternal(CommandEnvironment env, OptionsParsingRe case SHOW_EXTENSION: modExecutor.showExtension(argsAsExtensions, usageKeys); break; + default: + throw new IllegalStateException("Unexpected subcommand: " + subcommand); } return BlazeCommandResult.success(); @@ -510,4 +575,21 @@ private static BlazeCommandResult createFailureResult(String message, Code detai .setMessage(message) .build())); } + + public static void dumpRepoMappings(List repoMappings, Writer writer) + throws IOException { + Gson gson = new GsonBuilder().disableHtmlEscaping().create(); + for (RepositoryMappingValue repoMapping : repoMappings) { + JsonWriter jsonWriter = gson.newJsonWriter(writer); + jsonWriter.beginObject(); + for (Entry entry : + repoMapping.getRepositoryMapping().entries().entrySet()) { + jsonWriter.name(entry.getKey()); + jsonWriter.value(entry.getValue().getName()); + } + jsonWriter.endObject(); + writer.write('\n'); + } + writer.flush(); + } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/mod.txt b/src/main/java/com/google/devtools/build/lib/bazel/commands/mod.txt index 01d576972cf61d..8cdc0fdc1aa1ab 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/mod.txt +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/mod.txt @@ -12,6 +12,7 @@ The command will display the external dependency graph or parts thereof, structu - explain ...: Prints all the places where the module is (or was) requested as a direct dependency, along with the reason why the respective final version was selected. It will display a pruned version of the `all_paths ...` command which only contains the direct deps of the root, the leaves, along with their dependants (can be modified with --depth). - show_repo ...: Prints the rule that generated the specified repos (i.e. http_archive()). The arguments may refer to extension-generated repos. - show_extension ...: Prints information about the given extension(s). Usages can be filtered down to only those from modules in --extension_usage. + - dump_repo_mapping ...: Prints the mappings from apparent repo names to canonical repo names for the given repos in NDJSON format. The order of entries within each JSON object is unspecified. This command is intended for use by tools such as IDEs and Starlark language servers. arguments must be one of the following: @@ -25,4 +26,6 @@ The command will display the external dependency graph or parts thereof, structu arguments must be of the form %. For example, both rules_java//java:extensions.bzl%toolchains and @rules_java//java:extensions.bzl%toolchains are valid specifications of extensions. + arguments are canonical repo names without any leading @ characters. The canonical repo name of the root module repository is the empty string. + %{options} \ No newline at end of file diff --git a/src/test/py/bazel/bzlmod/mod_command_test.py b/src/test/py/bazel/bzlmod/mod_command_test.py index d17264ed2f7923..74172b0c765e7d 100644 --- a/src/test/py/bazel/bzlmod/mod_command_test.py +++ b/src/test/py/bazel/bzlmod/mod_command_test.py @@ -14,6 +14,7 @@ # limitations under the License. """Tests the mod command.""" +import json import os import tempfile from absl.testing import absltest @@ -454,6 +455,72 @@ def testShowRepoThrowsUnusedModule(self): stderr, ) + def testDumpRepoMapping(self): + _, stdout, _ = self.RunBazel( + [ + 'mod', + 'dump_repo_mapping', + '', + 'foo~2.0', + ], + ) + root_mapping, foo_mapping = [json.loads(l) for l in stdout] + + self.assertContainsSubset( + { + 'my_project': '', + 'foo1': 'foo~1.0', + 'foo2': 'foo~2.0', + 'myrepo2': 'ext2~1.0~ext~repo1', + 'bazel_tools': 'bazel_tools', + }.items(), + root_mapping.items(), + ) + + self.assertContainsSubset( + { + 'foo': 'foo~2.0', + 'ext_mod': 'ext~1.0', + 'my_repo3': 'ext~1.0~ext~repo3', + 'bazel_tools': 'bazel_tools', + }.items(), + foo_mapping.items(), + ) + + def testDumpRepoMappingThrowsNoRepos(self): + _, _, stderr = self.RunBazel( + ['mod', 'dump_repo_mapping'], + allow_failure=True, + ) + self.assertIn( + "ERROR: No repository name(s) specified. Type 'bazel help mod' for" + ' syntax and help.', + stderr, + ) + + def testDumpRepoMappingThrowsInvalidRepoName(self): + _, _, stderr = self.RunBazel( + ['mod', 'dump_repo_mapping', '{}'], + allow_failure=True, + ) + self.assertIn( + "ERROR: invalid repository name '{}': repo names may contain only A-Z," + " a-z, 0-9, '-', '_', '.' and '~' and must not start with '~'. Type" + " 'bazel help mod' for syntax and help.", + stderr, + ) + + def testDumpRepoMappingThrowsUnknownRepoName(self): + _, _, stderr = self.RunBazel( + ['mod', 'dump_repo_mapping', 'does_not_exist'], + allow_failure=True, + ) + self.assertIn( + "ERROR: Repositories not found: does_not_exist. Type 'bazel help mod'" + ' for syntax and help.', + stderr, + ) + if __name__ == '__main__': absltest.main()