[5.1] Adding Starlark dependencies to the package //external (#14991)

* Adding Starlark dependencies to the package //external

This is an alternative approach to fix #14280. It adds transitive closure of Starlark dependencies to `//external` package when loading `WORKSPACE` file, so it can be processed in the same way as `BUILD` files during query execution. Comparing to the approach taken in #14497, this approach is less intrusive, but not able to distinguish the extension files needed by `//external:foo` and `//external:bar`, meaning `buildfiles(//external:foo)` returns the same result as `buildfiles(//external:*)`. However, this behavior is consistent with other packages. For example, `buildfiles(//foo:bar)` has the same result as `buildfiles(//foo:*)`.

Closes #14630.

PiperOrigin-RevId: 424092916
(cherry picked from commit a6c3f23)

* Avoid bazel_module_test timeout on macos

https://buildkite.com/bazel/bazel-bazel/builds/17785#9a1fb564-c6f9-4e69-ac8f-87c422a002b0
By setting the test size to "large".

RELNOTES:None
PiperOrigin-RevId: 409114345
(cherry picked from commit 1d99391)

Co-authored-by: Zhongpeng Lin <zplin@uber.com>
Co-authored-by: pcloudy <pcloudy@google.com>

src/main/java/com/google/devtools/build/lib/packages/Package.java

@@ -1358,7 +1358,7 @@ FailureDetail getFailureDetail() {
       return null;
     }
 
-    Builder setStarlarkFileDependencies(ImmutableList<Label> starlarkFileDependencies) {
+    public Builder setStarlarkFileDependencies(ImmutableList<Label> starlarkFileDependencies) {
       this.starlarkFileDependencies = starlarkFileDependencies;
       return this;
     }

src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java

@@ -706,7 +706,7 @@ private static ImmutableList<Label> transitiveClosureOfLabels(
     return ImmutableList.copyOf(set);
   }
 
-  private static void transitiveClosureOfLabelsRec(
+  public static void transitiveClosureOfLabelsRec(
       Set<Label> set, ImmutableMap<String, Module> loads) {
     for (Module m : loads.values()) {
       BazelModuleContext ctx = BazelModuleContext.of(m);

src/main/java/com/google/devtools/build/lib/skyframe/WorkspaceFileFunction.java

@@ -24,6 +24,7 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSortedSet;
+import com.google.common.collect.Sets;
 import com.google.devtools.build.lib.actions.FileValue;
 import com.google.devtools.build.lib.analysis.BlazeDirectories;
 import com.google.devtools.build.lib.cmdline.Label;
@@ -58,6 +59,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import net.starlark.java.eval.Module;
 import net.starlark.java.eval.Mutability;
 import net.starlark.java.eval.Starlark;
@@ -325,14 +327,21 @@ public SkyValue compute(SkyKey skyKey, Environment env)
               directories.getWorkspace(),
               directories.getLocalJavabase(),
               starlarkSemantics);
+      Set<Label> starlarkFileDependencies;
       if (prevValue != null) {
+        starlarkFileDependencies =
+            Sets.newLinkedHashSet(prevValue.getPackage().getStarlarkFileDependencies());
         try {
           parser.setParent(
               prevValue.getPackage(), prevValue.getLoadedModules(), prevValue.getBindings());
         } catch (NameConflictException e) {
           throw new WorkspaceFileFunctionException(e, Transience.PERSISTENT);
         }
+      } else {
+        starlarkFileDependencies = Sets.newLinkedHashSet();
       }
+      PackageFactory.transitiveClosureOfLabelsRec(starlarkFileDependencies, loadedModules);
+      builder.setStarlarkFileDependencies(ImmutableList.copyOf(starlarkFileDependencies));
       // Execute the partial files that comprise this chunk.
       for (StarlarkFile partialFile : chunk) {
         parser.execute(partialFile, loadedModules, key);

src/test/py/bazel/BUILD

@@ -259,6 +259,7 @@ py_library(
 
 py_test(
     name = "bazel_module_test",
+    size = "large",
     srcs = ["bzlmod/bazel_module_test.py"],
     deps = [
         ":bzlmod_test_utils",

src/test/py/bazel/query_test.py

@@ -38,6 +38,87 @@ def testSimpleQuery(self):
     self._AssertQueryOutput('deps(//foo:top-rule, 1)', '//foo:top-rule',
                             '//foo:dep-rule')
 
+  def testBuildFilesForExternalRepos_Simple(self):
+    self.ScratchFile('WORKSPACE', [
+        'load("//:deps.bzl", "repos")',
+        'repos()',
+    ])
+    self.ScratchFile('BUILD.bazel')
+    self.ScratchFile('deps.bzl', [
+        'def repos():',
+        '    native.new_local_repository(',
+        '        name = "io_bazel_rules_go",',
+        '        path = ".",',
+        """        build_file_content = "exports_files(glob(['*.go']))",""",
+        '    )',
+    ])
+    self._AssertQueryOutputContains('buildfiles(//external:io_bazel_rules_go)',
+                                    '//external:WORKSPACE', '//:deps.bzl',
+                                    '//:BUILD.bazel')
+
+  def testBuildFilesForExternalRepos_IndirectLoads(self):
+    self.ScratchFile('WORKSPACE', [
+        'load("//:deps.bzl", "repos")',
+        'repos()',
+    ])
+    self.ScratchFile('BUILD.bazel')
+    self.ScratchFile('deps.bzl', [
+        'load("//:private_deps.bzl", "other_repos")',
+        'def repos():',
+        '    native.new_local_repository(',
+        '        name = "io_bazel_rules_go",',
+        '        path = ".",',
+        """        build_file_content = "exports_files(glob(['*.go']))",""",
+        '    )',
+        '    other_repos()',
+        '',
+    ])
+    self.ScratchFile('private_deps.bzl', [
+        'def other_repos():',
+        '    native.new_local_repository(',
+        '        name = "io_bazel_rules_python",',
+        '        path = ".",',
+        """        build_file_content = "exports_files(glob(['*.py']))",""",
+        '    )',
+    ])
+
+    self._AssertQueryOutputContains(
+        'buildfiles(//external:io_bazel_rules_python)', '//external:WORKSPACE',
+        '//:deps.bzl', '//:private_deps.bzl', '//:BUILD.bazel')
+
+  def testBuildFilesForExternalRepos_NoDuplicates(self):
+    self.ScratchFile('WORKSPACE', [
+        'load("//:deps.bzl", "repos")',
+        'repos()',
+    ])
+    self.ScratchFile('BUILD.bazel')
+    self.ScratchFile('deps.bzl', [
+        'def repos():',
+        '    native.new_local_repository(',
+        '        name = "io_bazel_rules_go",',
+        '        path = ".",',
+        """        build_file_content = "exports_files(glob(['*.go']))",""",
+        '    )',
+        '    other_repos()',
+        '',
+        'def other_repos():',
+        '    native.new_local_repository(',
+        '        name = "io_bazel_rules_python",',
+        '        path = ".",',
+        """        build_file_content = "exports_files(glob(['*.py']))",""",
+        '    )',
+    ])
+
+    exit_code, stdout, stderr = self.RunBazel(
+        ['query', 'buildfiles(//external:io_bazel_rules_python)'])
+    self.AssertExitCode(exit_code, 0, stderr)
+    result = set()
+    for item in stdout:
+      if not item:
+        continue
+      self.assertNotIn(item, result)
+      result.add(item)
+
   def _AssertQueryOutput(self, query_expr, *expected_results):
     exit_code, stdout, stderr = self.RunBazel(['query', query_expr])
     self.AssertExitCode(exit_code, 0, stderr)
@@ -46,6 +127,14 @@ def _AssertQueryOutput(self, query_expr, *expected_results):
     self.assertEqual(len(stdout), len(expected_results))
     self.assertListEqual(stdout, sorted(expected_results))
 
+  def _AssertQueryOutputContains(self, query_expr, *expected_content):
+    exit_code, stdout, stderr = self.RunBazel(['query', query_expr])
+    self.AssertExitCode(exit_code, 0, stderr)
+
+    stdout = {x for x in stdout if x}
+    for item in expected_content:
+      self.assertIn(item, stdout)
+
 
 if __name__ == '__main__':
   unittest.main()