Vendoring

Vendor command fetches all external repositories into a folder specified by the flag --vendor_dir.
When building using the same flag & value the vendored repos are used as long as they are up to date.

Related #19563

PiperOrigin-RevId: 585105301
Change-Id: I08196c7ad924c1d22918ba09dba2ae774802a691

src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java

@@ -56,6 +56,7 @@
 import com.google.devtools.build.lib.bazel.commands.FetchCommand;
 import com.google.devtools.build.lib.bazel.commands.ModCommand;
 import com.google.devtools.build.lib.bazel.commands.SyncCommand;
+import com.google.devtools.build.lib.bazel.commands.VendorCommand;
 import com.google.devtools.build.lib.bazel.repository.LocalConfigPlatformFunction;
 import com.google.devtools.build.lib.bazel.repository.LocalConfigPlatformRule;
 import com.google.devtools.build.lib.bazel.repository.RepositoryOptions;
@@ -156,6 +157,8 @@ public class BazelRepositoryModule extends BlazeModule {
   private CheckDirectDepsMode checkDirectDepsMode = CheckDirectDepsMode.WARNING;
   private BazelCompatibilityMode bazelCompatibilityMode = BazelCompatibilityMode.ERROR;
   private LockfileMode bazelLockfileMode = LockfileMode.UPDATE;
+
+  private Optional<Path> vendorDirectory;
   private List<String> allowedYankedVersions = ImmutableList.of();
   private SingleExtensionEvalFunction singleExtensionEvalFunction;
   private final ExecutorService repoFetchingWorkerThreadPool =
@@ -216,6 +219,7 @@ public void serverInit(OptionsParsingResult startupOptions, ServerBuilder builde
     builder.addCommands(new FetchCommand());
     builder.addCommands(new ModCommand());
     builder.addCommands(new SyncCommand());
+    builder.addCommands(new VendorCommand());
     builder.addInfoItems(new RepositoryCacheInfoItem(repositoryCache));
   }
 
@@ -516,6 +520,16 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
       bazelLockfileMode = repoOptions.lockfileMode;
       allowedYankedVersions = repoOptions.allowedYankedVersions;
 
+      if (repoOptions.vendorDirectory != null) {
+        vendorDirectory =
+            Optional.of(
+                repoOptions.vendorDirectory.isAbsolute()
+                    ? filesystem.getPath(repoOptions.vendorDirectory)
+                    : env.getWorkspace().getRelative(repoOptions.vendorDirectory));
+      } else {
+        vendorDirectory = Optional.empty();
+      }
+
       if (repoOptions.registries != null && !repoOptions.registries.isEmpty()) {
         registries = repoOptions.registries;
       } else {
@@ -580,13 +594,15 @@ public ImmutableList<Injected> getPrecomputedValues() {
         PrecomputedValue.injected(
             RepositoryDelegatorFunction.FORCE_FETCH_CONFIGURE,
             RepositoryDelegatorFunction.FORCE_FETCH_DISABLED),
+        PrecomputedValue.injected(RepositoryDelegatorFunction.IS_VENDOR_COMMAND, false),
         PrecomputedValue.injected(ModuleFileFunction.REGISTRIES, registries),
         PrecomputedValue.injected(ModuleFileFunction.IGNORE_DEV_DEPS, ignoreDevDeps.get()),
         PrecomputedValue.injected(
             BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, checkDirectDepsMode),
         PrecomputedValue.injected(
             BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, bazelCompatibilityMode),
         PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, bazelLockfileMode),
+        PrecomputedValue.injected(RepositoryDelegatorFunction.VENDOR_DIRECTORY, vendorDirectory),
         PrecomputedValue.injected(
             YankedVersionsUtil.ALLOWED_YANKED_VERSIONS, allowedYankedVersions));
   }

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelFetchAllFunction.java

@@ -18,6 +18,7 @@
 import static com.google.common.collect.ImmutableList.toImmutableList;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import com.google.devtools.build.lib.bazel.repository.starlark.StarlarkRepositoryFunction;
 import com.google.devtools.build.lib.cmdline.RepositoryName;
@@ -31,8 +32,8 @@
 import javax.annotation.Nullable;
 
 /**
- * Void function designed to gather and fetch all the repositories without returning any specific
- * result (empty value is returned).
+ * Gather and fetch all the repositories from MODULE.bazel resolution and extensions evaluation. If
+ * this is fetch configure, only configure repos will be fetched and returned
  */
 public class BazelFetchAllFunction implements SkyFunction {
 
@@ -53,14 +54,12 @@ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedExcept
             .filter(repo -> !repo.isMain())
             .collect(toImmutableList()));
 
-    // 2. Run every extension found in the modules
+    // 2. Run every extension found in the modules & collect its generated repos
     ImmutableSet<ModuleExtensionId> extensionIds =
         depGraphValue.getExtensionUsagesTable().rowKeySet();
     ImmutableSet<SkyKey> singleEvalKeys =
         extensionIds.stream().map(SingleExtensionEvalValue::key).collect(toImmutableSet());
     SkyframeLookupResult singleEvalValues = env.getValuesAndExceptions(singleEvalKeys);
-
-    // 3. For each extension, collect its generated repos
     for (SkyKey singleEvalKey : singleEvalKeys) {
       SingleExtensionEvalValue singleEvalValue =
           (SingleExtensionEvalValue) singleEvalValues.get(singleEvalKey);
@@ -70,7 +69,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedExcept
       reposToFetch.addAll(singleEvalValue.getCanonicalRepoNameToInternalNames().keySet());
     }
 
-    // 4. If this is fetch configure, get repo rules and only collect repos marked as configure
+    // 3. If this is fetch configure, get repo rules and only collect repos marked as configure
     Boolean fetchConfigure = (Boolean) skyKey.argument();
     if (fetchConfigure) {
       ImmutableSet<SkyKey> repoRuleKeys =
@@ -83,12 +82,13 @@ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedExcept
           return null;
         }
         if (StarlarkRepositoryFunction.isConfigureRule(repoRuleValue.getRule())) {
-          reposToFetch.add(RepositoryName.createUnvalidated(repoRuleValue.getRule().getName()));
+          reposToFetch.add((RepositoryName) repoRuleKey.argument());
         }
       }
     }
 
-    // 5. Fetch all the collected repos
+    // 4. Fetch all the collected repos
+    List<RepositoryName> shouldVendor = new ArrayList<>();
     ImmutableSet<SkyKey> repoDelegatorKeys =
         reposToFetch.stream().map(RepositoryDirectoryValue::key).collect(toImmutableSet());
     SkyframeLookupResult repoDirValues = env.getValuesAndExceptions(repoDelegatorKeys);
@@ -98,9 +98,13 @@ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedExcept
       if (repoDirValue == null) {
         return null;
       }
+      if (!repoDirValue.excludeFromVendoring()) {
+        shouldVendor.add((RepositoryName) repoDelegatorKey.argument());
+      }
     }
 
-    return BazelFetchAllValue.create();
+    return BazelFetchAllValue.create(
+        ImmutableList.copyOf(reposToFetch), ImmutableList.copyOf(shouldVendor));
   }
 
 }

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelFetchAllValue.java

@@ -16,6 +16,8 @@
 package com.google.devtools.build.lib.bazel.bzlmod;
 
 import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.cmdline.RepositoryName;
 import com.google.devtools.build.lib.skyframe.SkyFunctions;
 import com.google.devtools.build.lib.skyframe.serialization.VisibleForSerialization;
 import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec;
@@ -36,8 +38,14 @@ public static BazelFetchAllValue.Key key(Boolean configureEnabled) {
     return BazelFetchAllValue.Key.create(configureEnabled);
   }
 
-  public static BazelFetchAllValue create() {
-    return new AutoValue_BazelFetchAllValue();
+  public abstract ImmutableList<RepositoryName> getFetchedRepos();
+
+  public abstract ImmutableList<RepositoryName> getShouldVendorRepos();
+
+  public static BazelFetchAllValue create(
+      ImmutableList<RepositoryName> fetchedRepos,
+      ImmutableList<RepositoryName> excludeFromVendoring) {
+    return new AutoValue_BazelFetchAllValue(fetchedRepos, excludeFromVendoring);
   }
 
   /** Key type for BazelFetchAllValue. */

src/main/java/com/google/devtools/build/lib/bazel/commands/BUILD

@@ -20,6 +20,7 @@ java_library(
         "fetch.txt",
         "mod.txt",
         "sync.txt",
+        "vendor.txt",
     ],
     deps = [
         "//src/main/java/com/google/devtools/build/lib:keep-going-option",
@@ -36,6 +37,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution",
         "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand",
         "//src/main/java/com/google/devtools/build/lib/bazel/repository",
+        "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options",
         "//src/main/java/com/google/devtools/build/lib/bazel/repository/starlark",
         "//src/main/java/com/google/devtools/build/lib/cmdline",
         "//src/main/java/com/google/devtools/build/lib/events",
@@ -60,6 +62,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/util:interrupted_failure_details",
         "//src/main/java/com/google/devtools/build/lib/util:maybe_complete_set",
         "//src/main/java/com/google/devtools/build/lib/vfs",
+        "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
         "//src/main/java/com/google/devtools/build/skyframe",
         "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
         "//src/main/java/com/google/devtools/common/options",