Wire credential helper to repository fetching.

After this change, credential helpers will be used when available to obtain
credentials for repository fetching, taking precedence over the `auth`
parameter to rctx.download and rctx.download_and_extract.

Tests that need a credential helper are skipped on Windows for now, as
otherwise the credential helper would have to be reimplemented in Batch
or Powershell.

Also improve the documentation for credential helper related flags.

Fixes #15013.

src/main/java/com/google/devtools/build/lib/authandtls/AuthAndTLSOptions.java

@@ -148,11 +148,12 @@ public class AuthAndTLSOptions extends OptionsBase {
       documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
       effectTags = {OptionEffectTag.UNKNOWN},
       help =
-          "Configures Credential Helpers to use for retrieving credentials for the provided scope"
-              + " (domain).\n\n"
-              + "Credentials from Credential Helpers take precedence over credentials from"
-              + " <code>--google_default_credentials</code>, `--google_credentials`, or"
-              + " <code>.netrc</code>.\n\n"
+          "Configures a credential helper to use for retrieving authorization credentials for "
+              + " repository fetching, remote caching and execution, and the build event service.\n\n"
+              + "Credentials supplied by a helper take precedence over credentials supplied by"
+              + " --google_default_credentials, --google_credentials, a .netrc file, or the auth"
+              + " parameter to repository_ctx.download and repository_ctx.download_and_extract.\n\n"
+              + "May be specified multiple times to set up multiple helpers.\n\n"
               + "See https://github.com/bazelbuild/proposals/blob/main/designs/2022-06-07-bazel-credential-helpers.md"
               + " for details.")
   public List<UnresolvedScopedCredentialHelper> credentialHelpers;
@@ -164,8 +165,8 @@ public class AuthAndTLSOptions extends OptionsBase {
       documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
       effectTags = {OptionEffectTag.UNKNOWN},
       help =
-          "Configures the timeout for the Credential Helper.\n\n"
-              + "Credential Helpers failing to respond within this timeout will fail the"
+          "Configures the timeout for a credential helper.\n\n"
+              + "Credential helpers failing to respond within this timeout will fail the"
               + " invocation.")
   public Duration credentialHelperTimeout;
 
@@ -176,31 +177,43 @@ public class AuthAndTLSOptions extends OptionsBase {
       documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
       effectTags = {OptionEffectTag.UNKNOWN},
       help =
-          "Configures the duration for which credentials from Credential Helpers are cached.\n\n"
+          "The duration for which credentials supplied by a credential helper are cached.\n\n"
               + "Invoking with a different value will adjust the lifetime of preexisting entries;"
               + " pass zero to clear the cache. A clean command always clears the cache, regardless"
               + " of this flag.")
   public Duration credentialHelperCacheTimeout;
 
-  /** One of the values of the `--credential_helper` flag. */
+  /**
+   * One of the values of the `--experimental_credential_helper` flag.
+   */
   @AutoValue
   public abstract static class UnresolvedScopedCredentialHelper {
-    /** Returns the scope of the credential helper (if any). */
+
+    /**
+     * Returns the scope of the credential helper (if any).
+     */
     public abstract Optional<String> getScope();
 
-    /** Returns the (unparsed) path of the credential helper. */
+    /**
+     * Returns the (unparsed) path of the credential helper.
+     */
     public abstract String getPath();
   }
 
-  /** A {@link Converter} for the `--credential_helper` flag. */
+  /**
+   * A {@link Converter} for the `--experimental_credential_helper` flag.
+   */
   public static final class UnresolvedScopedCredentialHelperConverter
       extends Converter.Contextless<UnresolvedScopedCredentialHelper> {
+
     public static final UnresolvedScopedCredentialHelperConverter INSTANCE =
         new UnresolvedScopedCredentialHelperConverter();
 
     @Override
     public String getTypeDescription() {
-      return "An (unresolved) path to a credential helper for a scope.";
+      return "Path to a credential helper, optionally prefixed by a scope (domain name) followed"
+          + " an '='. The path may be absolute or relative. If no scope is provided, the helper"
+          + " has universal scope.";
     }
 
     @Override

src/main/java/com/google/devtools/build/lib/authandtls/GoogleAuthUtils.java

@@ -372,7 +372,6 @@ static Optional<Credentials> newCredentialsFromNetrc(
     }
   }
 
-  @VisibleForTesting
   public static CredentialHelperProvider newCredentialHelperProvider(
       CredentialHelperEnvironment environment,
       CommandLinePathFactory pathFactory,

src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/BUILD

@@ -20,6 +20,7 @@ java_library(
     deps = [
         "//src/main/java/com/google/devtools/build/lib:runtime",
         "//src/main/java/com/google/devtools/build/lib/authandtls",
+        "//src/main/java/com/google/devtools/common/options:options_internal",
         "//third_party:caffeine",
         "//third_party:guava",
     ],

src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper/CredentialModule.java

@@ -21,22 +21,33 @@
 import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
 import com.google.devtools.build.lib.runtime.BlazeModule;
 import com.google.devtools.build.lib.runtime.CommandEnvironment;
+import com.google.devtools.common.options.OptionsBase;
 import java.net.URI;
 import java.time.Duration;
 
-/** A module whose sole purpose is to hold the credential cache which is shared by other modules. */
+/**
+ * A module whose sole purpose is to hold the credential cache which is shared by other modules.
+ */
 public class CredentialModule extends BlazeModule {
+
   private final Cache<URI, ImmutableMap<String, ImmutableList<String>>> credentialCache =
       Caffeine.newBuilder()
           .expireAfterWrite(Duration.ZERO)
           .ticker(SystemMillisTicker.INSTANCE)
           .build();
 
-  /** Returns the credential cache. */
+  /**
+   * Returns the credential cache.
+   */
   public Cache<URI, ImmutableMap<String, ImmutableList<String>>> getCredentialCache() {
     return credentialCache;
   }
 
+  @Override
+  public Iterable<Class<? extends OptionsBase>> getCommonCommandOptions() {
+    return ImmutableList.of(AuthAndTLSOptions.class);
+  }
+
   @Override
   public void beforeCommand(CommandEnvironment env) {
     // Update the cache expiration policy according to the command options.

src/main/java/com/google/devtools/build/lib/bazel/BUILD

@@ -27,6 +27,9 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/analysis:analysis_cluster",
         "//src/main/java/com/google/devtools/build/lib/analysis:blaze_directories",
         "//src/main/java/com/google/devtools/build/lib/analysis:config/build_configuration",
+        "//src/main/java/com/google/devtools/build/lib/authandtls",
+        "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper",
+        "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper:credential_module",
         "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:common",
         "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:inspection",
         "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:inspection_impl",
@@ -55,6 +58,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/starlarkbuildapi/repository",
         "//src/main/java/com/google/devtools/build/lib/util:abrupt_exit_exception",
         "//src/main/java/com/google/devtools/build/lib/util:detailed_exit_code",
+        "//src/main/java/com/google/devtools/build/lib/util:exit_code",
         "//src/main/java/com/google/devtools/build/lib/vfs",
         "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
         "//src/main/java/com/google/devtools/common/options",

src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java

@@ -15,6 +15,7 @@
 
 package com.google.devtools.build.lib.bazel;
 
+import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
 import com.google.common.base.Supplier;
 import com.google.common.collect.ImmutableList;
@@ -25,6 +26,13 @@
 import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
 import com.google.devtools.build.lib.analysis.RuleDefinition;
 import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue;
+import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions;
+import com.google.devtools.build.lib.authandtls.GoogleAuthUtils;
+import com.google.devtools.build.lib.authandtls.StaticCredentials;
+import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperCredentials;
+import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperEnvironment;
+import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialHelperProvider;
+import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialModule;
 import com.google.devtools.build.lib.bazel.bzlmod.AttributeValues;
 import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphFunction;
 import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction;
@@ -110,6 +118,7 @@
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.Collectors;
+import javax.annotation.Nullable;
 
 /** Adds support for fetching external code. */
 public class BazelRepositoryModule extends BlazeModule {
@@ -146,6 +155,9 @@ public class BazelRepositoryModule extends BlazeModule {
   private List<String> allowedYankedVersions = ImmutableList.of();
   private SingleExtensionEvalFunction singleExtensionEvalFunction;
 
+  @Nullable
+  private CredentialModule credentialModule;
+
   public BazelRepositoryModule() {
     this.starlarkRepositoryFunction = new StarlarkRepositoryFunction(downloadManager);
     this.repositoryHandlers = repositoryRules();
@@ -263,6 +275,8 @@ SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(directories.getWorkspace
         .addSkyFunction(SkyFunctions.SINGLE_EXTENSION_EVAL, singleExtensionEvalFunction)
         .addSkyFunction(SkyFunctions.SINGLE_EXTENSION_USAGES, new SingleExtensionUsagesFunction());
     filesystem = runtime.getFileSystem();
+
+    credentialModule = Preconditions.checkNotNull(runtime.getBlazeModule(CredentialModule.class));
   }
 
   @Override
@@ -373,6 +387,41 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
                 Code.BAD_DOWNLOADER_CONFIG));
       }
 
+      try {
+        AuthAndTLSOptions authAndTlsOptions = env.getOptions().getOptions(AuthAndTLSOptions.class);
+        var credentialHelperEnvironment =
+            CredentialHelperEnvironment.newBuilder()
+                .setEventReporter(env.getReporter())
+                .setWorkspacePath(env.getWorkspace())
+                .setClientEnvironment(env.getClientEnv())
+                .setHelperExecutionTimeout(authAndTlsOptions.credentialHelperTimeout)
+                .build();
+        CredentialHelperProvider credentialHelperProvider =
+            GoogleAuthUtils.newCredentialHelperProvider(
+                credentialHelperEnvironment,
+                env.getCommandLinePathFactory(),
+                authAndTlsOptions.credentialHelpers);
+
+        downloadManager.setCredentialFactory(headers -> {
+          Preconditions.checkNotNull(headers);
+
+          return new CredentialHelperCredentials(
+              credentialHelperProvider,
+              credentialHelperEnvironment,
+              credentialModule.getCredentialCache(),
+              Optional.of(new StaticCredentials(headers)));
+        });
+      } catch (IOException e) {
+        env.getReporter().handle(Event.error(e.getMessage()));
+        env.getBlazeModuleEnvironment()
+            .exit(
+                new AbruptExitException(
+                    detailedExitCode(
+                        "Error initializing credential helper",
+                        Code.CREDENTIALS_INIT_FAILURE)));
+        return;
+      }
+
       if (repoOptions.experimentalDistdir != null) {
         downloadManager.setDistdir(
             repoOptions.experimentalDistdir.stream()

src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java

@@ -60,7 +60,17 @@ public class DownloadManager {
   private boolean disableDownload = false;
   private int retries = 0;
   private boolean urlsAsDefaultCanonicalId;
-  @Nullable private Credentials netrcCreds;
+  @Nullable
+  private Credentials netrcCreds;
+  private CredentialFactory credentialFactory = StaticCredentials::new;
+
+  /**
+   * Creates {@code Credentials} from a map of per-{@code URI} authentication headers.
+   */
+  public interface CredentialFactory {
+
+    Credentials create(Map<URI, Map<String, List<String>>> authHeaders);
+  }
 
   public DownloadManager(RepositoryCache repositoryCache, Downloader downloader) {
     this.repositoryCache = repositoryCache;
@@ -92,6 +102,10 @@ public void setNetrcCreds(Credentials netrcCreds) {
     this.netrcCreds = netrcCreds;
   }
 
+  public void setCredentialFactory(CredentialFactory credentialFactory) {
+    this.credentialFactory = credentialFactory;
+  }
+
   /**
    * Downloads file to disk and returns path.
    *
@@ -257,7 +271,7 @@ public Path download(
       try {
         downloader.download(
             rewrittenUrls,
-            new StaticCredentials(rewrittenAuthHeaders),
+            credentialFactory.create(rewrittenAuthHeaders),
             checksum,
             canonicalId,
             destination,
@@ -338,7 +352,7 @@ public byte[] downloadAndReadOneUrl(
     for (int attempt = 0; attempt <= retries; ++attempt) {
       try {
         return httpDownloader.downloadAndReadOneUrl(
-            rewrittenUrls.get(0), new StaticCredentials(authHeaders), eventHandler, clientEnv);
+            rewrittenUrls.get(0), credentialFactory.create(authHeaders), eventHandler, clientEnv);
       } catch (ContentLengthMismatchException e) {
         if (attempt == retries) {
           throw e;

src/main/protobuf/failure_details.proto

@@ -249,6 +249,7 @@ message ExternalRepository {
     OVERRIDE_DISALLOWED_MANAGED_DIRECTORIES = 1 [(metadata) = { exit_code: 2 }];
     BAD_DOWNLOADER_CONFIG = 2 [(metadata) = { exit_code: 2 }];
     REPOSITORY_MAPPING_RESOLUTION_FAILED = 3 [(metadata) = { exit_code: 37 }];
+    CREDENTIALS_INIT_FAILURE = 4 [(metadata) = { exit_code: 2 }];
   }
   Code code = 1;
   // Additional data could include external repository names.

src/test/java/com/google/devtools/build/lib/buildtool/util/BUILD

@@ -54,6 +54,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/analysis:top_level_artifact_context",
         "//src/main/java/com/google/devtools/build/lib/analysis:transitive_info_collection",
         "//src/main/java/com/google/devtools/build/lib/analysis:workspace_status_action",
+        "//src/main/java/com/google/devtools/build/lib/authandtls/credentialhelper:credential_module",
         "//src/main/java/com/google/devtools/build/lib/bazel:main",
         "//src/main/java/com/google/devtools/build/lib/bazel:repository_module",
         "//src/main/java/com/google/devtools/build/lib/bugreport",

src/test/java/com/google/devtools/build/lib/buildtool/util/BuildIntegrationTestCase.java

@@ -53,6 +53,7 @@
 import com.google.devtools.build.lib.analysis.util.AnalysisMock;
 import com.google.devtools.build.lib.analysis.util.AnalysisTestUtil;
 import com.google.devtools.build.lib.analysis.util.AnalysisTestUtil.DummyWorkspaceStatusActionContext;
+import com.google.devtools.build.lib.authandtls.credentialhelper.CredentialModule;
 import com.google.devtools.build.lib.bazel.BazelRepositoryModule;
 import com.google.devtools.build.lib.bugreport.BugReport;
 import com.google.devtools.build.lib.bugreport.BugReporter;
@@ -571,7 +572,8 @@ protected BlazeRuntime.Builder getRuntimeBuilder() throws Exception {
             .addBlazeModule(new OutputFilteringModule())
             .addBlazeModule(connectivityModule)
             .addBlazeModule(new SkymeldModule())
-            .addBlazeModule(getMockBazelRepositoryModule());
+            .addBlazeModule(getMockBazelRepositoryModule())
+            .addBlazeModule(new CredentialModule());
     getSpawnModules().forEach(builder::addBlazeModule);
     builder
         .addBlazeModule(getBuildInfoModule())

src/test/shell/bazel/remote_helpers.sh

@@ -38,8 +38,7 @@ function serve_file() {
   cd -
 }
 
-# Serves $1 as a file on localhost:$nc_port insisting on authentication (but
-# accepting any credentials.
+# Serves $1 as a file on localhost:$nc_port expecting authentication.
 #   * nc_port - the port nc is listening on.
 #   * nc_log - the path to nc's log.
 #   * nc_pid - the PID of nc.