Add option to throttle the action cache checks.

With a high number of jobs, and a large action cache, we have seen machines being overloaded with a load > 500.

This is a good candidate to play with virtual threads and Loom in the future, but we want to be able to experiment with it now.

PiperOrigin-RevId: 500753925
Change-Id: Ibaad4ce075743db792dc802b499316590cfaf508

src/main/java/com/google/devtools/build/lib/analysis/config/CoreOptions.java

@@ -897,6 +897,16 @@ public OutputPathsConverter() {
               + " of failing. This is to help use cquery diagnose failures in select.")
   public boolean debugSelectsAlwaysSucceed;
 
+  @Option(
+      name = "experimental_throttle_action_cache_check",
+      defaultValue = "false",
+      converter = BooleanConverter.class,
+      documentationCategory = OptionDocumentationCategory.UNDOCUMENTED,
+      metadataTags = OptionMetadataTag.EXPERIMENTAL,
+      effectTags = {OptionEffectTag.EXECUTION},
+      help = "Whether to throttle the check whether an action is cached.")
+  public boolean throttleActionCacheCheck;
+
   /** Ways configured targets may provide the {@link Fragment}s they require. */
   public enum IncludeConfigFragmentsEnum {
     /**

src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java

@@ -94,6 +94,7 @@
 import com.google.devtools.build.lib.skyframe.ActionOutputDirectoryHelper.CreateOutputDirectoryException;
 import com.google.devtools.build.lib.util.CrashFailureDetails;
 import com.google.devtools.build.lib.util.DetailedExitCode;
+import com.google.devtools.build.lib.util.ResourceUsage;
 import com.google.devtools.build.lib.util.io.FileOutErr;
 import com.google.devtools.build.lib.vfs.FileSystem;
 import com.google.devtools.build.lib.vfs.FileSystem.NotASymlinkException;
@@ -113,6 +114,7 @@
 import java.util.Set;
 import java.util.SortedMap;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.Semaphore;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
 import java.util.function.Supplier;
@@ -210,6 +212,8 @@ public void injectTree(SpecialArtifact output, TreeArtifactValue tree) {
 
   private DiscoveredModulesPruner discoveredModulesPruner;
 
+  @Nullable private Semaphore cacheHitSemaphore;
+
   SkyframeActionExecutor(
       ActionKeyContext actionKeyContext,
       MetadataConsumerForMetrics outputArtifactsSeen,
@@ -280,6 +284,11 @@ void prepareForExecution(
     // actions, which consume their shadowed action's discovered inputs.
     freeDiscoveredInputsAfterExecution =
         !trackIncrementalState && options.getOptions(CoreOptions.class).actionListeners.isEmpty();
+
+    this.cacheHitSemaphore =
+        options.getOptions(CoreOptions.class).throttleActionCacheCheck
+            ? new Semaphore(ResourceUsage.getAvailableProcessors())
+            : null;
   }
 
   public void setActionLogBufferPathGenerator(
@@ -586,6 +595,12 @@ Token checkActionCache(
     SortedMap<String, String> remoteDefaultProperties;
     EventHandler handler;
     boolean loadCachedOutputMetadata;
+
+    if (cacheHitSemaphore != null) {
+      try (SilentCloseable c = profiler.profile(ProfilerTask.ACTION_CHECK, "acquiring semaphore")) {
+        cacheHitSemaphore.acquire();
+      }
+    }
     try (SilentCloseable c = profiler.profile(ProfilerTask.ACTION_CHECK, action.describe())) {
       remoteOptions = this.options.getOptions(RemoteOptions.class);
       remoteDefaultProperties =
@@ -664,6 +679,10 @@ public <T extends ActionContext> T getContext(Class<? extends T> type) {
       }
     } catch (UserExecException e) {
       throw ActionExecutionException.fromExecException(e, action);
+    } finally {
+      if (cacheHitSemaphore != null) {
+        cacheHitSemaphore.release();
+      }
     }
     return token;
   }