diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java index 2cf75124ce8252..e6890842321789 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java @@ -535,13 +535,29 @@ private ToolSignature getToolSignature(Spawn spawn, SpawnExecutionContext contex : null; } + private void maybeAcquireRemoteActionBuildingSemaphore(ProfilerTask task) + throws InterruptedException { + if (!remoteOptions.throttleRemoteActionBuilding) { + return; + } + + try (var c = Profiler.instance().profile(task, "acquiring semaphore")) { + remoteActionBuildingSemaphore.acquire(); + } + } + + private void maybeReleaseRemoteActionBuildingSemaphore() { + if (!remoteOptions.throttleRemoteActionBuilding) { + return; + } + + remoteActionBuildingSemaphore.release(); + } + /** Creates a new {@link RemoteAction} instance from spawn. */ public RemoteAction buildRemoteAction(Spawn spawn, SpawnExecutionContext context) throws IOException, ExecException, ForbiddenActionInputException, InterruptedException { - try (SilentCloseable c = - Profiler.instance().profile(ProfilerTask.REMOTE_SETUP, "acquiring semaphore")) { - remoteActionBuildingSemaphore.acquire(); - } + maybeAcquireRemoteActionBuildingSemaphore(ProfilerTask.REMOTE_SETUP); try { // Create a remote path resolver that is aware of the spawn's path mapper, which rewrites // the paths of the inputs and outputs as well as paths appearing in the command line for @@ -603,7 +619,7 @@ public RemoteAction buildRemoteAction(Spawn spawn, SpawnExecutionContext context actionKey, remoteOptions.remoteDiscardMerkleTrees); } finally { - remoteActionBuildingSemaphore.release(); + maybeReleaseRemoteActionBuildingSemaphore(); } } @@ -1436,10 +1452,7 @@ public void uploadInputsIfNotPresent(RemoteAction action, boolean force) // concurrency. This prevents memory exhaustion. We assume that // ensureInputsPresent() provides enough parallelism to saturate the // network connection. - try (SilentCloseable c = - Profiler.instance().profile(ProfilerTask.UPLOAD_TIME, "acquiring semaphore")) { - remoteActionBuildingSemaphore.acquire(); - } + maybeAcquireRemoteActionBuildingSemaphore(ProfilerTask.UPLOAD_TIME); try { MerkleTree merkleTree = action.getMerkleTree(); if (merkleTree == null) { @@ -1462,7 +1475,7 @@ public void uploadInputsIfNotPresent(RemoteAction action, boolean force) additionalInputs, force); } finally { - remoteActionBuildingSemaphore.release(); + maybeReleaseRemoteActionBuildingSemaphore(); } } diff --git a/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java b/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java index 4d222479430d07..39827802b22bb7 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java +++ b/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java @@ -28,6 +28,7 @@ import com.google.devtools.common.options.Converter; import com.google.devtools.common.options.Converters; import com.google.devtools.common.options.Converters.AssignmentConverter; +import com.google.devtools.common.options.Converters.BooleanConverter; import com.google.devtools.common.options.EnumConverter; import com.google.devtools.common.options.Option; import com.google.devtools.common.options.OptionDocumentationCategory; @@ -725,6 +726,19 @@ public RemoteOutputsStrategyConverter() { + " variables).") public Scrubber scrubber; + @Option( + name = "experimental_throttle_remote_action_building", + defaultValue = "true", + converter = BooleanConverter.class, + documentationCategory = OptionDocumentationCategory.UNDOCUMENTED, + metadataTags = OptionMetadataTag.EXPERIMENTAL, + effectTags = {OptionEffectTag.EXECUTION}, + help = + "Whether to throttle the building of remote action to avoid OOM. Defaults to true.\n\n" + + "This is a temporary flag to allow users switch off the behaviour. Once Bazel is" + + " smart enough about the RAM/CPU usages, this flag will be removed.") + public boolean throttleRemoteActionBuilding; + private static final class ScrubberConverter extends Converter.Contextless { @Override