forked from bazelbuild/bazel
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Introduce options for remote cache key scrubbing.
This PR introduces a set of --experimental_remote_scrub_* flags that can be used to scrub platform-dependent data from the key used to retrieve and store action results from a remote or disk cache, so that actions executing on different platforms but targeting the same platform may be able to share cache entries. This is a simplified implementation of one of the ideas described in [1], highly influenced by Olivier Notteghem's earlier proposal [2], intended to provide a simple yet flexible API to enable further experimentation. It must be used with care, as incorrect settings can compromise build correctness. [1] https://docs.google.com/document/d/1uMPj2s0TlHSIKSngqOkWJoeqOtKzaxQLtBrRfYif3Lo/edit?usp=sharing [2] bazelbuild#18669
- Loading branch information
Showing
13 changed files
with
839 additions
and
19 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
156 changes: 156 additions & 0 deletions
156
src/main/java/com/google/devtools/build/lib/remote/Scrubber.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,156 @@ | ||
| // Copyright 2023 The Bazel Authors. All rights reserved. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
| package com.google.devtools.build.lib.remote; | ||
|
|
||
| import static com.google.common.base.Preconditions.checkState; | ||
|
|
||
| import com.google.common.base.Predicates; | ||
| import com.google.common.collect.ImmutableMap; | ||
| import com.google.devtools.build.lib.actions.ActionInput; | ||
| import com.google.devtools.build.lib.actions.Spawn; | ||
| import com.google.devtools.build.lib.actions.cache.VirtualActionInput; | ||
| import com.google.devtools.build.lib.remote.options.RemoteOptions; | ||
| import com.google.devtools.common.options.RegexPatternOption; | ||
| import java.util.Collection; | ||
| import java.util.Map; | ||
| import java.util.function.Predicate; | ||
| import java.util.regex.Matcher; | ||
| import java.util.regex.Pattern; | ||
| import javax.annotation.Nullable; | ||
|
|
||
| /** | ||
| * The {@link Scrubber} implements scrubbing of remote cache keys. | ||
| * | ||
| * <p>See the documentation for the {@code --experimental_remote_scrub_*} flags for more | ||
| * information. | ||
| */ | ||
| public class Scrubber { | ||
|
|
||
| /** | ||
| * A {@link SpawnScrubber} determines how to scrub the cache key for a {@link Spawn}. | ||
| */ | ||
| public static class SpawnScrubber { | ||
|
|
||
| private final Predicate<ActionInput> inputMatcher; | ||
| private final ImmutableMap<RegexPatternOption, String> argReplacements; | ||
| private final String salt; | ||
|
|
||
| private SpawnScrubber(RemoteOptions options) { | ||
| this.inputMatcher = buildInputMatcher(options); | ||
| this.argReplacements = ImmutableMap.copyOf(options.scrubArgReplacements); | ||
| this.salt = options.scrubSalt; | ||
| } | ||
|
|
||
| private static Predicate<ActionInput> buildInputMatcher(RemoteOptions options) { | ||
| Predicate<String> execPathMatcher = buildStringMatcher(options.scrubInput); | ||
| return (input) -> !input.equals(VirtualActionInput.EMPTY_MARKER) && execPathMatcher.test(input.getExecPathString()); | ||
| } | ||
|
|
||
| /** | ||
| * Whether the given input should be omitted from the cache key. | ||
| */ | ||
| public boolean shouldOmitInput(ActionInput input) { | ||
| return inputMatcher.test(input); | ||
| } | ||
|
|
||
| /** | ||
| * Transforms an action command line argument. | ||
| */ | ||
| public String transformArgument(String arg) { | ||
| for (Map.Entry<RegexPatternOption, String> entry : argReplacements.entrySet()) { | ||
| Pattern pattern = entry.getKey().regexPattern(); | ||
| String replacement = entry.getValue(); | ||
| // Don't use Pattern#replaceFirst because it allows references to capture groups. | ||
| Matcher m = pattern.matcher(arg); | ||
| if (m.find()) { | ||
| arg = arg.substring(0, m.start()) + replacement + arg.substring(m.end()); | ||
| } | ||
| } | ||
| return arg; | ||
| } | ||
|
|
||
| /** | ||
| * Returns the scrubbing salt. | ||
| */ | ||
| public String getSalt() { | ||
| return salt; | ||
| } | ||
| } | ||
|
|
||
| private final Predicate<Spawn> spawnMatcher; | ||
|
|
||
| private final SpawnScrubber spawnScrubber; | ||
|
|
||
| private Scrubber(RemoteOptions options) { | ||
| this.spawnMatcher = buildSpawnMatcher(options); | ||
| this.spawnScrubber = new SpawnScrubber(options); | ||
| } | ||
|
|
||
| /** | ||
| * Returns a {@link Scrubber} that performs scrubbing according to the {@link RemoteOptions}. | ||
| */ | ||
| @Nullable | ||
| public static Scrubber forOptions(RemoteOptions options) { | ||
| return new Scrubber(options); | ||
| } | ||
|
|
||
| private static Predicate<Spawn> buildSpawnMatcher(RemoteOptions options) { | ||
| if (!options.scrubEnabled) { | ||
| return Predicates.alwaysFalse(); | ||
| } | ||
|
|
||
| Predicate<String> mnemonicMatcher = buildStringMatcher(options.scrubMnemonic); | ||
| Predicate<String> repoMatcher = buildStringMatcher(options.scrubRepo); | ||
| boolean scrubExec = options.scrubExec; | ||
|
|
||
| return (spawn) -> { | ||
| String mnemonic = spawn.getMnemonic(); | ||
| String repo = spawn.getResourceOwner().getOwner().getLabel().getRepository().getName(); | ||
| boolean isForTool = spawn.getResourceOwner().getOwner().isBuildConfigurationForTool(); | ||
|
|
||
| return (!isForTool || scrubExec) && mnemonicMatcher.test(mnemonic) && repoMatcher.test(repo); | ||
| }; | ||
| } | ||
|
|
||
| private static Predicate<String> buildStringMatcher(Collection<RegexPatternOption> options) { | ||
| if (options.isEmpty()) { | ||
| // If no patterns are specified, match nothing. | ||
| return Predicates.alwaysFalse(); | ||
| } | ||
| // Combine multiple patterns into a single one for efficiency. | ||
| StringBuilder sb = new StringBuilder(); | ||
| for (RegexPatternOption opt : options) { | ||
| if (sb.length() > 0) { | ||
| sb.append("|"); | ||
| } | ||
| sb.append("(?:"); | ||
| sb.append(opt.regexPattern().pattern()); | ||
| sb.append(")"); | ||
| } | ||
| Pattern pattern = Pattern.compile(sb.toString()); | ||
| return (str) -> pattern.matcher(str).find(); | ||
| } | ||
|
|
||
| /** | ||
| * Returns a {@link SpawnScrubber} suitable for a {@link Spawn}, or {@code null} if the spawn does | ||
| * not need to be scrubbed. | ||
| */ | ||
| @Nullable | ||
| public SpawnScrubber forSpawn(Spawn spawn) { | ||
| if (spawnMatcher.test(spawn)) { | ||
| return spawnScrubber; | ||
| } | ||
| return null; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.