-
Notifications
You must be signed in to change notification settings - Fork 4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Push down incremental logic from
InMemoryMemoizingEvaluator to `Abs…
…tractIncrementalInMemoryMemoizingEvaluator`. PiperOrigin-RevId: 521590383 Change-Id: I3c0f4956119edf8a41c6871095cca9382bfc6322
- Loading branch information
1 parent
4b194d2
commit 1221742
Showing
2 changed files
with
161 additions
and
111 deletions.
There are no files selected for viewing
151 changes: 151 additions & 0 deletions
151
...ava/com/google/devtools/build/skyframe/AbstractIncrementalInMemoryMemoizingEvaluator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,151 @@ | ||
| // Copyright 2023 The Bazel Authors. All rights reserved. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
| package com.google.devtools.build.skyframe; | ||
|
|
||
| import static com.google.common.base.Preconditions.checkNotNull; | ||
|
|
||
| import com.google.common.collect.ImmutableMap; | ||
| import com.google.common.collect.Iterables; | ||
| import com.google.devtools.build.lib.collect.nestedset.NestedSetVisitor; | ||
| import com.google.devtools.build.skyframe.InvalidatingNodeVisitor.DeletingInvalidationState; | ||
| import com.google.devtools.build.skyframe.InvalidatingNodeVisitor.DirtyingInvalidationState; | ||
| import com.google.devtools.build.skyframe.InvalidatingNodeVisitor.InvalidationState; | ||
| import com.google.devtools.build.skyframe.QueryableGraph.Reason; | ||
| import java.util.HashMap; | ||
| import java.util.Iterator; | ||
| import java.util.LinkedHashSet; | ||
| import java.util.Map; | ||
| import java.util.Map.Entry; | ||
| import java.util.Set; | ||
|
|
||
| /** | ||
| * Partial implementation of {@link MemoizingEvaluator} with expanded support for incremental and | ||
| * non-incremental evaluations on an {@link InMemoryGraph}. | ||
| */ | ||
| abstract class AbstractIncrementalInMemoryMemoizingEvaluator | ||
| extends AbstractInMemoryMemoizingEvaluator { | ||
|
|
||
| final ImmutableMap<SkyFunctionName, SkyFunction> skyFunctions; | ||
| final DirtyTrackingProgressReceiver progressReceiver; | ||
|
|
||
| // State related to invalidation and deletion. | ||
| Set<SkyKey> valuesToDelete = new LinkedHashSet<>(); | ||
| private Set<SkyKey> valuesToDirty = new LinkedHashSet<>(); | ||
| Map<SkyKey, SkyValue> valuesToInject = new HashMap<>(); | ||
| private final DeletingInvalidationState deleterState = new DeletingInvalidationState(); | ||
| final Differencer differencer; | ||
| final GraphInconsistencyReceiver graphInconsistencyReceiver; | ||
| final EventFilter eventFilter; | ||
|
|
||
| // Keep edges in graph. Can be false to save memory, in which case incremental builds are | ||
| // not possible. | ||
| private final boolean keepEdges; | ||
|
|
||
| // Values that the caller explicitly specified are assumed to be changed -- they will be | ||
| // re-evaluated even if none of their children are changed. | ||
| private final InvalidationState invalidatorState = new DirtyingInvalidationState(); | ||
|
|
||
| final NestedSetVisitor.VisitedState emittedEventState; | ||
|
|
||
| AbstractIncrementalInMemoryMemoizingEvaluator( | ||
| ImmutableMap<SkyFunctionName, SkyFunction> skyFunctions, | ||
| Differencer differencer, | ||
| DirtyTrackingProgressReceiver dirtyTrackingProgressReceiver, | ||
| EventFilter eventFilter, | ||
| NestedSetVisitor.VisitedState emittedEventState, | ||
| GraphInconsistencyReceiver graphInconsistencyReceiver, | ||
| boolean keepEdges) { | ||
| this.skyFunctions = checkNotNull(skyFunctions); | ||
| this.differencer = checkNotNull(differencer); | ||
| this.progressReceiver = checkNotNull(dirtyTrackingProgressReceiver); | ||
| this.emittedEventState = checkNotNull(emittedEventState); | ||
| this.eventFilter = checkNotNull(eventFilter); | ||
| this.graphInconsistencyReceiver = checkNotNull(graphInconsistencyReceiver); | ||
| this.keepEdges = keepEdges; | ||
| } | ||
|
|
||
| void invalidate(Iterable<SkyKey> diff) { | ||
| Iterables.addAll(valuesToDirty, diff); | ||
| } | ||
|
|
||
| /** | ||
| * Removes entries in {@code valuesToInject} whose values are equal to the present values in the | ||
| * graph. | ||
| */ | ||
| void pruneInjectedValues(Map<SkyKey, SkyValue> valuesToInject) { | ||
| for (Iterator<Entry<SkyKey, SkyValue>> it = valuesToInject.entrySet().iterator(); | ||
| it.hasNext(); ) { | ||
| Map.Entry<SkyKey, SkyValue> entry = it.next(); | ||
| SkyKey key = entry.getKey(); | ||
| SkyValue newValue = entry.getValue(); | ||
| NodeEntry prevEntry = getInMemoryGraph().get(null, Reason.OTHER, key); | ||
| if (prevEntry != null && prevEntry.isDone()) { | ||
| if (keepEdges) { | ||
| try { | ||
| if (!prevEntry.hasAtLeastOneDep()) { | ||
| if (newValue.equals(prevEntry.getValue()) | ||
| && !valuesToDirty.contains(key) | ||
| && !valuesToDelete.contains(key)) { | ||
| it.remove(); | ||
| } | ||
| } else { | ||
| // Rare situation of an injected dep that depends on another node. Usually the dep is | ||
| // the error transience node. When working with external repositories, it can also be | ||
| // an external workspace file. Don't bother injecting it, just invalidate it. | ||
| // We'll wastefully evaluate the node freshly during evaluation, but this happens very | ||
| // rarely. | ||
| valuesToDirty.add(key); | ||
| it.remove(); | ||
| } | ||
| } catch (InterruptedException e) { | ||
| throw new IllegalStateException( | ||
| "InMemoryGraph does not throw: " + entry + ", " + prevEntry, e); | ||
| } | ||
| } else { | ||
| // No incrementality. Just delete the old value from the graph. The new value is about to | ||
| // be injected. | ||
| getInMemoryGraph().remove(key); | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /** Injects values in {@code valuesToInject} into the graph. */ | ||
| void injectValues(IntVersion version) { | ||
| if (valuesToInject.isEmpty()) { | ||
| return; | ||
| } | ||
| try { | ||
| ParallelEvaluator.injectValues(valuesToInject, version, getInMemoryGraph(), progressReceiver); | ||
| } catch (InterruptedException e) { | ||
| throw new IllegalStateException("InMemoryGraph doesn't throw interrupts", e); | ||
| } | ||
| // Start with a new map to avoid bloat since clear() does not downsize the map. | ||
| valuesToInject = new HashMap<>(); | ||
| } | ||
|
|
||
| void performInvalidation() throws InterruptedException { | ||
| EagerInvalidator.delete( | ||
| getInMemoryGraph(), valuesToDelete, progressReceiver, deleterState, keepEdges); | ||
| // Note that clearing the valuesToDelete would not do an internal resizing. Therefore, if any | ||
| // build has a large set of dirty values, subsequent operations (even clearing) will be slower. | ||
| // Instead, just start afresh with a new LinkedHashSet. | ||
| valuesToDelete = new LinkedHashSet<>(); | ||
|
|
||
| EagerInvalidator.invalidate( | ||
| getInMemoryGraph(), valuesToDirty, progressReceiver, invalidatorState); | ||
| // Ditto. | ||
| valuesToDirty = new LinkedHashSet<>(); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters