Skip to content

Commit

Permalink
Inject metadata when creating a filesystem symlink for a non-symlink …
Browse files Browse the repository at this point in the history 
…artifact.

A ctx.actions.symlink whose output is a declare_file or declare_directory
(as opposed to a declare_symlink) has "copy" semantics, i.e., the output
artifact is indistinguishable from the referent except for its name; the
symlink is just a filesystem-level optimization to avoid an actual copy,
and is transparently resolved when collecting the action output metadata.

When the symlink points to an artifact that was built remotely and without the
bytes, we currently must download it before executing the symlink action, so
that the output metadata can be constructed from the local filesystem. This
change short-circuits the filesystem traversal by injecting output metadata,
which is identical to the input plus a pointer to the original path. This is
used by the prefetcher to avoid downloading the same files multiple times when
they're symlinked more than once.

Fixes #15678.

Closes #16283.

PiperOrigin-RevId: 482479466
Change-Id: I3d10bef315b76a99a9e97077f07cc467e084984a
  • Loading branch information
@tjgq @copybara-github
tjgq authored and copybara-github committed Oct 20, 2022
1 parent 633348c commit 32b0f5a
Show file tree
Hide file tree
Showing 16 changed files with 1,310 additions and 167 deletions.
100 changes: 92 additions & 8 deletions src/main/java/com/google/devtools/build/lib/actions/FileArtifactValue.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import java.io.IOException;
import java.util.Arrays;
import java.util.Objects;
import java.util.Optional;
import javax.annotation.Nullable;

/**
Expand Down Expand Up @@ -172,6 +173,19 @@ protected boolean couldBeModifiedByMetadata(FileArtifactValue lastKnown) {
return true;
}

/**
* Optional materialization path.
*
* <p>If present, this artifact is a copy of another artifact. It is still tracked as a
* non-symlink by Bazel, but materialized in the local filesystem as a symlink to the original
* artifact, whose contents live at this location. This is used by {@link
* com.google.devtools.build.lib.remote.AbstractActionInputPrefetcher} to implement zero-cost
* copies of remotely stored artifacts.
*/
public Optional<PathFragment> getMaterializationExecPath() {
return Optional.empty();
}

/**
* Marker interface for singleton implementations of this class.
*
Expand Down Expand Up @@ -514,9 +528,7 @@ public long getModifiedTime() {
@Override
public String toString() {
return MoreObjects.toStringHelper(this)
.add(
"digest",
digest == null ? "(null)" : BaseEncoding.base16().lowerCase().encode(digest))
.add("digest", BaseEncoding.base16().lowerCase().encode(digest))
.add("size", size)
.add("proxy", proxy)
.toString();
Expand All @@ -535,10 +547,10 @@ protected boolean couldBeModifiedByMetadata(FileArtifactValue o) {

/** Metadata for remotely stored files. */
public static class RemoteFileArtifactValue extends FileArtifactValue {
private final byte[] digest;
private final long size;
private final int locationIndex;
private final String actionId;
protected final byte[] digest;
protected final long size;
protected final int locationIndex;
protected final String actionId;

private RemoteFileArtifactValue(byte[] digest, long size, int locationIndex, String actionId) {
this.digest = Preconditions.checkNotNull(digest, actionId);
Expand All @@ -556,6 +568,19 @@ public static RemoteFileArtifactValue create(byte[] digest, long size, int locat
return new RemoteFileArtifactValue(digest, size, locationIndex, /* actionId= */ "");
}

public static RemoteFileArtifactValue create(
byte[] digest,
long size,
int locationIndex,
String actionId,
@Nullable PathFragment materializationExecPath) {
if (materializationExecPath != null) {
return new RemoteFileArtifactValueWithMaterializationPath(
digest, size, locationIndex, actionId, materializationExecPath);
}
return new RemoteFileArtifactValue(digest, size, locationIndex, actionId);
}

@Override
public boolean equals(Object o) {
if (!(o instanceof RemoteFileArtifactValue)) {
Expand Down Expand Up @@ -602,7 +627,7 @@ public String getActionId() {
@Override
public long getModifiedTime() {
throw new UnsupportedOperationException(
"RemoteFileArifactValue doesn't support getModifiedTime");
"RemoteFileArtifactValue doesn't support getModifiedTime");
}

@Override
Expand All @@ -626,6 +651,65 @@ public String toString() {
.add("digest", bytesToString(digest))
.add("size", size)
.add("locationIndex", locationIndex)
.add("actionId", actionId)
.toString();
}
}

/**
* A remote artifact that should be materialized in the local filesystem as a symlink to another
* location.
*
* <p>See the documentation for {@link FileArtifactValue#getMaterializationExecPath}.
*/
public static final class RemoteFileArtifactValueWithMaterializationPath
extends RemoteFileArtifactValue {
private final PathFragment materializationExecPath;

private RemoteFileArtifactValueWithMaterializationPath(
byte[] digest,
long size,
int locationIndex,
String actionId,
PathFragment materializationExecPath) {
super(digest, size, locationIndex, actionId);
this.materializationExecPath = Preconditions.checkNotNull(materializationExecPath);
}

@Override
public Optional<PathFragment> getMaterializationExecPath() {
return Optional.ofNullable(materializationExecPath);
}

@Override
public boolean equals(Object o) {
if (!(o instanceof RemoteFileArtifactValueWithMaterializationPath)) {
return false;
}

RemoteFileArtifactValueWithMaterializationPath that =
(RemoteFileArtifactValueWithMaterializationPath) o;
return Arrays.equals(digest, that.digest)
&& size == that.size
&& locationIndex == that.locationIndex
&& Objects.equals(actionId, that.actionId)
&& Objects.equals(materializationExecPath, that.materializationExecPath);
}

@Override
public int hashCode() {
return Objects.hash(
Arrays.hashCode(digest), size, locationIndex, actionId, materializationExecPath);
}

@Override
public String toString() {
return MoreObjects.toStringHelper(this)
.add("digest", bytesToString(digest))
.add("size", size)
.add("locationIndex", locationIndex)
.add("actionId", actionId)
.add("materializationExecPath", materializationExecPath)
.toString();
}
}
Expand Down
17 changes: 13 additions & 4 deletions src/main/java/com/google/devtools/build/lib/actions/cache/ActionCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,9 +109,10 @@ final class Entry {
public abstract static class SerializableTreeArtifactValue {
public static SerializableTreeArtifactValue create(
ImmutableMap<String, RemoteFileArtifactValue> childValues,
Optional<RemoteFileArtifactValue> archivedFileValue) {
Optional<RemoteFileArtifactValue> archivedFileValue,
Optional<PathFragment> materializationExecPath) {
return new AutoValue_ActionCache_Entry_SerializableTreeArtifactValue(
childValues, archivedFileValue);
childValues, archivedFileValue, materializationExecPath);
}

/**
Expand All @@ -138,17 +139,25 @@ public static Optional<SerializableTreeArtifactValue> createSerializable(
.filter(ar -> ar.archivedFileValue().isRemote())
.map(ar -> (RemoteFileArtifactValue) ar.archivedFileValue());

if (childValues.isEmpty() && archivedFileValue.isEmpty()) {
Optional<PathFragment> materializationExecPath = treeMetadata.getMaterializationExecPath();

if (childValues.isEmpty()
&& archivedFileValue.isEmpty()
&& materializationExecPath.isEmpty()) {
return Optional.empty();
}

return Optional.of(SerializableTreeArtifactValue.create(childValues, archivedFileValue));
return Optional.of(
SerializableTreeArtifactValue.create(
childValues, archivedFileValue, materializationExecPath));
}

// A map from parentRelativePath to the file metadata
public abstract ImmutableMap<String, RemoteFileArtifactValue> childValues();

public abstract Optional<RemoteFileArtifactValue> archivedFileValue();

public abstract Optional<PathFragment> materializationExecPath();
}

public Entry(String key, Map<String, String> usedClientEnv, boolean discoversInputs) {
Expand Down
56 changes: 50 additions & 6 deletions src/main/java/com/google/devtools/build/lib/actions/cache/CompactPersistentActionCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import com.google.devtools.build.lib.util.VarInt;
import com.google.devtools.build.lib.vfs.DigestUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.SyscallCache;
import com.google.devtools.build.lib.vfs.UnixGlob;
import java.io.ByteArrayOutputStream;
Expand Down Expand Up @@ -75,7 +76,7 @@ public class CompactPersistentActionCache implements ActionCache {

private static final int NO_INPUT_DISCOVERY_COUNT = -1;

private static final int VERSION = 13;
private static final int VERSION = 14;

private static final class ActionMap extends PersistentMap<Integer, byte[]> {
private final Clock clock;
Expand Down Expand Up @@ -466,6 +467,14 @@ private static void encodeRemoteMetadata(
VarInt.putVarInt(value.getLocationIndex(), sink);

VarInt.putVarInt(indexer.getOrCreateIndex(value.getActionId()), sink);

Optional<PathFragment> materializationExecPath = value.getMaterializationExecPath();
if (materializationExecPath.isPresent()) {
VarInt.putVarInt(1, sink);
VarInt.putVarInt(indexer.getOrCreateIndex(materializationExecPath.get().toString()), sink);
} else {
VarInt.putVarInt(0, sink);
}
}

private static final int MAX_REMOTE_METADATA_SIZE =
Expand All @@ -484,7 +493,18 @@ private static RemoteFileArtifactValue decodeRemoteMetadata(

String actionId = getStringForIndex(indexer, VarInt.getVarInt(source));

return RemoteFileArtifactValue.create(digest, size, locationIndex, actionId);
PathFragment materializationExecPath = null;
int numMaterializationExecPath = VarInt.getVarInt(source);
if (numMaterializationExecPath > 0) {
if (numMaterializationExecPath != 1) {
throw new IOException("Invalid presence marker for materialization path");
}
materializationExecPath =
PathFragment.create(getStringForIndex(indexer, VarInt.getVarInt(source)));
}

return RemoteFileArtifactValue.create(
digest, size, locationIndex, actionId, materializationExecPath);
}

/** @return action data encoded as a byte[] array. */
Expand Down Expand Up @@ -513,9 +533,12 @@ private static byte[] encode(StringIndexer indexer, ActionCache.Entry entry) thr
+ MAX_REMOTE_METADATA_SIZE)
* value.childValues().size();

maxOutputTreesSize += VarInt.MAX_VARINT_SIZE; // value.archivedFileValue() optional
maxOutputTreesSize +=
value.archivedFileValue().map(ignored -> MAX_REMOTE_METADATA_SIZE).orElse(0);
(1 + VarInt.MAX_VARINT_SIZE) // value.archivedFileValue() optional
+ value.archivedFileValue().map(ignored -> MAX_REMOTE_METADATA_SIZE).orElse(0);
maxOutputTreesSize +=
(1 + VarInt.MAX_VARINT_SIZE) // value.materializationExecPath() optional
+ value.materializationExecPath().map(ignored -> MAX_REMOTE_METADATA_SIZE).orElse(0);
}

// Estimate the size of the buffer:
Expand Down Expand Up @@ -578,6 +601,15 @@ private static byte[] encode(StringIndexer indexer, ActionCache.Entry entry) thr
} else {
VarInt.putVarInt(0, sink);
}

Optional<PathFragment> materializationExecPath =
serializableTreeArtifactValue.materializationExecPath();
if (materializationExecPath.isPresent()) {
VarInt.putVarInt(1, sink);
VarInt.putVarInt(indexer.getOrCreateIndex(materializationExecPath.get().toString()), sink);
} else {
VarInt.putVarInt(0, sink);
}
}

return sink.toByteArray();
Expand Down Expand Up @@ -649,13 +681,25 @@ private static ActionCache.Entry decode(StringIndexer indexer, byte[] data) thro
int numArchivedFileValue = VarInt.getVarInt(source);
if (numArchivedFileValue > 0) {
if (numArchivedFileValue != 1) {
throw new IOException("Invalid number of archived artifacts");
throw new IOException("Invalid presence marker for archived representation");
}
archivedFileValue = Optional.of(decodeRemoteMetadata(indexer, source));
}

Optional<PathFragment> materializationExecPath = Optional.empty();
int numMaterializationExecPath = VarInt.getVarInt(source);
if (numMaterializationExecPath > 0) {
if (numMaterializationExecPath != 1) {
throw new IOException("Invalid presence marker for materialization path");
}
materializationExecPath =
Optional.of(
PathFragment.create(getStringForIndex(indexer, VarInt.getVarInt(source))));
}

SerializableTreeArtifactValue value =
SerializableTreeArtifactValue.create(childValues.buildOrThrow(), archivedFileValue);
SerializableTreeArtifactValue.create(
childValues.buildOrThrow(), archivedFileValue, materializationExecPath);
outputTrees.put(treeKey, value);
}

Expand Down
Loading

0 comments on commit 32b0f5a

Please sign in to comment.