Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[7.1.0] Make it possible to avoid an extra stat() when obtaining a digest from the cache. #21353

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import com.google.devtools.build.lib.remote.options.RemoteOptions;
import com.google.devtools.build.lib.vfs.DigestHashFunction;
import com.google.devtools.build.lib.vfs.DigestUtils;
import com.google.devtools.build.lib.vfs.FileStatus;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
Expand Down Expand Up @@ -172,14 +173,13 @@ protected Digest computeDigest(
}
}

long fileSize = path.getFileSize();

// Try to obtain a digest from the filesystem.
// Obtain a digest from the filesystem.
FileStatus status = path.stat();
return builder
.setHash(
HashCode.fromBytes(DigestUtils.getDigestWithManualFallback(path, xattrProvider))
HashCode.fromBytes(DigestUtils.getDigestWithManualFallback(path, xattrProvider, status))
.toString())
.setSizeBytes(fileSize)
.setSizeBytes(status.getSize())
.build();
}

Expand Down
41 changes: 36 additions & 5 deletions src/main/java/com/google/devtools/build/lib/vfs/DigestUtils.java
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import com.google.common.base.Preconditions;
import com.google.common.primitives.Longs;
import java.io.IOException;
import javax.annotation.Nullable;

/**
* Utility class for getting digests of files.
Expand Down Expand Up @@ -159,27 +160,57 @@ public static CacheStats getCacheStats() {
* <p>If {@link Path#getFastDigest} has already been attempted and was not available, call {@link
* #manuallyComputeDigest} to skip an additional attempt to obtain the fast digest.
*
* @param path Path of the file.
* <p>Prefer calling {@link #manuallyComputeDigest(Path, FileStatus)} when a recently obtained
* {@link FileStatus} is available.
*
* @param path the file path
*/
public static byte[] getDigestWithManualFallback(Path path, XattrProvider xattrProvider)
throws IOException {
return getDigestWithManualFallback(path, xattrProvider, null);
}

/**
* Same as {@link #getDigestWithManualFallback(Path, XattrProvider)}, but providing the ability to
* reuse a recently obtained {@link FileStatus}.
*
* @param path the file path
* @param status a recently obtained file status, if available
*/
public static byte[] getDigestWithManualFallback(
Path path, XattrProvider xattrProvider, @Nullable FileStatus status) throws IOException {
byte[] digest = xattrProvider.getFastDigest(path);
return digest != null ? digest : manuallyComputeDigest(path);
return digest != null ? digest : manuallyComputeDigest(path, status);
}

/**
* Calculates the digest manually.
* Calculates a digest manually (i.e., assuming that a fast digest can't obtained).
*
* <p>Prefer calling {@link #manuallyComputeDigest(Path, FileStatus)} when a recently obtained
* {@link FileStatus} is available.
*
* @param path Path of the file.
* @param path the file path
*/
public static byte[] manuallyComputeDigest(Path path) throws IOException {
return manuallyComputeDigest(path, null);
}

/**
* Same as {@link #manuallyComputeDigest(Path)}, but providing the ability to reuse a recently
* obtained {@link FileStatus}.
*
* @param path the file path
* @param status a recently obtained file status, if available
*/
public static byte[] manuallyComputeDigest(Path path, @Nullable FileStatus status)
throws IOException {
byte[] digest;

// Attempt a cache lookup if the cache is enabled.
Cache<CacheKey, byte[]> cache = globalCache;
CacheKey key = null;
if (cache != null) {
key = new CacheKey(path, path.stat());
key = new CacheKey(path, status != null ? status : path.stat());
digest = cache.getIfPresent(key);
if (digest != null) {
return digest;
Expand Down
Loading