Skip to content

Commit

Permalink
Streamline uploading of stdout and stderr
Browse files Browse the repository at this point in the history
This change removes extra upload logic for stdout/stderr. Besides
the streamlined code this also saves two round trips uploading an
action that produced both stdout and stderr.

Closes bazelbuild#9025.

PiperOrigin-RevId: 261288280
  • Loading branch information
buchgr authored and copybara-github committed Aug 2, 2019
1 parent 09ec893 commit a6b5b05
Show file tree
Hide file tree
Showing 2 changed files with 29 additions and 30 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -640,8 +640,10 @@ static class UploadManifest {
private final Path execRoot;
private final boolean allowSymlinks;
private final boolean uploadSymlinks;
private final Map<Digest, Path> digestToFile;
private final Map<Digest, Chunker> digestToChunkers;
private final Map<Digest, Path> digestToFile = new HashMap<>();
private final Map<Digest, Chunker> digestToChunkers = new HashMap<>();
private Digest stderrDigest;
private Digest stdoutDigest;

/**
* Create an UploadManifest from an ActionResult builder and an exec root. The ActionResult
Expand All @@ -658,9 +660,17 @@ public UploadManifest(
this.execRoot = execRoot;
this.uploadSymlinks = uploadSymlinks;
this.allowSymlinks = allowSymlinks;
}

this.digestToFile = new HashMap<>();
this.digestToChunkers = new HashMap<>();
public void setStdoutStderr(FileOutErr outErr) throws IOException {
if (outErr.getErrorPath().exists()) {
stderrDigest = digestUtil.compute(outErr.getErrorPath());
addFile(stderrDigest, outErr.getErrorPath());
}
if (outErr.getOutputPath().exists()) {
stdoutDigest = digestUtil.compute(outErr.getOutputPath());
addFile(stdoutDigest, outErr.getOutputPath());
}
}

/**
Expand Down Expand Up @@ -747,6 +757,16 @@ public Map<Digest, Chunker> getDigestToChunkers() {
return digestToChunkers;
}

@Nullable
public Digest getStdoutDigest() {
return stdoutDigest;
}

@Nullable
public Digest getStderrDigest() {
return stderrDigest;
}

private void addFileSymbolicLink(Path file, PathFragment target) throws IOException {
result
.addOutputFileSymlinksBuilder()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@
import com.google.common.base.Ascii;
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
Expand Down Expand Up @@ -419,6 +418,7 @@ void upload(
options.incompatibleRemoteSymlinks,
options.allowSymlinkUpload);
manifest.addFiles(files);
manifest.setStdoutStderr(outErr);
manifest.addAction(actionKey, action, command);

Map<HashCode, Chunker> filesToUpload = Maps.newHashMap();
Expand Down Expand Up @@ -449,34 +449,13 @@ void upload(
uploader.uploadBlobs(filesToUpload, /*forceUpload=*/true);
}

// TODO(olaola): inline small stdout/stderr here.
if (outErr.getErrorPath().exists()) {
Digest stderr = uploadFileContents(outErr.getErrorPath());
result.setStderrDigest(stderr);
if (manifest.getStderrDigest() != null) {
result.setStderrDigest(manifest.getStderrDigest());
}
if (outErr.getOutputPath().exists()) {
Digest stdout = uploadFileContents(outErr.getOutputPath());
result.setStdoutDigest(stdout);
if (manifest.getStdoutDigest() != null) {
result.setStdoutDigest(manifest.getStdoutDigest());
}
}

/**
* Put the file contents cache if it is not already in it. No-op if the file is already stored in
* cache. The given path must be a full absolute path.
*
* @return The key for fetching the file contents blob from cache.
*/
private Digest uploadFileContents(Path file) throws IOException, InterruptedException {
Digest digest = digestUtil.compute(file);
ImmutableSet<Digest> missing = getMissingDigests(ImmutableList.of(digest));
if (!missing.isEmpty()) {
uploader.uploadBlob(
HashCode.fromString(digest.getHash()),
Chunker.builder().setInput(digest.getSizeBytes(), file).build(),
/* forceUpload=*/ true);
}
return digest;
}

// Execution Cache API

Expand Down

0 comments on commit a6b5b05

Please sign in to comment.