Skip to content

Commit

Permalink
Report background download for BwoB (bazelbuild#17619)
Browse files Browse the repository at this point in the history
When building without the bytes, downloads can happen when Bazel need to fetch inputs for local actions, download outputs of toplevel targets, and etc. The downloads happen silently at the background.

This PR makes output downloads of toplevel targets visible to the UI.

For downloads during the build, the UI looks like:

```
[2 / 5] 2 actions, 1 running
    [Prepa] Executing genrule //:foo
    Executing genrule //:bar; 2s
    Fetching bazel-out/darwin-fastbuild/bin/baz; 50.8 MiB / 1.0 GiB; 4s
```

For downloads after build is completed, the UI looks like:

```
INFO: 2 processes: 1 remote cache hit, 1 internal.
INFO: Build completed successfully, 2 total actions
    Fetching bazel-out/darwin-fastbuild/bin/baz; 48.8 MiB / 1.0 GiB
```

Fixes bazelbuild#17417.

Closes bazelbuild#17604.

PiperOrigin-RevId: 512934756
Change-Id: I502489420ab9b84efb74ceabbe3dd4cb4a7a62e2
  • Loading branch information
coeuvre authored Feb 28, 2023
1 parent 938e348 commit 034a281
Show file tree
Hide file tree
Showing 7 changed files with 98 additions and 22 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,13 @@ public static SpawnProgressEvent create(String resourceId, String progress, bool
}

/** The id that uniquely determines the progress among all progress events for this spawn. */
abstract String progressId();
public abstract String progressId();

/** Human readable description of the progress. */
abstract String progress();
public abstract String progress();

/** Whether the progress reported about is finished already. */
abstract boolean finished();
public abstract boolean finished();

@Override
public void postTo(ExtendedEventHandler eventHandler, ActionExecutionMetadata action) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,11 @@ private boolean shouldDownloadFile(Path path, FileArtifactValue metadata) {
* @param tempPath the temporary path which the input should be written to.
*/
protected abstract ListenableFuture<Void> doDownloadFile(
Path tempPath, PathFragment execPath, FileArtifactValue metadata, Priority priority)
Reporter reporter,
Path tempPath,
PathFragment execPath,
FileArtifactValue metadata,
Priority priority)
throws IOException;

protected void prefetchVirtualActionInput(VirtualActionInput input) throws IOException {}
Expand Down Expand Up @@ -309,7 +313,8 @@ private Completable prefetchInputTree(
return toTransferResult(
toCompletable(
() ->
doDownloadFile(tempPath, treeFile.getExecPath(), metadata, priority),
doDownloadFile(
reporter, tempPath, treeFile.getExecPath(), metadata, priority),
directExecutor()));
});

Expand Down Expand Up @@ -455,7 +460,11 @@ private Completable downloadFileNoCheckRx(
toCompletable(
() ->
doDownloadFile(
tempPath, finalPath.relativeTo(execRoot), metadata, priority),
reporter,
tempPath,
finalPath.relativeTo(execRoot),
metadata,
priority),
directExecutor())
.doOnComplete(
() -> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
package com.google.devtools.build.lib.remote;

import static com.google.common.base.Preconditions.checkArgument;
import static com.google.devtools.build.lib.remote.common.ProgressStatusListener.NO_ACTION;

import build.bazel.remote.execution.v2.Digest;
import build.bazel.remote.execution.v2.RequestMetadata;
Expand All @@ -24,7 +25,10 @@
import com.google.devtools.build.lib.actions.FileArtifactValue;
import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
import com.google.devtools.build.lib.actions.cache.VirtualActionInput.EmptyActionInput;
import com.google.devtools.build.lib.events.ExtendedEventHandler.FetchProgress;
import com.google.devtools.build.lib.events.Reporter;
import com.google.devtools.build.lib.exec.SpawnProgressEvent;
import com.google.devtools.build.lib.remote.RemoteCache.DownloadProgressReporter;
import com.google.devtools.build.lib.remote.common.BulkTransferException;
import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
import com.google.devtools.build.lib.remote.util.DigestUtil;
Expand Down Expand Up @@ -90,15 +94,56 @@ protected boolean canDownloadFile(Path path, FileArtifactValue metadata) {

@Override
protected ListenableFuture<Void> doDownloadFile(
Path tempPath, PathFragment execPath, FileArtifactValue metadata, Priority priority)
Reporter reporter,
Path tempPath,
PathFragment execPath,
FileArtifactValue metadata,
Priority priority)
throws IOException {
checkArgument(metadata.isRemote(), "Cannot download file that is not a remote file.");
RequestMetadata requestMetadata =
TracingMetadataUtils.buildMetadata(buildRequestId, commandId, metadata.getActionId(), null);
RemoteActionExecutionContext context = RemoteActionExecutionContext.create(requestMetadata);

Digest digest = DigestUtil.buildDigest(metadata.getDigest(), metadata.getSize());
return remoteCache.downloadFile(context, tempPath, digest);

DownloadProgressReporter downloadProgressReporter;
if (priority == Priority.LOW) {
// Only report download progress for toplevel outputs
downloadProgressReporter =
new DownloadProgressReporter(
/* includeFile= */ false,
progress -> reporter.post(new DownloadProgress(progress)),
execPath.toString(),
metadata.getSize());
} else {
downloadProgressReporter = new DownloadProgressReporter(NO_ACTION, "", 0);
}

return remoteCache.downloadFile(context, tempPath, digest, downloadProgressReporter);
}

public static class DownloadProgress implements FetchProgress {
private final SpawnProgressEvent progress;

public DownloadProgress(SpawnProgressEvent progress) {
this.progress = progress;
}

@Override
public String getResourceIdentifier() {
return progress.progressId();
}

@Override
public String getProgress() {
return progress.progress();
}

@Override
public boolean isFinished() {
return progress.finished();
}
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -248,13 +248,20 @@ private ListenableFuture<Void> downloadBlob(
/** A reporter that reports download progresses. */
public static class DownloadProgressReporter {
private static final Pattern PATTERN = Pattern.compile("^bazel-out/[^/]+/[^/]+/");
private final boolean includeFile;
private final ProgressStatusListener listener;
private final String id;
private final String file;
private final String totalSize;
private final AtomicLong downloadedBytes = new AtomicLong(0);

public DownloadProgressReporter(ProgressStatusListener listener, String file, long totalSize) {
this(/* includeFile= */ true, listener, file, totalSize);
}

public DownloadProgressReporter(
boolean includeFile, ProgressStatusListener listener, String file, long totalSize) {
this.includeFile = includeFile;
this.listener = listener;
this.id = file;
this.totalSize = bytesCountToDisplayString(totalSize);
Expand All @@ -279,12 +286,21 @@ void finished() {
private void reportProgress(boolean includeBytes, boolean finished) {
String progress;
if (includeBytes) {
progress =
String.format(
"Downloading %s, %s / %s",
file, bytesCountToDisplayString(downloadedBytes.get()), totalSize);
if (includeFile) {
progress =
String.format(
"Downloading %s, %s / %s",
file, bytesCountToDisplayString(downloadedBytes.get()), totalSize);
} else {
progress =
String.format("%s / %s", bytesCountToDisplayString(downloadedBytes.get()), totalSize);
}
} else {
progress = String.format("Downloading %s", file);
if (includeFile) {
progress = String.format("Downloading %s", file);
} else {
progress = "";
}
}
listener.onProgressStatus(SpawnProgressEvent.create(id, progress, finished));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,14 @@
// limitations under the License.
package com.google.devtools.build.lib.remote.common;

import com.google.devtools.build.lib.exec.SpawnProgressEvent;
import com.google.devtools.build.lib.exec.SpawnRunner.ProgressStatus;

/** An interface that is used to receive {@link ProgressStatus} updates during spawn execution. */
@FunctionalInterface
public interface ProgressStatusListener {

void onProgressStatus(ProgressStatus progress);
void onProgressStatus(SpawnProgressEvent progress);

/** A {@link ProgressStatusListener} that does nothing. */
ProgressStatusListener NO_ACTION =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -993,7 +993,8 @@ synchronized boolean hasActivities() {
return !(buildCompleted()
&& bepOpenTransports.isEmpty()
&& activeActionUploads.get() == 0
&& activeActionDownloads.get() == 0);
&& activeActionDownloads.get() == 0
&& runningDownloads.isEmpty());
}

/**
Expand Down Expand Up @@ -1106,7 +1107,8 @@ private void reportOnOneDownload(
terminalWriter.append(url + postfix);
}

protected void reportOnDownloads(AnsiTerminalWriter terminalWriter) throws IOException {
protected void reportOnDownloads(PositionAwareAnsiTerminalWriter terminalWriter)
throws IOException {
int count = 0;
long nanoTime = clock.nanoTime();
int downloadCount = runningDownloads.size();
Expand All @@ -1116,7 +1118,10 @@ protected void reportOnDownloads(AnsiTerminalWriter terminalWriter) throws IOExc
break;
}
count++;
terminalWriter.newline().append(FETCH_PREFIX);
if (terminalWriter.getPosition() != 0) {
terminalWriter.newline();
}
terminalWriter.append(FETCH_PREFIX);
reportOnOneDownload(
url,
nanoTime,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ public void prefetchFiles_fileExists_doNotDownload()

wait(prefetcher.prefetchFiles(metadata.keySet(), metadataProvider));

verify(prefetcher, never()).doDownloadFile(any(), any(), any(), any());
verify(prefetcher, never()).doDownloadFile(any(), any(), any(), any(), any());
assertThat(prefetcher.downloadedFiles()).containsExactly(a.getPath());
assertThat(prefetcher.downloadsInProgress()).isEmpty();
}
Expand All @@ -190,7 +190,7 @@ public void prefetchFiles_fileExistsButContentMismatches_download()

wait(prefetcher.prefetchFiles(metadata.keySet(), metadataProvider));

verify(prefetcher).doDownloadFile(any(), eq(a.getExecPath()), any(), any());
verify(prefetcher).doDownloadFile(any(), any(), eq(a.getExecPath()), any(), any());
assertThat(prefetcher.downloadedFiles()).containsExactly(a.getPath());
assertThat(prefetcher.downloadsInProgress()).isEmpty();
assertThat(FileSystemUtils.readContent(a.getPath(), UTF_8)).isEqualTo("hello world remote");
Expand Down Expand Up @@ -531,8 +531,8 @@ protected static void mockDownload(
throws IOException {
doAnswer(
invocation -> {
Path path = invocation.getArgument(0);
FileArtifactValue metadata = invocation.getArgument(2);
Path path = invocation.getArgument(1);
FileArtifactValue metadata = invocation.getArgument(3);
byte[] content = cas.get(HashCode.fromBytes(metadata.getDigest()));
if (content == null) {
return Futures.immediateFailedFuture(new IOException("Not found"));
Expand All @@ -541,7 +541,7 @@ protected static void mockDownload(
return resultSupplier.get();
})
.when(prefetcher)
.doDownloadFile(any(), any(), any(), any());
.doDownloadFile(any(), any(), any(), any(), any());
}

private void assertReadableNonWritableAndExecutable(Path path) throws IOException {
Expand Down

0 comments on commit 034a281

Please sign in to comment.