Skip to content

Commit

Permalink
Introduce a SpawnLogContext interface.
Browse files Browse the repository at this point in the history 
For now, the sole implementation is ExpandedSpawnLogContext, which subsumes the old SpawnLogContext class, as well as some of the logic around log formats currently in SpawnLogModule.

In a followup, a separate CompactSpawnLogContext implementation will be introduced. This will let us experiment with a new log format while minimizing the chance of accidentally breaking the existing formats.

PiperOrigin-RevId: 588444807
Change-Id: I75bc2a577ec45202baf95c950a257eaf420cbeb0
  • Loading branch information
@tjgq @copybara-github
tjgq authored and copybara-github committed Dec 6, 2023
1 parent 962db94 commit 020b85e
Show file tree
Hide file tree
Showing 6 changed files with 609 additions and 532 deletions.
4 changes: 0 additions & 4 deletions src/main/java/com/google/devtools/build/lib/bazel/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,6 @@ java_library(
],
deps = [
"//src/main/java/com/google/devtools/build/lib:runtime",
"//src/main/java/com/google/devtools/build/lib/bazel/execlog:stable_sort",
"//src/main/java/com/google/devtools/build/lib/events",
"//src/main/java/com/google/devtools/build/lib/exec:execution_options",
"//src/main/java/com/google/devtools/build/lib/exec:executor_builder",
Expand All @@ -138,11 +137,8 @@ java_library(
"//src/main/java/com/google/devtools/build/lib/remote/options",
"//src/main/java/com/google/devtools/build/lib/util:abrupt_exit_exception",
"//src/main/java/com/google/devtools/build/lib/util:detailed_exit_code",
"//src/main/java/com/google/devtools/build/lib/util/io",
"//src/main/java/com/google/devtools/build/lib/util/io:io-proto",
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/main/protobuf:failure_details_java_proto",
"//src/main/protobuf:spawn_java_proto",
"//third_party:guava",
"//third_party:jsr305",
],
Expand Down
107 changes: 18 additions & 89 deletions src/main/java/com/google/devtools/build/lib/bazel/SpawnLogModule.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,13 @@

import static com.google.common.base.Preconditions.checkNotNull;

import com.google.devtools.build.lib.bazel.execlog.StableSort;
import com.google.devtools.build.lib.buildtool.BuildRequest;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.exec.ExecutionOptions;
import com.google.devtools.build.lib.exec.ExecutorBuilder;
import com.google.devtools.build.lib.exec.ExpandedSpawnLogContext;
import com.google.devtools.build.lib.exec.ExpandedSpawnLogContext.Encoding;
import com.google.devtools.build.lib.exec.ModuleActionContextRegistry;
import com.google.devtools.build.lib.exec.Protos.SpawnExec;
import com.google.devtools.build.lib.exec.SpawnLogContext;
import com.google.devtools.build.lib.remote.options.RemoteOptions;
import com.google.devtools.build.lib.runtime.BlazeModule;
Expand All @@ -32,41 +32,16 @@
import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
import com.google.devtools.build.lib.util.AbruptExitException;
import com.google.devtools.build.lib.util.DetailedExitCode;
import com.google.devtools.build.lib.util.io.AsynchronousMessageOutputStream;
import com.google.devtools.build.lib.util.io.MessageOutputStream;
import com.google.devtools.build.lib.util.io.MessageOutputStreamWrapper.BinaryOutputStreamWrapper;
import com.google.devtools.build.lib.util.io.MessageOutputStreamWrapper.JsonOutputStreamWrapper;
import com.google.devtools.build.lib.vfs.Path;
import java.io.IOException;
import java.io.InputStream;
import javax.annotation.Nullable;

/** Module providing on-demand spawn logging. */
public final class SpawnLogModule extends BlazeModule {
@Nullable private SpawnLogContext spawnLogContext;

/** Output path for the raw output stream. */
@Nullable private Path rawOutputPath;

/** Output stream to write directly into during execution. */
@Nullable private MessageOutputStream<SpawnExec> rawOutputStream;

/**
* Output stream to convert the raw output into after the execution is done.
*
* <p>We open the stream at the beginning of the command so that any errors (e.g., unwritable
* location) are surfaced before execution begins.
*/
@Nullable private MessageOutputStream<SpawnExec> convertedOutputStream;

private CommandEnvironment env;

private void clear() {
spawnLogContext = null;
rawOutputPath = null;
rawOutputStream = null;
convertedOutputStream = null;
env = null;
}

private void initOutputs(CommandEnvironment env) throws IOException {
Expand Down Expand Up @@ -99,45 +74,30 @@ private void initOutputs(CommandEnvironment env) throws IOException {
return;
}

this.env = env;

Path workingDirectory = env.getWorkingDirectory();
Path outputBase = env.getOutputBase();

// Set up the raw output stream.
// This stream performs the writes in a separate thread to avoid blocking execution.
// If the unsorted binary format was requested, use the respective output path to avoid a
// pointless conversion at the end. Otherwise, use a temporary path.
if (executionOptions.executionLogBinaryFile != null && !executionOptions.executionLogSort) {
rawOutputPath = workingDirectory.getRelative(executionOptions.executionLogBinaryFile);
} else {
rawOutputPath = outputBase.getRelative("execution.log");
}
rawOutputStream = new AsynchronousMessageOutputStream<>(rawOutputPath);

// Set up the binary output stream, if distinct from the raw output stream.
if (executionOptions.executionLogBinaryFile != null && executionOptions.executionLogSort) {
convertedOutputStream =
new BinaryOutputStreamWrapper<>(
workingDirectory
.getRelative(executionOptions.executionLogBinaryFile)
.getOutputStream());
Path outputPath = null;
Encoding encoding = null;
if (executionOptions.executionLogBinaryFile != null) {
encoding = Encoding.BINARY;
outputPath = workingDirectory.getRelative(executionOptions.executionLogBinaryFile);
} else if (executionOptions.executionLogJsonFile != null) {
encoding = Encoding.JSON;
outputPath = workingDirectory.getRelative(executionOptions.executionLogJsonFile);
}

// Set up the text output stream.
if (executionOptions.executionLogJsonFile != null) {
convertedOutputStream =
new JsonOutputStreamWrapper<>(
workingDirectory
.getRelative(executionOptions.executionLogJsonFile)
.getOutputStream());
}
// Use a well-known temporary path to avoid accumulation of potentially large files in /tmp
// due to abnormally terminated invocations (e.g., when running out of memory).
Path tempPath = outputBase.getRelative("execution.log");

spawnLogContext =
new SpawnLogContext(
new ExpandedSpawnLogContext(
checkNotNull(outputPath),
tempPath,
checkNotNull(encoding),
/* sorted= */ executionOptions.executionLogSort,
env.getExecRoot().asFragment(),
rawOutputStream,
env.getOptions().getOptions(ExecutionOptions.class),
env.getOptions().getOptions(RemoteOptions.class),
env.getRuntime().getFileSystem().getDigestFunction(),
env.getXattrProvider());
Expand Down Expand Up @@ -180,44 +140,13 @@ public void afterCommand() throws AbruptExitException {
return;
}

checkNotNull(rawOutputPath);

boolean done = false;
try {
spawnLogContext.close();
if (convertedOutputStream != null) {
InputStream in = rawOutputPath.getInputStream();
if (spawnLogContext.shouldSort()) {
StableSort.stableSort(in, convertedOutputStream);
} else {
while (in.available() > 0) {
SpawnExec ex = SpawnExec.parseDelimitedFrom(in);
convertedOutputStream.write(ex);
}
}
convertedOutputStream.close();
}
done = true;
} catch (IOException e) {
String message = e.getMessage() == null ? "Error writing execution log" : e.getMessage();
throw new AbruptExitException(
createDetailedExitCode(message, Code.EXECUTION_LOG_WRITE_FAILURE), e);
} finally {
if (convertedOutputStream != null) {
if (!done) {
env.getReporter()
.handle(
Event.warn(
"Execution log might not have been populated. Raw execution log is at "
+ rawOutputPath));
} else {
try {
rawOutputPath.delete();
} catch (IOException e) {
// Intentionally ignored.
}
}
}
clear();
}
}
Expand Down
8 changes: 6 additions & 2 deletions src/main/java/com/google/devtools/build/lib/exec/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -260,15 +260,19 @@ java_library(

java_library(
name = "spawn_log_context",
srcs = ["SpawnLogContext.java"],
srcs = [
"ExpandedSpawnLogContext.java",
"SpawnLogContext.java",
],
deps = [
":execution_options",
"//src/main/java/com/google/devtools/build/lib/actions",
"//src/main/java/com/google/devtools/build/lib/actions:artifacts",
"//src/main/java/com/google/devtools/build/lib/actions:file_metadata",
"//src/main/java/com/google/devtools/build/lib/analysis/platform:platform_utils",
"//src/main/java/com/google/devtools/build/lib/bazel/execlog:stable_sort",
"//src/main/java/com/google/devtools/build/lib/profiler",
"//src/main/java/com/google/devtools/build/lib/remote/options",
"//src/main/java/com/google/devtools/build/lib/util/io",
"//src/main/java/com/google/devtools/build/lib/util/io:io-proto",
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
Expand Down
Loading

0 comments on commit 020b85e

Please sign in to comment.