Skip to content

Commit

Permalink
Add a callback to enable more aggressive detection of `OutOfMemoryErr…
Browse files Browse the repository at this point in the history
…or`.

Add an optional predicate to categorize crashes as `OutOfMemoryError`s even if they don't have the error in the causal chain. This can be used to more aggressively categorize the crashes in cases when code paths lose the original `OutOfMemoryError`.

PiperOrigin-RevId: 504617124
Change-Id: I155c952dfd8afea863cb93ae417ec698a937b2b4
  • Loading branch information
alexjski authored and copybara-github committed Jan 25, 2023
1 parent d658908 commit c51d2bd
Show file tree
Hide file tree
Showing 3 changed files with 44 additions and 4 deletions.
3 changes: 3 additions & 0 deletions src/main/cpp/blaze_util_posix.cc
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,9 @@ void SignalHandler::Install(const string& product_name,
sigemptyset(&sigset);
sigprocmask(SIG_SETMASK, &sigset, nullptr);

// SIGWINCH is reserved for Bazel server internal use and cannot be passed to
// it. The JVM is not attached to a terminal, making a signal insufficient to
// react to window size change event anyway.
signal(SIGINT, handler);
signal(SIGTERM, handler);
signal(SIGPIPE, handler);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
import com.google.devtools.build.lib.server.FailureDetails.ThrowableOrBuilder;
import java.util.Set;
import java.util.function.BooleanSupplier;
import java.util.stream.Collectors;

/** Factory methods for producing {@link Crash}-type {@link FailureDetail} messages. */
Expand All @@ -45,8 +46,15 @@ public class CrashFailureDetails {
*/
private static final int MAX_STACK_TRACE_SIZE = 1000;

private static BooleanSupplier oomDetector = () -> false;

private CrashFailureDetails() {}

/** Registers a predicate to use for more aggressive {@link OutOfMemoryError} detection. */
public static void setOomDetector(BooleanSupplier oomDetector) {
CrashFailureDetails.oomDetector = oomDetector;
}

public static DetailedExitCode detailedExitCodeForThrowable(Throwable throwable) {
return DetailedExitCode.of(forThrowable(throwable));
}
Expand All @@ -59,7 +67,8 @@ public static FailureDetail forThrowable(Throwable throwable) {
Crash.Builder crashBuilder =
Crash.newBuilder()
.setCode(
(getRootCauseToleratingCycles(throwable) instanceof OutOfMemoryError)
(getRootCauseToleratingCycles(throwable) instanceof OutOfMemoryError
|| oomDetector.getAsBoolean())
? Crash.Code.CRASH_OOM
: Crash.Code.CRASH_UNKNOWN);
addCause(crashBuilder, throwable, Sets.newIdentityHashSet());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,18 +22,25 @@
import com.google.devtools.build.lib.server.FailureDetails.Crash.Code;
import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
import com.google.protobuf.ProtocolStringList;
import com.google.testing.junit.testparameterinjector.TestParameter;
import com.google.testing.junit.testparameterinjector.TestParameterInjector;
import java.util.List;
import org.junit.After;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;

/** Tests for {@link CrashFailureDetails}. */
@RunWith(JUnit4.class)
@RunWith(TestParameterInjector.class)
public final class CrashFailureDetailsTest {

private static final String TEST_EXCEPTION_NAME =
"com.google.devtools.build.lib.util.CrashFailureDetailsTest$TestException";

@After
public void restoreDefaultOomDetector() {
CrashFailureDetails.setOomDetector(() -> false);
}

@Test
public void nestedThrowables() {
// This test confirms that throwables' details are recorded: their messages, types, stack
Expand Down Expand Up @@ -147,13 +154,34 @@ public void detailedExitConstruction_wrappedOom() {
}

@Test
public void detailedExtitConstruction_otherCrash() {
public void detailedExitConstruction_otherCrash() {
assertThat(
CrashFailureDetails.detailedExitCodeForThrowable(new IllegalStateException())
.getExitCode())
.isEqualTo(ExitCode.BLAZE_INTERNAL_ERROR);
}

private enum ThrowableType {
OUT_OF_MEMORY_ERROR(new OutOfMemoryError()),
ILLEGAL_STATE_EXCEPTION(new IllegalStateException());

ThrowableType(Throwable throwable) {
this.throwable = throwable;
}

@SuppressWarnings("ImmutableEnumChecker")
final Throwable throwable;
}

@Test
public void detailExitConstruction_crashWithOomDetector_returnsOomCrash(
@TestParameter ThrowableType throwableType) {
CrashFailureDetails.setOomDetector(() -> true);
assertThat(
CrashFailureDetails.detailedExitCodeForThrowable(throwableType.throwable).getExitCode())
.isEqualTo(ExitCode.OOM_ERROR);
}

private static TestException functionForStackFrameTests_A(TestException cause) {
return new TestException("myMessage_A", cause);
}
Expand Down

0 comments on commit c51d2bd

Please sign in to comment.