Skip to content

Commit

Permalink
util: Status desc for outlier detection ejection (#11036)
Browse files Browse the repository at this point in the history
Including a Status description makes it easier to debug subchannel
closure issues if it's clear that a subchannel became unavailable because
of an outlier detection ejection.
  • Loading branch information
temawi committed Mar 22, 2024
1 parent bdb6230 commit 10cb4a3
Show file tree
Hide file tree
Showing 2 changed files with 35 additions and 9 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -377,8 +377,9 @@ void clearEndpointTracker() {

void eject() {
ejected = true;
subchannelStateListener.onSubchannelState(
ConnectivityStateInfo.forTransientFailure(Status.UNAVAILABLE));
subchannelStateListener.onSubchannelState(ConnectivityStateInfo.forTransientFailure(
Status.UNAVAILABLE.withDescription(
"The subchannel has been ejected by outlier detection")));
logger.log(ChannelLogLevel.INFO, "Subchannel ejected: {0}", this);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth.assertWithMessage;
import static io.grpc.ConnectivityState.READY;
import static io.grpc.ConnectivityState.TRANSIENT_FAILURE;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
Expand Down Expand Up @@ -50,6 +50,7 @@
import io.grpc.LoadBalancerProvider;
import io.grpc.Metadata;
import io.grpc.Status;
import io.grpc.Status.Code;
import io.grpc.SynchronizationContext;
import io.grpc.internal.FakeClock;
import io.grpc.internal.FakeClock.ScheduledTask;
Expand Down Expand Up @@ -1203,9 +1204,21 @@ public void successRateAndFailurePercentage_successRateOutlier_() { // with heal
// The one subchannel that was returning errors should be ejected.
assertEjectedSubchannels(ImmutableSet.of(ImmutableSet.copyOf(servers.get(0).getAddresses())));
if (hasHealthConsumer) {
verify(healthListeners.get(servers.get(0))).onSubchannelState(eq(
ConnectivityStateInfo.forTransientFailure(Status.UNAVAILABLE)
));
ArgumentCaptor<ConnectivityStateInfo> csiCaptor = ArgumentCaptor.forClass(
ConnectivityStateInfo.class);
verify(healthListeners.get(servers.get(0)), times(2)).onSubchannelState(csiCaptor.capture());
List<ConnectivityStateInfo> connectivityStateInfos = csiCaptor.getAllValues();

// The subchannel went through two state transitions...
assertThat(connectivityStateInfos).hasSize(2);
// ...it first went to the READY state...
assertThat(connectivityStateInfos.get(0).getState()).isEqualTo(READY);

// ...and then to TRANSIENT_FAILURE as outlier detection ejected it.
assertThat(connectivityStateInfos.get(1).getState()).isEqualTo(TRANSIENT_FAILURE);
assertThat(connectivityStateInfos.get(1).getStatus().getCode()).isEqualTo(Code.UNAVAILABLE);
assertThat(connectivityStateInfos.get(1).getStatus().getDescription()).isEqualTo(
"The subchannel has been ejected by outlier detection");
}
}

Expand Down Expand Up @@ -1264,9 +1277,21 @@ public void successRateAndFailurePercentage_errorPercentageOutlier_() { // with
// The one subchannel that was returning errors should be ejected.
assertEjectedSubchannels(ImmutableSet.of(ImmutableSet.copyOf(servers.get(0).getAddresses())));
if (hasHealthConsumer) {
verify(healthListeners.get(servers.get(0))).onSubchannelState(eq(
ConnectivityStateInfo.forTransientFailure(Status.UNAVAILABLE)
));
ArgumentCaptor<ConnectivityStateInfo> csiCaptor = ArgumentCaptor.forClass(
ConnectivityStateInfo.class);
verify(healthListeners.get(servers.get(0)), times(2)).onSubchannelState(csiCaptor.capture());
List<ConnectivityStateInfo> connectivityStateInfos = csiCaptor.getAllValues();

// The subchannel went through two state transitions...
assertThat(connectivityStateInfos).hasSize(2);
// ...it first went to the READY state...
assertThat(connectivityStateInfos.get(0).getState()).isEqualTo(READY);

// ...and then to TRANSIENT_FAILURE as outlier detection ejected it.
assertThat(connectivityStateInfos.get(1).getState()).isEqualTo(TRANSIENT_FAILURE);
assertThat(connectivityStateInfos.get(1).getStatus().getCode()).isEqualTo(Code.UNAVAILABLE);
assertThat(connectivityStateInfos.get(1).getStatus().getDescription()).isEqualTo(
"The subchannel has been ejected by outlier detection");
}
}

Expand Down

0 comments on commit 10cb4a3

Please sign in to comment.