Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added a simple transmission retry mechanism and capturing of exception stacktrace #501

Closed
wants to merge 1 commit into from
Closed

Added a simple transmission retry mechanism and capturing of exception stacktrace #501

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 21 additions & 11 deletions .../com/microsoft/applicationinsights/internal/channel/common/TransmissionNetworkOutput.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import org.apache.http.client.methods.HttpPost;
import org.apache.http.conn.ConnectionPoolTimeoutException;
import org.apache.http.entity.ByteArrayEntity;
import org.apache.commons.lang3.exception.ExceptionUtils;

import java.io.BufferedReader;
import java.io.IOException;
Expand Down Expand Up @@ -62,7 +63,8 @@ public final class TransmissionNetworkOutput implements TransmissionOutput {

private final static String DEFAULT_SERVER_URI = "https://dc.services.visualstudio.com/v2/track";
private final static int DEFAULT_BACKOFF_TIME_SECONDS = 300;

private final static int DEFAULT_RETRY_ATTEMPTS = 3;

// For future use: re-send a failed transmission back to the dispatcher
private TransmissionDispatcher transmissionDispatcher;

Expand Down Expand Up @@ -124,6 +126,8 @@ public synchronized void stop(long timeout, TimeUnit timeUnit) {
*/
@Override
public boolean send(Transmission transmission) {
int retry = 0;

while (!stopped) {
if (transmissionPolicyManager.getTransmissionPolicyState().getCurrentState() != TransmissionPolicy.UNBLOCKED) {
return false;
Expand Down Expand Up @@ -156,22 +160,22 @@ public boolean send(Transmission transmission) {
return true;
}
} catch (ConnectionPoolTimeoutException e) {
InternalLogger.INSTANCE.error("Failed to send, connection pool timeout exception");
InternalLogger.INSTANCE.error("Failed to send, connection pool timeout exception: %s", ExceptionUtils.getStackTrace(e));
shouldBackoff = true;
} catch (SocketException e) {
InternalLogger.INSTANCE.error("Failed to send, socket timeout exception");
InternalLogger.INSTANCE.error("Failed to send, socket timeout exception: %s", ExceptionUtils.getStackTrace(e));
shouldBackoff = true;
} catch (UnknownHostException e) {
InternalLogger.INSTANCE.error("Failed to send, wrong host address or cannot reach address due to network issues, exception: %s", e.getMessage());
InternalLogger.INSTANCE.error("Failed to send, wrong host address or cannot reach address due to network issues, exception: %s - %s", e.getMessage(), ExceptionUtils.getStackTrace(e));
shouldBackoff = true;
} catch (IOException ioe) {
InternalLogger.INSTANCE.error("Failed to send, exception: %s", ioe.getMessage());
InternalLogger.INSTANCE.error("Failed to send, exception: %s - %s", ioe.getMessage(), ExceptionUtils.getStackTrace(ioe));
shouldBackoff = true;
} catch (Exception e) {
InternalLogger.INSTANCE.error("Failed to send, unexpected exception: %s", e.getMessage());
InternalLogger.INSTANCE.error("Failed to send, unexpected exception: %s - %s", e.getMessage(), ExceptionUtils.getStackTrace(e));
shouldBackoff = true;
} catch (Throwable t) {
InternalLogger.INSTANCE.error("Failed to send, unexpected error: %s", t.getMessage());
InternalLogger.INSTANCE.error("Failed to send, unexpected error: %s - %s", t.getMessage(), ExceptionUtils.getStackTrace(t));
shouldBackoff = true;
}
finally {
Expand All @@ -181,10 +185,16 @@ public boolean send(Transmission transmission) {
httpClient.dispose(response);
// backoff before trying again
if (shouldBackoff) {
InternalLogger.INSTANCE.trace("Backing off for %s seconds", DEFAULT_BACKOFF_TIME_SECONDS);
transmissionPolicyManager.suspendInSeconds(TransmissionPolicy.BLOCKED_BUT_CAN_BE_PERSISTED, DEFAULT_BACKOFF_TIME_SECONDS);
if (retry > DEFAULT_RETRY_ATTEMPTS) {
InternalLogger.INSTANCE.trace("Backing off for %s seconds", DEFAULT_BACKOFF_TIME_SECONDS);
transmissionPolicyManager.suspendInSeconds(TransmissionPolicy.BLOCKED_BUT_CAN_BE_PERSISTED, DEFAULT_BACKOFF_TIME_SECONDS);
} else {
InternalLogger.INSTANCE.trace("Retrying transmission (retry attempt: %d)", (retry+1));
}
}
}

retry++;
}

return true;
Expand All @@ -209,7 +219,7 @@ private void suspendTransmissions(TransmissionPolicy suspensionPolicy, HttpRespo
long retryAfterAsSeconds = (date.getTime() - convertToDateToGmt(now).getTime())/1000;
transmissionPolicyManager.suspendInSeconds(suspensionPolicy, retryAfterAsSeconds);
} catch (Throwable e) {
InternalLogger.INSTANCE.logAlways(InternalLogger.LoggingLevel.ERROR, "Throttled but failed to block transmission, exception: %s", e.getMessage());
InternalLogger.INSTANCE.logAlways(InternalLogger.LoggingLevel.ERROR, "Throttled but failed to block transmission, exception: %s - %s", e.getMessage(), ExceptionUtils.getStackTrace(e));
}
}

Expand Down Expand Up @@ -302,7 +312,7 @@ private void logError(String baseErrorMessage, HttpEntity respEntity) {

InternalLogger.INSTANCE.error("Failed to send, %s : %s", baseErrorMessage, responseLine);
} catch (IOException e) {
InternalLogger.INSTANCE.error("Failed to send, %s, failed to log the error", baseErrorMessage);
InternalLogger.INSTANCE.error("Failed to send, %s, failed to log the error: %s", baseErrorMessage, ExceptionUtils.getStackTrace(e));
} finally {
if (inputStream != null) {
try {
Expand Down