Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

reduce number of getPasswordData calls #471

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

reduce number of getPasswordData calls #471

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8d37c9a
reduce number of getPasswordData calls
May 22, 2020
ca6e9b8
use full jitter
Jun 2, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
96 changes: 78 additions & 18 deletions src/main/java/hudson/plugins/ec2/win/EC2WindowsLauncher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import java.io.OutputStream;
import java.io.PrintStream;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;

import hudson.slaves.OfflineCause;
Expand All @@ -37,7 +38,19 @@
public class EC2WindowsLauncher extends EC2ComputerLauncher {
private static final String AGENT_JAR = "remoting.jar";

final long sleepBetweenAttempts = TimeUnit.SECONDS.toMillis(10);
private static final String DEFAULT_MAX_SLEEP = Long.toString(TimeUnit.MINUTES.toMillis(30));

private static long maxGetPassThreadSleep;

private static long maxWinRMThreadSleep;

private long sleepBetweenGetPassRange = TimeUnit.SECONDS.toMillis(1);

private long sleepBetweenGetPassAttempts = TimeUnit.SECONDS.toMillis(1);

private long sleepBetweenWinRMRange = TimeUnit.SECONDS.toMillis(1);

private long sleepBetweenWinRMAttempts = TimeUnit.SECONDS.toMillis(1);

@Override
protected void launchScript(EC2Computer computer, TaskListener listener) throws IOException,
Expand All @@ -54,7 +67,7 @@ protected void launchScript(EC2Computer computer, TaskListener listener) throws
}

final WinConnection connection = connectToWinRM(computer, node, template, logger);

try {
String initScript = node.initScript;
String tmpDir = (node.tmpDir != null && !node.tmpDir.equals("") ? WindowsUtil.quoteArgument(Util.ensureEndsWith(node.tmpDir,"\\"))
Expand Down Expand Up @@ -131,11 +144,23 @@ private WinConnection connectToWinRM(EC2Computer computer, EC2AbstractSlave node
if (timeout < minTimeout) {
timeout = minTimeout;
}
logger.println(String.format("Launch Timeout set to %ds", TimeUnit.MILLISECONDS.toSeconds(timeout)));
final long startTime = System.currentTimeMillis();

logger.println(node.getDisplayName() + " booted at " + node.getCreatedTime());
boolean alreadyBooted = (startTime - node.getCreatedTime()) > TimeUnit.MINUTES.toMillis(3);
WinConnection connection = null;
maxGetPassThreadSleep = Long.parseLong(System.getProperty(EC2WindowsLauncher.class.getName() + ".maxGetPassThreadSleep", DEFAULT_MAX_SLEEP));
maxWinRMThreadSleep = Long.parseLong(System.getProperty(EC2WindowsLauncher.class.getName() + ".maxWinRMThreadSleep", DEFAULT_MAX_SLEEP));

if (maxGetPassThreadSleep <= 0) {
maxGetPassThreadSleep = TimeUnit.SECONDS.toMillis(1);;
}

if (maxWinRMThreadSleep <= 0) {
maxWinRMThreadSleep = TimeUnit.SECONDS.toMillis(1);;
}

while (true) {
boolean allowSelfSignedCertificate = node.isAllowSelfSignedCertificate();

Expand All @@ -158,35 +183,46 @@ private WinConnection connectToWinRM(EC2Computer computer, EC2AbstractSlave node

if (!node.isSpecifyPassword()) {
GetPasswordDataResult result;
logger.print(String.format("GetPass sleep range: %ds. ", TimeUnit.MILLISECONDS.toSeconds(sleepBetweenGetPassRange)));
logger.print(String.format("GetPass max sleep: %ds. ", TimeUnit.MILLISECONDS.toSeconds(maxGetPassThreadSleep)));
try {
result = node.getCloud().connect().getPasswordData(new GetPasswordDataRequest(instance.getInstanceId()));
} catch (Exception e) {
logger.println("Unexpected Exception: " + e.toString());
Thread.sleep(sleepBetweenAttempts);
logger.println(String.format("Unexpected Exception: %s. Sleeping %ds.", e.toString(), TimeUnit.MILLISECONDS.toSeconds(sleepBetweenGetPassAttempts)));
Thread.sleep(sleepBetweenGetPassAttempts);
getPassBackoff();
continue;
}
String passwordData = result.getPasswordData();
if (passwordData == null || passwordData.isEmpty()) {
logger.println("Waiting for password to be available. Sleeping 10s.");
Thread.sleep(sleepBetweenAttempts);
logger.println(String.format("Waiting for password to be available. Sleeping %ds.", TimeUnit.MILLISECONDS.toSeconds(sleepBetweenGetPassAttempts)));
Thread.sleep(sleepBetweenGetPassAttempts);
getPassBackoff();
continue;
}
String password = node.getCloud().getPrivateKey().decryptWindowsPassword(passwordData);
if (!node.getRemoteAdmin().equals("Administrator")) {
logger.println("WARNING: For password retrieval remote admin must be Administrator, ignoring user provided value");
String username = System.getProperty(EC2WindowsLauncher.class.getName() + ".amiTemplateUsername", "david_webb6");

if (!node.getRemoteAdmin().equals(username)) {
logger.println("WARNING: For password retrieval remote admin must be " + username + ", ignoring user provided value");
}
logger.println("Connecting to " + "(" + host + ") with WinRM as Administrator");
connection = new WinConnection(host, "Administrator", password, allowSelfSignedCertificate);
logger.println("Connecting to " + "(" + host + ") with WinRM as " + username);
connection = new WinConnection(host, username, password, allowSelfSignedCertificate);

} else { //password Specified
logger.println("Connecting to " + "(" + host + ") with WinRM as " + node.getRemoteAdmin());
connection = new WinConnection(host, node.getRemoteAdmin(), node.getAdminPassword().getPlainText(), allowSelfSignedCertificate);
}
resetGetPassBackoff();
connection.setUseHTTPS(node.isUseHTTPS());
}

logger.print(String.format("WinRM sleep range: %ds. ", TimeUnit.MILLISECONDS.toSeconds(sleepBetweenWinRMRange)));
logger.print(String.format("WinRM max sleep: %ds. ", TimeUnit.MILLISECONDS.toSeconds(maxWinRMThreadSleep)));
if (!connection.pingFailingIfSSHHandShakeError()) {
logger.println("Waiting for WinRM to come up. Sleeping 10s.");
Thread.sleep(sleepBetweenAttempts);
logger.println(String.format("Waiting for WinRM to come up. Sleeping %ds.", TimeUnit.MILLISECONDS.toSeconds(sleepBetweenWinRMAttempts)));
Thread.sleep(sleepBetweenWinRMAttempts);
winRMBackoff();

continue;
}

Expand All @@ -197,13 +233,15 @@ private WinConnection connectToWinRM(EC2Computer computer, EC2AbstractSlave node
alreadyBooted = true;
logger.println("WinRM should now be ok on " + node.getDisplayName());
if (!connection.pingFailingIfSSHHandShakeError()) {
logger.println("WinRM not yet up. Sleeping 10s.");
Thread.sleep(sleepBetweenAttempts);
logger.println(String.format("WinRM not yet up. Sleeping %ds.", TimeUnit.MILLISECONDS.toSeconds(sleepBetweenWinRMAttempts)));
Thread.sleep(sleepBetweenWinRMAttempts);
winRMBackoff();
continue;
}
}

logger.println("Connected with WinRM.");
resetWinRMBackoff();
return connection; // successfully connected
} catch (IOException e) {
if (e instanceof SSLException) {
Expand All @@ -212,8 +250,9 @@ private WinConnection connectToWinRM(EC2Computer computer, EC2AbstractSlave node
// avoid waiting and trying again, this connection needs human intervention to change the certificate
throw new AmazonClientException("The SSL connection failed while negotiating SSL", e);
}
logger.println("Waiting for WinRM to come up. Sleeping 10s.");
Thread.sleep(sleepBetweenAttempts);
logger.println(String.format("Waiting for WinRM to come up. Sleeping %ds.", TimeUnit.MILLISECONDS.toSeconds(sleepBetweenWinRMAttempts)));
Thread.sleep(sleepBetweenWinRMAttempts);
winRMBackoff();
}
}
}
Expand All @@ -222,4 +261,25 @@ private WinConnection connectToWinRM(EC2Computer computer, EC2AbstractSlave node
public Descriptor<ComputerLauncher> getDescriptor() {
throw new UnsupportedOperationException();
}
}

private void getPassBackoff() {
sleepBetweenGetPassRange = Math.min(maxGetPassThreadSleep, sleepBetweenGetPassRange * 2);
sleepBetweenGetPassAttempts = ThreadLocalRandom.current().nextLong(0, sleepBetweenGetPassRange);
}

private void winRMBackoff() {
sleepBetweenWinRMRange = Math.min(maxWinRMThreadSleep, sleepBetweenWinRMRange * 2);
sleepBetweenWinRMAttempts = ThreadLocalRandom.current().nextLong(0, sleepBetweenWinRMRange);
}

private void resetGetPassBackoff() {
sleepBetweenGetPassRange = TimeUnit.SECONDS.toMillis(1);
sleepBetweenGetPassAttempts = TimeUnit.SECONDS.toMillis(1);
}

private void resetWinRMBackoff() {
sleepBetweenWinRMRange = TimeUnit.SECONDS.toMillis(1);
sleepBetweenWinRMAttempts = TimeUnit.SECONDS.toMillis(1);
}

}