From 8d37c9a1ec500d2d187a697c333f6dfcad86ce95 Mon Sep 17 00:00:00 2001 From: Alex Lin Date: Fri, 22 May 2020 13:02:24 -0700 Subject: [PATCH 1/2] reduce number of getPasswordData calls --- .../plugins/ec2/win/EC2WindowsLauncher.java | 89 +++++++++++++++---- 1 file changed, 71 insertions(+), 18 deletions(-) diff --git a/src/main/java/hudson/plugins/ec2/win/EC2WindowsLauncher.java b/src/main/java/hudson/plugins/ec2/win/EC2WindowsLauncher.java index a9fad33d3..60b84027b 100644 --- a/src/main/java/hudson/plugins/ec2/win/EC2WindowsLauncher.java +++ b/src/main/java/hudson/plugins/ec2/win/EC2WindowsLauncher.java @@ -19,6 +19,7 @@ import java.io.OutputStream; import java.io.PrintStream; import java.nio.charset.StandardCharsets; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; import hudson.slaves.OfflineCause; @@ -37,7 +38,19 @@ public class EC2WindowsLauncher extends EC2ComputerLauncher { private static final String AGENT_JAR = "remoting.jar"; - final long sleepBetweenAttempts = TimeUnit.SECONDS.toMillis(10); + private static final String DEFAULT_MAX_SLEEP = Long.toString(TimeUnit.MINUTES.toMillis(30)); + + private static long maxGetPassThreadSleep = Long.parseLong(System.getProperty(EC2WindowsLauncher.class.getName() + ".maxGetPassThreadSleep", DEFAULT_MAX_SLEEP)); + + private static long maxWinRMThreadSleep = Long.parseLong(System.getProperty(EC2WindowsLauncher.class.getName() + ".maxWinRMThreadSleep", DEFAULT_MAX_SLEEP)); + + private long sleepBetweenGetPassRange = TimeUnit.SECONDS.toMillis(1); + + private long sleepBetweenGetPassAttempts = TimeUnit.SECONDS.toMillis(1); + + private long sleepBetweenWinRMRange = TimeUnit.SECONDS.toMillis(1); + + private long sleepBetweenWinRMAttempts = TimeUnit.SECONDS.toMillis(1); @Override protected void launchScript(EC2Computer computer, TaskListener listener) throws IOException, @@ -54,7 +67,7 @@ protected void launchScript(EC2Computer computer, TaskListener listener) throws } final WinConnection connection = connectToWinRM(computer, node, template, logger); - + try { String initScript = node.initScript; String tmpDir = (node.tmpDir != null && !node.tmpDir.equals("") ? WindowsUtil.quoteArgument(Util.ensureEndsWith(node.tmpDir,"\\")) @@ -131,6 +144,7 @@ private WinConnection connectToWinRM(EC2Computer computer, EC2AbstractSlave node if (timeout < minTimeout) { timeout = minTimeout; } + logger.println(String.format("Launch Timeout set to %ds", TimeUnit.MILLISECONDS.toSeconds(timeout))); final long startTime = System.currentTimeMillis(); logger.println(node.getDisplayName() + " booted at " + node.getCreatedTime()); @@ -158,35 +172,46 @@ private WinConnection connectToWinRM(EC2Computer computer, EC2AbstractSlave node if (!node.isSpecifyPassword()) { GetPasswordDataResult result; + logger.print(String.format("GetPass sleep range: %ds. ", TimeUnit.MILLISECONDS.toSeconds(sleepBetweenGetPassRange))); + logger.print(String.format("GetPass max sleep: %ds. ", TimeUnit.MILLISECONDS.toSeconds(maxGetPassThreadSleep))); try { result = node.getCloud().connect().getPasswordData(new GetPasswordDataRequest(instance.getInstanceId())); } catch (Exception e) { - logger.println("Unexpected Exception: " + e.toString()); - Thread.sleep(sleepBetweenAttempts); + logger.println(String.format("Unexpected Exception: %s. Sleeping %ds.", e.toString(), TimeUnit.MILLISECONDS.toSeconds(sleepBetweenGetPassAttempts))); + Thread.sleep(sleepBetweenGetPassAttempts); + getPassBackoff(); continue; } String passwordData = result.getPasswordData(); if (passwordData == null || passwordData.isEmpty()) { - logger.println("Waiting for password to be available. Sleeping 10s."); - Thread.sleep(sleepBetweenAttempts); + logger.println(String.format("Waiting for password to be available. Sleeping %ds.", TimeUnit.MILLISECONDS.toSeconds(sleepBetweenGetPassAttempts))); + Thread.sleep(sleepBetweenGetPassAttempts); + getPassBackoff(); continue; } String password = node.getCloud().getPrivateKey().decryptWindowsPassword(passwordData); - if (!node.getRemoteAdmin().equals("Administrator")) { - logger.println("WARNING: For password retrieval remote admin must be Administrator, ignoring user provided value"); + String username = System.getProperty(EC2WindowsLauncher.class.getName() + ".amiTemplateUsername", "david_webb6"); + + if (!node.getRemoteAdmin().equals(username)) { + logger.println("WARNING: For password retrieval remote admin must be " + username + ", ignoring user provided value"); } - logger.println("Connecting to " + "(" + host + ") with WinRM as Administrator"); - connection = new WinConnection(host, "Administrator", password, allowSelfSignedCertificate); + logger.println("Connecting to " + "(" + host + ") with WinRM as " + username); + connection = new WinConnection(host, username, password, allowSelfSignedCertificate); + } else { //password Specified logger.println("Connecting to " + "(" + host + ") with WinRM as " + node.getRemoteAdmin()); connection = new WinConnection(host, node.getRemoteAdmin(), node.getAdminPassword().getPlainText(), allowSelfSignedCertificate); } + resetGetPassBackoff(); connection.setUseHTTPS(node.isUseHTTPS()); } - + logger.print(String.format("WinRM sleep range: %ds. ", TimeUnit.MILLISECONDS.toSeconds(sleepBetweenWinRMRange))); + logger.print(String.format("WinRM max sleep: %ds. ", TimeUnit.MILLISECONDS.toSeconds(maxWinRMThreadSleep))); if (!connection.pingFailingIfSSHHandShakeError()) { - logger.println("Waiting for WinRM to come up. Sleeping 10s."); - Thread.sleep(sleepBetweenAttempts); + logger.println(String.format("Waiting for WinRM to come up. Sleeping %ds.", TimeUnit.MILLISECONDS.toSeconds(sleepBetweenWinRMAttempts))); + Thread.sleep(sleepBetweenWinRMAttempts); + winRMBackoff(); + continue; } @@ -197,13 +222,15 @@ private WinConnection connectToWinRM(EC2Computer computer, EC2AbstractSlave node alreadyBooted = true; logger.println("WinRM should now be ok on " + node.getDisplayName()); if (!connection.pingFailingIfSSHHandShakeError()) { - logger.println("WinRM not yet up. Sleeping 10s."); - Thread.sleep(sleepBetweenAttempts); + logger.println(String.format("WinRM not yet up. Sleeping %ds.", TimeUnit.MILLISECONDS.toSeconds(sleepBetweenWinRMAttempts))); + Thread.sleep(sleepBetweenWinRMAttempts); + winRMBackoff(); continue; } } logger.println("Connected with WinRM."); + resetWinRMBackoff(); return connection; // successfully connected } catch (IOException e) { if (e instanceof SSLException) { @@ -212,8 +239,9 @@ private WinConnection connectToWinRM(EC2Computer computer, EC2AbstractSlave node // avoid waiting and trying again, this connection needs human intervention to change the certificate throw new AmazonClientException("The SSL connection failed while negotiating SSL", e); } - logger.println("Waiting for WinRM to come up. Sleeping 10s."); - Thread.sleep(sleepBetweenAttempts); + logger.println(String.format("Waiting for WinRM to come up. Sleeping %ds.", TimeUnit.MILLISECONDS.toSeconds(sleepBetweenWinRMAttempts))); + Thread.sleep(sleepBetweenWinRMAttempts); + winRMBackoff(); } } } @@ -222,4 +250,29 @@ private WinConnection connectToWinRM(EC2Computer computer, EC2AbstractSlave node public Descriptor getDescriptor() { throw new UnsupportedOperationException(); } -} + + private void getPassBackoff() { + long previousSleepBetweenGetPassRange = sleepBetweenGetPassRange; + sleepBetweenGetPassRange = sleepBetweenGetPassRange * 2; + sleepBetweenGetPassAttempts = ThreadLocalRandom.current().nextLong(previousSleepBetweenGetPassRange + 1, + Math.min(maxGetPassThreadSleep, sleepBetweenGetPassRange)); + } + + private void winRMBackoff() { + long previousSleepBetweenWinRMRange = sleepBetweenWinRMRange; + sleepBetweenWinRMRange = sleepBetweenWinRMRange * 2; + sleepBetweenWinRMAttempts = ThreadLocalRandom.current().nextLong(previousSleepBetweenWinRMRange + 1, + Math.min(maxWinRMThreadSleep, sleepBetweenWinRMRange)); + } + + private void resetGetPassBackoff() { + sleepBetweenGetPassRange = TimeUnit.SECONDS.toMillis(1); + sleepBetweenGetPassAttempts = TimeUnit.SECONDS.toMillis(1); + } + + private void resetWinRMBackoff() { + sleepBetweenWinRMRange = TimeUnit.SECONDS.toMillis(1); + sleepBetweenWinRMAttempts = TimeUnit.SECONDS.toMillis(1); + } + +} \ No newline at end of file From ca6e9b8ece4f450f24f6845785073cd298f49614 Mon Sep 17 00:00:00 2001 From: Alex Lin Date: Tue, 2 Jun 2020 13:05:22 -0700 Subject: [PATCH 2/2] use full jitter --- .../plugins/ec2/win/EC2WindowsLauncher.java | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/main/java/hudson/plugins/ec2/win/EC2WindowsLauncher.java b/src/main/java/hudson/plugins/ec2/win/EC2WindowsLauncher.java index 60b84027b..a55e27773 100644 --- a/src/main/java/hudson/plugins/ec2/win/EC2WindowsLauncher.java +++ b/src/main/java/hudson/plugins/ec2/win/EC2WindowsLauncher.java @@ -40,9 +40,9 @@ public class EC2WindowsLauncher extends EC2ComputerLauncher { private static final String DEFAULT_MAX_SLEEP = Long.toString(TimeUnit.MINUTES.toMillis(30)); - private static long maxGetPassThreadSleep = Long.parseLong(System.getProperty(EC2WindowsLauncher.class.getName() + ".maxGetPassThreadSleep", DEFAULT_MAX_SLEEP)); + private static long maxGetPassThreadSleep; - private static long maxWinRMThreadSleep = Long.parseLong(System.getProperty(EC2WindowsLauncher.class.getName() + ".maxWinRMThreadSleep", DEFAULT_MAX_SLEEP)); + private static long maxWinRMThreadSleep; private long sleepBetweenGetPassRange = TimeUnit.SECONDS.toMillis(1); @@ -150,6 +150,17 @@ private WinConnection connectToWinRM(EC2Computer computer, EC2AbstractSlave node logger.println(node.getDisplayName() + " booted at " + node.getCreatedTime()); boolean alreadyBooted = (startTime - node.getCreatedTime()) > TimeUnit.MINUTES.toMillis(3); WinConnection connection = null; + maxGetPassThreadSleep = Long.parseLong(System.getProperty(EC2WindowsLauncher.class.getName() + ".maxGetPassThreadSleep", DEFAULT_MAX_SLEEP)); + maxWinRMThreadSleep = Long.parseLong(System.getProperty(EC2WindowsLauncher.class.getName() + ".maxWinRMThreadSleep", DEFAULT_MAX_SLEEP)); + + if (maxGetPassThreadSleep <= 0) { + maxGetPassThreadSleep = TimeUnit.SECONDS.toMillis(1);; + } + + if (maxWinRMThreadSleep <= 0) { + maxWinRMThreadSleep = TimeUnit.SECONDS.toMillis(1);; + } + while (true) { boolean allowSelfSignedCertificate = node.isAllowSelfSignedCertificate(); @@ -252,17 +263,13 @@ public Descriptor getDescriptor() { } private void getPassBackoff() { - long previousSleepBetweenGetPassRange = sleepBetweenGetPassRange; - sleepBetweenGetPassRange = sleepBetweenGetPassRange * 2; - sleepBetweenGetPassAttempts = ThreadLocalRandom.current().nextLong(previousSleepBetweenGetPassRange + 1, - Math.min(maxGetPassThreadSleep, sleepBetweenGetPassRange)); + sleepBetweenGetPassRange = Math.min(maxGetPassThreadSleep, sleepBetweenGetPassRange * 2); + sleepBetweenGetPassAttempts = ThreadLocalRandom.current().nextLong(0, sleepBetweenGetPassRange); } private void winRMBackoff() { - long previousSleepBetweenWinRMRange = sleepBetweenWinRMRange; - sleepBetweenWinRMRange = sleepBetweenWinRMRange * 2; - sleepBetweenWinRMAttempts = ThreadLocalRandom.current().nextLong(previousSleepBetweenWinRMRange + 1, - Math.min(maxWinRMThreadSleep, sleepBetweenWinRMRange)); + sleepBetweenWinRMRange = Math.min(maxWinRMThreadSleep, sleepBetweenWinRMRange * 2); + sleepBetweenWinRMAttempts = ThreadLocalRandom.current().nextLong(0, sleepBetweenWinRMRange); } private void resetGetPassBackoff() {