Skip to content

Commit

Permalink
Add option to shutdown idle agent just before billing hour completes
Browse files Browse the repository at this point in the history
Implements #30
  • Loading branch information
rkosegi committed Jun 10, 2022
1 parent 2d88663 commit 88af647
Show file tree
Hide file tree
Showing 15 changed files with 307 additions and 19 deletions.
8 changes: 7 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,9 @@ These additional attributes can be specified, but are not required:

- `Number of Executors`

- `Keep around minutes` - Time that agent will be kept online after it become idle.
- `Shutdown policy` - Defines how agent will be shut down after it becomes idle
- `Removes server after it's idle for period of time` - you can define how many minutes will idle agent kept around
- `Removes idle server just before current hour of billing cycle completes`

### Scripted configuration using Groovy

Expand Down Expand Up @@ -171,6 +173,7 @@ jenkins:
connector:
root:
sshCredentialsId: 'ssh-private-key'
shutdownPolicy: "hour-wrap"
- name: ubuntu2-cx31
serverType: cx31
remoteFs: /var/lib/jenkins
Expand All @@ -182,6 +185,9 @@ jenkins:
connector:
root:
sshCredentialsId: 'ssh-private-key'
shutdownPolicy:
idle:
idleMinutes: 10
credentials:
system:
domainCredentials:
Expand Down
29 changes: 29 additions & 0 deletions src/main/java/cloud/dnation/jenkins/plugins/hetzner/Helper.java
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
*/
package cloud.dnation.jenkins.plugins.hetzner;

import cloud.dnation.jenkins.plugins.hetzner.client.ServerDetail;
import com.cloudbees.jenkins.plugins.sshcredentials.impl.BasicSSHUserPrivateKey;
import com.cloudbees.plugins.credentials.CredentialsMatchers;
import com.cloudbees.plugins.credentials.CredentialsProvider;
Expand All @@ -39,6 +40,8 @@
import java.nio.charset.StandardCharsets;
import java.security.KeyPair;
import java.security.interfaces.RSAPublicKey;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Base64;
import java.util.Collections;
import java.util.List;
Expand All @@ -48,6 +51,8 @@
import java.util.logging.LogRecord;
import java.util.logging.SimpleFormatter;

import static cloud.dnation.jenkins.plugins.hetzner.HetznerConstants.SHUTDOWN_TIME_BUFFER;

@UtilityClass
public class Helper {
private static final String SSH_RSA = "ssh-rsa";
Expand Down Expand Up @@ -154,4 +159,28 @@ public void error(String message, Throwable cause) {
stream.println(FORMATTER.format(rec));
}
}

/**
* Check if idle server can be shut down.
* <p>
* According to <a href="https://docs.hetzner.com/cloud/billing/faq#how-do-you-bill-your-servers">Hetzner billing policy</a>,
* you are billed for every hour of existence of server, so it makes sense to keep server running as long as next hour did
* not start yet.
*
* @param createdStr RFC3339-formatted instant when server was created. See ServerDetail#getCreated().
* @param currentMinute current minute. Kept as argument to allow unit-testing.
* @return <code>true</code> if server should be shut down, <code>false</code> otherwise.
* Note: we keep small time buffer for corner cases like clock skew or Jenkins's queue manager overload, which could
* lead to unnecessary 1-hour over-billing.
*/
public static boolean canShutdownServer(@Nonnull String createdStr, int currentMinute) {
int billingMinute = LocalDateTime.from(DateTimeFormatter.ISO_DATE_TIME.parse(createdStr)).getMinute();
if (billingMinute < SHUTDOWN_TIME_BUFFER) {
billingMinute += 60;
}
if (currentMinute < SHUTDOWN_TIME_BUFFER) {
currentMinute += 60;
}
return(currentMinute < billingMinute) && (currentMinute >= (billingMinute - SHUTDOWN_TIME_BUFFER));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@
*/
package cloud.dnation.jenkins.plugins.hetzner;

import cloud.dnation.jenkins.plugins.hetzner.shutdown.IdlePeriodPolicy;
import com.google.common.collect.ImmutableSet;
import hudson.slaves.CloudRetentionStrategy;
import lombok.experimental.UtilityClass;

import java.util.Set;
Expand Down Expand Up @@ -82,7 +82,13 @@ public class HetznerConstants {
.build();

/**
* Default retention strategy to use.
* Default shutdown policy to use.
*/
static final CloudRetentionStrategy DEFAULT_RETENTION_STRATEGY = new CloudRetentionStrategy(10);
static final IdlePeriodPolicy DEFAULT_SHUTDOWN_POLICY = new IdlePeriodPolicy(10);

/*
* Arbitrary value in minutes which gives us some time to shut down server before usage hour wraps.
*/
public static final int SHUTDOWN_TIME_BUFFER = 5;

}
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
package cloud.dnation.jenkins.plugins.hetzner;

import cloud.dnation.jenkins.plugins.hetzner.launcher.HetznerServerComputerLauncher;
import com.google.common.base.Strings;
import edu.umd.cs.findbugs.annotations.NonNull;
import hudson.Extension;
import hudson.model.Descriptor;
Expand Down Expand Up @@ -60,11 +59,7 @@ public HetznerServerAgent(@NonNull ProvisioningActivity.Id provisioningId,
setLabelString(template.getLabelStr());
setNumExecutors(template.getNumExecutors());
setMode(template.getMode() == null ? Mode.EXCLUSIVE : template.getMode());
if (Strings.isNullOrEmpty(template.getKeepAroundMinutes())) {
setRetentionStrategy(HetznerConstants.DEFAULT_RETENTION_STRATEGY);
} else {
setRetentionStrategy(new CloudRetentionStrategy(Integer.parseInt(template.getKeepAroundMinutes())));
}
setRetentionStrategy(template.getShutdownPolicy().getRetentionStrategy());
readResolve();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
package cloud.dnation.jenkins.plugins.hetzner;

import cloud.dnation.jenkins.plugins.hetzner.launcher.AbstractHetznerSshConnector;
import cloud.dnation.jenkins.plugins.hetzner.shutdown.AbstractShutdownPolicy;
import cloud.dnation.jenkins.plugins.hetzner.shutdown.IdlePeriodPolicy;
import com.google.common.base.Strings;
import hudson.Extension;
import hudson.Util;
Expand Down Expand Up @@ -83,7 +85,7 @@ public class HetznerServerTemplate extends AbstractDescribableImpl<HetznerServer
@Getter
private String jvmOpts;

@Setter(onMethod = @__({@DataBoundSetter}))
@Deprecated
@Getter
private String keepAroundMinutes;

Expand All @@ -103,6 +105,10 @@ public class HetznerServerTemplate extends AbstractDescribableImpl<HetznerServer
@Setter(onMethod = @__({@DataBoundSetter}))
private Mode mode = Mode.EXCLUSIVE;

@Getter
@Setter(onMethod = @__({@DataBoundSetter}))
private AbstractShutdownPolicy shutdownPolicy;

@DataBoundConstructor
public HetznerServerTemplate(String name, String labelStr, String image,
String location, String serverType) {
Expand All @@ -126,9 +132,22 @@ protected Object readResolve() {
if (bootDeadline == 0) {
setBootDeadline(HetznerConstants.DEFAULT_BOOT_DEADLINE);
}
if (shutdownPolicy == null) {
shutdownPolicy = HetznerConstants.DEFAULT_SHUTDOWN_POLICY;
}
return this;
}

@Deprecated
@DataBoundSetter
public void setKeepAroundMinutes(String keepAroundMinutes) {
if (!Strings.isNullOrEmpty(keepAroundMinutes)) {
log.info("{} : Migrating keepAroundMinutes to shutdown policy {}", name, keepAroundMinutes);
shutdownPolicy = new IdlePeriodPolicy(Integer.parseInt(keepAroundMinutes));
}
this.keepAroundMinutes = null;
}

/**
* Create new {@link HetznerServerAgent}.
*
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
/*
* Copyright 2022 https://dnation.cloud
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cloud.dnation.jenkins.plugins.hetzner.shutdown;

import hudson.model.AbstractDescribableImpl;
import hudson.slaves.AbstractCloudComputer;
import hudson.slaves.RetentionStrategy;
import lombok.Getter;

@SuppressWarnings("rawtypes")
public abstract class AbstractShutdownPolicy extends AbstractDescribableImpl<AbstractShutdownPolicy> {
@Getter
protected final transient RetentionStrategy<AbstractCloudComputer> retentionStrategy;

protected AbstractShutdownPolicy(RetentionStrategy<AbstractCloudComputer> strategy) {
this.retentionStrategy = strategy;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
/*
* Copyright 2022 https://dnation.cloud
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cloud.dnation.jenkins.plugins.hetzner.shutdown;

import cloud.dnation.jenkins.plugins.hetzner.Helper;
import cloud.dnation.jenkins.plugins.hetzner.HetznerServerAgent;
import cloud.dnation.jenkins.plugins.hetzner.Messages;
import edu.umd.cs.findbugs.annotations.NonNull;
import hudson.Extension;
import hudson.model.Descriptor;
import hudson.slaves.AbstractCloudComputer;
import hudson.slaves.RetentionStrategy;
import lombok.extern.slf4j.Slf4j;
import net.jcip.annotations.GuardedBy;
import org.jenkinsci.Symbol;
import org.kohsuke.stapler.DataBoundConstructor;

import java.io.IOException;
import java.time.LocalDateTime;

@Slf4j
public class BeforeHourWrapsPolicy extends AbstractShutdownPolicy {
@SuppressWarnings("rawtypes")
private static final RetentionStrategy<AbstractCloudComputer> STRATEGY_SINGLETON = new RetentionStrategyImpl();

@DataBoundConstructor
public BeforeHourWrapsPolicy() {
super(STRATEGY_SINGLETON);
}


@SuppressWarnings("rawtypes")
private static class RetentionStrategyImpl extends RetentionStrategy<AbstractCloudComputer> {
@Override
public void start(AbstractCloudComputer c) {
c.connect(false);
}

@Override
@GuardedBy("hudson.model.Queue.lock")
public long check(final AbstractCloudComputer c) {
final HetznerServerAgent agent = (HetznerServerAgent) c.getNode();
if (c.isIdle() && agent != null && agent.getServerInstance() != null) {
if (Helper.canShutdownServer(agent.getServerInstance().getServerDetail().getCreated(),
LocalDateTime.now().getMinute())) {
log.info("Disconnecting {}", c.getName());
try {
agent.terminate();
} catch (InterruptedException | IOException e) {
log.warn("Failed to terminate {}", c.getName(), e);
}
}
}
return 1;
}
}

@Extension
@Symbol("hour-wrap")
public static final class DescriptorImpl extends Descriptor<AbstractShutdownPolicy> {
@NonNull
@Override
public String getDisplayName() {
return Messages.policy_shutdown_beforeHourWrap();
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/*
* Copyright 2022 https://dnation.cloud
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cloud.dnation.jenkins.plugins.hetzner.shutdown;

import cloud.dnation.jenkins.plugins.hetzner.Messages;
import edu.umd.cs.findbugs.annotations.NonNull;
import hudson.Extension;
import hudson.model.Descriptor;
import hudson.slaves.CloudRetentionStrategy;
import lombok.Getter;
import org.jenkinsci.Symbol;
import org.kohsuke.stapler.DataBoundConstructor;

public class IdlePeriodPolicy extends AbstractShutdownPolicy {
@Getter
private final int idleMinutes;

@DataBoundConstructor
public IdlePeriodPolicy(int idleMinutes) {
super(new CloudRetentionStrategy(idleMinutes));
this.idleMinutes = idleMinutes;
}

@Extension
@Symbol("idle")
public static final class DescriptorImpl extends Descriptor<AbstractShutdownPolicy> {
@NonNull
@Override
public String getDisplayName() {
return Messages.policy_shutdown_idle();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,6 @@
<f:entry title="${%Number of Executors}" field="numExecutors">
<f:textbox default="1"/>
</f:entry>
<f:entry title="${%Keep around minutes}" field="keepAroundMinutes">
<f:textbox/>
</f:entry>
<f:dropdownDescriptorSelector field="shutdownPolicy" title="Shutdown policy"/>
</f:advanced>
</j:jelly>
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
<!--
Copyright 2021 https://dnation.cloud
Copyright 2022 https://dnation.cloud
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
Expand All @@ -14,6 +14,5 @@
limitations under the License.
-->
<div>
When empty, server will be kept alive for 10 minutes after being used.
This field allows to override this retention period.
Defines how idle server is shutdown.
</div>
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,6 @@ plugin.description=Plugin for launching build Agents as Hetzner compute resource
plugin.displayName=Hetzner
serverTemplate.displayName=Hetzner server template
connector.SshAsRoot=Connect via SSH as root, but launch agent as user configured in credentials
connector.SshAsNonRoot=Connect via SSH as user configured in credentials
connector.SshAsNonRoot=Connect via SSH as user configured in credentials
policy.shutdown.idle=Removes server after it's idle for period of time
policy.shutdown.beforeHourWrap=Removes idle server just before current hour of billing cycle completes
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
<!--
Copyright 2022 https://dnation.cloud
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<div>
Server will be kept alive until current hour of billing cycle completes.
Make sure that Jenkins controller's clock are configured correctly as skew may lead to 1 hour over-billing.
</div>
Loading

0 comments on commit 88af647

Please sign in to comment.