Skip to content

Commit

Permalink
Support api graceful shutdown (#2684)
Browse files Browse the repository at this point in the history
  • Loading branch information
tcibinan committed Aug 19, 2022
1 parent 72e1b76 commit 46ba80c
Show file tree
Hide file tree
Showing 4 changed files with 105 additions and 0 deletions.
2 changes: 2 additions & 0 deletions api/profiles/dev/application.properties
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#tomcat configuration
server.port=9999
server.shutdown=${CP_API_SRV_SHUTDOWN_TYPE:graceful}
server.context-path=/pipeline
server.compression.enabled=true
server.compression.min-response-size=2048
Expand All @@ -10,6 +11,7 @@ spring.http.encoding.charset=UTF-8
spring.http.encoding.force=true
spring.http.encoding.force-response=true
spring.session.store-type=jdbc
spring.lifecycle.timeout-per-shutdown-phase=${CP_API_SRV_SHUTDOWN_TIMEOUT:30}s

#Security
api.security.anonymous.urls=${CP_API_SECURITY_ANONYMOUS_URLS:/restapi/route}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
package com.epam.pipeline.app;

import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.catalina.connector.Connector;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.coyote.ProtocolHandler;
import org.apache.tomcat.util.threads.ThreadPoolExecutor;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.embedded.EmbeddedServletContainerCustomizer;
import org.springframework.boot.context.embedded.tomcat.TomcatConnectorCustomizer;
import org.springframework.boot.context.embedded.tomcat.TomcatEmbeddedServletContainerFactory;
import org.springframework.context.ApplicationListener;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.event.ContextClosedEvent;

import java.util.Optional;
import java.util.concurrent.TimeUnit;

@Configuration
@ConditionalOnProperty(value = "server.shutdown", havingValue = "graceful")
public class GracefulShutdownConfiguration {

@Bean
public GracefulShutdownListener gracefulShutdownListener(
@Value("${spring.lifecycle.timeout-per-shutdown-phase:30s}")
final String gracefulShutdownTimeoutString) {
final long gracefulShutdownTimeoutInSeconds = toSeconds(gracefulShutdownTimeoutString);
return new GracefulShutdownListener(gracefulShutdownTimeoutInSeconds);
}

private int toSeconds(final String timeoutString) {
return NumberUtils.toInt(StringUtils.removeEnd(timeoutString, "s"));
}

@Bean
public EmbeddedServletContainerCustomizer embeddedServletContainerCustomizer(
final GracefulShutdownListener gracefulShutdownListener) {
return container -> Optional.of(container)
.filter(TomcatEmbeddedServletContainerFactory.class::isInstance)
.map(TomcatEmbeddedServletContainerFactory.class::cast)
.ifPresent(factory -> factory.addConnectorCustomizers(gracefulShutdownListener));
}

@Slf4j
@RequiredArgsConstructor
private static class GracefulShutdownListener implements TomcatConnectorCustomizer,
ApplicationListener<ContextClosedEvent> {

private final long timeoutSeconds;

private volatile Connector connector;

@Override
public void customize(final Connector connector) {
this.connector = connector;
}

@Override
public void onApplicationEvent(final ContextClosedEvent event) {
log.info("Gracefully shutting down application...");

log.debug("Resetting new connections...");
connector.pause();

log.debug("Terminating connections pool...");
getThreadPoolExecutor(connector).ifPresent(this::terminate);

log.debug("Proceeding with application shutting down...");
}

private Optional<ThreadPoolExecutor> getThreadPoolExecutor(final Connector connector) {
return Optional.of(connector)
.map(Connector::getProtocolHandler)
.map(ProtocolHandler::getExecutor)
.filter(ThreadPoolExecutor.class::isInstance)
.map(ThreadPoolExecutor.class::cast);
}

private void terminate(final ThreadPoolExecutor executor) {
try {
executor.shutdown();
log.debug("Waiting for connections pool to terminate...");
if (executor.awaitTermination(timeoutSeconds, TimeUnit.SECONDS)) {
log.debug("Connections pool has been terminated successfully.");
} else {
log.warn("Connections pool has not been terminated after %s seconds. " +
"Proceeding with forceful connections shutting down...");
}
} catch (InterruptedException e) {
log.warn("Connections pool termination has been interrupted. " +
"Proceeding with forceful connections shutting down...");
Thread.currentThread().interrupt();
}
}
}
}
1 change: 1 addition & 0 deletions deploy/contents/k8s/cp-api-srv/cp-api-srv-dpl.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ spec:
labels:
cloud-pipeline/cp-api-srv: "true"
spec:
terminationGracePeriodSeconds: 40
nodeSelector:
cloud-pipeline/cp-api-srv: "true"
tolerations:
Expand Down
2 changes: 2 additions & 0 deletions deploy/docker/cp-api-srv/config/application.properties
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# Connectivity config
server.port=8080
server.shutdown=${CP_API_SRV_SHUTDOWN_TYPE:graceful}
server.context-path=/pipeline
server.compression.enabled=true
server.compression.min-response-size=2048
Expand All @@ -15,6 +16,7 @@ api.security.impersonation.operations.root.url=${CP_API_SECURITY_IMPERSONATION_R
api.security.public.urls=${CP_API_SECURITY_PUBLIC_URLS:/launch.sh,/PipelineCLI.tar.gz,/pipe-common.tar.gz,/commit-run-scripts/**,/pipe,/fsbrowser.tar.gz,/pipe.zip,/pipe.tar.gz,/pipe-el6,/pipe-el6.tar.gz,/pipe-osx,/pipe-osx.tar.gz,/cloud-data-linux.tar.gz,/cloud-data-win64.zip,/fsautoscale.sh}
# supported values - jdbc, HASH_MAP
spring.session.store-type=${CP_API_SRV_SESSION_STORE_TYPE:jdbc}
spring.lifecycle.timeout-per-shutdown-phase=${CP_API_SRV_SHUTDOWN_TIMEOUT:30}s

# DB config
database.url=jdbc:postgresql://${PSG_HOST:cp-api-db.default.svc.cluster.local}:${PSG_PORT:5432}/${PSG_DB:pipeline}
Expand Down

0 comments on commit 46ba80c

Please sign in to comment.