From 729284f714cd4120ea9040eb5d0afaa7187b14bc Mon Sep 17 00:00:00 2001 From: tmanninger Date: Thu, 26 Sep 2024 14:33:31 +0200 Subject: [PATCH 01/10] graceful shutdown Signed-off-by: tmanninger --- .../common/settings/ClusterSettings.java | 1 + .../http/AbstractHttpServerTransport.java | 37 +++++++++++++++++++ .../opensearch/http/DefaultRestChannel.java | 7 +++- .../http/HttpTransportSettings.java | 8 ++++ 4 files changed, 52 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index ecdd23530c648..3088e322bdd39 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -370,6 +370,7 @@ public void apply(Settings value, Settings current, Settings previous) { HttpTransportSettings.SETTING_HTTP_BIND_HOST, HttpTransportSettings.SETTING_HTTP_PORT, HttpTransportSettings.SETTING_HTTP_PUBLISH_PORT, + HttpTransportSettings.SETTING_HTTP_GRACEFUL_SHUTDOWN, HttpTransportSettings.SETTING_PIPELINING_MAX_EVENTS, HttpTransportSettings.SETTING_HTTP_COMPRESSION, HttpTransportSettings.SETTING_HTTP_COMPRESSION_LEVEL, diff --git a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java index 991fbf12072be..6a18b382d2917 100644 --- a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java +++ b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java @@ -36,6 +36,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.ExceptionsHelper; +import org.opensearch.OpenSearchException; import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.network.CloseableChannel; import org.opensearch.common.network.NetworkAddress; @@ -76,6 +77,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -84,6 +86,7 @@ import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_PORT; import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_PUBLISH_HOST; import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_PUBLISH_PORT; +import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_GRACEFUL_SHUTDOWN; /** * Base HttpServer class @@ -240,6 +243,40 @@ protected void doStop() { } } + long gracefulShutdownMillis = SETTING_HTTP_GRACEFUL_SHUTDOWN.get(settings).getMillis(); + if (gracefulShutdownMillis > 0 && httpChannels.size() > 0) { + + logger.info("There are {} open client connections, try to close gracefully within {}ms.", httpChannels.size(), gracefulShutdownMillis); + + // Set all httpchannels to close gracefully + for (HttpChannel httpChannel : httpChannels) { + if (httpChannel instanceof DefaultRestChannel) { + ((DefaultRestChannel) httpChannel).gracefulCloseConnection(); + } + } + final long startTimeNS = System.currentTimeMillis(); + boolean closedAll = false; + while (System.currentTimeMillis() - startTimeNS < gracefulShutdownMillis) { + if (httpChannels.isEmpty()) { + closedAll = true; + break; + } + try { + TimeUnit.MILLISECONDS.sleep(30); + } catch (InterruptedException ie) { + throw new OpenSearchException("Interrupted waiting for completion of [{}]", ie); + } + } + + if (!closedAll) { + logger.info("Timeout reached, {} connections not closed gracefully.", httpChannels.size()); + } + else { + logger.info("Closed all connections gracefully."); + } + } + + // Close all channels that are not yet closed try { CloseableChannel.closeChannels(new ArrayList<>(httpChannels), true); } catch (Exception e) { diff --git a/server/src/main/java/org/opensearch/http/DefaultRestChannel.java b/server/src/main/java/org/opensearch/http/DefaultRestChannel.java index 497ec799ff937..7d974251fabff 100644 --- a/server/src/main/java/org/opensearch/http/DefaultRestChannel.java +++ b/server/src/main/java/org/opensearch/http/DefaultRestChannel.java @@ -84,6 +84,7 @@ class DefaultRestChannel extends AbstractRestChannel implements RestChannel { private final HttpChannel httpChannel; private final CorsHandler corsHandler; private final Map> SERVER_VERSION_HEADER = Map.of(SERVER_VERSION, List.of(SERVER_VERSION_VALUE)); + private boolean gracefulCloseConnection = false; @Nullable private final HttpTracer tracerLog; @@ -119,7 +120,7 @@ public void sendResponse(RestResponse restResponse) { Releasables.closeWhileHandlingException(httpRequest::release); final ArrayList toClose = new ArrayList<>(3); - if (HttpUtils.shouldCloseConnection(httpRequest)) { + if (this.gracefulCloseConnection || HttpUtils.shouldCloseConnection(httpRequest)) { toClose.add(() -> CloseableChannel.closeChannel(httpChannel)); } @@ -213,4 +214,8 @@ private void addCookies(HttpResponse response) { } } } + + public void gracefulCloseConnection() { + this.gracefulCloseConnection = true; + } } diff --git a/server/src/main/java/org/opensearch/http/HttpTransportSettings.java b/server/src/main/java/org/opensearch/http/HttpTransportSettings.java index 621ef36692178..ba99cf3941f6e 100644 --- a/server/src/main/java/org/opensearch/http/HttpTransportSettings.java +++ b/server/src/main/java/org/opensearch/http/HttpTransportSettings.java @@ -117,6 +117,14 @@ public final class HttpTransportSettings { Property.NodeScope ); public static final Setting SETTING_HTTP_PUBLISH_PORT = intSetting("http.publish_port", -1, -1, Property.NodeScope); + + public static final Setting SETTING_HTTP_GRACEFUL_SHUTDOWN = Setting.timeSetting( + "http.graceful_shutdown", + new TimeValue(0), + new TimeValue(0), + Property.NodeScope + ); + public static final Setting SETTING_HTTP_DETAILED_ERRORS_ENABLED = Setting.boolSetting( "http.detailed_errors.enabled", true, From 9621c158c31ca2cd065c6f5c51d37acbfd5da475 Mon Sep 17 00:00:00 2001 From: tmanninger Date: Thu, 26 Sep 2024 14:47:03 +0200 Subject: [PATCH 02/10] fix typo in var Signed-off-by: tmanninger --- .../java/org/opensearch/http/AbstractHttpServerTransport.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java index 6a18b382d2917..4e904ce9f564d 100644 --- a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java +++ b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java @@ -254,9 +254,9 @@ protected void doStop() { ((DefaultRestChannel) httpChannel).gracefulCloseConnection(); } } - final long startTimeNS = System.currentTimeMillis(); + final long startTimeMillis = System.currentTimeMillis(); boolean closedAll = false; - while (System.currentTimeMillis() - startTimeNS < gracefulShutdownMillis) { + while (System.currentTimeMillis() - startTimeMillis < gracefulShutdownMillis) { if (httpChannels.isEmpty()) { closedAll = true; break; From f7b4ddd7884c279b811fdd0741f8344481e67997 Mon Sep 17 00:00:00 2001 From: tmanninger Date: Thu, 26 Sep 2024 15:03:25 +0200 Subject: [PATCH 03/10] spotless style Signed-off-by: tmanninger --- .../opensearch/http/AbstractHttpServerTransport.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java index 4e904ce9f564d..87eccde48469c 100644 --- a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java +++ b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java @@ -82,11 +82,11 @@ import java.util.concurrent.atomic.AtomicReference; import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_BIND_HOST; +import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_GRACEFUL_SHUTDOWN; import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_MAX_CONTENT_LENGTH; import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_PORT; import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_PUBLISH_HOST; import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_PUBLISH_PORT; -import static org.opensearch.http.HttpTransportSettings.SETTING_HTTP_GRACEFUL_SHUTDOWN; /** * Base HttpServer class @@ -246,7 +246,11 @@ protected void doStop() { long gracefulShutdownMillis = SETTING_HTTP_GRACEFUL_SHUTDOWN.get(settings).getMillis(); if (gracefulShutdownMillis > 0 && httpChannels.size() > 0) { - logger.info("There are {} open client connections, try to close gracefully within {}ms.", httpChannels.size(), gracefulShutdownMillis); + logger.info( + "There are {} open client connections, try to close gracefully within {}ms.", + httpChannels.size(), + gracefulShutdownMillis + ); // Set all httpchannels to close gracefully for (HttpChannel httpChannel : httpChannels) { @@ -270,8 +274,7 @@ protected void doStop() { if (!closedAll) { logger.info("Timeout reached, {} connections not closed gracefully.", httpChannels.size()); - } - else { + } else { logger.info("Closed all connections gracefully."); } } From 732d091c181c1cbe57df1b90a1abef92545563b0 Mon Sep 17 00:00:00 2001 From: tmanninger Date: Fri, 27 Sep 2024 11:26:05 +0200 Subject: [PATCH 04/10] cleanup code Signed-off-by: tmanninger --- .../java/org/opensearch/http/AbstractHttpServerTransport.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java index 87eccde48469c..cc3f5eea5bfa9 100644 --- a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java +++ b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java @@ -259,10 +259,8 @@ protected void doStop() { } } final long startTimeMillis = System.currentTimeMillis(); - boolean closedAll = false; while (System.currentTimeMillis() - startTimeMillis < gracefulShutdownMillis) { if (httpChannels.isEmpty()) { - closedAll = true; break; } try { @@ -272,7 +270,7 @@ protected void doStop() { } } - if (!closedAll) { + if (!httpChannels.isEmpty()) { logger.info("Timeout reached, {} connections not closed gracefully.", httpChannels.size()); } else { logger.info("Closed all connections gracefully."); From c7cb462c7a28003d343684fb78be4e9da090650f Mon Sep 17 00:00:00 2001 From: tmanninger Date: Fri, 27 Sep 2024 15:36:29 +0200 Subject: [PATCH 05/10] remove graceful stuff Signed-off-by: tmanninger --- .../org/opensearch/http/AbstractHttpServerTransport.java | 2 +- .../main/java/org/opensearch/http/DefaultRestChannel.java | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java index cc3f5eea5bfa9..0d0b86c99c0ce 100644 --- a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java +++ b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java @@ -255,7 +255,7 @@ protected void doStop() { // Set all httpchannels to close gracefully for (HttpChannel httpChannel : httpChannels) { if (httpChannel instanceof DefaultRestChannel) { - ((DefaultRestChannel) httpChannel).gracefulCloseConnection(); + httpChannel.close(); } } final long startTimeMillis = System.currentTimeMillis(); diff --git a/server/src/main/java/org/opensearch/http/DefaultRestChannel.java b/server/src/main/java/org/opensearch/http/DefaultRestChannel.java index 7d974251fabff..497ec799ff937 100644 --- a/server/src/main/java/org/opensearch/http/DefaultRestChannel.java +++ b/server/src/main/java/org/opensearch/http/DefaultRestChannel.java @@ -84,7 +84,6 @@ class DefaultRestChannel extends AbstractRestChannel implements RestChannel { private final HttpChannel httpChannel; private final CorsHandler corsHandler; private final Map> SERVER_VERSION_HEADER = Map.of(SERVER_VERSION, List.of(SERVER_VERSION_VALUE)); - private boolean gracefulCloseConnection = false; @Nullable private final HttpTracer tracerLog; @@ -120,7 +119,7 @@ public void sendResponse(RestResponse restResponse) { Releasables.closeWhileHandlingException(httpRequest::release); final ArrayList toClose = new ArrayList<>(3); - if (this.gracefulCloseConnection || HttpUtils.shouldCloseConnection(httpRequest)) { + if (HttpUtils.shouldCloseConnection(httpRequest)) { toClose.add(() -> CloseableChannel.closeChannel(httpChannel)); } @@ -214,8 +213,4 @@ private void addCookies(HttpResponse response) { } } } - - public void gracefulCloseConnection() { - this.gracefulCloseConnection = true; - } } From 73c29d1f0e39a9b77faa60a60699b78d0bb0d1fb Mon Sep 17 00:00:00 2001 From: tmanninger Date: Fri, 27 Sep 2024 16:21:04 +0200 Subject: [PATCH 06/10] fix keep-alive connections Signed-off-by: tmanninger --- .../http/AbstractHttpServerTransport.java | 15 +++++++++------ .../org/opensearch/http/DefaultRestChannel.java | 11 ++++++++++- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java index 0d0b86c99c0ce..4f7974d0dcc9e 100644 --- a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java +++ b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java @@ -119,6 +119,8 @@ public abstract class AbstractHttpServerTransport extends AbstractLifecycleCompo private final HttpTracer httpTracer; private final Tracer tracer; + private boolean stopGracefully = false; + protected AbstractHttpServerTransport( Settings settings, NetworkService networkService, @@ -252,12 +254,9 @@ protected void doStop() { gracefulShutdownMillis ); - // Set all httpchannels to close gracefully - for (HttpChannel httpChannel : httpChannels) { - if (httpChannel instanceof DefaultRestChannel) { - httpChannel.close(); - } - } + // This will trigger this.handleIncomingRequest() to close all connections after sending response to client. + this.stopGracefully = true; + final long startTimeMillis = System.currentTimeMillis(); while (System.currentTimeMillis() - startTimeMillis < gracefulShutdownMillis) { if (httpChannels.isEmpty()) { @@ -538,6 +537,10 @@ private void handleIncomingRequest(final HttpRequest httpRequest, final HttpChan } } channel = innerChannel; + if (this.stopGracefully) { + // After sending response to client, close the connection, including keep-alive connections + ((DefaultRestChannel) channel).gracefulCloseConnection(); + } } dispatchRequest(restRequest, channel, badRequestCause); diff --git a/server/src/main/java/org/opensearch/http/DefaultRestChannel.java b/server/src/main/java/org/opensearch/http/DefaultRestChannel.java index 497ec799ff937..51c527061ae17 100644 --- a/server/src/main/java/org/opensearch/http/DefaultRestChannel.java +++ b/server/src/main/java/org/opensearch/http/DefaultRestChannel.java @@ -32,6 +32,8 @@ package org.opensearch.http; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.Build; import org.opensearch.common.Nullable; import org.opensearch.common.io.stream.BytesStreamOutput; @@ -84,6 +86,9 @@ class DefaultRestChannel extends AbstractRestChannel implements RestChannel { private final HttpChannel httpChannel; private final CorsHandler corsHandler; private final Map> SERVER_VERSION_HEADER = Map.of(SERVER_VERSION, List.of(SERVER_VERSION_VALUE)); + private boolean gracefulCloseConnection = false; + + private static final Logger logger = LogManager.getLogger(DefaultRestChannel.class); @Nullable private final HttpTracer tracerLog; @@ -119,7 +124,7 @@ public void sendResponse(RestResponse restResponse) { Releasables.closeWhileHandlingException(httpRequest::release); final ArrayList toClose = new ArrayList<>(3); - if (HttpUtils.shouldCloseConnection(httpRequest)) { + if (this.gracefulCloseConnection || HttpUtils.shouldCloseConnection(httpRequest)) { toClose.add(() -> CloseableChannel.closeChannel(httpChannel)); } @@ -213,4 +218,8 @@ private void addCookies(HttpResponse response) { } } } + + public void gracefulCloseConnection() { + this.gracefulCloseConnection = true; + } } From ab55dbfb79178f5867a979ea1f1518210cde04c3 Mon Sep 17 00:00:00 2001 From: tmanninger Date: Fri, 27 Sep 2024 17:33:18 +0200 Subject: [PATCH 07/10] fix keep-alive connections Signed-off-by: tmanninger --- .../opensearch/http/AbstractHttpServerTransport.java | 8 ++------ .../java/org/opensearch/http/DefaultRestChannel.java | 2 +- .../org/opensearch/http/HttpHandlingSettings.java | 12 ++++++++++++ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java index 4f7974d0dcc9e..6470a2859b896 100644 --- a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java +++ b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java @@ -254,8 +254,8 @@ protected void doStop() { gracefulShutdownMillis ); - // This will trigger this.handleIncomingRequest() to close all connections after sending response to client. - this.stopGracefully = true; + // Close connection after the client is getting a response + handlingSettings.setForceCloseConnection(); final long startTimeMillis = System.currentTimeMillis(); while (System.currentTimeMillis() - startTimeMillis < gracefulShutdownMillis) { @@ -537,10 +537,6 @@ private void handleIncomingRequest(final HttpRequest httpRequest, final HttpChan } } channel = innerChannel; - if (this.stopGracefully) { - // After sending response to client, close the connection, including keep-alive connections - ((DefaultRestChannel) channel).gracefulCloseConnection(); - } } dispatchRequest(restRequest, channel, badRequestCause); diff --git a/server/src/main/java/org/opensearch/http/DefaultRestChannel.java b/server/src/main/java/org/opensearch/http/DefaultRestChannel.java index 51c527061ae17..f782098a1c063 100644 --- a/server/src/main/java/org/opensearch/http/DefaultRestChannel.java +++ b/server/src/main/java/org/opensearch/http/DefaultRestChannel.java @@ -124,7 +124,7 @@ public void sendResponse(RestResponse restResponse) { Releasables.closeWhileHandlingException(httpRequest::release); final ArrayList toClose = new ArrayList<>(3); - if (this.gracefulCloseConnection || HttpUtils.shouldCloseConnection(httpRequest)) { + if (settings.forceCloseConnection() || HttpUtils.shouldCloseConnection(httpRequest)) { toClose.add(() -> CloseableChannel.closeChannel(httpChannel)); } diff --git a/server/src/main/java/org/opensearch/http/HttpHandlingSettings.java b/server/src/main/java/org/opensearch/http/HttpHandlingSettings.java index 37f754d591b22..14b1313659d8b 100644 --- a/server/src/main/java/org/opensearch/http/HttpHandlingSettings.java +++ b/server/src/main/java/org/opensearch/http/HttpHandlingSettings.java @@ -65,6 +65,9 @@ public class HttpHandlingSettings { private final long readTimeoutMillis; private boolean corsEnabled; + // Close the connection after response is sent to the client (even if keep-alive is set) + private boolean forceCloseConnection; + public HttpHandlingSettings( int maxContentLength, int maxChunkSize, @@ -89,6 +92,7 @@ public HttpHandlingSettings( this.pipeliningMaxEvents = pipeliningMaxEvents; this.readTimeoutMillis = readTimeoutMillis; this.corsEnabled = corsEnabled; + this.forceCloseConnection = false; } public static HttpHandlingSettings fromSettings(Settings settings) { @@ -150,4 +154,12 @@ public long getReadTimeoutMillis() { public boolean isCorsEnabled() { return corsEnabled; } + + public void setForceCloseConnection() { + forceCloseConnection = true; + } + + public boolean forceCloseConnection() { + return forceCloseConnection; + } } From d308955b245dd36d3939728c45378bb2d6afcc36 Mon Sep 17 00:00:00 2001 From: tmanninger Date: Fri, 27 Sep 2024 17:37:34 +0200 Subject: [PATCH 08/10] fix keep-alive connections Signed-off-by: tmanninger --- .../org/opensearch/http/AbstractHttpServerTransport.java | 5 ++--- .../java/org/opensearch/http/DefaultRestChannel.java | 9 --------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java index 6470a2859b896..e7e83b392326c 100644 --- a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java +++ b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java @@ -119,8 +119,6 @@ public abstract class AbstractHttpServerTransport extends AbstractLifecycleCompo private final HttpTracer httpTracer; private final Tracer tracer; - private boolean stopGracefully = false; - protected AbstractHttpServerTransport( Settings settings, NetworkService networkService, @@ -254,9 +252,10 @@ protected void doStop() { gracefulShutdownMillis ); - // Close connection after the client is getting a response + // Close connection after the client is getting a response. handlingSettings.setForceCloseConnection(); + // Wait until all connections are closed or the graceful timeout is reached. final long startTimeMillis = System.currentTimeMillis(); while (System.currentTimeMillis() - startTimeMillis < gracefulShutdownMillis) { if (httpChannels.isEmpty()) { diff --git a/server/src/main/java/org/opensearch/http/DefaultRestChannel.java b/server/src/main/java/org/opensearch/http/DefaultRestChannel.java index f782098a1c063..641fb44693297 100644 --- a/server/src/main/java/org/opensearch/http/DefaultRestChannel.java +++ b/server/src/main/java/org/opensearch/http/DefaultRestChannel.java @@ -32,8 +32,6 @@ package org.opensearch.http; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.opensearch.Build; import org.opensearch.common.Nullable; import org.opensearch.common.io.stream.BytesStreamOutput; @@ -86,9 +84,6 @@ class DefaultRestChannel extends AbstractRestChannel implements RestChannel { private final HttpChannel httpChannel; private final CorsHandler corsHandler; private final Map> SERVER_VERSION_HEADER = Map.of(SERVER_VERSION, List.of(SERVER_VERSION_VALUE)); - private boolean gracefulCloseConnection = false; - - private static final Logger logger = LogManager.getLogger(DefaultRestChannel.class); @Nullable private final HttpTracer tracerLog; @@ -218,8 +213,4 @@ private void addCookies(HttpResponse response) { } } } - - public void gracefulCloseConnection() { - this.gracefulCloseConnection = true; - } } From 65df2af51b6d26e1d0912c2eede9d6f12d30955c Mon Sep 17 00:00:00 2001 From: tmanninger Date: Fri, 27 Sep 2024 19:58:58 +0200 Subject: [PATCH 09/10] fix keep-alive connections Signed-off-by: tmanninger --- .../org/opensearch/http/AbstractHttpServerTransport.java | 1 + .../main/java/org/opensearch/http/DefaultRestChannel.java | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java index e7e83b392326c..d34649945f9a9 100644 --- a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java +++ b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java @@ -263,6 +263,7 @@ protected void doStop() { } try { TimeUnit.MILLISECONDS.sleep(30); + TimeUnit.MILLISECONDS.sleep(30); } catch (InterruptedException ie) { throw new OpenSearchException("Interrupted waiting for completion of [{}]", ie); } diff --git a/server/src/main/java/org/opensearch/http/DefaultRestChannel.java b/server/src/main/java/org/opensearch/http/DefaultRestChannel.java index 641fb44693297..aaf93e5e4570c 100644 --- a/server/src/main/java/org/opensearch/http/DefaultRestChannel.java +++ b/server/src/main/java/org/opensearch/http/DefaultRestChannel.java @@ -151,6 +151,14 @@ public void sendResponse(RestResponse restResponse) { setHeaderField(httpResponse, X_OPAQUE_ID, opaque); } + if (settings.forceCloseConnection()) { + // Server is in shutdown mode. Send client to close the connection. + // ------------- + // Only works with http1 + // How to check request httpversion is 2 AND send goaway signal? + setHeaderField(httpResponse, CONNECTION, CLOSE); + } + // Add all custom headers addCustomHeaders(httpResponse, restResponse.getHeaders()); addCustomHeaders(httpResponse, threadContext.getResponseHeaders()); From d2c8c6a7a8cd0b03fa75b6f26606f323886b75c2 Mon Sep 17 00:00:00 2001 From: tmanninger Date: Fri, 27 Sep 2024 21:15:37 +0200 Subject: [PATCH 10/10] fix keep-alive connections Signed-off-by: tmanninger --- .../java/org/opensearch/http/AbstractHttpServerTransport.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java index d34649945f9a9..e7e83b392326c 100644 --- a/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java +++ b/server/src/main/java/org/opensearch/http/AbstractHttpServerTransport.java @@ -263,7 +263,6 @@ protected void doStop() { } try { TimeUnit.MILLISECONDS.sleep(30); - TimeUnit.MILLISECONDS.sleep(30); } catch (InterruptedException ie) { throw new OpenSearchException("Interrupted waiting for completion of [{}]", ie); }