Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Limit count of HTTP channels with tracked stats #77303

Merged
merged 2 commits into from
Sep 8, 2021
Merged

Limit count of HTTP channels with tracked stats #77303

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7ae3904
Limit count of HTTP channels with tracked stats
DaveCTurner Sep 6, 2021
86ebfc1
Test fixes
DaveCTurner Sep 6, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions docs/reference/modules/http.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,3 +190,14 @@ Defaults to `network.tcp.receive_buffer_size`.
`http.client_stats.enabled`::
(<<dynamic-cluster-setting,Dynamic>>)
Enable or disable collection of HTTP client stats. Defaults to `true`.

`http.client_stats.closed_channels.max_count`::
(<<static-cluster-setting,Static>>)
When `http.client_stats.enabled` is `true`, sets the maximum number of closed
HTTP channels for which {es} reports statistics. Defaults to `10000`.

`http.client_stats.closed_channels.max_age`::
(<<static-cluster-setting,Static>>)
When `http.client_stats.enabled` is `true`, sets the maximum length of time
after closing a HTTP channel that {es} will report that channel's statistics.
Defaults to `5m`.
13 changes: 11 additions & 2 deletions modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpChannel.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import org.elasticsearch.transport.netty4.Netty4TcpChannel;

import java.net.InetSocketAddress;
import java.net.SocketAddress;

public class Netty4HttpChannel implements HttpChannel {

Expand All @@ -34,12 +35,20 @@ public void sendResponse(HttpResponse response, ActionListener<Void> listener) {

@Override
public InetSocketAddress getLocalAddress() {
return (InetSocketAddress) channel.localAddress();
return castAddressOrNull(channel.localAddress());
}

@Override
public InetSocketAddress getRemoteAddress() {
return (InetSocketAddress) channel.remoteAddress();
return castAddressOrNull(channel.remoteAddress());
}

private static InetSocketAddress castAddressOrNull(SocketAddress socketAddress) {
if (socketAddress instanceof InetSocketAddress) {
return (InetSocketAddress) socketAddress;
} else {
return null;
}
}

@Override
Expand Down
2 changes: 2 additions & 0 deletions server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,8 @@ public void apply(Settings value, Settings current, Settings previous) {
HttpTransportSettings.SETTING_HTTP_TRACE_LOG_INCLUDE,
HttpTransportSettings.SETTING_HTTP_TRACE_LOG_EXCLUDE,
HttpTransportSettings.SETTING_HTTP_CLIENT_STATS_ENABLED,
HttpTransportSettings.SETTING_HTTP_CLIENT_STATS_MAX_CLOSED_CHANNEL_AGE,
HttpTransportSettings.SETTING_HTTP_CLIENT_STATS_MAX_CLOSED_CHANNEL_COUNT,
HierarchyCircuitBreakerService.USE_REAL_MEMORY_USAGE_SETTING,
HierarchyCircuitBreakerService.TOTAL_CIRCUIT_BREAKER_LIMIT_SETTING,
HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING,
Expand Down
283 changes: 178 additions & 105 deletions server/src/main/java/org/elasticsearch/http/HttpClientStatsTracker.java
View file
Open in desktop

Large diffs are not rendered by default.

69 changes: 38 additions & 31 deletions server/src/main/java/org/elasticsearch/http/HttpStats.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@

import java.io.IOException;
import java.util.List;
import java.util.concurrent.atomic.LongAdder;

public class HttpStats implements Writeable, ToXContentFragment {

Expand Down Expand Up @@ -93,28 +92,36 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
}

public static class ClientStats implements Writeable, ToXContentFragment {
final int id;
String agent;
String localAddress;
String remoteAddress;
String lastUri;
String forwardedFor;
String opaqueId;
long openedTimeMillis;
long closedTimeMillis = -1;
volatile long lastRequestTimeMillis = -1;
final LongAdder requestCount = new LongAdder();
final LongAdder requestSizeBytes = new LongAdder();

ClientStats(long openedTimeMillis) {
this.id = System.identityHashCode(this);
this.openedTimeMillis = openedTimeMillis;
}
public static final long NOT_CLOSED = -1L;

// visible for testing
public ClientStats(String agent, String localAddress, String remoteAddress, String lastUri, String forwardedFor, String opaqueId,
long openedTimeMillis, long closedTimeMillis, long lastRequestTimeMillis, long requestCount, long requestSizeBytes) {
this.id = System.identityHashCode(this);
final int id;
final String agent;
final String localAddress;
final String remoteAddress;
final String lastUri;
final String forwardedFor;
final String opaqueId;
final long openedTimeMillis;
final long closedTimeMillis;
final long lastRequestTimeMillis;
final long requestCount;
final long requestSizeBytes;

public ClientStats(
int id,
String agent,
String localAddress,
String remoteAddress,
String lastUri,
String forwardedFor,
String opaqueId,
long openedTimeMillis,
long closedTimeMillis,
long lastRequestTimeMillis,
long requestCount,
long requestSizeBytes
) {
this.id = id;
this.agent = agent;
this.localAddress = localAddress;
this.remoteAddress = remoteAddress;
Expand All @@ -124,8 +131,8 @@ public ClientStats(String agent, String localAddress, String remoteAddress, Stri
this.openedTimeMillis = openedTimeMillis;
this.closedTimeMillis = closedTimeMillis;
this.lastRequestTimeMillis = lastRequestTimeMillis;
this.requestCount.add(requestCount);
this.requestSizeBytes.add(requestSizeBytes);
this.requestCount = requestCount;
this.requestSizeBytes = requestSizeBytes;
}

ClientStats(StreamInput in) throws IOException {
Expand All @@ -139,8 +146,8 @@ public ClientStats(String agent, String localAddress, String remoteAddress, Stri
this.openedTimeMillis = in.readLong();
this.closedTimeMillis = in.readLong();
this.lastRequestTimeMillis = in.readLong();
this.requestCount.add(in.readLong());
this.requestSizeBytes.add(in.readLong());
this.requestCount = in.readLong();
this.requestSizeBytes = in.readLong();
}

@Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
Expand All @@ -165,12 +172,12 @@ public ClientStats(String agent, String localAddress, String remoteAddress, Stri
builder.field(Fields.CLIENT_OPAQUE_ID, opaqueId);
}
builder.field(Fields.CLIENT_OPENED_TIME_MILLIS, openedTimeMillis);
if (closedTimeMillis != -1) {
if (closedTimeMillis != NOT_CLOSED) {
builder.field(Fields.CLIENT_CLOSED_TIME_MILLIS, closedTimeMillis);
}
builder.field(Fields.CLIENT_LAST_REQUEST_TIME_MILLIS, lastRequestTimeMillis);
builder.field(Fields.CLIENT_REQUEST_COUNT, requestCount.longValue());
builder.field(Fields.CLIENT_REQUEST_SIZE_BYTES, requestSizeBytes.longValue());
builder.field(Fields.CLIENT_REQUEST_COUNT, requestCount);
builder.field(Fields.CLIENT_REQUEST_SIZE_BYTES, requestSizeBytes);
builder.endObject();
return builder;
}
Expand All @@ -187,8 +194,8 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeLong(openedTimeMillis);
out.writeLong(closedTimeMillis);
out.writeLong(lastRequestTimeMillis);
out.writeLong(requestCount.longValue());
out.writeLong(requestSizeBytes.longValue());
out.writeLong(requestCount);
out.writeLong(requestSizeBytes);
}
}
}
4 changes: 4 additions & 0 deletions server/src/main/java/org/elasticsearch/http/HttpTransportSettings.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,10 @@ public final class HttpTransportSettings {

public static final Setting<Boolean> SETTING_HTTP_CLIENT_STATS_ENABLED =
boolSetting("http.client_stats.enabled", true, Property.Dynamic, Property.NodeScope);
public static final Setting<Integer> SETTING_HTTP_CLIENT_STATS_MAX_CLOSED_CHANNEL_COUNT =
intSetting("http.client_stats.closed_channels.max_count", 10000, Property.NodeScope);
public static final Setting<TimeValue> SETTING_HTTP_CLIENT_STATS_MAX_CLOSED_CHANNEL_AGE =
Setting.timeSetting("http.client_stats.closed_channels.max_age", TimeValue.timeValueMinutes(5), Property.NodeScope);

private HttpTransportSettings() {
}
Expand Down
1 change: 1 addition & 0 deletions server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -522,6 +522,7 @@ public static NodeStats createNodeStats() {
List<HttpStats.ClientStats> clientStats = new ArrayList<>(numClients);
for (int k = 0; k < numClients; k++) {
var cs = new HttpStats.ClientStats(
randomInt(),
randomAlphaOfLength(6),
randomAlphaOfLength(6),
randomAlphaOfLength(6),
Expand Down
5 changes: 5 additions & 0 deletions server/src/test/java/org/elasticsearch/http/AbstractHttpServerTransportTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -462,6 +462,7 @@ protected void stopInternal() {
.withPath("/internal/stats_test")
.withHeaders(Map.of(Task.X_OPAQUE_ID, Collections.singletonList(opaqueId)))
.build();
transport.serverAcceptedChannel(fakeRestRequest.getHttpChannel());
transport.incomingRequest(fakeRestRequest.getHttpRequest(), fakeRestRequest.getHttpChannel());

HttpStats httpStats = transport.stats();
Expand All @@ -478,6 +479,7 @@ protected void stopInternal() {
.withPath("/internal/stats_test2")
.withHeaders(Map.of(Task.X_OPAQUE_ID.toUpperCase(Locale.ROOT), Collections.singletonList(opaqueId)))
.build();
transport.serverAcceptedChannel(fakeRestRequest.getHttpChannel());
transport.incomingRequest(fakeRestRequest.getHttpRequest(), fakeRestRequest.getHttpChannel());
httpStats = transport.stats();
assertThat(httpStats.getClientStats().size(), equalTo(2));
Expand Down Expand Up @@ -532,6 +534,7 @@ public void testDisablingHttpClientStats() {
.withPath("/internal/stats_test")
.withHeaders(Map.of(Task.X_OPAQUE_ID, Collections.singletonList(opaqueId)))
.build();
transport.serverAcceptedChannel(fakeRestRequest.getHttpChannel());
transport.incomingRequest(fakeRestRequest.getHttpRequest(), fakeRestRequest.getHttpChannel());

// HTTP client stats should default to enabled
Expand All @@ -556,6 +559,7 @@ public void testDisablingHttpClientStats() {
.withPath("/internal/stats_test")
.withHeaders(Map.of(Task.X_OPAQUE_ID, Collections.singletonList(opaqueId)))
.build();
transport.serverAcceptedChannel(fakeRestRequest.getHttpChannel());
transport.incomingRequest(fakeRestRequest.getHttpRequest(), fakeRestRequest.getHttpChannel());
httpStats = transport.stats();
assertThat(httpStats.getClientStats().size(), equalTo(0));
Expand All @@ -573,6 +577,7 @@ public void testDisablingHttpClientStats() {
.withPath("/internal/stats_test")
.withHeaders(Map.of(Task.X_OPAQUE_ID, Collections.singletonList(opaqueId)))
.build();
transport.serverAcceptedChannel(fakeRestRequest.getHttpChannel());
transport.incomingRequest(fakeRestRequest.getHttpRequest(), fakeRestRequest.getHttpChannel());
httpStats = transport.stats();
assertThat(httpStats.getClientStats().size(), equalTo(1));
Expand Down
Loading