From c34aa0e2ca05154f40baa4ab08add4768611b94f Mon Sep 17 00:00:00 2001 From: Kaushal Kumar Date: Wed, 21 Aug 2024 15:21:42 -0700 Subject: [PATCH 1/8] add query group stats constructs Signed-off-by: Kaushal Kumar --- .../opensearch/wlm/stats/QueryGroupState.java | 125 +++++++++++ .../opensearch/wlm/stats/QueryGroupStats.java | 197 ++++++++++++++++++ .../wlm/stats/QueryGroupStateTests.java | 56 +++++ .../wlm/stats/QueryGroupStatsTests.java | 38 ++++ 4 files changed, 416 insertions(+) create mode 100644 server/src/main/java/org/opensearch/wlm/stats/QueryGroupState.java create mode 100644 server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java create mode 100644 server/src/test/java/org/opensearch/wlm/stats/QueryGroupStateTests.java create mode 100644 server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java diff --git a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupState.java b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupState.java new file mode 100644 index 0000000000000..f2f3c1ff1f2ce --- /dev/null +++ b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupState.java @@ -0,0 +1,125 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm.stats; + +import org.opensearch.search.ResourceType; + +import java.util.EnumMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +/** + * This class will keep the point in time view of the query group stats + */ +public class QueryGroupState { + /** + * completions at the query group level, this is a cumulative counter since the Opensearch start time + */ + private final AtomicLong completions = new AtomicLong(); + + /** + * rejections at the query group level, this is a cumulative counter since the Opensearch start time + */ + private final AtomicLong rejections = new AtomicLong(); + + /** + * this will track the cumulative failures in a query group + */ + private final AtomicLong failures = new AtomicLong(); + + /** + * This is used to store the resource type state both for CPU and MEMORY + */ + private final Map resourceState; + + public QueryGroupState() { + resourceState = new EnumMap<>(ResourceType.class); + resourceState.put(ResourceType.CPU, new ResourceTypeState(ResourceType.CPU)); + resourceState.put(ResourceType.MEMORY, new ResourceTypeState(ResourceType.MEMORY)); + } + + /** + * + * @return completions in the query group + */ + public long getCompletions() { + return completions.get(); + } + + /** + * + * @return rejections in the query group + */ + public long getRejections() { + return rejections.get(); + } + + /** + * + * @return failures in the query group + */ + public long getFailures() { + return failures.get(); + } + + /** + * getter for query group resource state + * @return the query group resource state + */ + public Map getResourceState() { + return resourceState; + } + + /** + * this is a call back to increment cancellations for a query group at task level + */ + public void incrementCompletions() { + completions.incrementAndGet(); + } + + /** + * this is a call back to increment rejections for a query group at incoming request + */ + public void incrementRejections() { + rejections.incrementAndGet(); + } + + /** + * this is a call back to increment failures for a query group + */ + public void incrementFailures() { + failures.incrementAndGet(); + } + + /** + * This class holds the resource level stats for the query group + */ + public static class ResourceTypeState { + private final ResourceType resourceType; + private final AtomicLong cancellations = new AtomicLong(); + + public ResourceTypeState(ResourceType resourceType) { + this.resourceType = resourceType; + } + + /** + * getter for resource type cancellations + */ + public long getCancellations() { + return cancellations.get(); + } + + /** + * this will be called when a task is cancelled due to this resource + */ + public void incrementCancellations() { + cancellations.incrementAndGet(); + } + } +} diff --git a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java new file mode 100644 index 0000000000000..fa10f7a473239 --- /dev/null +++ b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java @@ -0,0 +1,197 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm.stats; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.search.ResourceType; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + +/** + * { + * "queryGroupID": { + * "completions": 1233234234, + * "rejections": 1243545, + * "failures": 97, + * "CPU": { "current_usage": 49.6, "cancellation": 432 }, + * "MEMORY": { "current_usage": 39.6, "cancellation": 42 } + * }, + * ... + * ... + * } + */ +public class QueryGroupStats implements ToXContentObject, Writeable { + private final Map stats; + + public QueryGroupStats(Map stats) { + this.stats = stats; + } + + public QueryGroupStats(StreamInput in) throws IOException { + stats = in.readMap(StreamInput::readString, QueryGroupStatsHolder::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeMap(stats, StreamOutput::writeString, QueryGroupStatsHolder::writeTo); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + for (Map.Entry queryGroupStats : stats.entrySet()) { + builder.startObject(queryGroupStats.getKey()); + queryGroupStats.getValue().toXContent(builder, params); + builder.endObject(); + } + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + QueryGroupStats that = (QueryGroupStats) o; + return Objects.equals(stats, that.stats); + } + + @Override + public int hashCode() { + return Objects.hash(stats); + } + + /** + * This is a stats holder object which will hold the data for a query group at a point in time + * the instance will only be created on demand through stats api + */ + public static class QueryGroupStatsHolder implements ToXContentObject, Writeable { + public static final String COMPLETIONS = "completions"; + public static final String REJECTIONS = "rejections"; + private final long completions; + private final long rejections; + private final Map resourceStats; + + public QueryGroupStatsHolder(long completions, long rejections, Map resourceStats) { + this.completions = completions; + this.rejections = rejections; + this.resourceStats = resourceStats; + } + + public QueryGroupStatsHolder(StreamInput in) throws IOException { + this.completions = in.readVLong(); + this.rejections = in.readVLong(); + this.resourceStats = in.readMap((i) -> ResourceType.fromName(i.readString()), ResourceStats::new); + } + + /** + * Writes the {@param statsHolder} to {@param out} + * @param out StreamOutput + * @param statsHolder QueryGroupStatsHolder + * @throws IOException exception + */ + public static void writeTo(StreamOutput out, QueryGroupStatsHolder statsHolder) throws IOException { + out.writeVLong(statsHolder.completions); + out.writeVLong(statsHolder.rejections); + out.writeMap(statsHolder.resourceStats, (o, val) -> o.writeString(val.getName()), ResourceStats::writeTo); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + QueryGroupStatsHolder.writeTo(out, this); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(COMPLETIONS, completions); + builder.field(REJECTIONS, rejections); + for (Map.Entry resourceStat : resourceStats.entrySet()) { + ResourceType resourceType = resourceStat.getKey(); + ResourceStats resourceStats1 = resourceStat.getValue(); + builder.startObject(resourceType.getName()); + resourceStats1.toXContent(builder, params); + builder.endObject(); + } + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + QueryGroupStatsHolder that = (QueryGroupStatsHolder) o; + return completions == that.completions && rejections == that.rejections && Objects.equals(resourceStats, that.resourceStats); + } + + @Override + public int hashCode() { + return Objects.hash(completions, rejections, resourceStats); + } + } + + /** + * point in time resource level stats holder + */ + public static class ResourceStats implements ToXContentObject, Writeable { + public static final String CURRENT_USAGE = "current_usage"; + public static final String CANCELLATIONS = "cancellations"; + private final double currentUsage; + private final long cancellations; + + public ResourceStats(double currentUsage, long cancellations) { + this.currentUsage = currentUsage; + this.cancellations = cancellations; + } + + public ResourceStats(StreamInput in) throws IOException { + this.currentUsage = in.readDouble(); + this.cancellations = in.readVLong(); + } + + /** + * Writes the {@param stats} to {@param out} + * @param out StreamOutput + * @param stats QueryGroupStatsHolder + * @throws IOException exception + */ + public static void writeTo(StreamOutput out, ResourceStats stats) throws IOException { + out.writeDouble(stats.currentUsage); + out.writeVLong(stats.cancellations); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + ResourceStats.writeTo(out, this); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(CURRENT_USAGE, currentUsage); + builder.field(CANCELLATIONS, cancellations); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ResourceStats that = (ResourceStats) o; + return (currentUsage - that.currentUsage) < 1e-9 && cancellations == that.cancellations; + } + + @Override + public int hashCode() { + return Objects.hash(currentUsage, cancellations); + } + } +} diff --git a/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStateTests.java b/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStateTests.java new file mode 100644 index 0000000000000..00d308beeed72 --- /dev/null +++ b/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStateTests.java @@ -0,0 +1,56 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm.stats; + +import org.opensearch.search.ResourceType; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.ArrayList; +import java.util.List; + +public class QueryGroupStateTests extends OpenSearchTestCase { + QueryGroupState queryGroupState; + + public void testRandomQueryGroupsStateUpdates() { + queryGroupState = new QueryGroupState(); + List updaterThreads = new ArrayList<>(); + + for (int i = 0; i < 25; i++) { + if (i % 5 == 0) { + updaterThreads.add(new Thread(() -> queryGroupState.incrementCompletions())); + } else if (i % 5 == 1) { + updaterThreads.add(new Thread(() -> queryGroupState.incrementRejections())); + } else if (i % 5 == 2) { + updaterThreads.add(new Thread(() -> queryGroupState.incrementFailures())); + } else if (i % 5 == 3) { + updaterThreads.add(new Thread(() -> queryGroupState.getResourceState().get(ResourceType.CPU).incrementCancellations())); + } else { + updaterThreads.add(new Thread(() -> queryGroupState.getResourceState().get(ResourceType.MEMORY).incrementCancellations())); + } + } + + // trigger the updates + updaterThreads.forEach(Thread::start); + // wait for updates to be finished + updaterThreads.forEach(thread -> { + try { + thread.join(); + } catch (InterruptedException ignored) { + + } + }); + + assertEquals(5, queryGroupState.getCompletions()); + assertEquals(5, queryGroupState.getRejections()); + assertEquals(5, queryGroupState.getFailures()); + assertEquals(5, queryGroupState.getResourceState().get(ResourceType.CPU).getCancellations()); + assertEquals(5, queryGroupState.getResourceState().get(ResourceType.MEMORY).getCancellations()); + } + +} diff --git a/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java b/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java new file mode 100644 index 0000000000000..39c98eb18514e --- /dev/null +++ b/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java @@ -0,0 +1,38 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.wlm.stats; + +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.search.ResourceType; +import org.opensearch.test.AbstractWireSerializingTestCase; + +import java.util.HashMap; +import java.util.Map; + +public class QueryGroupStatsTests extends AbstractWireSerializingTestCase { + + @Override + protected Writeable.Reader instanceReader() { + return QueryGroupStats::new; + } + + @Override + protected QueryGroupStats createTestInstance() { + Map stats = new HashMap<>(); + stats.put( + randomAlphaOfLength(10), + new QueryGroupStats.QueryGroupStatsHolder( + randomNonNegativeLong(), + randomNonNegativeLong(), + Map.of(ResourceType.CPU, new QueryGroupStats.ResourceStats(randomDoubleBetween(0.0, 0.90, false), randomNonNegativeLong())) + ) + ); + return new QueryGroupStats(stats); + } +} From 676096ffa7bca49f2bc5b963cb7ae1719017e8ec Mon Sep 17 00:00:00 2001 From: Kaushal Kumar Date: Wed, 21 Aug 2024 15:30:52 -0700 Subject: [PATCH 2/8] add changelog entry Signed-off-by: Kaushal Kumar --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1f3d9287e4a8..1c645a9c26f3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add allowlist setting for ingest-geoip and ingest-useragent ([#15325](https://github.com/opensearch-project/OpenSearch/pull/15325)) - Adding access to noSubMatches and noOverlappingMatches in Hyphenation ([#13895](https://github.com/opensearch-project/OpenSearch/pull/13895)) - Add support for index level max slice count setting for concurrent segment search ([#15336](https://github.com/opensearch-project/OpenSearch/pull/15336)) +- [Workload Management] Add query group stats constructs ([#15343](https://github.com/opensearch-project/OpenSearch/pull/15343))) ### Dependencies - Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081)) From 7fada745471937630ba690844935bc513d24258d Mon Sep 17 00:00:00 2001 From: Kaushal Kumar Date: Thu, 22 Aug 2024 11:32:25 -0700 Subject: [PATCH 3/8] add packageinfo for stats Signed-off-by: Kaushal Kumar --- .../java/org/opensearch/wlm/stats/package-info.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 server/src/main/java/org/opensearch/wlm/stats/package-info.java diff --git a/server/src/main/java/org/opensearch/wlm/stats/package-info.java b/server/src/main/java/org/opensearch/wlm/stats/package-info.java new file mode 100644 index 0000000000000..2facf8d16df22 --- /dev/null +++ b/server/src/main/java/org/opensearch/wlm/stats/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Query group stats related artifacts + */ +package org.opensearch.wlm.stats; From ce813f11c2387f59f460e1b70a6b1736dea9aba6 Mon Sep 17 00:00:00 2001 From: Kaushal Kumar Date: Sun, 25 Aug 2024 18:14:45 -0700 Subject: [PATCH 4/8] add total cancellations Signed-off-by: Kaushal Kumar --- .../opensearch/wlm/stats/QueryGroupState.java | 16 ++++++++++++++++ .../wlm/stats/QueryGroupStateTests.java | 5 +++++ 2 files changed, 21 insertions(+) diff --git a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupState.java b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupState.java index f2f3c1ff1f2ce..b53cdc11ea2ea 100644 --- a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupState.java +++ b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupState.java @@ -33,6 +33,11 @@ public class QueryGroupState { */ private final AtomicLong failures = new AtomicLong(); + /** + * This will track total number of cancellations in the query group due to all resource type breaches + */ + private final AtomicLong totalCancellations = new AtomicLong(); + /** * This is used to store the resource type state both for CPU and MEMORY */ @@ -68,6 +73,10 @@ public long getFailures() { return failures.get(); } + public long getTotalCancellations() { + return totalCancellations.get(); + } + /** * getter for query group resource state * @return the query group resource state @@ -97,6 +106,13 @@ public void incrementFailures() { failures.incrementAndGet(); } + /** + * this is a call back to increment total cancellations for a query group + */ + public void incrementTotalCancellations() { + totalCancellations.incrementAndGet(); + } + /** * This class holds the resource level stats for the query group */ diff --git a/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStateTests.java b/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStateTests.java index 00d308beeed72..722a8bf9235fe 100644 --- a/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStateTests.java +++ b/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStateTests.java @@ -33,6 +33,10 @@ public void testRandomQueryGroupsStateUpdates() { } else { updaterThreads.add(new Thread(() -> queryGroupState.getResourceState().get(ResourceType.MEMORY).incrementCancellations())); } + + if (i%5 == 3 || i%5 == 4) { + updaterThreads.add(new Thread(() -> queryGroupState.incrementTotalCancellations())); + } } // trigger the updates @@ -49,6 +53,7 @@ public void testRandomQueryGroupsStateUpdates() { assertEquals(5, queryGroupState.getCompletions()); assertEquals(5, queryGroupState.getRejections()); assertEquals(5, queryGroupState.getFailures()); + assertEquals(10, queryGroupState.getTotalCancellations()); assertEquals(5, queryGroupState.getResourceState().get(ResourceType.CPU).getCancellations()); assertEquals(5, queryGroupState.getResourceState().get(ResourceType.MEMORY).getCancellations()); } From a783e8f318ac66bca16ac81bc2e2bf5db6ed1610 Mon Sep 17 00:00:00 2001 From: Kaushal Kumar Date: Tue, 27 Aug 2024 17:39:12 -0700 Subject: [PATCH 5/8] add more granular level rejections Signed-off-by: Kaushal Kumar --- .../opensearch/wlm/stats/QueryGroupState.java | 69 ++++--------------- .../opensearch/wlm/stats/QueryGroupStats.java | 41 +++++++++-- .../wlm/stats/QueryGroupStateTests.java | 32 ++++++--- .../wlm/stats/QueryGroupStatsTests.java | 11 ++- 4 files changed, 81 insertions(+), 72 deletions(-) diff --git a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupState.java b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupState.java index b53cdc11ea2ea..44d2c13b58388 100644 --- a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupState.java +++ b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupState.java @@ -8,11 +8,11 @@ package org.opensearch.wlm.stats; +import org.opensearch.common.metrics.CounterMetric; import org.opensearch.search.ResourceType; import java.util.EnumMap; import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; /** * This class will keep the point in time view of the query group stats @@ -21,22 +21,22 @@ public class QueryGroupState { /** * completions at the query group level, this is a cumulative counter since the Opensearch start time */ - private final AtomicLong completions = new AtomicLong(); + final CounterMetric completions = new CounterMetric(); /** - * rejections at the query group level, this is a cumulative counter since the Opensearch start time + * rejections at the query group level, this is a cumulative counter since the OpenSearch start time */ - private final AtomicLong rejections = new AtomicLong(); + final CounterMetric totalRejections = new CounterMetric(); /** * this will track the cumulative failures in a query group */ - private final AtomicLong failures = new AtomicLong(); + final CounterMetric failures = new CounterMetric(); /** * This will track total number of cancellations in the query group due to all resource type breaches */ - private final AtomicLong totalCancellations = new AtomicLong(); + final CounterMetric totalCancellations = new CounterMetric(); /** * This is used to store the resource type state both for CPU and MEMORY @@ -54,15 +54,15 @@ public QueryGroupState() { * @return completions in the query group */ public long getCompletions() { - return completions.get(); + return completions.count(); } /** * * @return rejections in the query group */ - public long getRejections() { - return rejections.get(); + public long getTotalRejections() { + return totalRejections.count(); } /** @@ -70,11 +70,11 @@ public long getRejections() { * @return failures in the query group */ public long getFailures() { - return failures.get(); + return failures.count(); } public long getTotalCancellations() { - return totalCancellations.get(); + return totalCancellations.count(); } /** @@ -85,57 +85,16 @@ public Map getResourceState() { return resourceState; } - /** - * this is a call back to increment cancellations for a query group at task level - */ - public void incrementCompletions() { - completions.incrementAndGet(); - } - - /** - * this is a call back to increment rejections for a query group at incoming request - */ - public void incrementRejections() { - rejections.incrementAndGet(); - } - - /** - * this is a call back to increment failures for a query group - */ - public void incrementFailures() { - failures.incrementAndGet(); - } - - /** - * this is a call back to increment total cancellations for a query group - */ - public void incrementTotalCancellations() { - totalCancellations.incrementAndGet(); - } - /** * This class holds the resource level stats for the query group */ public static class ResourceTypeState { - private final ResourceType resourceType; - private final AtomicLong cancellations = new AtomicLong(); + final ResourceType resourceType; + final CounterMetric cancellations = new CounterMetric(); + final CounterMetric rejections = new CounterMetric(); public ResourceTypeState(ResourceType resourceType) { this.resourceType = resourceType; } - - /** - * getter for resource type cancellations - */ - public long getCancellations() { - return cancellations.get(); - } - - /** - * this will be called when a task is cancelled due to this resource - */ - public void incrementCancellations() { - cancellations.incrementAndGet(); - } } } diff --git a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java index fa10f7a473239..a4ccea353c7c5 100644 --- a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java +++ b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java @@ -25,6 +25,7 @@ * "completions": 1233234234, * "rejections": 1243545, * "failures": 97, + * "total_cancellations": 474, * "CPU": { "current_usage": 49.6, "cancellation": 432 }, * "MEMORY": { "current_usage": 39.6, "cancellation": 42 } * }, @@ -78,19 +79,33 @@ public int hashCode() { public static class QueryGroupStatsHolder implements ToXContentObject, Writeable { public static final String COMPLETIONS = "completions"; public static final String REJECTIONS = "rejections"; + public static final String TOTAL_CANCELLATIONS = "total_cancellations"; + public static final String FAILURES = "failures"; private final long completions; private final long rejections; + private final long failures; + private final long totalCancellations; private final Map resourceStats; - public QueryGroupStatsHolder(long completions, long rejections, Map resourceStats) { + public QueryGroupStatsHolder( + long completions, + long rejections, + long failures, + long totalCancellations, + Map resourceStats + ) { this.completions = completions; this.rejections = rejections; + this.failures = failures; + this.totalCancellations = totalCancellations; this.resourceStats = resourceStats; } public QueryGroupStatsHolder(StreamInput in) throws IOException { this.completions = in.readVLong(); this.rejections = in.readVLong(); + this.failures = in.readVLong(); + this.totalCancellations = in.readVLong(); this.resourceStats = in.readMap((i) -> ResourceType.fromName(i.readString()), ResourceStats::new); } @@ -103,6 +118,8 @@ public QueryGroupStatsHolder(StreamInput in) throws IOException { public static void writeTo(StreamOutput out, QueryGroupStatsHolder statsHolder) throws IOException { out.writeVLong(statsHolder.completions); out.writeVLong(statsHolder.rejections); + out.writeVLong(statsHolder.failures); + out.writeVLong(statsHolder.totalCancellations); out.writeMap(statsHolder.resourceStats, (o, val) -> o.writeString(val.getName()), ResourceStats::writeTo); } @@ -115,6 +132,8 @@ public void writeTo(StreamOutput out) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field(COMPLETIONS, completions); builder.field(REJECTIONS, rejections); + builder.field(FAILURES, failures); + builder.field(TOTAL_CANCELLATIONS, totalCancellations); for (Map.Entry resourceStat : resourceStats.entrySet()) { ResourceType resourceType = resourceStat.getKey(); ResourceStats resourceStats1 = resourceStat.getValue(); @@ -130,12 +149,16 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; QueryGroupStatsHolder that = (QueryGroupStatsHolder) o; - return completions == that.completions && rejections == that.rejections && Objects.equals(resourceStats, that.resourceStats); + return completions == that.completions + && rejections == that.rejections + && Objects.equals(resourceStats, that.resourceStats) + && failures == that.failures + && totalCancellations == that.totalCancellations; } @Override public int hashCode() { - return Objects.hash(completions, rejections, resourceStats); + return Objects.hash(completions, rejections, totalCancellations, failures, resourceStats); } } @@ -145,17 +168,21 @@ public int hashCode() { public static class ResourceStats implements ToXContentObject, Writeable { public static final String CURRENT_USAGE = "current_usage"; public static final String CANCELLATIONS = "cancellations"; + public static final double PRECISION = 1e-9; private final double currentUsage; private final long cancellations; + private final long rejections; - public ResourceStats(double currentUsage, long cancellations) { + public ResourceStats(double currentUsage, long cancellations, long rejections) { this.currentUsage = currentUsage; this.cancellations = cancellations; + this.rejections = rejections; } public ResourceStats(StreamInput in) throws IOException { this.currentUsage = in.readDouble(); this.cancellations = in.readVLong(); + this.rejections = in.readVLong(); } /** @@ -167,6 +194,7 @@ public ResourceStats(StreamInput in) throws IOException { public static void writeTo(StreamOutput out, ResourceStats stats) throws IOException { out.writeDouble(stats.currentUsage); out.writeVLong(stats.cancellations); + out.writeVLong(stats.rejections); } @Override @@ -178,6 +206,7 @@ public void writeTo(StreamOutput out) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field(CURRENT_USAGE, currentUsage); builder.field(CANCELLATIONS, cancellations); + builder.field(QueryGroupStatsHolder.REJECTIONS, rejections); return builder; } @@ -186,12 +215,12 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; ResourceStats that = (ResourceStats) o; - return (currentUsage - that.currentUsage) < 1e-9 && cancellations == that.cancellations; + return (currentUsage - that.currentUsage) < PRECISION && cancellations == that.cancellations && rejections == that.rejections; } @Override public int hashCode() { - return Objects.hash(currentUsage, cancellations); + return Objects.hash(currentUsage, cancellations, rejections); } } } diff --git a/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStateTests.java b/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStateTests.java index 722a8bf9235fe..76f4b787a34f4 100644 --- a/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStateTests.java +++ b/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStateTests.java @@ -23,19 +23,26 @@ public void testRandomQueryGroupsStateUpdates() { for (int i = 0; i < 25; i++) { if (i % 5 == 0) { - updaterThreads.add(new Thread(() -> queryGroupState.incrementCompletions())); + updaterThreads.add(new Thread(() -> queryGroupState.completions.inc())); } else if (i % 5 == 1) { - updaterThreads.add(new Thread(() -> queryGroupState.incrementRejections())); + updaterThreads.add(new Thread(() -> { + queryGroupState.totalRejections.inc(); + if (randomBoolean()) { + queryGroupState.getResourceState().get(ResourceType.CPU).rejections.inc(); + } else { + queryGroupState.getResourceState().get(ResourceType.MEMORY).rejections.inc(); + } + })); } else if (i % 5 == 2) { - updaterThreads.add(new Thread(() -> queryGroupState.incrementFailures())); + updaterThreads.add(new Thread(() -> queryGroupState.failures.inc())); } else if (i % 5 == 3) { - updaterThreads.add(new Thread(() -> queryGroupState.getResourceState().get(ResourceType.CPU).incrementCancellations())); + updaterThreads.add(new Thread(() -> queryGroupState.getResourceState().get(ResourceType.CPU).cancellations.inc())); } else { - updaterThreads.add(new Thread(() -> queryGroupState.getResourceState().get(ResourceType.MEMORY).incrementCancellations())); + updaterThreads.add(new Thread(() -> queryGroupState.getResourceState().get(ResourceType.MEMORY).cancellations.inc())); } - if (i%5 == 3 || i%5 == 4) { - updaterThreads.add(new Thread(() -> queryGroupState.incrementTotalCancellations())); + if (i % 5 == 3 || i % 5 == 4) { + updaterThreads.add(new Thread(() -> queryGroupState.totalCancellations.inc())); } } @@ -51,11 +58,16 @@ public void testRandomQueryGroupsStateUpdates() { }); assertEquals(5, queryGroupState.getCompletions()); - assertEquals(5, queryGroupState.getRejections()); + assertEquals(5, queryGroupState.getTotalRejections()); + + final long sumOfRejectionsDueToResourceTypes = queryGroupState.getResourceState().get(ResourceType.CPU).rejections.count() + + queryGroupState.getResourceState().get(ResourceType.MEMORY).rejections.count(); + assertEquals(sumOfRejectionsDueToResourceTypes, queryGroupState.getTotalRejections()); + assertEquals(5, queryGroupState.getFailures()); assertEquals(10, queryGroupState.getTotalCancellations()); - assertEquals(5, queryGroupState.getResourceState().get(ResourceType.CPU).getCancellations()); - assertEquals(5, queryGroupState.getResourceState().get(ResourceType.MEMORY).getCancellations()); + assertEquals(5, queryGroupState.getResourceState().get(ResourceType.CPU).cancellations.count()); + assertEquals(5, queryGroupState.getResourceState().get(ResourceType.MEMORY).cancellations.count()); } } diff --git a/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java b/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java index 39c98eb18514e..95f5a96a302a9 100644 --- a/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java +++ b/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java @@ -30,7 +30,16 @@ protected QueryGroupStats createTestInstance() { new QueryGroupStats.QueryGroupStatsHolder( randomNonNegativeLong(), randomNonNegativeLong(), - Map.of(ResourceType.CPU, new QueryGroupStats.ResourceStats(randomDoubleBetween(0.0, 0.90, false), randomNonNegativeLong())) + randomNonNegativeLong(), + randomNonNegativeLong(), + Map.of( + ResourceType.CPU, + new QueryGroupStats.ResourceStats( + randomDoubleBetween(0.0, 0.90, false), + randomNonNegativeLong(), + randomNonNegativeLong() + ) + ) ) ); return new QueryGroupStats(stats); From c53daf8b11bfb6fdb5e25078d1a62e006f361a3d Mon Sep 17 00:00:00 2001 From: Kaushal Kumar Date: Tue, 27 Aug 2024 18:06:11 -0700 Subject: [PATCH 6/8] add toXContent test cases Signed-off-by: Kaushal Kumar --- .../opensearch/wlm/stats/QueryGroupStats.java | 2 ++ .../wlm/stats/QueryGroupStatsTests.java | 28 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java index a4ccea353c7c5..6b9fef979c53e 100644 --- a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java +++ b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java @@ -51,11 +51,13 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject("query_groups"); for (Map.Entry queryGroupStats : stats.entrySet()) { builder.startObject(queryGroupStats.getKey()); queryGroupStats.getValue().toXContent(builder, params); builder.endObject(); } + builder.endObject(); return builder; } diff --git a/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java b/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java index 95f5a96a302a9..c114c811bbe07 100644 --- a/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java +++ b/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java @@ -8,15 +8,43 @@ package org.opensearch.wlm.stats; +import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.ResourceType; import org.opensearch.test.AbstractWireSerializingTestCase; +import java.io.IOException; import java.util.HashMap; import java.util.Map; public class QueryGroupStatsTests extends AbstractWireSerializingTestCase { + public void testToXContent() throws IOException { + final Map stats = new HashMap<>(); + final String queryGroupId = "afakjklaj304041-afaka"; + stats.put( + queryGroupId, + new QueryGroupStats.QueryGroupStatsHolder( + 123456789, + 2, + 0, + 13, + Map.of(ResourceType.CPU, new QueryGroupStats.ResourceStats(0.3, 13, 2)) + ) + ); + XContentBuilder builder = JsonXContent.contentBuilder(); + QueryGroupStats queryGroupStats = new QueryGroupStats(stats); + builder.startObject(); + queryGroupStats.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + assertEquals( + "{\"query_groups\":{\"afakjklaj304041-afaka\":{\"completions\":123456789,\"rejections\":2,\"failures\":0,\"total_cancellations\":13,\"cpu\":{\"current_usage\":0.3,\"cancellations\":13,\"rejections\":2}}}}", + builder.toString() + ); + } + @Override protected Writeable.Reader instanceReader() { return QueryGroupStats::new; From 353b1c82c3386c236cdbf9de97e3be280e933138 Mon Sep 17 00:00:00 2001 From: Kaushal Kumar Date: Tue, 27 Aug 2024 18:20:41 -0700 Subject: [PATCH 7/8] move ResourceType enum to wlm Signed-off-by: Kaushal Kumar --- .../wlm/service/QueryGroupPersistenceService.java | 2 +- .../opensearch/plugin/wlm/QueryGroupTestUtils.java | 2 +- .../service/QueryGroupPersistenceServiceTests.java | 2 +- .../opensearch/cluster/metadata/QueryGroup.java | 2 +- .../backpressure/SearchBackpressureService.java | 2 +- .../backpressure/trackers/NodeDuressTrackers.java | 2 +- .../wlm/QueryGroupLevelResourceUsageView.java | 1 - .../opensearch/{search => wlm}/ResourceType.java | 14 ++++++++++---- .../org/opensearch/wlm/stats/QueryGroupState.java | 9 ++++++--- .../org/opensearch/wlm/stats/QueryGroupStats.java | 2 +- .../QueryGroupResourceUsageTrackerService.java | 2 +- .../cluster/metadata/QueryGroupMetadataTests.java | 2 +- .../cluster/metadata/QueryGroupTests.java | 2 +- .../SearchBackpressureServiceTests.java | 6 +++--- .../trackers/NodeDuressTrackersTests.java | 2 +- .../wlm/QueryGroupLevelResourceUsageViewTests.java | 1 - .../{search => wlm}/ResourceTypeTests.java | 3 +-- .../opensearch/wlm/stats/QueryGroupStateTests.java | 2 +- .../opensearch/wlm/stats/QueryGroupStatsTests.java | 2 +- ...QueryGroupResourceUsageTrackerServiceTests.java | 2 +- 20 files changed, 34 insertions(+), 28 deletions(-) rename server/src/main/java/org/opensearch/{search => wlm}/ResourceType.java (87%) rename server/src/test/java/org/opensearch/{search => wlm}/ResourceTypeTests.java (96%) diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/service/QueryGroupPersistenceService.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/service/QueryGroupPersistenceService.java index ba5161a2c855e..7561a2f6f99c3 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/service/QueryGroupPersistenceService.java +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/service/QueryGroupPersistenceService.java @@ -29,7 +29,7 @@ import org.opensearch.core.rest.RestStatus; import org.opensearch.plugin.wlm.action.CreateQueryGroupResponse; import org.opensearch.plugin.wlm.action.DeleteQueryGroupRequest; -import org.opensearch.search.ResourceType; +import org.opensearch.wlm.ResourceType; import java.util.Collection; import java.util.EnumMap; diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/QueryGroupTestUtils.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/QueryGroupTestUtils.java index 5ba1ad5334712..e165645775d5c 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/QueryGroupTestUtils.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/QueryGroupTestUtils.java @@ -31,7 +31,7 @@ import java.util.Set; import static org.opensearch.cluster.metadata.QueryGroup.builder; -import static org.opensearch.search.ResourceType.fromName; +import static org.opensearch.wlm.ResourceType.fromName; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/service/QueryGroupPersistenceServiceTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/service/QueryGroupPersistenceServiceTests.java index a516ffdde839e..5cb3d8fc6d11f 100644 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/service/QueryGroupPersistenceServiceTests.java +++ b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/service/QueryGroupPersistenceServiceTests.java @@ -24,9 +24,9 @@ import org.opensearch.plugin.wlm.QueryGroupTestUtils; import org.opensearch.plugin.wlm.action.CreateQueryGroupResponse; import org.opensearch.plugin.wlm.action.DeleteQueryGroupRequest; -import org.opensearch.search.ResourceType; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.wlm.ResourceType; import java.util.ArrayList; import java.util.Collection; diff --git a/server/src/main/java/org/opensearch/cluster/metadata/QueryGroup.java b/server/src/main/java/org/opensearch/cluster/metadata/QueryGroup.java index 9b5c6bc2369a6..a971aa58940ba 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/QueryGroup.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/QueryGroup.java @@ -17,7 +17,7 @@ import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.search.ResourceType; +import org.opensearch.wlm.ResourceType; import org.joda.time.Instant; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java index c26c5d63a3573..a85bc69b766cb 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java +++ b/server/src/main/java/org/opensearch/search/backpressure/SearchBackpressureService.java @@ -18,7 +18,6 @@ import org.opensearch.common.settings.Setting; import org.opensearch.monitor.jvm.JvmStats; import org.opensearch.monitor.process.ProcessProbe; -import org.opensearch.search.ResourceType; import org.opensearch.search.backpressure.settings.SearchBackpressureMode; import org.opensearch.search.backpressure.settings.SearchBackpressureSettings; import org.opensearch.search.backpressure.settings.SearchShardTaskSettings; @@ -43,6 +42,7 @@ import org.opensearch.tasks.TaskResourceTrackingService.TaskCompletionListener; import org.opensearch.threadpool.Scheduler; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.wlm.ResourceType; import java.io.IOException; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackers.java b/server/src/main/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackers.java index ae60a82fc2816..c27c50ac12c0f 100644 --- a/server/src/main/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackers.java +++ b/server/src/main/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackers.java @@ -9,7 +9,7 @@ package org.opensearch.search.backpressure.trackers; import org.opensearch.common.util.Streak; -import org.opensearch.search.ResourceType; +import org.opensearch.wlm.ResourceType; import java.util.Map; import java.util.function.BooleanSupplier; diff --git a/server/src/main/java/org/opensearch/wlm/QueryGroupLevelResourceUsageView.java b/server/src/main/java/org/opensearch/wlm/QueryGroupLevelResourceUsageView.java index 2fd743dc3f83f..7577c8573ec10 100644 --- a/server/src/main/java/org/opensearch/wlm/QueryGroupLevelResourceUsageView.java +++ b/server/src/main/java/org/opensearch/wlm/QueryGroupLevelResourceUsageView.java @@ -8,7 +8,6 @@ package org.opensearch.wlm; -import org.opensearch.search.ResourceType; import org.opensearch.tasks.Task; import java.util.List; diff --git a/server/src/main/java/org/opensearch/search/ResourceType.java b/server/src/main/java/org/opensearch/wlm/ResourceType.java similarity index 87% rename from server/src/main/java/org/opensearch/search/ResourceType.java rename to server/src/main/java/org/opensearch/wlm/ResourceType.java index 0cba2222a6e20..adf384995c91d 100644 --- a/server/src/main/java/org/opensearch/search/ResourceType.java +++ b/server/src/main/java/org/opensearch/wlm/ResourceType.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.search; +package org.opensearch.wlm; import org.opensearch.common.annotation.PublicApi; import org.opensearch.core.common.io.stream.StreamOutput; @@ -21,15 +21,17 @@ */ @PublicApi(since = "2.x") public enum ResourceType { - CPU("cpu", task -> task.getTotalResourceUtilization(ResourceStats.CPU)), - MEMORY("memory", task -> task.getTotalResourceUtilization(ResourceStats.MEMORY)); + CPU("cpu", task -> task.getTotalResourceUtilization(ResourceStats.CPU), true), + MEMORY("memory", task -> task.getTotalResourceUtilization(ResourceStats.MEMORY), true); private final String name; private final Function getResourceUsage; + private final boolean statsEnabled; - ResourceType(String name, Function getResourceUsage) { + ResourceType(String name, Function getResourceUsage, boolean statsEnabled) { this.name = name; this.getResourceUsage = getResourceUsage; + this.statsEnabled = statsEnabled; } /** @@ -63,4 +65,8 @@ public String getName() { public long getResourceUsage(Task task) { return getResourceUsage.apply(task); } + + public boolean hasStatsEnabled() { + return statsEnabled; + } } diff --git a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupState.java b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupState.java index 44d2c13b58388..93cfcea697c43 100644 --- a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupState.java +++ b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupState.java @@ -9,7 +9,7 @@ package org.opensearch.wlm.stats; import org.opensearch.common.metrics.CounterMetric; -import org.opensearch.search.ResourceType; +import org.opensearch.wlm.ResourceType; import java.util.EnumMap; import java.util.Map; @@ -45,8 +45,11 @@ public class QueryGroupState { public QueryGroupState() { resourceState = new EnumMap<>(ResourceType.class); - resourceState.put(ResourceType.CPU, new ResourceTypeState(ResourceType.CPU)); - resourceState.put(ResourceType.MEMORY, new ResourceTypeState(ResourceType.MEMORY)); + for (ResourceType resourceType : ResourceType.values()) { + if (resourceType.hasStatsEnabled()) { + resourceState.put(resourceType, new ResourceTypeState(resourceType)); + } + } } /** diff --git a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java index 6b9fef979c53e..6a7da53f4a986 100644 --- a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java +++ b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java @@ -13,7 +13,7 @@ import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.search.ResourceType; +import org.opensearch.wlm.ResourceType; import java.io.IOException; import java.util.Map; diff --git a/server/src/main/java/org/opensearch/wlm/tracker/QueryGroupResourceUsageTrackerService.java b/server/src/main/java/org/opensearch/wlm/tracker/QueryGroupResourceUsageTrackerService.java index bfbf5d8a452d1..15852b5bbe6a8 100644 --- a/server/src/main/java/org/opensearch/wlm/tracker/QueryGroupResourceUsageTrackerService.java +++ b/server/src/main/java/org/opensearch/wlm/tracker/QueryGroupResourceUsageTrackerService.java @@ -8,11 +8,11 @@ package org.opensearch.wlm.tracker; -import org.opensearch.search.ResourceType; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskResourceTrackingService; import org.opensearch.wlm.QueryGroupLevelResourceUsageView; import org.opensearch.wlm.QueryGroupTask; +import org.opensearch.wlm.ResourceType; import java.util.EnumMap; import java.util.EnumSet; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/QueryGroupMetadataTests.java b/server/src/test/java/org/opensearch/cluster/metadata/QueryGroupMetadataTests.java index 06734b8e0bac2..f5e667de73d93 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/QueryGroupMetadataTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/QueryGroupMetadataTests.java @@ -14,8 +14,8 @@ import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.search.ResourceType; import org.opensearch.test.AbstractDiffableSerializationTestCase; +import org.opensearch.wlm.ResourceType; import java.io.IOException; import java.util.Collections; diff --git a/server/src/test/java/org/opensearch/cluster/metadata/QueryGroupTests.java b/server/src/test/java/org/opensearch/cluster/metadata/QueryGroupTests.java index 884b364fb26b8..f4d3e5ceb1784 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/QueryGroupTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/QueryGroupTests.java @@ -14,8 +14,8 @@ import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.search.ResourceType; import org.opensearch.test.AbstractSerializingTestCase; +import org.opensearch.wlm.ResourceType; import org.joda.time.Instant; import java.io.IOException; diff --git a/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java b/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java index 15d0fcd10d701..a444eb42eac2e 100644 --- a/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java +++ b/server/src/test/java/org/opensearch/search/backpressure/SearchBackpressureServiceTests.java @@ -16,7 +16,6 @@ import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.search.ResourceType; import org.opensearch.search.backpressure.settings.SearchBackpressureMode; import org.opensearch.search.backpressure.settings.SearchBackpressureSettings; import org.opensearch.search.backpressure.settings.SearchShardTaskSettings; @@ -40,6 +39,7 @@ import org.opensearch.test.transport.MockTransportService; import org.opensearch.threadpool.TestThreadPool; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.wlm.ResourceType; import org.junit.After; import org.junit.Before; @@ -56,9 +56,9 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.LongSupplier; -import static org.opensearch.search.ResourceType.CPU; -import static org.opensearch.search.ResourceType.MEMORY; import static org.opensearch.search.backpressure.SearchBackpressureTestHelpers.createMockTaskWithResourceStats; +import static org.opensearch.wlm.ResourceType.CPU; +import static org.opensearch.wlm.ResourceType.MEMORY; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.any; diff --git a/server/src/test/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackersTests.java b/server/src/test/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackersTests.java index 801576bdf89d4..7c52840c099d4 100644 --- a/server/src/test/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackersTests.java +++ b/server/src/test/java/org/opensearch/search/backpressure/trackers/NodeDuressTrackersTests.java @@ -8,9 +8,9 @@ package org.opensearch.search.backpressure.trackers; -import org.opensearch.search.ResourceType; import org.opensearch.search.backpressure.trackers.NodeDuressTrackers.NodeDuressTracker; import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.wlm.ResourceType; import java.util.EnumMap; diff --git a/server/src/test/java/org/opensearch/wlm/QueryGroupLevelResourceUsageViewTests.java b/server/src/test/java/org/opensearch/wlm/QueryGroupLevelResourceUsageViewTests.java index 7f6419505fec2..532bf3de95bd6 100644 --- a/server/src/test/java/org/opensearch/wlm/QueryGroupLevelResourceUsageViewTests.java +++ b/server/src/test/java/org/opensearch/wlm/QueryGroupLevelResourceUsageViewTests.java @@ -10,7 +10,6 @@ import org.opensearch.action.search.SearchAction; import org.opensearch.core.tasks.TaskId; -import org.opensearch.search.ResourceType; import org.opensearch.tasks.Task; import org.opensearch.test.OpenSearchTestCase; diff --git a/server/src/test/java/org/opensearch/search/ResourceTypeTests.java b/server/src/test/java/org/opensearch/wlm/ResourceTypeTests.java similarity index 96% rename from server/src/test/java/org/opensearch/search/ResourceTypeTests.java rename to server/src/test/java/org/opensearch/wlm/ResourceTypeTests.java index 78827b8b1bdad..737cbb37b554c 100644 --- a/server/src/test/java/org/opensearch/search/ResourceTypeTests.java +++ b/server/src/test/java/org/opensearch/wlm/ResourceTypeTests.java @@ -6,14 +6,13 @@ * compatible open source license. */ -package org.opensearch.search; +package org.opensearch.wlm; import org.opensearch.action.search.SearchShardTask; import org.opensearch.core.tasks.resourcetracker.ResourceStats; import org.opensearch.tasks.CancellableTask; import org.opensearch.test.OpenSearchTestCase; -import static org.junit.Assert.assertThrows; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; diff --git a/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStateTests.java b/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStateTests.java index 76f4b787a34f4..576eec7be1888 100644 --- a/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStateTests.java +++ b/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStateTests.java @@ -8,8 +8,8 @@ package org.opensearch.wlm.stats; -import org.opensearch.search.ResourceType; import org.opensearch.test.OpenSearchTestCase; +import org.opensearch.wlm.ResourceType; import java.util.ArrayList; import java.util.List; diff --git a/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java b/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java index c114c811bbe07..661c3a7beae40 100644 --- a/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java +++ b/server/src/test/java/org/opensearch/wlm/stats/QueryGroupStatsTests.java @@ -12,8 +12,8 @@ import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.search.ResourceType; import org.opensearch.test.AbstractWireSerializingTestCase; +import org.opensearch.wlm.ResourceType; import java.io.IOException; import java.util.HashMap; diff --git a/server/src/test/java/org/opensearch/wlm/tracker/QueryGroupResourceUsageTrackerServiceTests.java b/server/src/test/java/org/opensearch/wlm/tracker/QueryGroupResourceUsageTrackerServiceTests.java index 967119583c25f..ca2891cb532f2 100644 --- a/server/src/test/java/org/opensearch/wlm/tracker/QueryGroupResourceUsageTrackerServiceTests.java +++ b/server/src/test/java/org/opensearch/wlm/tracker/QueryGroupResourceUsageTrackerServiceTests.java @@ -12,7 +12,6 @@ import org.opensearch.action.search.SearchTask; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.tasks.resourcetracker.ResourceStats; -import org.opensearch.search.ResourceType; import org.opensearch.tasks.CancellableTask; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskResourceTrackingService; @@ -21,6 +20,7 @@ import org.opensearch.threadpool.ThreadPool; import org.opensearch.wlm.QueryGroupLevelResourceUsageView; import org.opensearch.wlm.QueryGroupTask; +import org.opensearch.wlm.ResourceType; import org.junit.After; import org.junit.Before; From 4f6ee7af4ff6d4fa5fde7ce22ea553484d890e05 Mon Sep 17 00:00:00 2001 From: Kaushal Kumar Date: Tue, 27 Aug 2024 18:27:14 -0700 Subject: [PATCH 8/8] update the comment for query group stats Signed-off-by: Kaushal Kumar --- .../main/java/org/opensearch/wlm/stats/QueryGroupStats.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java index 6a7da53f4a986..d39bf104332da 100644 --- a/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java +++ b/server/src/main/java/org/opensearch/wlm/stats/QueryGroupStats.java @@ -23,11 +23,11 @@ * { * "queryGroupID": { * "completions": 1233234234, - * "rejections": 1243545, + * "rejections": 12, * "failures": 97, * "total_cancellations": 474, - * "CPU": { "current_usage": 49.6, "cancellation": 432 }, - * "MEMORY": { "current_usage": 39.6, "cancellation": 42 } + * "CPU": { "current_usage": 49.6, "cancellation": 432, "rejections": 8 }, + * "MEMORY": { "current_usage": 39.6, "cancellation": 42, "rejections": 4 } * }, * ... * ...