diff --git a/docs/aggregations.asciidoc b/docs/aggregations.asciidoc index 0d4fd0098cb..322d5bb1843 100644 --- a/docs/aggregations.asciidoc +++ b/docs/aggregations.asciidoc @@ -277,6 +277,8 @@ There are many different types of pipeline aggregation, each computing different * <> +* <> + * <> * <> @@ -323,6 +325,8 @@ include::aggregations/pipeline/moving-average/moving-average-simple-aggregation- include::aggregations/pipeline/moving-function/moving-function-aggregation-usage.asciidoc[] +include::aggregations/pipeline/moving-percentiles/moving-percentiles-aggregation-usage.asciidoc[] + include::aggregations/pipeline/normalize/normalize-aggregation-usage.asciidoc[] include::aggregations/pipeline/percentiles-bucket/percentiles-bucket-aggregation-usage.asciidoc[] diff --git a/docs/aggregations/pipeline/moving-percentiles/moving-percentiles-aggregation-usage.asciidoc b/docs/aggregations/pipeline/moving-percentiles/moving-percentiles-aggregation-usage.asciidoc new file mode 100644 index 00000000000..fa45aeea67d --- /dev/null +++ b/docs/aggregations/pipeline/moving-percentiles/moving-percentiles-aggregation-usage.asciidoc @@ -0,0 +1,110 @@ +:ref_current: https://www.elastic.co/guide/en/elasticsearch/reference/master + +:github: https://github.com/elastic/elasticsearch-net + +:nuget: https://www.nuget.org/packages + +//// +IMPORTANT NOTE +============== +This file has been generated from https://github.com/elastic/elasticsearch-net/tree/master/src/Tests/Tests/Aggregations/Pipeline/MovingPercentiles/MovingPercentilesAggregationUsageTests.cs. +If you wish to submit a PR for any spelling mistakes, typos or grammatical errors for this file, +please modify the original csharp file found at the link and submit the PR with that change. Thanks! +//// + +[[moving-percentiles-aggregation-usage]] +=== Moving Percentiles Aggregation Usage + +Given an ordered series of percentiles, the Moving Percentile aggregation will slide a window across those +percentiles and allow the user to compute the cumulative percentile. + +This is conceptually very similar to the Moving Function pipeline aggregation, except it works on the percentiles sketches instead of the actual buckets values. + +NOTE: Available in Elasticsearch 7.9.0+ with at least basic license level + +==== Fluent DSL example + +[source,csharp] +---- +a => a +.DateHistogram("projects_started_per_month", dh => dh + .Field(p => p.StartedOn) + .CalendarInterval(DateInterval.Month) + .MinimumDocumentCount(0) + .Aggregations(aa => aa + .Percentiles("percentiles", sm => sm + .Field(p => p.NumberOfCommits) + ) + .MovingPercentiles("moving_percentiles", mv => mv + .BucketsPath("percentiles") + .Window(10) + ) + ) +) +---- + +==== Object Initializer syntax example + +[source,csharp] +---- +new DateHistogramAggregation("projects_started_per_month") +{ + Field = "startedOn", + CalendarInterval = DateInterval.Month, + MinimumDocumentCount = 0, + Aggregations = + new PercentilesAggregation("percentiles", "numberOfCommits") + && new MovingPercentilesAggregation("moving_percentiles", "percentiles") + { + Window = 10 + } +} +---- + +[source,javascript] +.Example json output +---- +{ + "projects_started_per_month": { + "date_histogram": { + "field": "startedOn", + "calendar_interval": "month", + "min_doc_count": 0 + }, + "aggs": { + "percentiles": { + "percentiles": { + "field": "numberOfCommits" + } + }, + "moving_percentiles": { + "moving_percentiles": { + "buckets_path": "percentiles", + "window": 10 + } + } + } + } +} +---- + +==== Handling Responses + +[source,csharp] +---- +response.ShouldBeValid(); + +var projectsPerMonth = response.Aggregations.DateHistogram("projects_started_per_month"); +projectsPerMonth.Should().NotBeNull(); +projectsPerMonth.Buckets.Should().NotBeNull(); +projectsPerMonth.Buckets.Count.Should().BeGreaterThan(0); + +// percentiles not calculated for the first bucket +foreach (var item in projectsPerMonth.Buckets.Skip(1)) +{ + var movingPercentiles = item.MovingPercentiles("moving_percentiles"); + movingPercentiles.Should().NotBeNull(); + movingPercentiles.Items.Should().NotBeNull(); +} +---- + diff --git a/src/Nest/Aggregations/AggregateDictionary.cs b/src/Nest/Aggregations/AggregateDictionary.cs index 88e27c919a3..d9c09746ce3 100644 --- a/src/Nest/Aggregations/AggregateDictionary.cs +++ b/src/Nest/Aggregations/AggregateDictionary.cs @@ -98,6 +98,8 @@ public ScriptedMetricAggregate ScriptedMetric(string key) public PercentilesAggregate PercentilesBucket(string key) => TryGet(key); + public PercentilesAggregate MovingPercentiles(string key) => TryGet(key); + public PercentilesAggregate PercentileRanks(string key) => TryGet(key); public TopHitsAggregate TopHits(string key) => TryGet(key); diff --git a/src/Nest/Aggregations/AggregationContainer.cs b/src/Nest/Aggregations/AggregationContainer.cs index 94691baf6e1..9c984e38359 100644 --- a/src/Nest/Aggregations/AggregationContainer.cs +++ b/src/Nest/Aggregations/AggregationContainer.cs @@ -199,6 +199,10 @@ public interface IAggregationContainer [DataMember(Name = "moving_fn")] IMovingFunctionAggregation MovingFunction { get; set; } + /// + [DataMember(Name = "moving_percentiles")] + IMovingPercentilesAggregation MovingPercentiles { get; set; } + [DataMember(Name = "nested")] INestedAggregation Nested { get; set; } @@ -358,6 +362,9 @@ public class AggregationContainer : IAggregationContainer public IMovingFunctionAggregation MovingFunction { get; set; } + /// + public IMovingPercentilesAggregation MovingPercentiles { get; set; } + public INestedAggregation Nested { get; set; } /// @@ -518,6 +525,8 @@ public class AggregationContainerDescriptor : DescriptorBase selector ) => _SetInnerAggregation(name, selector, (a, d) => a.MovingFunction = d); + public AggregationContainerDescriptor MovingPercentiles(string name, + Func selector + ) => + _SetInnerAggregation(name, selector, (a, d) => a.MovingPercentiles = d); + public AggregationContainerDescriptor CumulativeSum(string name, Func selector ) => diff --git a/src/Nest/Aggregations/Pipeline/MovingPercentiles/MovingPercentilesAggregation.cs b/src/Nest/Aggregations/Pipeline/MovingPercentiles/MovingPercentilesAggregation.cs new file mode 100644 index 00000000000..56530126e13 --- /dev/null +++ b/src/Nest/Aggregations/Pipeline/MovingPercentiles/MovingPercentilesAggregation.cs @@ -0,0 +1,61 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Runtime.Serialization; +using Elasticsearch.Net.Utf8Json; + +namespace Nest +{ + /// + /// the Moving Percentile aggregation will slide a window across those percentiles and allow the user to compute the cumulative percentile. + /// + [InterfaceDataContract] + [ReadAs(typeof(MovingPercentilesAggregation))] + public interface IMovingPercentilesAggregation : IPipelineAggregation + { + /// + /// The size of window to "slide" across the histogram. + /// + [DataMember(Name ="window")] + int? Window { get; set; } + + /// + /// Shift of window position. + /// + [DataMember(Name ="shift")] + int? Shift { get; set; } + } + + /// + public class MovingPercentilesAggregation + : PipelineAggregationBase, IMovingPercentilesAggregation + { + internal MovingPercentilesAggregation() { } + + public MovingPercentilesAggregation(string name, SingleBucketsPath bucketsPath) + : base(name, bucketsPath) { } + + /// + public int? Window { get; set; } + + /// + public int? Shift { get; set; } + + internal override void WrapInContainer(AggregationContainer c) => c.MovingPercentiles = this; + } + + public class MovingPercentilesAggregationDescriptor + : PipelineAggregationDescriptorBase + , IMovingPercentilesAggregation + { + int? IMovingPercentilesAggregation.Window { get; set; } + int? IMovingPercentilesAggregation.Shift { get; set; } + + /// + public MovingPercentilesAggregationDescriptor Window(int? windowSize) => Assign(windowSize, (a, v) => a.Window = v); + + /// + public MovingPercentilesAggregationDescriptor Shift(int? shift) => Assign(shift, (a, v) => a.Shift = v); + } +} diff --git a/src/Nest/Aggregations/Visitor/AggregationVisitor.cs b/src/Nest/Aggregations/Visitor/AggregationVisitor.cs index 7580c22fef8..8154e2c20cf 100644 --- a/src/Nest/Aggregations/Visitor/AggregationVisitor.cs +++ b/src/Nest/Aggregations/Visitor/AggregationVisitor.cs @@ -114,6 +114,8 @@ public interface IAggregationVisitor void Visit(IMovingAverageAggregation aggregation); + void Visit(IMovingPercentilesAggregation aggregation); + void Visit(ICumulativeSumAggregation aggregation); void Visit(ICumulativeCardinalityAggregation aggregation); @@ -219,6 +221,8 @@ public virtual void Visit(ISerialDifferencingAggregation aggregation) { } public virtual void Visit(IMovingAverageAggregation aggregation) { } + public virtual void Visit(IMovingPercentilesAggregation aggregation) { } + public virtual void Visit(IMinBucketAggregation aggregation) { } public virtual void Visit(IDerivativeAggregation aggregation) { } diff --git a/src/Nest/Aggregations/Visitor/AggregationWalker.cs b/src/Nest/Aggregations/Visitor/AggregationWalker.cs index ea57de3b90e..621771ed962 100644 --- a/src/Nest/Aggregations/Visitor/AggregationWalker.cs +++ b/src/Nest/Aggregations/Visitor/AggregationWalker.cs @@ -117,6 +117,7 @@ public void Walk(IAggregationContainer aggregation, IAggregationVisitor visitor) Accept(v, d.Aggregations); }); AcceptAggregation(aggregation.MovingAverage, visitor, (v, d) => v.Visit(d)); + AcceptAggregation(aggregation.MovingPercentiles, visitor, (v, d) => v.Visit(d)); AcceptAggregation(aggregation.Nested, visitor, (v, d) => { v.Visit(d); diff --git a/tests/Tests/Aggregations/Pipeline/MovingPercentiles/MovingPercentilesAggregationUsageTests.cs b/tests/Tests/Aggregations/Pipeline/MovingPercentiles/MovingPercentilesAggregationUsageTests.cs new file mode 100644 index 00000000000..92e34d911bd --- /dev/null +++ b/tests/Tests/Aggregations/Pipeline/MovingPercentiles/MovingPercentilesAggregationUsageTests.cs @@ -0,0 +1,109 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System; +using System.Linq; +using Elastic.Elasticsearch.Xunit.XunitPlumbing; +using FluentAssertions; +using Nest; +using Tests.Core.Extensions; +using Tests.Core.ManagedElasticsearch.Clusters; +using Tests.Domain; +using Tests.Framework.EndpointTests.TestState; + +namespace Tests.Aggregations.Pipeline.MovingPercentiles +{ + /** + * Given an ordered series of percentiles, the Moving Percentile aggregation will slide a window across those + * percentiles and allow the user to compute the cumulative percentile. + * + * This is conceptually very similar to the Moving Function pipeline aggregation, except it works on the percentiles sketches instead of the actual buckets values. + * + * NOTE: Available in Elasticsearch 7.9.0+ with at least basic license level + */ + [SkipVersion("<7.9.0", "introduced in 7.9.0+")] + public class MovingPercentilesAggregationUsageTests : AggregationUsageTestBase + { + public MovingPercentilesAggregationUsageTests(ReadOnlyCluster cluster, EndpointUsage usage) : base(cluster, usage) { } + + protected override object AggregationJson => new + { + projects_started_per_month = new + { + date_histogram = new + { + field = "startedOn", + calendar_interval = "month", + min_doc_count = 0 + }, + aggs = new + { + percentiles = new + { + percentiles = new + { + field = "numberOfCommits" + } + }, + moving_percentiles = new + { + moving_percentiles = new + { + buckets_path = "percentiles", + window = 10, + } + } + } + } + }; + + protected override Func, IAggregationContainer> FluentAggs => a => a + .DateHistogram("projects_started_per_month", dh => dh + .Field(p => p.StartedOn) + .CalendarInterval(DateInterval.Month) + .MinimumDocumentCount(0) + .Aggregations(aa => aa + .Percentiles("percentiles", sm => sm + .Field(p => p.NumberOfCommits) + ) + .MovingPercentiles("moving_percentiles", mv => mv + .BucketsPath("percentiles") + .Window(10) + ) + ) + ); + + protected override AggregationDictionary InitializerAggs => + new DateHistogramAggregation("projects_started_per_month") + { + Field = "startedOn", + CalendarInterval = DateInterval.Month, + MinimumDocumentCount = 0, + Aggregations = + new PercentilesAggregation("percentiles", "numberOfCommits") + && new MovingPercentilesAggregation("moving_percentiles", "percentiles") + { + Window = 10 + } + }; + + protected override void ExpectResponse(ISearchResponse response) + { + response.ShouldBeValid(); + + var projectsPerMonth = response.Aggregations.DateHistogram("projects_started_per_month"); + projectsPerMonth.Should().NotBeNull(); + projectsPerMonth.Buckets.Should().NotBeNull(); + projectsPerMonth.Buckets.Count.Should().BeGreaterThan(0); + + // percentiles not calculated for the first bucket + foreach (var item in projectsPerMonth.Buckets.Skip(1)) + { + var movingPercentiles = item.MovingPercentiles("moving_percentiles"); + movingPercentiles.Should().NotBeNull(); + movingPercentiles.Items.Should().NotBeNull(); + } + } + } +}