From 4156cc3faefe3a1c0b0af589de349beb4ad4855f Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Wed, 5 Sep 2018 12:08:28 +0200 Subject: [PATCH 01/10] MINOR+CORE: Remove Dead Methods ClusterService (#33346) * None of these methods are used anywhere --- .../service/ClusterApplierService.java | 21 --------------- .../cluster/service/ClusterService.java | 26 ------------------- 2 files changed, 47 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java index 5dd36b9b1bc20..7272c9ed30200 100644 --- a/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java +++ b/server/src/main/java/org/elasticsearch/cluster/service/ClusterApplierService.java @@ -255,13 +255,6 @@ public void addLocalNodeMasterListener(LocalNodeMasterListener listener) { localNodeMasterListeners.add(listener); } - /** - * Remove the given listener for on/off local master events - */ - public void removeLocalNodeMasterListener(LocalNodeMasterListener listener) { - localNodeMasterListeners.remove(listener); - } - /** * Adds a cluster state listener that is expected to be removed during a short period of time. * If provided, the listener will be notified once a specific time has elapsed. @@ -349,13 +342,6 @@ private void submitStateUpdateTask(final String source, final ClusterStateTaskCo } } - /** asserts that the current thread is the cluster state update thread */ - public static boolean assertClusterStateUpdateThread() { - assert Thread.currentThread().getName().contains(ClusterApplierService.CLUSTER_UPDATE_THREAD_NAME) : - "not called from the cluster state update thread"; - return true; - } - /** asserts that the current thread is NOT the cluster state update thread */ public static boolean assertNotClusterStateUpdateThread(String reason) { assert Thread.currentThread().getName().contains(CLUSTER_UPDATE_THREAD_NAME) == false : @@ -607,13 +593,6 @@ private void add(LocalNodeMasterListener listener) { listeners.add(listener); } - private void remove(LocalNodeMasterListener listener) { - listeners.remove(listener); - } - - private void clear() { - listeners.clear(); - } } private static class OnMasterRunnable implements Runnable { diff --git a/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java b/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java index 5317a2c26c9ad..58cace44754de 100644 --- a/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java +++ b/server/src/main/java/org/elasticsearch/cluster/service/ClusterService.java @@ -28,10 +28,8 @@ import org.elasticsearch.cluster.ClusterStateTaskListener; import org.elasticsearch.cluster.LocalNodeMasterListener; import org.elasticsearch.cluster.NodeConnectionsService; -import org.elasticsearch.cluster.TimeoutClusterStateListener; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.routing.OperationRouting; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractLifecycleComponent; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Setting; @@ -167,13 +165,6 @@ public void removeListener(ClusterStateListener listener) { clusterApplierService.removeListener(listener); } - /** - * Removes a timeout listener for updated cluster states. - */ - public void removeTimeoutListener(TimeoutClusterStateListener listener) { - clusterApplierService.removeTimeoutListener(listener); - } - /** * Add a listener for on/off local node master events */ @@ -181,23 +172,6 @@ public void addLocalNodeMasterListener(LocalNodeMasterListener listener) { clusterApplierService.addLocalNodeMasterListener(listener); } - /** - * Remove the given listener for on/off local master events - */ - public void removeLocalNodeMasterListener(LocalNodeMasterListener listener) { - clusterApplierService.removeLocalNodeMasterListener(listener); - } - - /** - * Adds a cluster state listener that is expected to be removed during a short period of time. - * If provided, the listener will be notified once a specific time has elapsed. - * - * NOTE: the listener is not removed on timeout. This is the responsibility of the caller. - */ - public void addTimeoutListener(@Nullable final TimeValue timeout, final TimeoutClusterStateListener listener) { - clusterApplierService.addTimeoutListener(timeout, listener); - } - public MasterService getMasterService() { return masterService; } From c303006e6b79e96e643473e5bc77e901e4972ffc Mon Sep 17 00:00:00 2001 From: Paul Sanwald Date: Wed, 5 Sep 2018 07:35:59 -0400 Subject: [PATCH 02/10] Add interval response parameter to AutoDateInterval histogram (#33254) Adds the interval used to the aggregation response. --- .../autodatehistogram-aggregation.asciidoc | 9 ++++-- .../AutoDateHistogramAggregationBuilder.java | 18 ++++++----- .../AutoDateHistogramAggregator.java | 5 +-- .../histogram/InternalAutoDateHistogram.java | 31 ++++++++++++------- .../histogram/ParsedAutoDateHistogram.java | 21 +++++++++++++ .../InternalAutoDateHistogramTests.java | 18 ++++++++--- 6 files changed, 74 insertions(+), 28 deletions(-) diff --git a/docs/reference/aggregations/bucket/autodatehistogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/autodatehistogram-aggregation.asciidoc index 3bd430d03d5ac..e371674228bb4 100644 --- a/docs/reference/aggregations/bucket/autodatehistogram-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/autodatehistogram-aggregation.asciidoc @@ -81,7 +81,8 @@ Response: "key": 1425168000000, "doc_count": 2 } - ] + ], + "interval": "1M" } } } @@ -174,7 +175,8 @@ starting at midnight UTC on 1 October 2015: "key": 1443664800000, "doc_count": 1 } - ] + ], + "interval": "1h" } } } @@ -229,7 +231,8 @@ the specified time zone. "key": 1443664800000, "doc_count": 1 } - ] + ], + "interval": "1h" } } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java index b97670ddb5730..87ba80af9a4b0 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregationBuilder.java @@ -73,17 +73,17 @@ public class AutoDateHistogramAggregationBuilder static RoundingInfo[] buildRoundings(DateTimeZone timeZone) { RoundingInfo[] roundings = new RoundingInfo[6]; roundings[0] = new RoundingInfo(createRounding(DateTimeUnit.SECOND_OF_MINUTE, timeZone), - 1000L, 1, 5, 10, 30); + 1000L, "s" , 1, 5, 10, 30); roundings[1] = new RoundingInfo(createRounding(DateTimeUnit.MINUTES_OF_HOUR, timeZone), - 60 * 1000L, 1, 5, 10, 30); + 60 * 1000L, "m", 1, 5, 10, 30); roundings[2] = new RoundingInfo(createRounding(DateTimeUnit.HOUR_OF_DAY, timeZone), - 60 * 60 * 1000L, 1, 3, 12); + 60 * 60 * 1000L, "h", 1, 3, 12); roundings[3] = new RoundingInfo(createRounding(DateTimeUnit.DAY_OF_MONTH, timeZone), - 24 * 60 * 60 * 1000L, 1, 7); + 24 * 60 * 60 * 1000L, "d", 1, 7); roundings[4] = new RoundingInfo(createRounding(DateTimeUnit.MONTH_OF_YEAR, timeZone), - 30 * 24 * 60 * 60 * 1000L, 1, 3); + 30 * 24 * 60 * 60 * 1000L, "M", 1, 3); roundings[5] = new RoundingInfo(createRounding(DateTimeUnit.YEAR_OF_CENTURY, timeZone), - 365 * 24 * 60 * 60 * 1000L, 1, 5, 10, 20, 50, 100); + 365 * 24 * 60 * 60 * 1000L, "y", 1, 5, 10, 20, 50, 100); return roundings; } @@ -186,10 +186,12 @@ public static class RoundingInfo implements Writeable { final Rounding rounding; final int[] innerIntervals; final long roughEstimateDurationMillis; + final String unitAbbreviation; - public RoundingInfo(Rounding rounding, long roughEstimateDurationMillis, int... innerIntervals) { + public RoundingInfo(Rounding rounding, long roughEstimateDurationMillis, String unitAbbreviation, int... innerIntervals) { this.rounding = rounding; this.roughEstimateDurationMillis = roughEstimateDurationMillis; + this.unitAbbreviation = unitAbbreviation; this.innerIntervals = innerIntervals; } @@ -197,6 +199,7 @@ public RoundingInfo(StreamInput in) throws IOException { rounding = Rounding.Streams.read(in); roughEstimateDurationMillis = in.readVLong(); innerIntervals = in.readIntArray(); + unitAbbreviation = in.readString(); } @Override @@ -204,6 +207,7 @@ public void writeTo(StreamOutput out) throws IOException { Rounding.Streams.write(rounding, out); out.writeVLong(roughEstimateDurationMillis); out.writeIntArray(innerIntervals); + out.writeString(unitAbbreviation); } public int getMaximumInnerInterval() { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java index f86145386f1df..95376710373f8 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java @@ -181,7 +181,8 @@ public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOE InternalAutoDateHistogram.BucketInfo emptyBucketInfo = new InternalAutoDateHistogram.BucketInfo(roundingInfos, roundingIdx, buildEmptySubAggregations()); - return new InternalAutoDateHistogram(name, buckets, targetBuckets, emptyBucketInfo, formatter, pipelineAggregators(), metaData()); + return new InternalAutoDateHistogram(name, buckets, targetBuckets, emptyBucketInfo, + formatter, pipelineAggregators(), metaData(), 1); } @Override @@ -189,7 +190,7 @@ public InternalAggregation buildEmptyAggregation() { InternalAutoDateHistogram.BucketInfo emptyBucketInfo = new InternalAutoDateHistogram.BucketInfo(roundingInfos, roundingIdx, buildEmptySubAggregations()); return new InternalAutoDateHistogram(name, Collections.emptyList(), targetBuckets, emptyBucketInfo, formatter, - pipelineAggregators(), metaData()); + pipelineAggregators(), metaData(), 1); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogram.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogram.java index 7bc2b9a317838..f2e450942c3ad 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogram.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogram.java @@ -208,15 +208,16 @@ public int hashCode() { private final DocValueFormat format; private final BucketInfo bucketInfo; private final int targetBuckets; - + private long bucketInnerInterval; InternalAutoDateHistogram(String name, List buckets, int targetBuckets, BucketInfo emptyBucketInfo, DocValueFormat formatter, - List pipelineAggregators, Map metaData) { + List pipelineAggregators, Map metaData, long bucketInnerInterval) { super(name, pipelineAggregators, metaData); this.buckets = buckets; this.bucketInfo = emptyBucketInfo; this.format = formatter; this.targetBuckets = targetBuckets; + this.bucketInnerInterval = bucketInnerInterval; } /** @@ -238,6 +239,13 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeVInt(targetBuckets); } + public DateHistogramInterval getInterval() { + + RoundingInfo roundingInfo = this.bucketInfo.roundingInfos[this.bucketInfo.roundingIdx]; + String unitAbbreviation = roundingInfo.unitAbbreviation; + return new DateHistogramInterval(Long.toString(bucketInnerInterval) + unitAbbreviation); + } + @Override public String getWriteableName() { return AutoDateHistogramAggregationBuilder.NAME; @@ -262,7 +270,7 @@ public BucketInfo getBucketInfo() { @Override public InternalAutoDateHistogram create(List buckets) { - return new InternalAutoDateHistogram(name, buckets, targetBuckets, bucketInfo, format, pipelineAggregators(), metaData); + return new InternalAutoDateHistogram(name, buckets, targetBuckets, bucketInfo, format, pipelineAggregators(), metaData, 1); } @Override @@ -279,7 +287,6 @@ private static class IteratorAndCurrent { this.iterator = iterator; current = iterator.next(); } - } /** @@ -365,7 +372,7 @@ private BucketReduceResult mergeBucketsIfNeeded(List reducedBuckets, int reduceRoundingInfo = bucketInfo.roundingInfos[reduceRoundingIdx]; reducedBuckets = mergeBuckets(reducedBuckets, reduceRoundingInfo.rounding, reduceContext); } - return new BucketReduceResult(reducedBuckets, reduceRoundingInfo, reduceRoundingIdx); + return new BucketReduceResult(reducedBuckets, reduceRoundingInfo, reduceRoundingIdx, 1); } private List mergeBuckets(List reducedBuckets, Rounding reduceRounding, ReduceContext reduceContext) { @@ -403,12 +410,13 @@ private static class BucketReduceResult { List buckets; RoundingInfo roundingInfo; int roundingIdx; + long innerInterval; - BucketReduceResult(List buckets, RoundingInfo roundingInfo, int roundingIdx) { + BucketReduceResult(List buckets, RoundingInfo roundingInfo, int roundingIdx, long innerInterval) { this.buckets = buckets; this.roundingInfo = roundingInfo; this.roundingIdx = roundingIdx; - + this.innerInterval = innerInterval; } } @@ -444,7 +452,7 @@ private BucketReduceResult addEmptyBuckets(BucketReduceResult currentResult, Red } lastBucket = iter.next(); } - return new BucketReduceResult(list, roundingInfo, roundingIdx); + return new BucketReduceResult(list, roundingInfo, roundingIdx, currentResult.innerInterval); } static int getAppropriateRounding(long minKey, long maxKey, int roundingIdx, @@ -507,7 +515,7 @@ public InternalAggregation doReduce(List aggregations, Redu this.bucketInfo.emptySubAggregations); return new InternalAutoDateHistogram(getName(), reducedBucketsResult.buckets, targetBuckets, bucketInfo, format, - pipelineAggregators(), getMetaData()); + pipelineAggregators(), getMetaData(), reducedBucketsResult.innerInterval); } private BucketReduceResult maybeMergeConsecutiveBuckets(BucketReduceResult reducedBucketsResult, @@ -547,7 +555,7 @@ private BucketReduceResult mergeConsecutiveBuckets(List reducedBuckets, reduceContext.consumeBucketsAndMaybeBreak(1); mergedBuckets.add(sameKeyedBuckets.get(0).reduce(sameKeyedBuckets, roundingInfo.rounding, reduceContext)); } - return new BucketReduceResult(mergedBuckets, roundingInfo, roundingIdx); + return new BucketReduceResult(mergedBuckets, roundingInfo, roundingIdx, mergeInterval); } @Override @@ -557,6 +565,7 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th bucket.toXContent(builder, params); } builder.endArray(); + builder.field("interval", getInterval().toString()); return builder; } @@ -580,7 +589,7 @@ public InternalAggregation createAggregation(List getBuckets() { return buckets; @@ -47,6 +58,8 @@ public List getBuckets() { declareMultiBucketAggregationFields(PARSER, parser -> ParsedBucket.fromXContent(parser, false), parser -> ParsedBucket.fromXContent(parser, true)); + PARSER.declareString((parsed, value) -> parsed.interval = value, + new ParseField("interval")); } public static ParsedAutoDateHistogram fromXContent(XContentParser parser, String name) throws IOException { @@ -55,6 +68,14 @@ public static ParsedAutoDateHistogram fromXContent(XContentParser parser, String return aggregation; } + @Override + protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + builder = super.doXContentBody(builder, params); + builder.field("interval", getInterval()); + return builder; + } + + public static class ParsedBucket extends ParsedMultiBucketAggregation.ParsedBucket implements Histogram.Bucket { private Long key; diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java index b7c5bf03ac552..dd3425c20f43c 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/histogram/InternalAutoDateHistogramTests.java @@ -80,7 +80,7 @@ protected InternalAutoDateHistogram createTestInstance(String name, } InternalAggregations subAggregations = new InternalAggregations(Collections.emptyList()); BucketInfo bucketInfo = new BucketInfo(roundingInfos, randomIntBetween(0, roundingInfos.length - 1), subAggregations); - return new InternalAutoDateHistogram(name, buckets, targetBuckets, bucketInfo, format, pipelineAggregators, metaData); + return new InternalAutoDateHistogram(name, buckets, targetBuckets, bucketInfo, format, pipelineAggregators, metaData, 1); } /* @@ -94,11 +94,11 @@ public void testGetAppropriateRoundingUsesCorrectIntervals() { // an innerInterval that is quite large, such that targetBuckets * roundings[i].getMaximumInnerInterval() // will be larger than the estimate. roundings[0] = new RoundingInfo(createRounding(DateTimeUnit.SECOND_OF_MINUTE, timeZone), - 1000L, 1000); + 1000L, "s", 1000); roundings[1] = new RoundingInfo(createRounding(DateTimeUnit.MINUTES_OF_HOUR, timeZone), - 60 * 1000L, 1, 5, 10, 30); + 60 * 1000L, "m", 1, 5, 10, 30); roundings[2] = new RoundingInfo(createRounding(DateTimeUnit.HOUR_OF_DAY, timeZone), - 60 * 60 * 1000L, 1, 3, 12); + 60 * 60 * 1000L, "h", 1, 3, 12); OffsetDateTime timestamp = Instant.parse("2018-01-01T00:00:01.000Z").atOffset(ZoneOffset.UTC); // We want to pass a roundingIdx of zero, because in order to reproduce this bug, we need the function @@ -198,6 +198,14 @@ protected void assertReduced(InternalAutoDateHistogram reduced, List (oldValue == null ? 0 : oldValue) + bucket.getDocCount()); } assertEquals(expectedCounts, actualCounts); + + DateHistogramInterval expectedInterval; + if (reduced.getBuckets().size() == 1) { + expectedInterval = reduced.getInterval(); + } else { + expectedInterval = new DateHistogramInterval(innerIntervalToUse+roundingInfo.unitAbbreviation); + } + assertThat(reduced.getInterval(), equalTo(expectedInterval)); } private int getBucketCount(long lowest, long highest, RoundingInfo roundingInfo, long intervalInMillis) { @@ -252,6 +260,6 @@ protected InternalAutoDateHistogram mutateInstance(InternalAutoDateHistogram ins default: throw new AssertionError("Illegal randomisation branch"); } - return new InternalAutoDateHistogram(name, buckets, targetBuckets, bucketInfo, format, pipelineAggregators, metaData); + return new InternalAutoDateHistogram(name, buckets, targetBuckets, bucketInfo, format, pipelineAggregators, metaData, 1); } } From a2968292053a8b3497c1ce384166b8bd33afe12f Mon Sep 17 00:00:00 2001 From: David Roberts Date: Wed, 5 Sep 2018 12:57:20 +0100 Subject: [PATCH 03/10] [ML] Add field stats to log structure finder (#33351) The log structure endpoint will return these in addition to pure structure information so that it can be used to drive pre-import data visualizer functionality. The statistics for every field are count, cardinality (distinct count) and top hits (most common values). Extra statistics are calculated if the field is numeric: min, max, mean and median. --- .../DelimitedLogStructureFinder.java | 9 +- .../ml/logstructurefinder/FieldStats.java | 147 ++++++++++++ .../FieldStatsCalculator.java | 182 +++++++++++++++ .../GrokPatternCreator.java | 42 ++-- .../JsonLogStructureFinder.java | 9 +- .../ml/logstructurefinder/LogStructure.java | 35 ++- .../logstructurefinder/LogStructureUtils.java | 58 +++-- .../TextLogStructureFinder.java | 5 +- .../XmlLogStructureFinder.java | 9 +- .../FieldStatsCalculatorTests.java | 218 ++++++++++++++++++ .../logstructurefinder/FieldStatsTests.java | 61 +++++ .../GrokPatternCreatorTests.java | 34 +-- .../logstructurefinder/LogStructureTests.java | 8 + .../LogStructureUtilsTests.java | 83 ++++--- 14 files changed, 815 insertions(+), 85 deletions(-) create mode 100644 x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/FieldStats.java create mode 100644 x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/FieldStatsCalculator.java create mode 100644 x-pack/plugin/ml/log-structure-finder/src/test/java/org/elasticsearch/xpack/ml/logstructurefinder/FieldStatsCalculatorTests.java create mode 100644 x-pack/plugin/ml/log-structure-finder/src/test/java/org/elasticsearch/xpack/ml/logstructurefinder/FieldStatsTests.java diff --git a/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/DelimitedLogStructureFinder.java b/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/DelimitedLogStructureFinder.java index 2f7bb41d0bae7..de010196808d4 100644 --- a/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/DelimitedLogStructureFinder.java +++ b/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/DelimitedLogStructureFinder.java @@ -123,9 +123,16 @@ static DelimitedLogStructureFinder makeDelimitedLogStructureFinder(List .setMultilineStartPattern(timeLineRegex); } - SortedMap mappings = LogStructureUtils.guessMappings(explanation, sampleRecords); + Tuple, SortedMap> mappingsAndFieldStats = + LogStructureUtils.guessMappingsAndCalculateFieldStats(explanation, sampleRecords); + + SortedMap mappings = mappingsAndFieldStats.v1(); mappings.put(LogStructureUtils.DEFAULT_TIMESTAMP_FIELD, Collections.singletonMap(LogStructureUtils.MAPPING_TYPE_SETTING, "date")); + if (mappingsAndFieldStats.v2() != null) { + structureBuilder.setFieldStats(mappingsAndFieldStats.v2()); + } + LogStructure structure = structureBuilder .setMappings(mappings) .setExplanation(explanation) diff --git a/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/FieldStats.java b/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/FieldStats.java new file mode 100644 index 0000000000000..8e8401123aa9f --- /dev/null +++ b/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/FieldStats.java @@ -0,0 +1,147 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.logstructurefinder; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class FieldStats implements ToXContentObject { + + static final ParseField COUNT = new ParseField("count"); + static final ParseField CARDINALITY = new ParseField("cardinality"); + static final ParseField MIN_VALUE = new ParseField("min_value"); + static final ParseField MAX_VALUE = new ParseField("max_value"); + static final ParseField MEAN_VALUE = new ParseField("mean_value"); + static final ParseField MEDIAN_VALUE = new ParseField("median_value"); + static final ParseField TOP_HITS = new ParseField("top_hits"); + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("field_stats", false, + a -> new FieldStats((long) a[0], (int) a[1], (Double) a[2], (Double) a[3], (Double) a[4], (Double) a[5], + (List>) a[6])); + + static { + PARSER.declareLong(ConstructingObjectParser.constructorArg(), COUNT); + PARSER.declareInt(ConstructingObjectParser.constructorArg(), CARDINALITY); + PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), MIN_VALUE); + PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), MAX_VALUE); + PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), MEAN_VALUE); + PARSER.declareDouble(ConstructingObjectParser.optionalConstructorArg(), MEDIAN_VALUE); + PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.mapOrdered(), TOP_HITS); + } + + private final long count; + private final int cardinality; + private final Double minValue; + private final Double maxValue; + private final Double meanValue; + private final Double medianValue; + private final List> topHits; + + FieldStats(long count, int cardinality, List> topHits) { + this(count, cardinality, null, null, null, null, topHits); + } + + FieldStats(long count, int cardinality, Double minValue, Double maxValue, Double meanValue, Double medianValue, + List> topHits) { + this.count = count; + this.cardinality = cardinality; + this.minValue = minValue; + this.maxValue = maxValue; + this.meanValue = meanValue; + this.medianValue = medianValue; + this.topHits = (topHits == null) ? Collections.emptyList() : Collections.unmodifiableList(topHits); + } + + public long getCount() { + return count; + } + + public int getCardinality() { + return cardinality; + } + + public Double getMinValue() { + return minValue; + } + + public Double getMaxValue() { + return maxValue; + } + + public Double getMeanValue() { + return meanValue; + } + + public Double getMedianValue() { + return medianValue; + } + + public List> getTopHits() { + return topHits; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + + builder.startObject(); + builder.field(COUNT.getPreferredName(), count); + builder.field(CARDINALITY.getPreferredName(), cardinality); + if (minValue != null) { + builder.field(MIN_VALUE.getPreferredName(), minValue); + } + if (maxValue != null) { + builder.field(MAX_VALUE.getPreferredName(), maxValue); + } + if (meanValue != null) { + builder.field(MEAN_VALUE.getPreferredName(), meanValue); + } + if (medianValue != null) { + builder.field(MEDIAN_VALUE.getPreferredName(), medianValue); + } + if (topHits.isEmpty() == false) { + builder.field(TOP_HITS.getPreferredName(), topHits); + } + builder.endObject(); + + return builder; + } + + @Override + public int hashCode() { + + return Objects.hash(count, cardinality, minValue, maxValue, meanValue, medianValue, topHits); + } + + @Override + public boolean equals(Object other) { + + if (this == other) { + return true; + } + + if (other == null || getClass() != other.getClass()) { + return false; + } + + FieldStats that = (FieldStats) other; + return this.count == that.count && + this.cardinality == that.cardinality && + Objects.equals(this.minValue, that.minValue) && + Objects.equals(this.maxValue, that.maxValue) && + Objects.equals(this.meanValue, that.meanValue) && + Objects.equals(this.medianValue, that.medianValue) && + Objects.equals(this.topHits, that.topHits); + } +} diff --git a/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/FieldStatsCalculator.java b/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/FieldStatsCalculator.java new file mode 100644 index 0000000000000..5f76e48f0c8b1 --- /dev/null +++ b/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/FieldStatsCalculator.java @@ -0,0 +1,182 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.logstructurefinder; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.stream.Collectors; + +/** + * Calculate statistics for a set of scalar field values. + * Count, cardinality (distinct count) and top hits (most common values) are always calculated. + * Extra statistics are calculated if the field is numeric: min, max, mean and median. + */ +public class FieldStatsCalculator { + + private long count; + private SortedMap countsByStringValue = new TreeMap<>(); + private SortedMap countsByNumericValue = new TreeMap<>(); + + /** + * Add a collection of values to the calculator. + * The values to be added can be combined by the caller and added in a + * single call to this method or added in multiple calls to this method. + * @param fieldValues Zero or more values to add. May not be null. + */ + public void accept(Collection fieldValues) { + + count += fieldValues.size(); + + for (String fieldValue : fieldValues) { + + countsByStringValue.compute(fieldValue, (k, v) -> (v == null) ? 1 : (1 + v)); + + if (countsByNumericValue != null) { + + try { + countsByNumericValue.compute(Double.valueOf(fieldValue), (k, v) -> (v == null) ? 1 : (1 + v)); + } catch (NumberFormatException e) { + countsByNumericValue = null; + } + } + } + } + + /** + * Calculate field statistics based on the previously accepted values. + * @param numTopHits The maximum number of entries to include in the top hits. + * @return The calculated field statistics. + */ + public FieldStats calculate(int numTopHits) { + + if (countsByNumericValue != null && countsByNumericValue.isEmpty() == false) { + return new FieldStats(count, countsByNumericValue.size(), countsByNumericValue.firstKey(), countsByNumericValue.lastKey(), + calculateMean(), calculateMedian(), findNumericTopHits(numTopHits)); + } else { + return new FieldStats(count, countsByStringValue.size(), findStringTopHits(numTopHits)); + } + } + + Double calculateMean() { + + assert countsByNumericValue != null; + + if (countsByNumericValue.isEmpty()) { + return null; + } + + double runningCount = 0.0; + double runningMean = Double.NaN; + + for (Map.Entry entry : countsByNumericValue.entrySet()) { + + double entryCount = (double) entry.getValue(); + double newRunningCount = runningCount + entryCount; + + // Updating a running mean like this is more numerically stable than using (sum / count) + if (runningCount > 0.0) { + runningMean = runningMean * (runningCount / newRunningCount) + entry.getKey() * (entryCount / newRunningCount); + } else if (entryCount > 0.0) { + runningMean = entry.getKey(); + } + + runningCount = newRunningCount; + } + + return runningMean; + } + + Double calculateMedian() { + + assert countsByNumericValue != null; + + if (count % 2 == 1) { + + // Simple case - median is middle value + long targetCount = count / 2 + 1; + long currentUpperBound = 0; + + for (Map.Entry entry : countsByNumericValue.entrySet()) { + + currentUpperBound += entry.getValue(); + + if (currentUpperBound >= targetCount) { + return entry.getKey(); + } + } + + } else { + + // More complicated case - median is average of two middle values + long target1Count = count / 2; + long target2Count = target1Count + 1; + double target1Value = Double.NaN; + long prevUpperBound = -1; + long currentUpperBound = 0; + + for (Map.Entry entry : countsByNumericValue.entrySet()) { + + currentUpperBound += entry.getValue(); + + if (currentUpperBound >= target2Count) { + + if (prevUpperBound < target1Count) { + // Both target values are the same + return entry.getKey(); + } else { + return (target1Value + entry.getKey()) / 2.0; + } + } + + if (currentUpperBound >= target1Count) { + target1Value = entry.getKey(); + } + + prevUpperBound = currentUpperBound; + } + } + + return null; + } + + List> findNumericTopHits(int numTopHits) { + assert countsByNumericValue != null; + return findTopHits(numTopHits, countsByNumericValue, Comparator.comparing(Map.Entry::getKey)); + } + + List> findStringTopHits(int numTopHits) { + return findTopHits(numTopHits, countsByStringValue, Comparator.comparing(Map.Entry::getKey)); + } + + /** + * Order by descending count, with a secondary sort to ensure reproducibility of results. + */ + private static List> findTopHits(int numTopHits, Map countsByValue, + Comparator> secondarySort) { + + List> sortedByCount = countsByValue.entrySet().stream() + .sorted(Comparator.comparing(Map.Entry::getValue, Comparator.reverseOrder()).thenComparing(secondarySort)) + .limit(numTopHits).collect(Collectors.toList()); + + List> topHits = new ArrayList<>(sortedByCount.size()); + + for (Map.Entry entry : sortedByCount) { + + Map topHit = new LinkedHashMap<>(3); + topHit.put("value", entry.getKey()); + topHit.put("count", entry.getValue()); + topHits.add(topHit); + } + + return topHits; + } +} diff --git a/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/GrokPatternCreator.java b/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/GrokPatternCreator.java index 186477507acce..b24e067b59d4b 100644 --- a/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/GrokPatternCreator.java +++ b/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/GrokPatternCreator.java @@ -119,6 +119,7 @@ public final class GrokPatternCreator { * Both this class and other classes will update it. */ private final Map mappings; + private final Map fieldStats; private final Map fieldNameCountStore = new HashMap<>(); private final StringBuilder overallGrokPatternBuilder = new StringBuilder(); @@ -128,22 +129,26 @@ public final class GrokPatternCreator { * can be appended by the methods of this class. * @param sampleMessages Sample messages that any Grok pattern found must match. * @param mappings Will be updated with mappings appropriate for the returned pattern, if non-null. + * @param fieldStats Will be updated with field stats for the fields in the returned pattern, if non-null. */ - public GrokPatternCreator(List explanation, Collection sampleMessages, Map mappings) { + public GrokPatternCreator(List explanation, Collection sampleMessages, Map mappings, + Map fieldStats) { this.explanation = explanation; this.sampleMessages = Collections.unmodifiableCollection(sampleMessages); this.mappings = mappings; + this.fieldStats = fieldStats; } /** * This method attempts to find a Grok pattern that will match all of the sample messages in their entirety. + * It will also update mappings and field stats if they are non-null. * @return A tuple of (time field name, Grok string), or null if no suitable Grok pattern was found. */ public Tuple findFullLineGrokPattern() { for (FullMatchGrokPatternCandidate candidate : FULL_MATCH_GROK_PATTERNS) { if (candidate.matchesAll(sampleMessages)) { - return candidate.processMatch(explanation, sampleMessages, mappings); + return candidate.processMatch(explanation, sampleMessages, mappings, fieldStats); } } @@ -186,7 +191,8 @@ private void processCandidateAndSplit(GrokPatternCandidate chosenPattern, boolea Collection prefaces = new ArrayList<>(); Collection epilogues = new ArrayList<>(); - String patternBuilderContent = chosenPattern.processCaptures(fieldNameCountStore, snippets, prefaces, epilogues, mappings); + String patternBuilderContent = + chosenPattern.processCaptures(fieldNameCountStore, snippets, prefaces, epilogues, mappings, fieldStats); appendBestGrokMatchForStrings(false, prefaces, ignoreKeyValueCandidateLeft, ignoreValueOnlyCandidatesLeft); overallGrokPatternBuilder.append(patternBuilderContent); appendBestGrokMatchForStrings(isLast, epilogues, ignoreKeyValueCandidateRight, ignoreValueOnlyCandidatesRight); @@ -375,11 +381,12 @@ interface GrokPatternCandidate { /** * After it has been determined that this Grok pattern candidate matches a collection of strings, * return collections of the bits that come before (prefaces) and after (epilogues) the bit - * that matches. Also update mappings with the most appropriate field name and type. + * that matches. Also update mappings with the most appropriate field name and type, and + * calculate field stats. * @return The string that needs to be incorporated into the overall Grok pattern for the line. */ String processCaptures(Map fieldNameCountStore, Collection snippets, Collection prefaces, - Collection epilogues, Map mappings); + Collection epilogues, Map mappings, Map fieldStats); } /** @@ -436,7 +443,7 @@ public boolean matchesAll(Collection snippets) { */ @Override public String processCaptures(Map fieldNameCountStore, Collection snippets, Collection prefaces, - Collection epilogues, Map mappings) { + Collection epilogues, Map mappings, Map fieldStats) { String sampleValue = null; for (String snippet : snippets) { Map captures = grok.captures(snippet); @@ -505,7 +512,7 @@ public boolean matchesAll(Collection snippets) { @Override public String processCaptures(Map fieldNameCountStore, Collection snippets, Collection prefaces, - Collection epilogues, Map mappings) { + Collection epilogues, Map mappings, Map fieldStats) { if (fieldName == null) { throw new IllegalStateException("Cannot process KV matches until a field name has been determined"); } @@ -526,6 +533,9 @@ public String processCaptures(Map fieldNameCountStore, Collecti if (mappings != null) { mappings.put(adjustedFieldName, LogStructureUtils.guessScalarMapping(explanation, adjustedFieldName, values)); } + if (fieldStats != null) { + fieldStats.put(adjustedFieldName, LogStructureUtils.calculateFieldStats(values)); + } return "\\b" + fieldName + "=%{USER:" + adjustedFieldName + "}"; } } @@ -541,8 +551,8 @@ static class NoMappingGrokPatternCandidate extends ValueOnlyGrokPatternCandidate @Override public String processCaptures(Map fieldNameCountStore, Collection snippets, Collection prefaces, - Collection epilogues, Map mappings) { - return super.processCaptures(fieldNameCountStore, snippets, prefaces, epilogues, null); + Collection epilogues, Map mappings, Map fieldStats) { + return super.processCaptures(fieldNameCountStore, snippets, prefaces, epilogues, null, fieldStats); } } @@ -570,11 +580,11 @@ public boolean matchesAll(Collection sampleMessages) { * @return A tuple of (time field name, Grok string). */ public Tuple processMatch(List explanation, Collection sampleMessages, - Map mappings) { + Map mappings, Map fieldStats) { explanation.add("A full message Grok pattern [" + grokString.substring(2, grokString.length() - 1) + "] looks appropriate"); - if (mappings != null) { + if (mappings != null || fieldStats != null) { Map> valuesPerField = new HashMap<>(); for (String sampleMessage : sampleMessages) { @@ -604,8 +614,14 @@ public Tuple processMatch(List explanation, Collection> valuesForField : valuesPerField.entrySet()) { String fieldName = valuesForField.getKey(); - mappings.put(fieldName, - LogStructureUtils.guessScalarMapping(explanation, fieldName, valuesForField.getValue())); + if (mappings != null) { + mappings.put(fieldName, + LogStructureUtils.guessScalarMapping(explanation, fieldName, valuesForField.getValue())); + } + if (fieldStats != null) { + fieldStats.put(fieldName, + LogStructureUtils.calculateFieldStats(valuesForField.getValue())); + } } } diff --git a/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/JsonLogStructureFinder.java b/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/JsonLogStructureFinder.java index 98e8a0213fbef..7f3cee3f0bd7b 100644 --- a/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/JsonLogStructureFinder.java +++ b/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/JsonLogStructureFinder.java @@ -56,9 +56,16 @@ static JsonLogStructureFinder makeJsonLogStructureFinder(List explanatio .setNeedClientTimezone(timeField.v2().hasTimezoneDependentParsing()); } - SortedMap mappings = LogStructureUtils.guessMappings(explanation, sampleRecords); + Tuple, SortedMap> mappingsAndFieldStats = + LogStructureUtils.guessMappingsAndCalculateFieldStats(explanation, sampleRecords); + + SortedMap mappings = mappingsAndFieldStats.v1(); mappings.put(LogStructureUtils.DEFAULT_TIMESTAMP_FIELD, Collections.singletonMap(LogStructureUtils.MAPPING_TYPE_SETTING, "date")); + if (mappingsAndFieldStats.v2() != null) { + structureBuilder.setFieldStats(mappingsAndFieldStats.v2()); + } + LogStructure structure = structureBuilder .setMappings(mappings) .setExplanation(explanation) diff --git a/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/LogStructure.java b/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/LogStructure.java index ea8fe37e62f9f..6d36da1180220 100644 --- a/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/LogStructure.java +++ b/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/LogStructure.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; import java.util.ArrayList; @@ -95,6 +96,7 @@ public String toString() { static final ParseField TIMESTAMP_FORMATS = new ParseField("timestamp_formats"); static final ParseField NEED_CLIENT_TIMEZONE = new ParseField("need_client_timezone"); static final ParseField MAPPINGS = new ParseField("mappings"); + static final ParseField FIELD_STATS = new ParseField("field_stats"); static final ParseField EXPLANATION = new ParseField("explanation"); public static final ObjectParser PARSER = new ObjectParser<>("log_file_structure", false, Builder::new); @@ -117,6 +119,13 @@ public String toString() { PARSER.declareStringArray(Builder::setTimestampFormats, TIMESTAMP_FORMATS); PARSER.declareBoolean(Builder::setNeedClientTimezone, NEED_CLIENT_TIMEZONE); PARSER.declareObject(Builder::setMappings, (p, c) -> new TreeMap<>(p.map()), MAPPINGS); + PARSER.declareObject(Builder::setFieldStats, (p, c) -> { + Map fieldStats = new TreeMap<>(); + while (p.nextToken() == XContentParser.Token.FIELD_NAME) { + fieldStats.put(p.currentName(), FieldStats.PARSER.apply(p, c)); + } + return fieldStats; + }, FIELD_STATS); PARSER.declareStringArray(Builder::setExplanation, EXPLANATION); } @@ -137,13 +146,14 @@ public String toString() { private final String timestampField; private final boolean needClientTimezone; private final SortedMap mappings; + private final SortedMap fieldStats; private final List explanation; public LogStructure(int numLinesAnalyzed, int numMessagesAnalyzed, String sampleStart, String charset, Boolean hasByteOrderMarker, Format format, String multilineStartPattern, String excludeLinesPattern, List inputFields, Boolean hasHeaderRow, Character delimiter, Boolean shouldTrimFields, String grokPattern, String timestampField, List timestampFormats, boolean needClientTimezone, Map mappings, - List explanation) { + Map fieldStats, List explanation) { this.numLinesAnalyzed = numLinesAnalyzed; this.numMessagesAnalyzed = numMessagesAnalyzed; @@ -162,6 +172,7 @@ public LogStructure(int numLinesAnalyzed, int numMessagesAnalyzed, String sample this.timestampFormats = (timestampFormats == null) ? null : Collections.unmodifiableList(new ArrayList<>(timestampFormats)); this.needClientTimezone = needClientTimezone; this.mappings = Collections.unmodifiableSortedMap(new TreeMap<>(mappings)); + this.fieldStats = Collections.unmodifiableSortedMap(new TreeMap<>(fieldStats)); this.explanation = Collections.unmodifiableList(new ArrayList<>(explanation)); } @@ -233,6 +244,10 @@ public SortedMap getMappings() { return mappings; } + public SortedMap getFieldStats() { + return fieldStats; + } + public List getExplanation() { return explanation; } @@ -278,6 +293,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } builder.field(NEED_CLIENT_TIMEZONE.getPreferredName(), needClientTimezone); builder.field(MAPPINGS.getPreferredName(), mappings); + if (fieldStats.isEmpty() == false) { + builder.startObject(FIELD_STATS.getPreferredName()); + for (Map.Entry entry : fieldStats.entrySet()) { + builder.field(entry.getKey(), entry.getValue()); + } + builder.endObject(); + } builder.field(EXPLANATION.getPreferredName(), explanation); builder.endObject(); @@ -289,7 +311,7 @@ public int hashCode() { return Objects.hash(numLinesAnalyzed, numMessagesAnalyzed, sampleStart, charset, hasByteOrderMarker, format, multilineStartPattern, excludeLinesPattern, inputFields, hasHeaderRow, delimiter, shouldTrimFields, grokPattern, timestampField, - timestampFormats, needClientTimezone, mappings, explanation); + timestampFormats, needClientTimezone, mappings, fieldStats, explanation); } @Override @@ -321,6 +343,7 @@ public boolean equals(Object other) { Objects.equals(this.timestampField, that.timestampField) && Objects.equals(this.timestampFormats, that.timestampFormats) && Objects.equals(this.mappings, that.mappings) && + Objects.equals(this.fieldStats, that.fieldStats) && Objects.equals(this.explanation, that.explanation); } @@ -343,6 +366,7 @@ public static class Builder { private List timestampFormats; private boolean needClientTimezone; private Map mappings; + private Map fieldStats = Collections.emptyMap(); private List explanation; public Builder() { @@ -438,6 +462,11 @@ public Builder setMappings(Map mappings) { return this; } + public Builder setFieldStats(Map fieldStats) { + this.fieldStats = Objects.requireNonNull(fieldStats); + return this; + } + public Builder setExplanation(List explanation) { this.explanation = Objects.requireNonNull(explanation); return this; @@ -540,7 +569,7 @@ public LogStructure build() { return new LogStructure(numLinesAnalyzed, numMessagesAnalyzed, sampleStart, charset, hasByteOrderMarker, format, multilineStartPattern, excludeLinesPattern, inputFields, hasHeaderRow, delimiter, shouldTrimFields, grokPattern, - timestampField, timestampFormats, needClientTimezone, mappings, explanation); + timestampField, timestampFormats, needClientTimezone, mappings, fieldStats, explanation); } } } diff --git a/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/LogStructureUtils.java b/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/LogStructureUtils.java index 71a68c399910b..69214a746ed79 100644 --- a/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/LogStructureUtils.java +++ b/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/LogStructureUtils.java @@ -16,6 +16,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; import java.util.stream.Collectors; @@ -28,6 +29,7 @@ public final class LogStructureUtils { public static final String MAPPING_FORMAT_SETTING = "format"; public static final String MAPPING_PROPERTIES_SETTING = "properties"; + private static final int NUM_TOP_HITS = 10; // NUMBER Grok pattern doesn't support scientific notation, so we extend it private static final Grok NUMBER_GROK = new Grok(Grok.getBuiltinPatterns(), "^%{NUMBER}(?:[eE][+-]?[0-3]?[0-9]{1,2})?$"); private static final Grok IP_GROK = new Grok(Grok.getBuiltinPatterns(), "^%{IP}$"); @@ -112,26 +114,39 @@ private static List> findCandidates(List e * @param sampleRecords The sampled records. * @return A map of field name to mapping settings. */ - static SortedMap guessMappings(List explanation, List> sampleRecords) { + static Tuple, SortedMap> + guessMappingsAndCalculateFieldStats(List explanation, List> sampleRecords) { SortedMap mappings = new TreeMap<>(); + SortedMap fieldStats = new TreeMap<>(); - for (Map sampleRecord : sampleRecords) { - for (String fieldName : sampleRecord.keySet()) { - mappings.computeIfAbsent(fieldName, key -> guessMapping(explanation, fieldName, - sampleRecords.stream().flatMap(record -> { - Object fieldValue = record.get(fieldName); - return (fieldValue == null) ? Stream.empty() : Stream.of(fieldValue); - } - ).collect(Collectors.toList()))); + Set uniqueFieldNames = sampleRecords.stream().flatMap(record -> record.keySet().stream()).collect(Collectors.toSet()); + + for (String fieldName : uniqueFieldNames) { + + List fieldValues = sampleRecords.stream().flatMap(record -> { + Object fieldValue = record.get(fieldName); + return (fieldValue == null) ? Stream.empty() : Stream.of(fieldValue); + } + ).collect(Collectors.toList()); + + Tuple, FieldStats> mappingAndFieldStats = + guessMappingAndCalculateFieldStats(explanation, fieldName, fieldValues); + if (mappingAndFieldStats != null) { + if (mappingAndFieldStats.v1() != null) { + mappings.put(fieldName, mappingAndFieldStats.v1()); + } + if (mappingAndFieldStats.v2() != null) { + fieldStats.put(fieldName, mappingAndFieldStats.v2()); + } } } - return mappings; + return new Tuple<>(mappings, fieldStats); } - static Map guessMapping(List explanation, String fieldName, List fieldValues) { - + static Tuple, FieldStats> guessMappingAndCalculateFieldStats(List explanation, + String fieldName, List fieldValues) { if (fieldValues == null || fieldValues.isEmpty()) { // We can get here if all the records that contained a given field had a null value for it. // In this case it's best not to make any statement about what the mapping type should be. @@ -140,7 +155,7 @@ static Map guessMapping(List explanation, String fieldNa if (fieldValues.stream().anyMatch(value -> value instanceof Map)) { if (fieldValues.stream().allMatch(value -> value instanceof Map)) { - return Collections.singletonMap(MAPPING_TYPE_SETTING, "object"); + return new Tuple<>(Collections.singletonMap(MAPPING_TYPE_SETTING, "object"), null); } throw new IllegalArgumentException("Field [" + fieldName + "] has both object and non-object values - this is not supported by Elasticsearch"); @@ -148,11 +163,12 @@ static Map guessMapping(List explanation, String fieldNa if (fieldValues.stream().anyMatch(value -> value instanceof List || value instanceof Object[])) { // Elasticsearch fields can be either arrays or single values, but array values must all have the same type - return guessMapping(explanation, fieldName, + return guessMappingAndCalculateFieldStats(explanation, fieldName, fieldValues.stream().flatMap(LogStructureUtils::flatten).collect(Collectors.toList())); } - return guessScalarMapping(explanation, fieldName, fieldValues.stream().map(Object::toString).collect(Collectors.toList())); + Collection fieldValuesAsStrings = fieldValues.stream().map(Object::toString).collect(Collectors.toList()); + return new Tuple<>(guessScalarMapping(explanation, fieldName, fieldValuesAsStrings), calculateFieldStats(fieldValuesAsStrings)); } private static Stream flatten(Object value) { @@ -227,6 +243,18 @@ else if (fieldValues.stream().allMatch(IP_GROK::match)) { return Collections.singletonMap(MAPPING_TYPE_SETTING, "keyword"); } + /** + * Calculate stats for a set of field values. + * @param fieldValues Values of the field for which field stats are to be calculated. + * @return The stats calculated from the field values. + */ + static FieldStats calculateFieldStats(Collection fieldValues) { + + FieldStatsCalculator calculator = new FieldStatsCalculator(); + calculator.accept(fieldValues); + return calculator.calculate(NUM_TOP_HITS); + } + /** * The thinking is that the longer the field value and the more spaces it contains, * the more likely it is that it should be indexed as text rather than keyword. diff --git a/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/TextLogStructureFinder.java b/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/TextLogStructureFinder.java index 722751a4cf49e..e830aa30a1e87 100644 --- a/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/TextLogStructureFinder.java +++ b/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/TextLogStructureFinder.java @@ -82,10 +82,12 @@ static TextLogStructureFinder makeTextLogStructureFinder(List explanatio mappings.put("message", Collections.singletonMap(LogStructureUtils.MAPPING_TYPE_SETTING, "text")); mappings.put(LogStructureUtils.DEFAULT_TIMESTAMP_FIELD, Collections.singletonMap(LogStructureUtils.MAPPING_TYPE_SETTING, "date")); + SortedMap fieldStats = new TreeMap<>(); + // We can't parse directly into @timestamp using Grok, so parse to some other time field, which the date filter will then remove String interimTimestampField; String grokPattern; - GrokPatternCreator grokPatternCreator = new GrokPatternCreator(explanation, sampleMessages, mappings); + GrokPatternCreator grokPatternCreator = new GrokPatternCreator(explanation, sampleMessages, mappings, fieldStats); Tuple timestampFieldAndFullMatchGrokPattern = grokPatternCreator.findFullLineGrokPattern(); if (timestampFieldAndFullMatchGrokPattern != null) { interimTimestampField = timestampFieldAndFullMatchGrokPattern.v1(); @@ -101,6 +103,7 @@ static TextLogStructureFinder makeTextLogStructureFinder(List explanatio .setNeedClientTimezone(bestTimestamp.v1().hasTimezoneDependentParsing()) .setGrokPattern(grokPattern) .setMappings(mappings) + .setFieldStats(fieldStats) .setExplanation(explanation) .build(); diff --git a/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/XmlLogStructureFinder.java b/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/XmlLogStructureFinder.java index d664a9ccb8213..6c81032c05baf 100644 --- a/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/XmlLogStructureFinder.java +++ b/x-pack/plugin/ml/log-structure-finder/src/main/java/org/elasticsearch/xpack/ml/logstructurefinder/XmlLogStructureFinder.java @@ -95,7 +95,14 @@ static XmlLogStructureFinder makeXmlLogStructureFinder(List explanation, .setNeedClientTimezone(timeField.v2().hasTimezoneDependentParsing()); } - SortedMap innerMappings = LogStructureUtils.guessMappings(explanation, sampleRecords); + Tuple, SortedMap> mappingsAndFieldStats = + LogStructureUtils.guessMappingsAndCalculateFieldStats(explanation, sampleRecords); + + if (mappingsAndFieldStats.v2() != null) { + structureBuilder.setFieldStats(mappingsAndFieldStats.v2()); + } + + SortedMap innerMappings = mappingsAndFieldStats.v1(); Map secondLevelProperties = new LinkedHashMap<>(); secondLevelProperties.put(LogStructureUtils.MAPPING_TYPE_SETTING, "object"); secondLevelProperties.put(LogStructureUtils.MAPPING_PROPERTIES_SETTING, innerMappings); diff --git a/x-pack/plugin/ml/log-structure-finder/src/test/java/org/elasticsearch/xpack/ml/logstructurefinder/FieldStatsCalculatorTests.java b/x-pack/plugin/ml/log-structure-finder/src/test/java/org/elasticsearch/xpack/ml/logstructurefinder/FieldStatsCalculatorTests.java new file mode 100644 index 0000000000000..6d8927c1c2b3a --- /dev/null +++ b/x-pack/plugin/ml/log-structure-finder/src/test/java/org/elasticsearch/xpack/ml/logstructurefinder/FieldStatsCalculatorTests.java @@ -0,0 +1,218 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.logstructurefinder; + +import java.util.Arrays; +import java.util.Collections; +import java.util.DoubleSummaryStatistics; +import java.util.List; +import java.util.Map; + +public class FieldStatsCalculatorTests extends LogStructureTestCase { + + public void testMean() { + + FieldStatsCalculator calculator = new FieldStatsCalculator(); + + calculator.accept(Arrays.asList("1", "3.5", "2.5", "9")); + + assertEquals(4.0, calculator.calculateMean(), 1e-10); + } + + public void testMedianGivenOddCount() { + + FieldStatsCalculator calculator = new FieldStatsCalculator(); + + calculator.accept(Arrays.asList("3", "23", "-1", "5", "1000")); + + assertEquals(5.0, calculator.calculateMedian(), 1e-10); + } + + public void testMedianGivenOddCountMinimal() { + + FieldStatsCalculator calculator = new FieldStatsCalculator(); + + calculator.accept(Collections.singletonList("3")); + + assertEquals(3.0, calculator.calculateMedian(), 1e-10); + } + + public void testMedianGivenEvenCountMiddleValuesDifferent() { + + FieldStatsCalculator calculator = new FieldStatsCalculator(); + + calculator.accept(Arrays.asList("3", "23", "-1", "5", "1000", "6")); + + assertEquals(5.5, calculator.calculateMedian(), 1e-10); + } + + public void testMedianGivenEvenCountMiddleValuesSame() { + + FieldStatsCalculator calculator = new FieldStatsCalculator(); + + calculator.accept(Arrays.asList("3", "23", "-1", "5", "1000", "5")); + + assertEquals(5.0, calculator.calculateMedian(), 1e-10); + } + + public void testMedianGivenEvenCountMinimal() { + + FieldStatsCalculator calculator = new FieldStatsCalculator(); + + calculator.accept(Arrays.asList("4", "4")); + + assertEquals(4.0, calculator.calculateMedian(), 1e-10); + } + + public void testTopHitsNumeric() { + + FieldStatsCalculator calculator = new FieldStatsCalculator(); + + calculator.accept(Arrays.asList("4", "4", "7", "4", "6", "5", "6", "5", "16", "4", "5")); + + List> topHits = calculator.findNumericTopHits(3); + + assertEquals(3, topHits.size()); + assertEquals(4.0, topHits.get(0).get("value")); + assertEquals(4, topHits.get(0).get("count")); + assertEquals(5.0, topHits.get(1).get("value")); + assertEquals(3, topHits.get(1).get("count")); + assertEquals(6.0, topHits.get(2).get("value")); + assertEquals(2, topHits.get(2).get("count")); + } + + public void testTopHitsString() { + + FieldStatsCalculator calculator = new FieldStatsCalculator(); + + calculator.accept(Arrays.asList("s", "s", "d", "s", "f", "x", "f", "x", "n", "s", "x")); + + List> topHits = calculator.findStringTopHits(3); + + assertEquals(3, topHits.size()); + assertEquals("s", topHits.get(0).get("value")); + assertEquals(4, topHits.get(0).get("count")); + assertEquals("x", topHits.get(1).get("value")); + assertEquals(3, topHits.get(1).get("count")); + assertEquals("f", topHits.get(2).get("value")); + assertEquals(2, topHits.get(2).get("count")); + } + + public void testCalculateGivenEmpty() { + + FieldStatsCalculator calculator = new FieldStatsCalculator(); + + calculator.accept(Collections.emptyList()); + + FieldStats stats = calculator.calculate(3); + + assertEquals(0L, stats.getCount()); + assertEquals(0, stats.getCardinality()); + assertNull(stats.getMinValue()); + assertNull(stats.getMaxValue()); + assertNull(stats.getMeanValue()); + assertNull(stats.getMedianValue()); + assertEquals(0, stats.getTopHits().size()); + } + + public void testCalculateGivenNumericField() { + + FieldStatsCalculator calculator = new FieldStatsCalculator(); + + calculator.accept(Arrays.asList("4", "4", "7", "4", "6", "5", "6", "5", "16", "4", "5")); + + FieldStats stats = calculator.calculate(3); + + assertEquals(11L, stats.getCount()); + assertEquals(5, stats.getCardinality()); + assertEquals(4.0, stats.getMinValue(), 1e-10); + assertEquals(16.0, stats.getMaxValue(), 1e-10); + assertEquals(6.0, stats.getMeanValue(), 1e-10); + assertEquals(5.0, stats.getMedianValue(), 1e-10); + + List> topHits = stats.getTopHits(); + + assertEquals(3, topHits.size()); + assertEquals(4.0, topHits.get(0).get("value")); + assertEquals(4, topHits.get(0).get("count")); + assertEquals(5.0, topHits.get(1).get("value")); + assertEquals(3, topHits.get(1).get("count")); + assertEquals(6.0, topHits.get(2).get("value")); + assertEquals(2, topHits.get(2).get("count")); + } + + public void testCalculateGivenStringField() { + + FieldStatsCalculator calculator = new FieldStatsCalculator(); + + calculator.accept(Arrays.asList("s", "s", "d", "s", "f", "x", "f", "x", "n", "s", "x")); + + FieldStats stats = calculator.calculate(3); + + assertEquals(11L, stats.getCount()); + assertEquals(5, stats.getCardinality()); + assertNull(stats.getMinValue()); + assertNull(stats.getMaxValue()); + assertNull(stats.getMeanValue()); + assertNull(stats.getMedianValue()); + + List> topHits = stats.getTopHits(); + + assertEquals(3, topHits.size()); + assertEquals("s", topHits.get(0).get("value")); + assertEquals(4, topHits.get(0).get("count")); + assertEquals("x", topHits.get(1).get("value")); + assertEquals(3, topHits.get(1).get("count")); + assertEquals("f", topHits.get(2).get("value")); + assertEquals(2, topHits.get(2).get("count")); + } + + public void testCalculateGivenMixedField() { + + FieldStatsCalculator calculator = new FieldStatsCalculator(); + + calculator.accept(Arrays.asList("4", "4", "d", "4", "f", "x", "f", "x", "16", "4", "x")); + + FieldStats stats = calculator.calculate(3); + + assertEquals(11L, stats.getCount()); + assertEquals(5, stats.getCardinality()); + assertNull(stats.getMinValue()); + assertNull(stats.getMaxValue()); + assertNull(stats.getMeanValue()); + assertNull(stats.getMedianValue()); + + List> topHits = stats.getTopHits(); + + assertEquals(3, topHits.size()); + assertEquals("4", topHits.get(0).get("value")); + assertEquals(4, topHits.get(0).get("count")); + assertEquals("x", topHits.get(1).get("value")); + assertEquals(3, topHits.get(1).get("count")); + assertEquals("f", topHits.get(2).get("value")); + assertEquals(2, topHits.get(2).get("count")); + } + + public void testJavaStatsEquivalence() { + + DoubleSummaryStatistics summaryStatistics = new DoubleSummaryStatistics(); + FieldStatsCalculator calculator = new FieldStatsCalculator(); + + for (int numValues = randomIntBetween(1000, 10000); numValues > 0; --numValues) { + + double value = randomDouble(); + summaryStatistics.accept(value); + calculator.accept(Collections.singletonList(Double.toString(value))); + } + + FieldStats stats = calculator.calculate(1); + + assertEquals(summaryStatistics.getCount(), stats.getCount()); + assertEquals(summaryStatistics.getMin(), stats.getMinValue(), 1e-10); + assertEquals(summaryStatistics.getMax(), stats.getMaxValue(), 1e-10); + assertEquals(summaryStatistics.getAverage(), stats.getMeanValue(), 1e-10); + } +} diff --git a/x-pack/plugin/ml/log-structure-finder/src/test/java/org/elasticsearch/xpack/ml/logstructurefinder/FieldStatsTests.java b/x-pack/plugin/ml/log-structure-finder/src/test/java/org/elasticsearch/xpack/ml/logstructurefinder/FieldStatsTests.java new file mode 100644 index 0000000000000..4a95e6631c96a --- /dev/null +++ b/x-pack/plugin/ml/log-structure-finder/src/test/java/org/elasticsearch/xpack/ml/logstructurefinder/FieldStatsTests.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.ml.logstructurefinder; + +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class FieldStatsTests extends AbstractXContentTestCase { + + protected FieldStats createTestInstance() { + return createTestFieldStats(); + } + + static FieldStats createTestFieldStats() { + + long count = randomIntBetween(1, 100000); + int cardinality = randomIntBetween(1, (int) count); + + Double minValue = null; + Double maxValue = null; + Double meanValue = null; + Double medianValue = null; + boolean isMetric = randomBoolean(); + if (isMetric) { + minValue = randomDouble(); + maxValue = randomDouble(); + meanValue = randomDouble(); + medianValue = randomDouble(); + } + + List> topHits = new ArrayList<>(); + for (int i = 0; i < Math.min(10, cardinality); ++i) { + Map topHit = new LinkedHashMap<>(); + if (isMetric) { + topHit.put("value", randomDouble()); + } else { + topHit.put("value", randomAlphaOfLength(20)); + } + topHit.put("count", randomIntBetween(1, cardinality)); + topHits.add(topHit); + } + + return new FieldStats(count, cardinality, minValue, maxValue, meanValue, medianValue, topHits); + } + + protected FieldStats doParseInstance(XContentParser parser) { + return FieldStats.PARSER.apply(parser, null); + } + + protected boolean supportsUnknownFields() { + return false; + } +} diff --git a/x-pack/plugin/ml/log-structure-finder/src/test/java/org/elasticsearch/xpack/ml/logstructurefinder/GrokPatternCreatorTests.java b/x-pack/plugin/ml/log-structure-finder/src/test/java/org/elasticsearch/xpack/ml/logstructurefinder/GrokPatternCreatorTests.java index 87f9f662698ef..9853efd41de84 100644 --- a/x-pack/plugin/ml/log-structure-finder/src/test/java/org/elasticsearch/xpack/ml/logstructurefinder/GrokPatternCreatorTests.java +++ b/x-pack/plugin/ml/log-structure-finder/src/test/java/org/elasticsearch/xpack/ml/logstructurefinder/GrokPatternCreatorTests.java @@ -43,7 +43,7 @@ public void testPopulatePrefacesAndEpiloguesGivenTimestamp() { Collection prefaces = new ArrayList<>(); Collection epilogues = new ArrayList<>(); - candidate.processCaptures(fieldNameCountStore, matchingStrings, prefaces, epilogues, null); + candidate.processCaptures(fieldNameCountStore, matchingStrings, prefaces, epilogues, null, null); assertThat(prefaces, containsInAnyOrder("[", "[", "junk [", "[")); assertThat(epilogues, containsInAnyOrder("] DEBUG ", "] ERROR ", "] INFO ", "] DEBUG ")); @@ -60,7 +60,7 @@ public void testPopulatePrefacesAndEpiloguesGivenEmailAddress() { Collection prefaces = new ArrayList<>(); Collection epilogues = new ArrayList<>(); - candidate.processCaptures(fieldNameCountStore, matchingStrings, prefaces, epilogues, null); + candidate.processCaptures(fieldNameCountStore, matchingStrings, prefaces, epilogues, null, null); assertThat(prefaces, containsInAnyOrder("before ", "abc ", "")); assertThat(epilogues, containsInAnyOrder(" after", " xyz", "")); @@ -73,7 +73,7 @@ public void testAppendBestGrokMatchForStringsGivenTimestampsAndLogLevels() { "junk [2018-01-22T07:33:23] INFO ", "[2018-01-21T03:33:23] DEBUG "); - GrokPatternCreator grokPatternCreator = new GrokPatternCreator(explanation, snippets, null); + GrokPatternCreator grokPatternCreator = new GrokPatternCreator(explanation, snippets, null, null); grokPatternCreator.appendBestGrokMatchForStrings(false, snippets, false, 0); assertEquals(".*?\\[%{TIMESTAMP_ISO8601:extra_timestamp}\\] %{LOGLEVEL:loglevel} ", @@ -87,7 +87,7 @@ public void testAppendBestGrokMatchForStringsGivenNumbersInBrackets() { " (4)", " (-5) "); - GrokPatternCreator grokPatternCreator = new GrokPatternCreator(explanation, snippets, null); + GrokPatternCreator grokPatternCreator = new GrokPatternCreator(explanation, snippets, null, null); grokPatternCreator.appendBestGrokMatchForStrings(false, snippets, false, 0); assertEquals(".*?\\(%{INT:field}\\).*?", grokPatternCreator.getOverallGrokPatternBuilder().toString()); @@ -99,7 +99,7 @@ public void testAppendBestGrokMatchForStringsGivenNegativeNumbersWithoutBreak() "prior to-3", "-4"); - GrokPatternCreator grokPatternCreator = new GrokPatternCreator(explanation, snippets, null); + GrokPatternCreator grokPatternCreator = new GrokPatternCreator(explanation, snippets, null, null); grokPatternCreator.appendBestGrokMatchForStrings(false, snippets, false, 0); // It seems sensible that we don't detect these suffices as either base 10 or base 16 numbers @@ -113,7 +113,7 @@ public void testAppendBestGrokMatchForStringsGivenHexNumbers() { " -123", "1f is hex"); - GrokPatternCreator grokPatternCreator = new GrokPatternCreator(explanation, snippets, null); + GrokPatternCreator grokPatternCreator = new GrokPatternCreator(explanation, snippets, null, null); grokPatternCreator.appendBestGrokMatchForStrings(false, snippets, false, 0); assertEquals(".*?%{BASE16NUM:field}.*?", grokPatternCreator.getOverallGrokPatternBuilder().toString()); @@ -124,7 +124,7 @@ public void testAppendBestGrokMatchForStringsGivenHostnamesWithNumbers() { Collection snippets = Arrays.asList(" mappings = new HashMap<>(); - GrokPatternCreator grokPatternCreator = new GrokPatternCreator(explanation, sampleMessages, mappings); + GrokPatternCreator grokPatternCreator = new GrokPatternCreator(explanation, sampleMessages, mappings, null); assertEquals("%{SYSLOGTIMESTAMP:timestamp} .*? .*?\\[%{INT:field}\\]: %{LOGLEVEL:loglevel} \\(.*? .*? .*?\\) .*? " + "%{QUOTEDSTRING:field2}: %{IP:ipaddress}#%{INT:field3}", @@ -215,7 +215,7 @@ public void testCreateGrokPatternFromExamplesGivenCatalinaLogs() { "Invalid chunk ignored."); Map mappings = new HashMap<>(); - GrokPatternCreator grokPatternCreator = new GrokPatternCreator(explanation, sampleMessages, mappings); + GrokPatternCreator grokPatternCreator = new GrokPatternCreator(explanation, sampleMessages, mappings, null); assertEquals("%{CATALINA_DATESTAMP:timestamp} .*? .*?\\n%{LOGLEVEL:loglevel}: .*", grokPatternCreator.createGrokPatternFromExamples("CATALINA_DATESTAMP", "timestamp")); @@ -237,7 +237,7 @@ public void testCreateGrokPatternFromExamplesGivenMultiTimestampLogs() { "Info\tsshd\tsubsystem request for sftp"); Map mappings = new HashMap<>(); - GrokPatternCreator grokPatternCreator = new GrokPatternCreator(explanation, sampleMessages, mappings); + GrokPatternCreator grokPatternCreator = new GrokPatternCreator(explanation, sampleMessages, mappings, null); assertEquals("%{INT:field}\\t%{TIMESTAMP_ISO8601:timestamp}\\t%{TIMESTAMP_ISO8601:extra_timestamp}\\t%{INT:field2}\\t.*?\\t" + "%{IP:ipaddress}\\t.*?\\t%{LOGLEVEL:loglevel}\\t.*", @@ -271,7 +271,7 @@ public void testFindFullLineGrokPatternGivenApacheCombinedLogs() { "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36\""); Map mappings = new HashMap<>(); - GrokPatternCreator grokPatternCreator = new GrokPatternCreator(explanation, sampleMessages, mappings); + GrokPatternCreator grokPatternCreator = new GrokPatternCreator(explanation, sampleMessages, mappings, null); assertEquals(new Tuple<>("timestamp", "%{COMBINEDAPACHELOG}"), grokPatternCreator.findFullLineGrokPattern()); assertEquals(10, mappings.size()); @@ -300,7 +300,7 @@ public void testAdjustForPunctuationGivenCommonPrefix() { ",\"rule1\",\"Accept\",\"\",\"\",\"\",\"0000000000000000\"" ); - GrokPatternCreator grokPatternCreator = new GrokPatternCreator(explanation, snippets, null); + GrokPatternCreator grokPatternCreator = new GrokPatternCreator(explanation, snippets, null, null); Collection adjustedSnippets = grokPatternCreator.adjustForPunctuation(snippets); assertEquals("\",", grokPatternCreator.getOverallGrokPatternBuilder().toString()); @@ -317,7 +317,7 @@ public void testAdjustForPunctuationGivenNoCommonPrefix() { "was added by 'User1'(id:2) to servergroup 'GAME'(id:9)" ); - GrokPatternCreator grokPatternCreator = new GrokPatternCreator(explanation, snippets, null); + GrokPatternCreator grokPatternCreator = new GrokPatternCreator(explanation, snippets, null, null); Collection adjustedSnippets = grokPatternCreator.adjustForPunctuation(snippets); assertEquals("", grokPatternCreator.getOverallGrokPatternBuilder().toString()); diff --git a/x-pack/plugin/ml/log-structure-finder/src/test/java/org/elasticsearch/xpack/ml/logstructurefinder/LogStructureTests.java b/x-pack/plugin/ml/log-structure-finder/src/test/java/org/elasticsearch/xpack/ml/logstructurefinder/LogStructureTests.java index 302946dcaa86c..2a10e11164f6e 100644 --- a/x-pack/plugin/ml/log-structure-finder/src/test/java/org/elasticsearch/xpack/ml/logstructurefinder/LogStructureTests.java +++ b/x-pack/plugin/ml/log-structure-finder/src/test/java/org/elasticsearch/xpack/ml/logstructurefinder/LogStructureTests.java @@ -66,6 +66,14 @@ protected LogStructure createTestInstance() { } builder.setMappings(mappings); + //if (randomBoolean()) { + Map fieldStats = new TreeMap<>(); + for (String field : generateRandomStringArray(5, 20, false, false)) { + fieldStats.put(field, FieldStatsTests.createTestFieldStats()); + } + builder.setFieldStats(fieldStats); + //} + builder.setExplanation(Arrays.asList(generateRandomStringArray(10, 150, false, false))); return builder.build(); diff --git a/x-pack/plugin/ml/log-structure-finder/src/test/java/org/elasticsearch/xpack/ml/logstructurefinder/LogStructureUtilsTests.java b/x-pack/plugin/ml/log-structure-finder/src/test/java/org/elasticsearch/xpack/ml/logstructurefinder/LogStructureUtilsTests.java index 7e92728f01aa0..8ebfe520d6621 100644 --- a/x-pack/plugin/ml/log-structure-finder/src/test/java/org/elasticsearch/xpack/ml/logstructurefinder/LogStructureUtilsTests.java +++ b/x-pack/plugin/ml/log-structure-finder/src/test/java/org/elasticsearch/xpack/ml/logstructurefinder/LogStructureUtilsTests.java @@ -12,7 +12,9 @@ import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; +import java.util.SortedMap; import static org.hamcrest.Matchers.contains; @@ -178,96 +180,83 @@ public void testSamplesWithManyFieldsInconsistentAndConsistentTimeFields() { } public void testGuessMappingGivenNothing() { - assertNull(LogStructureUtils.guessMapping(explanation, "foo", Collections.emptyList())); + assertNull(guessMapping(explanation, "foo", Collections.emptyList())); } public void testGuessMappingGivenKeyword() { Map expected = Collections.singletonMap(LogStructureUtils.MAPPING_TYPE_SETTING, "keyword"); - assertEquals(expected, - LogStructureUtils.guessMapping(explanation, "foo", Arrays.asList("ERROR", "INFO", "DEBUG"))); - assertEquals(expected, - LogStructureUtils.guessMapping(explanation, "foo", Arrays.asList("2018-06-11T13:26:47Z", "not a date"))); + assertEquals(expected, guessMapping(explanation, "foo", Arrays.asList("ERROR", "INFO", "DEBUG"))); + assertEquals(expected, guessMapping(explanation, "foo", Arrays.asList("2018-06-11T13:26:47Z", "not a date"))); } public void testGuessMappingGivenText() { Map expected = Collections.singletonMap(LogStructureUtils.MAPPING_TYPE_SETTING, "text"); - assertEquals(expected, LogStructureUtils.guessMapping(explanation, "foo", - Arrays.asList("a", "the quick brown fox jumped over the lazy dog"))); + assertEquals(expected, guessMapping(explanation, "foo", Arrays.asList("a", "the quick brown fox jumped over the lazy dog"))); } public void testGuessMappingGivenIp() { Map expected = Collections.singletonMap(LogStructureUtils.MAPPING_TYPE_SETTING, "ip"); - assertEquals(expected, - LogStructureUtils.guessMapping(explanation, "foo", Arrays.asList("10.0.0.1", "172.16.0.1", "192.168.0.1"))); + assertEquals(expected, guessMapping(explanation, "foo", Arrays.asList("10.0.0.1", "172.16.0.1", "192.168.0.1"))); } public void testGuessMappingGivenDouble() { Map expected = Collections.singletonMap(LogStructureUtils.MAPPING_TYPE_SETTING, "double"); - assertEquals(expected, - LogStructureUtils.guessMapping(explanation, "foo", Arrays.asList("3.14159265359", "0", "-8"))); + assertEquals(expected, guessMapping(explanation, "foo", Arrays.asList("3.14159265359", "0", "-8"))); // 12345678901234567890 is too long for long - assertEquals(expected, - LogStructureUtils.guessMapping(explanation, "foo", Arrays.asList("1", "2", "12345678901234567890"))); - assertEquals(expected, - LogStructureUtils.guessMapping(explanation, "foo", Arrays.asList(3.14159265359, 0.0, 1e-308))); - assertEquals(expected, - LogStructureUtils.guessMapping(explanation, "foo", Arrays.asList("-1e-1", "-1e308", "1e-308"))); + assertEquals(expected, guessMapping(explanation, "foo", Arrays.asList("1", "2", "12345678901234567890"))); + assertEquals(expected, guessMapping(explanation, "foo", Arrays.asList(3.14159265359, 0.0, 1e-308))); + assertEquals(expected, guessMapping(explanation, "foo", Arrays.asList("-1e-1", "-1e308", "1e-308"))); } public void testGuessMappingGivenLong() { Map expected = Collections.singletonMap(LogStructureUtils.MAPPING_TYPE_SETTING, "long"); - assertEquals(expected, - LogStructureUtils.guessMapping(explanation, "foo", Arrays.asList("500", "3", "-3"))); - assertEquals(expected, - LogStructureUtils.guessMapping(explanation, "foo", Arrays.asList(500, 6, 0))); + assertEquals(expected, guessMapping(explanation, "foo", Arrays.asList("500", "3", "-3"))); + assertEquals(expected, guessMapping(explanation, "foo", Arrays.asList(500, 6, 0))); } public void testGuessMappingGivenDate() { Map expected = Collections.singletonMap(LogStructureUtils.MAPPING_TYPE_SETTING, "date"); - assertEquals(expected, LogStructureUtils.guessMapping(explanation, "foo", - Arrays.asList("2018-06-11T13:26:47Z", "2018-06-11T13:27:12Z"))); + assertEquals(expected, guessMapping(explanation, "foo", Arrays.asList("2018-06-11T13:26:47Z", "2018-06-11T13:27:12Z"))); } public void testGuessMappingGivenBoolean() { Map expected = Collections.singletonMap(LogStructureUtils.MAPPING_TYPE_SETTING, "boolean"); - assertEquals(expected, LogStructureUtils.guessMapping(explanation, "foo", Arrays.asList("false", "true"))); - assertEquals(expected, LogStructureUtils.guessMapping(explanation, "foo", Arrays.asList(true, false))); + assertEquals(expected, guessMapping(explanation, "foo", Arrays.asList("false", "true"))); + assertEquals(expected, guessMapping(explanation, "foo", Arrays.asList(true, false))); } public void testGuessMappingGivenArray() { Map expected = Collections.singletonMap(LogStructureUtils.MAPPING_TYPE_SETTING, "long"); - assertEquals(expected, - LogStructureUtils.guessMapping(explanation, "foo", Arrays.asList(42, Arrays.asList(1, -99)))); + assertEquals(expected, guessMapping(explanation, "foo", Arrays.asList(42, Arrays.asList(1, -99)))); expected = Collections.singletonMap(LogStructureUtils.MAPPING_TYPE_SETTING, "keyword"); - assertEquals(expected, - LogStructureUtils.guessMapping(explanation, "foo", Arrays.asList(new String[]{ "x", "y" }, "z"))); + assertEquals(expected, guessMapping(explanation, "foo", Arrays.asList(new String[]{ "x", "y" }, "z"))); } public void testGuessMappingGivenObject() { Map expected = Collections.singletonMap(LogStructureUtils.MAPPING_TYPE_SETTING, "object"); - assertEquals(expected, LogStructureUtils.guessMapping(explanation, "foo", + assertEquals(expected, guessMapping(explanation, "foo", Arrays.asList(Collections.singletonMap("name", "value1"), Collections.singletonMap("name", "value2")))); } public void testGuessMappingGivenObjectAndNonObject() { - RuntimeException e = expectThrows(RuntimeException.class, () -> LogStructureUtils.guessMapping(explanation, + RuntimeException e = expectThrows(RuntimeException.class, () -> guessMapping(explanation, "foo", Arrays.asList(Collections.singletonMap("name", "value1"), "value2"))); assertEquals("Field [foo] has both object and non-object values - this is not supported by Elasticsearch", e.getMessage()); } - public void testGuessMappings() { + public void testGuessMappingsAndCalculateFieldStats() { Map sample1 = new LinkedHashMap<>(); sample1.put("foo", "not a time"); sample1.put("time", "2018-05-24 17:28:31,735"); @@ -279,7 +268,11 @@ public void testGuessMappings() { sample2.put("bar", 17); sample2.put("nothing", null); - Map mappings = LogStructureUtils.guessMappings(explanation, Arrays.asList(sample1, sample2)); + Tuple, SortedMap> mappingsAndFieldStats = + LogStructureUtils.guessMappingsAndCalculateFieldStats(explanation, Arrays.asList(sample1, sample2)); + assertNotNull(mappingsAndFieldStats); + + Map mappings = mappingsAndFieldStats.v1(); assertNotNull(mappings); assertEquals(Collections.singletonMap(LogStructureUtils.MAPPING_TYPE_SETTING, "keyword"), mappings.get("foo")); Map expectedTimeMapping = new HashMap<>(); @@ -288,5 +281,29 @@ public void testGuessMappings() { assertEquals(expectedTimeMapping, mappings.get("time")); assertEquals(Collections.singletonMap(LogStructureUtils.MAPPING_TYPE_SETTING, "long"), mappings.get("bar")); assertNull(mappings.get("nothing")); + + Map fieldStats = mappingsAndFieldStats.v2(); + assertNotNull(fieldStats); + assertEquals(3, fieldStats.size()); + assertEquals(new FieldStats(2, 2, makeTopHits("not a time", 1, "whatever", 1)), fieldStats.get("foo")); + assertEquals(new FieldStats(2, 2, makeTopHits("2018-05-24 17:28:31,735", 1, "2018-05-29 11:53:02,837", 1)), fieldStats.get("time")); + assertEquals(new FieldStats(2, 2, 17.0, 42.0, 29.5, 29.5, makeTopHits(17.0, 1, 42.0, 1)), fieldStats.get("bar")); + assertNull(fieldStats.get("nothing")); + } + + private Map guessMapping(List explanation, String fieldName, List fieldValues) { + Tuple, FieldStats> mappingAndFieldStats = + LogStructureUtils.guessMappingAndCalculateFieldStats(explanation, fieldName, fieldValues); + return (mappingAndFieldStats == null) ? null : mappingAndFieldStats.v1(); + } + + private List> makeTopHits(Object value1, int count1, Object value2, int count2) { + Map topHit1 = new LinkedHashMap<>(); + topHit1.put("value", value1); + topHit1.put("count", count1); + Map topHit2 = new LinkedHashMap<>(); + topHit2.put("value", value2); + topHit2.put("count", count2); + return Arrays.asList(topHit1, topHit2); } } From 46774098d9fcdbf7b983c05b6afd76b425be7f60 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Wed, 5 Sep 2018 14:25:29 +0200 Subject: [PATCH 04/10] INGEST: Implement Drop Processor (#32278) * INGEST: Implement Drop Processor * Adjust Processor API * Implement Drop Processor * Closes #23726 --- .../common/AbstractStringProcessor.java | 5 +- .../ingest/common/AppendProcessor.java | 3 +- .../ingest/common/ConvertProcessor.java | 5 +- .../ingest/common/DateIndexNameProcessor.java | 3 +- .../ingest/common/DateProcessor.java | 3 +- .../ingest/common/DissectProcessor.java | 5 +- .../ingest/common/DotExpanderProcessor.java | 3 +- .../ingest/common/DropProcessor.java | 57 +++++++++++++++++++ .../ingest/common/FailProcessor.java | 2 +- .../ingest/common/ForEachProcessor.java | 13 +++-- .../ingest/common/GrokProcessor.java | 5 +- .../ingest/common/IngestCommonPlugin.java | 1 + .../ingest/common/JoinProcessor.java | 3 +- .../ingest/common/JsonProcessor.java | 3 +- .../ingest/common/KeyValueProcessor.java | 3 +- .../ingest/common/PipelineProcessor.java | 4 +- .../ingest/common/Processors.java | 1 + .../ingest/common/RemoveProcessor.java | 3 +- .../ingest/common/RenameProcessor.java | 5 +- .../ingest/common/ScriptProcessor.java | 3 +- .../ingest/common/SetProcessor.java | 3 +- .../ingest/common/SortProcessor.java | 3 +- .../ingest/common/SplitProcessor.java | 5 +- .../ingest/common/ForEachProcessorTests.java | 3 +- .../ingest/common/PipelineProcessorTests.java | 3 +- .../attachment/AttachmentProcessor.java | 5 +- .../ingest/geoip/GeoIpProcessor.java | 5 +- .../ingest/useragent/UserAgentProcessor.java | 5 +- .../action/bulk/TransportBulkAction.java | 57 ++++++++++++------- .../ingest/SimulateExecutionService.java | 5 +- .../ingest/TrackingResultProcessor.java | 3 +- .../ingest/CompoundProcessor.java | 11 ++-- .../ingest/ConditionalProcessor.java | 5 +- .../elasticsearch/ingest/IngestDocument.java | 4 +- .../elasticsearch/ingest/IngestService.java | 37 ++++++------ .../org/elasticsearch/ingest/Pipeline.java | 4 +- .../org/elasticsearch/ingest/Processor.java | 2 +- .../bulk/TransportBulkActionIngestTests.java | 12 ++-- .../ingest/ConditionalProcessorTests.java | 3 +- .../ingest/IngestServiceTests.java | 36 ++++++------ .../elasticsearch/ingest/TestProcessor.java | 3 +- .../monitoring/test/MockIngestPlugin.java | 3 +- .../ingest/SetSecurityUserProcessor.java | 3 +- 43 files changed, 235 insertions(+), 115 deletions(-) create mode 100644 modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DropProcessor.java diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java index 23c98ca1e0c0e..792e5e4ebed2d 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AbstractStringProcessor.java @@ -57,16 +57,17 @@ String getTargetField() { } @Override - public final void execute(IngestDocument document) { + public final IngestDocument execute(IngestDocument document) { String val = document.getFieldValue(field, String.class, ignoreMissing); if (val == null && ignoreMissing) { - return; + return document; } else if (val == null) { throw new IllegalArgumentException("field [" + field + "] is null, cannot process it."); } document.setFieldValue(targetField, process(val)); + return document; } protected abstract T process(String value); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java index 0543ae8591f97..058d1bf22d81c 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/AppendProcessor.java @@ -56,8 +56,9 @@ public ValueSource getValue() { } @Override - public void execute(IngestDocument ingestDocument) throws Exception { + public IngestDocument execute(IngestDocument ingestDocument) throws Exception { ingestDocument.appendFieldValue(field, value); + return ingestDocument; } @Override diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java index 2e881b82b59de..aca48efe6c11c 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ConvertProcessor.java @@ -173,12 +173,12 @@ boolean isIgnoreMissing() { } @Override - public void execute(IngestDocument document) { + public IngestDocument execute(IngestDocument document) { Object oldValue = document.getFieldValue(field, Object.class, ignoreMissing); Object newValue; if (oldValue == null && ignoreMissing) { - return; + return document; } else if (oldValue == null) { throw new IllegalArgumentException("Field [" + field + "] is null, cannot be converted to type [" + convertType + "]"); } @@ -194,6 +194,7 @@ public void execute(IngestDocument document) { newValue = convertType.convert(oldValue); } document.setFieldValue(targetField, newValue); + return document; } @Override diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java index 0d6253c88f9fa..4a88f15b6410d 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java @@ -63,7 +63,7 @@ public final class DateIndexNameProcessor extends AbstractProcessor { } @Override - public void execute(IngestDocument ingestDocument) throws Exception { + public IngestDocument execute(IngestDocument ingestDocument) throws Exception { // Date can be specified as a string or long: Object obj = ingestDocument.getFieldValue(field, Object.class); String date = null; @@ -101,6 +101,7 @@ public void execute(IngestDocument ingestDocument) throws Exception { .append('>'); String dynamicIndexName = builder.toString(); ingestDocument.setFieldValue(IngestDocument.MetaData.INDEX.getFieldName(), dynamicIndexName); + return ingestDocument; } @Override diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java index 4a9654f8cd0fe..dd6e6006eeb6d 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateProcessor.java @@ -74,7 +74,7 @@ private Locale newLocale(Map params) { } @Override - public void execute(IngestDocument ingestDocument) { + public IngestDocument execute(IngestDocument ingestDocument) { Object obj = ingestDocument.getFieldValue(field, Object.class); String value = null; if (obj != null) { @@ -98,6 +98,7 @@ public void execute(IngestDocument ingestDocument) { } ingestDocument.setFieldValue(targetField, ISODateTimeFormat.dateTime().print(dateTime)); + return ingestDocument; } @Override diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java index 58f04ccdd431f..fa51d047e73e3 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DissectProcessor.java @@ -47,14 +47,15 @@ public final class DissectProcessor extends AbstractProcessor { } @Override - public void execute(IngestDocument ingestDocument) { + public IngestDocument execute(IngestDocument ingestDocument) { String input = ingestDocument.getFieldValue(field, String.class, ignoreMissing); if (input == null && ignoreMissing) { - return; + return ingestDocument; } else if (input == null) { throw new IllegalArgumentException("field [" + field + "] is null, cannot process it."); } dissectParser.parse(input).forEach(ingestDocument::setFieldValue); + return ingestDocument; } @Override diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java index bfc32311733da..0698f6ed0a6c9 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DotExpanderProcessor.java @@ -41,7 +41,7 @@ public final class DotExpanderProcessor extends AbstractProcessor { @Override @SuppressWarnings("unchecked") - public void execute(IngestDocument ingestDocument) throws Exception { + public IngestDocument execute(IngestDocument ingestDocument) throws Exception { String path; Map map; if (this.path != null) { @@ -75,6 +75,7 @@ public void execute(IngestDocument ingestDocument) throws Exception { Object value = map.remove(field); ingestDocument.setFieldValue(path, value); } + return ingestDocument; } @Override diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DropProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DropProcessor.java new file mode 100644 index 0000000000000..a0eabe38979eb --- /dev/null +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DropProcessor.java @@ -0,0 +1,57 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.ingest.common; + +import java.util.Map; +import org.elasticsearch.ingest.AbstractProcessor; +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.Processor; + +/** + * Drop processor only returns {@code null} for the execution result to indicate that any document + * executed by it should not be indexed. + */ +public final class DropProcessor extends AbstractProcessor { + + public static final String TYPE = "drop"; + + private DropProcessor(final String tag) { + super(tag); + } + + @Override + public IngestDocument execute(final IngestDocument ingestDocument) throws Exception { + return null; + } + + @Override + public String getType() { + return TYPE; + } + + public static final class Factory implements Processor.Factory { + + @Override + public Processor create(final Map processorFactories, final String tag, + final Map config) { + return new DropProcessor(tag); + } + } +} diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java index b1f946c10a239..0b62fbf72c8e5 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/FailProcessor.java @@ -48,7 +48,7 @@ public TemplateScript.Factory getMessage() { } @Override - public void execute(IngestDocument document) { + public IngestDocument execute(IngestDocument document) { throw new FailProcessorException(document.renderTemplate(message)); } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java index 31c0ae8cc3dc8..ad93298c646e1 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ForEachProcessor.java @@ -63,24 +63,29 @@ boolean isIgnoreMissing() { } @Override - public void execute(IngestDocument ingestDocument) throws Exception { + public IngestDocument execute(IngestDocument ingestDocument) throws Exception { List values = ingestDocument.getFieldValue(field, List.class, ignoreMissing); if (values == null) { if (ignoreMissing) { - return; + return ingestDocument; } throw new IllegalArgumentException("field [" + field + "] is null, cannot loop over its elements."); } List newValues = new ArrayList<>(values.size()); + IngestDocument document = ingestDocument; for (Object value : values) { Object previousValue = ingestDocument.getIngestMetadata().put("_value", value); try { - processor.execute(ingestDocument); + document = processor.execute(document); + if (document == null) { + return null; + } } finally { newValues.add(ingestDocument.getIngestMetadata().put("_value", previousValue)); } } - ingestDocument.setFieldValue(field, newValues); + document.setFieldValue(field, newValues); + return document; } @Override diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java index 88cba512b86cb..19883053d2a09 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/GrokProcessor.java @@ -54,11 +54,11 @@ public final class GrokProcessor extends AbstractProcessor { } @Override - public void execute(IngestDocument ingestDocument) throws Exception { + public IngestDocument execute(IngestDocument ingestDocument) throws Exception { String fieldValue = ingestDocument.getFieldValue(matchField, String.class, ignoreMissing); if (fieldValue == null && ignoreMissing) { - return; + return ingestDocument; } else if (fieldValue == null) { throw new IllegalArgumentException("field [" + matchField + "] is null, cannot process it."); } @@ -81,6 +81,7 @@ public void execute(IngestDocument ingestDocument) throws Exception { ingestDocument.setFieldValue(PATTERN_MATCH_KEY, "0"); } } + return ingestDocument; } @Override diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java index 8b048282814ea..d9dba2cc10073 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java @@ -84,6 +84,7 @@ public Map getProcessors(Processor.Parameters paramet processors.put(BytesProcessor.TYPE, new BytesProcessor.Factory()); processors.put(PipelineProcessor.TYPE, new PipelineProcessor.Factory(parameters.ingestService)); processors.put(DissectProcessor.TYPE, new DissectProcessor.Factory()); + processors.put(DropProcessor.TYPE, new DropProcessor.Factory()); return Collections.unmodifiableMap(processors); } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java index 57216a71e022c..f29a688886194 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JoinProcessor.java @@ -60,7 +60,7 @@ String getTargetField() { } @Override - public void execute(IngestDocument document) { + public IngestDocument execute(IngestDocument document) { List list = document.getFieldValue(field, List.class); if (list == null) { throw new IllegalArgumentException("field [" + field + "] is null, cannot join."); @@ -69,6 +69,7 @@ public void execute(IngestDocument document) { .map(Object::toString) .collect(Collectors.joining(separator)); document.setFieldValue(targetField, joined); + return document; } @Override diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java index c0a9d37abdab7..90a648347cdfd 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/JsonProcessor.java @@ -107,12 +107,13 @@ public static void apply(Map ctx, String fieldName) { } @Override - public void execute(IngestDocument document) throws Exception { + public IngestDocument execute(IngestDocument document) throws Exception { if (addToRoot) { apply(document.getSourceAndMetadata(), field); } else { document.setFieldValue(targetField, apply(document.getFieldValue(field, Object.class))); } + return document; } @Override diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java index 9cce3cedf3d02..69c7e9ff75187 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/KeyValueProcessor.java @@ -188,8 +188,9 @@ private static void append(IngestDocument document, String targetField, String v } @Override - public void execute(IngestDocument document) { + public IngestDocument execute(IngestDocument document) { execution.accept(document); + return document; } @Override diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/PipelineProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/PipelineProcessor.java index 77ffdb919193f..1958a3e5232b8 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/PipelineProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/PipelineProcessor.java @@ -42,12 +42,12 @@ private PipelineProcessor(String tag, String pipelineName, IngestService ingestS } @Override - public void execute(IngestDocument ingestDocument) throws Exception { + public IngestDocument execute(IngestDocument ingestDocument) throws Exception { Pipeline pipeline = ingestService.getPipeline(pipelineName); if (pipeline == null) { throw new IllegalStateException("Pipeline processor configured for non-existent pipeline [" + pipelineName + ']'); } - ingestDocument.executePipeline(pipeline); + return ingestDocument.executePipeline(pipeline); } @Override diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Processors.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Processors.java index 8a0b152989241..00209f5560090 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Processors.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/Processors.java @@ -46,4 +46,5 @@ public static void json(Map ctx, String field) { public static String urlDecode(String value) { return URLDecodeProcessor.apply(value); } + } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java index 2b9eaa9a13d18..6002abb9e67a3 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java @@ -52,7 +52,7 @@ public List getFields() { } @Override - public void execute(IngestDocument document) { + public IngestDocument execute(IngestDocument document) { if (ignoreMissing) { fields.forEach(field -> { String path = document.renderTemplate(field); @@ -63,6 +63,7 @@ public void execute(IngestDocument document) { } else { fields.forEach(document::removeField); } + return document; } @Override diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java index a35a164ddd3f1..2abd920048f73 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java @@ -59,11 +59,11 @@ boolean isIgnoreMissing() { } @Override - public void execute(IngestDocument document) { + public IngestDocument execute(IngestDocument document) { String path = document.renderTemplate(field); if (document.hasField(path, true) == false) { if (ignoreMissing) { - return; + return document; } else { throw new IllegalArgumentException("field [" + path + "] doesn't exist"); } @@ -86,6 +86,7 @@ public void execute(IngestDocument document) { document.setFieldValue(path, value); throw e; } + return document; } @Override diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java index 169b2ab646a7d..12ef53cdcfcfa 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/ScriptProcessor.java @@ -69,9 +69,10 @@ public final class ScriptProcessor extends AbstractProcessor { * @param document The Ingest document passed into the script context under the "ctx" object. */ @Override - public void execute(IngestDocument document) { + public IngestDocument execute(IngestDocument document) { IngestScript.Factory factory = scriptService.compile(script, IngestScript.CONTEXT); factory.newInstance(script.getParams()).execute(document.getSourceAndMetadata()); + return document; } @Override diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java index 7aefa28861830..0af51e5b895e4 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SetProcessor.java @@ -65,10 +65,11 @@ public ValueSource getValue() { } @Override - public void execute(IngestDocument document) { + public IngestDocument execute(IngestDocument document) { if (overrideEnabled || document.hasField(field) == false || document.getFieldValue(field, Object.class) == null) { document.setFieldValue(field, value); } + return document; } @Override diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java index 7ff266efe6b91..a29cc34652479 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SortProcessor.java @@ -94,7 +94,7 @@ String getTargetField() { @Override @SuppressWarnings("unchecked") - public void execute(IngestDocument document) { + public IngestDocument execute(IngestDocument document) { List> list = document.getFieldValue(field, List.class); if (list == null) { @@ -110,6 +110,7 @@ public void execute(IngestDocument document) { } document.setFieldValue(targetField, copy); + return document; } @Override diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java index cdd90f937fd09..96a765b5ba7a3 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/SplitProcessor.java @@ -68,11 +68,11 @@ String getTargetField() { } @Override - public void execute(IngestDocument document) { + public IngestDocument execute(IngestDocument document) { String oldVal = document.getFieldValue(field, String.class, ignoreMissing); if (oldVal == null && ignoreMissing) { - return; + return document; } else if (oldVal == null) { throw new IllegalArgumentException("field [" + field + "] is null, cannot split."); } @@ -81,6 +81,7 @@ public void execute(IngestDocument document) { List splitList = new ArrayList<>(strings.length); Collections.addAll(splitList, strings); document.setFieldValue(targetField, splitList); + return document; } @Override diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java index ffc5bcd4ac930..282994d8eb354 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ForEachProcessorTests.java @@ -154,9 +154,10 @@ public void testRestOfTheDocumentIsAvailable() throws Exception { public void testRandom() throws Exception { Processor innerProcessor = new Processor() { @Override - public void execute(IngestDocument ingestDocument) throws Exception { + public IngestDocument execute(IngestDocument ingestDocument) throws Exception { String existingValue = ingestDocument.getFieldValue("_ingest._value", String.class); ingestDocument.setFieldValue("_ingest._value", existingValue + "."); + return ingestDocument; } @Override diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/PipelineProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/PipelineProcessorTests.java index 5baf3cf822d72..3103fb0392e96 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/PipelineProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/PipelineProcessorTests.java @@ -45,8 +45,9 @@ public void testExecutesPipeline() throws Exception { pipelineId, null, null, new CompoundProcessor(new Processor() { @Override - public void execute(final IngestDocument ingestDocument) throws Exception { + public IngestDocument execute(final IngestDocument ingestDocument) throws Exception { invoked.complete(ingestDocument); + return ingestDocument; } @Override diff --git a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java index 9fb2debcb5481..c8a24ad3c8719 100644 --- a/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java +++ b/plugins/ingest-attachment/src/main/java/org/elasticsearch/ingest/attachment/AttachmentProcessor.java @@ -73,13 +73,13 @@ boolean isIgnoreMissing() { } @Override - public void execute(IngestDocument ingestDocument) { + public IngestDocument execute(IngestDocument ingestDocument) { Map additionalFields = new HashMap<>(); byte[] input = ingestDocument.getFieldValueAsBytes(field, ignoreMissing); if (input == null && ignoreMissing) { - return; + return ingestDocument; } else if (input == null) { throw new IllegalArgumentException("field [" + field + "] is null, cannot parse."); } @@ -164,6 +164,7 @@ public void execute(IngestDocument ingestDocument) { } ingestDocument.setFieldValue(targetField, additionalFields); + return ingestDocument; } @Override diff --git a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java index 366b6ffc1d241..b5dbf5a7f34de 100644 --- a/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java +++ b/plugins/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java @@ -81,11 +81,11 @@ boolean isIgnoreMissing() { } @Override - public void execute(IngestDocument ingestDocument) { + public IngestDocument execute(IngestDocument ingestDocument) { String ip = ingestDocument.getFieldValue(field, String.class, ignoreMissing); if (ip == null && ignoreMissing) { - return; + return ingestDocument; } else if (ip == null) { throw new IllegalArgumentException("field [" + field + "] is null, cannot extract geoip information."); } @@ -120,6 +120,7 @@ public void execute(IngestDocument ingestDocument) { if (geoData.isEmpty() == false) { ingestDocument.setFieldValue(targetField, geoData); } + return ingestDocument; } @Override diff --git a/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java b/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java index 93f210c427b51..6e7f588f0bd8a 100644 --- a/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java +++ b/plugins/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java @@ -63,11 +63,11 @@ boolean isIgnoreMissing() { } @Override - public void execute(IngestDocument ingestDocument) throws Exception { + public IngestDocument execute(IngestDocument ingestDocument) throws Exception { String userAgent = ingestDocument.getFieldValue(field, String.class, ignoreMissing); if (userAgent == null && ignoreMissing) { - return; + return ingestDocument; } else if (userAgent == null) { throw new IllegalArgumentException("field [" + field + "] is null, cannot parse user-agent."); } @@ -144,6 +144,7 @@ public void execute(IngestDocument ingestDocument) throws Exception { } ingestDocument.setFieldValue(targetField, uaDetails); + return ingestDocument; } /** To maintain compatibility with logstash-filter-useragent */ diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index ea4a5086d7b98..a3d7d50f3e22a 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -27,6 +27,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.RoutingMissingException; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; @@ -37,6 +38,7 @@ import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.action.update.TransportUpdateAction; import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateObserver; @@ -521,28 +523,30 @@ private long relativeTime() { void processBulkIndexIngestRequest(Task task, BulkRequest original, ActionListener listener) { long ingestStartTimeInNanos = System.nanoTime(); BulkRequestModifier bulkRequestModifier = new BulkRequestModifier(original); - ingestService.executeBulkRequest(() -> bulkRequestModifier, (indexRequest, exception) -> { - logger.debug(() -> new ParameterizedMessage("failed to execute pipeline [{}] for document [{}/{}/{}]", - indexRequest.getPipeline(), indexRequest.index(), indexRequest.type(), indexRequest.id()), exception); - bulkRequestModifier.markCurrentItemAsFailed(exception); - }, (exception) -> { - if (exception != null) { - logger.error("failed to execute pipeline for a bulk request", exception); - listener.onFailure(exception); - } else { - long ingestTookInMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - ingestStartTimeInNanos); - BulkRequest bulkRequest = bulkRequestModifier.getBulkRequest(); - ActionListener actionListener = bulkRequestModifier.wrapActionListenerIfNeeded(ingestTookInMillis, listener); - if (bulkRequest.requests().isEmpty()) { - // at this stage, the transport bulk action can't deal with a bulk request with no requests, - // so we stop and send an empty response back to the client. - // (this will happen if pre-processing all items in the bulk failed) - actionListener.onResponse(new BulkResponse(new BulkItemResponse[0], 0)); + ingestService.executeBulkRequest(() -> bulkRequestModifier, + (indexRequest, exception) -> { + logger.debug(() -> new ParameterizedMessage("failed to execute pipeline [{}] for document [{}/{}/{}]", + indexRequest.getPipeline(), indexRequest.index(), indexRequest.type(), indexRequest.id()), exception); + bulkRequestModifier.markCurrentItemAsFailed(exception); + }, (exception) -> { + if (exception != null) { + logger.error("failed to execute pipeline for a bulk request", exception); + listener.onFailure(exception); } else { - doExecute(task, bulkRequest, actionListener); + long ingestTookInMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - ingestStartTimeInNanos); + BulkRequest bulkRequest = bulkRequestModifier.getBulkRequest(); + ActionListener actionListener = bulkRequestModifier.wrapActionListenerIfNeeded(ingestTookInMillis, listener); + if (bulkRequest.requests().isEmpty()) { + // at this stage, the transport bulk action can't deal with a bulk request with no requests, + // so we stop and send an empty response back to the client. + // (this will happen if pre-processing all items in the bulk failed) + actionListener.onResponse(new BulkResponse(new BulkItemResponse[0], 0)); + } else { + doExecute(task, bulkRequest, actionListener); + } } - } - }); + }, + indexRequest -> bulkRequestModifier.markCurrentItemAsDropped()); } static final class BulkRequestModifier implements Iterator> { @@ -604,6 +608,19 @@ ActionListener wrapActionListenerIfNeeded(long ingestTookInMillis, } } + void markCurrentItemAsDropped() { + IndexRequest indexRequest = (IndexRequest) bulkRequest.requests().get(currentSlot); + failedSlots.set(currentSlot); + itemResponses.add( + new BulkItemResponse(currentSlot, indexRequest.opType(), + new UpdateResponse( + new ShardId(indexRequest.index(), IndexMetaData.INDEX_UUID_NA_VALUE, 0), + indexRequest.type(), indexRequest.id(), indexRequest.version(), DocWriteResponse.Result.NOOP + ) + ) + ); + } + void markCurrentItemAsFailed(Exception e) { IndexRequest indexRequest = (IndexRequest) bulkRequest.requests().get(currentSlot); // We hit a error during preprocessing a request, so we: diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java index db7397ba1f86b..430da9955bafa 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java @@ -67,7 +67,10 @@ public void execute(SimulatePipelineRequest.Parsed request, ActionListener responses = new ArrayList<>(); for (IngestDocument ingestDocument : request.getDocuments()) { - responses.add(executeDocument(request.getPipeline(), ingestDocument, request.isVerbose())); + SimulateDocumentResult response = executeDocument(request.getPipeline(), ingestDocument, request.isVerbose()); + if (response != null) { + responses.add(response); + } } listener.onResponse(new SimulatePipelineResponse(request.getPipeline().getId(), request.isVerbose(), responses)); } diff --git a/server/src/main/java/org/elasticsearch/action/ingest/TrackingResultProcessor.java b/server/src/main/java/org/elasticsearch/action/ingest/TrackingResultProcessor.java index abf617ffb1aac..04c0fe7ca49dc 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/TrackingResultProcessor.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/TrackingResultProcessor.java @@ -42,7 +42,7 @@ public TrackingResultProcessor(boolean ignoreFailure, Processor actualProcessor, } @Override - public void execute(IngestDocument ingestDocument) throws Exception { + public IngestDocument execute(IngestDocument ingestDocument) throws Exception { try { actualProcessor.execute(ingestDocument); processorResultList.add(new SimulateProcessorResult(actualProcessor.getTag(), new IngestDocument(ingestDocument))); @@ -54,6 +54,7 @@ public void execute(IngestDocument ingestDocument) throws Exception { } throw e; } + return ingestDocument; } @Override diff --git a/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java b/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java index 3ab7c078cd7ad..f576667f44109 100644 --- a/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java +++ b/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java @@ -94,17 +94,19 @@ public String getTag() { } @Override - public void execute(IngestDocument ingestDocument) throws Exception { + public IngestDocument execute(IngestDocument ingestDocument) throws Exception { for (Processor processor : processors) { try { - processor.execute(ingestDocument); + if (processor.execute(ingestDocument) == null) { + return null; + } } catch (Exception e) { if (ignoreFailure) { continue; } ElasticsearchException compoundProcessorException = - newCompoundProcessorException(e, processor.getType(), processor.getTag()); + newCompoundProcessorException(e, processor.getType(), processor.getTag()); if (onFailureProcessors.isEmpty()) { throw compoundProcessorException; } else { @@ -113,6 +115,7 @@ public void execute(IngestDocument ingestDocument) throws Exception { } } } + return ingestDocument; } void executeOnFailure(IngestDocument ingestDocument, ElasticsearchException exception) throws Exception { @@ -149,7 +152,7 @@ private void removeFailureMetadata(IngestDocument ingestDocument) { } private ElasticsearchException newCompoundProcessorException(Exception e, String processorType, String processorTag) { - if (e instanceof ElasticsearchException && ((ElasticsearchException)e).getHeader("processor_type") != null) { + if (e instanceof ElasticsearchException && ((ElasticsearchException) e).getHeader("processor_type") != null) { return (ElasticsearchException) e; } diff --git a/server/src/main/java/org/elasticsearch/ingest/ConditionalProcessor.java b/server/src/main/java/org/elasticsearch/ingest/ConditionalProcessor.java index d1eb651acae03..b6f6612344a39 100644 --- a/server/src/main/java/org/elasticsearch/ingest/ConditionalProcessor.java +++ b/server/src/main/java/org/elasticsearch/ingest/ConditionalProcessor.java @@ -51,12 +51,13 @@ public class ConditionalProcessor extends AbstractProcessor { } @Override - public void execute(IngestDocument ingestDocument) throws Exception { + public IngestDocument execute(IngestDocument ingestDocument) throws Exception { IngestConditionalScript script = scriptService.compile(condition, IngestConditionalScript.CONTEXT).newInstance(condition.getParams()); if (script.execute(new UnmodifiableIngestData(ingestDocument.getSourceAndMetadata()))) { - processor.execute(ingestDocument); + return processor.execute(ingestDocument); } + return ingestDocument; } @Override diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java index e218168eeb7b5..5f122358d0c43 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java @@ -644,11 +644,11 @@ private static Object deepCopy(Object value) { * @param pipeline Pipeline to execute * @throws Exception On exception in pipeline execution */ - public void executePipeline(Pipeline pipeline) throws Exception { + public IngestDocument executePipeline(Pipeline pipeline) throws Exception { if (this.executedPipelines.add(pipeline) == false) { throw new IllegalStateException("Recursive invocation of pipeline [" + pipeline.getId() + "] detected."); } - pipeline.execute(this); + return pipeline.execute(this); } @Override diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index f0f5d76caaba8..5623cf30f3642 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -270,7 +270,7 @@ private static Pipeline substitutePipeline(String id, ElasticsearchParseExceptio String errorMessage = "pipeline with id [" + id + "] could not be loaded, caused by [" + e.getDetailedMessage() + "]"; Processor failureProcessor = new AbstractProcessor(tag) { @Override - public void execute(IngestDocument ingestDocument) { + public IngestDocument execute(IngestDocument ingestDocument) { throw new IllegalStateException(errorMessage); } @@ -323,7 +323,8 @@ void validatePipeline(Map ingestInfos, PutPipelineReq } public void executeBulkRequest(Iterable> actionRequests, - BiConsumer itemFailureHandler, Consumer completionHandler) { + BiConsumer itemFailureHandler, Consumer completionHandler, + Consumer itemDroppedHandler) { threadPool.executor(ThreadPool.Names.WRITE).execute(new AbstractRunnable() { @Override @@ -351,7 +352,7 @@ protected void doRun() { if (pipeline == null) { throw new IllegalArgumentException("pipeline with id [" + pipelineId + "] does not exist"); } - innerExecute(indexRequest, pipeline); + innerExecute(indexRequest, pipeline, itemDroppedHandler); //this shouldn't be needed here but we do it for consistency with index api // which requires it to prevent double execution indexRequest.setPipeline(NOOP_PIPELINE_NAME); @@ -399,7 +400,7 @@ void updatePipelineStats(IngestMetadata ingestMetadata) { } } - private void innerExecute(IndexRequest indexRequest, Pipeline pipeline) throws Exception { + private void innerExecute(IndexRequest indexRequest, Pipeline pipeline, Consumer itemDroppedHandler) throws Exception { if (pipeline.getProcessors().isEmpty()) { return; } @@ -419,20 +420,22 @@ private void innerExecute(IndexRequest indexRequest, Pipeline pipeline) throws E VersionType versionType = indexRequest.versionType(); Map sourceAsMap = indexRequest.sourceAsMap(); IngestDocument ingestDocument = new IngestDocument(index, type, id, routing, version, versionType, sourceAsMap); - pipeline.execute(ingestDocument); - - Map metadataMap = ingestDocument.extractMetadata(); - //it's fine to set all metadata fields all the time, as ingest document holds their starting values - //before ingestion, which might also get modified during ingestion. - indexRequest.index((String) metadataMap.get(IngestDocument.MetaData.INDEX)); - indexRequest.type((String) metadataMap.get(IngestDocument.MetaData.TYPE)); - indexRequest.id((String) metadataMap.get(IngestDocument.MetaData.ID)); - indexRequest.routing((String) metadataMap.get(IngestDocument.MetaData.ROUTING)); - indexRequest.version(((Number) metadataMap.get(IngestDocument.MetaData.VERSION)).longValue()); - if (metadataMap.get(IngestDocument.MetaData.VERSION_TYPE) != null) { - indexRequest.versionType(VersionType.fromString((String) metadataMap.get(IngestDocument.MetaData.VERSION_TYPE))); + if (pipeline.execute(ingestDocument) == null) { + itemDroppedHandler.accept(indexRequest); + } else { + Map metadataMap = ingestDocument.extractMetadata(); + //it's fine to set all metadata fields all the time, as ingest document holds their starting values + //before ingestion, which might also get modified during ingestion. + indexRequest.index((String) metadataMap.get(IngestDocument.MetaData.INDEX)); + indexRequest.type((String) metadataMap.get(IngestDocument.MetaData.TYPE)); + indexRequest.id((String) metadataMap.get(IngestDocument.MetaData.ID)); + indexRequest.routing((String) metadataMap.get(IngestDocument.MetaData.ROUTING)); + indexRequest.version(((Number) metadataMap.get(IngestDocument.MetaData.VERSION)).longValue()); + if (metadataMap.get(IngestDocument.MetaData.VERSION_TYPE) != null) { + indexRequest.versionType(VersionType.fromString((String) metadataMap.get(IngestDocument.MetaData.VERSION_TYPE))); + } + indexRequest.source(ingestDocument.getSourceAndMetadata()); } - indexRequest.source(ingestDocument.getSourceAndMetadata()); } catch (Exception e) { totalStats.ingestFailed(); pipelineStats.ifPresent(StatsHolder::ingestFailed); diff --git a/server/src/main/java/org/elasticsearch/ingest/Pipeline.java b/server/src/main/java/org/elasticsearch/ingest/Pipeline.java index 0a8f9fbc0d894..9f13cb1280aac 100644 --- a/server/src/main/java/org/elasticsearch/ingest/Pipeline.java +++ b/server/src/main/java/org/elasticsearch/ingest/Pipeline.java @@ -77,8 +77,8 @@ public static Pipeline create(String id, Map config, /** * Modifies the data of a document to be indexed based on the processor this pipeline holds */ - public void execute(IngestDocument ingestDocument) throws Exception { - compoundProcessor.execute(ingestDocument); + public IngestDocument execute(IngestDocument ingestDocument) throws Exception { + return compoundProcessor.execute(ingestDocument); } /** diff --git a/server/src/main/java/org/elasticsearch/ingest/Processor.java b/server/src/main/java/org/elasticsearch/ingest/Processor.java index 15a26d3749191..498ec3a77104f 100644 --- a/server/src/main/java/org/elasticsearch/ingest/Processor.java +++ b/server/src/main/java/org/elasticsearch/ingest/Processor.java @@ -40,7 +40,7 @@ public interface Processor { /** * Introspect and potentially modify the incoming data. */ - void execute(IngestDocument ingestDocument) throws Exception; + IngestDocument execute(IngestDocument ingestDocument) throws Exception; /** * Gets the type of a processor diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java index 8b68d2b6bb9bd..7fdb12ff1356a 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java @@ -259,7 +259,7 @@ public void testIngestLocal() throws Exception { assertFalse(action.isExecuted); // haven't executed yet assertFalse(responseCalled.get()); assertFalse(failureCalled.get()); - verify(ingestService).executeBulkRequest(bulkDocsItr.capture(), failureHandler.capture(), completionHandler.capture()); + verify(ingestService).executeBulkRequest(bulkDocsItr.capture(), failureHandler.capture(), completionHandler.capture(), any()); completionHandler.getValue().accept(exception); assertTrue(failureCalled.get()); @@ -293,7 +293,7 @@ public void testSingleItemBulkActionIngestLocal() throws Exception { assertFalse(action.isExecuted); // haven't executed yet assertFalse(responseCalled.get()); assertFalse(failureCalled.get()); - verify(ingestService).executeBulkRequest(bulkDocsItr.capture(), failureHandler.capture(), completionHandler.capture()); + verify(ingestService).executeBulkRequest(bulkDocsItr.capture(), failureHandler.capture(), completionHandler.capture(), any()); completionHandler.getValue().accept(exception); assertTrue(failureCalled.get()); @@ -325,7 +325,7 @@ public void testIngestForward() throws Exception { action.execute(null, bulkRequest, listener); // should not have executed ingest locally - verify(ingestService, never()).executeBulkRequest(any(), any(), any()); + verify(ingestService, never()).executeBulkRequest(any(), any(), any(), any()); // but instead should have sent to a remote node with the transport service ArgumentCaptor node = ArgumentCaptor.forClass(DiscoveryNode.class); verify(transportService).sendRequest(node.capture(), eq(BulkAction.NAME), any(), remoteResponseHandler.capture()); @@ -369,7 +369,7 @@ public void testSingleItemBulkActionIngestForward() throws Exception { singleItemBulkWriteAction.execute(null, indexRequest, listener); // should not have executed ingest locally - verify(ingestService, never()).executeBulkRequest(any(), any(), any()); + verify(ingestService, never()).executeBulkRequest(any(), any(), any(), any()); // but instead should have sent to a remote node with the transport service ArgumentCaptor node = ArgumentCaptor.forClass(DiscoveryNode.class); verify(transportService).sendRequest(node.capture(), eq(BulkAction.NAME), any(), remoteResponseHandler.capture()); @@ -417,7 +417,7 @@ public void testUseDefaultPipeline() throws Exception { assertFalse(action.isExecuted); // haven't executed yet assertFalse(responseCalled.get()); assertFalse(failureCalled.get()); - verify(ingestService).executeBulkRequest(bulkDocsItr.capture(), failureHandler.capture(), completionHandler.capture()); + verify(ingestService).executeBulkRequest(bulkDocsItr.capture(), failureHandler.capture(), completionHandler.capture(), any()); completionHandler.getValue().accept(exception); assertTrue(failureCalled.get()); @@ -449,7 +449,7 @@ public void testCreateIndexBeforeRunPipeline() throws Exception { assertFalse(action.isExecuted); // haven't executed yet assertFalse(responseCalled.get()); assertFalse(failureCalled.get()); - verify(ingestService).executeBulkRequest(bulkDocsItr.capture(), failureHandler.capture(), completionHandler.capture()); + verify(ingestService).executeBulkRequest(bulkDocsItr.capture(), failureHandler.capture(), completionHandler.capture(), any()); completionHandler.getValue().accept(exception); assertTrue(failureCalled.get()); diff --git a/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java index 2cb13af7a2808..12b4078ddf8bb 100644 --- a/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/ConditionalProcessorTests.java @@ -65,8 +65,9 @@ public void testChecksCondition() throws Exception { scriptName, Collections.emptyMap()), scriptService, new Processor() { @Override - public void execute(final IngestDocument ingestDocument) throws Exception { + public IngestDocument execute(final IngestDocument ingestDocument) throws Exception { ingestDocument.setFieldValue("foo", "bar"); + return ingestDocument; } @Override diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index 83a5bef4de279..e3f52f35b79cd 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -126,7 +126,7 @@ public void testExecuteIndexPipelineDoesNotExist() { @SuppressWarnings("unchecked") final Consumer completionHandler = mock(Consumer.class); - ingestService.executeBulkRequest(Collections.singletonList(indexRequest), failureHandler, completionHandler); + ingestService.executeBulkRequest(Collections.singletonList(indexRequest), failureHandler, completionHandler, indexReq -> {}); assertTrue(failure.get()); verify(completionHandler, times(1)).accept(null); @@ -424,7 +424,7 @@ public void testExecuteIndexPipelineExistsButFailedParsing() { IngestService ingestService = createWithProcessors(Collections.singletonMap( "mock", (factories, tag, config) -> new AbstractProcessor("mock") { @Override - public void execute(IngestDocument ingestDocument) { + public IngestDocument execute(IngestDocument ingestDocument) { throw new IllegalStateException("error"); } @@ -453,7 +453,7 @@ public String getType() { @SuppressWarnings("unchecked") final Consumer completionHandler = mock(Consumer.class); - ingestService.executeBulkRequest(Collections.singletonList(indexRequest), failureHandler, completionHandler); + ingestService.executeBulkRequest(Collections.singletonList(indexRequest), failureHandler, completionHandler, indexReq -> {}); assertTrue(failure.get()); verify(completionHandler, times(1)).accept(null); @@ -481,7 +481,7 @@ public void testExecuteBulkPipelineDoesNotExist() { BiConsumer failureHandler = mock(BiConsumer.class); @SuppressWarnings("unchecked") Consumer completionHandler = mock(Consumer.class); - ingestService.executeBulkRequest(bulkRequest.requests(), failureHandler, completionHandler); + ingestService.executeBulkRequest(bulkRequest.requests(), failureHandler, completionHandler, indexReq -> {}); verify(failureHandler, times(1)).accept( argThat(new CustomTypeSafeMatcher("failure handler was not called with the expected arguments") { @Override @@ -514,7 +514,7 @@ public void testExecuteSuccess() { final BiConsumer failureHandler = mock(BiConsumer.class); @SuppressWarnings("unchecked") final Consumer completionHandler = mock(Consumer.class); - ingestService.executeBulkRequest(Collections.singletonList(indexRequest), failureHandler, completionHandler); + ingestService.executeBulkRequest(Collections.singletonList(indexRequest), failureHandler, completionHandler, indexReq -> {}); verify(failureHandler, never()).accept(any(), any()); verify(completionHandler, times(1)).accept(null); } @@ -532,7 +532,7 @@ public void testExecuteEmptyPipeline() throws Exception { final BiConsumer failureHandler = mock(BiConsumer.class); @SuppressWarnings("unchecked") final Consumer completionHandler = mock(Consumer.class); - ingestService.executeBulkRequest(Collections.singletonList(indexRequest), failureHandler, completionHandler); + ingestService.executeBulkRequest(Collections.singletonList(indexRequest), failureHandler, completionHandler, indexReq -> {}); verify(failureHandler, never()).accept(any(), any()); verify(completionHandler, times(1)).accept(null); } @@ -560,14 +560,14 @@ public void testExecutePropagateAllMetaDataUpdates() throws Exception { ingestDocument.setFieldValue(metaData.getFieldName(), "update" + metaData.getFieldName()); } } - return null; + return ingestDocument; }).when(processor).execute(any()); final IndexRequest indexRequest = new IndexRequest("_index", "_type", "_id").source(emptyMap()).setPipeline("_id"); @SuppressWarnings("unchecked") final BiConsumer failureHandler = mock(BiConsumer.class); @SuppressWarnings("unchecked") final Consumer completionHandler = mock(Consumer.class); - ingestService.executeBulkRequest(Collections.singletonList(indexRequest), failureHandler, completionHandler); + ingestService.executeBulkRequest(Collections.singletonList(indexRequest), failureHandler, completionHandler, indexReq -> {}); verify(processor).execute(any()); verify(failureHandler, never()).accept(any(), any()); verify(completionHandler, times(1)).accept(null); @@ -597,7 +597,7 @@ public void testExecuteFailure() throws Exception { final BiConsumer failureHandler = mock(BiConsumer.class); @SuppressWarnings("unchecked") final Consumer completionHandler = mock(Consumer.class); - ingestService.executeBulkRequest(Collections.singletonList(indexRequest), failureHandler, completionHandler); + ingestService.executeBulkRequest(Collections.singletonList(indexRequest), failureHandler, completionHandler, indexReq -> {}); verify(processor).execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), emptyMap())); verify(failureHandler, times(1)).accept(eq(indexRequest), any(RuntimeException.class)); verify(completionHandler, times(1)).accept(null); @@ -624,7 +624,7 @@ public void testExecuteSuccessWithOnFailure() throws Exception { final BiConsumer failureHandler = mock(BiConsumer.class); @SuppressWarnings("unchecked") final Consumer completionHandler = mock(Consumer.class); - ingestService.executeBulkRequest(Collections.singletonList(indexRequest), failureHandler, completionHandler); + ingestService.executeBulkRequest(Collections.singletonList(indexRequest), failureHandler, completionHandler, indexReq -> {}); verify(failureHandler, never()).accept(eq(indexRequest), any(ElasticsearchException.class)); verify(completionHandler, times(1)).accept(null); } @@ -661,7 +661,7 @@ public void testExecuteFailureWithNestedOnFailure() throws Exception { final BiConsumer failureHandler = mock(BiConsumer.class); @SuppressWarnings("unchecked") final Consumer completionHandler = mock(Consumer.class); - ingestService.executeBulkRequest(Collections.singletonList(indexRequest), failureHandler, completionHandler); + ingestService.executeBulkRequest(Collections.singletonList(indexRequest), failureHandler, completionHandler, indexReq -> {}); verify(processor).execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), emptyMap())); verify(failureHandler, times(1)).accept(eq(indexRequest), any(RuntimeException.class)); verify(completionHandler, times(1)).accept(null); @@ -707,7 +707,7 @@ public void testBulkRequestExecutionWithFailures() throws Exception { BiConsumer requestItemErrorHandler = mock(BiConsumer.class); @SuppressWarnings("unchecked") Consumer completionHandler = mock(Consumer.class); - ingestService.executeBulkRequest(bulkRequest.requests(), requestItemErrorHandler, completionHandler); + ingestService.executeBulkRequest(bulkRequest.requests(), requestItemErrorHandler, completionHandler, indexReq -> {}); verify(requestItemErrorHandler, times(numIndexRequests)).accept(any(IndexRequest.class), argThat(new ArgumentMatcher() { @Override @@ -741,7 +741,7 @@ public void testBulkRequestExecution() { BiConsumer requestItemErrorHandler = mock(BiConsumer.class); @SuppressWarnings("unchecked") Consumer completionHandler = mock(Consumer.class); - ingestService.executeBulkRequest(bulkRequest.requests(), requestItemErrorHandler, completionHandler); + ingestService.executeBulkRequest(bulkRequest.requests(), requestItemErrorHandler, completionHandler, indexReq -> {}); verify(requestItemErrorHandler, never()).accept(any(), any()); verify(completionHandler, times(1)).accept(null); @@ -779,7 +779,7 @@ public void testStats() { final IndexRequest indexRequest = new IndexRequest("_index"); indexRequest.setPipeline("_id1"); - ingestService.executeBulkRequest(Collections.singletonList(indexRequest), failureHandler, completionHandler); + ingestService.executeBulkRequest(Collections.singletonList(indexRequest), failureHandler, completionHandler, indexReq -> {}); final IngestStats afterFirstRequestStats = ingestService.stats(); assertThat(afterFirstRequestStats.getStatsPerPipeline().size(), equalTo(2)); assertThat(afterFirstRequestStats.getStatsPerPipeline().get("_id1").getIngestCount(), equalTo(1L)); @@ -787,7 +787,7 @@ public void testStats() { assertThat(afterFirstRequestStats.getTotalStats().getIngestCount(), equalTo(1L)); indexRequest.setPipeline("_id2"); - ingestService.executeBulkRequest(Collections.singletonList(indexRequest), failureHandler, completionHandler); + ingestService.executeBulkRequest(Collections.singletonList(indexRequest), failureHandler, completionHandler, indexReq -> {}); final IngestStats afterSecondRequestStats = ingestService.stats(); assertThat(afterSecondRequestStats.getStatsPerPipeline().size(), equalTo(2)); assertThat(afterSecondRequestStats.getStatsPerPipeline().get("_id1").getIngestCount(), equalTo(1L)); @@ -827,8 +827,9 @@ private static IngestService createWithProcessors() { String value = (String) config.remove("value"); return new Processor() { @Override - public void execute(IngestDocument ingestDocument) { + public IngestDocument execute(IngestDocument ingestDocument) { ingestDocument.setFieldValue(field, value); + return ingestDocument; } @Override @@ -846,8 +847,9 @@ public String getTag() { String field = (String) config.remove("field"); return new Processor() { @Override - public void execute(IngestDocument ingestDocument) { + public IngestDocument execute(IngestDocument ingestDocument) { ingestDocument.removeField(field); + return ingestDocument; } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java b/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java index 4e4c5a24c0cb6..a1feb3e1f73be 100644 --- a/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java +++ b/test/framework/src/main/java/org/elasticsearch/ingest/TestProcessor.java @@ -45,9 +45,10 @@ public TestProcessor(String tag, String type, Consumer ingestDoc } @Override - public void execute(IngestDocument ingestDocument) throws Exception { + public IngestDocument execute(IngestDocument ingestDocument) throws Exception { invokedCounter.incrementAndGet(); ingestDocumentConsumer.accept(ingestDocument); + return ingestDocument; } @Override diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MockIngestPlugin.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MockIngestPlugin.java index 818ab374d3495..b4521ad58b222 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MockIngestPlugin.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MockIngestPlugin.java @@ -74,8 +74,9 @@ static class MockProcessor implements Processor { } @Override - public void execute(IngestDocument ingestDocument) throws Exception { + public IngestDocument execute(IngestDocument ingestDocument) throws Exception { // mock processor does nothing + return ingestDocument; } @Override diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessor.java index 15ac88b4d9462..0c30af1879cc9 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessor.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessor.java @@ -43,7 +43,7 @@ public SetSecurityUserProcessor(String tag, ThreadContext threadContext, String } @Override - public void execute(IngestDocument ingestDocument) throws Exception { + public IngestDocument execute(IngestDocument ingestDocument) throws Exception { Authentication authentication = Authentication.getAuthentication(threadContext); if (authentication == null) { throw new IllegalStateException("No user authenticated, only use this processor via authenticated user"); @@ -86,6 +86,7 @@ public void execute(IngestDocument ingestDocument) throws Exception { } } ingestDocument.setFieldValue(field, userObject); + return ingestDocument; } @Override From 913d5fd820477c9277c6c34a757cab258bbe9041 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 5 Sep 2018 14:52:47 +0200 Subject: [PATCH 05/10] Disable IndexRecoveryIT.testRerouteRecovery. Relates #32686. --- .../java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java index 81afab4bb8f7e..213aa644665dd 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java @@ -248,6 +248,7 @@ public void testReplicaRecovery() throws Exception { validateIndexRecoveryState(nodeBRecoveryState.getIndex()); } + @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/32686") @TestLogging( "_root:DEBUG," + "org.elasticsearch.cluster.service:TRACE," From 74b87989d976395b333bb478bd99e760b0878344 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 5 Sep 2018 09:17:02 -0400 Subject: [PATCH 06/10] Build: Merge xpack checkstyle config into core (#33399) Solves all of the xpack line length suppressions and then merges the remainder of the xpack checkstyle_suppressions.xml file into the core checkstyle_suppressions.xml file. At this point that just means the antlr generated files for sql. It also adds an exclusion to the line length tests for javadocs that are just a URL. We have one such javadoc and breaking up the line would make the link difficult to use. --- buildSrc/src/main/resources/checkstyle.xml | 1 + .../resources/checkstyle_suppressions.xml | 1 + x-pack/build.gradle | 9 ------ x-pack/dev-tools/checkstyle_suppressions.xml | 29 ------------------- .../authc/ldap/ActiveDirectorySIDUtil.java | 3 +- .../user/TransportGetUsersActionTests.java | 16 +++++----- .../security/authc/file/FileRealmTests.java | 3 +- .../authc/ldap/LdapSessionFactoryTests.java | 28 +++++++++++------- .../security/authc/pki/PkiRealmTests.java | 3 +- 9 files changed, 33 insertions(+), 60 deletions(-) delete mode 100644 x-pack/dev-tools/checkstyle_suppressions.xml diff --git a/buildSrc/src/main/resources/checkstyle.xml b/buildSrc/src/main/resources/checkstyle.xml index 033f020fde0fa..e1000b3e4a9f8 100644 --- a/buildSrc/src/main/resources/checkstyle.xml +++ b/buildSrc/src/main/resources/checkstyle.xml @@ -23,6 +23,7 @@ unfair. --> + diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml index 418d366a5cd6d..e9dc4918d6932 100644 --- a/buildSrc/src/main/resources/checkstyle_suppressions.xml +++ b/buildSrc/src/main/resources/checkstyle_suppressions.xml @@ -9,6 +9,7 @@ + diff --git a/x-pack/build.gradle b/x-pack/build.gradle index 4ba4c28ee630e..d2a19be2136fb 100644 --- a/x-pack/build.gradle +++ b/x-pack/build.gradle @@ -31,16 +31,7 @@ subprojects { } } -File checkstyleSuppressions = file('dev-tools/checkstyle_suppressions.xml') subprojects { - tasks.withType(Checkstyle) { - inputs.file(checkstyleSuppressions) - // Use x-pack-elasticsearch specific suppressions file rather than the open source one. - configProperties = [ - suppressions: checkstyleSuppressions - ] - } - ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-ccr:${version}": xpackModule('ccr')] ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-core:${version}": xpackModule('core')] ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-deprecation:${version}": xpackModule('deprecation')] diff --git a/x-pack/dev-tools/checkstyle_suppressions.xml b/x-pack/dev-tools/checkstyle_suppressions.xml deleted file mode 100644 index 4748436a84979..0000000000000 --- a/x-pack/dev-tools/checkstyle_suppressions.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySIDUtil.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySIDUtil.java index e8f418d891bc3..d93ba9e017e1f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySIDUtil.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySIDUtil.java @@ -18,7 +18,8 @@ */ /* - * This code sourced from:http://svn.apache.org/repos/asf/directory/studio/tags/2.0.0.v20170904-M13/plugins/valueeditors/src/main/java/org/apache/directory/studio/valueeditors/msad/InPlaceMsAdObjectSidValueEditor.java + * This code sourced from: + * http://svn.apache.org/repos/asf/directory/studio/tags/2.0.0.v20170904-M13/plugins/valueeditors/src/main/java/org/apache/directory/studio/valueeditors/msad/InPlaceMsAdObjectSidValueEditor.java */ package org.elasticsearch.xpack.security.authc.ldap; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersActionTests.java index 1c5f93187c059..33cec72060886 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportGetUsersActionTests.java @@ -169,8 +169,8 @@ public void testReservedUsersOnly() { final int size = randomIntBetween(1, allReservedUsers.size()); final List reservedUsers = randomSubsetOf(size, allReservedUsers); final List names = reservedUsers.stream().map(User::principal).collect(Collectors.toList()); - TransportService transportService = new TransportService(Settings.EMPTY, mock(Transport.class), null, TransportService.NOOP_TRANSPORT_INTERCEPTOR, - x -> null, null, Collections.emptySet()); + TransportService transportService = new TransportService(Settings.EMPTY, mock(Transport.class), null, + TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, Collections.emptySet()); TransportGetUsersAction action = new TransportGetUsersAction(Settings.EMPTY, mock(ActionFilters.class), usersStore, transportService, reservedRealm); @@ -209,8 +209,8 @@ public void testGetAllUsers() { ReservedRealmTests.mockGetAllReservedUserInfo(usersStore, Collections.emptyMap()); ReservedRealm reservedRealm = new ReservedRealm(mock(Environment.class), settings, usersStore, new AnonymousUser(settings), securityIndex, threadPool); - TransportService transportService = new TransportService(Settings.EMPTY, mock(Transport.class), null, TransportService.NOOP_TRANSPORT_INTERCEPTOR, - x -> null, null, Collections.emptySet()); + TransportService transportService = new TransportService(Settings.EMPTY, mock(Transport.class), null, + TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, Collections.emptySet()); TransportGetUsersAction action = new TransportGetUsersAction(Settings.EMPTY, mock(ActionFilters.class), usersStore, transportService, reservedRealm); @@ -256,8 +256,8 @@ public void testGetStoreOnlyUsers() { randomFrom(Collections.singletonList(new User("joe")), Arrays.asList(new User("jane"), new User("fred")), randomUsers()); final String[] storeUsernames = storeUsers.stream().map(User::principal).collect(Collectors.toList()).toArray(Strings.EMPTY_ARRAY); NativeUsersStore usersStore = mock(NativeUsersStore.class); - TransportService transportService = new TransportService(Settings.EMPTY, mock(Transport.class), null, TransportService.NOOP_TRANSPORT_INTERCEPTOR, - x -> null, null, Collections.emptySet()); + TransportService transportService = new TransportService(Settings.EMPTY, mock(Transport.class), null, + TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, Collections.emptySet()); TransportGetUsersAction action = new TransportGetUsersAction(Settings.EMPTY, mock(ActionFilters.class), usersStore, transportService, mock(ReservedRealm.class)); @@ -304,8 +304,8 @@ public void testException() { randomFrom(Collections.singletonList(new User("joe")), Arrays.asList(new User("jane"), new User("fred")), randomUsers()); final String[] storeUsernames = storeUsers.stream().map(User::principal).collect(Collectors.toList()).toArray(Strings.EMPTY_ARRAY); NativeUsersStore usersStore = mock(NativeUsersStore.class); - TransportService transportService = new TransportService(Settings.EMPTY, mock(Transport.class), null, TransportService.NOOP_TRANSPORT_INTERCEPTOR, - x -> null, null, Collections.emptySet()); + TransportService transportService = new TransportService(Settings.EMPTY, mock(Transport.class), null, + TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> null, null, Collections.emptySet()); TransportGetUsersAction action = new TransportGetUsersAction(Settings.EMPTY, mock(ActionFilters.class), usersStore, transportService, mock(ReservedRealm.class)); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileRealmTests.java index f5dad8b7c684c..1310980fc5f7c 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/file/FileRealmTests.java @@ -86,7 +86,8 @@ public void testAuthenticate() throws Exception { public void testAuthenticateCaching() throws Exception { Settings settings = Settings.builder() - .put("cache.hash_algo", Hasher.values()[randomIntBetween(0, Hasher.values().length - 1)].name().toLowerCase(Locale.ROOT)).build(); + .put("cache.hash_algo", Hasher.values()[randomIntBetween(0, Hasher.values().length - 1)].name().toLowerCase(Locale.ROOT)) + .build(); RealmConfig config = new RealmConfig("file-test", settings, globalSettings, TestEnvironment.newEnvironment(globalSettings), threadContext); when(userPasswdStore.verifyPassword(eq("user1"), eq(new SecureString("test123")), any(Supplier.class))) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapSessionFactoryTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapSessionFactoryTests.java index f719546cade87..a22cc9fba1779 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapSessionFactoryTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapSessionFactoryTests.java @@ -63,7 +63,8 @@ public void testBindWithReadTimeout() throws Exception { .put("path.home", createTempDir()) .build(); - RealmConfig config = new RealmConfig("ldap_realm", settings, globalSettings, TestEnvironment.newEnvironment(globalSettings), new ThreadContext(globalSettings)); + RealmConfig config = new RealmConfig("ldap_realm", settings, globalSettings, + TestEnvironment.newEnvironment(globalSettings), new ThreadContext(globalSettings)); LdapSessionFactory sessionFactory = new LdapSessionFactory(config, sslService, threadPool); String user = "Horatio Hornblower"; SecureString userPass = new SecureString("pass"); @@ -87,8 +88,9 @@ public void testBindWithTemplates() throws Exception { "wrongname={0},ou=people,o=sevenSeas", "cn={0},ou=people,o=sevenSeas", //this last one should work }; - RealmConfig config = new RealmConfig("ldap_realm", buildLdapSettings(ldapUrls(), userTemplates, groupSearchBase, - LdapSearchScope.SUB_TREE), globalSettings, TestEnvironment.newEnvironment(globalSettings), new ThreadContext(globalSettings)); + RealmConfig config = new RealmConfig("ldap_realm", + buildLdapSettings(ldapUrls(), userTemplates, groupSearchBase, LdapSearchScope.SUB_TREE), + globalSettings, TestEnvironment.newEnvironment(globalSettings), new ThreadContext(globalSettings)); LdapSessionFactory sessionFactory = new LdapSessionFactory(config, sslService, threadPool); @@ -110,8 +112,9 @@ public void testBindWithBogusTemplates() throws Exception { "wrongname={0},ou=people,o=sevenSeas", "asdf={0},ou=people,o=sevenSeas", //none of these should work }; - RealmConfig config = new RealmConfig("ldap_realm", buildLdapSettings(ldapUrls(), userTemplates, groupSearchBase, - LdapSearchScope.SUB_TREE), globalSettings, TestEnvironment.newEnvironment(globalSettings), new ThreadContext(globalSettings)); + RealmConfig config = new RealmConfig("ldap_realm", + buildLdapSettings(ldapUrls(), userTemplates, groupSearchBase, LdapSearchScope.SUB_TREE), + globalSettings, TestEnvironment.newEnvironment(globalSettings), new ThreadContext(globalSettings)); LdapSessionFactory ldapFac = new LdapSessionFactory(config, sslService, threadPool); @@ -128,8 +131,9 @@ public void testBindWithBogusTemplates() throws Exception { public void testGroupLookupSubtree() throws Exception { String groupSearchBase = "o=sevenSeas"; String userTemplate = "cn={0},ou=people,o=sevenSeas"; - RealmConfig config = new RealmConfig("ldap_realm", buildLdapSettings(ldapUrls(), userTemplate, groupSearchBase, - LdapSearchScope.SUB_TREE), globalSettings, TestEnvironment.newEnvironment(globalSettings), new ThreadContext(globalSettings)); + RealmConfig config = new RealmConfig("ldap_realm", + buildLdapSettings(ldapUrls(), userTemplate, groupSearchBase, LdapSearchScope.SUB_TREE), + globalSettings, TestEnvironment.newEnvironment(globalSettings), new ThreadContext(globalSettings)); LdapSessionFactory ldapFac = new LdapSessionFactory(config, sslService, threadPool); @@ -147,8 +151,9 @@ public void testGroupLookupSubtree() throws Exception { public void testGroupLookupOneLevel() throws Exception { String groupSearchBase = "ou=crews,ou=groups,o=sevenSeas"; String userTemplate = "cn={0},ou=people,o=sevenSeas"; - RealmConfig config = new RealmConfig("ldap_realm", buildLdapSettings(ldapUrls(), userTemplate, groupSearchBase, - LdapSearchScope.ONE_LEVEL), globalSettings, TestEnvironment.newEnvironment(globalSettings), new ThreadContext(globalSettings)); + RealmConfig config = new RealmConfig("ldap_realm", + buildLdapSettings(ldapUrls(), userTemplate, groupSearchBase, LdapSearchScope.ONE_LEVEL), + globalSettings, TestEnvironment.newEnvironment(globalSettings), new ThreadContext(globalSettings)); LdapSessionFactory ldapFac = new LdapSessionFactory(config, sslService, threadPool); @@ -165,8 +170,9 @@ public void testGroupLookupOneLevel() throws Exception { public void testGroupLookupBase() throws Exception { String groupSearchBase = "cn=HMS Lydia,ou=crews,ou=groups,o=sevenSeas"; String userTemplate = "cn={0},ou=people,o=sevenSeas"; - RealmConfig config = new RealmConfig("ldap_realm", buildLdapSettings(ldapUrls(), userTemplate, groupSearchBase, - LdapSearchScope.BASE), globalSettings, TestEnvironment.newEnvironment(globalSettings), new ThreadContext(globalSettings)); + RealmConfig config = new RealmConfig("ldap_realm", + buildLdapSettings(ldapUrls(), userTemplate, groupSearchBase, LdapSearchScope.BASE), + globalSettings, TestEnvironment.newEnvironment(globalSettings), new ThreadContext(globalSettings)); LdapSessionFactory ldapFac = new LdapSessionFactory(config, sslService, threadPool); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiRealmTests.java index 8d4c5d75c7351..45ccaf6a14725 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiRealmTests.java @@ -74,7 +74,8 @@ public void setup() throws Exception { } public void testTokenSupport() { - RealmConfig config = new RealmConfig("", Settings.EMPTY, globalSettings, TestEnvironment.newEnvironment(globalSettings), new ThreadContext(globalSettings)); + RealmConfig config = new RealmConfig("", Settings.EMPTY, globalSettings, + TestEnvironment.newEnvironment(globalSettings), new ThreadContext(globalSettings)); PkiRealm realm = new PkiRealm(config, mock(UserRoleMapper.class)); assertThat(realm.supports(null), is(false)); From 636442700c02bdf929cad6dd47f18a84635309c4 Mon Sep 17 00:00:00 2001 From: Alan Woodward Date: Wed, 5 Sep 2018 14:52:43 +0100 Subject: [PATCH 07/10] Add conditional token filter to elasticsearch (#31958) This allows tokenfilters to be applied selectively, depending on the status of the current token in the tokenstream. The filter takes a scripted predicate, and only applies its subfilter when the predicate returns true. --- .../painless/painless-contexts/index.asciidoc | 2 + ...inless-analysis-predicate-context.asciidoc | 43 +++++++ docs/reference/analysis/tokenfilters.asciidoc | 2 + .../condition-tokenfilter.asciidoc | 90 ++++++++++++++ modules/analysis-common/build.gradle | 9 ++ .../common/AnalysisPainlessExtension.java | 40 ++++++ .../common/AnalysisPredicateScript.java | 87 +++++++++++++ .../analysis/common/CommonAnalysisPlugin.java | 35 +++++- .../ScriptedConditionTokenFilterFactory.java | 117 ++++++++++++++++++ ...asticsearch.painless.spi.PainlessExtension | 1 + .../analysis/common/painless_whitelist.txt | 28 +++++ .../ScriptedConditionTokenFilterTests.java | 89 +++++++++++++ .../analysis-common/60_analysis_scripting.yml | 36 ++++++ 13 files changed, 578 insertions(+), 1 deletion(-) create mode 100644 docs/painless/painless-contexts/painless-analysis-predicate-context.asciidoc create mode 100644 docs/reference/analysis/tokenfilters/condition-tokenfilter.asciidoc create mode 100644 modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AnalysisPainlessExtension.java create mode 100644 modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AnalysisPredicateScript.java create mode 100644 modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterFactory.java create mode 100644 modules/analysis-common/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension create mode 100644 modules/analysis-common/src/main/resources/org/elasticsearch/analysis/common/painless_whitelist.txt create mode 100644 modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterTests.java create mode 100644 modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/60_analysis_scripting.yml diff --git a/docs/painless/painless-contexts/index.asciidoc b/docs/painless/painless-contexts/index.asciidoc index a9d3982133e1b..a71fde0be32a0 100644 --- a/docs/painless/painless-contexts/index.asciidoc +++ b/docs/painless/painless-contexts/index.asciidoc @@ -30,6 +30,8 @@ include::painless-metric-agg-reduce-context.asciidoc[] include::painless-bucket-agg-context.asciidoc[] +include::painless-analysis-predicate-context.asciidoc[] + include::painless-watcher-condition-context.asciidoc[] include::painless-watcher-transform-context.asciidoc[] diff --git a/docs/painless/painless-contexts/painless-analysis-predicate-context.asciidoc b/docs/painless/painless-contexts/painless-analysis-predicate-context.asciidoc new file mode 100644 index 0000000000000..07914b671e781 --- /dev/null +++ b/docs/painless/painless-contexts/painless-analysis-predicate-context.asciidoc @@ -0,0 +1,43 @@ +[[painless-analysis-predicate-context]] +=== Analysis Predicate Context + +Use a painless script to determine whether or not the current token in an +analysis chain matches a predicate. + +*Variables* + +`params` (`Map`, read-only):: + User-defined parameters passed in as part of the query. + +`token.term` (`CharSequence`, read-only):: + The characters of the current token + +`token.position` (`int`, read-only):: + The position of the current token + +`token.positionIncrement` (`int`, read-only):: + The position increment of the current token + +`token.positionLength` (`int`, read-only):: + The position length of the current token + +`token.startOffset` (`int`, read-only):: + The start offset of the current token + +`token.endOffset` (`int`, read-only):: + The end offset of the current token + +`token.type` (`String`, read-only):: + The type of the current token + +`token.keyword` ('boolean`, read-only):: + Whether or not the current token is marked as a keyword + +*Return* + +`boolean`:: + Whether or not the current token matches the predicate + +*API* + +The standard <> is available. \ No newline at end of file diff --git a/docs/reference/analysis/tokenfilters.asciidoc b/docs/reference/analysis/tokenfilters.asciidoc index ee891fdd09aa7..5899744247899 100644 --- a/docs/reference/analysis/tokenfilters.asciidoc +++ b/docs/reference/analysis/tokenfilters.asciidoc @@ -37,6 +37,8 @@ include::tokenfilters/word-delimiter-graph-tokenfilter.asciidoc[] include::tokenfilters/multiplexer-tokenfilter.asciidoc[] +include::tokenfilters/condition-tokenfilter.asciidoc[] + include::tokenfilters/stemmer-tokenfilter.asciidoc[] include::tokenfilters/stemmer-override-tokenfilter.asciidoc[] diff --git a/docs/reference/analysis/tokenfilters/condition-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/condition-tokenfilter.asciidoc new file mode 100644 index 0000000000000..cff05559ab9e6 --- /dev/null +++ b/docs/reference/analysis/tokenfilters/condition-tokenfilter.asciidoc @@ -0,0 +1,90 @@ +[[analysis-condition-tokenfilter]] +=== Conditional Token Filter + +The conditional token filter takes a predicate script and a list of subfilters, and +only applies the subfilters to the current token if it matches the predicate. + +[float] +=== Options +[horizontal] +filter:: a chain of token filters to apply to the current token if the predicate + matches. These can be any token filters defined elsewhere in the index mappings. + +script:: a predicate script that determines whether or not the filters will be applied + to the current token. Note that only inline scripts are supported + +[float] +=== Settings example + +You can set it up like: + +[source,js] +-------------------------------------------------- +PUT /condition_example +{ + "settings" : { + "analysis" : { + "analyzer" : { + "my_analyzer" : { + "tokenizer" : "standard", + "filter" : [ "my_condition" ] + } + }, + "filter" : { + "my_condition" : { + "type" : "condition", + "filter" : [ "lowercase" ], + "script" : { + "source" : "token.getTerm().length() < 5" <1> + } + } + } + } + } +} +-------------------------------------------------- +// CONSOLE + +<1> This will only apply the lowercase filter to terms that are less than 5 +characters in length + +And test it like: + +[source,js] +-------------------------------------------------- +POST /condition_example/_analyze +{ + "analyzer" : "my_analyzer", + "text" : "What Flapdoodle" +} +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +And it'd respond: + +[source,js] +-------------------------------------------------- +{ + "tokens": [ + { + "token": "what", <1> + "start_offset": 0, + "end_offset": 4, + "type": "", + "position": 0 + }, + { + "token": "Flapdoodle", <2> + "start_offset": 5, + "end_offset": 15, + "type": "", + "position": 1 + } + ] +} +-------------------------------------------------- +// TESTRESPONSE +<1> The term `What` has been lowercased, because it is only 4 characters long +<2> The term `Flapdoodle` has been left in its original case, because it doesn't pass + the predicate \ No newline at end of file diff --git a/modules/analysis-common/build.gradle b/modules/analysis-common/build.gradle index 391b74934c97d..e5193ab3c8451 100644 --- a/modules/analysis-common/build.gradle +++ b/modules/analysis-common/build.gradle @@ -20,4 +20,13 @@ esplugin { description 'Adds "built in" analyzers to Elasticsearch.' classname 'org.elasticsearch.analysis.common.CommonAnalysisPlugin' + extendedPlugins = ['lang-painless'] +} + +dependencies { + compileOnly project(':modules:lang-painless') +} + +integTestCluster { + module project(':modules:lang-painless') } diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AnalysisPainlessExtension.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AnalysisPainlessExtension.java new file mode 100644 index 0000000000000..85abec4ce915c --- /dev/null +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AnalysisPainlessExtension.java @@ -0,0 +1,40 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.analysis.common; + +import org.elasticsearch.painless.spi.PainlessExtension; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScriptContext; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class AnalysisPainlessExtension implements PainlessExtension { + + private static final Whitelist WHITELIST = + WhitelistLoader.loadFromResourceFiles(AnalysisPainlessExtension.class, "painless_whitelist.txt"); + + @Override + public Map, List> getContextWhitelists() { + return Collections.singletonMap(AnalysisPredicateScript.CONTEXT, Collections.singletonList(WHITELIST)); + } +} diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AnalysisPredicateScript.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AnalysisPredicateScript.java new file mode 100644 index 0000000000000..7de588a958c77 --- /dev/null +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/AnalysisPredicateScript.java @@ -0,0 +1,87 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.analysis.common; + +import org.elasticsearch.script.ScriptContext; + +/** + * A predicate based on the current token in a TokenStream + */ +public abstract class AnalysisPredicateScript { + + /** + * Encapsulation of the state of the current token + */ + public static class Token { + public CharSequence term; + public int pos; + public int posInc; + public int posLen; + public int startOffset; + public int endOffset; + public String type; + public boolean isKeyword; + + public CharSequence getTerm() { + return term; + } + + public int getPositionIncrement() { + return posInc; + } + + public int getPosition() { + return pos; + } + + public int getPositionLength() { + return posLen; + } + + public int getStartOffset() { + return startOffset; + } + + public int getEndOffset() { + return endOffset; + } + + public String getType() { + return type; + } + + public boolean isKeyword() { + return isKeyword; + } + } + + /** + * Returns {@code true} if the current term matches the predicate + */ + public abstract boolean execute(Token token); + + public interface Factory { + AnalysisPredicateScript newInstance(); + } + + public static final String[] PARAMETERS = new String[]{ "token" }; + public static final ScriptContext CONTEXT = new ScriptContext<>("analysis", Factory.class); + +} diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java index d95af920a307b..bbd721169c6c7 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java @@ -111,9 +111,16 @@ import org.apache.lucene.analysis.tr.ApostropheFilter; import org.apache.lucene.analysis.tr.TurkishAnalyzer; import org.apache.lucene.analysis.util.ElisionFilter; +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.analysis.AnalyzerProvider; import org.elasticsearch.index.analysis.CharFilterFactory; import org.elasticsearch.index.analysis.PreBuiltAnalyzerProviderFactory; @@ -127,20 +134,44 @@ import org.elasticsearch.indices.analysis.PreBuiltCacheFactory.CachingStrategy; import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.ScriptPlugin; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; import org.tartarus.snowball.ext.DutchStemmer; import org.tartarus.snowball.ext.FrenchStemmer; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.TreeMap; import static org.elasticsearch.plugins.AnalysisPlugin.requiresAnalysisSettings; -public class CommonAnalysisPlugin extends Plugin implements AnalysisPlugin { +public class CommonAnalysisPlugin extends Plugin implements AnalysisPlugin, ScriptPlugin { private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(CommonAnalysisPlugin.class)); + private final SetOnce scriptService = new SetOnce<>(); + + @Override + public Collection createComponents(Client client, ClusterService clusterService, ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, ScriptService scriptService, + NamedXContentRegistry xContentRegistry, Environment environment, + NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry) { + this.scriptService.set(scriptService); + return Collections.emptyList(); + } + + @Override + @SuppressWarnings("rawtypes") // TODO ScriptPlugin needs to change this to pass precommit? + public List getContexts() { + return Collections.singletonList(AnalysisPredicateScript.CONTEXT); + } + @Override public Map>> getAnalyzers() { Map>> analyzers = new TreeMap<>(); @@ -202,6 +233,8 @@ public Map> getTokenFilters() { filters.put("classic", ClassicFilterFactory::new); filters.put("czech_stem", CzechStemTokenFilterFactory::new); filters.put("common_grams", requiresAnalysisSettings(CommonGramsTokenFilterFactory::new)); + filters.put("condition", + requiresAnalysisSettings((i, e, n, s) -> new ScriptedConditionTokenFilterFactory(i, n, s, scriptService.get()))); filters.put("decimal_digit", DecimalDigitFilterFactory::new); filters.put("delimited_payload_filter", LegacyDelimitedPayloadTokenFilterFactory::new); filters.put("delimited_payload", DelimitedPayloadTokenFilterFactory::new); diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterFactory.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterFactory.java new file mode 100644 index 0000000000000..cf7fd5b047a89 --- /dev/null +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterFactory.java @@ -0,0 +1,117 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.analysis.common; + +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.miscellaneous.ConditionalTokenFilter; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.KeywordAttribute; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute; +import org.apache.lucene.analysis.tokenattributes.TypeAttribute; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.AbstractTokenFilterFactory; +import org.elasticsearch.index.analysis.ReferringFilterFactory; +import org.elasticsearch.index.analysis.TokenFilterFactory; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.ScriptType; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +/** + * A factory for a conditional token filter that only applies child filters if the underlying token + * matches an {@link AnalysisPredicateScript} + */ +public class ScriptedConditionTokenFilterFactory extends AbstractTokenFilterFactory implements ReferringFilterFactory { + + private final AnalysisPredicateScript.Factory factory; + private final List filters = new ArrayList<>(); + private final List filterNames; + + ScriptedConditionTokenFilterFactory(IndexSettings indexSettings, String name, + Settings settings, ScriptService scriptService) { + super(indexSettings, name, settings); + + Settings scriptSettings = settings.getAsSettings("script"); + Script script = Script.parse(scriptSettings); + if (script.getType() != ScriptType.INLINE) { + throw new IllegalArgumentException("Cannot use stored scripts in tokenfilter [" + name + "]"); + } + this.factory = scriptService.compile(script, AnalysisPredicateScript.CONTEXT); + + this.filterNames = settings.getAsList("filter"); + if (this.filterNames.isEmpty()) { + throw new IllegalArgumentException("Empty list of filters provided to tokenfilter [" + name + "]"); + } + } + + @Override + public TokenStream create(TokenStream tokenStream) { + Function filter = in -> { + for (TokenFilterFactory tff : filters) { + in = tff.create(in); + } + return in; + }; + AnalysisPredicateScript script = factory.newInstance(); + final AnalysisPredicateScript.Token token = new AnalysisPredicateScript.Token(); + return new ConditionalTokenFilter(tokenStream, filter) { + + CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class); + PositionLengthAttribute posLenAtt = addAttribute(PositionLengthAttribute.class); + OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); + TypeAttribute typeAtt = addAttribute(TypeAttribute.class); + KeywordAttribute keywordAtt = addAttribute(KeywordAttribute.class); + + @Override + protected boolean shouldFilter() { + token.term = termAtt; + token.posInc = posIncAtt.getPositionIncrement(); + token.pos += token.posInc; + token.posLen = posLenAtt.getPositionLength(); + token.startOffset = offsetAtt.startOffset(); + token.endOffset = offsetAtt.endOffset(); + token.type = typeAtt.type(); + token.isKeyword = keywordAtt.isKeyword(); + return script.execute(token); + } + }; + } + + @Override + public void setReferences(Map factories) { + for (String filter : filterNames) { + TokenFilterFactory tff = factories.get(filter); + if (tff == null) { + throw new IllegalArgumentException("ScriptedConditionTokenFilter [" + name() + + "] refers to undefined token filter [" + filter + "]"); + } + filters.add(tff); + } + } + +} diff --git a/modules/analysis-common/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension b/modules/analysis-common/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension new file mode 100644 index 0000000000000..44e98a3dd9c68 --- /dev/null +++ b/modules/analysis-common/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension @@ -0,0 +1 @@ +org.elasticsearch.analysis.common.AnalysisPainlessExtension \ No newline at end of file diff --git a/modules/analysis-common/src/main/resources/org/elasticsearch/analysis/common/painless_whitelist.txt b/modules/analysis-common/src/main/resources/org/elasticsearch/analysis/common/painless_whitelist.txt new file mode 100644 index 0000000000000..83b70be58774e --- /dev/null +++ b/modules/analysis-common/src/main/resources/org/elasticsearch/analysis/common/painless_whitelist.txt @@ -0,0 +1,28 @@ +# +# Licensed to Elasticsearch under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +class org.elasticsearch.analysis.common.AnalysisPredicateScript$Token { + CharSequence getTerm() + int getPosition() + int getPositionIncrement() + int getPositionLength() + int getStartOffset() + int getEndOffset() + String getType() + boolean isKeyword() +} \ No newline at end of file diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterTests.java new file mode 100644 index 0000000000000..39134ef1f532b --- /dev/null +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/ScriptedConditionTokenFilterTests.java @@ -0,0 +1,89 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.analysis.common; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.IndexAnalyzers; +import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.indices.analysis.AnalysisModule; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.test.ESTokenStreamTestCase; +import org.elasticsearch.test.IndexSettingsModule; + +import java.util.Collections; + +public class ScriptedConditionTokenFilterTests extends ESTokenStreamTestCase { + + public void testSimpleCondition() throws Exception { + Settings settings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .build(); + Settings indexSettings = Settings.builder() + .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT) + .put("index.analysis.filter.cond.type", "condition") + .put("index.analysis.filter.cond.script.source", "token.getTerm().length() > 5") + .putList("index.analysis.filter.cond.filter", "uppercase") + .put("index.analysis.analyzer.myAnalyzer.type", "custom") + .put("index.analysis.analyzer.myAnalyzer.tokenizer", "standard") + .putList("index.analysis.analyzer.myAnalyzer.filter", "cond") + .build(); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings("index", indexSettings); + + AnalysisPredicateScript.Factory factory = () -> new AnalysisPredicateScript() { + @Override + public boolean execute(Token token) { + return token.getTerm().length() > 5; + } + }; + + @SuppressWarnings("unchecked") + ScriptService scriptService = new ScriptService(indexSettings, Collections.emptyMap(), Collections.emptyMap()){ + @Override + public FactoryType compile(Script script, ScriptContext context) { + assertEquals(context, AnalysisPredicateScript.CONTEXT); + assertEquals(new Script("token.getTerm().length() > 5"), script); + return (FactoryType) factory; + } + }; + + CommonAnalysisPlugin plugin = new CommonAnalysisPlugin(); + plugin.createComponents(null, null, null, null, scriptService, null, null, null, null); + AnalysisModule module + = new AnalysisModule(TestEnvironment.newEnvironment(settings), Collections.singletonList(plugin)); + + IndexAnalyzers analyzers = module.getAnalysisRegistry().build(idxSettings); + + try (NamedAnalyzer analyzer = analyzers.get("myAnalyzer")) { + assertNotNull(analyzer); + assertAnalyzesTo(analyzer, "Vorsprung Durch Technik", new String[]{ + "VORSPRUNG", "Durch", "TECHNIK" + }); + } + + } + +} diff --git a/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/60_analysis_scripting.yml b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/60_analysis_scripting.yml new file mode 100644 index 0000000000000..4305e5db0af37 --- /dev/null +++ b/modules/analysis-common/src/test/resources/rest-api-spec/test/analysis-common/60_analysis_scripting.yml @@ -0,0 +1,36 @@ +## Test analysis scripts + +"condition": + - do: + indices.analyze: + body: + text: "Vorsprung Durch Technik" + tokenizer: "whitespace" + filter: + - type: condition + filter: [ "lowercase" ] + script: + source: "token.term.length() > 5" + + - length: { tokens: 3 } + - match: { tokens.0.token: "vorsprung" } + - match: { tokens.1.token: "Durch" } + - match: { tokens.2.token: "technik" } + +--- +"condition-vars": + - do: + indices.analyze: + body: + text: "Vorsprung Durch Technik" + tokenizer: "whitespace" + filter: + - type: condition + filter: [ "lowercase" ] + script: + source: "token.position > 1 && token.positionIncrement > 0 && token.startOffset > 0 && token.endOffset > 0 && (token.positionLength == 1 || token.type == \"a\" || token.keyword)" + + - length: { tokens: 3 } + - match: { tokens.0.token: "Vorsprung" } + - match: { tokens.1.token: "durch" } + - match: { tokens.2.token: "technik" } From 24776b2b8017f083be8d848bfa2f33bb2014d4db Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Wed, 5 Sep 2018 15:06:55 +0100 Subject: [PATCH 08/10] HLRC: Add ML get influencers API (#33389) Relates #29827 --- .../client/MLRequestConverters.java | 15 ++ .../client/MachineLearningClient.java | 51 +++- .../client/ml/GetBucketsRequest.java | 17 +- .../client/ml/GetInfluencersRequest.java | 227 ++++++++++++++++++ .../client/ml/GetInfluencersResponse.java | 78 ++++++ .../client/ml/GetOverallBucketsRequest.java | 4 +- .../client/ml/GetRecordsRequest.java | 18 +- .../client/ml/GetRecordsResponse.java | 6 +- .../client/MLRequestConvertersTests.java | 21 ++ .../client/MachineLearningGetResultsIT.java | 103 ++++++++ .../MlClientDocumentationIT.java | 94 ++++++++ .../client/ml/GetInfluencersRequestTests.java | 71 ++++++ .../ml/GetInfluencersResponseTests.java | 53 ++++ .../client/ml/GetRecordsRequestTests.java | 35 ++- .../client/ml/GetRecordsResponseTests.java | 5 +- .../ml/job/results/InfluencerTests.java | 2 +- .../high-level/ml/get-influencers.asciidoc | 112 +++++++++ .../high-level/supported-apis.asciidoc | 2 + 18 files changed, 864 insertions(+), 50 deletions(-) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetInfluencersRequest.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetInfluencersResponse.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetInfluencersRequestTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetInfluencersResponseTests.java create mode 100644 docs/java-rest/high-level/ml/get-influencers.asciidoc diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java index 35898a8a8e4d8..dc4d550e7b9ad 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java @@ -28,6 +28,7 @@ import org.elasticsearch.client.ml.DeleteJobRequest; import org.elasticsearch.client.ml.FlushJobRequest; import org.elasticsearch.client.ml.GetBucketsRequest; +import org.elasticsearch.client.ml.GetInfluencersRequest; import org.elasticsearch.client.ml.GetJobRequest; import org.elasticsearch.client.ml.GetJobStatsRequest; import org.elasticsearch.client.ml.GetOverallBucketsRequest; @@ -186,4 +187,18 @@ static Request getRecords(GetRecordsRequest getRecordsRequest) throws IOExceptio request.setEntity(createEntity(getRecordsRequest, REQUEST_BODY_CONTENT_TYPE)); return request; } + + static Request getInfluencers(GetInfluencersRequest getInfluencersRequest) throws IOException { + String endpoint = new EndpointBuilder() + .addPathPartAsIs("_xpack") + .addPathPartAsIs("ml") + .addPathPartAsIs("anomaly_detectors") + .addPathPart(getInfluencersRequest.getJobId()) + .addPathPartAsIs("results") + .addPathPartAsIs("influencers") + .build(); + Request request = new Request(HttpGet.METHOD_NAME, endpoint); + request.setEntity(createEntity(getInfluencersRequest, REQUEST_BODY_CONTENT_TYPE)); + return request; + } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java index 4757ec3182b40..be5f81076ae90 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java @@ -19,19 +19,20 @@ package org.elasticsearch.client; import org.elasticsearch.action.ActionListener; -import org.elasticsearch.client.ml.FlushJobRequest; -import org.elasticsearch.client.ml.FlushJobResponse; -import org.elasticsearch.client.ml.GetJobStatsRequest; -import org.elasticsearch.client.ml.GetJobStatsResponse; -import org.elasticsearch.client.ml.job.stats.JobStats; import org.elasticsearch.client.ml.CloseJobRequest; import org.elasticsearch.client.ml.CloseJobResponse; import org.elasticsearch.client.ml.DeleteJobRequest; import org.elasticsearch.client.ml.DeleteJobResponse; +import org.elasticsearch.client.ml.FlushJobRequest; +import org.elasticsearch.client.ml.FlushJobResponse; import org.elasticsearch.client.ml.GetBucketsRequest; import org.elasticsearch.client.ml.GetBucketsResponse; +import org.elasticsearch.client.ml.GetInfluencersRequest; +import org.elasticsearch.client.ml.GetInfluencersResponse; import org.elasticsearch.client.ml.GetJobRequest; import org.elasticsearch.client.ml.GetJobResponse; +import org.elasticsearch.client.ml.GetJobStatsRequest; +import org.elasticsearch.client.ml.GetJobStatsResponse; import org.elasticsearch.client.ml.GetOverallBucketsRequest; import org.elasticsearch.client.ml.GetOverallBucketsResponse; import org.elasticsearch.client.ml.GetRecordsRequest; @@ -40,6 +41,7 @@ import org.elasticsearch.client.ml.OpenJobResponse; import org.elasticsearch.client.ml.PutJobRequest; import org.elasticsearch.client.ml.PutJobResponse; +import org.elasticsearch.client.ml.job.stats.JobStats; import java.io.IOException; import java.util.Collections; @@ -464,4 +466,43 @@ public void getRecordsAsync(GetRecordsRequest request, RequestOptions options, A listener, Collections.emptySet()); } + + /** + * Gets the influencers for a Machine Learning Job. + *

+ * For additional info + * see + * ML GET influencers documentation + * + * @param request the request + * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + */ + public GetInfluencersResponse getInfluencers(GetInfluencersRequest request, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(request, + MLRequestConverters::getInfluencers, + options, + GetInfluencersResponse::fromXContent, + Collections.emptySet()); + } + + /** + * Gets the influencers for a Machine Learning Job, notifies listener once the requested influencers are retrieved. + *

+ * For additional info + * * see + * ML GET influencers documentation + * + * @param request the request + * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener Listener to be notified upon request completion + */ + public void getInfluencersAsync(GetInfluencersRequest request, RequestOptions options, + ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(request, + MLRequestConverters::getInfluencers, + options, + GetInfluencersResponse::fromXContent, + listener, + Collections.emptySet()); + } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetBucketsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetBucketsRequest.java index f50d92d58dda5..927fd08c1ca79 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetBucketsRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetBucketsRequest.java @@ -41,7 +41,6 @@ public class GetBucketsRequest extends ActionRequest implements ToXContentObject public static final ParseField START = new ParseField("start"); public static final ParseField END = new ParseField("end"); public static final ParseField ANOMALY_SCORE = new ParseField("anomaly_score"); - public static final ParseField TIMESTAMP = new ParseField("timestamp"); public static final ParseField SORT = new ParseField("sort"); public static final ParseField DESCENDING = new ParseField("desc"); @@ -87,7 +86,7 @@ public String getJobId() { /** * Sets the timestamp of a specific bucket to be retrieved. - * @param timestamp the timestamp of a specific bucket to be retrieved + * @param timestamp String representation of a timestamp; may be an epoch seconds, epoch millis or an ISO string */ public void setTimestamp(String timestamp) { this.timestamp = timestamp; @@ -106,11 +105,11 @@ public boolean isExpand() { * When {@code true}, buckets will be expanded to include their records. * @param expand value of "expand" to be set */ - public void setExpand(boolean expand) { + public void setExpand(Boolean expand) { this.expand = expand; } - public boolean isExcludeInterim() { + public Boolean isExcludeInterim() { return excludeInterim; } @@ -119,7 +118,7 @@ public boolean isExcludeInterim() { * When {@code true}, interim buckets will be filtered out. * @param excludeInterim value of "exclude_interim" to be set */ - public void setExcludeInterim(boolean excludeInterim) { + public void setExcludeInterim(Boolean excludeInterim) { this.excludeInterim = excludeInterim; } @@ -130,7 +129,7 @@ public String getStart() { /** * Sets the value of "start" which is a timestamp. * Only buckets whose timestamp is on or after the "start" value will be returned. - * @param start value of "start" to be set + * @param start String representation of a timestamp; may be an epoch seconds, epoch millis or an ISO string */ public void setStart(String start) { this.start = start; @@ -143,7 +142,7 @@ public String getEnd() { /** * Sets the value of "end" which is a timestamp. * Only buckets whose timestamp is before the "end" value will be returned. - * @param end value of "end" to be set + * @param end String representation of a timestamp; may be an epoch seconds, epoch millis or an ISO string */ public void setEnd(String end) { this.end = end; @@ -170,7 +169,7 @@ public Double getAnomalyScore() { * Only buckets with "anomaly_score" equal or greater will be returned. * @param anomalyScore value of "anomaly_score". */ - public void setAnomalyScore(double anomalyScore) { + public void setAnomalyScore(Double anomalyScore) { this.anomalyScore = anomalyScore; } @@ -187,7 +186,7 @@ public void setSort(String sort) { this.sort = sort; } - public boolean isDescending() { + public Boolean isDescending() { return descending; } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetInfluencersRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetInfluencersRequest.java new file mode 100644 index 0000000000000..f57d327db3a36 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetInfluencersRequest.java @@ -0,0 +1,227 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.ml; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.client.ml.job.config.Job; +import org.elasticsearch.client.ml.job.util.PageParams; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +/** + * A request to retrieve influencers of a given job + */ +public class GetInfluencersRequest extends ActionRequest implements ToXContentObject { + + public static final ParseField EXCLUDE_INTERIM = new ParseField("exclude_interim"); + public static final ParseField START = new ParseField("start"); + public static final ParseField END = new ParseField("end"); + public static final ParseField INFLUENCER_SCORE = new ParseField("influencer_score"); + public static final ParseField SORT = new ParseField("sort"); + public static final ParseField DESCENDING = new ParseField("desc"); + + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "get_influencers_request", a -> new GetInfluencersRequest((String) a[0])); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), Job.ID); + PARSER.declareBoolean(GetInfluencersRequest::setExcludeInterim, EXCLUDE_INTERIM); + PARSER.declareStringOrNull(GetInfluencersRequest::setStart, START); + PARSER.declareStringOrNull(GetInfluencersRequest::setEnd, END); + PARSER.declareObject(GetInfluencersRequest::setPageParams, PageParams.PARSER, PageParams.PAGE); + PARSER.declareDouble(GetInfluencersRequest::setInfluencerScore, INFLUENCER_SCORE); + PARSER.declareString(GetInfluencersRequest::setSort, SORT); + PARSER.declareBoolean(GetInfluencersRequest::setDescending, DESCENDING); + } + + private final String jobId; + private Boolean excludeInterim; + private String start; + private String end; + private Double influencerScore; + private PageParams pageParams; + private String sort; + private Boolean descending; + + /** + * Constructs a request to retrieve influencers of a given job + * @param jobId id of the job to retrieve influencers of + */ + public GetInfluencersRequest(String jobId) { + this.jobId = Objects.requireNonNull(jobId); + } + + public String getJobId() { + return jobId; + } + + public Boolean isExcludeInterim() { + return excludeInterim; + } + + /** + * Sets the value of "exclude_interim". + * When {@code true}, interim influencers will be filtered out. + * @param excludeInterim value of "exclude_interim" to be set + */ + public void setExcludeInterim(Boolean excludeInterim) { + this.excludeInterim = excludeInterim; + } + + public String getStart() { + return start; + } + + /** + * Sets the value of "start" which is a timestamp. + * Only influencers whose timestamp is on or after the "start" value will be returned. + * @param start String representation of a timestamp; may be an epoch seconds, epoch millis or an ISO string + */ + public void setStart(String start) { + this.start = start; + } + + public String getEnd() { + return end; + } + + /** + * Sets the value of "end" which is a timestamp. + * Only influencers whose timestamp is before the "end" value will be returned. + * @param end String representation of a timestamp; may be an epoch seconds, epoch millis or an ISO string + */ + public void setEnd(String end) { + this.end = end; + } + + public PageParams getPageParams() { + return pageParams; + } + + /** + * Sets the paging parameters + * @param pageParams The paging parameters + */ + public void setPageParams(PageParams pageParams) { + this.pageParams = pageParams; + } + + public Double getInfluencerScore() { + return influencerScore; + } + + /** + * Sets the value of "influencer_score". + * Only influencers with "influencer_score" equal or greater will be returned. + * @param influencerScore value of "influencer_score". + */ + public void setInfluencerScore(Double influencerScore) { + this.influencerScore = influencerScore; + } + + public String getSort() { + return sort; + } + + /** + * Sets the value of "sort". + * Specifies the influencer field to sort on. + * @param sort value of "sort". + */ + public void setSort(String sort) { + this.sort = sort; + } + + public Boolean isDescending() { + return descending; + } + + /** + * Sets the value of "desc". + * Specifies the sorting order. + * @param descending value of "desc" + */ + public void setDescending(Boolean descending) { + this.descending = descending; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(Job.ID.getPreferredName(), jobId); + if (excludeInterim != null) { + builder.field(EXCLUDE_INTERIM.getPreferredName(), excludeInterim); + } + if (start != null) { + builder.field(START.getPreferredName(), start); + } + if (end != null) { + builder.field(END.getPreferredName(), end); + } + if (pageParams != null) { + builder.field(PageParams.PAGE.getPreferredName(), pageParams); + } + if (influencerScore != null) { + builder.field(INFLUENCER_SCORE.getPreferredName(), influencerScore); + } + if (sort != null) { + builder.field(SORT.getPreferredName(), sort); + } + if (descending != null) { + builder.field(DESCENDING.getPreferredName(), descending); + } + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(jobId, excludeInterim, influencerScore, pageParams, start, end, sort, descending); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + GetInfluencersRequest other = (GetInfluencersRequest) obj; + return Objects.equals(jobId, other.jobId) && + Objects.equals(excludeInterim, other.excludeInterim) && + Objects.equals(influencerScore, other.influencerScore) && + Objects.equals(pageParams, other.pageParams) && + Objects.equals(start, other.start) && + Objects.equals(end, other.end) && + Objects.equals(sort, other.sort) && + Objects.equals(descending, other.descending); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetInfluencersResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetInfluencersResponse.java new file mode 100644 index 0000000000000..113d960008c76 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetInfluencersResponse.java @@ -0,0 +1,78 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.ml; + +import org.elasticsearch.client.ml.job.results.Influencer; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; + +/** + * A response containing the requested influencers + */ +public class GetInfluencersResponse extends AbstractResultResponse { + + public static final ParseField INFLUENCERS = new ParseField("influencers"); + + @SuppressWarnings("unchecked") + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "get_influencers_response", true, a -> new GetInfluencersResponse((List) a[0], (long) a[1])); + + static { + PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), Influencer.PARSER, INFLUENCERS); + PARSER.declareLong(ConstructingObjectParser.constructorArg(), COUNT); + } + + public static GetInfluencersResponse fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + GetInfluencersResponse(List influencers, long count) { + super(INFLUENCERS, influencers, count); + } + + /** + * The retrieved influencers + * @return the retrieved influencers + */ + public List influencers() { + return results; + } + + @Override + public int hashCode() { + return Objects.hash(count, results); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + GetInfluencersResponse other = (GetInfluencersResponse) obj; + return count == other.count && Objects.equals(results, other.results); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetOverallBucketsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetOverallBucketsRequest.java index e58b57605e6f2..97bde11d8c6cd 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetOverallBucketsRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetOverallBucketsRequest.java @@ -154,7 +154,7 @@ public String getStart() { /** * Sets the value of "start" which is a timestamp. * Only overall buckets whose timestamp is on or after the "start" value will be returned. - * @param start value of "start" to be set + * @param start String representation of a timestamp; may be an epoch seconds, epoch millis or an ISO string */ public void setStart(String start) { this.start = start; @@ -167,7 +167,7 @@ public String getEnd() { /** * Sets the value of "end" which is a timestamp. * Only overall buckets whose timestamp is before the "end" value will be returned. - * @param end value of "end" to be set + * @param end String representation of a timestamp; may be an epoch seconds, epoch millis or an ISO string */ public void setEnd(String end) { this.end = end; diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetRecordsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetRecordsRequest.java index 0a701f5a1433a..3c11cbd2c10f5 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetRecordsRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetRecordsRequest.java @@ -41,7 +41,7 @@ public class GetRecordsRequest implements ToXContentObject, Validatable { public static final ParseField SORT = new ParseField("sort"); public static final ParseField DESCENDING = new ParseField("desc"); - public static final ObjectParser PARSER = new ObjectParser<>("get_buckets_request", GetRecordsRequest::new); + public static final ObjectParser PARSER = new ObjectParser<>("get_records_request", GetRecordsRequest::new); static { PARSER.declareString((request, jobId) -> request.jobId = jobId, Job.ID); @@ -77,7 +77,7 @@ public String getJobId() { return jobId; } - public boolean isExcludeInterim() { + public Boolean isExcludeInterim() { return excludeInterim; } @@ -86,7 +86,7 @@ public boolean isExcludeInterim() { * When {@code true}, interim records will be filtered out. * @param excludeInterim value of "exclude_interim" to be set */ - public void setExcludeInterim(boolean excludeInterim) { + public void setExcludeInterim(Boolean excludeInterim) { this.excludeInterim = excludeInterim; } @@ -97,7 +97,7 @@ public String getStart() { /** * Sets the value of "start" which is a timestamp. * Only records whose timestamp is on or after the "start" value will be returned. - * @param start value of "start" to be set + * @param start String representation of a timestamp; may be an epoch seconds, epoch millis or an ISO string */ public void setStart(String start) { this.start = start; @@ -110,7 +110,7 @@ public String getEnd() { /** * Sets the value of "end" which is a timestamp. * Only records whose timestamp is before the "end" value will be returned. - * @param end value of "end" to be set + * @param end String representation of a timestamp; may be an epoch seconds, epoch millis or an ISO string */ public void setEnd(String end) { this.end = end; @@ -137,7 +137,7 @@ public Double getRecordScore() { * Only records with "record_score" equal or greater will be returned. * @param recordScore value of "record_score". */ - public void setRecordScore(double recordScore) { + public void setRecordScore(Double recordScore) { this.recordScore = recordScore; } @@ -147,14 +147,14 @@ public String getSort() { /** * Sets the value of "sort". - * Specifies the bucket field to sort on. + * Specifies the record field to sort on. * @param sort value of "sort". */ public void setSort(String sort) { this.sort = sort; } - public boolean isDescending() { + public Boolean isDescending() { return descending; } @@ -163,7 +163,7 @@ public boolean isDescending() { * Specifies the sorting order. * @param descending value of "desc" */ - public void setDescending(boolean descending) { + public void setDescending(Boolean descending) { this.descending = descending; } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetRecordsResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetRecordsResponse.java index 99e1152422609..0d8efd5c6ea2a 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetRecordsResponse.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetRecordsResponse.java @@ -28,7 +28,7 @@ import java.util.Objects; /** - * A response containing the requested buckets + * A response containing the requested records */ public class GetRecordsResponse extends AbstractResultResponse { @@ -47,8 +47,8 @@ public static GetRecordsResponse fromXContent(XContentParser parser) throws IOEx return PARSER.parse(parser, null); } - GetRecordsResponse(List buckets, long count) { - super(RECORDS, buckets, count); + GetRecordsResponse(List records, long count) { + super(RECORDS, records, count); } /** diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java index bd997224bebda..0822db33505c4 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.client.ml.DeleteJobRequest; import org.elasticsearch.client.ml.FlushJobRequest; import org.elasticsearch.client.ml.GetBucketsRequest; +import org.elasticsearch.client.ml.GetInfluencersRequest; import org.elasticsearch.client.ml.GetJobRequest; import org.elasticsearch.client.ml.GetJobStatsRequest; import org.elasticsearch.client.ml.GetOverallBucketsRequest; @@ -220,6 +221,26 @@ public void testGetRecords() throws IOException { } } + public void testGetInfluencers() throws IOException { + String jobId = randomAlphaOfLength(10); + GetInfluencersRequest getInfluencersRequest = new GetInfluencersRequest(jobId); + getInfluencersRequest.setStart("2018-08-08T00:00:00Z"); + getInfluencersRequest.setEnd("2018-09-08T00:00:00Z"); + getInfluencersRequest.setPageParams(new PageParams(100, 300)); + getInfluencersRequest.setInfluencerScore(75.0); + getInfluencersRequest.setSort("anomaly_score"); + getInfluencersRequest.setDescending(true); + getInfluencersRequest.setExcludeInterim(true); + + Request request = MLRequestConverters.getInfluencers(getInfluencersRequest); + assertEquals(HttpGet.METHOD_NAME, request.getMethod()); + assertEquals("/_xpack/ml/anomaly_detectors/" + jobId + "/results/influencers", request.getEndpoint()); + try (XContentParser parser = createParser(JsonXContent.jsonXContent, request.getEntity().getContent())) { + GetInfluencersRequest parsedRequest = GetInfluencersRequest.PARSER.apply(parser, null); + assertThat(parsedRequest, equalTo(getInfluencersRequest)); + } + } + private static Job createValidJob(String jobId) { AnalysisConfig.Builder analysisConfig = AnalysisConfig.builder(Collections.singletonList( Detector.builder().setFunction("count").build())); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningGetResultsIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningGetResultsIT.java index b1c743098db4a..40d8596d1ba86 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningGetResultsIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningGetResultsIT.java @@ -23,6 +23,8 @@ import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.ml.GetBucketsRequest; import org.elasticsearch.client.ml.GetBucketsResponse; +import org.elasticsearch.client.ml.GetInfluencersRequest; +import org.elasticsearch.client.ml.GetInfluencersResponse; import org.elasticsearch.client.ml.GetOverallBucketsRequest; import org.elasticsearch.client.ml.GetOverallBucketsResponse; import org.elasticsearch.client.ml.GetRecordsRequest; @@ -34,6 +36,7 @@ import org.elasticsearch.client.ml.job.config.Job; import org.elasticsearch.client.ml.job.results.AnomalyRecord; import org.elasticsearch.client.ml.job.results.Bucket; +import org.elasticsearch.client.ml.job.results.Influencer; import org.elasticsearch.client.ml.job.results.OverallBucket; import org.elasticsearch.client.ml.job.util.PageParams; import org.elasticsearch.common.unit.TimeValue; @@ -387,6 +390,106 @@ public void testGetRecords() throws IOException { } } + public void testGetInfluencers() throws IOException { + MachineLearningClient machineLearningClient = highLevelClient().machineLearning(); + + // Let us index a few influencer docs + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + long timestamp = START_TIME_EPOCH_MS; + long end = START_TIME_EPOCH_MS + 5 * 3600000L; + while (timestamp < end) { + boolean isLast = timestamp == end - 3600000L; + // Last one is interim + boolean isInterim = isLast; + // Last one score is higher + double score = isLast ? 90.0 : 42.0; + + IndexRequest indexRequest = new IndexRequest(RESULTS_INDEX, DOC); + indexRequest.source("{\"job_id\":\"" + JOB_ID + "\", \"result_type\":\"influencer\", \"timestamp\": " + + timestamp + "," + "\"bucket_span\": 3600,\"is_interim\": " + isInterim + ", \"influencer_score\": " + score + ", " + + "\"influencer_field_name\":\"my_influencer\", \"influencer_field_value\": \"inf_1\", \"probability\":" + + randomDouble() + "}", XContentType.JSON); + bulkRequest.add(indexRequest); + timestamp += 3600000L; + } + highLevelClient().bulk(bulkRequest, RequestOptions.DEFAULT); + + { + GetInfluencersRequest request = new GetInfluencersRequest(JOB_ID); + request.setDescending(false); + + GetInfluencersResponse response = execute(request, machineLearningClient::getInfluencers, + machineLearningClient::getInfluencersAsync); + + assertThat(response.count(), equalTo(5L)); + } + { + long requestStart = START_TIME_EPOCH_MS + 3600000L; + long requestEnd = end - 3600000L; + GetInfluencersRequest request = new GetInfluencersRequest(JOB_ID); + request.setStart(String.valueOf(requestStart)); + request.setEnd(String.valueOf(requestEnd)); + + GetInfluencersResponse response = execute(request, machineLearningClient::getInfluencers, + machineLearningClient::getInfluencersAsync); + + assertThat(response.count(), equalTo(3L)); + for (Influencer influencer : response.influencers()) { + assertThat(influencer.getTimestamp().getTime(), greaterThanOrEqualTo(START_TIME_EPOCH_MS)); + assertThat(influencer.getTimestamp().getTime(), lessThan(end)); + } + } + { + GetInfluencersRequest request = new GetInfluencersRequest(JOB_ID); + request.setSort("timestamp"); + request.setDescending(false); + request.setPageParams(new PageParams(1, 2)); + + GetInfluencersResponse response = execute(request, machineLearningClient::getInfluencers, + machineLearningClient::getInfluencersAsync); + + assertThat(response.influencers().size(), equalTo(2)); + assertThat(response.influencers().get(0).getTimestamp().getTime(), equalTo(START_TIME_EPOCH_MS + 3600000L)); + assertThat(response.influencers().get(1).getTimestamp().getTime(), equalTo(START_TIME_EPOCH_MS + 2 * 3600000L)); + } + { + GetInfluencersRequest request = new GetInfluencersRequest(JOB_ID); + request.setExcludeInterim(true); + + GetInfluencersResponse response = execute(request, machineLearningClient::getInfluencers, + machineLearningClient::getInfluencersAsync); + + assertThat(response.count(), equalTo(4L)); + assertThat(response.influencers().stream().anyMatch(Influencer::isInterim), is(false)); + } + { + GetInfluencersRequest request = new GetInfluencersRequest(JOB_ID); + request.setInfluencerScore(75.0); + + GetInfluencersResponse response = execute(request, machineLearningClient::getInfluencers, + machineLearningClient::getInfluencersAsync); + + assertThat(response.count(), equalTo(1L)); + assertThat(response.influencers().get(0).getInfluencerScore(), greaterThanOrEqualTo(75.0)); + } + { + GetInfluencersRequest request = new GetInfluencersRequest(JOB_ID); + request.setSort("probability"); + request.setDescending(true); + + GetInfluencersResponse response = execute(request, machineLearningClient::getInfluencers, + machineLearningClient::getInfluencersAsync); + + assertThat(response.influencers().size(), equalTo(5)); + double previousProb = 1.0; + for (Influencer influencer : response.influencers()) { + assertThat(influencer.getProbability(), lessThanOrEqualTo(previousProb)); + previousProb = influencer.getProbability(); + } + } + } + public static Job buildJob(String jobId) { Job.Builder builder = new Job.Builder(jobId); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java index 427f75a80d029..2e1fc6c2711d2 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java @@ -37,6 +37,8 @@ import org.elasticsearch.client.ml.FlushJobResponse; import org.elasticsearch.client.ml.GetBucketsRequest; import org.elasticsearch.client.ml.GetBucketsResponse; +import org.elasticsearch.client.ml.GetInfluencersRequest; +import org.elasticsearch.client.ml.GetInfluencersResponse; import org.elasticsearch.client.ml.GetJobRequest; import org.elasticsearch.client.ml.GetJobResponse; import org.elasticsearch.client.ml.GetJobStatsRequest; @@ -55,6 +57,7 @@ import org.elasticsearch.client.ml.job.config.Job; import org.elasticsearch.client.ml.job.results.AnomalyRecord; import org.elasticsearch.client.ml.job.results.Bucket; +import org.elasticsearch.client.ml.job.results.Influencer; import org.elasticsearch.client.ml.job.results.OverallBucket; import org.elasticsearch.client.ml.job.stats.JobStats; import org.elasticsearch.client.ml.job.util.PageParams; @@ -781,4 +784,95 @@ public void onFailure(Exception e) { assertTrue(latch.await(30L, TimeUnit.SECONDS)); } } + + public void testGetInfluencers() throws IOException, InterruptedException { + RestHighLevelClient client = highLevelClient(); + + String jobId = "test-get-influencers"; + Job job = MachineLearningIT.buildJob(jobId); + client.machineLearning().putJob(new PutJobRequest(job), RequestOptions.DEFAULT); + + // Let us index a record + IndexRequest indexRequest = new IndexRequest(".ml-anomalies-shared", "doc"); + indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + indexRequest.source("{\"job_id\":\"test-get-influencers\", \"result_type\":\"influencer\", \"timestamp\": 1533081600000," + + "\"bucket_span\": 600,\"is_interim\": false, \"influencer_score\": 80.0, \"influencer_field_name\": \"my_influencer\"," + + "\"influencer_field_value\":\"foo\"}", XContentType.JSON); + client.index(indexRequest, RequestOptions.DEFAULT); + + { + // tag::x-pack-ml-get-influencers-request + GetInfluencersRequest request = new GetInfluencersRequest(jobId); // <1> + // end::x-pack-ml-get-influencers-request + + // tag::x-pack-ml-get-influencers-desc + request.setDescending(true); // <1> + // end::x-pack-ml-get-influencers-desc + + // tag::x-pack-ml-get-influencers-end + request.setEnd("2018-08-21T00:00:00Z"); // <1> + // end::x-pack-ml-get-influencers-end + + // tag::x-pack-ml-get-influencers-exclude-interim + request.setExcludeInterim(true); // <1> + // end::x-pack-ml-get-influencers-exclude-interim + + // tag::x-pack-ml-get-influencers-influencer-score + request.setInfluencerScore(75.0); // <1> + // end::x-pack-ml-get-influencers-influencer-score + + // tag::x-pack-ml-get-influencers-page + request.setPageParams(new PageParams(100, 200)); // <1> + // end::x-pack-ml-get-influencers-page + + // Set page params back to null so the response contains the influencer we indexed + request.setPageParams(null); + + // tag::x-pack-ml-get-influencers-sort + request.setSort("probability"); // <1> + // end::x-pack-ml-get-influencers-sort + + // tag::x-pack-ml-get-influencers-start + request.setStart("2018-08-01T00:00:00Z"); // <1> + // end::x-pack-ml-get-influencers-start + + // tag::x-pack-ml-get-influencers-execute + GetInfluencersResponse response = client.machineLearning().getInfluencers(request, RequestOptions.DEFAULT); + // end::x-pack-ml-get-influencers-execute + + // tag::x-pack-ml-get-influencers-response + long count = response.count(); // <1> + List influencers = response.influencers(); // <2> + // end::x-pack-ml-get-influencers-response + assertEquals(1, influencers.size()); + } + { + GetInfluencersRequest request = new GetInfluencersRequest(jobId); + + // tag::x-pack-ml-get-influencers-listener + ActionListener listener = + new ActionListener() { + @Override + public void onResponse(GetInfluencersResponse getInfluencersResponse) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::x-pack-ml-get-influencers-listener + + // Replace the empty listener by a blocking listener in test + final CountDownLatch latch = new CountDownLatch(1); + listener = new LatchedActionListener<>(listener, latch); + + // tag::x-pack-ml-get-influencers-execute-async + client.machineLearning().getInfluencersAsync(request, RequestOptions.DEFAULT, listener); // <1> + // end::x-pack-ml-get-influencers-execute-async + + assertTrue(latch.await(30L, TimeUnit.SECONDS)); + } + } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetInfluencersRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetInfluencersRequestTests.java new file mode 100644 index 0000000000000..94937cd78155f --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetInfluencersRequestTests.java @@ -0,0 +1,71 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.ml; + +import org.elasticsearch.client.ml.job.util.PageParams; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.io.IOException; + +public class GetInfluencersRequestTests extends AbstractXContentTestCase { + + @Override + protected GetInfluencersRequest createTestInstance() { + GetInfluencersRequest request = new GetInfluencersRequest(randomAlphaOfLengthBetween(1, 20)); + + if (randomBoolean()) { + request.setStart(String.valueOf(randomLong())); + } + if (randomBoolean()) { + request.setEnd(String.valueOf(randomLong())); + } + if (randomBoolean()) { + request.setExcludeInterim(randomBoolean()); + } + if (randomBoolean()) { + request.setInfluencerScore(randomDouble()); + } + if (randomBoolean()) { + int from = randomInt(10000); + int size = randomInt(10000); + request.setPageParams(new PageParams(from, size)); + } + if (randomBoolean()) { + request.setSort("influencer_score"); + } + if (randomBoolean()) { + request.setDescending(randomBoolean()); + } + if (randomBoolean()) { + request.setExcludeInterim(randomBoolean()); + } + return request; + } + + @Override + protected GetInfluencersRequest doParseInstance(XContentParser parser) throws IOException { + return GetInfluencersRequest.PARSER.apply(parser, null); + } + + @Override + protected boolean supportsUnknownFields() { + return false; + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetInfluencersResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetInfluencersResponseTests.java new file mode 100644 index 0000000000000..5f1fa3c1ab5ec --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetInfluencersResponseTests.java @@ -0,0 +1,53 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.ml; + +import org.elasticsearch.client.ml.job.results.Influencer; +import org.elasticsearch.client.ml.job.results.InfluencerTests; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class GetInfluencersResponseTests extends AbstractXContentTestCase { + + @Override + protected GetInfluencersResponse createTestInstance() { + String jobId = randomAlphaOfLength(20); + int listSize = randomInt(10); + List influencers = new ArrayList<>(listSize); + for (int j = 0; j < listSize; j++) { + Influencer influencer = InfluencerTests.createTestInstance(jobId); + influencers.add(influencer); + } + return new GetInfluencersResponse(influencers, listSize); + } + + @Override + protected GetInfluencersResponse doParseInstance(XContentParser parser) throws IOException { + return GetInfluencersResponse.fromXContent(parser); + } + + @Override + protected boolean supportsUnknownFields() { + return true; + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetRecordsRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetRecordsRequestTests.java index 226ffe75b01e6..f6f4b49889a47 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetRecordsRequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetRecordsRequestTests.java @@ -21,7 +21,6 @@ import org.elasticsearch.client.ml.job.util.PageParams; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.AbstractXContentTestCase; -import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -29,33 +28,33 @@ public class GetRecordsRequestTests extends AbstractXContentTestCase records = new ArrayList<>(listSize); for (int j = 0; j < listSize; j++) { AnomalyRecord record = AnomalyRecordTests.createTestInstance(jobId); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/results/InfluencerTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/results/InfluencerTests.java index ef83af39958de..33d1a33e9f153 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/results/InfluencerTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/job/results/InfluencerTests.java @@ -29,7 +29,7 @@ public class InfluencerTests extends AbstractXContentTestCase { - public Influencer createTestInstance(String jobId) { + public static Influencer createTestInstance(String jobId) { Influencer influencer = new Influencer(jobId, randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20), new Date(randomNonNegativeLong()), randomNonNegativeLong()); influencer.setInterim(randomBoolean()); diff --git a/docs/java-rest/high-level/ml/get-influencers.asciidoc b/docs/java-rest/high-level/ml/get-influencers.asciidoc new file mode 100644 index 0000000000000..e53e92ff1df07 --- /dev/null +++ b/docs/java-rest/high-level/ml/get-influencers.asciidoc @@ -0,0 +1,112 @@ +[[java-rest-high-x-pack-ml-get-influencers]] +=== Get Influencers API + +The Get Influencers API retrieves one or more influencer results. +It accepts a `GetInfluencersRequest` object and responds +with a `GetInfluencersResponse` object. + +[[java-rest-high-x-pack-ml-get-influencers-request]] +==== Get Influencers Request + +A `GetInfluencersRequest` object gets created with an existing non-null `jobId`. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-get-influencers-request] +-------------------------------------------------- +<1> Constructing a new request referencing an existing `jobId` + +==== Optional Arguments +The following arguments are optional: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-get-influencers-desc] +-------------------------------------------------- +<1> If `true`, the influencers are sorted in descending order. Defaults to `false`. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-get-influencers-end] +-------------------------------------------------- +<1> Influencers with timestamps earlier than this time will be returned. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-get-influencers-exclude-interim] +-------------------------------------------------- +<1> If `true`, interim results will be excluded. Defaults to `false`. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-get-influencers-influencer-score] +-------------------------------------------------- +<1> Influencers with influencer_score greater or equal than this value will be returned. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-get-influencers-page] +-------------------------------------------------- +<1> The page parameters `from` and `size`. `from` specifies the number of influencers to skip. +`size` specifies the maximum number of influencers to get. Defaults to `0` and `100` respectively. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-get-influencers-sort] +-------------------------------------------------- +<1> The field to sort influencers on. Defaults to `influencer_score`. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-get-influencers-start] +-------------------------------------------------- +<1> Influencers with timestamps on or after this time will be returned. + +[[java-rest-high-x-pack-ml-get-influencers-execution]] +==== Execution + +The request can be executed through the `MachineLearningClient` contained +in the `RestHighLevelClient` object, accessed via the `machineLearningClient()` method. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-get-influencers-execute] +-------------------------------------------------- + +[[java-rest-high-x-pack-ml-get-influencers-execution-async]] +==== Asynchronous Execution + +The request can also be executed asynchronously: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-get-influencers-execute-async] +-------------------------------------------------- +<1> The `GetInfluencersRequest` to execute and the `ActionListener` to use when +the execution completes + +The asynchronous method does not block and returns immediately. Once it is +completed the `ActionListener` is called back with the `onResponse` method +if the execution is successful or the `onFailure` method if the execution +failed. + +A typical listener for `GetInfluencersResponse` looks like: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-get-influencers-listener] +-------------------------------------------------- +<1> `onResponse` is called back when the action is completed successfully +<2> `onFailure` is called back when some unexpected error occurs + +[[java-rest-high-snapshot-ml-get-influencers-response]] +==== Get Influencers Response + +The returned `GetInfluencersResponse` contains the requested influencers: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/MlClientDocumentationIT.java[x-pack-ml-get-influencers-response] +-------------------------------------------------- +<1> The count of influencers that were matched +<2> The influencers retrieved \ No newline at end of file diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc index 481a2470aa2d7..f01e12c4c9812 100644 --- a/docs/java-rest/high-level/supported-apis.asciidoc +++ b/docs/java-rest/high-level/supported-apis.asciidoc @@ -220,6 +220,7 @@ The Java High Level REST Client supports the following Machine Learning APIs: * <> * <> * <> +* <> include::ml/put-job.asciidoc[] include::ml/get-job.asciidoc[] @@ -231,6 +232,7 @@ include::ml/get-job-stats.asciidoc[] include::ml/get-buckets.asciidoc[] include::ml/get-overall-buckets.asciidoc[] include::ml/get-records.asciidoc[] +include::ml/get-influencers.asciidoc[] == Migration APIs From 7319bc7411fb289f854ebbbdb4ac0e91235b3e34 Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Wed, 5 Sep 2018 09:34:47 -0500 Subject: [PATCH 09/10] HLRC: split cluster request converters (#33400) In an effort to encapsulate the different clients, the request converters are being shuffled around. This splits the ClusterClient request converters. --- .../elasticsearch/client/ClusterClient.java | 12 +- .../client/ClusterRequestConverters.java | 77 +++++++++ .../client/RequestConverters.java | 46 ------ .../client/ClusterRequestConvertersTests.java | 150 ++++++++++++++++++ .../client/ESRestHighLevelClientTestCase.java | 3 +- .../client/RequestConvertersTests.java | 124 +-------------- 6 files changed, 241 insertions(+), 171 deletions(-) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ClusterRequestConverters.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterRequestConvertersTests.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ClusterClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ClusterClient.java index b72a21ed7d1c4..f9b1474c69ae4 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ClusterClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ClusterClient.java @@ -56,7 +56,7 @@ public final class ClusterClient { */ public ClusterUpdateSettingsResponse putSettings(ClusterUpdateSettingsRequest clusterUpdateSettingsRequest, RequestOptions options) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(clusterUpdateSettingsRequest, RequestConverters::clusterPutSettings, + return restHighLevelClient.performRequestAndParseEntity(clusterUpdateSettingsRequest, ClusterRequestConverters::clusterPutSettings, options, ClusterUpdateSettingsResponse::fromXContent, emptySet()); } @@ -70,7 +70,7 @@ public ClusterUpdateSettingsResponse putSettings(ClusterUpdateSettingsRequest cl */ public void putSettingsAsync(ClusterUpdateSettingsRequest clusterUpdateSettingsRequest, RequestOptions options, ActionListener listener) { - restHighLevelClient.performRequestAsyncAndParseEntity(clusterUpdateSettingsRequest, RequestConverters::clusterPutSettings, + restHighLevelClient.performRequestAsyncAndParseEntity(clusterUpdateSettingsRequest, ClusterRequestConverters::clusterPutSettings, options, ClusterUpdateSettingsResponse::fromXContent, listener, emptySet()); } @@ -85,7 +85,7 @@ public void putSettingsAsync(ClusterUpdateSettingsRequest clusterUpdateSettingsR */ public ClusterGetSettingsResponse getSettings(ClusterGetSettingsRequest clusterGetSettingsRequest, RequestOptions options) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(clusterGetSettingsRequest, RequestConverters::clusterGetSettings, + return restHighLevelClient.performRequestAndParseEntity(clusterGetSettingsRequest, ClusterRequestConverters::clusterGetSettings, options, ClusterGetSettingsResponse::fromXContent, emptySet()); } @@ -99,7 +99,7 @@ public ClusterGetSettingsResponse getSettings(ClusterGetSettingsRequest clusterG */ public void getSettingsAsync(ClusterGetSettingsRequest clusterGetSettingsRequest, RequestOptions options, ActionListener listener) { - restHighLevelClient.performRequestAsyncAndParseEntity(clusterGetSettingsRequest, RequestConverters::clusterGetSettings, + restHighLevelClient.performRequestAsyncAndParseEntity(clusterGetSettingsRequest, ClusterRequestConverters::clusterGetSettings, options, ClusterGetSettingsResponse::fromXContent, listener, emptySet()); } @@ -115,7 +115,7 @@ public void getSettingsAsync(ClusterGetSettingsRequest clusterGetSettingsRequest * @throws IOException in case there is a problem sending the request or parsing back the response */ public ClusterHealthResponse health(ClusterHealthRequest healthRequest, RequestOptions options) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(healthRequest, RequestConverters::clusterHealth, options, + return restHighLevelClient.performRequestAndParseEntity(healthRequest, ClusterRequestConverters::clusterHealth, options, ClusterHealthResponse::fromXContent, singleton(RestStatus.REQUEST_TIMEOUT.getStatus())); } @@ -129,7 +129,7 @@ public ClusterHealthResponse health(ClusterHealthRequest healthRequest, RequestO * @param listener the listener to be notified upon request completion */ public void healthAsync(ClusterHealthRequest healthRequest, RequestOptions options, ActionListener listener) { - restHighLevelClient.performRequestAsyncAndParseEntity(healthRequest, RequestConverters::clusterHealth, options, + restHighLevelClient.performRequestAsyncAndParseEntity(healthRequest, ClusterRequestConverters::clusterHealth, options, ClusterHealthResponse::fromXContent, listener, singleton(RestStatus.REQUEST_TIMEOUT.getStatus())); } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ClusterRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ClusterRequestConverters.java new file mode 100644 index 0000000000000..d6c41e804df61 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ClusterRequestConverters.java @@ -0,0 +1,77 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPut; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsRequest; +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; +import org.elasticsearch.action.support.ActiveShardCount; +import org.elasticsearch.common.Strings; + +import java.io.IOException; + +final class ClusterRequestConverters { + + static Request clusterPutSettings(ClusterUpdateSettingsRequest clusterUpdateSettingsRequest) throws IOException { + Request request = new Request(HttpPut.METHOD_NAME, "/_cluster/settings"); + + RequestConverters.Params parameters = new RequestConverters.Params(request); + parameters.withTimeout(clusterUpdateSettingsRequest.timeout()); + parameters.withMasterTimeout(clusterUpdateSettingsRequest.masterNodeTimeout()); + + request.setEntity(RequestConverters.createEntity(clusterUpdateSettingsRequest, RequestConverters.REQUEST_BODY_CONTENT_TYPE)); + return request; + } + + static Request clusterGetSettings(ClusterGetSettingsRequest clusterGetSettingsRequest) throws IOException { + Request request = new Request(HttpGet.METHOD_NAME, "/_cluster/settings"); + + RequestConverters.Params parameters = new RequestConverters.Params(request); + parameters.withLocal(clusterGetSettingsRequest.local()); + parameters.withIncludeDefaults(clusterGetSettingsRequest.includeDefaults()); + parameters.withMasterTimeout(clusterGetSettingsRequest.masterNodeTimeout()); + + return request; + } + + static Request clusterHealth(ClusterHealthRequest healthRequest) { + String[] indices = healthRequest.indices() == null ? Strings.EMPTY_ARRAY : healthRequest.indices(); + String endpoint = new RequestConverters.EndpointBuilder() + .addPathPartAsIs("_cluster/health") + .addCommaSeparatedPathParts(indices) + .build(); + Request request = new Request(HttpGet.METHOD_NAME, endpoint); + + new RequestConverters.Params(request) + .withWaitForStatus(healthRequest.waitForStatus()) + .withWaitForNoRelocatingShards(healthRequest.waitForNoRelocatingShards()) + .withWaitForNoInitializingShards(healthRequest.waitForNoInitializingShards()) + .withWaitForActiveShards(healthRequest.waitForActiveShards(), ActiveShardCount.NONE) + .withWaitForNodes(healthRequest.waitForNodes()) + .withWaitForEvents(healthRequest.waitForEvents()) + .withTimeout(healthRequest.timeout()) + .withMasterTimeout(healthRequest.masterNodeTimeout()) + .withLocal(healthRequest.local()) + .withLevel(healthRequest.level()); + return request; + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index 41caf2625a231..89521b5e9b06f 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -36,8 +36,6 @@ import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest; import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest; import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryRequest; -import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsRequest; -import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; @@ -724,28 +722,6 @@ private static Request resize(ResizeRequest resizeRequest) throws IOException { return request; } - static Request clusterPutSettings(ClusterUpdateSettingsRequest clusterUpdateSettingsRequest) throws IOException { - Request request = new Request(HttpPut.METHOD_NAME, "/_cluster/settings"); - - Params parameters = new Params(request); - parameters.withTimeout(clusterUpdateSettingsRequest.timeout()); - parameters.withMasterTimeout(clusterUpdateSettingsRequest.masterNodeTimeout()); - - request.setEntity(createEntity(clusterUpdateSettingsRequest, REQUEST_BODY_CONTENT_TYPE)); - return request; - } - - static Request clusterGetSettings(ClusterGetSettingsRequest clusterGetSettingsRequest) throws IOException { - Request request = new Request(HttpGet.METHOD_NAME, "/_cluster/settings"); - - Params parameters = new Params(request); - parameters.withLocal(clusterGetSettingsRequest.local()); - parameters.withIncludeDefaults(clusterGetSettingsRequest.includeDefaults()); - parameters.withMasterTimeout(clusterGetSettingsRequest.masterNodeTimeout()); - - return request; - } - static Request getPipeline(GetPipelineRequest getPipelineRequest) { String endpoint = new EndpointBuilder() .addPathPartAsIs("_ingest/pipeline") @@ -803,28 +779,6 @@ static Request listTasks(ListTasksRequest listTaskRequest) { return request; } - static Request clusterHealth(ClusterHealthRequest healthRequest) { - String[] indices = healthRequest.indices() == null ? Strings.EMPTY_ARRAY : healthRequest.indices(); - String endpoint = new EndpointBuilder() - .addPathPartAsIs("_cluster/health") - .addCommaSeparatedPathParts(indices) - .build(); - Request request = new Request(HttpGet.METHOD_NAME, endpoint); - - new Params(request) - .withWaitForStatus(healthRequest.waitForStatus()) - .withWaitForNoRelocatingShards(healthRequest.waitForNoRelocatingShards()) - .withWaitForNoInitializingShards(healthRequest.waitForNoInitializingShards()) - .withWaitForActiveShards(healthRequest.waitForActiveShards(), ActiveShardCount.NONE) - .withWaitForNodes(healthRequest.waitForNodes()) - .withWaitForEvents(healthRequest.waitForEvents()) - .withTimeout(healthRequest.timeout()) - .withMasterTimeout(healthRequest.masterNodeTimeout()) - .withLocal(healthRequest.local()) - .withLevel(healthRequest.level()); - return request; - } - static Request reindex(ReindexRequest reindexRequest) throws IOException { String endpoint = new EndpointBuilder().addPathPart("_reindex").build(); Request request = new Request(HttpPost.METHOD_NAME, endpoint); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterRequestConvertersTests.java new file mode 100644 index 0000000000000..9a7596957d02a --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterRequestConvertersTests.java @@ -0,0 +1,150 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPut; +import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; +import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsRequest; +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; +import org.elasticsearch.action.support.ActiveShardCount; +import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.cluster.health.ClusterHealthStatus; +import org.elasticsearch.common.Priority; +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.CoreMatchers; +import org.junit.Assert; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.Matchers.nullValue; + +public class ClusterRequestConvertersTests extends ESTestCase { + + public void testClusterPutSettings() throws IOException { + ClusterUpdateSettingsRequest request = new ClusterUpdateSettingsRequest(); + Map expectedParams = new HashMap<>(); + RequestConvertersTests.setRandomMasterTimeout(request, expectedParams); + RequestConvertersTests.setRandomTimeout(request::timeout, AcknowledgedRequest.DEFAULT_ACK_TIMEOUT, expectedParams); + + Request expectedRequest = ClusterRequestConverters.clusterPutSettings(request); + Assert.assertEquals("/_cluster/settings", expectedRequest.getEndpoint()); + Assert.assertEquals(HttpPut.METHOD_NAME, expectedRequest.getMethod()); + Assert.assertEquals(expectedParams, expectedRequest.getParameters()); + } + + public void testClusterGetSettings() throws IOException { + ClusterGetSettingsRequest request = new ClusterGetSettingsRequest(); + Map expectedParams = new HashMap<>(); + RequestConvertersTests.setRandomMasterTimeout(request, expectedParams); + request.includeDefaults(ESTestCase.randomBoolean()); + if (request.includeDefaults()) { + expectedParams.put("include_defaults", String.valueOf(true)); + } + + Request expectedRequest = ClusterRequestConverters.clusterGetSettings(request); + Assert.assertEquals("/_cluster/settings", expectedRequest.getEndpoint()); + Assert.assertEquals(HttpGet.METHOD_NAME, expectedRequest.getMethod()); + Assert.assertEquals(expectedParams, expectedRequest.getParameters()); + } + + public void testClusterHealth() { + ClusterHealthRequest healthRequest = new ClusterHealthRequest(); + Map expectedParams = new HashMap<>(); + RequestConvertersTests.setRandomLocal(healthRequest, expectedParams); + String timeoutType = ESTestCase.randomFrom("timeout", "masterTimeout", "both", "none"); + String timeout = ESTestCase.randomTimeValue(); + String masterTimeout = ESTestCase.randomTimeValue(); + switch (timeoutType) { + case "timeout": + healthRequest.timeout(timeout); + expectedParams.put("timeout", timeout); + // If Master Timeout wasn't set it uses the same value as Timeout + expectedParams.put("master_timeout", timeout); + break; + case "masterTimeout": + expectedParams.put("timeout", "30s"); + healthRequest.masterNodeTimeout(masterTimeout); + expectedParams.put("master_timeout", masterTimeout); + break; + case "both": + healthRequest.timeout(timeout); + expectedParams.put("timeout", timeout); + healthRequest.masterNodeTimeout(timeout); + expectedParams.put("master_timeout", timeout); + break; + case "none": + expectedParams.put("timeout", "30s"); + expectedParams.put("master_timeout", "30s"); + break; + default: + throw new UnsupportedOperationException(); + } + RequestConvertersTests.setRandomWaitForActiveShards(healthRequest::waitForActiveShards, ActiveShardCount.NONE, expectedParams); + if (ESTestCase.randomBoolean()) { + ClusterHealthRequest.Level level = ESTestCase.randomFrom(ClusterHealthRequest.Level.values()); + healthRequest.level(level); + expectedParams.put("level", level.name().toLowerCase(Locale.ROOT)); + } else { + expectedParams.put("level", "cluster"); + } + if (ESTestCase.randomBoolean()) { + Priority priority = ESTestCase.randomFrom(Priority.values()); + healthRequest.waitForEvents(priority); + expectedParams.put("wait_for_events", priority.name().toLowerCase(Locale.ROOT)); + } + if (ESTestCase.randomBoolean()) { + ClusterHealthStatus status = ESTestCase.randomFrom(ClusterHealthStatus.values()); + healthRequest.waitForStatus(status); + expectedParams.put("wait_for_status", status.name().toLowerCase(Locale.ROOT)); + } + if (ESTestCase.randomBoolean()) { + boolean waitForNoInitializingShards = ESTestCase.randomBoolean(); + healthRequest.waitForNoInitializingShards(waitForNoInitializingShards); + if (waitForNoInitializingShards) { + expectedParams.put("wait_for_no_initializing_shards", Boolean.TRUE.toString()); + } + } + if (ESTestCase.randomBoolean()) { + boolean waitForNoRelocatingShards = ESTestCase.randomBoolean(); + healthRequest.waitForNoRelocatingShards(waitForNoRelocatingShards); + if (waitForNoRelocatingShards) { + expectedParams.put("wait_for_no_relocating_shards", Boolean.TRUE.toString()); + } + } + String[] indices = ESTestCase.randomBoolean() ? null : RequestConvertersTests.randomIndicesNames(0, 5); + healthRequest.indices(indices); + + Request request = ClusterRequestConverters.clusterHealth(healthRequest); + Assert.assertThat(request, CoreMatchers.notNullValue()); + Assert.assertThat(request.getMethod(), equalTo(HttpGet.METHOD_NAME)); + Assert.assertThat(request.getEntity(), nullValue()); + if (indices != null && indices.length > 0) { + Assert.assertThat(request.getEndpoint(), equalTo("/_cluster/health/" + String.join(",", indices))); + } else { + Assert.assertThat(request.getEndpoint(), equalTo("/_cluster/health")); + } + Assert.assertThat(request.getParameters(), equalTo(expectedParams)); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java index e05fa9fa79b90..d917102d43d2a 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java @@ -134,6 +134,7 @@ protected static void clusterUpdateSettings(Settings persistentSettings, ClusterUpdateSettingsRequest request = new ClusterUpdateSettingsRequest(); request.persistentSettings(persistentSettings); request.transientSettings(transientSettings); - assertOK(client().performRequest(RequestConverters.clusterPutSettings(request))); + assertTrue(execute( + request, highLevelClient().cluster()::putSettings, highLevelClient().cluster()::putSettingsAsync).isAcknowledged()); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index a0597d3b79471..61d5866b8e1ee 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -29,15 +29,12 @@ import org.apache.http.util.EntityUtils; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.DocWriteRequest; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest; import org.elasticsearch.action.admin.cluster.repositories.get.GetRepositoriesRequest; import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest; import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryRequest; -import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsRequest; -import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest; import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; @@ -97,10 +94,8 @@ import org.elasticsearch.action.support.replication.ReplicationRequest; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.RequestConverters.EndpointBuilder; -import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.CheckedFunction; -import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -156,7 +151,6 @@ import org.elasticsearch.tasks.TaskId; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.RandomObjects; -import org.hamcrest.CoreMatchers; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -1838,33 +1832,6 @@ private static void resizeTest(ResizeType resizeType, CheckedFunction expectedParams = new HashMap<>(); - setRandomMasterTimeout(request, expectedParams); - setRandomTimeout(request::timeout, AcknowledgedRequest.DEFAULT_ACK_TIMEOUT, expectedParams); - - Request expectedRequest = RequestConverters.clusterPutSettings(request); - assertEquals("/_cluster/settings", expectedRequest.getEndpoint()); - assertEquals(HttpPut.METHOD_NAME, expectedRequest.getMethod()); - assertEquals(expectedParams, expectedRequest.getParameters()); - } - - public void testClusterGetSettings() throws IOException { - ClusterGetSettingsRequest request = new ClusterGetSettingsRequest(); - Map expectedParams = new HashMap<>(); - setRandomMasterTimeout(request, expectedParams); - request.includeDefaults(randomBoolean()); - if (request.includeDefaults()) { - expectedParams.put("include_defaults", String.valueOf(true)); - } - - Request expectedRequest = RequestConverters.clusterGetSettings(request); - assertEquals("/_cluster/settings", expectedRequest.getEndpoint()); - assertEquals(HttpGet.METHOD_NAME, expectedRequest.getMethod()); - assertEquals(expectedParams, expectedRequest.getParameters()); - } - public void testPutPipeline() throws IOException { String pipelineId = "some_pipeline_id"; PutPipelineRequest request = new PutPipelineRequest( @@ -1942,85 +1909,6 @@ public void testSimulatePipeline() throws IOException { assertToXContentBody(request, expectedRequest.getEntity()); } - public void testClusterHealth() { - ClusterHealthRequest healthRequest = new ClusterHealthRequest(); - Map expectedParams = new HashMap<>(); - setRandomLocal(healthRequest, expectedParams); - String timeoutType = randomFrom("timeout", "masterTimeout", "both", "none"); - String timeout = randomTimeValue(); - String masterTimeout = randomTimeValue(); - switch (timeoutType) { - case "timeout": - healthRequest.timeout(timeout); - expectedParams.put("timeout", timeout); - // If Master Timeout wasn't set it uses the same value as Timeout - expectedParams.put("master_timeout", timeout); - break; - case "masterTimeout": - expectedParams.put("timeout", "30s"); - healthRequest.masterNodeTimeout(masterTimeout); - expectedParams.put("master_timeout", masterTimeout); - break; - case "both": - healthRequest.timeout(timeout); - expectedParams.put("timeout", timeout); - healthRequest.masterNodeTimeout(timeout); - expectedParams.put("master_timeout", timeout); - break; - case "none": - expectedParams.put("timeout", "30s"); - expectedParams.put("master_timeout", "30s"); - break; - default: - throw new UnsupportedOperationException(); - } - setRandomWaitForActiveShards(healthRequest::waitForActiveShards, ActiveShardCount.NONE, expectedParams); - if (randomBoolean()) { - ClusterHealthRequest.Level level = randomFrom(ClusterHealthRequest.Level.values()); - healthRequest.level(level); - expectedParams.put("level", level.name().toLowerCase(Locale.ROOT)); - } else { - expectedParams.put("level", "cluster"); - } - if (randomBoolean()) { - Priority priority = randomFrom(Priority.values()); - healthRequest.waitForEvents(priority); - expectedParams.put("wait_for_events", priority.name().toLowerCase(Locale.ROOT)); - } - if (randomBoolean()) { - ClusterHealthStatus status = randomFrom(ClusterHealthStatus.values()); - healthRequest.waitForStatus(status); - expectedParams.put("wait_for_status", status.name().toLowerCase(Locale.ROOT)); - } - if (randomBoolean()) { - boolean waitForNoInitializingShards = randomBoolean(); - healthRequest.waitForNoInitializingShards(waitForNoInitializingShards); - if (waitForNoInitializingShards) { - expectedParams.put("wait_for_no_initializing_shards", Boolean.TRUE.toString()); - } - } - if (randomBoolean()) { - boolean waitForNoRelocatingShards = randomBoolean(); - healthRequest.waitForNoRelocatingShards(waitForNoRelocatingShards); - if (waitForNoRelocatingShards) { - expectedParams.put("wait_for_no_relocating_shards", Boolean.TRUE.toString()); - } - } - String[] indices = randomBoolean() ? null : randomIndicesNames(0, 5); - healthRequest.indices(indices); - - Request request = RequestConverters.clusterHealth(healthRequest); - assertThat(request, CoreMatchers.notNullValue()); - assertThat(request.getMethod(), equalTo(HttpGet.METHOD_NAME)); - assertThat(request.getEntity(), nullValue()); - if (indices != null && indices.length > 0) { - assertThat(request.getEndpoint(), equalTo("/_cluster/health/" + String.join(",", indices))); - } else { - assertThat(request.getEndpoint(), equalTo("/_cluster/health")); - } - assertThat(request.getParameters(), equalTo(expectedParams)); - } - public void testRollover() throws IOException { RolloverRequest rolloverRequest = new RolloverRequest(randomAlphaOfLengthBetween(3, 10), randomBoolean() ? null : randomAlphaOfLengthBetween(3, 10)); @@ -2920,11 +2808,11 @@ private static void setRandomLocal(Consumer setter, Map } } - private static void setRandomLocal(MasterNodeReadRequest request, Map expectedParams) { + static void setRandomLocal(MasterNodeReadRequest request, Map expectedParams) { setRandomLocal(request::local, expectedParams); } - private static void setRandomTimeout(Consumer setter, TimeValue defaultTimeout, Map expectedParams) { + static void setRandomTimeout(Consumer setter, TimeValue defaultTimeout, Map expectedParams) { if (randomBoolean()) { String timeout = randomTimeValue(); setter.accept(timeout); @@ -2934,7 +2822,7 @@ private static void setRandomTimeout(Consumer setter, TimeValue defaultT } } - private static void setRandomMasterTimeout(MasterNodeRequest request, Map expectedParams) { + static void setRandomMasterTimeout(MasterNodeRequest request, Map expectedParams) { if (randomBoolean()) { String masterTimeout = randomTimeValue(); request.masterNodeTimeout(masterTimeout); @@ -2948,8 +2836,8 @@ private static void setRandomWaitForActiveShards(Consumer sett setRandomWaitForActiveShards(setter, ActiveShardCount.DEFAULT, expectedParams); } - private static void setRandomWaitForActiveShards(Consumer setter, ActiveShardCount defaultActiveShardCount, - Map expectedParams) { + static void setRandomWaitForActiveShards(Consumer setter, ActiveShardCount defaultActiveShardCount, + Map expectedParams) { if (randomBoolean()) { int waitForActiveShardsInt = randomIntBetween(-1, 5); String waitForActiveShardsString; @@ -3009,7 +2897,7 @@ private static String randomFields(String[] fields) { return excludesParam.toString(); } - private static String[] randomIndicesNames(int minIndicesNum, int maxIndicesNum) { + static String[] randomIndicesNames(int minIndicesNum, int maxIndicesNum) { int numIndices = randomIntBetween(minIndicesNum, maxIndicesNum); String[] indices = new String[numIndices]; for (int i = 0; i < numIndices; i++) { From 23934e39d2911c987bd7c91cbf5e8281ced361a9 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 5 Sep 2018 11:01:58 -0400 Subject: [PATCH 10/10] Fix deprecated setting specializations (#33412) Deprecating a some setting specializations (e.g., list settings) does not cause deprecation warning headers and deprecation log messages to appear. This is due to a missed check for deprecation. This commit fixes this for all setting specializations, and ensures that this can not be missed again. --- .../common/settings/SecureSetting.java | 2 +- .../common/settings/Setting.java | 19 ++++++++++++++---- .../common/settings/SettingTests.java | 20 +++++++++++++++++++ 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/settings/SecureSetting.java b/server/src/main/java/org/elasticsearch/common/settings/SecureSetting.java index c23a0bd42e3e0..33f4718aa45e4 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/SecureSetting.java +++ b/server/src/main/java/org/elasticsearch/common/settings/SecureSetting.java @@ -69,7 +69,7 @@ public T getDefault(Settings settings) { } @Override - public String getRaw(Settings settings) { + String innerGetRaw(final Settings settings) { throw new UnsupportedOperationException("secure settings are not strings"); } diff --git a/server/src/main/java/org/elasticsearch/common/settings/Setting.java b/server/src/main/java/org/elasticsearch/common/settings/Setting.java index ceeb60f8edd04..ff6a5b8fe0f96 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -426,8 +426,19 @@ public void diff(Settings.Builder builder, Settings source, Settings defaultSett * Returns the raw (string) settings value. If the setting is not present in the given settings object the default value is returned * instead. This is useful if the value can't be parsed due to an invalid value to access the actual value. */ - public String getRaw(Settings settings) { + public final String getRaw(final Settings settings) { checkDeprecation(settings); + return innerGetRaw(settings); + } + + /** + * The underlying implementation for {@link #getRaw(Settings)}. Setting specializations can override this as needed to convert the + * actual settings value to raw strings. + * + * @param settings the settings instance + * @return the raw string representation of the setting value + */ + String innerGetRaw(final Settings settings) { return settings.get(getKey(), defaultValue.apply(settings)); } @@ -713,7 +724,7 @@ public T get(Settings settings) { } @Override - public String getRaw(Settings settings) { + public String innerGetRaw(final Settings settings) { throw new UnsupportedOperationException("affix settings can't return values" + " use #getConcreteSetting to obtain a concrete setting"); } @@ -820,7 +831,7 @@ public boolean isGroupSetting() { } @Override - public String getRaw(Settings settings) { + public String innerGetRaw(final Settings settings) { Settings subSettings = get(settings); try { XContentBuilder builder = XContentFactory.jsonBuilder(); @@ -913,7 +924,7 @@ private ListSetting(String key, Function> defaultStringVa } @Override - public String getRaw(Settings settings) { + String innerGetRaw(final Settings settings) { List array = settings.getAsList(getKey(), null); return array == null ? defaultValue.apply(settings) : arrayToParsableString(array); } diff --git a/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java b/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java index bedb2857b6055..d82b620660249 100644 --- a/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java +++ b/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java @@ -462,6 +462,26 @@ public void testCompositeValidator() { } + public void testListSettingsDeprecated() { + final Setting> deprecatedListSetting = + Setting.listSetting( + "foo.deprecated", + Collections.singletonList("foo.deprecated"), + Function.identity(), + Property.Deprecated, + Property.NodeScope); + final Setting> nonDeprecatedListSetting = + Setting.listSetting( + "foo.non_deprecated", Collections.singletonList("foo.non_deprecated"), Function.identity(), Property.NodeScope); + final Settings settings = Settings.builder() + .put("foo.deprecated", "foo.deprecated1,foo.deprecated2") + .put("foo.deprecated", "foo.non_deprecated1,foo.non_deprecated2") + .build(); + deprecatedListSetting.get(settings); + nonDeprecatedListSetting.get(settings); + assertSettingDeprecationsAndWarnings(new Setting[]{deprecatedListSetting}); + } + public void testListSettings() { Setting> listSetting = Setting.listSetting("foo.bar", Arrays.asList("foo,bar"), (s) -> s.toString(), Property.Dynamic, Property.NodeScope);