From 7535d086f4005d56808ac765fa809279177b137e Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Tue, 20 Nov 2018 23:07:10 +0100 Subject: [PATCH 1/5] Handles exists query in composite aggs This commit adds the support for exists query in the sorted execution mode of the `composite` aggregation. We'll execute the aggregation from the sorted points and use early termination if the main query is an `exists` query over the first source of the `composite` aggregation. --- .../bucket/composite/LongValuesSource.java | 34 +++++++++++++++---- .../SingleDimensionValuesSourceTests.java | 12 +++++++ 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java index 6b5329e0a70b4..f840d9acd8e2b 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java @@ -25,6 +25,8 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.ConstantScoreQuery; +import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.PointRangeQuery; @@ -37,6 +39,7 @@ import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; +import org.elasticsearch.index.query.ExistsQueryBuilder; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.LeafBucketCollector; @@ -180,24 +183,41 @@ public void collect(int doc, long bucket) throws IOException { }; } - static Query extractQuery(Query query) { + private static Query extractQuery(Query query) { if (query instanceof BoostQuery) { return extractQuery(((BoostQuery) query).getQuery()); } else if (query instanceof IndexOrDocValuesQuery) { return extractQuery(((IndexOrDocValuesQuery) query).getIndexQuery()); + } else if (query instanceof ConstantScoreQuery){ + return extractQuery(((ConstantScoreQuery) query).getQuery()); } else { return query; } } + /** + * Returns true if we can use query with a {@link SortedDocsProducer} on fieldName. + */ + private static boolean checkQuery(Query query, String fieldName) { + query = extractQuery(query); + if (query == null) { + return true; + } else if (query.getClass() == MatchAllDocsQuery.class) { + return true; + } else if (query instanceof PointRangeQuery) { + PointRangeQuery pointQuery = (PointRangeQuery) query; + return fieldName.equals(pointQuery.getField()); + } else if (query instanceof DocValuesFieldExistsQuery) { + DocValuesFieldExistsQuery existsQuery = (DocValuesFieldExistsQuery) query; + return fieldName.equals(existsQuery.getField()); + } else { + return false; + } + } + @Override SortedDocsProducer createSortedDocsProducerOrNull(IndexReader reader, Query query) { - query = extractQuery(query); - if (checkIfSortedDocsIsApplicable(reader, fieldType) == false || - (query != null && - query.getClass() != MatchAllDocsQuery.class && - // if the query is a range query over the same field - (query instanceof PointRangeQuery && fieldType.name().equals((((PointRangeQuery) query).getField()))) == false)) { + if (checkIfSortedDocsIsApplicable(reader, fieldType) == false || checkQuery(query, fieldType.name()) == false) { return null; } final byte[] lowerPoint; diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/SingleDimensionValuesSourceTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/SingleDimensionValuesSourceTests.java index e7703265d7b2d..13e7f6362e8fd 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/SingleDimensionValuesSourceTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/SingleDimensionValuesSourceTests.java @@ -23,6 +23,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.TermQuery; @@ -55,6 +56,7 @@ public void testBinarySorted() { IndexReader reader = mockIndexReader(1, 1); assertNotNull(source.createSortedDocsProducerOrNull(reader, new MatchAllDocsQuery())); assertNotNull(source.createSortedDocsProducerOrNull(reader, null)); + assertNull(source.createSortedDocsProducerOrNull(reader, new TermQuery(new Term("foo", "bar")))); assertNull(source.createSortedDocsProducerOrNull(reader, new TermQuery(new Term("keyword", "toto)")))); @@ -112,6 +114,7 @@ public void testGlobalOrdinalsSorted() { IndexReader reader = mockIndexReader(1, 1); assertNotNull(source.createSortedDocsProducerOrNull(reader, new MatchAllDocsQuery())); assertNotNull(source.createSortedDocsProducerOrNull(reader, null)); + assertNull(source.createSortedDocsProducerOrNull(reader, new TermQuery(new Term("foo", "bar")))); assertNull(source.createSortedDocsProducerOrNull(reader, new TermQuery(new Term("keyword", "toto)")))); @@ -126,6 +129,7 @@ public void testGlobalOrdinalsSorted() { ); assertNull(source.createSortedDocsProducerOrNull(reader, new MatchAllDocsQuery())); assertNull(source.createSortedDocsProducerOrNull(reader, null)); + assertNull(source.createSortedDocsProducerOrNull(reader, new TermQuery(new Term("foo", "bar")))); source = new GlobalOrdinalValuesSource( BigArrays.NON_RECYCLING_INSTANCE, @@ -137,6 +141,7 @@ public void testGlobalOrdinalsSorted() { -1 ); assertNull(source.createSortedDocsProducerOrNull(reader, null)); + assertNull(source.createSortedDocsProducerOrNull(reader, new TermQuery(new Term("foo", "bar")))); final MappedFieldType ip = new IpFieldMapper.IpFieldType(); ip.setName("ip"); @@ -150,6 +155,7 @@ public void testGlobalOrdinalsSorted() { 1 ); assertNull(source.createSortedDocsProducerOrNull(reader, null)); + assertNull(source.createSortedDocsProducerOrNull(reader, new TermQuery(new Term("foo", "bar")))); } public void testNumericSorted() { @@ -179,6 +185,7 @@ public void testNumericSorted() { assertNotNull(source.createSortedDocsProducerOrNull(reader, LongPoint.newRangeQuery("number", 0, 1))); assertNotNull(source.createSortedDocsProducerOrNull(reader, new IndexOrDocValuesQuery( LongPoint.newRangeQuery("number", 0, 1), new MatchAllDocsQuery()))); + assertNotNull(source.createSortedDocsProducerOrNull(reader, new DocValuesFieldExistsQuery("number"))); assertNotNull(source.createSortedDocsProducerOrNull(reader, new BoostQuery(new IndexOrDocValuesQuery( LongPoint.newRangeQuery("number", 0, 1), new MatchAllDocsQuery()), 2.0f))); assertNull(source.createSortedDocsProducerOrNull(reader, new TermQuery(new Term("keyword", "toto)")))); @@ -195,6 +202,7 @@ public void testNumericSorted() { assertNull(sourceWithMissing.createSortedDocsProducerOrNull(reader, new MatchAllDocsQuery())); assertNull(sourceWithMissing.createSortedDocsProducerOrNull(reader, null)); assertNull(sourceWithMissing.createSortedDocsProducerOrNull(reader, new TermQuery(new Term("keyword", "toto)")))); + assertNull(sourceWithMissing.createSortedDocsProducerOrNull(reader, new DocValuesFieldExistsQuery("number"))); LongValuesSource sourceRev = new LongValuesSource( BigArrays.NON_RECYCLING_INSTANCE, @@ -207,6 +215,8 @@ public void testNumericSorted() { -1 ); assertNull(sourceRev.createSortedDocsProducerOrNull(reader, null)); + assertNull(sourceRev.createSortedDocsProducerOrNull(reader, new DocValuesFieldExistsQuery("number"))); + assertNull(sourceWithMissing.createSortedDocsProducerOrNull(reader, new TermQuery(new Term("keyword", "toto)")))); } else if (numberType == NumberFieldMapper.NumberType.HALF_FLOAT || numberType == NumberFieldMapper.NumberType.FLOAT || numberType == NumberFieldMapper.NumberType.DOUBLE) { @@ -221,6 +231,8 @@ public void testNumericSorted() { ); IndexReader reader = mockIndexReader(1, 1); assertNull(source.createSortedDocsProducerOrNull(reader, null)); + assertNull(source.createSortedDocsProducerOrNull(reader, new DocValuesFieldExistsQuery("number"))); + assertNull(source.createSortedDocsProducerOrNull(reader, new TermQuery(new Term("keyword", "toto)")))); } else{ throw new AssertionError ("missing type:" + numberType.typeName()); } From 573d5da9ecb4c4ce8d289c708af4f1f1596c6ddd Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Tue, 20 Nov 2018 23:09:38 +0100 Subject: [PATCH 2/5] add tests for constantscorequery --- .../composite/SingleDimensionValuesSourceTests.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/SingleDimensionValuesSourceTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/SingleDimensionValuesSourceTests.java index 13e7f6362e8fd..a34ec7ada20a6 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/SingleDimensionValuesSourceTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/composite/SingleDimensionValuesSourceTests.java @@ -23,6 +23,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchAllDocsQuery; @@ -186,6 +187,8 @@ public void testNumericSorted() { assertNotNull(source.createSortedDocsProducerOrNull(reader, new IndexOrDocValuesQuery( LongPoint.newRangeQuery("number", 0, 1), new MatchAllDocsQuery()))); assertNotNull(source.createSortedDocsProducerOrNull(reader, new DocValuesFieldExistsQuery("number"))); + assertNotNull(source.createSortedDocsProducerOrNull(reader, + new ConstantScoreQuery(new DocValuesFieldExistsQuery("number")))); assertNotNull(source.createSortedDocsProducerOrNull(reader, new BoostQuery(new IndexOrDocValuesQuery( LongPoint.newRangeQuery("number", 0, 1), new MatchAllDocsQuery()), 2.0f))); assertNull(source.createSortedDocsProducerOrNull(reader, new TermQuery(new Term("keyword", "toto)")))); @@ -203,6 +206,8 @@ public void testNumericSorted() { assertNull(sourceWithMissing.createSortedDocsProducerOrNull(reader, null)); assertNull(sourceWithMissing.createSortedDocsProducerOrNull(reader, new TermQuery(new Term("keyword", "toto)")))); assertNull(sourceWithMissing.createSortedDocsProducerOrNull(reader, new DocValuesFieldExistsQuery("number"))); + assertNull(sourceWithMissing.createSortedDocsProducerOrNull(reader, + new ConstantScoreQuery(new DocValuesFieldExistsQuery("number")))); LongValuesSource sourceRev = new LongValuesSource( BigArrays.NON_RECYCLING_INSTANCE, @@ -216,6 +221,8 @@ public void testNumericSorted() { ); assertNull(sourceRev.createSortedDocsProducerOrNull(reader, null)); assertNull(sourceRev.createSortedDocsProducerOrNull(reader, new DocValuesFieldExistsQuery("number"))); + assertNull(sourceRev.createSortedDocsProducerOrNull(reader, + new ConstantScoreQuery(new DocValuesFieldExistsQuery("number")))); assertNull(sourceWithMissing.createSortedDocsProducerOrNull(reader, new TermQuery(new Term("keyword", "toto)")))); } else if (numberType == NumberFieldMapper.NumberType.HALF_FLOAT || numberType == NumberFieldMapper.NumberType.FLOAT || @@ -233,6 +240,8 @@ public void testNumericSorted() { assertNull(source.createSortedDocsProducerOrNull(reader, null)); assertNull(source.createSortedDocsProducerOrNull(reader, new DocValuesFieldExistsQuery("number"))); assertNull(source.createSortedDocsProducerOrNull(reader, new TermQuery(new Term("keyword", "toto)")))); + assertNull(source.createSortedDocsProducerOrNull(reader, + new ConstantScoreQuery(new DocValuesFieldExistsQuery("number")))); } else{ throw new AssertionError ("missing type:" + numberType.typeName()); } From 4c2146bc2c142e7c98c2936861066cec9f9e0b06 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Wed, 21 Nov 2018 08:46:17 +0100 Subject: [PATCH 3/5] remove unused import --- .../search/aggregations/bucket/composite/LongValuesSource.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java index f840d9acd8e2b..35721b1a2eff8 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java @@ -39,7 +39,6 @@ import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper; -import org.elasticsearch.index.query.ExistsQueryBuilder; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.LeafBucketCollector; From 38ef31dcc7f14e92ab9fe1624b3da49689f612ae Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Mon, 26 Nov 2018 09:35:09 +0100 Subject: [PATCH 4/5] fix missing query extraction --- .../aggregations/bucket/composite/LongValuesSource.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java index 35721b1a2eff8..69c45a170916d 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java @@ -198,7 +198,6 @@ private static Query extractQuery(Query query) { * Returns true if we can use query with a {@link SortedDocsProducer} on fieldName. */ private static boolean checkQuery(Query query, String fieldName) { - query = extractQuery(query); if (query == null) { return true; } else if (query.getClass() == MatchAllDocsQuery.class) { @@ -216,7 +215,9 @@ private static boolean checkQuery(Query query, String fieldName) { @Override SortedDocsProducer createSortedDocsProducerOrNull(IndexReader reader, Query query) { - if (checkIfSortedDocsIsApplicable(reader, fieldType) == false || checkQuery(query, fieldType.name()) == false) { + query = extractQuery(query); + if (checkIfSortedDocsIsApplicable(reader, fieldType) == false || + checkQuery(query, fieldType.name()) == false) { return null; } final byte[] lowerPoint; From 66ebd485dd3c7778950740d84d9107645a711f48 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Mon, 26 Nov 2018 10:28:35 +0100 Subject: [PATCH 5/5] address review --- .../aggregations/bucket/composite/LongValuesSource.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java index 69c45a170916d..8e20fe917094b 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/LongValuesSource.java @@ -197,7 +197,7 @@ private static Query extractQuery(Query query) { /** * Returns true if we can use query with a {@link SortedDocsProducer} on fieldName. */ - private static boolean checkQuery(Query query, String fieldName) { + private static boolean checkMatchAllOrRangeQuery(Query query, String fieldName) { if (query == null) { return true; } else if (query.getClass() == MatchAllDocsQuery.class) { @@ -217,7 +217,7 @@ private static boolean checkQuery(Query query, String fieldName) { SortedDocsProducer createSortedDocsProducerOrNull(IndexReader reader, Query query) { query = extractQuery(query); if (checkIfSortedDocsIsApplicable(reader, fieldType) == false || - checkQuery(query, fieldType.name()) == false) { + checkMatchAllOrRangeQuery(query, fieldType.name()) == false) { return null; } final byte[] lowerPoint;