From 7d02df15dedf0ebeb37a9481e88aac2d9769f577 Mon Sep 17 00:00:00 2001 From: Navneet Verma Date: Mon, 1 Aug 2022 16:23:15 -0700 Subject: [PATCH] Refactored the src and test of GeoHashGrid and GeoTileGrid Aggregations on GeoPoint from server folder to geo module.(#4071) (#4072) The changes also include: * Updated Search plugin to provide the interface so that plugins can also register the compositie aggregations * Added YAML test for the geo_grid, geo_tile and composite aggregation Signed-off-by: Navneet Verma --- .../client/RestHighLevelClient.java | 6 - modules/geo/build.gradle | 9 +- .../aggregations/bucket/GeoHashGridIT.java | 39 +- .../aggregations/bucket/ShardReduceIT.java | 107 +++++ .../aggregations/metrics/GeoCentroidIT.java | 84 ++++ .../org/opensearch/geo/GeoModulePlugin.java | 42 +- .../GeoTileGridValuesSourceBuilder.java | 29 +- .../bucket/composite/GeoTileValuesSource.java | 8 +- .../bucket/geogrid/BoundedCellValues.java | 2 +- .../bucket/geogrid/BucketPriorityQueue.java | 2 +- .../bucket/geogrid/CellIdSource.java | 2 +- .../bucket/geogrid/CellValues.java | 2 +- .../aggregations/bucket/geogrid/GeoGrid.java | 2 +- .../geogrid/GeoGridAggregationBuilder.java | 2 +- .../bucket/geogrid/GeoGridAggregator.java | 2 +- .../GeoHashGridAggregationBuilder.java | 4 +- .../bucket/geogrid/GeoHashGridAggregator.java | 2 +- .../geogrid/GeoHashGridAggregatorFactory.java | 2 +- .../GeoTileGridAggregationBuilder.java | 5 +- .../bucket/geogrid/GeoTileGridAggregator.java | 2 +- .../geogrid/GeoTileGridAggregatorFactory.java | 3 +- .../bucket/geogrid/InternalGeoGrid.java | 2 +- .../bucket/geogrid/InternalGeoGridBucket.java | 2 +- .../bucket/geogrid/InternalGeoHashGrid.java | 2 +- .../geogrid/InternalGeoHashGridBucket.java | 4 +- .../bucket/geogrid/InternalGeoTileGrid.java | 2 +- .../geogrid/InternalGeoTileGridBucket.java | 5 +- .../bucket/geogrid/ParsedGeoGrid.java | 2 +- .../bucket/geogrid/ParsedGeoGridBucket.java | 2 +- .../bucket/geogrid/ParsedGeoHashGrid.java | 2 +- .../geogrid/ParsedGeoHashGridBucket.java | 2 +- .../bucket/geogrid/ParsedGeoTileGrid.java | 2 +- .../geogrid/ParsedGeoTileGridBucket.java | 3 +- .../bucket/geogrid/UnboundedCellValues.java | 2 +- .../bucket/geogrid/package-info.java | 2 +- .../metrics/GeoGridAggregatorSupplier.java | 4 +- .../GeoHashGridAggregationBuilderTests.java | 21 +- .../GeoTileGridAggregationBuilderTests.java | 23 +- ...idAggregationCompositeAggregatorTests.java | 174 ++++++++ ...eGridCompositeAggregationBuilderTests.java | 50 +++ .../GeoTileGridValuesSourceBuilderTests.java | 3 +- .../geogrid/GeoGridAggregatorTestCase.java | 20 +- .../bucket/geogrid/GeoGridTestCase.java | 39 +- .../geogrid/GeoHashGridAggregatorTests.java | 2 +- .../geogrid/GeoHashGridParserTests.java | 2 +- .../bucket/geogrid/GeoHashGridTests.java | 2 +- .../geogrid/GeoTileGridAggregatorTests.java | 4 +- .../geogrid/GeoTileGridParserTests.java | 3 +- .../bucket/geogrid/GeoTileGridTests.java | 3 +- .../geo/tests/common/AggregationBuilders.java | 18 + .../common/AggregationInspectionHelper.java | 5 + .../geo/tests/common/RandomGeoGenerator.java | 11 + .../test/geo_shape/230_composite.yml | 168 ++++++++ .../test/geo_shape}/280_geohash_grid.yml | 0 .../test/geo_shape}/290_geotile_grid.yml | 0 .../test/search.aggregation/230_composite.yml | 86 ---- .../aggregations/bucket/ShardReduceIT.java | 35 -- .../aggregations/metrics/GeoCentroidIT.java | 33 -- .../org/opensearch/plugins/SearchPlugin.java | 82 ++++ .../org/opensearch/search/DocValueFormat.java | 2 +- .../org/opensearch/search/SearchModule.java | 22 +- .../aggregations/AggregationBuilders.java | 18 - .../bucket/{geogrid => }/GeoTileUtils.java | 8 +- .../CompositeAggregationBuilder.java | 50 ++- .../CompositeAggregationParsingFunction.java | 22 + .../CompositeValuesSourceBuilder.java | 6 +- .../CompositeValuesSourceConfig.java | 18 +- .../CompositeValuesSourceParserHelper.java | 78 +++- .../bucket/composite/LongValuesSource.java | 6 +- .../SingleDimensionValuesSource.java | 2 +- .../support/AggregationInspectionHelper.java | 5 - .../aggregations/support/package-info.java | 2 +- .../search/DocValueFormatTests.java | 2 +- .../aggregations/AggregationsTests.java | 4 - .../CompositeAggregationBuilderTests.java | 20 +- .../composite/CompositeAggregatorTests.java | 384 +----------------- .../bucket/geogrid/GeoTileUtilsTests.java | 13 +- .../BaseCompositeAggregatorTestCase.java | 310 ++++++++++++++ .../test/InternalAggregationTestCase.java | 6 - 79 files changed, 1395 insertions(+), 761 deletions(-) rename {server/src/internalClusterTest/java/org/opensearch => modules/geo/src/internalClusterTest/java/org/opensearch/geo}/search/aggregations/bucket/GeoHashGridIT.java (89%) create mode 100644 modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/ShardReduceIT.java create mode 100644 modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoCentroidIT.java rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java (87%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/composite/GeoTileValuesSource.java (88%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/BoundedCellValues.java (97%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/BucketPriorityQueue.java (96%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/CellIdSource.java (98%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/CellValues.java (97%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoGrid.java (96%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java (99%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoGridAggregator.java (99%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java (96%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java (97%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java (98%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java (95%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java (97%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java (97%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/InternalGeoGrid.java (99%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/InternalGeoGridBucket.java (98%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java (97%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/InternalGeoHashGridBucket.java (95%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/InternalGeoTileGrid.java (97%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java (93%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/ParsedGeoGrid.java (97%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/ParsedGeoGridBucket.java (96%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/ParsedGeoHashGrid.java (96%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/ParsedGeoHashGridBucket.java (96%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/ParsedGeoTileGrid.java (96%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java (93%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/UnboundedCellValues.java (96%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/package-info.java (79%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/metrics/GeoGridAggregatorSupplier.java (93%) rename server/src/test/java/org/opensearch/search/aggregations/bucket/GeoHashGridTests.java => modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridAggregationBuilderTests.java (72%) rename server/src/test/java/org/opensearch/search/aggregations/bucket/GeoTileGridTests.java => modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/GeoTileGridAggregationBuilderTests.java (70%) create mode 100644 modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridAggregationCompositeAggregatorTests.java create mode 100644 modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridCompositeAggregationBuilderTests.java rename {server/src/test/java/org/opensearch => modules/geo/src/test/java/org/opensearch/geo}/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilderTests.java (90%) rename {server/src/test/java/org/opensearch => modules/geo/src/test/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java (95%) rename {server/src/test/java/org/opensearch => modules/geo/src/test/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoGridTestCase.java (79%) rename {server/src/test/java/org/opensearch => modules/geo/src/test/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java (96%) rename {server/src/test/java/org/opensearch => modules/geo/src/test/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java (99%) rename {server/src/test/java/org/opensearch => modules/geo/src/test/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoHashGridTests.java (97%) rename {server/src/test/java/org/opensearch => modules/geo/src/test/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java (94%) rename {server/src/test/java/org/opensearch => modules/geo/src/test/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java (97%) rename {server/src/test/java/org/opensearch => modules/geo/src/test/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoTileGridTests.java (94%) create mode 100644 modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape/230_composite.yml rename {rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation => modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape}/280_geohash_grid.yml (100%) rename {rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation => modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape}/290_geotile_grid.yml (100%) rename server/src/main/java/org/opensearch/search/aggregations/bucket/{geogrid => }/GeoTileUtils.java (97%) create mode 100644 server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationParsingFunction.java create mode 100644 test/framework/src/main/java/org/opensearch/search/aggregations/composite/BaseCompositeAggregatorTestCase.java diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java index 7ae8f8826c5a4..28a441bdf7f7f 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java @@ -108,10 +108,6 @@ import org.opensearch.search.aggregations.bucket.filter.FiltersAggregationBuilder; import org.opensearch.search.aggregations.bucket.filter.ParsedFilter; import org.opensearch.search.aggregations.bucket.filter.ParsedFilters; -import org.opensearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.ParsedGeoHashGrid; -import org.opensearch.search.aggregations.bucket.geogrid.ParsedGeoTileGrid; import org.opensearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.opensearch.search.aggregations.bucket.global.ParsedGlobal; import org.opensearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; @@ -2130,8 +2126,6 @@ static List getDefaultNamedXContents() { map.put(GlobalAggregationBuilder.NAME, (p, c) -> ParsedGlobal.fromXContent(p, (String) c)); map.put(FilterAggregationBuilder.NAME, (p, c) -> ParsedFilter.fromXContent(p, (String) c)); map.put(InternalSampler.PARSER_NAME, (p, c) -> ParsedSampler.fromXContent(p, (String) c)); - map.put(GeoHashGridAggregationBuilder.NAME, (p, c) -> ParsedGeoHashGrid.fromXContent(p, (String) c)); - map.put(GeoTileGridAggregationBuilder.NAME, (p, c) -> ParsedGeoTileGrid.fromXContent(p, (String) c)); map.put(RangeAggregationBuilder.NAME, (p, c) -> ParsedRange.fromXContent(p, (String) c)); map.put(DateRangeAggregationBuilder.NAME, (p, c) -> ParsedDateRange.fromXContent(p, (String) c)); map.put(GeoDistanceAggregationBuilder.NAME, (p, c) -> ParsedGeoDistance.fromXContent(p, (String) c)); diff --git a/modules/geo/build.gradle b/modules/geo/build.gradle index 0b8e623c24ac6..7f687a414e566 100644 --- a/modules/geo/build.gradle +++ b/modules/geo/build.gradle @@ -37,9 +37,16 @@ opensearchplugin { restResources { restApi { - includeCore '_common', 'indices', 'index', 'search' + includeCore '_common', 'indices', 'index', 'search', 'bulk' } } artifacts { restTests(project.file('src/yamlRestTest/resources/rest-api-spec/test')) } +/** + * These compiler arguments needs to be removed, as there are raw types being used in the GeoGrid and GeoTile aggregations. + */ +tasks.withType(JavaCompile).configureEach { + options.compilerArgs -= '-Xlint:rawtypes' + options.compilerArgs -= '-Xlint:unchecked' +} diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/GeoHashGridIT.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridIT.java similarity index 89% rename from server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/GeoHashGridIT.java rename to modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridIT.java index 56d918feef9d8..6ab7dd5254679 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/GeoHashGridIT.java +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridIT.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket; +package org.opensearch.geo.search.aggregations.bucket; import com.carrotsearch.hppc.ObjectIntHashMap; import com.carrotsearch.hppc.ObjectIntMap; @@ -41,12 +41,12 @@ import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.geo.GeoModulePluginIntegTestCase; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGrid; +import org.opensearch.geo.tests.common.AggregationBuilders; import org.opensearch.index.query.GeoBoundingBoxQueryBuilder; -import org.opensearch.search.aggregations.AggregationBuilders; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.bucket.filter.Filter; -import org.opensearch.search.aggregations.bucket.geogrid.GeoGrid; -import org.opensearch.search.aggregations.bucket.geogrid.GeoGrid.Bucket; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.VersionUtils; @@ -57,17 +57,16 @@ import java.util.Random; import java.util.Set; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.geometry.utils.Geohash.PRECISION; import static org.opensearch.geometry.utils.Geohash.stringEncode; -import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.opensearch.search.aggregations.AggregationBuilders.geohashGrid; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class GeoHashGridIT extends OpenSearchIntegTestCase { +public class GeoHashGridIT extends GeoModulePluginIntegTestCase { @Override protected boolean forbidPrivateIndexSettings() { @@ -158,13 +157,13 @@ public void setupSuiteScopeCluster() throws Exception { public void testSimple() throws Exception { for (int precision = 1; precision <= PRECISION; precision++) { SearchResponse response = client().prepareSearch("idx") - .addAggregation(geohashGrid("geohashgrid").field("location").precision(precision)) + .addAggregation(AggregationBuilders.geohashGrid("geohashgrid").field("location").precision(precision)) .get(); assertSearchResponse(response); GeoGrid geoGrid = response.getAggregations().get("geohashgrid"); - List buckets = geoGrid.getBuckets(); + List buckets = geoGrid.getBuckets(); Object[] propertiesKeys = (Object[]) ((InternalAggregation) geoGrid).getProperty("_key"); Object[] propertiesDocCounts = (Object[]) ((InternalAggregation) geoGrid).getProperty("_count"); for (int i = 0; i < buckets.size(); i++) { @@ -185,7 +184,7 @@ public void testSimple() throws Exception { public void testMultivalued() throws Exception { for (int precision = 1; precision <= PRECISION; precision++) { SearchResponse response = client().prepareSearch("multi_valued_idx") - .addAggregation(geohashGrid("geohashgrid").field("location").precision(precision)) + .addAggregation(AggregationBuilders.geohashGrid("geohashgrid").field("location").precision(precision)) .get(); assertSearchResponse(response); @@ -208,8 +207,8 @@ public void testFiltered() throws Exception { for (int precision = 1; precision <= PRECISION; precision++) { SearchResponse response = client().prepareSearch("idx") .addAggregation( - AggregationBuilders.filter("filtered", bbox) - .subAggregation(geohashGrid("geohashgrid").field("location").precision(precision)) + org.opensearch.search.aggregations.AggregationBuilders.filter("filtered", bbox) + .subAggregation(AggregationBuilders.geohashGrid("geohashgrid").field("location").precision(precision)) ) .get(); @@ -233,7 +232,7 @@ public void testFiltered() throws Exception { public void testUnmapped() throws Exception { for (int precision = 1; precision <= PRECISION; precision++) { SearchResponse response = client().prepareSearch("idx_unmapped") - .addAggregation(geohashGrid("geohashgrid").field("location").precision(precision)) + .addAggregation(AggregationBuilders.geohashGrid("geohashgrid").field("location").precision(precision)) .get(); assertSearchResponse(response); @@ -247,7 +246,7 @@ public void testUnmapped() throws Exception { public void testPartiallyUnmapped() throws Exception { for (int precision = 1; precision <= PRECISION; precision++) { SearchResponse response = client().prepareSearch("idx", "idx_unmapped") - .addAggregation(geohashGrid("geohashgrid").field("location").precision(precision)) + .addAggregation(AggregationBuilders.geohashGrid("geohashgrid").field("location").precision(precision)) .get(); assertSearchResponse(response); @@ -267,7 +266,9 @@ public void testPartiallyUnmapped() throws Exception { public void testTopMatch() throws Exception { for (int precision = 1; precision <= PRECISION; precision++) { SearchResponse response = client().prepareSearch("idx") - .addAggregation(geohashGrid("geohashgrid").field("location").size(1).shardSize(100).precision(precision)) + .addAggregation( + AggregationBuilders.geohashGrid("geohashgrid").field("location").size(1).shardSize(100).precision(precision) + ) .get(); assertSearchResponse(response); @@ -296,7 +297,7 @@ public void testSizeIsZero() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> client().prepareSearch("idx") - .addAggregation(geohashGrid("geohashgrid").field("location").size(size).shardSize(shardSize)) + .addAggregation(AggregationBuilders.geohashGrid("geohashgrid").field("location").size(size).shardSize(shardSize)) .get() ); assertThat(exception.getMessage(), containsString("[size] must be greater than 0. Found [0] in [geohashgrid]")); @@ -308,7 +309,7 @@ public void testShardSizeIsZero() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> client().prepareSearch("idx") - .addAggregation(geohashGrid("geohashgrid").field("location").size(size).shardSize(shardSize)) + .addAggregation(AggregationBuilders.geohashGrid("geohashgrid").field("location").size(size).shardSize(shardSize)) .get() ); assertThat(exception.getMessage(), containsString("[shardSize] must be greater than 0. Found [0] in [geohashgrid]")); diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/ShardReduceIT.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/ShardReduceIT.java new file mode 100644 index 0000000000000..5b4dd052a2f65 --- /dev/null +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/ShardReduceIT.java @@ -0,0 +1,107 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.search.aggregations.bucket; + +import org.opensearch.action.index.IndexRequestBuilder; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.geo.GeoModulePluginIntegTestCase; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGrid; +import org.opensearch.geo.tests.common.AggregationBuilders; +import org.opensearch.geometry.utils.Geohash; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.opensearch.search.aggregations.bucket.histogram.Histogram; +import org.opensearch.test.OpenSearchIntegTestCase; + +import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; + +/** + * Tests making sure that the reduce is propagated to all aggregations in the hierarchy when executing on a single shard + * These tests are based on the date histogram in combination of min_doc_count=0. In order for the date histogram to + * compute empty buckets, its {@code reduce()} method must be called. So by adding the date histogram under other buckets, + * we can make sure that the reduce is properly propagated by checking that empty buckets were created. + */ +@OpenSearchIntegTestCase.SuiteScopeTestCase +public class ShardReduceIT extends GeoModulePluginIntegTestCase { + + private IndexRequestBuilder indexDoc(String date, int value) throws Exception { + return client().prepareIndex("idx") + .setSource( + jsonBuilder().startObject() + .field("value", value) + .field("ip", "10.0.0." + value) + .field("location", Geohash.stringEncode(5, 52, Geohash.PRECISION)) + .field("date", date) + .field("term-l", 1) + .field("term-d", 1.5) + .field("term-s", "term") + .startObject("nested") + .field("date", date) + .endObject() + .endObject() + ); + } + + @Override + public void setupSuiteScopeCluster() throws Exception { + assertAcked( + prepareCreate("idx").setMapping( + "nested", + "type=nested", + "ip", + "type=ip", + "location", + "type=geo_point", + "term-s", + "type=keyword" + ) + ); + + indexRandom(true, indexDoc("2014-01-01", 1), indexDoc("2014-01-02", 2), indexDoc("2014-01-04", 3)); + ensureSearchable(); + } + + public void testGeoHashGrid() throws Exception { + SearchResponse response = client().prepareSearch("idx") + .setQuery(QueryBuilders.matchAllQuery()) + .addAggregation( + AggregationBuilders.geohashGrid("grid") + .field("location") + .subAggregation(dateHistogram("histo").field("date").fixedInterval(DateHistogramInterval.DAY).minDocCount(0)) + ) + .get(); + + assertSearchResponse(response); + + GeoGrid grid = response.getAggregations().get("grid"); + Histogram histo = grid.getBuckets().iterator().next().getAggregations().get("histo"); + assertThat(histo.getBuckets().size(), equalTo(4)); + } + + public void testGeoTileGrid() throws Exception { + SearchResponse response = client().prepareSearch("idx") + .setQuery(QueryBuilders.matchAllQuery()) + .addAggregation( + AggregationBuilders.geotileGrid("grid") + .field("location") + .subAggregation(dateHistogram("histo").field("date").fixedInterval(DateHistogramInterval.DAY).minDocCount(0)) + ) + .get(); + + assertSearchResponse(response); + + GeoGrid grid = response.getAggregations().get("grid"); + Histogram histo = grid.getBuckets().iterator().next().getAggregations().get("histo"); + assertThat(histo.getBuckets().size(), equalTo(4)); + } +} diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoCentroidIT.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoCentroidIT.java new file mode 100644 index 0000000000000..76f59afa67726 --- /dev/null +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoCentroidIT.java @@ -0,0 +1,84 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.geo.search.aggregations.metrics; + +import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.geo.GeoPoint; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGrid; +import org.opensearch.geo.tests.common.AggregationBuilders; +import org.opensearch.search.aggregations.metrics.GeoCentroid; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.util.List; + +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.opensearch.search.aggregations.AggregationBuilders.geoCentroid; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; + +@OpenSearchIntegTestCase.SuiteScopeTestCase +public class GeoCentroidIT extends AbstractGeoAggregatorTestCaseModulePlugin { + private static final String aggName = "geoCentroid"; + + public void testSingleValueFieldAsSubAggToGeohashGrid() throws Exception { + SearchResponse response = client().prepareSearch(HIGH_CARD_IDX_NAME) + .addAggregation( + AggregationBuilders.geohashGrid("geoGrid") + .field(SINGLE_VALUED_FIELD_NAME) + .subAggregation(geoCentroid(aggName).field(SINGLE_VALUED_FIELD_NAME)) + ) + .get(); + assertSearchResponse(response); + + GeoGrid grid = response.getAggregations().get("geoGrid"); + assertThat(grid, notNullValue()); + assertThat(grid.getName(), equalTo("geoGrid")); + List buckets = grid.getBuckets(); + for (GeoGrid.Bucket cell : buckets) { + String geohash = cell.getKeyAsString(); + GeoPoint expectedCentroid = expectedCentroidsForGeoHash.get(geohash); + GeoCentroid centroidAgg = cell.getAggregations().get(aggName); + assertThat( + "Geohash " + geohash + " has wrong centroid latitude ", + expectedCentroid.lat(), + closeTo(centroidAgg.centroid().lat(), GEOHASH_TOLERANCE) + ); + assertThat( + "Geohash " + geohash + " has wrong centroid longitude", + expectedCentroid.lon(), + closeTo(centroidAgg.centroid().lon(), GEOHASH_TOLERANCE) + ); + } + } +} diff --git a/modules/geo/src/main/java/org/opensearch/geo/GeoModulePlugin.java b/modules/geo/src/main/java/org/opensearch/geo/GeoModulePlugin.java index 64aac66b7eef3..25dcf8db2c407 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/GeoModulePlugin.java +++ b/modules/geo/src/main/java/org/opensearch/geo/GeoModulePlugin.java @@ -32,6 +32,12 @@ package org.opensearch.geo; +import org.opensearch.geo.search.aggregations.bucket.composite.GeoTileGridValuesSourceBuilder; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoTileGridAggregator; +import org.opensearch.geo.search.aggregations.bucket.geogrid.InternalGeoHashGrid; +import org.opensearch.geo.search.aggregations.bucket.geogrid.InternalGeoTileGrid; import org.opensearch.geo.search.aggregations.metrics.GeoBounds; import org.opensearch.geo.search.aggregations.metrics.GeoBoundsAggregationBuilder; import org.opensearch.geo.search.aggregations.metrics.InternalGeoBounds; @@ -40,6 +46,7 @@ import org.opensearch.plugins.MapperPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.SearchPlugin; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregation; import java.util.Collections; import java.util.List; @@ -57,11 +64,42 @@ public Map getMappers() { */ @Override public List getAggregations() { - final AggregationSpec spec = new AggregationSpec( + final AggregationSpec geoBounds = new AggregationSpec( GeoBoundsAggregationBuilder.NAME, GeoBoundsAggregationBuilder::new, GeoBoundsAggregationBuilder.PARSER ).addResultReader(InternalGeoBounds::new).setAggregatorRegistrar(GeoBoundsAggregationBuilder::registerAggregators); - return Collections.singletonList(spec); + + final AggregationSpec geoHashGrid = new AggregationSpec( + GeoHashGridAggregationBuilder.NAME, + GeoHashGridAggregationBuilder::new, + GeoHashGridAggregationBuilder.PARSER + ).addResultReader(InternalGeoHashGrid::new).setAggregatorRegistrar(GeoHashGridAggregationBuilder::registerAggregators); + + final AggregationSpec geoTileGrid = new AggregationSpec( + GeoTileGridAggregationBuilder.NAME, + GeoTileGridAggregationBuilder::new, + GeoTileGridAggregationBuilder.PARSER + ).addResultReader(InternalGeoTileGrid::new).setAggregatorRegistrar(GeoTileGridAggregationBuilder::registerAggregators); + return List.of(geoBounds, geoHashGrid, geoTileGrid); + } + + /** + * Registering the {@link GeoTileGridAggregator} in the {@link CompositeAggregation}. + * + * @return a {@link List} of {@link CompositeAggregationSpec} + */ + @Override + public List getCompositeAggregations() { + return Collections.singletonList( + new CompositeAggregationSpec( + GeoTileGridValuesSourceBuilder::register, + GeoTileGridValuesSourceBuilder.class, + GeoTileGridValuesSourceBuilder.COMPOSITE_AGGREGATION_SERIALISATION_BYTE_CODE, + GeoTileGridValuesSourceBuilder::new, + GeoTileGridValuesSourceBuilder::parse, + GeoTileGridValuesSourceBuilder.TYPE + ) + ); } } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java similarity index 87% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java index 4b01a08d29a43..84d5943da287f 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.composite; +package org.opensearch.geo.search.aggregations.bucket.composite; import org.apache.lucene.index.IndexReader; import org.opensearch.LegacyESVersion; @@ -43,12 +43,15 @@ import org.opensearch.common.xcontent.ObjectParser; import org.opensearch.common.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.geo.search.aggregations.bucket.geogrid.CellIdSource; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.query.QueryShardContext; import org.opensearch.search.DocValueFormat; -import org.opensearch.search.aggregations.bucket.geogrid.CellIdSource; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; +import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceConfig; +import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceParserHelper; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import org.opensearch.search.aggregations.bucket.missing.MissingOrder; import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.search.aggregations.support.ValuesSource; @@ -88,13 +91,19 @@ CompositeValuesSourceConfig apply( ); } - static final String TYPE = "geotile_grid"; + public static final String TYPE = "geotile_grid"; + /* + use the TYPE parameter instead of Byte code. The byte code is added for backward compatibility and will be + removed in the next version. + */ + @Deprecated + public static final Byte COMPOSITE_AGGREGATION_SERIALISATION_BYTE_CODE = 3; static final ValuesSourceRegistry.RegistryKey REGISTRY_KEY = new ValuesSourceRegistry.RegistryKey( TYPE, GeoTileCompositeSuppier.class ); - private static final ObjectParser PARSER; + static final ObjectParser PARSER; static { PARSER = new ObjectParser<>(GeoTileGridValuesSourceBuilder.TYPE); PARSER.declareInt(GeoTileGridValuesSourceBuilder::precision, new ParseField("precision")); @@ -106,11 +115,11 @@ CompositeValuesSourceConfig apply( CompositeValuesSourceParserHelper.declareValuesSourceFields(PARSER); } - static GeoTileGridValuesSourceBuilder parse(String name, XContentParser parser) throws IOException { + public static GeoTileGridValuesSourceBuilder parse(String name, XContentParser parser) throws IOException { return PARSER.parse(parser, new GeoTileGridValuesSourceBuilder(name), null); } - static void register(ValuesSourceRegistry.Builder builder) { + public static void register(ValuesSourceRegistry.Builder builder) { builder.register( REGISTRY_KEY, @@ -163,7 +172,7 @@ static void register(ValuesSourceRegistry.Builder builder) { super(name); } - GeoTileGridValuesSourceBuilder(StreamInput in) throws IOException { + public GeoTileGridValuesSourceBuilder(StreamInput in) throws IOException { super(in); this.precision = in.readInt(); if (in.getVersion().onOrAfter(LegacyESVersion.V_7_6_0)) { @@ -203,7 +212,7 @@ protected void doXContentBody(XContentBuilder builder, Params params) throws IOE } @Override - String type() { + protected String type() { return TYPE; } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/GeoTileValuesSource.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileValuesSource.java similarity index 88% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/composite/GeoTileValuesSource.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileValuesSource.java index 819dfc573bbe4..303e577e99e7b 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/GeoTileValuesSource.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileValuesSource.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.composite; +package org.opensearch.geo.search.aggregations.bucket.composite; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedNumericDocValues; @@ -38,7 +38,9 @@ import org.opensearch.common.util.BigArrays; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.search.DocValueFormat; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.opensearch.search.aggregations.bucket.composite.LongValuesSource; +import org.opensearch.search.aggregations.bucket.composite.SingleDimensionValuesSource; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import org.opensearch.search.aggregations.bucket.missing.MissingOrder; import java.io.IOException; @@ -68,7 +70,7 @@ class GeoTileValuesSource extends LongValuesSource { } @Override - void setAfter(Comparable value) { + protected void setAfter(Comparable value) { if (missingBucket && value == null) { afterValue = null; } else if (value instanceof Number) { diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/BoundedCellValues.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BoundedCellValues.java similarity index 97% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/BoundedCellValues.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BoundedCellValues.java index ba824fc8f21dd..06d2dcaee3932 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/BoundedCellValues.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BoundedCellValues.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.index.fielddata.MultiGeoPointValues; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/BucketPriorityQueue.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BucketPriorityQueue.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/BucketPriorityQueue.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BucketPriorityQueue.java index d6cfde0c46eae..70d0552b3e80b 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/BucketPriorityQueue.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BucketPriorityQueue.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.apache.lucene.util.PriorityQueue; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/CellIdSource.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/CellIdSource.java similarity index 98% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/CellIdSource.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/CellIdSource.java index 12d9043a2fd5f..d40029e9a762d 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/CellIdSource.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/CellIdSource.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedNumericDocValues; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/CellValues.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/CellValues.java similarity index 97% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/CellValues.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/CellValues.java index 9dc357659aae8..d01896c8136fa 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/CellValues.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/CellValues.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.index.fielddata.AbstractSortingNumericDocValues; import org.opensearch.index.fielddata.MultiGeoPointValues; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGrid.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGrid.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGrid.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGrid.java index cfdb08f9ee3d7..4ae888640efc8 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGrid.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGrid.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java similarity index 99% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java index b08c40268c5cf..4a904b3aa2b16 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.LegacyESVersion; import org.opensearch.OpenSearchException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridAggregator.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregator.java similarity index 99% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridAggregator.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregator.java index 1ef8ba6c697f4..909772c61a960 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridAggregator.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregator.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedNumericDocValues; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java index 4049bf2c73640..bbaf9613fb216 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.common.geo.GeoUtils; @@ -40,7 +40,7 @@ import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.AggregatorFactory; -import org.opensearch.search.aggregations.metrics.GeoGridAggregatorSupplier; +import org.opensearch.geo.search.aggregations.metrics.GeoGridAggregatorSupplier; import org.opensearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.opensearch.search.aggregations.support.ValuesSourceConfig; import org.opensearch.search.aggregations.support.ValuesSourceRegistry; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java similarity index 97% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java index 1106320c7431f..6ca7a4d8a9cb8 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.AggregatorFactories; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java similarity index 98% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java index cdc801aaedffb..1914c07e831f7 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.geometry.utils.Geohash; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java similarity index 95% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java index f73360e3cb826..76ad515f34fe5 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.common.io.stream.StreamInput; @@ -39,7 +39,8 @@ import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.AggregatorFactory; -import org.opensearch.search.aggregations.metrics.GeoGridAggregatorSupplier; +import org.opensearch.geo.search.aggregations.metrics.GeoGridAggregatorSupplier; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import org.opensearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.opensearch.search.aggregations.support.ValuesSourceConfig; import org.opensearch.search.aggregations.support.ValuesSourceRegistry; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java similarity index 97% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java index 7a2b908148c4c..a205a9afde41e 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.AggregatorFactories; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java similarity index 97% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java index ef8cd11a22498..b830988a3d410 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.index.query.QueryShardContext; @@ -40,6 +40,7 @@ import org.opensearch.search.aggregations.CardinalityUpperBound; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.NonCollectingAggregator; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.search.aggregations.support.ValuesSource; import org.opensearch.search.aggregations.support.ValuesSourceAggregatorFactory; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoGrid.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoGrid.java similarity index 99% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoGrid.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoGrid.java index 94a5ad5717854..9dbed7b27307a 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoGrid.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoGrid.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoGridBucket.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoGridBucket.java similarity index 98% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoGridBucket.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoGridBucket.java index a187bfefb661f..93fcdbd098400 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoGridBucket.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoGridBucket.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java similarity index 97% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java index 7811b8774d04f..ff1247300939a 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.search.aggregations.InternalAggregations; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoHashGridBucket.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoHashGridBucket.java similarity index 95% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoHashGridBucket.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoHashGridBucket.java index f9c45dc41ceb1..ea8678fb51673 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoHashGridBucket.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoHashGridBucket.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.io.stream.StreamInput; @@ -43,7 +43,7 @@ * * @opensearch.internal */ -public class InternalGeoHashGridBucket extends InternalGeoGridBucket { +public class InternalGeoHashGridBucket extends InternalGeoGridBucket { InternalGeoHashGridBucket(long hashAsLong, long docCount, InternalAggregations aggregations) { super(hashAsLong, docCount, aggregations); } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoTileGrid.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoTileGrid.java similarity index 97% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoTileGrid.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoTileGrid.java index efbd9a05d6a4d..fa544b5893f0c 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoTileGrid.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoTileGrid.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.search.aggregations.InternalAggregations; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java similarity index 93% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java index f200f55232e00..6d6c8b499dbe8 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java @@ -30,11 +30,12 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.search.aggregations.InternalAggregations; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import java.io.IOException; @@ -43,7 +44,7 @@ * * @opensearch.internal */ -public class InternalGeoTileGridBucket extends InternalGeoGridBucket { +public class InternalGeoTileGridBucket extends InternalGeoGridBucket { InternalGeoTileGridBucket(long hashAsLong, long docCount, InternalAggregations aggregations) { super(hashAsLong, docCount, aggregations); } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoGrid.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoGrid.java similarity index 97% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoGrid.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoGrid.java index 3f85cf350c89c..adfffeddba59d 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoGrid.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoGrid.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.CheckedFunction; import org.opensearch.common.xcontent.ObjectParser; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoGridBucket.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoGridBucket.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoGridBucket.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoGridBucket.java index 08e5c15188ee6..80124cda50b19 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoGridBucket.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoGridBucket.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.xcontent.XContentBuilder; import org.opensearch.search.aggregations.Aggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoHashGrid.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoHashGrid.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoHashGrid.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoHashGrid.java index f20f972c1ce0a..109524e755c4d 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoHashGrid.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoHashGrid.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.xcontent.ObjectParser; import org.opensearch.common.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoHashGridBucket.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoHashGridBucket.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoHashGridBucket.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoHashGridBucket.java index 05c7a1c8d1663..4e6e454b08324 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoHashGridBucket.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoHashGridBucket.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoTileGrid.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoTileGrid.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoTileGrid.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoTileGrid.java index 06915cc4210e1..8734c96a15578 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoTileGrid.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoTileGrid.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.xcontent.ObjectParser; import org.opensearch.common.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java similarity index 93% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java index c8dec16f322ef..fd47c35f13de1 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java @@ -30,10 +30,11 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/UnboundedCellValues.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/UnboundedCellValues.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/UnboundedCellValues.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/UnboundedCellValues.java index f5a139cdb8d9d..c628c7bfdc8ec 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/UnboundedCellValues.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/UnboundedCellValues.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.index.fielddata.MultiGeoPointValues; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/package-info.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/package-info.java similarity index 79% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/package-info.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/package-info.java index c59685e06cf79..d9183a0f742ef 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/package-info.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/package-info.java @@ -7,4 +7,4 @@ */ /** geo_grid Aggregation package. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/GeoGridAggregatorSupplier.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoGridAggregatorSupplier.java similarity index 93% rename from server/src/main/java/org/opensearch/search/aggregations/metrics/GeoGridAggregatorSupplier.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoGridAggregatorSupplier.java index 183c64f4e4af2..43ccb8b89545a 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/GeoGridAggregatorSupplier.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoGridAggregatorSupplier.java @@ -30,13 +30,13 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.metrics; +package org.opensearch.geo.search.aggregations.metrics; import org.opensearch.common.geo.GeoBoundingBox; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGridAggregator; import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.CardinalityUpperBound; -import org.opensearch.search.aggregations.bucket.geogrid.GeoGridAggregator; import org.opensearch.search.aggregations.support.ValuesSource; import org.opensearch.search.internal.SearchContext; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/GeoHashGridTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridAggregationBuilderTests.java similarity index 72% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/GeoHashGridTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridAggregationBuilderTests.java index 5e230a445ec98..00cb162e64c19 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/GeoHashGridTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridAggregationBuilderTests.java @@ -30,14 +30,23 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket; +package org.opensearch.geo.search.aggregations.bucket; -import org.opensearch.common.geo.GeoBoundingBoxTests; +import org.opensearch.geo.GeoModulePlugin; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; +import org.opensearch.geo.tests.common.RandomGeoGenerator; +import org.opensearch.plugins.Plugin; import org.opensearch.search.aggregations.BaseAggregationTestCase; -import org.opensearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; -public class GeoHashGridTests extends BaseAggregationTestCase { +import java.util.Collection; +import java.util.Collections; + +public class GeoHashGridAggregationBuilderTests extends BaseAggregationTestCase { + + protected Collection> getPlugins() { + return Collections.singletonList(GeoModulePlugin.class); + } @Override protected GeoHashGridAggregationBuilder createTestAggregatorBuilder() { @@ -55,7 +64,7 @@ protected GeoHashGridAggregationBuilder createTestAggregatorBuilder() { factory.shardSize(randomIntBetween(1, Integer.MAX_VALUE)); } if (randomBoolean()) { - factory.setGeoBoundingBox(GeoBoundingBoxTests.randomBBox()); + factory.setGeoBoundingBox(RandomGeoGenerator.randomBBox()); } return factory; } diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/GeoTileGridTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/GeoTileGridAggregationBuilderTests.java similarity index 70% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/GeoTileGridTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/GeoTileGridAggregationBuilderTests.java index d54667fb4f1a6..c7c0be21273bd 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/GeoTileGridTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/GeoTileGridAggregationBuilderTests.java @@ -30,15 +30,24 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket; +package org.opensearch.geo.search.aggregations.bucket; -import org.opensearch.common.geo.GeoBoundingBoxTests; +import org.opensearch.geo.GeoModulePlugin; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; +import org.opensearch.geo.tests.common.RandomGeoGenerator; +import org.opensearch.plugins.Plugin; import org.opensearch.search.aggregations.BaseAggregationTestCase; -import org.opensearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; -public class GeoTileGridTests extends BaseAggregationTestCase { +import java.util.Collection; +import java.util.Collections; + +public class GeoTileGridAggregationBuilderTests extends BaseAggregationTestCase { + + protected Collection> getPlugins() { + return Collections.singletonList(GeoModulePlugin.class); + } @Override protected GeoTileGridAggregationBuilder createTestAggregatorBuilder() { @@ -55,7 +64,7 @@ protected GeoTileGridAggregationBuilder createTestAggregatorBuilder() { factory.shardSize(randomIntBetween(1, Integer.MAX_VALUE)); } if (randomBoolean()) { - factory.setGeoBoundingBox(GeoBoundingBoxTests.randomBBox()); + factory.setGeoBoundingBox(RandomGeoGenerator.randomBBox()); } return factory; } diff --git a/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridAggregationCompositeAggregatorTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridAggregationCompositeAggregatorTests.java new file mode 100644 index 0000000000000..3c7c292f9d193 --- /dev/null +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridAggregationCompositeAggregatorTests.java @@ -0,0 +1,174 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.search.aggregations.bucket.composite; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.LatLonPoint; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.search.DocValuesFieldExistsQuery; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.junit.Before; +import org.opensearch.common.geo.GeoPoint; +import org.opensearch.geo.GeoModulePlugin; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoTileGridAggregator; +import org.opensearch.index.mapper.GeoPointFieldMapper; +import org.opensearch.plugins.SearchPlugin; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; +import org.opensearch.search.aggregations.composite.BaseCompositeAggregatorTestCase; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Testing the {@link GeoTileGridAggregator} as part of CompositeAggregation. + */ +public class GeoTileGridAggregationCompositeAggregatorTests extends BaseCompositeAggregatorTestCase { + + protected List getSearchPlugins() { + return Collections.singletonList(new GeoModulePlugin()); + } + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + FIELD_TYPES.add(new GeoPointFieldMapper.GeoPointFieldType("geo_point")); + } + + public void testUnmappedFieldWithGeopoint() throws Exception { + final List>> dataset = new ArrayList<>(); + final String mappedFieldName = "geo_point"; + dataset.addAll( + Arrays.asList( + createDocument(mappedFieldName, new GeoPoint(48.934059, 41.610741)), + createDocument(mappedFieldName, new GeoPoint(-23.065941, 113.610741)), + createDocument(mappedFieldName, new GeoPoint(90.0, 0.0)), + createDocument(mappedFieldName, new GeoPoint(37.2343, -115.8067)), + createDocument(mappedFieldName, new GeoPoint(90.0, 0.0)) + ) + ); + + // just unmapped = no results + testSearchCase( + Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery(mappedFieldName)), + dataset, + () -> new CompositeAggregationBuilder("name", Arrays.asList(new GeoTileGridValuesSourceBuilder("unmapped").field("unmapped"))), + (result) -> assertEquals(0, result.getBuckets().size()) + ); + + // unmapped missing bucket = one result + testSearchCase( + Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery(mappedFieldName)), + dataset, + () -> new CompositeAggregationBuilder( + "name", + Arrays.asList(new GeoTileGridValuesSourceBuilder("unmapped").field("unmapped").missingBucket(true)) + ), + (result) -> { + assertEquals(1, result.getBuckets().size()); + assertEquals("{unmapped=null}", result.afterKey().toString()); + assertEquals("{unmapped=null}", result.getBuckets().get(0).getKeyAsString()); + assertEquals(5L, result.getBuckets().get(0).getDocCount()); + } + ); + + // field + unmapped, no missing bucket = no results + testSearchCase( + Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery(mappedFieldName)), + dataset, + () -> new CompositeAggregationBuilder( + "name", + Arrays.asList( + new GeoTileGridValuesSourceBuilder(mappedFieldName).field(mappedFieldName), + new GeoTileGridValuesSourceBuilder("unmapped").field("unmapped") + ) + ), + (result) -> assertEquals(0, result.getBuckets().size()) + ); + + // field + unmapped with missing bucket = multiple results + testSearchCase( + Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery(mappedFieldName)), + dataset, + () -> new CompositeAggregationBuilder( + "name", + Arrays.asList( + new GeoTileGridValuesSourceBuilder(mappedFieldName).field(mappedFieldName), + new GeoTileGridValuesSourceBuilder("unmapped").field("unmapped").missingBucket(true) + ) + ), + (result) -> { + assertEquals(2, result.getBuckets().size()); + assertEquals("{geo_point=7/64/56, unmapped=null}", result.afterKey().toString()); + assertEquals("{geo_point=7/32/56, unmapped=null}", result.getBuckets().get(0).getKeyAsString()); + assertEquals(2L, result.getBuckets().get(0).getDocCount()); + assertEquals("{geo_point=7/64/56, unmapped=null}", result.getBuckets().get(1).getKeyAsString()); + assertEquals(3L, result.getBuckets().get(1).getDocCount()); + } + ); + + } + + public void testWithGeoPoint() throws Exception { + final List>> dataset = new ArrayList<>(); + dataset.addAll( + Arrays.asList( + createDocument("geo_point", new GeoPoint(48.934059, 41.610741)), + createDocument("geo_point", new GeoPoint(-23.065941, 113.610741)), + createDocument("geo_point", new GeoPoint(90.0, 0.0)), + createDocument("geo_point", new GeoPoint(37.2343, -115.8067)), + createDocument("geo_point", new GeoPoint(90.0, 0.0)) + ) + ); + testSearchCase(Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery("geo_point")), dataset, () -> { + GeoTileGridValuesSourceBuilder geoTile = new GeoTileGridValuesSourceBuilder("geo_point").field("geo_point"); + return new CompositeAggregationBuilder("name", Collections.singletonList(geoTile)); + }, (result) -> { + assertEquals(2, result.getBuckets().size()); + assertEquals("{geo_point=7/64/56}", result.afterKey().toString()); + assertEquals("{geo_point=7/32/56}", result.getBuckets().get(0).getKeyAsString()); + assertEquals(2L, result.getBuckets().get(0).getDocCount()); + assertEquals("{geo_point=7/64/56}", result.getBuckets().get(1).getKeyAsString()); + assertEquals(3L, result.getBuckets().get(1).getDocCount()); + }); + + testSearchCase(Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery("geo_point")), dataset, () -> { + GeoTileGridValuesSourceBuilder geoTile = new GeoTileGridValuesSourceBuilder("geo_point").field("geo_point"); + return new CompositeAggregationBuilder("name", Collections.singletonList(geoTile)).aggregateAfter( + Collections.singletonMap("geo_point", "7/32/56") + ); + }, (result) -> { + assertEquals(1, result.getBuckets().size()); + assertEquals("{geo_point=7/64/56}", result.afterKey().toString()); + assertEquals("{geo_point=7/64/56}", result.getBuckets().get(0).getKeyAsString()); + assertEquals(3L, result.getBuckets().get(0).getDocCount()); + }); + } + + @Override + protected boolean addValueToDocument(final Document doc, final String name, final Object value) { + if (value instanceof GeoPoint) { + GeoPoint point = (GeoPoint) value; + doc.add( + new SortedNumericDocValuesField( + name, + GeoTileUtils.longEncode(point.lon(), point.lat(), GeoTileGridAggregationBuilder.DEFAULT_PRECISION) + ) + ); + doc.add(new LatLonPoint(name, point.lat(), point.lon())); + return true; + } + return false; + } +} diff --git a/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridCompositeAggregationBuilderTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridCompositeAggregationBuilderTests.java new file mode 100644 index 0000000000000..ea7a2a83945c2 --- /dev/null +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridCompositeAggregationBuilderTests.java @@ -0,0 +1,50 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.search.aggregations.bucket.composite; + +import org.opensearch.geo.GeoModulePlugin; +import org.opensearch.geo.tests.common.RandomGeoGenerator; +import org.opensearch.plugins.Plugin; +import org.opensearch.search.aggregations.BaseAggregationTestCase; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; +import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +public class GeoTileGridCompositeAggregationBuilderTests extends BaseAggregationTestCase { + + protected Collection> getPlugins() { + return Collections.singletonList(GeoModulePlugin.class); + } + + private GeoTileGridValuesSourceBuilder randomGeoTileGridValuesSourceBuilder() { + GeoTileGridValuesSourceBuilder geoTile = new GeoTileGridValuesSourceBuilder(randomAlphaOfLengthBetween(5, 10)); + if (randomBoolean()) { + geoTile.precision(randomIntBetween(0, GeoTileUtils.MAX_ZOOM)); + } + if (randomBoolean()) { + geoTile.geoBoundingBox(RandomGeoGenerator.randomBBox()); + } + return geoTile; + } + + @Override + protected CompositeAggregationBuilder createTestAggregatorBuilder() { + int numSources = randomIntBetween(1, 10); + List> sources = new ArrayList<>(); + for (int i = 0; i < numSources; i++) { + sources.add(randomGeoTileGridValuesSourceBuilder()); + } + return new CompositeAggregationBuilder(randomAlphaOfLength(10), sources); + } +} diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilderTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilderTests.java similarity index 90% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilderTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilderTests.java index 2b1700676f549..c6276c06c4511 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilderTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilderTests.java @@ -30,8 +30,9 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.composite; +package org.opensearch.geo.search.aggregations.bucket.composite; +import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; import org.opensearch.test.OpenSearchTestCase; public class GeoTileGridValuesSourceBuilderTests extends OpenSearchTestCase { diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java similarity index 95% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java index 17fddb8978499..d6153637f656d 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.apache.lucene.document.LatLonDocValuesField; import org.apache.lucene.document.SortedSetDocValuesField; @@ -45,17 +45,19 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.common.CheckedConsumer; import org.opensearch.common.geo.GeoBoundingBox; -import org.opensearch.common.geo.GeoBoundingBoxTests; import org.opensearch.common.geo.GeoUtils; +import org.opensearch.geo.GeoModulePlugin; +import org.opensearch.geo.tests.common.AggregationInspectionHelper; +import org.opensearch.geo.tests.common.RandomGeoGenerator; import org.opensearch.index.mapper.GeoPointFieldMapper; import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.plugins.SearchPlugin; import org.opensearch.search.aggregations.Aggregation; import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.AggregatorTestCase; import org.opensearch.search.aggregations.MultiBucketConsumerService; import org.opensearch.search.aggregations.bucket.terms.StringTerms; import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.opensearch.search.aggregations.support.AggregationInspectionHelper; import java.io.IOException; import java.util.ArrayList; @@ -91,6 +93,16 @@ public abstract class GeoGridAggregatorTestCase */ protected abstract GeoGridAggregationBuilder createBuilder(String name); + /** + * Overriding the Search Plugins list with {@link GeoModulePlugin} so that the testcase will know that this plugin is + * to be loaded during the tests. + * @return List of {@link SearchPlugin} + */ + @Override + protected List getSearchPlugins() { + return Collections.singletonList(new GeoModulePlugin()); + } + public void testNoDocs() throws IOException { testCase( new MatchAllDocsQuery(), @@ -225,7 +237,7 @@ public void testBounds() throws IOException { // only consider bounding boxes that are at least GEOHASH_TOLERANCE wide and have quantized coordinates GeoBoundingBox bbox = randomValueOtherThanMany( (b) -> Math.abs(GeoUtils.normalizeLon(b.right()) - GeoUtils.normalizeLon(b.left())) < GEOHASH_TOLERANCE, - GeoBoundingBoxTests::randomBBox + RandomGeoGenerator::randomBBox ); Function encodeDecodeLat = (lat) -> GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(lat)); Function encodeDecodeLon = (lon) -> GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(lon)); diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridTestCase.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridTestCase.java similarity index 79% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridTestCase.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridTestCase.java index ce286a4443660..432736a2b43fe 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridTestCase.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridTestCase.java @@ -29,9 +29,16 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.apache.lucene.index.IndexWriter; +import org.opensearch.common.ParseField; +import org.opensearch.common.xcontent.ContextParser; +import org.opensearch.common.xcontent.NamedXContentRegistry; +import org.opensearch.geo.GeoModulePlugin; +import org.opensearch.geo.search.aggregations.metrics.ParsedGeoBounds; +import org.opensearch.plugins.SearchPlugin; +import org.opensearch.search.aggregations.Aggregation; import org.opensearch.search.aggregations.InternalAggregations; import org.opensearch.search.aggregations.ParsedMultiBucketAggregation; import org.opensearch.test.InternalMultiBucketAggregationTestCase; @@ -76,6 +83,36 @@ protected int maxNumberOfBuckets() { return 3; } + /** + * Overriding the method so that tests can get the aggregation specs for namedWriteable. + * + * @return GeoPlugin + */ + @Override + protected SearchPlugin registerPlugin() { + return new GeoModulePlugin(); + } + + /** + * Overriding with the {@link ParsedGeoBounds} so that it can be parsed. We need to do this as {@link GeoModulePlugin} + * is registering this Aggregation. + * + * @return a List of {@link NamedXContentRegistry.Entry} + */ + @Override + protected List getNamedXContents() { + final List namedXContents = new ArrayList<>(getDefaultNamedXContents()); + final ContextParser hashGridParser = (p, c) -> ParsedGeoHashGrid.fromXContent(p, (String) c); + final ContextParser geoTileParser = (p, c) -> ParsedGeoTileGrid.fromXContent(p, (String) c); + namedXContents.add( + new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(GeoHashGridAggregationBuilder.NAME), hashGridParser) + ); + namedXContents.add( + new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(GeoTileGridAggregationBuilder.NAME), geoTileParser) + ); + return namedXContents; + } + @Override protected T createTestInstance(String name, Map metadata, InternalAggregations aggregations) { final int precision = randomPrecision(); diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java similarity index 96% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java index 5c63b15c7f614..04fa815366f6b 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import static org.opensearch.geometry.utils.Geohash.stringEncode; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java similarity index 99% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java index e81e22b3b562f..44f292e898a61 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.greaterThanOrEqualTo; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridTests.java similarity index 97% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridTests.java index 5a26ec759281c..c84c6ef5ec076 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridTests.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.geometry.utils.Geohash; import org.opensearch.search.aggregations.InternalAggregations; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java similarity index 94% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java index 4e88111ac2dfc..f2f641ea794c0 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java @@ -30,7 +30,9 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; + +import org.opensearch.search.aggregations.bucket.GeoTileUtils; public class GeoTileGridAggregatorTests extends GeoGridAggregatorTestCase { diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java similarity index 97% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java index 567bcd57d23e5..a5b000d5e6ab3 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.ExceptionsHelper; import org.opensearch.common.xcontent.XContentParseException; @@ -37,6 +37,7 @@ import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.geo.GeometryTestUtils; import org.opensearch.geometry.Rectangle; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import org.opensearch.test.OpenSearchTestCase; import static org.hamcrest.Matchers.containsString; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridTests.java similarity index 94% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridTests.java index 50b9a8cd762d1..ead67e0455d94 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridTests.java @@ -29,9 +29,10 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.search.aggregations.InternalAggregations; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import java.util.List; import java.util.Map; diff --git a/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationBuilders.java b/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationBuilders.java index c1f27b71c326d..c0d7e51047c6b 100644 --- a/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationBuilders.java +++ b/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationBuilders.java @@ -8,6 +8,10 @@ package org.opensearch.geo.tests.common; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; +import org.opensearch.geo.search.aggregations.bucket.geogrid.InternalGeoHashGrid; +import org.opensearch.geo.search.aggregations.bucket.geogrid.InternalGeoTileGrid; import org.opensearch.geo.search.aggregations.metrics.GeoBounds; import org.opensearch.geo.search.aggregations.metrics.GeoBoundsAggregationBuilder; @@ -18,4 +22,18 @@ public class AggregationBuilders { public static GeoBoundsAggregationBuilder geoBounds(String name) { return new GeoBoundsAggregationBuilder(name); } + + /** + * Create a new {@link InternalGeoHashGrid} aggregation with the given name. + */ + public static GeoHashGridAggregationBuilder geohashGrid(String name) { + return new GeoHashGridAggregationBuilder(name); + } + + /** + * Create a new {@link InternalGeoTileGrid} aggregation with the given name. + */ + public static GeoTileGridAggregationBuilder geotileGrid(String name) { + return new GeoTileGridAggregationBuilder(name); + } } diff --git a/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationInspectionHelper.java b/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationInspectionHelper.java index 208187bf34a5c..3473cf2d94b76 100644 --- a/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationInspectionHelper.java +++ b/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationInspectionHelper.java @@ -8,6 +8,7 @@ package org.opensearch.geo.tests.common; +import org.opensearch.geo.search.aggregations.bucket.geogrid.InternalGeoGrid; import org.opensearch.geo.search.aggregations.metrics.InternalGeoBounds; public class AggregationInspectionHelper { @@ -15,4 +16,8 @@ public class AggregationInspectionHelper { public static boolean hasValue(InternalGeoBounds agg) { return (agg.topLeft() == null && agg.bottomRight() == null) == false; } + + public static boolean hasValue(InternalGeoGrid agg) { + return agg.getBuckets().stream().anyMatch(bucket -> bucket.getDocCount() > 0); + } } diff --git a/modules/geo/src/test/java/org/opensearch/geo/tests/common/RandomGeoGenerator.java b/modules/geo/src/test/java/org/opensearch/geo/tests/common/RandomGeoGenerator.java index 2cf32c36b97ec..2fb403155e2bc 100644 --- a/modules/geo/src/test/java/org/opensearch/geo/tests/common/RandomGeoGenerator.java +++ b/modules/geo/src/test/java/org/opensearch/geo/tests/common/RandomGeoGenerator.java @@ -8,7 +8,10 @@ package org.opensearch.geo.tests.common; +import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.common.geo.GeoPoint; +import org.opensearch.geo.GeometryTestUtils; +import org.opensearch.geometry.Rectangle; import java.util.Random; @@ -83,4 +86,12 @@ private static double normalizeLongitude(double longitude) { return -180 + off; } } + + public static GeoBoundingBox randomBBox() { + Rectangle rectangle = GeometryTestUtils.randomRectangle(); + return new GeoBoundingBox( + new GeoPoint(rectangle.getMaxLat(), rectangle.getMinLon()), + new GeoPoint(rectangle.getMinLat(), rectangle.getMaxLon()) + ); + } } diff --git a/modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape/230_composite.yml b/modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape/230_composite.yml new file mode 100644 index 0000000000000..211f3c3f46b88 --- /dev/null +++ b/modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape/230_composite.yml @@ -0,0 +1,168 @@ +--- +setup: + - do: + indices.create: + index: test + body: + mappings: + properties: + date: + type: date + keyword: + type: keyword + long: + type: long + geo_point: + type: geo_point + nested: + type: nested + properties: + nested_long: + type: long + + - do: + indices.create: + index: other + body: + mappings: + properties: + date: + type: date + long: + type: long + nested: + type: nested + properties: + nested_long: + type: long + + - do: + index: + index: test + id: 1 + body: { "keyword": "foo", "long": [10, 20], "geo_point": "37.2343,-115.8067", "nested": [{"nested_long": 10}, {"nested_long": 20}] } + + - do: + index: + index: test + id: 2 + body: { "keyword": ["foo", "bar"], "geo_point": "41.12,-71.34" } + + - do: + index: + index: test + id: 3 + body: { "keyword": "bar", "long": [100, 0], "geo_point": "90.0,0.0", "nested": [{"nested_long": 10}, {"nested_long": 0}] } + + - do: + index: + index: test + id: 4 + body: { "keyword": "bar", "long": [1000, 0], "geo_point": "41.12,-71.34", "nested": [{"nested_long": 1000}, {"nested_long": 20}] } + + - do: + index: + index: test + id: 5 + body: { "date": "2017-10-20T03:08:45" } + + - do: + index: + index: test + id: 6 + body: { "date": "2017-10-21T07:00:00" } + + - do: + index: + index: other + id: 0 + body: { "date": "2017-10-20T03:08:45" } + + - do: + indices.refresh: + index: [test, other] +--- +"Simple Composite aggregation with GeoTile grid": + - skip: + version: " - 7.4.99" + reason: geotile_grid is not supported until 7.5.0 + - do: + search: + rest_total_hits_as_int: true + index: test + body: + aggregations: + test: + composite: + sources: [ + "geo": { + "geotile_grid": { + "field": "geo_point", + "precision": 12 + } + }, + { + "kw": { + "terms": { + "field": "keyword" + } + } + } + ] + + - match: {hits.total: 6} + - length: { aggregations.test.buckets: 4 } + - match: { aggregations.test.buckets.0.key.geo: "12/730/1590" } + - match: { aggregations.test.buckets.0.key.kw: "foo" } + - match: { aggregations.test.buckets.0.doc_count: 1 } + - match: { aggregations.test.buckets.1.key.geo: "12/1236/1533" } + - match: { aggregations.test.buckets.1.key.kw: "bar" } + - match: { aggregations.test.buckets.1.doc_count: 2 } + - match: { aggregations.test.buckets.2.key.geo: "12/1236/1533" } + - match: { aggregations.test.buckets.2.key.kw: "foo" } + - match: { aggregations.test.buckets.2.doc_count: 1 } + - match: { aggregations.test.buckets.3.key.geo: "12/2048/0" } + - match: { aggregations.test.buckets.3.key.kw: "bar" } + - match: { aggregations.test.buckets.3.doc_count: 1 } + +--- +"Simple Composite aggregation with geotile grid add aggregate after": + - skip: + version: " - 7.4.99" + reason: geotile_grid is not supported until 7.5.0 + - do: + search: + index: test + body: + aggregations: + test: + composite: + sources: [ + "geo": { + "geotile_grid": { + "field": "geo_point", + "precision": 12 + } + }, + { + "kw": { + "terms": { + "field": "keyword" + } + } + } + ] + after: { "geo": "12/730/1590", "kw": "foo" } + + - match: { hits.total.value: 6 } + - match: { hits.total.relation: "eq" } + - length: { aggregations.test.buckets: 3 } + - match: { aggregations.test.buckets.0.key.geo: "12/1236/1533" } + - match: { aggregations.test.buckets.0.key.kw: "bar" } + - match: { aggregations.test.buckets.0.doc_count: 2 } + - match: { aggregations.test.buckets.1.key.geo: "12/1236/1533" } + - match: { aggregations.test.buckets.1.key.kw: "foo" } + - match: { aggregations.test.buckets.1.doc_count: 1 } + - match: { aggregations.test.buckets.2.key.geo: "12/2048/0" } + - match: { aggregations.test.buckets.2.key.kw: "bar" } + - match: { aggregations.test.buckets.2.doc_count: 1 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/280_geohash_grid.yml b/modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape/280_geohash_grid.yml similarity index 100% rename from rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/280_geohash_grid.yml rename to modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape/280_geohash_grid.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/290_geotile_grid.yml b/modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape/290_geotile_grid.yml similarity index 100% rename from rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/290_geotile_grid.yml rename to modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape/290_geotile_grid.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml index 2e298441918bc..09278690f5d05 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml @@ -651,92 +651,6 @@ setup: } ] ---- -"Simple Composite aggregation with GeoTile grid": - - skip: - version: " - 7.4.99" - reason: geotile_grid is not supported until 7.5.0 - - do: - search: - rest_total_hits_as_int: true - index: test - body: - aggregations: - test: - composite: - sources: [ - "geo": { - "geotile_grid": { - "field": "geo_point", - "precision": 12 - } - }, - { - "kw": { - "terms": { - "field": "keyword" - } - } - } - ] - - - match: {hits.total: 6} - - length: { aggregations.test.buckets: 4 } - - match: { aggregations.test.buckets.0.key.geo: "12/730/1590" } - - match: { aggregations.test.buckets.0.key.kw: "foo" } - - match: { aggregations.test.buckets.0.doc_count: 1 } - - match: { aggregations.test.buckets.1.key.geo: "12/1236/1533" } - - match: { aggregations.test.buckets.1.key.kw: "bar" } - - match: { aggregations.test.buckets.1.doc_count: 2 } - - match: { aggregations.test.buckets.2.key.geo: "12/1236/1533" } - - match: { aggregations.test.buckets.2.key.kw: "foo" } - - match: { aggregations.test.buckets.2.doc_count: 1 } - - match: { aggregations.test.buckets.3.key.geo: "12/2048/0" } - - match: { aggregations.test.buckets.3.key.kw: "bar" } - - match: { aggregations.test.buckets.3.doc_count: 1 } - ---- -"Simple Composite aggregation with geotile grid add aggregate after": - - skip: - version: " - 7.4.99" - reason: geotile_grid is not supported until 7.5.0 - - do: - search: - index: test - body: - aggregations: - test: - composite: - sources: [ - "geo": { - "geotile_grid": { - "field": "geo_point", - "precision": 12 - } - }, - { - "kw": { - "terms": { - "field": "keyword" - } - } - } - ] - after: { "geo": "12/730/1590", "kw": "foo" } - - - match: { hits.total.value: 6 } - - match: { hits.total.relation: "eq" } - - length: { aggregations.test.buckets: 3 } - - match: { aggregations.test.buckets.0.key.geo: "12/1236/1533" } - - match: { aggregations.test.buckets.0.key.kw: "bar" } - - match: { aggregations.test.buckets.0.doc_count: 2 } - - match: { aggregations.test.buckets.1.key.geo: "12/1236/1533" } - - match: { aggregations.test.buckets.1.key.kw: "foo" } - - match: { aggregations.test.buckets.1.doc_count: 1 } - - match: { aggregations.test.buckets.2.key.geo: "12/2048/0" } - - match: { aggregations.test.buckets.2.key.kw: "bar" } - - match: { aggregations.test.buckets.2.doc_count: 1 } - --- "Mixed ip and unmapped fields": - skip: diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/ShardReduceIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/ShardReduceIT.java index 7352dc7170a21..faa6a54394b00 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/ShardReduceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/ShardReduceIT.java @@ -37,7 +37,6 @@ import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.opensearch.search.aggregations.bucket.filter.Filter; -import org.opensearch.search.aggregations.bucket.geogrid.GeoGrid; import org.opensearch.search.aggregations.bucket.global.Global; import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.opensearch.search.aggregations.bucket.histogram.Histogram; @@ -51,8 +50,6 @@ import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram; import static org.opensearch.search.aggregations.AggregationBuilders.dateRange; import static org.opensearch.search.aggregations.AggregationBuilders.filter; -import static org.opensearch.search.aggregations.AggregationBuilders.geohashGrid; -import static org.opensearch.search.aggregations.AggregationBuilders.geotileGrid; import static org.opensearch.search.aggregations.AggregationBuilders.global; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.ipRange; @@ -338,36 +335,4 @@ public void testDateHistogram() throws Exception { } - public void testGeoHashGrid() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .setQuery(QueryBuilders.matchAllQuery()) - .addAggregation( - geohashGrid("grid").field("location") - .subAggregation(dateHistogram("histo").field("date").dateHistogramInterval(DateHistogramInterval.DAY).minDocCount(0)) - ) - .get(); - - assertSearchResponse(response); - - GeoGrid grid = response.getAggregations().get("grid"); - Histogram histo = grid.getBuckets().iterator().next().getAggregations().get("histo"); - assertThat(histo.getBuckets().size(), equalTo(4)); - } - - public void testGeoTileGrid() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .setQuery(QueryBuilders.matchAllQuery()) - .addAggregation( - geotileGrid("grid").field("location") - .subAggregation(dateHistogram("histo").field("date").dateHistogramInterval(DateHistogramInterval.DAY).minDocCount(0)) - ) - .get(); - - assertSearchResponse(response); - - GeoGrid grid = response.getAggregations().get("grid"); - Histogram histo = grid.getBuckets().iterator().next().getAggregations().get("histo"); - assertThat(histo.getBuckets().size(), equalTo(4)); - } - } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/GeoCentroidIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/GeoCentroidIT.java index 7cd8b3ed39051..ffc31b7cdb7c4 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/GeoCentroidIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/GeoCentroidIT.java @@ -35,15 +35,11 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.common.geo.GeoPoint; import org.opensearch.search.aggregations.InternalAggregation; -import org.opensearch.search.aggregations.bucket.geogrid.GeoGrid; import org.opensearch.search.aggregations.bucket.global.Global; import org.opensearch.test.OpenSearchIntegTestCase; -import java.util.List; - import static org.opensearch.index.query.QueryBuilders.matchAllQuery; import static org.opensearch.search.aggregations.AggregationBuilders.geoCentroid; -import static org.opensearch.search.aggregations.AggregationBuilders.geohashGrid; import static org.opensearch.search.aggregations.AggregationBuilders.global; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.closeTo; @@ -168,33 +164,4 @@ public void testMultiValuedField() throws Exception { assertThat(centroid.lon(), closeTo(multiCentroid.lon(), GEOHASH_TOLERANCE)); assertEquals(2 * numDocs, geoCentroid.count()); } - - public void testSingleValueFieldAsSubAggToGeohashGrid() throws Exception { - SearchResponse response = client().prepareSearch(HIGH_CARD_IDX_NAME) - .addAggregation( - geohashGrid("geoGrid").field(SINGLE_VALUED_FIELD_NAME).subAggregation(geoCentroid(aggName).field(SINGLE_VALUED_FIELD_NAME)) - ) - .get(); - assertSearchResponse(response); - - GeoGrid grid = response.getAggregations().get("geoGrid"); - assertThat(grid, notNullValue()); - assertThat(grid.getName(), equalTo("geoGrid")); - List buckets = grid.getBuckets(); - for (GeoGrid.Bucket cell : buckets) { - String geohash = cell.getKeyAsString(); - GeoPoint expectedCentroid = expectedCentroidsForGeoHash.get(geohash); - GeoCentroid centroidAgg = cell.getAggregations().get(aggName); - assertThat( - "Geohash " + geohash + " has wrong centroid latitude ", - expectedCentroid.lat(), - closeTo(centroidAgg.centroid().lat(), GEOHASH_TOLERANCE) - ); - assertThat( - "Geohash " + geohash + " has wrong centroid longitude", - expectedCentroid.lon(), - closeTo(centroidAgg.centroid().lon(), GEOHASH_TOLERANCE) - ); - } - } } diff --git a/server/src/main/java/org/opensearch/plugins/SearchPlugin.java b/server/src/main/java/org/opensearch/plugins/SearchPlugin.java index a743360e1e90c..af7d4fc2e9fe5 100644 --- a/server/src/main/java/org/opensearch/plugins/SearchPlugin.java +++ b/server/src/main/java/org/opensearch/plugins/SearchPlugin.java @@ -55,6 +55,9 @@ import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.PipelineAggregationBuilder; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregation; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregationParsingFunction; +import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; import org.opensearch.search.aggregations.bucket.terms.SignificantTerms; import org.opensearch.search.aggregations.bucket.terms.heuristic.SignificanceHeuristic; import org.opensearch.search.aggregations.pipeline.MovAvgModel; @@ -172,6 +175,15 @@ default List> getAggregationExtentions() return emptyList(); } + /** + * Allows plugins to register new Aggregation in the {@link CompositeAggregation}. + * + * @return A {@link List} of {@link CompositeAggregationSpec} + */ + default List getCompositeAggregations() { + return emptyList(); + } + /** * The new {@link PipelineAggregator}s added by this plugin. */ @@ -532,6 +544,76 @@ public AggregationSpec setAggregatorRegistrar(Consumer aggregatorRegistrar; + private final Class valueSourceBuilderClass; + @Deprecated + /** This is added for backward compatibility, you don't need to set it, as we use aggregationType instead of + * byte code + */ + private Byte byteCode; + private final CompositeAggregationParsingFunction parsingFunction; + private final String aggregationType; + private final Writeable.Reader> reader; + + /** + * Specification for registering an aggregation in Composite Aggregation + * + * @param aggregatorRegistrar function to register the + * {@link org.opensearch.search.aggregations.support.ValuesSource} to aggregator mappings for Composite + * aggregation + * @param valueSourceBuilderClass ValueSourceBuilder class name which is building the aggregation + * @param byteCode byte code which is used in serialisation and de-serialisation to indentify which + * aggregation builder to use + * @param reader Typically, a reference to a constructor that takes a {@link StreamInput}, which is + * registered with the aggregation + * @param parsingFunction a reference function which will be used to parse the Aggregation input. + * @param aggregationType a {@link String} defined in the AggregationBuilder as type. + */ + public CompositeAggregationSpec( + final Consumer aggregatorRegistrar, + final Class> valueSourceBuilderClass, + final Byte byteCode, + final Writeable.Reader> reader, + final CompositeAggregationParsingFunction parsingFunction, + final String aggregationType + ) { + this.aggregatorRegistrar = aggregatorRegistrar; + this.valueSourceBuilderClass = valueSourceBuilderClass; + this.byteCode = byteCode; + this.parsingFunction = parsingFunction; + this.aggregationType = aggregationType; + this.reader = reader; + } + + public Consumer getAggregatorRegistrar() { + return aggregatorRegistrar; + } + + public Class getValueSourceBuilderClass() { + return valueSourceBuilderClass; + } + + public Byte getByteCode() { + return byteCode; + } + + public CompositeAggregationParsingFunction getParsingFunction() { + return parsingFunction; + } + + public String getAggregationType() { + return aggregationType; + } + + public Writeable.Reader> getReader() { + return reader; + } + } + /** * Specification for a {@link PipelineAggregator}. */ diff --git a/server/src/main/java/org/opensearch/search/DocValueFormat.java b/server/src/main/java/org/opensearch/search/DocValueFormat.java index 7e7e4f83334f5..84c46e400543a 100644 --- a/server/src/main/java/org/opensearch/search/DocValueFormat.java +++ b/server/src/main/java/org/opensearch/search/DocValueFormat.java @@ -47,7 +47,7 @@ import org.opensearch.common.time.DateUtils; import org.opensearch.geometry.utils.Geohash; import org.opensearch.index.mapper.DateFieldMapper; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import java.io.IOException; import java.math.BigInteger; diff --git a/server/src/main/java/org/opensearch/search/SearchModule.java b/server/src/main/java/org/opensearch/search/SearchModule.java index 80e025a3651a8..0149f9a025bcd 100644 --- a/server/src/main/java/org/opensearch/search/SearchModule.java +++ b/server/src/main/java/org/opensearch/search/SearchModule.java @@ -126,10 +126,6 @@ import org.opensearch.search.aggregations.bucket.filter.FiltersAggregationBuilder; import org.opensearch.search.aggregations.bucket.filter.InternalFilter; import org.opensearch.search.aggregations.bucket.filter.InternalFilters; -import org.opensearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.InternalGeoHashGrid; -import org.opensearch.search.aggregations.bucket.geogrid.InternalGeoTileGrid; import org.opensearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.opensearch.search.aggregations.bucket.global.InternalGlobal; import org.opensearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; @@ -628,22 +624,6 @@ private ValuesSourceRegistry registerAggregations(List plugins) { ).addResultReader(InternalGeoDistance::new).setAggregatorRegistrar(GeoDistanceAggregationBuilder::registerAggregators), builder ); - registerAggregation( - new AggregationSpec( - GeoHashGridAggregationBuilder.NAME, - GeoHashGridAggregationBuilder::new, - GeoHashGridAggregationBuilder.PARSER - ).addResultReader(InternalGeoHashGrid::new).setAggregatorRegistrar(GeoHashGridAggregationBuilder::registerAggregators), - builder - ); - registerAggregation( - new AggregationSpec( - GeoTileGridAggregationBuilder.NAME, - GeoTileGridAggregationBuilder::new, - GeoTileGridAggregationBuilder.PARSER - ).addResultReader(InternalGeoTileGrid::new).setAggregatorRegistrar(GeoTileGridAggregationBuilder::registerAggregators), - builder - ); registerAggregation( new AggregationSpec(NestedAggregationBuilder.NAME, NestedAggregationBuilder::new, NestedAggregationBuilder::parse) .addResultReader(InternalNested::new), @@ -681,7 +661,7 @@ private ValuesSourceRegistry registerAggregations(List plugins) { registerAggregation( new AggregationSpec(CompositeAggregationBuilder.NAME, CompositeAggregationBuilder::new, CompositeAggregationBuilder.PARSER) .addResultReader(InternalComposite::new) - .setAggregatorRegistrar(CompositeAggregationBuilder::registerAggregators), + .setAggregatorRegistrar(reg -> CompositeAggregationBuilder.registerAggregators(reg, plugins)), builder ); registerAggregation( diff --git a/server/src/main/java/org/opensearch/search/aggregations/AggregationBuilders.java b/server/src/main/java/org/opensearch/search/aggregations/AggregationBuilders.java index 382455093309d..9886e423bbc76 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/AggregationBuilders.java +++ b/server/src/main/java/org/opensearch/search/aggregations/AggregationBuilders.java @@ -43,10 +43,6 @@ import org.opensearch.search.aggregations.bucket.filter.Filters; import org.opensearch.search.aggregations.bucket.filter.FiltersAggregationBuilder; import org.opensearch.search.aggregations.bucket.filter.FiltersAggregator.KeyedFilter; -import org.opensearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.InternalGeoHashGrid; -import org.opensearch.search.aggregations.bucket.geogrid.InternalGeoTileGrid; import org.opensearch.search.aggregations.bucket.global.Global; import org.opensearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.opensearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; @@ -261,20 +257,6 @@ public static HistogramAggregationBuilder histogram(String name) { return new HistogramAggregationBuilder(name); } - /** - * Create a new {@link InternalGeoHashGrid} aggregation with the given name. - */ - public static GeoHashGridAggregationBuilder geohashGrid(String name) { - return new GeoHashGridAggregationBuilder(name); - } - - /** - * Create a new {@link InternalGeoTileGrid} aggregation with the given name. - */ - public static GeoTileGridAggregationBuilder geotileGrid(String name) { - return new GeoTileGridAggregationBuilder(name); - } - /** * Create a new {@link SignificantTerms} aggregation with the given name. */ diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileUtils.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/GeoTileUtils.java similarity index 97% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileUtils.java rename to server/src/main/java/org/opensearch/search/aggregations/bucket/GeoTileUtils.java index 5498b2b1a7109..6cd1823622f01 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileUtils.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/GeoTileUtils.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.search.aggregations.bucket; import org.apache.lucene.geo.GeoEncodingUtils; import org.apache.lucene.util.SloppyMath; @@ -104,7 +104,7 @@ private GeoTileUtils() {} * @param parser {@link XContentParser} to parse the value from * @return int representing precision */ - static int parsePrecision(XContentParser parser) throws IOException, OpenSearchParseException { + public static int parsePrecision(XContentParser parser) throws IOException, OpenSearchParseException { final Object node = parser.currentToken().equals(XContentParser.Token.VALUE_NUMBER) ? Integer.valueOf(parser.intValue()) : parser.text(); @@ -252,7 +252,7 @@ public static String stringEncode(long hash) { /** * Decode long hash as a GeoPoint (center of the tile) */ - static GeoPoint hashToGeoPoint(long hash) { + public static GeoPoint hashToGeoPoint(long hash) { int[] res = parseHash(hash); return zxyToGeoPoint(res[0], res[1], res[2]); } @@ -260,7 +260,7 @@ static GeoPoint hashToGeoPoint(long hash) { /** * Decode a string bucket key in "zoom/x/y" format to a GeoPoint (center of the tile) */ - static GeoPoint keyToGeoPoint(String hashAsString) { + public static GeoPoint keyToGeoPoint(String hashAsString) { int[] hashAsInts = parseHash(hashAsString); return zxyToGeoPoint(hashAsInts[0], hashAsInts[1], hashAsInts[2]); } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java index 8b07df3f689bf..093c2ad42722e 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java @@ -35,9 +35,11 @@ import org.opensearch.common.ParseField; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.common.io.stream.Writeable; import org.opensearch.common.xcontent.ConstructingObjectParser; import org.opensearch.common.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; +import org.opensearch.plugins.SearchPlugin; import org.opensearch.search.aggregations.AbstractAggregationBuilder; import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.AggregatorFactories; @@ -47,11 +49,14 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.Consumer; +import java.util.function.Function; import static org.opensearch.common.xcontent.ConstructingObjectParser.constructorArg; @@ -82,14 +87,55 @@ public class CompositeAggregationBuilder extends AbstractAggregationBuilder p.map(), AFTER_FIELD_NAME); } - public static void registerAggregators(ValuesSourceRegistry.Builder builder) { + static final Map, Byte> BUILDER_CLASS_TO_BYTE_CODE = new HashMap<>(); + static final Map BUILDER_TYPE_TO_PARSER = new HashMap<>(); + static final Map>> BYTE_CODE_TO_COMPOSITE_VALUE_SOURCE_READER = + new HashMap<>(); + static final Map< + String, + Writeable.Reader>> AGGREGATION_TYPE_TO_COMPOSITE_VALUE_SOURCE_READER = new HashMap<>(); + static final Map, String> BUILDER_CLASS_TO_AGGREGATION_TYPE = new HashMap<>(); + + public static void registerAggregators(ValuesSourceRegistry.Builder builder, final List plugins) { DateHistogramValuesSourceBuilder.register(builder); HistogramValuesSourceBuilder.register(builder); - GeoTileGridValuesSourceBuilder.register(builder); TermsValuesSourceBuilder.register(builder); + // Register All other aggregations that wants to be part of Composite Aggregation which are provided in + // Plugins along with their parsers and serialisation codes + registerCompositeAggregatorsPlugins(plugins, SearchPlugin::getCompositeAggregations, (compositeAggregationSpec) -> { + compositeAggregationSpec.getAggregatorRegistrar().accept(builder); + BUILDER_TYPE_TO_PARSER.put(compositeAggregationSpec.getAggregationType(), compositeAggregationSpec.getParsingFunction()); + // This is added for backward compatibility, so that we can move away from byte code in the serialisation + if (compositeAggregationSpec.getByteCode() != null) { + BYTE_CODE_TO_COMPOSITE_VALUE_SOURCE_READER.put( + (int) compositeAggregationSpec.getByteCode(), + compositeAggregationSpec.getReader() + ); + BUILDER_CLASS_TO_BYTE_CODE.put( + compositeAggregationSpec.getValueSourceBuilderClass(), + compositeAggregationSpec.getByteCode() + ); + } + AGGREGATION_TYPE_TO_COMPOSITE_VALUE_SOURCE_READER.put( + compositeAggregationSpec.getAggregationType(), + compositeAggregationSpec.getReader() + ); + BUILDER_CLASS_TO_AGGREGATION_TYPE.put( + compositeAggregationSpec.getValueSourceBuilderClass(), + compositeAggregationSpec.getAggregationType() + ); + }); builder.registerUsage(NAME); } + private static void registerCompositeAggregatorsPlugins( + final List plugins, + final Function> producer, + final Consumer consumer + ) { + plugins.forEach(searchPlugin -> producer.apply(searchPlugin).forEach(consumer)); + } + private List> sources; private Map after; private int size = 10; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationParsingFunction.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationParsingFunction.java new file mode 100644 index 0000000000000..344563ad20309 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationParsingFunction.java @@ -0,0 +1,22 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.aggregations.bucket.composite; + +import org.opensearch.common.xcontent.XContentParser; + +import java.io.IOException; + +/** + * A functional interface which encapsulates the parsing function to be called for the aggregation which is + * also registered as CompositeAggregation. + */ +@FunctionalInterface +public interface CompositeAggregationParsingFunction { + CompositeValuesSourceBuilder parse(final String name, final XContentParser parser) throws IOException; +} diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java index 7764d367a0cec..26015ae04cf76 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java @@ -69,11 +69,11 @@ public abstract class CompositeValuesSourceBuilder createValuesSource( * @param missingBucket If true an explicit null bucket will represent documents with missing values. * @param hasScript true if the source contains a script that can change the value. */ - CompositeValuesSourceConfig( + public CompositeValuesSourceConfig( String name, @Nullable MappedFieldType fieldType, ValuesSource vs, @@ -113,21 +113,21 @@ SingleDimensionValuesSource createValuesSource( /** * Returns the name associated with this configuration. */ - String name() { + protected String name() { return name; } /** * Returns the {@link MappedFieldType} for this config. */ - MappedFieldType fieldType() { + public MappedFieldType fieldType() { return fieldType; } /** * Returns the {@link ValuesSource} for this configuration. */ - ValuesSource valuesSource() { + public ValuesSource valuesSource() { return vs; } @@ -135,35 +135,35 @@ ValuesSource valuesSource() { * The {@link DocValueFormat} to use for formatting the keys. * {@link DocValueFormat#RAW} means no formatting. */ - DocValueFormat format() { + public DocValueFormat format() { return format; } /** * If true, an explicit `null bucket represents documents with missing values. */ - boolean missingBucket() { + public boolean missingBucket() { return missingBucket; } /** * Return the {@link MissingOrder} for the config. */ - MissingOrder missingOrder() { + public MissingOrder missingOrder() { return missingOrder; } /** * Returns true if the source contains a script that can change the value. */ - boolean hasScript() { + protected boolean hasScript() { return hasScript; } /** * The sort order for the values source (e.g. -1 for descending and 1 for ascending). */ - int reverseMul() { + public int reverseMul() { assert reverseMul == -1 || reverseMul == 1; return reverseMul; } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java index 60d7f277f7650..d8526e684f391 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java @@ -49,6 +49,11 @@ import java.io.IOException; import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder.AGGREGATION_TYPE_TO_COMPOSITE_VALUE_SOURCE_READER; +import static org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder.BUILDER_CLASS_TO_AGGREGATION_TYPE; +import static org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder.BUILDER_CLASS_TO_BYTE_CODE; +import static org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder.BUILDER_TYPE_TO_PARSER; +import static org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder.BYTE_CODE_TO_COMPOSITE_VALUE_SOURCE_READER; /** * Helper class for obtaining values source parsers for different aggs @@ -57,7 +62,11 @@ */ public class CompositeValuesSourceParserHelper { - static , T> void declareValuesSourceFields(AbstractObjectParser objectParser) { + private static final int AGGREGATION_TYPE_REFERENCE = Byte.MAX_VALUE; + + public static , T> void declareValuesSourceFields( + AbstractObjectParser objectParser + ) { objectParser.declareField(VB::field, XContentParser::text, new ParseField("field"), ObjectParser.ValueType.STRING); objectParser.declareBoolean(VB::missingBucket, new ParseField("missing_bucket")); objectParser.declareString(VB::missingOrder, new ParseField(MissingOrder.NAME)); @@ -78,28 +87,45 @@ static , T> void declareValuesSource } public static void writeTo(CompositeValuesSourceBuilder builder, StreamOutput out) throws IOException { - final byte code; + int code = Byte.MIN_VALUE; + String aggregationType = null; if (builder.getClass() == TermsValuesSourceBuilder.class) { code = 0; } else if (builder.getClass() == DateHistogramValuesSourceBuilder.class) { code = 1; } else if (builder.getClass() == HistogramValuesSourceBuilder.class) { code = 2; - } else if (builder.getClass() == GeoTileGridValuesSourceBuilder.class) { - if (out.getVersion().before(LegacyESVersion.V_7_5_0)) { - throw new IOException( - "Attempting to serialize [" - + builder.getClass().getSimpleName() - + "] to a node with unsupported version [" - + out.getVersion() - + "]" - ); - } - code = 3; } else { - throw new IOException("invalid builder type: " + builder.getClass().getSimpleName()); + if (!BUILDER_CLASS_TO_BYTE_CODE.containsKey(builder.getClass()) + && !BUILDER_CLASS_TO_AGGREGATION_TYPE.containsKey(builder.getClass())) { + throw new IOException("invalid builder type: " + builder.getClass().getSimpleName()); + } + aggregationType = BUILDER_CLASS_TO_AGGREGATION_TYPE.get(builder.getClass()); + if (BUILDER_CLASS_TO_BYTE_CODE.containsKey(builder.getClass())) { + code = BUILDER_CLASS_TO_BYTE_CODE.get(builder.getClass()); + if (code == 3 && out.getVersion().before(LegacyESVersion.V_7_5_0)) { + throw new IOException( + "Attempting to serialize [" + + builder.getClass().getSimpleName() + + "] to a node with unsupported version [" + + out.getVersion() + + "]" + ); + } + } + } + + if (code != Byte.MIN_VALUE) { + out.writeByte((byte) code); + } else if (!BUILDER_CLASS_TO_BYTE_CODE.containsKey(builder.getClass())) { + /* + * This is added for backward compatibility when 1 data node is using the new code which is using the + * aggregation type and another is using the only byte code in the serialisation. + */ + out.writeByte((byte) AGGREGATION_TYPE_REFERENCE); + assert aggregationType != null; + out.writeString(aggregationType); } - out.writeByte(code); builder.writeTo(out); } @@ -112,10 +138,17 @@ public static CompositeValuesSourceBuilder readFrom(StreamInput in) throws IO return new DateHistogramValuesSourceBuilder(in); case 2: return new HistogramValuesSourceBuilder(in); - case 3: - return new GeoTileGridValuesSourceBuilder(in); + case AGGREGATION_TYPE_REFERENCE: + final String aggregationType = in.readString(); + if (!AGGREGATION_TYPE_TO_COMPOSITE_VALUE_SOURCE_READER.containsKey(aggregationType)) { + throw new IOException("Invalid aggregation type " + aggregationType); + } + return (CompositeValuesSourceBuilder) AGGREGATION_TYPE_TO_COMPOSITE_VALUE_SOURCE_READER.get(aggregationType).read(in); default: - throw new IOException("Invalid code " + code); + if (!BYTE_CODE_TO_COMPOSITE_VALUE_SOURCE_READER.containsKey(code)) { + throw new IOException("Invalid code " + code); + } + return (CompositeValuesSourceBuilder) BYTE_CODE_TO_COMPOSITE_VALUE_SOURCE_READER.get(code).read(in); } } @@ -143,11 +176,11 @@ public static CompositeValuesSourceBuilder fromXContent(XContentParser parser case HistogramValuesSourceBuilder.TYPE: builder = HistogramValuesSourceBuilder.parse(name, parser); break; - case GeoTileGridValuesSourceBuilder.TYPE: - builder = GeoTileGridValuesSourceBuilder.parse(name, parser); - break; default: - throw new ParsingException(parser.getTokenLocation(), "invalid source type: " + type); + if (!BUILDER_TYPE_TO_PARSER.containsKey(type)) { + throw new ParsingException(parser.getTokenLocation(), "invalid source type: " + type); + } + builder = BUILDER_TYPE_TO_PARSER.get(type).parse(name, parser); } parser.nextToken(); parser.nextToken(); @@ -163,4 +196,5 @@ public static XContentBuilder toXContent(CompositeValuesSourceBuilder source, builder.endObject(); return builder; } + } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/LongValuesSource.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/LongValuesSource.java index a7ed50507288d..ec6410c2a9377 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/LongValuesSource.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/LongValuesSource.java @@ -66,7 +66,7 @@ * * @opensearch.internal */ -class LongValuesSource extends SingleDimensionValuesSource { +public class LongValuesSource extends SingleDimensionValuesSource { private final BigArrays bigArrays; private final CheckedFunction docValuesFunc; private final LongUnaryOperator rounding; @@ -76,7 +76,7 @@ class LongValuesSource extends SingleDimensionValuesSource { private long currentValue; private boolean missingCurrentValue; - LongValuesSource( + public LongValuesSource( BigArrays bigArrays, MappedFieldType fieldType, CheckedFunction docValuesFunc, @@ -165,7 +165,7 @@ private int compareValues(long v1, long v2) { } @Override - void setAfter(Comparable value) { + protected void setAfter(Comparable value) { if (missingBucket && value == null) { afterValue = null; } else { diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/SingleDimensionValuesSource.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/SingleDimensionValuesSource.java index 747a7017ec872..fe0801d6d230e 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/SingleDimensionValuesSource.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/SingleDimensionValuesSource.java @@ -53,7 +53,7 @@ * * @opensearch.internal */ -abstract class SingleDimensionValuesSource> implements Releasable { +public abstract class SingleDimensionValuesSource> implements Releasable { protected final BigArrays bigArrays; protected final DocValueFormat format; @Nullable diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/AggregationInspectionHelper.java b/server/src/main/java/org/opensearch/search/aggregations/support/AggregationInspectionHelper.java index f36c4620d5b33..b4da1d10b4b68 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/AggregationInspectionHelper.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/AggregationInspectionHelper.java @@ -35,7 +35,6 @@ import org.opensearch.search.aggregations.bucket.composite.InternalComposite; import org.opensearch.search.aggregations.bucket.filter.InternalFilter; import org.opensearch.search.aggregations.bucket.filter.InternalFilters; -import org.opensearch.search.aggregations.bucket.geogrid.InternalGeoGrid; import org.opensearch.search.aggregations.bucket.global.InternalGlobal; import org.opensearch.search.aggregations.bucket.histogram.InternalVariableWidthHistogram; import org.opensearch.search.aggregations.bucket.histogram.InternalAutoDateHistogram; @@ -119,10 +118,6 @@ public static boolean hasValue(InternalFilter agg) { return agg.getDocCount() > 0; } - public static boolean hasValue(InternalGeoGrid agg) { - return agg.getBuckets().stream().anyMatch(bucket -> bucket.getDocCount() > 0); - } - public static boolean hasValue(InternalGlobal agg) { return agg.getDocCount() > 0; } diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/package-info.java b/server/src/main/java/org/opensearch/search/aggregations/support/package-info.java index e16e8c91b3fd0..dd2c16f1daa0e 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/package-info.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/package-info.java @@ -43,7 +43,7 @@ * output). A class hierarchy defines the type of values returned by the source. The top level sub-classes define type-specific behavior, * such as {@link org.opensearch.search.aggregations.support.ValuesSource.Numeric#isFloatingPoint()}. Second level subclasses are * then specialized based on where they read values from, e.g. script or field cases. There are also adapter classes like - * {@link org.opensearch.search.aggregations.bucket.geogrid.CellIdSource} which do run-time conversion from one type to another, often + * org.opensearch.search.aggregations.bucket.geogrid.CellIdSource which do run-time conversion from one type to another, often * dependent on a user specified parameter (precision in that case). *

* diff --git a/server/src/test/java/org/opensearch/search/DocValueFormatTests.java b/server/src/test/java/org/opensearch/search/DocValueFormatTests.java index 36a6eb3ae87b0..bd0fbfe69960c 100644 --- a/server/src/test/java/org/opensearch/search/DocValueFormatTests.java +++ b/server/src/test/java/org/opensearch/search/DocValueFormatTests.java @@ -48,7 +48,7 @@ import java.util.ArrayList; import java.util.List; -import static org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils.longEncode; +import static org.opensearch.search.aggregations.bucket.GeoTileUtils.longEncode; public class DocValueFormatTests extends OpenSearchTestCase { diff --git a/server/src/test/java/org/opensearch/search/aggregations/AggregationsTests.java b/server/src/test/java/org/opensearch/search/aggregations/AggregationsTests.java index 111ce23f8a0cb..94fb6cded637d 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/AggregationsTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/AggregationsTests.java @@ -48,8 +48,6 @@ import org.opensearch.search.aggregations.bucket.composite.InternalCompositeTests; import org.opensearch.search.aggregations.bucket.filter.InternalFilterTests; import org.opensearch.search.aggregations.bucket.filter.InternalFiltersTests; -import org.opensearch.search.aggregations.bucket.geogrid.GeoHashGridTests; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileGridTests; import org.opensearch.search.aggregations.bucket.global.InternalGlobalTests; import org.opensearch.search.aggregations.bucket.histogram.InternalAutoDateHistogramTests; import org.opensearch.search.aggregations.bucket.histogram.InternalDateHistogramTests; @@ -157,8 +155,6 @@ private static List> getAggsTests() { aggsTests.add(new InternalGlobalTests()); aggsTests.add(new InternalFilterTests()); aggsTests.add(new InternalSamplerTests()); - aggsTests.add(new GeoHashGridTests()); - aggsTests.add(new GeoTileGridTests()); aggsTests.add(new InternalRangeTests()); aggsTests.add(new InternalDateRangeTests()); aggsTests.add(new InternalGeoDistanceTests()); diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilderTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilderTests.java index c4a87f3993bb4..9290183ec7312 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilderTests.java @@ -32,10 +32,8 @@ package org.opensearch.search.aggregations.bucket.composite; -import org.opensearch.common.geo.GeoBoundingBoxTests; import org.opensearch.script.Script; import org.opensearch.search.aggregations.BaseAggregationTestCase; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.opensearch.search.aggregations.bucket.missing.MissingOrder; import org.opensearch.search.sort.SortOrder; @@ -74,17 +72,6 @@ private DateHistogramValuesSourceBuilder randomDateHistogramSourceBuilder() { return histo; } - private GeoTileGridValuesSourceBuilder randomGeoTileGridValuesSourceBuilder() { - GeoTileGridValuesSourceBuilder geoTile = new GeoTileGridValuesSourceBuilder(randomAlphaOfLengthBetween(5, 10)); - if (randomBoolean()) { - geoTile.precision(randomIntBetween(0, GeoTileUtils.MAX_ZOOM)); - } - if (randomBoolean()) { - geoTile.geoBoundingBox(GeoBoundingBoxTests.randomBBox()); - } - return geoTile; - } - private TermsValuesSourceBuilder randomTermsSourceBuilder() { TermsValuesSourceBuilder terms = new TermsValuesSourceBuilder(randomAlphaOfLengthBetween(5, 10)); if (randomBoolean()) { @@ -118,11 +105,9 @@ private HistogramValuesSourceBuilder randomHistogramSourceBuilder() { @Override protected CompositeAggregationBuilder createTestAggregatorBuilder() { int numSources = randomIntBetween(1, 10); - numSources = 1; List> sources = new ArrayList<>(); for (int i = 0; i < numSources; i++) { - int type = randomIntBetween(0, 3); - type = 3; + int type = randomIntBetween(0, 2); switch (type) { case 0: sources.add(randomTermsSourceBuilder()); @@ -133,9 +118,6 @@ protected CompositeAggregationBuilder createTestAggregatorBuilder() { case 2: sources.add(randomHistogramSourceBuilder()); break; - case 3: - sources.add(randomGeoTileGridValuesSourceBuilder()); - break; default: throw new AssertionError("wrong branch"); } diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java index 88b2323b8adfc..25003e0b84567 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java @@ -32,68 +32,24 @@ package org.opensearch.search.aggregations.bucket.composite; -import org.apache.lucene.tests.analysis.MockAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.DoublePoint; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.InetAddressPoint; -import org.apache.lucene.document.IntPoint; -import org.apache.lucene.document.LatLonPoint; import org.apache.lucene.document.LongPoint; -import org.apache.lucene.document.SortedNumericDocValuesField; -import org.apache.lucene.document.SortedSetDocValuesField; -import org.apache.lucene.document.StringField; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.search.DocValuesFieldExistsQuery; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.SortedNumericSortField; -import org.apache.lucene.search.SortedSetSortField; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.NumericUtils; -import org.apache.lucene.tests.util.TestUtil; import org.opensearch.OpenSearchParseException; -import org.opensearch.common.geo.GeoPoint; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.text.Text; -import org.opensearch.common.time.DateFormatter; -import org.opensearch.common.time.DateFormatters; -import org.opensearch.index.Index; -import org.opensearch.index.IndexSettings; -import org.opensearch.index.mapper.DateFieldMapper; -import org.opensearch.index.mapper.DocumentMapper; -import org.opensearch.index.mapper.GeoPointFieldMapper; -import org.opensearch.index.mapper.IpFieldMapper; -import org.opensearch.index.mapper.KeywordFieldMapper; -import org.opensearch.index.mapper.MappedFieldType; -import org.opensearch.index.mapper.MapperService; -import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.search.aggregations.Aggregator; -import org.opensearch.search.aggregations.AggregatorTestCase; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.opensearch.search.aggregations.bucket.missing.MissingOrder; import org.opensearch.search.aggregations.bucket.terms.StringTerms; import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.opensearch.search.aggregations.composite.BaseCompositeAggregatorTestCase; import org.opensearch.search.aggregations.metrics.InternalMax; import org.opensearch.search.aggregations.metrics.MaxAggregationBuilder; import org.opensearch.search.aggregations.metrics.TopHits; import org.opensearch.search.aggregations.metrics.TopHitsAggregationBuilder; import org.opensearch.search.aggregations.support.ValueType; import org.opensearch.search.sort.SortOrder; -import org.opensearch.test.IndexSettingsModule; -import org.junit.After; -import org.junit.Before; import java.io.IOException; import java.net.InetAddress; @@ -109,51 +65,14 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; -import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class CompositeAggregatorTests extends AggregatorTestCase { - private static MappedFieldType[] FIELD_TYPES; - - @Override - @Before - public void setUp() throws Exception { - super.setUp(); - FIELD_TYPES = new MappedFieldType[8]; - FIELD_TYPES[0] = new KeywordFieldMapper.KeywordFieldType("keyword"); - FIELD_TYPES[1] = new NumberFieldMapper.NumberFieldType("long", NumberFieldMapper.NumberType.LONG); - FIELD_TYPES[2] = new NumberFieldMapper.NumberFieldType("double", NumberFieldMapper.NumberType.DOUBLE); - FIELD_TYPES[3] = new DateFieldMapper.DateFieldType("date", DateFormatter.forPattern("yyyy-MM-dd||epoch_millis")); - FIELD_TYPES[4] = new NumberFieldMapper.NumberFieldType("price", NumberFieldMapper.NumberType.INTEGER); - FIELD_TYPES[5] = new KeywordFieldMapper.KeywordFieldType("terms"); - FIELD_TYPES[6] = new IpFieldMapper.IpFieldType("ip"); - FIELD_TYPES[7] = new GeoPointFieldMapper.GeoPointFieldType("geo_point"); - } - - @Override - @After - public void tearDown() throws Exception { - super.tearDown(); - FIELD_TYPES = null; - } - @Override - protected MapperService mapperServiceMock() { - MapperService mapperService = mock(MapperService.class); - DocumentMapper mapper = mock(DocumentMapper.class); - when(mapper.typeText()).thenReturn(new Text("_doc")); - when(mapper.type()).thenReturn("_doc"); - when(mapperService.documentMapper()).thenReturn(mapper); - return mapperService; - } +public class CompositeAggregatorTests extends BaseCompositeAggregatorTestCase { public void testUnmappedFieldWithTerms() throws Exception { final List>> dataset = new ArrayList<>(); @@ -234,80 +153,6 @@ public void testUnmappedFieldWithTerms() throws Exception { ); } - public void testUnmappedFieldWithGeopoint() throws Exception { - final List>> dataset = new ArrayList<>(); - final String mappedFieldName = "geo_point"; - dataset.addAll( - Arrays.asList( - createDocument(mappedFieldName, new GeoPoint(48.934059, 41.610741)), - createDocument(mappedFieldName, new GeoPoint(-23.065941, 113.610741)), - createDocument(mappedFieldName, new GeoPoint(90.0, 0.0)), - createDocument(mappedFieldName, new GeoPoint(37.2343, -115.8067)), - createDocument(mappedFieldName, new GeoPoint(90.0, 0.0)) - ) - ); - - // just unmapped = no results - testSearchCase( - Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery(mappedFieldName)), - dataset, - () -> new CompositeAggregationBuilder("name", Arrays.asList(new GeoTileGridValuesSourceBuilder("unmapped").field("unmapped"))), - (result) -> assertEquals(0, result.getBuckets().size()) - ); - - // unmapped missing bucket = one result - testSearchCase( - Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery(mappedFieldName)), - dataset, - () -> new CompositeAggregationBuilder( - "name", - Arrays.asList(new GeoTileGridValuesSourceBuilder("unmapped").field("unmapped").missingBucket(true)) - ), - (result) -> { - assertEquals(1, result.getBuckets().size()); - assertEquals("{unmapped=null}", result.afterKey().toString()); - assertEquals("{unmapped=null}", result.getBuckets().get(0).getKeyAsString()); - assertEquals(5L, result.getBuckets().get(0).getDocCount()); - } - ); - - // field + unmapped, no missing bucket = no results - testSearchCase( - Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery(mappedFieldName)), - dataset, - () -> new CompositeAggregationBuilder( - "name", - Arrays.asList( - new GeoTileGridValuesSourceBuilder(mappedFieldName).field(mappedFieldName), - new GeoTileGridValuesSourceBuilder("unmapped").field("unmapped") - ) - ), - (result) -> assertEquals(0, result.getBuckets().size()) - ); - - // field + unmapped with missing bucket = multiple results - testSearchCase( - Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery(mappedFieldName)), - dataset, - () -> new CompositeAggregationBuilder( - "name", - Arrays.asList( - new GeoTileGridValuesSourceBuilder(mappedFieldName).field(mappedFieldName), - new GeoTileGridValuesSourceBuilder("unmapped").field("unmapped").missingBucket(true) - ) - ), - (result) -> { - assertEquals(2, result.getBuckets().size()); - assertEquals("{geo_point=7/64/56, unmapped=null}", result.afterKey().toString()); - assertEquals("{geo_point=7/32/56, unmapped=null}", result.getBuckets().get(0).getKeyAsString()); - assertEquals(2L, result.getBuckets().get(0).getDocCount()); - assertEquals("{geo_point=7/64/56, unmapped=null}", result.getBuckets().get(1).getKeyAsString()); - assertEquals(3L, result.getBuckets().get(1).getDocCount()); - } - ); - - } - public void testUnmappedFieldWithHistogram() throws Exception { final List>> dataset = new ArrayList<>(); final String mappedFieldName = "price"; @@ -2483,42 +2328,6 @@ public void testWithIP() throws Exception { }); } - public void testWithGeoPoint() throws Exception { - final List>> dataset = new ArrayList<>(); - dataset.addAll( - Arrays.asList( - createDocument("geo_point", new GeoPoint(48.934059, 41.610741)), - createDocument("geo_point", new GeoPoint(-23.065941, 113.610741)), - createDocument("geo_point", new GeoPoint(90.0, 0.0)), - createDocument("geo_point", new GeoPoint(37.2343, -115.8067)), - createDocument("geo_point", new GeoPoint(90.0, 0.0)) - ) - ); - testSearchCase(Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery("geo_point")), dataset, () -> { - GeoTileGridValuesSourceBuilder geoTile = new GeoTileGridValuesSourceBuilder("geo_point").field("geo_point"); - return new CompositeAggregationBuilder("name", Collections.singletonList(geoTile)); - }, (result) -> { - assertEquals(2, result.getBuckets().size()); - assertEquals("{geo_point=7/64/56}", result.afterKey().toString()); - assertEquals("{geo_point=7/32/56}", result.getBuckets().get(0).getKeyAsString()); - assertEquals(2L, result.getBuckets().get(0).getDocCount()); - assertEquals("{geo_point=7/64/56}", result.getBuckets().get(1).getKeyAsString()); - assertEquals(3L, result.getBuckets().get(1).getDocCount()); - }); - - testSearchCase(Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery("geo_point")), dataset, () -> { - GeoTileGridValuesSourceBuilder geoTile = new GeoTileGridValuesSourceBuilder("geo_point").field("geo_point"); - return new CompositeAggregationBuilder("name", Collections.singletonList(geoTile)).aggregateAfter( - Collections.singletonMap("geo_point", "7/32/56") - ); - }, (result) -> { - assertEquals(1, result.getBuckets().size()); - assertEquals("{geo_point=7/64/56}", result.afterKey().toString()); - assertEquals("{geo_point=7/64/56}", result.getBuckets().get(0).getKeyAsString()); - assertEquals(3L, result.getBuckets().get(0).getDocCount()); - }); - } - public void testEarlyTermination() throws Exception { final List>> dataset = new ArrayList<>(); dataset.addAll( @@ -2648,193 +2457,4 @@ public void testIndexSortWithDuplicate() throws Exception { ); } } - - private void testSearchCase( - List queries, - List>> dataset, - Supplier create, - Consumer verify - ) throws IOException { - for (Query query : queries) { - executeTestCase(false, false, query, dataset, create, verify); - executeTestCase(false, true, query, dataset, create, verify); - } - } - - private void executeTestCase( - boolean forceMerge, - boolean useIndexSort, - Query query, - List>> dataset, - Supplier create, - Consumer verify - ) throws IOException { - Map types = Arrays.stream(FIELD_TYPES) - .collect(Collectors.toMap(MappedFieldType::name, Function.identity())); - CompositeAggregationBuilder aggregationBuilder = create.get(); - Sort indexSort = useIndexSort ? buildIndexSort(aggregationBuilder.sources(), types) : null; - IndexSettings indexSettings = createIndexSettings(indexSort); - try (Directory directory = newDirectory()) { - IndexWriterConfig config = newIndexWriterConfig(random(), new MockAnalyzer(random())); - if (indexSort != null) { - config.setIndexSort(indexSort); - config.setCodec(TestUtil.getDefaultCodec()); - } - try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory, config)) { - Document document = new Document(); - int id = 0; - for (Map> fields : dataset) { - document.clear(); - addToDocument(id, document, fields); - indexWriter.addDocument(document); - id++; - } - if (forceMerge || rarely()) { - // forceMerge randomly or if the collector-per-leaf testing stuff would break the tests. - indexWriter.forceMerge(1); - } else { - if (dataset.size() > 0) { - int numDeletes = randomIntBetween(1, 25); - for (int i = 0; i < numDeletes; i++) { - id = randomIntBetween(0, dataset.size() - 1); - indexWriter.deleteDocuments(new Term("id", Integer.toString(id))); - document.clear(); - addToDocument(id, document, dataset.get(id)); - indexWriter.addDocument(document); - } - } - - } - } - try (IndexReader indexReader = DirectoryReader.open(directory)) { - IndexSearcher indexSearcher = new IndexSearcher(indexReader); - InternalComposite composite = searchAndReduce(indexSettings, indexSearcher, query, aggregationBuilder, FIELD_TYPES); - verify.accept(composite); - } - } - } - - private static IndexSettings createIndexSettings(Sort sort) { - Settings.Builder builder = Settings.builder(); - if (sort != null) { - String[] fields = Arrays.stream(sort.getSort()).map(SortField::getField).toArray(String[]::new); - String[] orders = Arrays.stream(sort.getSort()).map((o) -> o.getReverse() ? "desc" : "asc").toArray(String[]::new); - builder.putList("index.sort.field", fields); - builder.putList("index.sort.order", orders); - } - return IndexSettingsModule.newIndexSettings(new Index("_index", "0"), builder.build()); - } - - private void addToDocument(int id, Document doc, Map> keys) { - doc.add(new StringField("id", Integer.toString(id), Field.Store.NO)); - for (Map.Entry> entry : keys.entrySet()) { - final String name = entry.getKey(); - for (Object value : entry.getValue()) { - if (value instanceof Integer) { - doc.add(new SortedNumericDocValuesField(name, (int) value)); - doc.add(new IntPoint(name, (int) value)); - } else if (value instanceof Long) { - doc.add(new SortedNumericDocValuesField(name, (long) value)); - doc.add(new LongPoint(name, (long) value)); - } else if (value instanceof Double) { - doc.add(new SortedNumericDocValuesField(name, NumericUtils.doubleToSortableLong((double) value))); - doc.add(new DoublePoint(name, (double) value)); - } else if (value instanceof String) { - doc.add(new SortedSetDocValuesField(name, new BytesRef((String) value))); - doc.add(new StringField(name, new BytesRef((String) value), Field.Store.NO)); - } else if (value instanceof InetAddress) { - doc.add(new SortedSetDocValuesField(name, new BytesRef(InetAddressPoint.encode((InetAddress) value)))); - doc.add(new InetAddressPoint(name, (InetAddress) value)); - } else if (value instanceof GeoPoint) { - GeoPoint point = (GeoPoint) value; - doc.add( - new SortedNumericDocValuesField( - name, - GeoTileUtils.longEncode(point.lon(), point.lat(), GeoTileGridAggregationBuilder.DEFAULT_PRECISION) - ) - ); - doc.add(new LatLonPoint(name, point.lat(), point.lon())); - } else { - throw new AssertionError("invalid object: " + value.getClass().getSimpleName()); - } - } - } - } - - private static Map createAfterKey(Object... fields) { - assert fields.length % 2 == 0; - final Map map = new HashMap<>(); - for (int i = 0; i < fields.length; i += 2) { - String field = (String) fields[i]; - map.put(field, fields[i + 1]); - } - return map; - } - - @SuppressWarnings("unchecked") - private static Map> createDocument(Object... fields) { - assert fields.length % 2 == 0; - final Map> map = new HashMap<>(); - for (int i = 0; i < fields.length; i += 2) { - String field = (String) fields[i]; - if (fields[i + 1] instanceof List) { - map.put(field, (List) fields[i + 1]); - } else { - map.put(field, Collections.singletonList(fields[i + 1])); - } - } - return map; - } - - private static long asLong(String dateTime) { - return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); - } - - private static Sort buildIndexSort(List> sources, Map fieldTypes) { - List sortFields = new ArrayList<>(); - Map remainingFieldTypes = new HashMap<>(fieldTypes); - for (CompositeValuesSourceBuilder source : sources) { - MappedFieldType type = fieldTypes.remove(source.field()); - remainingFieldTypes.remove(source.field()); - SortField sortField = sortFieldFrom(type); - if (sortField == null) { - break; - } - sortFields.add(sortField); - } - while (remainingFieldTypes.size() > 0 && randomBoolean()) { - // Add extra unused sorts - List fields = new ArrayList<>(remainingFieldTypes.keySet()); - Collections.sort(fields); - String field = fields.get(between(0, fields.size() - 1)); - SortField sortField = sortFieldFrom(remainingFieldTypes.remove(field)); - if (sortField != null) { - sortFields.add(sortField); - } - } - return sortFields.size() > 0 ? new Sort(sortFields.toArray(new SortField[0])) : null; - } - - private static SortField sortFieldFrom(MappedFieldType type) { - if (type instanceof KeywordFieldMapper.KeywordFieldType) { - return new SortedSetSortField(type.name(), false); - } else if (type instanceof DateFieldMapper.DateFieldType) { - return new SortedNumericSortField(type.name(), SortField.Type.LONG, false); - } else if (type instanceof NumberFieldMapper.NumberFieldType) { - switch (type.typeName()) { - case "byte": - case "short": - case "integer": - return new SortedNumericSortField(type.name(), SortField.Type.INT, false); - case "long": - return new SortedNumericSortField(type.name(), SortField.Type.LONG, false); - case "float": - case "double": - return new SortedNumericSortField(type.name(), SortField.Type.DOUBLE, false); - default: - return null; - } - } - return null; - } } diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java index dfe4034650594..1443208a1d2fc 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java @@ -34,14 +34,15 @@ import org.opensearch.common.geo.GeoPoint; import org.opensearch.geometry.Rectangle; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import org.opensearch.test.OpenSearchTestCase; -import static org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils.MAX_ZOOM; -import static org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils.checkPrecisionRange; -import static org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils.hashToGeoPoint; -import static org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils.keyToGeoPoint; -import static org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils.longEncode; -import static org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils.stringEncode; +import static org.opensearch.search.aggregations.bucket.GeoTileUtils.MAX_ZOOM; +import static org.opensearch.search.aggregations.bucket.GeoTileUtils.checkPrecisionRange; +import static org.opensearch.search.aggregations.bucket.GeoTileUtils.hashToGeoPoint; +import static org.opensearch.search.aggregations.bucket.GeoTileUtils.keyToGeoPoint; +import static org.opensearch.search.aggregations.bucket.GeoTileUtils.longEncode; +import static org.opensearch.search.aggregations.bucket.GeoTileUtils.stringEncode; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.containsString; diff --git a/test/framework/src/main/java/org/opensearch/search/aggregations/composite/BaseCompositeAggregatorTestCase.java b/test/framework/src/main/java/org/opensearch/search/aggregations/composite/BaseCompositeAggregatorTestCase.java new file mode 100644 index 0000000000000..7d00772913d6e --- /dev/null +++ b/test/framework/src/main/java/org/opensearch/search/aggregations/composite/BaseCompositeAggregatorTestCase.java @@ -0,0 +1,310 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.aggregations.composite; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoublePoint; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.SortedNumericSortField; +import org.apache.lucene.search.SortedSetSortField; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.analysis.MockAnalyzer; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.tests.util.TestUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.NumericUtils; +import org.junit.After; +import org.junit.Before; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.text.Text; +import org.opensearch.common.time.DateFormatter; +import org.opensearch.common.time.DateFormatters; +import org.opensearch.index.Index; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.mapper.DateFieldMapper; +import org.opensearch.index.mapper.DocumentMapper; +import org.opensearch.index.mapper.IpFieldMapper; +import org.opensearch.index.mapper.KeywordFieldMapper; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.search.aggregations.AggregatorTestCase; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; +import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; +import org.opensearch.search.aggregations.bucket.composite.InternalComposite; +import org.opensearch.test.IndexSettingsModule; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Base class for the Aggregator Tests which are registered under Composite Aggregation. + */ +public class BaseCompositeAggregatorTestCase extends AggregatorTestCase { + + protected static List FIELD_TYPES; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + FIELD_TYPES = new ArrayList<>(); + FIELD_TYPES.add(new KeywordFieldMapper.KeywordFieldType("keyword")); + FIELD_TYPES.add(new NumberFieldMapper.NumberFieldType("long", NumberFieldMapper.NumberType.LONG)); + FIELD_TYPES.add(new NumberFieldMapper.NumberFieldType("double", NumberFieldMapper.NumberType.DOUBLE)); + FIELD_TYPES.add(new DateFieldMapper.DateFieldType("date", DateFormatter.forPattern("yyyy-MM-dd||epoch_millis"))); + FIELD_TYPES.add(new NumberFieldMapper.NumberFieldType("price", NumberFieldMapper.NumberType.INTEGER)); + FIELD_TYPES.add(new KeywordFieldMapper.KeywordFieldType("terms")); + FIELD_TYPES.add(new IpFieldMapper.IpFieldType("ip")); + } + + @Override + @After + public void tearDown() throws Exception { + super.tearDown(); + FIELD_TYPES = null; + } + + @Override + protected MapperService mapperServiceMock() { + MapperService mapperService = mock(MapperService.class); + DocumentMapper mapper = mock(DocumentMapper.class); + when(mapper.typeText()).thenReturn(new Text("_doc")); + when(mapper.type()).thenReturn("_doc"); + when(mapperService.documentMapper()).thenReturn(mapper); + return mapperService; + } + + protected static Map> createDocument(Object... fields) { + assert fields.length % 2 == 0; + final Map> map = new HashMap<>(); + for (int i = 0; i < fields.length; i += 2) { + String field = (String) fields[i]; + if (fields[i + 1] instanceof List) { + map.put(field, (List) fields[i + 1]); + } else { + map.put(field, Collections.singletonList(fields[i + 1])); + } + } + return map; + } + + protected void testSearchCase( + List queries, + List>> dataset, + Supplier create, + Consumer verify + ) throws IOException { + for (Query query : queries) { + executeTestCase(false, false, query, dataset, create, verify); + executeTestCase(false, true, query, dataset, create, verify); + } + } + + protected void executeTestCase( + boolean forceMerge, + boolean useIndexSort, + Query query, + List>> dataset, + Supplier create, + Consumer verify + ) throws IOException { + Map types = FIELD_TYPES.stream().collect(Collectors.toMap(MappedFieldType::name, Function.identity())); + CompositeAggregationBuilder aggregationBuilder = create.get(); + Sort indexSort = useIndexSort ? buildIndexSort(aggregationBuilder.sources(), types) : null; + IndexSettings indexSettings = createIndexSettings(indexSort); + try (Directory directory = newDirectory()) { + IndexWriterConfig config = newIndexWriterConfig(random(), new MockAnalyzer(random())); + if (indexSort != null) { + config.setIndexSort(indexSort); + config.setCodec(TestUtil.getDefaultCodec()); + } + try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory, config)) { + Document document = new Document(); + int id = 0; + for (Map> fields : dataset) { + document.clear(); + addToDocument(id, document, fields); + indexWriter.addDocument(document); + id++; + } + if (forceMerge || rarely()) { + // forceMerge randomly or if the collector-per-leaf testing stuff would break the tests. + indexWriter.forceMerge(1); + } else { + if (dataset.size() > 0) { + int numDeletes = randomIntBetween(1, 25); + for (int i = 0; i < numDeletes; i++) { + id = randomIntBetween(0, dataset.size() - 1); + indexWriter.deleteDocuments(new Term("id", Integer.toString(id))); + document.clear(); + addToDocument(id, document, dataset.get(id)); + indexWriter.addDocument(document); + } + } + + } + } + try (IndexReader indexReader = DirectoryReader.open(directory)) { + IndexSearcher indexSearcher = new IndexSearcher(indexReader); + InternalComposite composite = searchAndReduce( + indexSettings, + indexSearcher, + query, + aggregationBuilder, + FIELD_TYPES.toArray(new MappedFieldType[0]) + ); + verify.accept(composite); + } + } + } + + protected void addToDocument(int id, Document doc, Map> keys) { + doc.add(new StringField("id", Integer.toString(id), Field.Store.NO)); + for (Map.Entry> entry : keys.entrySet()) { + final String name = entry.getKey(); + for (Object value : entry.getValue()) { + if (value instanceof Integer) { + doc.add(new SortedNumericDocValuesField(name, (int) value)); + doc.add(new IntPoint(name, (int) value)); + } else if (value instanceof Long) { + doc.add(new SortedNumericDocValuesField(name, (long) value)); + doc.add(new LongPoint(name, (long) value)); + } else if (value instanceof Double) { + doc.add(new SortedNumericDocValuesField(name, NumericUtils.doubleToSortableLong((double) value))); + doc.add(new DoublePoint(name, (double) value)); + } else if (value instanceof String) { + doc.add(new SortedSetDocValuesField(name, new BytesRef((String) value))); + doc.add(new StringField(name, new BytesRef((String) value), Field.Store.NO)); + } else if (value instanceof InetAddress) { + doc.add(new SortedSetDocValuesField(name, new BytesRef(InetAddressPoint.encode((InetAddress) value)))); + doc.add(new InetAddressPoint(name, (InetAddress) value)); + } else { + if (!addValueToDocument(doc, name, value)) throw new AssertionError( + "invalid object: " + value.getClass().getSimpleName() + ); + } + } + } + } + + /** + * Override this function to handle any specific type of value you want to add in the document for doing the + * composite aggregation. If you have added another Composite Aggregation Type then you must override this + * function so that your field value can be added in the document correctly. + * + * @param doc {@link Document} + * @param name {@link String} Field Name + * @param value {@link Object} Field value + * @return boolean true or false, based on if value is added or not + */ + protected boolean addValueToDocument(final Document doc, final String name, final Object value) { + return false; + } + + protected static Sort buildIndexSort(List> sources, Map fieldTypes) { + List sortFields = new ArrayList<>(); + Map remainingFieldTypes = new HashMap<>(fieldTypes); + for (CompositeValuesSourceBuilder source : sources) { + MappedFieldType type = fieldTypes.remove(source.field()); + remainingFieldTypes.remove(source.field()); + SortField sortField = sortFieldFrom(type); + if (sortField == null) { + break; + } + sortFields.add(sortField); + } + while (remainingFieldTypes.size() > 0 && randomBoolean()) { + // Add extra unused sorts + List fields = new ArrayList<>(remainingFieldTypes.keySet()); + Collections.sort(fields); + String field = fields.get(between(0, fields.size() - 1)); + SortField sortField = sortFieldFrom(remainingFieldTypes.remove(field)); + if (sortField != null) { + sortFields.add(sortField); + } + } + return sortFields.size() > 0 ? new Sort(sortFields.toArray(new SortField[0])) : null; + } + + protected static SortField sortFieldFrom(MappedFieldType type) { + if (type instanceof KeywordFieldMapper.KeywordFieldType) { + return new SortedSetSortField(type.name(), false); + } else if (type instanceof DateFieldMapper.DateFieldType) { + return new SortedNumericSortField(type.name(), SortField.Type.LONG, false); + } else if (type instanceof NumberFieldMapper.NumberFieldType) { + switch (type.typeName()) { + case "byte": + case "short": + case "integer": + return new SortedNumericSortField(type.name(), SortField.Type.INT, false); + case "long": + return new SortedNumericSortField(type.name(), SortField.Type.LONG, false); + case "float": + case "double": + return new SortedNumericSortField(type.name(), SortField.Type.DOUBLE, false); + default: + return null; + } + } + return null; + } + + protected static IndexSettings createIndexSettings(Sort sort) { + Settings.Builder builder = Settings.builder(); + if (sort != null) { + String[] fields = Arrays.stream(sort.getSort()).map(SortField::getField).toArray(String[]::new); + String[] orders = Arrays.stream(sort.getSort()).map((o) -> o.getReverse() ? "desc" : "asc").toArray(String[]::new); + builder.putList("index.sort.field", fields); + builder.putList("index.sort.order", orders); + } + return IndexSettingsModule.newIndexSettings(new Index("_index", "0"), builder.build()); + } + + protected static Map createAfterKey(Object... fields) { + assert fields.length % 2 == 0; + final Map map = new HashMap<>(); + for (int i = 0; i < fields.length; i += 2) { + String field = (String) fields[i]; + map.put(field, fields[i + 1]); + } + return map; + } + + protected static long asLong(String dateTime) { + return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); + } +} diff --git a/test/framework/src/main/java/org/opensearch/test/InternalAggregationTestCase.java b/test/framework/src/main/java/org/opensearch/test/InternalAggregationTestCase.java index a4099d66de28e..5325c48e16913 100644 --- a/test/framework/src/main/java/org/opensearch/test/InternalAggregationTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/InternalAggregationTestCase.java @@ -68,10 +68,6 @@ import org.opensearch.search.aggregations.bucket.filter.FiltersAggregationBuilder; import org.opensearch.search.aggregations.bucket.filter.ParsedFilter; import org.opensearch.search.aggregations.bucket.filter.ParsedFilters; -import org.opensearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.ParsedGeoHashGrid; -import org.opensearch.search.aggregations.bucket.geogrid.ParsedGeoTileGrid; import org.opensearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.opensearch.search.aggregations.bucket.global.ParsedGlobal; import org.opensearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; @@ -275,8 +271,6 @@ public ReduceContext forFinalReduction() { map.put(GlobalAggregationBuilder.NAME, (p, c) -> ParsedGlobal.fromXContent(p, (String) c)); map.put(FilterAggregationBuilder.NAME, (p, c) -> ParsedFilter.fromXContent(p, (String) c)); map.put(InternalSampler.PARSER_NAME, (p, c) -> ParsedSampler.fromXContent(p, (String) c)); - map.put(GeoHashGridAggregationBuilder.NAME, (p, c) -> ParsedGeoHashGrid.fromXContent(p, (String) c)); - map.put(GeoTileGridAggregationBuilder.NAME, (p, c) -> ParsedGeoTileGrid.fromXContent(p, (String) c)); map.put(RangeAggregationBuilder.NAME, (p, c) -> ParsedRange.fromXContent(p, (String) c)); map.put(DateRangeAggregationBuilder.NAME, (p, c) -> ParsedDateRange.fromXContent(p, (String) c)); map.put(GeoDistanceAggregationBuilder.NAME, (p, c) -> ParsedGeoDistance.fromXContent(p, (String) c));