-
Notifications
You must be signed in to change notification settings - Fork 1.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Star tree mapping changes with feature flag
Signed-off-by: Bharathwaj G <bharath78910@gmail.com>
- Loading branch information
1 parent
2e13e9c
commit 1bb63c7
Showing
29 changed files
with
1,309 additions
and
12 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
55 changes: 55 additions & 0 deletions
55
server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexSettings.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.index.compositeindex; | ||
|
|
||
| import org.opensearch.common.annotation.ExperimentalApi; | ||
| import org.opensearch.common.settings.ClusterSettings; | ||
| import org.opensearch.common.settings.Setting; | ||
| import org.opensearch.common.settings.Settings; | ||
| import org.opensearch.common.util.FeatureFlags; | ||
|
|
||
| /** | ||
| * Cluster level settings for composite indices | ||
| * | ||
| * @opensearch.experimental | ||
| */ | ||
| @ExperimentalApi | ||
| public class CompositeIndexSettings { | ||
| public static final Setting<Boolean> STAR_TREE_INDEX_ENABLED_SETTING = Setting.boolSetting( | ||
| "indices.composite.star_tree.enabled", | ||
| false, | ||
| value -> { | ||
| if (FeatureFlags.isEnabled(FeatureFlags.STAR_TREE_INDEX_SETTING) == false && value == true) { | ||
| throw new IllegalArgumentException( | ||
| "star tree index is under an experimental feature and can be activated only by enabling " | ||
| + FeatureFlags.STAR_TREE_INDEX_SETTING.getKey() | ||
| + " feature flag in the JVM options" | ||
| ); | ||
| } | ||
| }, | ||
| Setting.Property.NodeScope, | ||
| Setting.Property.Dynamic | ||
| ); | ||
|
|
||
| private volatile boolean starTreeIndexCreationEnabled; | ||
|
|
||
| public CompositeIndexSettings(Settings settings, ClusterSettings clusterSettings) { | ||
| this.starTreeIndexCreationEnabled = STAR_TREE_INDEX_ENABLED_SETTING.get(settings); | ||
| clusterSettings.addSettingsUpdateConsumer(STAR_TREE_INDEX_ENABLED_SETTING, this::starTreeIndexCreationEnabled); | ||
|
|
||
| } | ||
|
|
||
| private void starTreeIndexCreationEnabled(boolean value) { | ||
| this.starTreeIndexCreationEnabled = value; | ||
| } | ||
|
|
||
| public boolean isStarTreeIndexCreationEnabled() { | ||
| return starTreeIndexCreationEnabled; | ||
| } | ||
| } |
77 changes: 77 additions & 0 deletions
77
server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexValidator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.index.compositeindex; | ||
|
|
||
| import org.opensearch.common.annotation.ExperimentalApi; | ||
| import org.opensearch.index.mapper.CompositeDataCubeFieldType; | ||
| import org.opensearch.index.mapper.CompositeMappedFieldType; | ||
| import org.opensearch.index.mapper.MappedFieldType; | ||
| import org.opensearch.index.mapper.MapperService; | ||
| import org.opensearch.index.mapper.StarTreeMapper; | ||
|
|
||
| import java.util.Locale; | ||
| import java.util.Set; | ||
|
|
||
| /** | ||
| * Validation for composite indices | ||
| * | ||
| * @opensearch.experimental | ||
| */ | ||
| @ExperimentalApi | ||
| public class CompositeIndexValidator { | ||
|
|
||
| public static void validate(MapperService mapperService, CompositeIndexSettings compositeIndexSettings) { | ||
| validateStarTreeMappings(mapperService, compositeIndexSettings); | ||
| } | ||
|
|
||
| private static void validateStarTreeMappings(MapperService mapperService, CompositeIndexSettings compositeIndexSettings) { | ||
| Set<CompositeMappedFieldType> compositeFieldTypes = mapperService.getCompositeFieldTypes(); | ||
| for (CompositeMappedFieldType compositeFieldType : compositeFieldTypes) { | ||
| if (!(compositeFieldType instanceof StarTreeMapper.StarTreeFieldType)) { | ||
| return; | ||
| } | ||
| if (!compositeIndexSettings.isStarTreeIndexCreationEnabled()) { | ||
| throw new IllegalArgumentException( | ||
| String.format( | ||
| Locale.ROOT, | ||
| "star tree index cannot be created, enable it using [%s] setting", | ||
| CompositeIndexSettings.STAR_TREE_INDEX_ENABLED_SETTING.getKey() | ||
| ) | ||
| ); | ||
| } | ||
| CompositeDataCubeFieldType dataCubeFieldType = (CompositeDataCubeFieldType) compositeFieldType; | ||
| for (Dimension dim : dataCubeFieldType.getDimensions()) { | ||
| MappedFieldType ft = mapperService.fieldType(dim.getField()); | ||
| if (!ft.isAggregatable()) { | ||
| throw new IllegalArgumentException( | ||
| String.format( | ||
| Locale.ROOT, | ||
| "Aggregations not supported for the dimension field [%s] with field type [%s] as part of star tree field", | ||
| dim.getField(), | ||
| ft.typeName() | ||
| ) | ||
| ); | ||
| } | ||
| } | ||
| for (Metric metric : dataCubeFieldType.getMetrics()) { | ||
| MappedFieldType ft = mapperService.fieldType(metric.getField()); | ||
| if (!ft.isAggregatable()) { | ||
| throw new IllegalArgumentException( | ||
| String.format( | ||
| Locale.ROOT, | ||
| "Aggregations not supported for the metrics field [%s] with field type [%s] as part of star tree field", | ||
| metric.getField(), | ||
| ft.typeName() | ||
| ) | ||
| ); | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } |
71 changes: 71 additions & 0 deletions
71
server/src/main/java/org/opensearch/index/compositeindex/DateDimension.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,71 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.index.compositeindex; | ||
|
|
||
| import org.opensearch.common.Rounding; | ||
| import org.opensearch.common.annotation.ExperimentalApi; | ||
| import org.opensearch.common.xcontent.support.XContentMapValues; | ||
| import org.opensearch.core.xcontent.XContentBuilder; | ||
| import org.opensearch.index.compositeindex.startree.StarTreeIndexSettings; | ||
| import org.opensearch.index.mapper.Mapper; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.stream.Collectors; | ||
|
|
||
| /** | ||
| * Date dimension class | ||
| * | ||
| * @opensearch.experimental | ||
| */ | ||
| @ExperimentalApi | ||
| public class DateDimension extends Dimension { | ||
| private final List<Rounding.DateTimeUnit> calendarIntervals; | ||
|
|
||
| public DateDimension(String name, List<Rounding.DateTimeUnit> intervals) { | ||
| super(name); | ||
| this.calendarIntervals = intervals; | ||
| } | ||
|
|
||
| @SuppressWarnings("unchecked") | ||
| public DateDimension(Map.Entry<String, Object> dimension, Mapper.TypeParser.ParserContext c) { | ||
| super(dimension.getKey()); | ||
| List<String> intervalStrings = XContentMapValues.extractRawValues("calendar_interval", (Map<String, Object>) dimension.getValue()) | ||
| .stream() | ||
| .map(Object::toString) | ||
| .collect(Collectors.toList()); | ||
| if (intervalStrings == null || intervalStrings.isEmpty()) { | ||
| this.calendarIntervals = StarTreeIndexSettings.DEFAULT_DATE_INTERVALS.get(c.getSettings()); | ||
| } else { | ||
| this.calendarIntervals = new ArrayList<>(); | ||
| for (String interval : intervalStrings) { | ||
| this.calendarIntervals.add(StarTreeIndexSettings.getTimeUnit(interval)); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| public List<Rounding.DateTimeUnit> getIntervals() { | ||
| return calendarIntervals; | ||
| } | ||
|
|
||
| @Override | ||
| public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { | ||
| builder.startObject(this.getField()); | ||
| builder.field("type", "date"); | ||
| builder.startArray("calendar_interval"); | ||
| for (Rounding.DateTimeUnit interval : calendarIntervals) { | ||
| builder.value(interval.shortName()); | ||
| } | ||
| builder.endArray(); | ||
| builder.endObject(); | ||
| return builder; | ||
| } | ||
| } |
Oops, something went wrong.