Skip to content

Commit

Permalink
Add unmatch_mapping_type; support array of types
Browse files Browse the repository at this point in the history 
Add an unmatch_mapping_type condition to dynamic templates,
and add support for specifying a list of types to
match_mapping_type.
  • Loading branch information
@axw
axw committed Dec 8, 2023
1 parent b9c2980 commit 7d2a5a4
Show file tree
Hide file tree
Showing 3 changed files with 126 additions and 29 deletions.
100 changes: 74 additions & 26 deletions server/src/main/java/org/elasticsearch/index/mapper/DynamicTemplate.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,13 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.function.Predicate;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import java.util.stream.Stream;
Expand Down Expand Up @@ -257,7 +260,8 @@ static DynamicTemplate parse(String name, Map<String, Object> conf) throws Mappe
List<String> pathUnmatch = new ArrayList<>(4);
Map<String, Object> mapping = null;
boolean runtime = false;
String matchMappingType = null;
Object matchMappingType = null;
Object unmatchMappingType = null;
String matchPattern = MatchType.DEFAULT.toString();

for (Map.Entry<String, Object> entry : conf.entrySet()) {
Expand All @@ -271,7 +275,9 @@ static DynamicTemplate parse(String name, Map<String, Object> conf) throws Mappe
} else if ("path_unmatch".equals(propName)) {
addEntriesToPatternList(pathUnmatch, propName, entry);
} else if ("match_mapping_type".equals(propName)) {
matchMappingType = entry.getValue().toString();
matchMappingType = entry.getValue();
} else if ("unmatch_mapping_type".equals(propName)) {
unmatchMappingType = entry.getValue();
} else if ("match_pattern".equals(propName)) {
matchPattern = entry.getValue().toString();
} else if ("mapping".equals(propName)) {
Expand Down Expand Up @@ -301,29 +307,47 @@ static DynamicTemplate parse(String name, Map<String, Object> conf) throws Mappe
throw new MapperParsingException("template [" + name + "] must have either mapping or runtime set");
}

final XContentFieldType[] xContentFieldTypes;
if ("*".equals(matchMappingType) || (matchMappingType == null && matchPatternsAreDefined(match, pathMatch))) {
if (runtime) {
xContentFieldTypes = Arrays.stream(XContentFieldType.values())
.filter(XContentFieldType::supportsRuntimeField)
.toArray(XContentFieldType[]::new);
} else {
xContentFieldTypes = XContentFieldType.values();
}
} else if (matchMappingType != null) {
final XContentFieldType xContentFieldType = XContentFieldType.fromString(matchMappingType);
if (runtime && xContentFieldType.supportsRuntimeField() == false) {
if (matchMappingType == null && (unmatchMappingType != null || matchPatternsAreDefined(match, pathMatch))) {
matchMappingType = "*";
}
XContentFieldType[] xContentFieldTypes = parseMatchMappingType(matchMappingType);

// unmatch_mapping_type filters down the matched mapping types.
if (unmatchMappingType != null) {
final Set<XContentFieldType> unmatchSet = Set.of(parseMatchMappingType(unmatchMappingType));
Set<XContentFieldType> matchSet = new LinkedHashSet<XContentFieldType>(Arrays.asList(xContentFieldTypes));
matchSet.removeAll(unmatchSet);
xContentFieldTypes = matchSet.toArray(XContentFieldType[]::new);
}

// If match_mapping_type is "*", filter down matched mapping types
// to those allowed by runtime fields. Otherwise, throw an exception
// if match_mapping_type explicitly matches field types that are not
// allowed as runtime fields.
if (runtime && xContentFieldTypes.length > 0) {
final boolean matchAny = matchMappingType.equals("*");
final XContentFieldType[] filteredXContentFieldTypes = Arrays.stream(xContentFieldTypes)
.filter(XContentFieldType::supportsRuntimeField)
.toArray(XContentFieldType[]::new);
final int diff = xContentFieldTypes.length - filteredXContentFieldTypes.length;
if (matchAny == false && diff > 0) {
final String[] unsupported = Arrays.stream(xContentFieldTypes)
.filter(Predicate.not(XContentFieldType::supportsRuntimeField))
.map(XContentFieldType::toString)
.toArray(String[]::new);
throw new MapperParsingException(
"Dynamic template ["
+ name
+ "] defines a runtime field but type ["
+ xContentFieldType
+ "] is not supported as runtime field"
+ "] defines a runtime field but type"
+ (diff == 1 ? "" : "s")
+ " ["
+ String.join(", ", unsupported)
+ "] "
+ (diff == 1 ? "is" : "are")
+ " not supported as runtime field"
);
}
xContentFieldTypes = new XContentFieldType[] { xContentFieldType };
} else {
xContentFieldTypes = new XContentFieldType[0];
xContentFieldTypes = filteredXContentFieldTypes;
}

final MatchType matchType = MatchType.fromString(matchPattern);
Expand All @@ -339,6 +363,21 @@ static DynamicTemplate parse(String name, Map<String, Object> conf) throws Mappe
return new DynamicTemplate(name, pathMatch, pathUnmatch, match, unmatch, xContentFieldTypes, matchType, mapping, runtime);
}

private static XContentFieldType[] parseMatchMappingType(Object matchMappingType) {
final XContentFieldType[] xContentFieldTypes;
if (matchMappingType instanceof List<?> ls) {
xContentFieldTypes = ls.stream().map(Object::toString).map(XContentFieldType::fromString).toArray(XContentFieldType[]::new);
} else if ("*".equals(matchMappingType)) {
xContentFieldTypes = XContentFieldType.values();
} else if (matchMappingType != null) {
final XContentFieldType xContentFieldType = XContentFieldType.fromString(matchMappingType.toString());
xContentFieldTypes = new XContentFieldType[] { xContentFieldType };
} else {
xContentFieldTypes = new XContentFieldType[0];
}
return xContentFieldTypes;
}

/**
* @param match list of match patterns (can be empty but not null)
* @param pathMatch list of pathMatch patterns (can be empty but not null)
Expand Down Expand Up @@ -553,13 +592,22 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
builder.field("path_unmatch", pathUnmatch);
}
}
// We have more than one types when (1) `match_mapping_type` is "*", and (2) match and/or path_match are defined but
// not `match_mapping_type`. In the latter the template implicitly accepts all types and we don't need to serialize
// the `match_mapping_type` values.
if (xContentFieldTypes.length > 1 && match.isEmpty() && pathMatch.isEmpty()) {
builder.field("match_mapping_type", "*");
} else if (xContentFieldTypes.length == 1) {
// If we can match all types (considering runtime support), then we can skip serializing "match_mapping_type".
if (xContentFieldTypes.length == 1) {
builder.field("match_mapping_type", xContentFieldTypes[0]);
} else if (xContentFieldTypes.length != 0) {
final long numPossibleXContentFieldTypes;
if (runtimeMapping) {
numPossibleXContentFieldTypes = List.of(XContentFieldType.values())
.stream()
.filter(XContentFieldType::supportsRuntimeField)
.count();
} else {
numPossibleXContentFieldTypes = XContentFieldType.values().length;
}
if (xContentFieldTypes.length < numPossibleXContentFieldTypes) {
builder.field("match_mapping_type", List.of(xContentFieldTypes).stream().map(XContentFieldType::toString).toList());
}
}
if (matchType != MatchType.DEFAULT) {
builder.field("match_pattern", matchType);
Expand Down
20 changes: 20 additions & 0 deletions server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplateParseTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,26 @@ public void testSerialization() throws Exception {
assertEquals("""
{"match_mapping_type":"string","mapping":{"store":true}}""", Strings.toString(builder));

// type-based template with single-entry array - still serializes as single string, rather than list
templateDef = new HashMap<>();
templateDef.put("match_mapping_type", List.of("string"));
templateDef.put("mapping", Collections.singletonMap("store", true));
template = DynamicTemplate.parse("my_template", templateDef);
builder = JsonXContent.contentBuilder();
template.toXContent(builder, ToXContent.EMPTY_PARAMS);
assertEquals("""
{"match_mapping_type":"string","mapping":{"store":true}}""", Strings.toString(builder));

// type-based template with multi-entry array - now serializes as list
templateDef = new HashMap<>();
templateDef.put("match_mapping_type", List.of("string", "long"));
templateDef.put("mapping", Collections.singletonMap("store", true));
template = DynamicTemplate.parse("my_template", templateDef);
builder = JsonXContent.contentBuilder();
template.toXContent(builder, ToXContent.EMPTY_PARAMS);
assertEquals("""
{"match_mapping_type":["string","long"],"mapping":{"store":true}}""", Strings.toString(builder));

// name-based template
templateDef = new HashMap<>();
if (randomBoolean()) {
Expand Down
35 changes: 32 additions & 3 deletions server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,11 +73,40 @@ public void testMatchTypeOnly() throws Exception {

assertThat(mapperService.fieldType("s"), notNullValue());
assertFalse(mapperService.fieldType("s").isIndexed());
assertFalse(mapperService.fieldType("s").isSearchable());

assertThat(mapperService.fieldType("l"), notNullValue());
assertFalse(mapperService.fieldType("s").isIndexed());
assertTrue(mapperService.fieldType("l").isSearchable());
assertTrue(mapperService.fieldType("l").isIndexed());
}

public void testUnmatchTypeOnly() throws Exception {
MapperService mapperService = createMapperService(topMapping(b -> {
b.startArray("dynamic_templates");
{
b.startObject();
{
b.startObject("test");
{
b.field("unmatch_mapping_type", "string");
b.startObject("mapping").field("index", false).endObject();
}
b.endObject();
}
b.endObject();
}
b.endArray();
}));
DocumentMapper docMapper = mapperService.documentMapper();
ParsedDocument parsedDoc = docMapper.parse(source(b -> {
b.field("s", "hello");
b.field("l", 1);
}));
merge(mapperService, dynamicMapping(parsedDoc.dynamicMappingsUpdate()));

assertThat(mapperService.fieldType("s"), notNullValue());
assertTrue(mapperService.fieldType("s").isIndexed());

assertThat(mapperService.fieldType("l"), notNullValue());
assertFalse(mapperService.fieldType("l").isIndexed());
}

public void testSimple() throws Exception {
Expand Down

0 comments on commit 7d2a5a4

Please sign in to comment.