diff --git a/.ci/scripts/resolve-dra-manifest.sh b/.ci/scripts/resolve-dra-manifest.sh index 4ac94122351aa..e42a22834cccb 100755 --- a/.ci/scripts/resolve-dra-manifest.sh +++ b/.ci/scripts/resolve-dra-manifest.sh @@ -30,7 +30,6 @@ if [ "$LATEST_VERSION" != "$ES_VERSION" ]; then fi echo "Using branch $NEW_BRANCH instead of $BRANCH." 1>&2 - echo "https://artifacts-$WORKFLOW.elastic.co/$ARTIFACT/latest/$NEW_BRANCH.json" LATEST_BUILD=$(fetch_build $WORKFLOW $ARTIFACT $NEW_BRANCH) fi diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 11c6010c22af0..30974ed2396a8 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -515,6 +515,8 @@ subprojects { Project subProject -> base = DockerBase.CLOUD_ESS } else if (subProject.name.contains('cloud-')) { base = DockerBase.CLOUD + } else if (subProject.name.contains('wolfi-')) { + base = DockerBase.WOLFI } final String arch = architecture == Architecture.AARCH64 ? '-aarch64' : '' @@ -522,7 +524,8 @@ subprojects { Project subProject -> (base == DockerBase.IRON_BANK ? 'ironbank.tar' : (base == DockerBase.CLOUD ? 'cloud.tar' : (base == DockerBase.CLOUD_ESS ? 'cloud-ess.tar' : - 'docker.tar'))) + (base == DockerBase.WOLFI ? 'wolfi.tar' : + 'docker.tar')))) final String artifactName = "elasticsearch${arch}${base.suffix}_test" final String exportTaskName = taskName("export", architecture, base, 'DockerImage') diff --git a/docs/changelog/112294.yaml b/docs/changelog/112294.yaml new file mode 100644 index 0000000000000..71ce9eeef584c --- /dev/null +++ b/docs/changelog/112294.yaml @@ -0,0 +1,8 @@ +pr: 112294 +summary: "Use fallback synthetic source for `copy_to` and doc_values: false cases" +area: Mapping +type: enhancement +issues: + - 110753 + - 110038 + - 109546 diff --git a/docs/changelog/112713.yaml b/docs/changelog/112713.yaml new file mode 100644 index 0000000000000..1ccf451b13f82 --- /dev/null +++ b/docs/changelog/112713.yaml @@ -0,0 +1,5 @@ +pr: 112713 +summary: Fix encoding of dynamic arrays in ignored source +area: Logs +type: bug +issues: [] diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java index d3a2867fe2ecd..f0279702812c4 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeCustomSettingsIT.java @@ -283,35 +283,6 @@ public void testOverrideIgnoreDynamicBeyondLimit() throws IOException { assertThat(ignoreDynamicBeyondLimitIndexSetting, equalTo("false")); } - public void testAddNonCompatibleMapping() throws IOException { - var nonCompatibleMappingAdditionTemplate = """ - { - "template": { - "mappings": { - "properties": { - "bomb": { - "type": "ip", - "doc_values": false - } - } - } - } - }"""; - - Exception e = assertThrows( - ResponseException.class, - () -> putComponentTemplate(client, "logs@custom", nonCompatibleMappingAdditionTemplate) - ); - assertThat( - e.getMessage(), - containsString("updating component template [logs@custom] results in invalid composable template [logs]") - ); - assertThat( - e.getMessage(), - containsString("field [bomb] of type [ip] doesn't support synthetic source because it doesn't have doc values") - ); - } - private static Map getMapping(final RestClient client, final String indexName) throws IOException { final Request request = new Request("GET", "/" + indexName + "/_mapping"); diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java index ad4302cb04b44..404914cda6a7e 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/StandardVersusLogsIndexModeRandomDataChallengeRestIT.java @@ -17,7 +17,6 @@ import org.elasticsearch.logsdb.datageneration.DataGenerator; import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification; import org.elasticsearch.logsdb.datageneration.FieldDataGenerator; -import org.elasticsearch.logsdb.datageneration.FieldType; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceHandler; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest; import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse; @@ -78,7 +77,18 @@ public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequ })) .withPredefinedFields( List.of( - new PredefinedField.WithType("host.name", FieldType.KEYWORD), + // Customized because it always needs doc_values for aggregations. + new PredefinedField.WithGenerator("host.name", new FieldDataGenerator() { + @Override + public CheckedConsumer mappingWriter() { + return b -> b.startObject().field("type", "keyword").endObject(); + } + + @Override + public CheckedConsumer fieldValueGenerator() { + return b -> b.value(randomAlphaOfLength(5)); + } + }), // Needed for terms query new PredefinedField.WithGenerator("method", new FieldDataGenerator() { @Override diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index d6225674c7626..a616fe9c20c26 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -45,7 +45,6 @@ import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MapperBuilderContext; -import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.StringFieldType; import org.elasticsearch.index.mapper.StringStoredFieldFieldLoader; @@ -433,22 +432,14 @@ public MatchOnlyTextFieldType fieldType() { } @Override - protected SyntheticSourceMode syntheticSourceMode() { - return SyntheticSourceMode.NATIVE; - } - - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - if (copyTo().copyToFields().isEmpty() != true) { - throw new IllegalArgumentException( - "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" - ); - } - return new StringStoredFieldFieldLoader(fieldType().storedFieldNameForSyntheticSource(), leafName()) { + protected SyntheticSourceSupport syntheticSourceSupport() { + var loader = new StringStoredFieldFieldLoader(fieldType().storedFieldNameForSyntheticSource(), leafName()) { @Override protected void write(XContentBuilder b, Object value) throws IOException { b.value((String) value); } }; + + return new SyntheticSourceSupport.Native(loader); } } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureMetaFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureMetaFieldMapper.java index c45065037b5a8..366145be02d82 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureMetaFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureMetaFieldMapper.java @@ -12,7 +12,6 @@ import org.apache.lucene.search.Query; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MetadataFieldMapper; -import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.SearchExecutionContext; @@ -75,9 +74,4 @@ private RankFeatureMetaFieldMapper() { protected String contentType() { return CONTENT_TYPE; } - - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - return SourceLoader.SyntheticFieldLoader.NOTHING; - } } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index 4e46105bd0534..ac236d13cc586 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -38,7 +38,6 @@ import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.SimpleMappedFieldType; import org.elasticsearch.index.mapper.SortedNumericDocValuesSyntheticFieldLoader; -import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.TimeSeriesParams; @@ -705,32 +704,19 @@ public int docValueCount() { } @Override - protected SyntheticSourceMode syntheticSourceMode() { - return SyntheticSourceMode.NATIVE; - } + protected SyntheticSourceSupport syntheticSourceSupport() { + if (hasDocValues) { + var loader = new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) { + @Override + protected void writeValue(XContentBuilder b, long value) throws IOException { + b.value(decodeForSyntheticSource(value, scalingFactor)); + } + }; - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - if (hasDocValues == false) { - throw new IllegalArgumentException( - "field [" - + fullPath() - + "] of type [" - + typeName() - + "] doesn't support synthetic source because it doesn't have doc values" - ); - } - if (copyTo().copyToFields().isEmpty() != true) { - throw new IllegalArgumentException( - "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" - ); + return new SyntheticSourceSupport.Native(loader); } - return new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) { - @Override - protected void writeValue(XContentBuilder b, long value) throws IOException { - b.value(decodeForSyntheticSource(value, scalingFactor)); - } - }; + + return super.syntheticSourceSupport(); } /** diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java index 56b9bb7f748b4..765e72091a1ba 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java @@ -470,12 +470,7 @@ private void mapping(XContentBuilder b) throws IOException { @Override public List invalidExample() throws IOException { - return List.of( - new SyntheticSourceInvalidExample( - equalTo("field [field] of type [scaled_float] doesn't support synthetic source because it doesn't have doc values"), - b -> b.field("type", "scaled_float").field("scaling_factor", 10).field("doc_values", false) - ) - ); + return List.of(); } } diff --git a/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/match_only_text/10_basic.yml b/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/match_only_text/10_basic.yml index 7d1d6e2edec30..b4ee226f72692 100644 --- a/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/match_only_text/10_basic.yml +++ b/modules/mapper-extras/src/yamlRestTest/resources/rest-api-spec/test/match_only_text/10_basic.yml @@ -351,3 +351,45 @@ tsdb: "@timestamp" : "2000-01-01T00:00:00.000Z" "dimension" : "a" foo: "Apache Lucene powers Elasticsearch" + +--- +synthetic_source with copy_to: + - requires: + cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: synthetic_source_test + body: + mappings: + _source: + mode: synthetic + properties: + foo: + type: match_only_text + copy_to: copy + copy: + type: keyword + + - do: + index: + index: synthetic_source_test + id: "1" + refresh: true + body: + foo: "Apache Lucene powers Elasticsearch" + + - do: + search: + index: synthetic_source_test + body: + fields: ["copy"] + + - match: { "hits.total.value": 1 } + - match: + hits.hits.0._source.foo: "Apache Lucene powers Elasticsearch" + - match: + hits.hits.0.fields.copy.0: "Apache Lucene powers Elasticsearch" + + diff --git a/muted-tests.yml b/muted-tests.yml index 1fe7cbb9540b3..589e49645c986 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -169,12 +169,6 @@ tests: - class: org.elasticsearch.xpack.esql.EsqlAsyncSecurityIT method: testIndexPatternErrorMessageComparison_ESQL_SearchDSL issue: https://github.com/elastic/elasticsearch/issues/112630 -- class: org.elasticsearch.compute.aggregation.blockhash.BlockHashTests - method: testBytesRefLongHashHugeCombinatorialExplosion {forcePackedHash=false} - issue: https://github.com/elastic/elasticsearch/issues/112442 -- class: org.elasticsearch.compute.aggregation.blockhash.BlockHashTests - method: testBytesRefLongHashHugeCombinatorialExplosion {forcePackedHash=true} - issue: https://github.com/elastic/elasticsearch/issues/112443 - class: org.elasticsearch.xpack.ml.integration.MlJobIT method: testPutJob_GivenFarequoteConfig issue: https://github.com/elastic/elasticsearch/issues/112382 diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index 4f077fdcde069..b66ce41d3259d 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -30,7 +30,6 @@ import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MapperBuilderContext; -import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.StringStoredFieldFieldLoader; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TextParams; @@ -46,7 +45,6 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -570,39 +568,23 @@ public FieldMapper.Builder getMergeBuilder() { } @Override - protected SyntheticSourceMode syntheticSourceMode() { - return SyntheticSourceMode.NATIVE; - } - - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - if (copyTo().copyToFields().isEmpty() != true) { - throw new IllegalArgumentException( - "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" - ); - } + protected SyntheticSourceSupport syntheticSourceSupport() { if (fieldType.stored()) { - return new StringStoredFieldFieldLoader(fullPath(), leafName()) { + var loader = new StringStoredFieldFieldLoader(fullPath(), leafName()) { @Override protected void write(XContentBuilder b, Object value) throws IOException { b.value((String) value); } }; + + return new SyntheticSourceSupport.Native(loader); } var kwd = TextFieldMapper.SyntheticSourceHelper.getKeywordFieldMapperForSyntheticSource(this); if (kwd != null) { - return kwd.syntheticFieldLoader(leafName()); + return new SyntheticSourceSupport.Native(kwd.syntheticFieldLoader(leafName())); } - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "field [%s] of type [%s] doesn't support synthetic source unless it is stored or has a sub-field of" - + " type [keyword] with doc values or stored and without a normalizer", - fullPath(), - typeName() - ) - ); + return super.syntheticSourceSupport(); } } diff --git a/plugins/mapper-annotated-text/src/yamlRestTest/resources/rest-api-spec/test/mapper_annotatedtext/20_synthetic_source.yml b/plugins/mapper-annotated-text/src/yamlRestTest/resources/rest-api-spec/test/mapper_annotatedtext/20_synthetic_source.yml index 54a51e60f56df..4aac881700e15 100644 --- a/plugins/mapper-annotated-text/src/yamlRestTest/resources/rest-api-spec/test/mapper_annotatedtext/20_synthetic_source.yml +++ b/plugins/mapper-annotated-text/src/yamlRestTest/resources/rest-api-spec/test/mapper_annotatedtext/20_synthetic_source.yml @@ -195,3 +195,34 @@ multiple values in stored annotated_text field with keyword multi-field: - match: hits.hits.0._source: annotated_text: ["world", "hello", "world"] + +--- +fallback synthetic source: + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + annotated_text: + type: annotated_text + store: false + + - do: + index: + index: test + id: 1 + refresh: true + body: + annotated_text: ["world", "hello", "world"] + + - do: + search: + index: test + + - match: + hits.hits.0._source: + annotated_text: ["world", "hello", "world"] + diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java index b188f4b148590..6a4869b8c89b7 100644 --- a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java +++ b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java @@ -16,7 +16,6 @@ import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; -import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.SearchExecutionContext; @@ -97,9 +96,4 @@ public void postParse(DocumentParserContext context) { public FieldMapper.Builder getMergeBuilder() { return new Builder().init(this); } - - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - return SourceLoader.SyntheticFieldLoader.NOTHING; - } } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml index 55605849de69c..f1e296ed8e304 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml @@ -129,50 +129,6 @@ force_synthetic_source_ok: kwd: foo - is_false: fields ---- -force_synthetic_source_bad_mapping: - - requires: - cluster_features: ["gte_v8.4.0"] - reason: introduced in 8.4.0 - - - do: - indices.create: - index: test - body: - settings: - number_of_shards: 1 # Use a single shard to get consistent error messages - mappings: - _source: - mode: stored - properties: - text: - type: text - - - do: - index: - index: test - id: 1 - refresh: true - body: - text: foo - - # When _source is used in the fetch the original _source is perfect - - do: - get: - index: test - id: 1 - - match: - _source: - text: foo - - # Forcing synthetic source fails because the mapping is invalid - - do: - catch: bad_request - get: - index: test - id: 1 - force_synthetic_source: true - --- stored text: - requires: @@ -1040,25 +996,6 @@ flattened field: - is_false: fields ---- -flattened field no doc values: - - requires: - cluster_features: ["gte_v8.8.0"] - reason: support for synthetic source on flattened fields added in 8.8.0 - - - do: - catch: /field \[flattened\] of type \[flattened\] doesn't support synthetic source because it doesn't have doc values/ - indices.create: - index: test - body: - mappings: - _source: - mode: synthetic - properties: - flattened: - type: flattened - doc_values: false - --- flattened field with ignore_above: - requires: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml index 265aec75dc9c2..9dd6cec6e657c 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml @@ -1,23 +1,3 @@ -invalid: - - requires: - cluster_features: ["gte_v8.3.0"] - reason: introduced in 8.3.0 - - - do: - catch: bad_request - indices.create: - index: test - body: - mappings: - _source: - mode: synthetic - properties: - kwd: - type: boolean - doc_values: false - - ---- object with unmapped fields: - requires: cluster_features: ["mapper.track_ignored_source"] @@ -990,3 +970,247 @@ subobjects auto: - match: { hits.hits.3._source.id: 4 } - match: { hits.hits.3._source.auto_obj.foo: 40 } - match: { hits.hits.3._source.auto_obj.foo\.bar: 400 } + +--- +synthetic_source with copy_to: + - requires: + cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + number: + type: integer + copy_to: number_copy + number_copy: + type: keyword + boolean: + type: boolean + copy_to: boolean_copy + boolean_copy: + type: keyword + keyword: + type: keyword + copy_to: keyword_copy + keyword_copy: + type: keyword + date: + type: date + copy_to: date_copy + date_copy: + type: keyword + text: + type: text + copy_to: text_copy + text_copy: + type: keyword + ip: + type: ip + copy_to: ip_copy + ip_copy: + type: keyword + ip_range: + type: ip_range + copy_to: ip_range_copy + ip_range_copy: + type: keyword + geo_point: + type: geo_point + copy_to: geo_point_copy + geo_point_copy: + type: keyword + binary: + type: binary + copy_to: binary_copy + binary_copy: + type: keyword + scaled_float: + type: scaled_float + scaling_factor: 10 + copy_to: scaled_float_copy + scaled_float_copy: + type: keyword + + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - >- + { + "number": 100, + "boolean": false, + "keyword": "hello_keyword", + "date": "2015-01-01T12:10:30Z", + "text": "hello_text", + "match_only_text": "hello_match_only_text", + "ip": "192.168.1.1", + "ip_range": "10.0.0.0/24", + "geo_point": "POINT (-71.34 41.12)", + "binary": "aGVsbG8gY3VyaW91cyBwZXJzb24=", + "scaled_float": 1.5 + } + + - match: { errors: false } + + - do: + search: + index: test + body: + fields: ["number_copy", "boolean_copy", "keyword_copy", "date_copy", "text_copy", "ip_copy", "ip_range_copy", "geo_point_copy", "binary_copy", "scaled_float_copy"] + + - match: { hits.hits.0._source.number: 100 } + - match: { hits.hits.0.fields.number_copy.0: "100" } + + - match: { hits.hits.0._source.boolean: false } + - match: { hits.hits.0.fields.boolean_copy.0: "false" } + + - match: { hits.hits.0._source.keyword: "hello_keyword" } + - match: { hits.hits.0.fields.keyword_copy.0: "hello_keyword" } + + - match: { hits.hits.0._source.date: "2015-01-01T12:10:30Z" } + - match: { hits.hits.0.fields.date_copy.0: "2015-01-01T12:10:30Z" } + + - match: { hits.hits.0._source.text: "hello_text" } + - match: { hits.hits.0.fields.text_copy.0: "hello_text" } + + - match: { hits.hits.0._source.ip: "192.168.1.1" } + - match: { hits.hits.0.fields.ip_copy.0: "192.168.1.1" } + + - match: { hits.hits.0._source.ip_range: "10.0.0.0/24" } + - match: { hits.hits.0.fields.ip_range_copy.0: "10.0.0.0/24" } + + - match: { hits.hits.0._source.geo_point: "POINT (-71.34 41.12)" } + - match: { hits.hits.0.fields.geo_point_copy.0: "POINT (-71.34 41.12)" } + + - match: { hits.hits.0._source.binary: "aGVsbG8gY3VyaW91cyBwZXJzb24=" } + - match: { hits.hits.0.fields.binary_copy.0: "aGVsbG8gY3VyaW91cyBwZXJzb24=" } + + - match: { hits.hits.0._source.scaled_float: 1.5 } + - match: { hits.hits.0.fields.scaled_float_copy.0: "1.5" } + +--- +synthetic_source with disabled doc_values: + - requires: + cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"] + reason: requires disabled doc_values support in synthetic source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + number: + type: integer + doc_values: false + boolean: + type: boolean + doc_values: false + keyword: + type: keyword + doc_values: false + date: + type: date + doc_values: false + ip: + type: ip + doc_values: false + ip_range: + type: ip_range + doc_values: false + flattened: + type: flattened + doc_values: false + geo_point: + type: geo_point + doc_values: false + binary: + type: binary + doc_values: false + scaled_float: + type: scaled_float + scaling_factor: 10 + doc_values: false + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - >- + { + "number": 100, + "boolean": false, + "keyword": "hello_keyword", + "date": "2015-01-01T12:10:30Z", + "ip": "192.168.1.1", + "ip_range": "10.0.0.0/24", + "flattened": { "f": "hey" }, + "geo_point": "POINT (-71.34 41.12)", + "binary": "aGVsbG8gY3VyaW91cyBwZXJzb24=", + "scaled_float": 1.5 + } + + - match: { errors: false } + + - do: + search: + index: test + + - match: { hits.hits.0._source.number: 100 } + - match: { hits.hits.0._source.boolean: false } + - match: { hits.hits.0._source.keyword: "hello_keyword" } + - match: { hits.hits.0._source.date: "2015-01-01T12:10:30Z" } + - match: { hits.hits.0._source.ip: "192.168.1.1" } + - match: { hits.hits.0._source.ip_range: "10.0.0.0/24" } + - match: { hits.hits.0._source.flattened.f: "hey" } + - match: { hits.hits.0._source.geo_point: "POINT (-71.34 41.12)" } + - match: { hits.hits.0._source.binary: "aGVsbG8gY3VyaW91cyBwZXJzb24=" } + - match: { hits.hits.0._source.scaled_float: 1.5 } + +--- +fallback synthetic_source for text field: + - requires: + cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"] + reason: requires disabled doc_values support in synthetic source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + text: + type: text + store: false + + - do: + index: + index: test + id: 1 + refresh: true + body: + text: [ "world", "hello", "world" ] + + - do: + search: + index: test + + - match: + hits.hits.0._source: + text: [ "world", "hello", "world" ] + diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml index ff17a92ed0fcc..2f3d2fa2f974d 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml @@ -247,58 +247,3 @@ force_synthetic_source_ok: obj: kwd: bar ---- -force_synthetic_source_bad_mapping: - - requires: - cluster_features: ["gte_v8.5.0"] - reason: message changed in 8.5 - - - do: - indices.create: - index: test - body: - settings: - number_of_shards: 1 # Use a single shard to get consistent error messages - mappings: - _source: - mode: stored - properties: - text: - type: text - - - do: - index: - index: test - id: 1 - body: - text: foo - - - do: - index: - index: test - id: 2 - body: - text: bar - - # When _source is used in the fetch the original _source is perfect - - do: - mget: - index: test - body: - ids: [ 1, 2 ] - - match: - docs.0._source: - text: foo - - match: - docs.1._source: - text: bar - - # Forcing synthetic source fails because the mapping is invalid - - do: - mget: - index: test - force_synthetic_source: true - body: - ids: [ 1, 2 ] - - match: {docs.0.error.reason: "field [text] of type [text] doesn't support synthetic source unless it is stored or has a sub-field of type [keyword] with doc values or stored and without a normalizer"} - - match: {docs.1.error.reason: "field [text] of type [text] doesn't support synthetic source unless it is stored or has a sub-field of type [keyword] with doc values or stored and without a normalizer"} diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/400_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/400_synthetic_source.yml index 75d488427a903..0cc1796bb47de 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/400_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/400_synthetic_source.yml @@ -208,56 +208,6 @@ force_synthetic_source_ok: obj: kwd: foo ---- -force_synthetic_source_bad_mapping: - - requires: - cluster_features: ["gte_v8.4.0"] - reason: introduced in 8.4.0 - - - do: - indices.create: - index: test - body: - settings: - number_of_shards: 1 # Use a single shard to get consistent error messages - mappings: - _source: - mode: stored - properties: - text: - type: text - - - do: - index: - index: test - id: 1 - refresh: true - body: - text: foo - - # When _source is used in the fetch the original _source is perfect - - do: - search: - index: test - body: - query: - ids: - values: [1] - - match: - hits.hits.0._source: - text: foo - - # Forcing synthetic source fails because the mapping is invalid - - do: - catch: bad_request - search: - index: test - force_synthetic_source: true - body: - query: - ids: - values: [1] - --- doc values keyword with ignore_above: - requires: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml index 976ac8f08f795..db718959919da 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/90_unsupported_operations.yml @@ -238,52 +238,3 @@ aggregate on _id: id: terms: field: _id - ---- -synthetic source text field: - - requires: - cluster_features: ["gte_v8.7.0"] - reason: "synthetic source introduced in 8.7.0" - - - do: - catch: /field \[k8s.pod.agent.name\] of type \[text\] doesn't support synthetic source unless it is stored or has a sub-field of type \[keyword\] with doc values or stored and without a normalizer/ - indices.create: - index: test-text-field - body: - settings: - number_of_shards: 1 - number_of_replicas: 0 - index: - mode: time_series - routing_path: [ metricset, k8s.pod.uid ] - time_series: - start_time: 2021-04-28T00:00:00Z - end_time: 2021-04-29T00:00:00Z - mappings: - properties: - "@timestamp": - type: date - metricset: - type: keyword - time_series_dimension: true - k8s: - properties: - pod: - properties: - uid: - type: keyword - time_series_dimension: true - agent: - type: object - properties: - id: - type: text - fields: - raw: - type: keyword - name: - type: text - store: false - value: - type: long - time_series_metric: gauge diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java index cc272042d5384..c89c7d8d749f6 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java @@ -11,7 +11,6 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.logging.log4j.Level; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.admin.cluster.reroute.ClusterRerouteUtils; import org.elasticsearch.action.fieldcaps.FieldCapabilities; @@ -38,8 +37,6 @@ import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MetadataFieldMapper; -import org.elasticsearch.index.mapper.SourceLoader; -import org.elasticsearch.index.mapper.StringStoredFieldFieldLoader; import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; @@ -915,17 +912,6 @@ protected String contentType() { return CONTENT_TYPE; } - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - return new StringStoredFieldFieldLoader(fullPath(), leafName()) { - @Override - protected void write(XContentBuilder b, Object value) throws IOException { - BytesRef ref = (BytesRef) value; - b.utf8Value(ref.bytes, ref.offset, ref.length); - } - }; - } - private static final TypeParser PARSER = new FixedTypeParser(c -> new TestMetadataMapper()); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index 99a04f5d08337..8d9dd99092a07 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -194,54 +194,40 @@ protected String contentType() { } @Override - protected SyntheticSourceMode syntheticSourceMode() { - return SyntheticSourceMode.NATIVE; - } - - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - if (copyTo().copyToFields().isEmpty() != true) { - throw new IllegalArgumentException( - "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" - ); - } - if (hasDocValues == false) { - throw new IllegalArgumentException( - "field [" - + fullPath() - + "] of type [" - + typeName() - + "] doesn't support synthetic source because it doesn't have doc values" - ); - } - - return new BinaryDocValuesSyntheticFieldLoader(fullPath()) { - @Override - protected void writeValue(XContentBuilder b, BytesRef value) throws IOException { - var in = new ByteArrayStreamInput(); - in.reset(value.bytes, value.offset, value.length); - - int count = in.readVInt(); - switch (count) { - case 0: - return; - case 1: - b.field(leafName()); - break; - default: - b.startArray(leafName()); + protected SyntheticSourceSupport syntheticSourceSupport() { + if (hasDocValues) { + var loader = new BinaryDocValuesSyntheticFieldLoader(fullPath()) { + @Override + protected void writeValue(XContentBuilder b, BytesRef value) throws IOException { + var in = new ByteArrayStreamInput(); + in.reset(value.bytes, value.offset, value.length); + + int count = in.readVInt(); + switch (count) { + case 0: + return; + case 1: + b.field(leafName()); + break; + default: + b.startArray(leafName()); + } + + for (int i = 0; i < count; i++) { + byte[] bytes = in.readByteArray(); + b.value(Base64.getEncoder().encodeToString(bytes)); + } + + if (count > 1) { + b.endArray(); + } } + }; - for (int i = 0; i < count; i++) { - byte[] bytes = in.readByteArray(); - b.value(Base64.getEncoder().encodeToString(bytes)); - } + return new SyntheticSourceSupport.Native(loader); + } - if (count > 1) { - b.endArray(); - } - } - }; + return super.syntheticSourceSupport(); } public static final class CustomBinaryDocValuesField extends CustomDocValuesField { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 59db5c35e40bb..e450d3916fc62 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -549,34 +549,18 @@ protected String contentType() { } @Override - protected SyntheticSourceMode syntheticSourceMode() { - return SyntheticSourceMode.NATIVE; - } + protected SyntheticSourceSupport syntheticSourceSupport() { + if (hasDocValues) { + var loader = new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) { + @Override + protected void writeValue(XContentBuilder b, long value) throws IOException { + b.value(value == 1); + } + }; - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - if (hasScript()) { - return SourceLoader.SyntheticFieldLoader.NOTHING; - } - if (hasDocValues == false) { - throw new IllegalArgumentException( - "field [" - + fullPath() - + "] of type [" - + typeName() - + "] doesn't support synthetic source because it doesn't have doc values" - ); + return new SyntheticSourceSupport.Native(loader); } - if (copyTo().copyToFields().isEmpty() != true) { - throw new IllegalArgumentException( - "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" - ); - } - return new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()) { - @Override - protected void writeValue(XContentBuilder b, long value) throws IOException { - b.value(value == 1); - } - }; + + return super.syntheticSourceSupport(); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java index 6f4a8ffa92cbe..a6048c22c2827 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java @@ -270,9 +270,4 @@ protected String contentType() { public boolean isEnabled() { return enabled; } - - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - return SourceLoader.SyntheticFieldLoader.NOTHING; - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 11f6e7b6ff93d..5bd79ca211280 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -1004,34 +1004,18 @@ public Long getNullValue() { } @Override - protected SyntheticSourceMode syntheticSourceMode() { - return SyntheticSourceMode.NATIVE; - } + protected SyntheticSourceSupport syntheticSourceSupport() { + if (hasDocValues) { + var loader = new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed) { + @Override + protected void writeValue(XContentBuilder b, long value) throws IOException { + b.value(fieldType().format(value, fieldType().dateTimeFormatter())); + } + }; - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - if (hasScript()) { - return SourceLoader.SyntheticFieldLoader.NOTHING; - } - if (hasDocValues == false) { - throw new IllegalArgumentException( - "field [" - + fullPath() - + "] of type [" - + typeName() - + "] doesn't support synthetic source because it doesn't have doc values" - ); + return new SyntheticSourceSupport.Native(loader); } - if (copyTo().copyToFields().isEmpty() != true) { - throw new IllegalArgumentException( - "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" - ); - } - return new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed) { - @Override - protected void writeValue(XContentBuilder b, long value) throws IOException { - b.value(fieldType().format(value, fieldType().dateTimeFormatter())); - } - }; + + return super.syntheticSourceSupport(); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocCountFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocCountFieldMapper.java index 6326e23c59b92..7a94df40daf76 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocCountFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocCountFieldMapper.java @@ -126,8 +126,8 @@ public static IndexableField field(int count) { } @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - return new SyntheticFieldLoader(); + protected SyntheticSourceSupport syntheticSourceSupport() { + return new SyntheticSourceSupport.Native(new SyntheticFieldLoader()); } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 150b2b68cc018..e2143aca0fd1d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -646,7 +646,7 @@ private static void parseArrayDynamic(DocumentParserContext context, String curr context.addIgnoredField( IgnoredSourceFieldMapper.NameValue.fromContext( context, - currentFieldName, + context.path().pathAsText(currentFieldName), XContentDataHelper.encodeToken(context.parser()) ) ); @@ -998,16 +998,10 @@ protected String contentType() { } @Override - protected SyntheticSourceMode syntheticSourceMode() { + protected SyntheticSourceSupport syntheticSourceSupport() { // Opt out of fallback synthetic source implementation - // since there is custom logic in #parseCreateField() - return SyntheticSourceMode.NATIVE; - } - - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - // Handled via IgnoredSourceFieldMapper infrastructure - return SourceLoader.SyntheticFieldLoader.NOTHING; + // since there is custom logic in #parseCreateField(). + return new SyntheticSourceSupport.Native(SourceLoader.SyntheticFieldLoader.NOTHING); } }; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 01e86b643d058..e2616832b7b14 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -443,21 +443,6 @@ public Map indexAnalyzers() { return Map.of(); } - /** - * Specifies the mode of synthetic source support by the mapper. - * - * 
-     * {@link SyntheticSourceMode#NATIVE} - mapper natively supports synthetic source, f.e. by constructing it from doc values.
-     *
-     * {@link SyntheticSourceMode#FALLBACK} - mapper does not have native support and uses generic fallback implementation
-     * that stores raw input source data as is.
-     *
- */ - protected enum SyntheticSourceMode { - NATIVE, - FALLBACK - } - /** *

* Specifies the mode of synthetic source support by the mapper. @@ -469,38 +454,112 @@ protected enum SyntheticSourceMode { * and then use it for synthetic source. *

*

- * Field mappers must override this method if they provide - * a custom implementation of {@link #syntheticFieldLoader()} - * in order to use a more efficient field-specific implementation. + * This method is final in order to support common use cases like fallback synthetic source and copy_to. + * Mappers that need custom support of synthetic source should override {@link #syntheticSourceSupport()}. *

* @return {@link SyntheticSourceMode} */ - protected SyntheticSourceMode syntheticSourceMode() { - return SyntheticSourceMode.FALLBACK; + final SyntheticSourceMode syntheticSourceMode() { + if (hasScript()) { + return SyntheticSourceMode.NATIVE; + } + + if (builderParams.copyTo.copyToFields().isEmpty() == false) { + // When copy_to is used, we need to use fallback logic to store source of the field exactly. + // Otherwise, due to possible differences between synthetic source and stored source, + // values of fields that are destinations of copy_to would be different after reindexing. + return SyntheticSourceMode.FALLBACK; + } + + return syntheticSourceSupport().mode(); } /** - * Mappers override this method with native synthetic source support. - * If mapper does not support synthetic source, it is generated using generic implementation + * Returns synthetic field loader for the mapper. + * If mapper does not support synthetic source, it is handled using generic implementation * in {@link DocumentParser#parseObjectOrField} and {@link ObjectMapper#syntheticFieldLoader()}. + *
+ * + * This method is final in order to support common use cases like fallback synthetic source. + * Mappers that need custom support of synthetic source should override {@link #syntheticSourceSupport()}. * * @return implementation of {@link SourceLoader.SyntheticFieldLoader} */ @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - // If mapper supports synthetic source natively, it overrides this method, - // so we won't see those here. + public final SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + if (hasScript()) { + return SourceLoader.SyntheticFieldLoader.NOTHING; + } + if (syntheticSourceMode() == SyntheticSourceMode.FALLBACK) { - if (builderParams.copyTo.copyToFields().isEmpty() != true) { - throw new IllegalArgumentException( - "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" - ); - } - // Nothing because it is handled at `ObjectMapper` level. + // Nothing because it is handled at `DocumentParser/ObjectMapper` level. return SourceLoader.SyntheticFieldLoader.NOTHING; } - return super.syntheticFieldLoader(); + return syntheticSourceSupport().loader(); + } + + /** + *

+ * Returns implementation of synthetic source support for the mapper. + *
+ * By default (meaning {@link SyntheticSourceSupport.Fallback}), + * an exact full copy of parsed field value is stored separately + * and used for synthetic source. + *

+ *

+ * Field mappers must override this method if they provide + * a more efficient field-specific implementation of synthetic source. + *

+ * @return {@link SyntheticSourceMode} + */ + protected SyntheticSourceSupport syntheticSourceSupport() { + return SyntheticSourceSupport.FALLBACK; + } + + /** + * Specifies the mode of synthetic source support by the mapper. + * + * 
+     * {@link SyntheticSourceMode#NATIVE} - mapper natively supports synthetic source, f.e. by constructing it from doc values.
+     *
+     * {@link SyntheticSourceMode#FALLBACK} - mapper does not have native support and uses generic fallback implementation
+     * that stores raw input source data as is.
+     *
+ */ + protected enum SyntheticSourceMode { + NATIVE, + FALLBACK + } + + /** + * Interface that defines how a field supports synthetic source. + */ + protected sealed interface SyntheticSourceSupport permits SyntheticSourceSupport.Fallback, SyntheticSourceSupport.Native { + final class Fallback implements SyntheticSourceSupport { + @Override + public SyntheticSourceMode mode() { + return SyntheticSourceMode.FALLBACK; + } + + @Override + public SourceLoader.SyntheticFieldLoader loader() { + return null; + } + } + + SyntheticSourceSupport FALLBACK = new Fallback(); + + record Native(SourceLoader.SyntheticFieldLoader loader) implements SyntheticSourceSupport { + @Override + public SyntheticSourceMode mode() { + return SyntheticSourceMode.NATIVE; + } + } + + SyntheticSourceMode mode(); + + SourceLoader.SyntheticFieldLoader loader(); } public static final class MultiFields implements Iterable, ToXContent { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java index 57a4885d44d5a..1229cce59aceb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldNamesFieldMapper.java @@ -187,9 +187,4 @@ private static boolean noDocValues(String field, DocumentParserContext context) protected String contentType() { return CONTENT_TYPE; } - - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - return SourceLoader.SyntheticFieldLoader.NOTHING; - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index 3c62410fa848a..0719d4ad7e484 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -607,37 +607,21 @@ protected void onMalformedValue(DocumentParserContext context, XContentBuilder m } @Override - protected SyntheticSourceMode syntheticSourceMode() { - return SyntheticSourceMode.NATIVE; - } + protected SyntheticSourceSupport syntheticSourceSupport() { + if (fieldType().hasDocValues()) { + var loader = new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed()) { + final GeoPoint point = new GeoPoint(); - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - if (hasScript()) { - return SourceLoader.SyntheticFieldLoader.NOTHING; - } - if (fieldType().hasDocValues() == false) { - throw new IllegalArgumentException( - "field [" - + fullPath() - + "] of type [" - + typeName() - + "] doesn't support synthetic source because it doesn't have doc values" - ); - } - if (copyTo().copyToFields().isEmpty() != true) { - throw new IllegalArgumentException( - "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" - ); + @Override + protected void writeValue(XContentBuilder b, long value) throws IOException { + point.reset(GeoEncodingUtils.decodeLatitude((int) (value >>> 32)), GeoEncodingUtils.decodeLongitude((int) value)); + point.toXContent(b, ToXContent.EMPTY_PARAMS); + } + }; + + return new SyntheticSourceSupport.Native(loader); } - return new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed()) { - final GeoPoint point = new GeoPoint(); - @Override - protected void writeValue(XContentBuilder b, long value) throws IOException { - point.reset(GeoEncodingUtils.decodeLatitude((int) (value >>> 32)), GeoEncodingUtils.decodeLongitude((int) value)); - point.toXContent(b, ToXContent.EMPTY_PARAMS); - } - }; + return super.syntheticSourceSupport(); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java index c5b1f575d941f..8b1e54ef1c7b0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java @@ -50,11 +50,6 @@ protected final String contentType() { return CONTENT_TYPE; } - @Override - public final SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - return SourceLoader.SyntheticFieldLoader.NOTHING; - } - /** * Description of the document being parsed used in error messages. Not * called unless there is an error. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java index b0f153252ac4e..2e6466e69b840 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldMapper.java @@ -140,9 +140,4 @@ public void postParse(DocumentParserContext context) { protected String contentType() { return CONTENT_TYPE; } - - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - return SourceLoader.SyntheticFieldLoader.NOTHING; - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java index f94a05b2a8658..5d8d13abafeab 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java @@ -199,12 +199,4 @@ public static byte[] encodeFromMap(MappedNameValue mappedNameValue, Map(); + layers.add(new SortedSetDocValuesSyntheticFieldLoaderLayer(fullPath()) { + @Override + protected BytesRef convert(BytesRef value) { + byte[] bytes = Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length); + return new BytesRef(NetworkAddress.format(InetAddressPoint.decode(bytes))); + } - var layers = new ArrayList(); - layers.add(new SortedSetDocValuesSyntheticFieldLoaderLayer(fullPath()) { - @Override - protected BytesRef convert(BytesRef value) { - byte[] bytes = Arrays.copyOfRange(value.bytes, value.offset, value.offset + value.length); - return new BytesRef(NetworkAddress.format(InetAddressPoint.decode(bytes))); - } + @Override + protected BytesRef preserve(BytesRef value) { + // No need to copy because convert has made a deep copy + return value; + } + }); - @Override - protected BytesRef preserve(BytesRef value) { - // No need to copy because convert has made a deep copy - return value; + if (ignoreMalformed) { + layers.add(new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath())); } - }); - if (ignoreMalformed) { - layers.add(new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath())); + return new SyntheticSourceSupport.Native(new CompositeSyntheticFieldLoader(leafName(), fullPath(), layers)); } - return new CompositeSyntheticFieldLoader(leafName(), fullPath(), layers); + return super.syntheticSourceSupport(); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 769d170423fa1..97c48c7bbfa36 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -1029,37 +1029,22 @@ private String originalName() { } @Override - protected SyntheticSourceMode syntheticSourceMode() { + protected SyntheticSourceSupport syntheticSourceSupport() { if (hasNormalizer()) { - // NOTE: no matter if we have doc values or not we use a stored field to reconstruct the original value - // whose doc values would be altered by the normalizer - return SyntheticSourceMode.FALLBACK; + // NOTE: no matter if we have doc values or not we use fallback synthetic source + // to store the original value whose doc values would be altered by the normalizer + return SyntheticSourceSupport.FALLBACK; } + if (fieldType.stored() || hasDocValues) { - return SyntheticSourceMode.NATIVE; + return new SyntheticSourceSupport.Native(syntheticFieldLoader(leafName())); } - return SyntheticSourceMode.FALLBACK; - } - - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - return syntheticFieldLoader(leafName()); + return super.syntheticSourceSupport(); } public SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String simpleName) { - if (hasScript()) { - return SourceLoader.SyntheticFieldLoader.NOTHING; - } - if (builderParams.copyTo().copyToFields().isEmpty() != true) { - throw new IllegalArgumentException( - "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" - ); - } - - if (syntheticSourceMode() != SyntheticSourceMode.NATIVE) { - return super.syntheticFieldLoader(); - } + assert fieldType.stored() || hasDocValues; var layers = new ArrayList(); if (fieldType.stored()) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java index 359faf2d6f4b0..9cd0482d133b6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyTypeFieldMapper.java @@ -105,9 +105,4 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) protected String contentType() { return CONTENT_TYPE; } - - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - return SourceLoader.SyntheticFieldLoader.NOTHING; - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index 2e250726b98ca..92412177d35e5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -35,7 +35,8 @@ public Set getFeatures() { ObjectMapper.SUBOBJECTS_AUTO, KeywordFieldMapper.KEYWORD_NORMALIZER_SYNTHETIC_SOURCE, SourceFieldMapper.SYNTHETIC_SOURCE_STORED_FIELDS_ADVANCE_FIX, - Mapper.SYNTHETIC_SOURCE_KEEP_FEATURE + Mapper.SYNTHETIC_SOURCE_KEEP_FEATURE, + SourceFieldMapper.SYNTHETIC_SOURCE_WITH_COPY_TO_AND_DOC_VALUES_FALSE_SUPPORT ); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java index 850448c208e25..356e800105143 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java @@ -216,10 +216,7 @@ public void postParse(DocumentParserContext context) throws IOException { } @Override - protected SyntheticSourceMode syntheticSourceMode() { - return SyntheticSourceMode.NATIVE; + protected SyntheticSourceSupport syntheticSourceSupport() { + return new SyntheticSourceSupport.Native(SourceLoader.SyntheticFieldLoader.NOTHING); } - - @Override - public abstract SourceLoader.SyntheticFieldLoader syntheticFieldLoader(); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java index 6a0f4a87eb890..d315fcba3ca0c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedPathFieldMapper.java @@ -83,9 +83,4 @@ private NestedPathFieldMapper(String name) { protected String contentType() { return NAME; } - - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - return SourceLoader.SyntheticFieldLoader.NOTHING; - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index cfcc7cdea4b31..a895c9d52fccd 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -1984,29 +1984,12 @@ public void doValidate(MappingLookup lookup) { } @Override - protected SyntheticSourceMode syntheticSourceMode() { + protected SyntheticSourceSupport syntheticSourceSupport() { if (hasDocValues) { - return SyntheticSourceMode.NATIVE; + return new SyntheticSourceSupport.Native(type.syntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value())); } - return SyntheticSourceMode.FALLBACK; - } - - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - if (hasScript()) { - return SourceLoader.SyntheticFieldLoader.NOTHING; - } - if (builderParams.copyTo().copyToFields().isEmpty() != true) { - throw new IllegalArgumentException( - "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" - ); - } - if (hasDocValues) { - return type.syntheticFieldLoader(fullPath(), leafName(), ignoreMalformed.value()); - } - - return super.syntheticFieldLoader(); + return super.syntheticSourceSupport(); } // For testing only: diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index d7befbfacf4b1..1aabb2959f0ff 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -462,47 +462,34 @@ private static Range parseIpRangeFromCidr(final XContentParser parser) throws IO } @Override - protected SyntheticSourceMode syntheticSourceMode() { - return SyntheticSourceMode.NATIVE; - } + protected SyntheticSourceSupport syntheticSourceSupport() { + if (hasDocValues) { + var loader = new BinaryDocValuesSyntheticFieldLoader(fullPath()) { + @Override + protected void writeValue(XContentBuilder b, BytesRef value) throws IOException { + List ranges = type.decodeRanges(value); + + switch (ranges.size()) { + case 0: + return; + case 1: + b.field(leafName()); + ranges.get(0).toXContent(b, fieldType().dateTimeFormatter); + break; + default: + b.startArray(leafName()); + for (var range : ranges) { + range.toXContent(b, fieldType().dateTimeFormatter); + } + b.endArray(); + } + } + }; - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - if (hasDocValues == false) { - throw new IllegalArgumentException( - "field [" - + fullPath() - + "] of type [" - + typeName() - + "] doesn't support synthetic source because it doesn't have doc values" - ); - } - if (copyTo().copyToFields().isEmpty() != true) { - throw new IllegalArgumentException( - "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" - ); + return new SyntheticSourceSupport.Native(loader); } - return new BinaryDocValuesSyntheticFieldLoader(fullPath()) { - @Override - protected void writeValue(XContentBuilder b, BytesRef value) throws IOException { - List ranges = type.decodeRanges(value); - - switch (ranges.size()) { - case 0: - return; - case 1: - b.field(leafName()); - ranges.get(0).toXContent(b, fieldType().dateTimeFormatter); - break; - default: - b.startArray(leafName()); - for (var range : ranges) { - range.toXContent(b, fieldType().dateTimeFormatter); - } - b.endArray(); - } - } - }; + + return super.syntheticSourceSupport(); } /** Class defining a range */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java index 39686c3f30555..0b80e582f6f9d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java @@ -119,9 +119,4 @@ public void preParse(DocumentParserContext context) { protected String contentType() { return CONTENT_TYPE; } - - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - return SourceLoader.SyntheticFieldLoader.NOTHING; - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java index a46a310d0770f..3b97b8ec51735 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java @@ -244,9 +244,4 @@ public void postParse(DocumentParserContext context) { protected String contentType() { return CONTENT_TYPE; } - - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - return SourceLoader.SyntheticFieldLoader.NOTHING; - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java index 9dcb5142fa725..1de4e11a1d3b1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java @@ -43,6 +43,9 @@ public class SourceFieldMapper extends MetadataFieldMapper { public static final NodeFeature SYNTHETIC_SOURCE_STORED_FIELDS_ADVANCE_FIX = new NodeFeature( "mapper.source.synthetic_source_stored_fields_advance_fix" ); + public static final NodeFeature SYNTHETIC_SOURCE_WITH_COPY_TO_AND_DOC_VALUES_FALSE_SUPPORT = new NodeFeature( + "mapper.source.synthetic_source_with_copy_to_and_doc_values_false" + ); public static final String NAME = "_source"; public static final String RECOVERY_SOURCE_NAME = "_recovery_source"; @@ -463,9 +466,4 @@ public SourceLoader newSourceLoader(Mapping mapping, SourceFieldMetrics metrics) public boolean isSynthetic() { return mode == Mode.SYNTHETIC; } - - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - return SourceLoader.SyntheticFieldLoader.NOTHING; - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 99974d1ad0b2d..8c8396c7a9bb7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -79,7 +79,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.function.IntPredicate; @@ -1449,40 +1448,24 @@ protected void doXContentBody(XContentBuilder builder, Params params) throws IOE } @Override - protected SyntheticSourceMode syntheticSourceMode() { - return SyntheticSourceMode.NATIVE; - } - - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - if (copyTo().copyToFields().isEmpty() != true) { - throw new IllegalArgumentException( - "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" - ); - } + protected SyntheticSourceSupport syntheticSourceSupport() { if (store) { - return new StringStoredFieldFieldLoader(fullPath(), leafName()) { + var loader = new StringStoredFieldFieldLoader(fullPath(), leafName()) { @Override protected void write(XContentBuilder b, Object value) throws IOException { b.value((String) value); } }; + + return new SyntheticSourceSupport.Native(loader); } var kwd = SyntheticSourceHelper.getKeywordFieldMapperForSyntheticSource(this); if (kwd != null) { - return kwd.syntheticFieldLoader(leafName()); + return new SyntheticSourceSupport.Native(kwd.syntheticFieldLoader(leafName())); } - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "field [%s] of type [%s] doesn't support synthetic source unless it is stored or has a sub-field of" - + " type [keyword] with doc values or stored and without a normalizer", - fullPath(), - typeName() - ) - ); + return super.syntheticSourceSupport(); } public static class SyntheticSourceHelper { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index f1c6a072c2d9e..0b8177b2408a4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -158,11 +158,6 @@ protected String contentType() { return CONTENT_TYPE; } - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - return SourceLoader.SyntheticFieldLoader.NOTHING; - } - /** * Decode the {@code _tsid} into a human readable map. */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java index b9629d7561982..5b2b095c51339 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java @@ -128,11 +128,6 @@ protected String contentType() { return NAME; } - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - return SourceLoader.SyntheticFieldLoader.NOTHING; - } - public static String encode(int routingId) { byte[] bytes = new byte[4]; ByteUtils.writeIntLE(routingId, bytes, 0); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java index 1d4f56b02ed74..daeec320514fb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java @@ -101,9 +101,4 @@ public void postParse(DocumentParserContext context) { protected String contentType() { return CONTENT_TYPE; } - - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - return SourceLoader.SyntheticFieldLoader.NOTHING; - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 63f82ac9bfcdf..3507db5db606a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -53,7 +53,6 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperBuilderContext; -import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.StringFieldType; import org.elasticsearch.index.mapper.TextParams; @@ -809,21 +808,13 @@ public FieldMapper.Builder getMergeBuilder() { } @Override - protected SyntheticSourceMode syntheticSourceMode() { - return SyntheticSourceMode.NATIVE; - } - - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - if (hasScript()) { - return SourceLoader.SyntheticFieldLoader.NOTHING; - } + protected SyntheticSourceSupport syntheticSourceSupport() { if (fieldType().hasDocValues()) { - return new FlattenedSortedSetDocValuesSyntheticFieldLoader(fullPath(), fullPath() + "._keyed", leafName()); + var loader = new FlattenedSortedSetDocValuesSyntheticFieldLoader(fullPath(), fullPath() + "._keyed", leafName()); + + return new SyntheticSourceSupport.Native(loader); } - throw new IllegalArgumentException( - "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it doesn't have doc values" - ); + return super.syntheticSourceSupport(); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 34fc9f537ad0f..1afd2f718daff 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -2153,21 +2153,12 @@ public String toString() { } @Override - protected SyntheticSourceMode syntheticSourceMode() { - return SyntheticSourceMode.NATIVE; - } + protected SyntheticSourceSupport syntheticSourceSupport() { + var loader = fieldType().indexed + ? new IndexedSyntheticFieldLoader(indexCreatedVersion, fieldType().similarity) + : new DocValuesSyntheticFieldLoader(indexCreatedVersion); - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - if (copyTo().copyToFields().isEmpty() != true) { - throw new IllegalArgumentException( - "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" - ); - } - if (fieldType().indexed) { - return new IndexedSyntheticFieldLoader(indexCreatedVersion, fieldType().similarity); - } - return new DocValuesSyntheticFieldLoader(indexCreatedVersion); + return new SyntheticSourceSupport.Native(loader); } private class IndexedSyntheticFieldLoader extends SourceLoader.DocValuesBasedSyntheticFieldLoader { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java index 77b37b2bde860..0c5590ff07094 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java @@ -31,7 +31,6 @@ import java.util.stream.Collectors; import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; public class BinaryFieldMapperTests extends MapperTestCase { @@ -228,16 +227,7 @@ public SyntheticSourceExample example(int maxValues) throws IOException { @Override public List invalidExample() throws IOException { - return List.of( - new SyntheticSourceInvalidExample( - equalTo("field [field] of type [binary] doesn't support synthetic source because it doesn't have doc values"), - b -> b.field("type", "binary") - ), - new SyntheticSourceInvalidExample( - equalTo("field [field] of type [binary] doesn't support synthetic source because it doesn't have doc values"), - b -> b.field("type", "binary").field("doc_values", false) - ) - ); + return List.of(); } private Tuple generateValue() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index 03f030a26992d..86bf5404faf61 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -344,12 +344,7 @@ private void mapping(XContentBuilder b) throws IOException { @Override public List invalidExample() throws IOException { - return List.of( - new SyntheticSourceInvalidExample( - equalTo("field [field] of type [boolean] doesn't support synthetic source because it doesn't have doc values"), - b -> b.field("type", "boolean").field("doc_values", false) - ) - ); + return List.of(); } }; } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index 4efdd43b5cd71..aa28e8a3bd5b4 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -30,7 +30,6 @@ import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; -import java.util.ArrayList; import java.util.Comparator; import java.util.List; import java.util.function.Function; @@ -675,20 +674,7 @@ private void mapping(XContentBuilder b) throws IOException { @Override public List invalidExample() throws IOException { - List examples = new ArrayList<>(); - for (String fieldType : new String[] { "date", "date_nanos" }) { - examples.add( - new SyntheticSourceInvalidExample( - equalTo( - "field [field] of type [" - + fieldType - + "] doesn't support synthetic source because it doesn't have doc values" - ), - b -> b.field("type", fieldType).field("doc_values", false) - ) - ); - } - return examples; + return List.of(); } }; } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java index 07a1f732d818f..8c99e51d02cc7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java @@ -3244,14 +3244,16 @@ protected String contentType() { } @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - return new StringStoredFieldFieldLoader(fullPath(), leafName()) { + protected SyntheticSourceSupport syntheticSourceSupport() { + var loader = new StringStoredFieldFieldLoader(fullPath(), leafName()) { @Override protected void write(XContentBuilder b, Object value) throws IOException { BytesRef ref = (BytesRef) value; b.utf8Value(ref.bytes, ref.offset, ref.length); } }; + + return new SyntheticSourceSupport.Native(loader); } private static final TypeParser PARSER = new FixedTypeParser(c -> new MockMetadataMapper()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java index a389e803e66b6..621bc1cd0515c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java @@ -726,16 +726,7 @@ private void mapping(XContentBuilder b) throws IOException { @Override public List invalidExample() throws IOException { - return List.of( - new SyntheticSourceInvalidExample( - equalTo("field [field] of type [geo_point] doesn't support synthetic source because it doesn't have doc values"), - b -> b.field("type", "geo_point").field("doc_values", false) - ), - new SyntheticSourceInvalidExample( - equalTo("field [field] of type [geo_point] doesn't support synthetic source because it declares copy_to"), - b -> b.field("type", "geo_point").field("copy_to", "foo") - ) - ); + return List.of(); } }; } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index 61c4068cedf4a..a10b47cc06a73 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -221,6 +221,18 @@ public void testMultipleIgnoredFieldsManyObjects() throws IOException { ); } + public void testIgnoredDynamicArrayNestedInObject() throws IOException { + int intValue = randomInt(); + + String syntheticSource = getSyntheticSourceWithFieldLimit(b -> { + b.startObject("bar"); + b.field("a", List.of(intValue, intValue)); + b.endObject(); + }); + assertEquals(String.format(Locale.ROOT, """ + {"bar":{"a":[%s,%s]}}""", intValue, intValue), syntheticSource); + } + public void testDisabledRootObjectSingleField() throws IOException { String name = randomAlphaOfLength(20); DocumentMapper documentMapper = createMapperService(topMapping(b -> { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index 296871e258cd7..a38775e76c689 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -395,12 +395,7 @@ private void mapping(XContentBuilder b) throws IOException { @Override public List invalidExample() throws IOException { - return List.of( - new SyntheticSourceInvalidExample( - equalTo("field [field] of type [ip] doesn't support synthetic source because it doesn't have doc values"), - b -> b.field("type", "ip").field("doc_values", false) - ) - ); + return List.of(); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java index 0bfa04a95f1f5..b5755854a9d52 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java @@ -25,7 +25,6 @@ import java.util.Map; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; public class IpRangeFieldMapperTests extends RangeFieldMapperTests { @@ -202,19 +201,6 @@ private void inclusiveTo(Map output, String to) { } } - public void testInvalidSyntheticSource() { - Exception e = expectThrows(IllegalArgumentException.class, () -> createDocumentMapper(syntheticSourceMapping(b -> { - b.startObject("field"); - b.field("type", "ip_range"); - b.field("doc_values", false); - b.endObject(); - }))); - assertThat( - e.getMessage(), - equalTo("field [field] of type [ip_range] doesn't support synthetic source because it doesn't have doc values") - ); - } - @Override protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) { throw new AssumptionViolatedException("custom version of synthetic source tests is implemented"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java index cda594326464d..0275d7bfe0e37 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java @@ -25,7 +25,6 @@ import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import static org.elasticsearch.index.query.RangeQueryBuilder.GTE_FIELD; @@ -34,7 +33,6 @@ import static org.elasticsearch.index.query.RangeQueryBuilder.LT_FIELD; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.endsWith; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.startsWith; public abstract class RangeFieldMapperTests extends MapperTestCase { @@ -291,18 +289,7 @@ private void mapping(XContentBuilder b) throws IOException { @Override public List invalidExample() throws IOException { - return List.of( - new SyntheticSourceInvalidExample( - equalTo( - String.format( - Locale.ROOT, - "field [field] of type [%s] doesn't support synthetic source because it doesn't have doc values", - rangeType().name - ) - ), - b -> b.field("type", rangeType().name).field("doc_values", false) - ) - ); + return List.of(); } }; } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java index f569a69246d9f..769347985cfa7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMetricsTests.java @@ -60,10 +60,8 @@ public void testSyntheticSourceLoadLatency() throws IOException { } public void testSyntheticSourceIncompatibleMapping() throws IOException { - var mapping = syntheticSourceMapping(b -> b.startObject("kwd").field("type", "text").field("store", "false").endObject()); var mapperMetrics = createTestMapperMetrics(); - var mapperService = new TestMapperServiceBuilder().mapperMetrics(mapperMetrics).build(); - assertThrows(IllegalArgumentException.class, () -> withMapping(mapperService, mapping)); + mapperMetrics.sourceFieldMetrics().recordSyntheticSourceIncompatibleMapping(); var measurements = telemetryPlugin.getLongCounterMeasurement(SourceFieldMetrics.SYNTHETIC_SOURCE_INCOMPATIBLE_MAPPING); assertEquals(1, measurements.size()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java index 848f8878ffb98..a454ddeb721d6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceLoaderTests.java @@ -33,20 +33,6 @@ public void testEmptyObject() throws IOException { {"kwd":"foo"}""")); } - public void testUnsupported() throws IOException { - Exception e = expectThrows( - IllegalArgumentException.class, - () -> createDocumentMapper(syntheticSourceMapping(b -> b.startObject("txt").field("type", "text").endObject())) - ); - assertThat( - e.getMessage(), - equalTo( - "field [txt] of type [text] doesn't support synthetic source unless it is stored or has a sub-field " - + "of type [keyword] with doc values or stored and without a normalizer" - ) - ); - } - public void testDotsInFieldName() throws IOException { DocumentMapper mapper = createDocumentMapper( syntheticSourceMapping(b -> b.startObject("foo.bar.baz").field("type", "keyword").endObject()) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java index 7b7044f528c89..3a0952afaed5b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java @@ -749,12 +749,7 @@ public SyntheticSourceExample example(int maxValues) throws IOException { @Override public List invalidExample() throws IOException { - return List.of( - new SyntheticSourceInvalidExample( - equalTo("field [field] of type [flattened] doesn't support synthetic " + "source because it doesn't have doc values"), - b -> b.field("type", "flattened").field("doc_values", false) - ) - ); + return List.of(); } private void mapping(XContentBuilder b) throws IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 9eaace8f93e58..86aaa66b85bd5 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -82,7 +82,6 @@ import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.matchesPattern; import static org.hamcrest.Matchers.nullValue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -1507,17 +1506,6 @@ private void assertNoDocValueLoader(CheckedConsumer examples = new ArrayList<>(syntheticSourceSupport(ignoreMalformed).invalidExample()); - if (supportsCopyTo()) { - examples.add( - new SyntheticSourceInvalidExample( - matchesPattern("field \\[field] of type \\[.+] doesn't support synthetic source because it declares copy_to"), - b -> { - syntheticSourceSupport(ignoreMalformed).example(5).mapping().accept(b); - b.field("copy_to", "bar"); - } - ) - ); - } for (SyntheticSourceInvalidExample example : examples) { Exception e = expectThrows( IllegalArgumentException.class, diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TextFieldFamilySyntheticSourceTestSetup.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TextFieldFamilySyntheticSourceTestSetup.java index 953d71b9a791b..14050150b9f9f 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/TextFieldFamilySyntheticSourceTestSetup.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/TextFieldFamilySyntheticSourceTestSetup.java @@ -13,18 +13,15 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xcontent.XContentBuilder; -import org.hamcrest.Matcher; import java.io.IOException; import java.util.List; -import java.util.Locale; import java.util.function.Function; import static org.elasticsearch.test.ESTestCase.between; import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength; import static org.elasticsearch.test.ESTestCase.randomAlphaOfLengthBetween; import static org.elasticsearch.test.ESTestCase.randomBoolean; -import static org.hamcrest.Matchers.equalTo; /** * Provides functionality needed to test synthetic source support in text and text-like fields (e.g. "text", "annotated_text"). @@ -160,73 +157,7 @@ private String randomString() { @Override public List invalidExample() throws IOException { - Matcher err = equalTo( - String.format( - Locale.ROOT, - "field [field] of type [%s] doesn't support synthetic source unless it is stored or" - + " has a sub-field of type [keyword] with doc values or stored and without a normalizer", - fieldType - ) - ); - return List.of( - new MapperTestCase.SyntheticSourceInvalidExample(err, b -> b.field("type", fieldType)), - new MapperTestCase.SyntheticSourceInvalidExample(err, b -> { - b.field("type", fieldType); - b.startObject("fields"); - { - b.startObject("l"); - b.field("type", "long"); - b.endObject(); - } - b.endObject(); - }), - new MapperTestCase.SyntheticSourceInvalidExample(err, b -> { - b.field("type", fieldType); - b.startObject("fields"); - { - b.startObject("kwd"); - b.field("type", "keyword"); - b.field("normalizer", "lowercase"); - b.endObject(); - } - b.endObject(); - }), - new MapperTestCase.SyntheticSourceInvalidExample(err, b -> { - b.field("type", fieldType); - b.startObject("fields"); - { - b.startObject("kwd"); - b.field("type", "keyword"); - b.field("doc_values", "false"); - b.endObject(); - } - b.endObject(); - }), - new MapperTestCase.SyntheticSourceInvalidExample(err, b -> { - b.field("type", fieldType); - b.field("store", "false"); - b.startObject("fields"); - { - b.startObject("kwd"); - b.field("type", "keyword"); - b.field("doc_values", "false"); - b.endObject(); - } - b.endObject(); - }), - new MapperTestCase.SyntheticSourceInvalidExample(err, b -> { - b.field("type", fieldType); - b.startObject("fields"); - { - b.startObject("kwd"); - b.field("type", "keyword"); - b.field("doc_values", "false"); - b.field("store", "false"); - b.endObject(); - } - b.endObject(); - }) - ); + return List.of(); } } } diff --git a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java index e57257f69da20..5babaec0cf43a 100644 --- a/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java @@ -25,10 +25,15 @@ public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceReques }); } - // TODO enable doc_values: false - // It is disabled because it hits a bug in synthetic source. private Supplier> keywordMapping() { - return () -> Map.of("store", ESTestCase.randomBoolean(), "index", ESTestCase.randomBoolean()); + return () -> Map.of( + "store", + ESTestCase.randomBoolean(), + "index", + ESTestCase.randomBoolean(), + "doc_values", + ESTestCase.randomBoolean() + ); } private Supplier> numberMapping() { @@ -43,13 +48,29 @@ private Supplier> numberMapping() { } private Supplier> unsignedLongMapping() { - return () -> Map.of("store", ESTestCase.randomBoolean(), "index", ESTestCase.randomBoolean()); + return () -> Map.of( + "store", + ESTestCase.randomBoolean(), + "index", + ESTestCase.randomBoolean(), + "doc_values", + ESTestCase.randomBoolean() + ); } private Supplier> scaledFloatMapping() { return () -> { var scalingFactor = ESTestCase.randomFrom(10, 1000, 100000, 100.5); - return Map.of("scaling_factor", scalingFactor, "store", ESTestCase.randomBoolean(), "index", ESTestCase.randomBoolean()); + return Map.of( + "scaling_factor", + scalingFactor, + "store", + ESTestCase.randomBoolean(), + "index", + ESTestCase.randomBoolean(), + "doc_values", + ESTestCase.randomBoolean() + ); }; } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index cbb75ee1ebe28..0940b3ef96ddd 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -500,24 +500,15 @@ public long count() { } @Override - protected SyntheticSourceMode syntheticSourceMode() { - return SyntheticSourceMode.NATIVE; - } - - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - if (copyTo().copyToFields().isEmpty() != true) { - throw new IllegalArgumentException( - "field [" + fullPath() + "] of type [histogram] doesn't support synthetic source because it declares copy_to" - ); - } - - return new CompositeSyntheticFieldLoader( + protected SyntheticSourceSupport syntheticSourceSupport() { + var loader = new CompositeSyntheticFieldLoader( leafName(), fullPath(), new HistogramSyntheticFieldLoader(), new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath()) ); + + return new SyntheticSourceSupport.Native(loader); } private class HistogramSyntheticFieldLoader implements CompositeSyntheticFieldLoader.DocValuesLayer { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java index a0aaf7f3bfeb5..527f8d1c176ec 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldMapper.java @@ -17,7 +17,6 @@ import org.elasticsearch.index.mapper.ConstantFieldType; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MetadataFieldMapper; -import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.SearchExecutionContext; @@ -108,9 +107,4 @@ public DataTierFieldMapper() { protected String contentType() { return CONTENT_TYPE; } - - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - return SourceLoader.SyntheticFieldLoader.NOTHING; - } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/AddBlock.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/AddPage.java similarity index 82% rename from x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/AddBlock.java rename to x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/AddPage.java index 496624fc1189d..4e051c73a3643 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/AddBlock.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/AddPage.java @@ -22,13 +22,17 @@ * for how to add values to it. After adding all values, call {@link #emitOrds} to * flush the last batch of values to the aggs. */ -public class AddBlock implements Releasable { +public class AddPage implements Releasable { private final BlockFactory blockFactory; - private final int emitBatchSize; + private final long emitBatchSize; private final GroupingAggregatorFunction.AddInput addInput; private int positionOffset = 0; - private int added = 0; + /** + * Number of added documents. This is a {@code long} because callers will + * often perform the combinatorial explosion of values. + */ + private long added = 0; private IntBlock.Builder ords; /** * State of the current position. @@ -50,7 +54,7 @@ public class AddBlock implements Releasable { */ private int firstOrd = -1; - public AddBlock(BlockFactory blockFactory, int emitBatchSize, GroupingAggregatorFunction.AddInput addInput) { + public AddPage(BlockFactory blockFactory, int emitBatchSize, GroupingAggregatorFunction.AddInput addInput) { this.blockFactory = blockFactory; this.emitBatchSize = emitBatchSize; this.addInput = addInput; @@ -58,6 +62,10 @@ public AddBlock(BlockFactory blockFactory, int emitBatchSize, GroupingAggregator this.ords = blockFactory.newIntBlockBuilder(emitBatchSize); } + long added() { + return added; + } + /** * Append a single valued ordinal. This will flush the ordinals to the aggs * if we've added {@link #emitBatchSize}. @@ -65,7 +73,7 @@ public AddBlock(BlockFactory blockFactory, int emitBatchSize, GroupingAggregator protected final void appendOrdSv(int position, int ord) { assert firstOrd == -1 : "currently in a multivalue position"; ords.appendInt(ord); - if (++added % emitBatchSize == 0) { + if (++added % emitBatchSize == 0L) { rollover(position + 1); } } @@ -78,7 +86,7 @@ protected final void appendOrdSv(int position, int ord) { @Deprecated protected final void appendNullSv(int position) { ords.appendNull(); - if (++added % emitBatchSize == 0) { + if (++added % emitBatchSize == 0L) { rollover(position + 1); } } @@ -95,7 +103,7 @@ protected final void appendNullSv(int position) { * } */ protected final void appendOrdInMv(int position, int ord) { - if (++added % emitBatchSize == 0) { + if (++added % emitBatchSize == 0L) { switch (firstOrd) { case -1 -> ords.appendInt(ord); case -2 -> { @@ -136,7 +144,20 @@ protected final void finishMv() { firstOrd = -1; } - protected final void emitOrds() { + /** + * Call when finished to emit all remaining ordinals to the aggs. + */ + protected final void flushRemaining() { + if (firstOrd != -1) { + throw new IllegalStateException("in the middle of a position"); + } + if (added % emitBatchSize != 0) { + // If the % is 0 then we just flushed and there isn't any need to flush an empty block. + emitOrds(); + } + } + + private void emitOrds() { try (IntBlock ordsBlock = ords.build()) { addInput.add(positionOffset, ordsBlock); } @@ -145,7 +166,7 @@ protected final void emitOrds() { private void rollover(int position) { emitOrds(); positionOffset = position; - ords = blockFactory.newIntBlockBuilder(emitBatchSize); // TODO add a clear method to the builder? + ords = blockFactory.newIntBlockBuilder(Math.toIntExact(emitBatchSize)); } @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BytesRef3BlockHash.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BytesRef3BlockHash.java index 07600cd7a1dc9..54bf068b4de33 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BytesRef3BlockHash.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/BytesRef3BlockHash.java @@ -98,7 +98,7 @@ private void addVectors(BytesRefVector v1, BytesRefVector v2, BytesRefVector v3, } } - private class AddWork extends AddBlock { + private class AddWork extends AddPage { final IntBlock b1; final IntBlock b2; final IntBlock b3; @@ -137,7 +137,7 @@ void add() { } finishMv(); } - emitOrds(); + flushRemaining(); } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/PackedValuesBlockHash.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/PackedValuesBlockHash.java index 85b3cec274e39..6eb3aef6dfc8b 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/PackedValuesBlockHash.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/blockhash/PackedValuesBlockHash.java @@ -117,7 +117,7 @@ public void close() { } } - class AddWork extends AddBlock { + class AddWork extends AddPage { final Group[] groups; final int positionCount; int position; @@ -142,7 +142,7 @@ void add() { addMultipleEntries(); } } - emitOrds(); + flushRemaining(); } private void addSingleEntry() { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/mvdedupe/IntLongBlockAdd.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/mvdedupe/IntLongBlockAdd.java index ecd7df1a7e6c4..eb0351c095795 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/mvdedupe/IntLongBlockAdd.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/mvdedupe/IntLongBlockAdd.java @@ -9,13 +9,13 @@ import org.elasticsearch.common.util.LongLongHash; import org.elasticsearch.compute.aggregation.GroupingAggregatorFunction; -import org.elasticsearch.compute.aggregation.blockhash.AddBlock; +import org.elasticsearch.compute.aggregation.blockhash.AddPage; import org.elasticsearch.compute.aggregation.blockhash.BlockHash; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.LongBlock; -public class IntLongBlockAdd extends AddBlock { +public class IntLongBlockAdd extends AddPage { private final LongLongHash hash; private final MultivalueDedupeInt block1; private final MultivalueDedupeLong block2; @@ -39,7 +39,7 @@ public void add() { for (int p = 0; p < positions; p++) { add1(p); } - emitOrds(); + flushRemaining(); } private void add1(int position) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/mvdedupe/LongLongBlockAdd.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/mvdedupe/LongLongBlockAdd.java index 1f117498b078e..b59a43f0f93a9 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/mvdedupe/LongLongBlockAdd.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/mvdedupe/LongLongBlockAdd.java @@ -9,12 +9,12 @@ import org.elasticsearch.common.util.LongLongHash; import org.elasticsearch.compute.aggregation.GroupingAggregatorFunction; -import org.elasticsearch.compute.aggregation.blockhash.AddBlock; +import org.elasticsearch.compute.aggregation.blockhash.AddPage; import org.elasticsearch.compute.aggregation.blockhash.BlockHash; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.LongBlock; -public class LongLongBlockAdd extends AddBlock { +public class LongLongBlockAdd extends AddPage { private final LongLongHash hash; private final MultivalueDedupeLong block1; private final MultivalueDedupeLong block2; @@ -38,7 +38,7 @@ public void add() { for (int p = 0; p < positions; p++) { add1(p); } - emitOrds(); + flushRemaining(); } private void add1(int position) { diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/AddBlockTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/AddPageTests.java similarity index 67% rename from x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/AddBlockTests.java rename to x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/AddPageTests.java index da9529cb761ef..810402d82c9d1 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/AddBlockTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/blockhash/AddPageTests.java @@ -19,23 +19,25 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Locale; import static org.hamcrest.Matchers.equalTo; -public class AddBlockTests extends ESTestCase { +public class AddPageTests extends ESTestCase { private final BlockFactory blockFactory = BlockFactoryTests.blockFactory(ByteSizeValue.ofGb(1)); public void testSv() { TestAddInput result = new TestAddInput(); List expected = new ArrayList<>(); - try (AddBlock add = new AddBlock(blockFactory, 3, result)) { + try (AddPage add = new AddPage(blockFactory, 3, result)) { add.appendOrdSv(0, 0); add.appendOrdSv(1, 2); add.appendOrdSv(2, 3); expected.add(added(0, 0, 2, 3)); assertThat(result.added, equalTo(expected)); add.appendOrdSv(3, 4); - add.emitOrds(); + add.flushRemaining(); + assertThat(add.added(), equalTo(4L)); } expected.add(added(3, 4)); assertThat(result.added, equalTo(expected)); @@ -45,7 +47,7 @@ public void testSv() { public void testMvBlockEndsOnBatchBoundary() { TestAddInput result = new TestAddInput(); List expected = new ArrayList<>(); - try (AddBlock add = new AddBlock(blockFactory, 3, result)) { + try (AddPage add = new AddPage(blockFactory, 3, result)) { add.appendOrdInMv(0, 0); add.appendOrdInMv(0, 2); add.appendOrdInMv(0, 3); @@ -58,10 +60,14 @@ public void testMvBlockEndsOnBatchBoundary() { expected.add(new Added(0, List.of(List.of(4), List.of(0, 2)))); assertThat(result.added, equalTo(expected)); add.finishMv(); - add.emitOrds(); + add.flushRemaining(); + assertThat(add.added(), equalTo(6L)); } - // We uselessly flush an empty position if emitBatchSize lines up with the total count - expected.add(new Added(1, List.of(List.of()))); + /* + * We do *not* uselessly flush an empty Block of ordinals. Doing so would + * be a slight performance hit, but, worse, makes testing harder to reason + * about. + */ assertThat(result.added, equalTo(expected)); assertThat(result.closed, equalTo(true)); } @@ -69,7 +75,7 @@ public void testMvBlockEndsOnBatchBoundary() { public void testMvPositionEndOnBatchBoundary() { TestAddInput result = new TestAddInput(); List expected = new ArrayList<>(); - try (AddBlock add = new AddBlock(blockFactory, 4, result)) { + try (AddPage add = new AddPage(blockFactory, 4, result)) { add.appendOrdInMv(0, 0); add.appendOrdInMv(0, 2); add.appendOrdInMv(0, 3); @@ -80,7 +86,8 @@ public void testMvPositionEndOnBatchBoundary() { add.appendOrdInMv(1, 0); add.appendOrdInMv(1, 2); add.finishMv(); - add.emitOrds(); + add.flushRemaining(); + assertThat(add.added(), equalTo(6L)); } // Because the first position ended on a block boundary we uselessly emit an empty position there expected.add(new Added(0, List.of(List.of(), List.of(0, 2)))); @@ -91,7 +98,7 @@ public void testMvPositionEndOnBatchBoundary() { public void testMv() { TestAddInput result = new TestAddInput(); List expected = new ArrayList<>(); - try (AddBlock add = new AddBlock(blockFactory, 5, result)) { + try (AddPage add = new AddPage(blockFactory, 5, result)) { add.appendOrdInMv(0, 0); add.appendOrdInMv(0, 2); add.appendOrdInMv(0, 3); @@ -102,13 +109,43 @@ public void testMv() { assertThat(result.added, equalTo(expected)); add.appendOrdInMv(1, 2); add.finishMv(); - add.emitOrds(); + add.flushRemaining(); + assertThat(add.added(), equalTo(6L)); } expected.add(new Added(1, List.of(List.of(2)))); assertThat(result.added, equalTo(expected)); assertThat(result.closed, equalTo(true)); } + /** + * Test that we can add more than {@link Integer#MAX_VALUE} values. That's + * more than two billion values. We've made the call as fast as we can. + * Locally this test takes about 40 seconds for Nik. + */ + public void testMvBillions() { + CountingAddInput counter = new CountingAddInput(); + try (AddPage add = new AddPage(blockFactory, 5, counter)) { + for (int i = 0; i < Integer.MAX_VALUE; i++) { + add.appendOrdInMv(0, 0); + assertThat(add.added(), equalTo((long) i + 1)); + if (i % 5 == 0) { + assertThat(counter.count, equalTo(i / 5)); + } + if (i % 10_000_000 == 0) { + logger.info(String.format(Locale.ROOT, "Progress: %02.0f%%", 100 * ((double) i / Integer.MAX_VALUE))); + } + } + add.finishMv(); + add.appendOrdInMv(1, 0); + assertThat(add.added(), equalTo(Integer.MAX_VALUE + 1L)); + add.appendOrdInMv(1, 0); + assertThat(add.added(), equalTo(Integer.MAX_VALUE + 2L)); + add.finishMv(); + add.flushRemaining(); + assertThat(counter.count, equalTo(Integer.MAX_VALUE / 5 + 1)); + } + } + @After public void breakerClear() { assertThat(blockFactory.breaker().getUsed(), equalTo(0L)); @@ -151,4 +188,21 @@ public void close() { closed = true; } } + + private class CountingAddInput implements GroupingAggregatorFunction.AddInput { + private int count; + + @Override + public void add(int positionOffset, IntBlock groupIds) { + count++; + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + count++; + } + + @Override + public void close() {} + } } diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java index 8bbc6958c8c75..b1f38191ba7b3 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java @@ -37,7 +37,6 @@ import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.SimpleMappedFieldType; import org.elasticsearch.index.mapper.SortedNumericDocValuesSyntheticFieldLoader; -import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.TimeSeriesParams; @@ -711,18 +710,15 @@ public FieldMapper.Builder getMergeBuilder() { } @Override - protected SyntheticSourceMode syntheticSourceMode() { - return SyntheticSourceMode.NATIVE; - } - - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - return new CompositeSyntheticFieldLoader( + protected SyntheticSourceSupport syntheticSourceSupport() { + var loader = new CompositeSyntheticFieldLoader( leafName(), fullPath(), new AggregateMetricSyntheticFieldLoader(fullPath(), metrics), new CompositeSyntheticFieldLoader.MalformedValuesLayer(fullPath()) ); + + return new SyntheticSourceSupport.Native(loader); } public static class AggregateMetricSyntheticFieldLoader implements CompositeSyntheticFieldLoader.DocValuesLayer { diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index f5a7c35f75eca..c594c9f553164 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -345,18 +345,14 @@ protected String contentType() { } @Override - protected SyntheticSourceMode syntheticSourceMode() { - return SyntheticSourceMode.NATIVE; - } - - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + protected SyntheticSourceSupport syntheticSourceSupport() { String value = fieldType().value(); - ; + if (value == null) { - return SourceLoader.SyntheticFieldLoader.NOTHING; + return new SyntheticSourceSupport.Native(SourceLoader.SyntheticFieldLoader.NOTHING); } - return new SourceLoader.SyntheticFieldLoader() { + + var loader = new SourceLoader.SyntheticFieldLoader() { @Override public Stream> storedFieldLoaders() { return Stream.of(); @@ -389,5 +385,7 @@ public String fieldName() { return fullPath(); } }; + + return new SyntheticSourceSupport.Native(loader); } } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index b350607e66b10..5b04225cee105 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -36,7 +36,6 @@ import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.SimpleMappedFieldType; import org.elasticsearch.index.mapper.SortedNumericDocValuesSyntheticFieldLoader; -import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.TimeSeriesParams; @@ -753,31 +752,18 @@ public void doValidate(MappingLookup lookup) { } @Override - protected SyntheticSourceMode syntheticSourceMode() { - return SyntheticSourceMode.NATIVE; - } + protected SyntheticSourceSupport syntheticSourceSupport() { + if (hasDocValues) { + var loader = new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed()) { + @Override + protected void writeValue(XContentBuilder b, long value) throws IOException { + b.value(DocValueFormat.UNSIGNED_LONG_SHIFTED.format(value)); + } + }; - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - if (hasDocValues == false) { - throw new IllegalArgumentException( - "field [" - + fullPath() - + "] of type [" - + typeName() - + "] doesn't support synthetic source because it doesn't have doc values" - ); - } - if (copyTo().copyToFields().isEmpty() != true) { - throw new IllegalArgumentException( - "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" - ); + return new SyntheticSourceSupport.Native(loader); } - return new SortedNumericDocValuesSyntheticFieldLoader(fullPath(), leafName(), ignoreMalformed()) { - @Override - protected void writeValue(XContentBuilder b, long value) throws IOException { - b.value(DocValueFormat.UNSIGNED_LONG_SHIFTED.format(value)); - } - }; + + return super.syntheticSourceSupport(); } } diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java index 46969d8dbb2ed..cdb25fbe995b2 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java @@ -42,7 +42,6 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.matchesPattern; public class UnsignedLongFieldMapperTests extends WholeNumberFieldMapperTests { @@ -504,15 +503,7 @@ private void mapping(XContentBuilder b) throws IOException { @Override public List invalidExample() { - return List.of( - new SyntheticSourceInvalidExample( - matchesPattern("field \\[field] of type \\[.+] doesn't support synthetic source because it doesn't have doc values"), - b -> { - minimalMapping(b); - b.field("doc_values", false); - } - ) - ); + return List.of(); } } } diff --git a/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/80_synthetic_source.yml b/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/80_synthetic_source.yml new file mode 100644 index 0000000000000..b88fca3c478a9 --- /dev/null +++ b/x-pack/plugin/mapper-unsigned-long/src/yamlRestTest/resources/rest-api-spec/test/80_synthetic_source.yml @@ -0,0 +1,152 @@ +synthetic source: + - requires: + cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"] + reason: requires synthetic source support + + - do: + indices.create: + index: synthetic_source_test + body: + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + ulong: + type: unsigned_long + ignore_malformed: true + + - do: + bulk: + index: synthetic_source_test + refresh: true + body: | + { "index": {"_id" : "1"} } + { "name": "A", "ulong": "120" } + { "index": {"_id" : "2"} } + { "name": "B", "ulong": "hello" } + { "index": {"_id" : "3"} } + { "name": "C", "ulong": [6, 5, 4] } + + - do: + search: + index: synthetic_source_test + sort: name + + - match: { "hits.total.value": 3 } + - match: + hits.hits.0._source.ulong: 120 + - match: + hits.hits.1._source.ulong: "hello" + - match: + hits.hits.2._source.ulong: [4, 5, 6] + +--- +synthetic source with copy_to: + - requires: + cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: synthetic_source_test + body: + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + ulong: + type: unsigned_long + ignore_malformed: true + copy_to: copy + copy: + type: keyword + + - do: + bulk: + index: synthetic_source_test + refresh: true + body: | + { "index": {"_id" : "1"} } + { "name": "A", "ulong": "120" } + { "index": {"_id" : "2"} } + { "name": "B", "ulong": "hello" } + { "index": {"_id" : "3"} } + { "name": "C", "ulong": [6, 5, 4] } + + - do: + search: + index: synthetic_source_test + sort: name + body: + docvalue_fields: ["copy"] + + - match: { "hits.total.value": 3 } + + - match: + hits.hits.0._source.ulong: "120" + - match: + hits.hits.0.fields.copy.0: "120" + + - match: + hits.hits.1._source.ulong: "hello" + - match: + hits.hits.1.fields.copy.0: "hello" + + - match: + hits.hits.2._source.ulong: [6, 5, 4] + - match: + hits.hits.2.fields.copy: ["4", "5", "6"] + +--- +synthetic source with disabled doc_values: + - requires: + cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"] + reason: requires disabled doc_values support in synthetic source + + - do: + indices.create: + index: synthetic_source_test + body: + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + ulong: + type: unsigned_long + ignore_malformed: true + doc_values: false + + - do: + bulk: + index: synthetic_source_test + refresh: true + body: | + { "index": {"_id" : "1"} } + { "name": "A", "ulong": "120" } + { "index": {"_id" : "2"} } + { "name": "B", "ulong": "hello" } + { "index": {"_id" : "3"} } + { "name": "C", "ulong": [6, 5, 4] } + + - do: + search: + index: synthetic_source_test + sort: name + + - match: { "hits.total.value": 3 } + + - match: + hits.hits.0._source.ulong: "120" + + - match: + hits.hits.1._source.ulong: "hello" + + - match: + hits.hits.2._source.ulong: [6, 5, 4] + diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index a8fe0bf4e3c0a..b49b4500ce7b7 100644 --- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -49,7 +49,6 @@ import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.SearchAfterTermsEnum; import org.elasticsearch.index.mapper.SortedSetDocValuesSyntheticFieldLoaderLayer; -import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.TermBasedFieldType; import org.elasticsearch.index.mapper.TextSearchInfo; @@ -435,18 +434,8 @@ public FieldMapper.Builder getMergeBuilder() { } @Override - protected SyntheticSourceMode syntheticSourceMode() { - return SyntheticSourceMode.NATIVE; - } - - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - if (copyTo().copyToFields().isEmpty() != true) { - throw new IllegalArgumentException( - "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" - ); - } - return new CompositeSyntheticFieldLoader(leafName(), fullPath(), new SortedSetDocValuesSyntheticFieldLoaderLayer(fullPath()) { + protected SyntheticSourceSupport syntheticSourceSupport() { + var loader = new CompositeSyntheticFieldLoader(leafName(), fullPath(), new SortedSetDocValuesSyntheticFieldLoaderLayer(fullPath()) { @Override protected BytesRef convert(BytesRef value) { return VersionEncoder.decodeVersion(value); @@ -458,5 +447,7 @@ protected BytesRef preserve(BytesRef value) { return value; } }); + + return new SyntheticSourceSupport.Native(loader); } } diff --git a/x-pack/plugin/mapper-version/src/yamlRestTest/resources/rest-api-spec/test/40_synthetic_source.yml b/x-pack/plugin/mapper-version/src/yamlRestTest/resources/rest-api-spec/test/40_synthetic_source.yml index 4ff2499ad7e72..1ec91f5fde8d1 100644 --- a/x-pack/plugin/mapper-version/src/yamlRestTest/resources/rest-api-spec/test/40_synthetic_source.yml +++ b/x-pack/plugin/mapper-version/src/yamlRestTest/resources/rest-api-spec/test/40_synthetic_source.yml @@ -65,3 +65,43 @@ script values: - match: { hits.hits.1.fields.field.0: "1.2.3.4.5" } - match: { hits.hits.2.fields.field.0: "1.2.3-abc+def" } - match: { hits.hits.3.fields.field.0: "1.0.0" } + +--- +synthetic source with copy_to: + - requires: + cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: synthetic_source_test + body: + mappings: + _source: + mode: synthetic + properties: + ver: + type: version + copy_to: copy + copy: + type: keyword + + - do: + bulk: + index: synthetic_source_test + refresh: true + body: | + { "index": {"_id" : "1"} } + { "ver": "1.2.3-abc+def" } + + - do: + search: + index: synthetic_source_test + body: + fields: ["copy"] + + - match: { "hits.total.value": 1 } + - match: + hits.hits.0._source.ver: "1.2.3-abc+def" + - match: + hits.hits.0.fields.copy.0: "1.2.3-abc+def" diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index e245d10e35dc8..8e4f56e299587 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -68,7 +68,6 @@ import org.elasticsearch.index.mapper.LuceneDocument; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; -import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.ValueFetcher; @@ -988,35 +987,21 @@ public FieldMapper.Builder getMergeBuilder() { } @Override - protected SyntheticSourceMode syntheticSourceMode() { - return SyntheticSourceMode.NATIVE; - } - - @Override - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - if (copyTo().copyToFields().isEmpty() != true) { - throw new IllegalArgumentException( - "field [" + fullPath() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" - ); - } - - var loader = new WildcardSyntheticFieldLoader(); + protected SyntheticSourceSupport syntheticSourceSupport() { + var layers = new ArrayList(); + layers.add(new WildcardSyntheticFieldLoader()); if (ignoreAbove != Defaults.IGNORE_ABOVE) { - return new CompositeSyntheticFieldLoader( - leafName(), - fullPath(), - loader, - new CompositeSyntheticFieldLoader.StoredFieldLayer(originalName()) { - @Override - protected void writeValue(Object value, XContentBuilder b) throws IOException { - BytesRef r = (BytesRef) value; - b.utf8Value(r.bytes, r.offset, r.length); - } + layers.add(new CompositeSyntheticFieldLoader.StoredFieldLayer(originalName()) { + @Override + protected void writeValue(Object value, XContentBuilder b) throws IOException { + BytesRef r = (BytesRef) value; + b.utf8Value(r.bytes, r.offset, r.length); } - ); + }); } - return new CompositeSyntheticFieldLoader(leafName(), fullPath(), loader); + var loader = new CompositeSyntheticFieldLoader(leafName(), fullPath(), layers); + return new SyntheticSourceSupport.Native(loader); } private class WildcardSyntheticFieldLoader implements CompositeSyntheticFieldLoader.DocValuesLayer { diff --git a/x-pack/plugin/wildcard/src/yamlRestTest/resources/rest-api-spec/test/30_synthetic_source.yml b/x-pack/plugin/wildcard/src/yamlRestTest/resources/rest-api-spec/test/30_synthetic_source.yml new file mode 100644 index 0000000000000..ffa76f7433985 --- /dev/null +++ b/x-pack/plugin/wildcard/src/yamlRestTest/resources/rest-api-spec/test/30_synthetic_source.yml @@ -0,0 +1,90 @@ +synthetic source: + - requires: + cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"] + reason: requires synthetic source support + + - do: + indices.create: + index: synthetic_source_test + body: + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + wildcard: + type: wildcard + ignore_above: 6 + + - do: + bulk: + index: synthetic_source_test + refresh: true + body: | + { "index": {"_id" : "1"} } + { "name": "A", "wildcard": "hello" } + { "index": {"_id" : "2"} } + { "name": "B", "wildcard": "long_hello" } + + - do: + search: + index: synthetic_source_test + sort: name + + - match: { "hits.total.value": 2 } + - match: + hits.hits.0._source.wildcard: "hello" + - match: + hits.hits.1._source.wildcard: "long_hello" + +--- +synthetic source with copy_to: + - requires: + cluster_features: ["mapper.source.synthetic_source_with_copy_to_and_doc_values_false"] + reason: requires copy_to support in synthetic source + + - do: + indices.create: + index: synthetic_source_test + body: + mappings: + _source: + mode: synthetic + properties: + name: + type: keyword + wildcard: + type: wildcard + copy_to: copy + copy: + type: keyword + + - do: + bulk: + index: synthetic_source_test + refresh: true + body: | + { "index": {"_id" : "1"} } + { "name": "A", "wildcard": "hello" } + { "index": {"_id" : "2"} } + { "name": "B", "wildcard": "long_hello" } + + - do: + search: + index: synthetic_source_test + sort: name + body: + fields: ["copy"] + + - match: { "hits.total.value": 2 } + + - match: + hits.hits.0._source.wildcard: "hello" + - match: + hits.hits.0.fields.copy.0: "hello" + + - match: + hits.hits.1._source.wildcard: "long_hello" + - match: + hits.hits.1.fields.copy.0: "long_hello"