Give precedence to index creation when mixing typed templates with ty…

…peless index creation and vice-versa. (#38055)

This is a backport of #37871 and #38032.

docs/reference/mapping/removal_of_types.asciidoc

@@ -424,3 +424,76 @@ POST _reindex
 ----
 // NOTCONSOLE
 
+[float]
+=== Index templates
+
+It is recommended to make index templates typeless before upgrading to 7.0 by
+re-adding them with `include_type_name` set to `false`.
+
+In case typeless templates are used with typed index creation calls or typed
+templates are used with typeless index creation calls, the template will still
+be applied but the index creation call decides whether there should be a type
+or not. For instance in the below example, `index-1-01` will have a type in
+spite of the fact that it matches a template that is typeless, and `index-2-01`
+will be typeless in spite of the fact that it matches a template that defines
+a type. Both `index-1-01` and `index-2-01` will inherit the `foo` field from
+the template that they match.
+
+[source,js]
+--------------------------------------------------
+PUT _template/template1?include_type_name=false
+{
+  "index_patterns":[ "index-1-*" ],
+  "mappings": {
+    "properties": {
+      "foo": {
+        "type": "keyword"
+      }
+    }
+  }
+}
+
+PUT _template/template2?include_type_name=true
+{
+  "index_patterns":[ "index-2-*" ],
+  "mappings": {
+    "type": {
+      "properties": {
+        "foo": {
+          "type": "keyword"
+        }
+      }
+    }
+  }
+}
+
+PUT index-1-01?include_type_name=true
+{
+  "mappings": {
+    "type": {
+      "properties": {
+        "bar": {
+          "type": "long"
+        }
+      }
+    }
+  }
+}
+
+PUT index-2-01?include_type_name=false
+{
+  "mappings": {
+    "properties": {
+      "bar": {
+        "type": "long"
+      }
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+
+In case of implicit index creation, because of documents that get indexed in
+an index that doesn't exist yet, the template is always honored. This is
+usually not a problem due to the fact that typless index calls work on typed
+indices.

rest-api-spec/src/main/resources/rest-api-spec/test/indices.create/20_mix_typeless_typeful.yml

@@ -0,0 +1,129 @@
+---
+"Create a typeless index while there is a typed template":
+
+ - skip:
+      version: " - 6.6.99"
+      reason: Merging typeless/typed mappings/templates was added in 6.7
+
+ - do:
+      indices.put_template:
+          include_type_name: true
+          name: test_template
+          body:
+              index_patterns: test-*
+              mappings:
+                  my_type:
+                      properties:
+                          foo:
+                              type: keyword
+
+ - do:
+      indices.create:
+          include_type_name: false
+          index: test-1
+          body:
+              mappings:
+                  properties:
+                      bar:
+                          type: "long"
+
+ - do:
+      indices.get_mapping:
+          include_type_name: true
+          index: test-1
+
+ - is_true: test-1.mappings._doc # the index creation call won
+ - is_false: test-1.mappings.my_type
+ - is_true: test-1.mappings._doc.properties.foo
+ - is_true: test-1.mappings._doc.properties.bar
+
+---
+"Create a typed index while there is a typeless template":
+
+ - skip:
+      version: " - 6.6.99"
+      reason: Merging typeless/typed mappings/templates was added in 6.7
+
+ - do:
+      indices.put_template:
+          include_type_name: false
+          name: test_template
+          body:
+              index_patterns: test-*
+              mappings:
+                  properties:
+                      foo:
+                          type: keyword
+
+ - do:
+      indices.create: 
+          include_type_name: true
+          index: test-1
+          body:
+              mappings:
+                  my_type:
+                      properties:
+                          bar:
+                              type: "long"
+
+ - do:
+      indices.get_mapping:
+          include_type_name: true
+          index: test-1
+
+ - is_true: test-1.mappings.my_type # the index creation call won
+ - is_false: test-1.mappings._doc
+ - is_true: test-1.mappings.my_type.properties.foo
+ - is_true: test-1.mappings.my_type.properties.bar
+
+---
+"Implicitly create a typed index while there is a typeless template":
+
+ - skip:
+      version: " - 6.6.99"
+      reason: include_type_name only supported as of 6.7
+
+ - do:
+      indices.put_template:
+          include_type_name: false
+          name: test_template
+          body:
+              index_patterns: test-*
+              mappings:
+                  properties:
+                      foo:
+                          type: keyword
+
+ - do:
+      catch: /the final mapping would have more than 1 type/
+      index:
+          index: test-1
+          type: my_type
+          body: { bar: 42 }
+
+---
+"Implicitly create a typeless index while there is a typed template":
+
+ - skip:
+      version: " - 6.6.99"
+      reason: include_type_name only supported as of 6.7
+
+ - do:
+      indices.put_template:
+          include_type_name: true
+          name: test_template
+          body:
+              index_patterns: test-*
+              mappings:
+                  my_type:
+                      properties:
+                          foo:
+                              type: keyword
+
+ - do:
+      catch: /the final mapping would have more than 1 type/
+      index:
+          index: test-1
+          type: _doc
+          body: { bar: 42 }
+

rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/20_mix_typeless_typeful.yml

@@ -42,11 +42,41 @@
                       type: "keyword" # also test no-op updates that trigger special logic wrt the mapping version
 
  - do:
-      catch: bad_request
+      catch: /the final mapping would have more than 1 type/
       indices.put_mapping:
           index: index
+          type: some_other_type
           body:
               some_other_type:
                   properties:
                       bar:
                           type: "long"
+
+
+---
+"PUT mapping with _doc on an index that has types":
+
+ - skip:
+      version: " - 5.99.99"
+      reason: 5.x indices can have types that start with an `_`
+
+ - do:
+      indices.create:
+          index: index
+          body:
+              mappings:
+                  my_type:
+                      properties:
+                          foo:
+                              type: "keyword"
+
+ - do:
+      catch: /the final mapping would have more than 1 type/
+      indices.put_mapping:
+          index: index
+          type: _doc
+          body:
+              _doc:
+                  properties:
+                      bar:
+                          type: "long"

server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java

@@ -321,6 +321,28 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                             if (mappings.containsKey(cursor.key)) {
                                 XContentHelper.mergeDefaults(mappings.get(cursor.key),
                                     MapperService.parseMapping(xContentRegistry, mappingString));
+                            } else if (mappings.size() == 1 && cursor.key.equals(MapperService.SINGLE_MAPPING_NAME)) {
+                                // Typeless template with typed mapping
+                                Map<String, Object> templateMapping = MapperService.parseMapping(xContentRegistry, mappingString);
+                                assert templateMapping.size() == 1 : templateMapping;
+                                assert cursor.key.equals(templateMapping.keySet().iterator().next()) :
+                                    cursor.key + " != " + templateMapping;
+                                Map.Entry<String, Map<String, Object>> mappingEntry = mappings.entrySet().iterator().next();
+                                templateMapping = Collections.singletonMap(
+                                        mappingEntry.getKey(),                       // reuse type name from the mapping
+                                        templateMapping.values().iterator().next()); // but actual mappings from the template
+                                XContentHelper.mergeDefaults(mappingEntry.getValue(), templateMapping);
+                            } else if (template.mappings().size() == 1 && mappings.containsKey(MapperService.SINGLE_MAPPING_NAME)) {
+                                // Typed template with typeless mapping
+                                Map<String, Object> templateMapping = MapperService.parseMapping(xContentRegistry, mappingString);
+                                assert templateMapping.size() == 1 : templateMapping;
+                                assert cursor.key.equals(templateMapping.keySet().iterator().next()) :
+                                    cursor.key + " != " + templateMapping;
+                                Map<String, Object> mapping = mappings.get(MapperService.SINGLE_MAPPING_NAME);
+                                templateMapping = Collections.singletonMap(
+                                        MapperService.SINGLE_MAPPING_NAME,           // make template mapping typeless
+                                        templateMapping.values().iterator().next());
+                                XContentHelper.mergeDefaults(mapping, templateMapping);
                             } else {
                                 mappings.put(cursor.key,
                                     MapperService.parseMapping(xContentRegistry, mappingString));

server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataMappingService.java

@@ -38,6 +38,8 @@
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexService;
@@ -273,7 +275,10 @@ private ClusterState applyRequest(ClusterState currentState, PutMappingClusterSt
                 updateList.add(indexMetaData);
                 // try and parse it (no need to add it here) so we can bail early in case of parsing exception
                 DocumentMapper newMapper;
-                DocumentMapper existingMapper = getMapperForUpdate(mapperService, mappingType);
+                DocumentMapper existingMapper = mapperService.documentMapper(mappingType);
+                if (existingMapper == null && isMappingSourceTyped(mapperService, mappingUpdateSource, request.type()) == false) {
+                    existingMapper = getMapperForUpdate(mapperService, mappingType);
+                }
                 String typeForUpdate = existingMapper == null ? mappingType : existingMapper.type();
 
                 if (MapperService.DEFAULT_MAPPING.equals(typeForUpdate)) {
@@ -325,9 +330,16 @@ private ClusterState applyRequest(ClusterState currentState, PutMappingClusterSt
                 // we use the exact same indexService and metadata we used to validate above here to actually apply the update
                 final Index index = indexMetaData.getIndex();
                 final MapperService mapperService = indexMapperServices.get(index);
+
+                // If the _type name is _doc and there is no _doc top-level key then this means that we
+                // are handling a typeless call. In such a case, we override _doc with the actual type
+                // name in the mappings. This allows to use typeless APIs on typed indices.
                 String typeForUpdate = mappingType;
                 CompressedXContent existingSource = null;
-                DocumentMapper existingMapper = getMapperForUpdate(mapperService, mappingType);
+                DocumentMapper existingMapper = mapperService.documentMapper(mappingType);
+                if (existingMapper == null && isMappingSourceTyped(mapperService, mappingUpdateSource, request.type()) == false) {
+                    existingMapper = getMapperForUpdate(mapperService, mappingType);
+                }
                 if (existingMapper != null) {
                     typeForUpdate = existingMapper.type();
                     existingSource = existingMapper.mappingSource();
@@ -388,6 +400,15 @@ public String describeTasks(List<PutMappingClusterStateUpdateRequest> tasks) {
         }
     }
 
+    /**
+     * Returns {@code true} if the given {@code mappingSource} includes a type
+     * as a top-level object.
+     */
+    private static boolean isMappingSourceTyped(MapperService mapperService, CompressedXContent mappingSource, String type) {
+        Map<String, Object> root = XContentHelper.convertToMap(mappingSource.compressedReference(), true, XContentType.JSON).v2();
+        return root.size() == 1 && root.keySet().iterator().next().equals(type);
+    }
+
     public void putMapping(final PutMappingClusterStateUpdateRequest request, final ActionListener<ClusterStateUpdateResponse> listener) {
         clusterService.submitStateUpdateTask("put-mapping",
                 request,

server/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java

@@ -348,6 +348,36 @@ public void testWriteIndexValidationException() throws Exception {
             + "you must manage this on the create index request or with an index template");
     }
 
+    public void testTypelessTemplateWithTypedIndexCreation() throws Exception {
+        reqSettings.put(SETTING_NUMBER_OF_SHARDS, 1);
+        addMatchingTemplate(builder -> builder.putMapping("type", "{\"type\": {}}"));
+        setupRequestMapping(MapperService.SINGLE_MAPPING_NAME, new CompressedXContent("{\"_doc\":{}}"));
+        executeTask();
+        assertThat(getMappingsFromResponse(), Matchers.hasKey(MapperService.SINGLE_MAPPING_NAME));
+    }
+
+    public void testTypedTemplateWithTypelessIndexCreation() throws Exception {
+        reqSettings.put(SETTING_NUMBER_OF_SHARDS, 1);
+        addMatchingTemplate(builder -> builder.putMapping(MapperService.SINGLE_MAPPING_NAME, "{\"_doc\": {}}"));
+        setupRequestMapping("type", new CompressedXContent("{\"type\":{}}"));
+        executeTask();
+        assertThat(getMappingsFromResponse(), Matchers.hasKey("type"));
+    }
+
+    public void testTypedTemplate() throws Exception {
+        reqSettings.put(SETTING_NUMBER_OF_SHARDS, 1);
+        addMatchingTemplate(builder -> builder.putMapping("type", "{\"type\": {}}"));
+        executeTask();
+        assertThat(getMappingsFromResponse(), Matchers.hasKey("type"));
+    }
+
+    public void testTypelessTemplate() throws Exception {
+        reqSettings.put(SETTING_NUMBER_OF_SHARDS, 1);
+        addMatchingTemplate(builder -> builder.putMapping(MapperService.SINGLE_MAPPING_NAME, "{\"_doc\": {}}"));
+        executeTask();
+        assertThat(getMappingsFromResponse(), Matchers.hasKey(MapperService.SINGLE_MAPPING_NAME));
+    }
+
     private IndexRoutingTable createIndexRoutingTableWithStartedShards(Index index) {
         final IndexRoutingTable idxRoutingTable = mock(IndexRoutingTable.class);