Merge remote-tracking branch 'elastic/master' into pr/31512

* elastic/master: (57 commits)
  HLRest: Fix test for explain API
  [TEST] Fix RemoteClusterConnectionTests
  Add Create Snapshot to High-Level Rest Client (elastic#31215)
  Remove legacy MetaDataStateFormat (elastic#31603)
  Add explain API to high-level REST client (elastic#31387)
  Preserve thread context when connecting to remote cluster (elastic#31574)
  Unify headers for full text queries
  Remove redundant 'minimum_should_match'
  JDBC driver prepared statement set* methods  (elastic#31494)
  [TEST] call yaml client close method from test suite (elastic#31591)
  ingest: Add ignore_missing property to foreach filter (elastic#22147) (elastic#31578)
  Fix a formatting issue in the docvalue_fields documentation. (elastic#31563)
  reduce log level at gradle configuration time
  [TEST] Close additional clients created while running yaml tests (elastic#31575)
  Docs: Clarify sensitive fields watcher encryption (elastic#31551)
  Watcher: Remove never executed code (elastic#31135)
  Add support for switching distribution for all integration tests (elastic#30874)
  Improve robustness of geo shape parser for malformed shapes (elastic#31449)
  QA: Create xpack yaml features (elastic#31403)
  Improve test times for tests using `RandomObjects::addFields` (elastic#31556)
  ...

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -71,7 +71,9 @@ public class PluginBuildPlugin extends BuildPlugin {
             if (isModule) {
                 project.integTestCluster.module(project)
                 project.tasks.run.clusterConfig.module(project)
-                project.tasks.run.clusterConfig.distribution = 'integ-test-zip'
+                project.tasks.run.clusterConfig.distribution = System.getProperty(
+                        'run.distribution', 'integ-test-zip'
+                )
             } else {
                 project.integTestCluster.plugin(project.path)
                 project.tasks.run.clusterConfig.plugin(project.path)
@@ -111,7 +113,7 @@ public class PluginBuildPlugin extends BuildPlugin {
     private static void createIntegTestTask(Project project) {
         RestIntegTestTask integTest = project.tasks.create('integTest', RestIntegTestTask.class)
         integTest.mustRunAfter(project.precommit, project.test)
-        project.integTestCluster.distribution = 'integ-test-zip'
+        project.integTestCluster.distribution = System.getProperty('tests.distribution', 'integ-test-zip')
         project.check.dependsOn(integTest)
     }
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -88,6 +88,9 @@ class ClusterFormationTasks {
         Configuration currentDistro = project.configurations.create("${prefix}_elasticsearchDistro")
         Configuration bwcDistro = project.configurations.create("${prefix}_elasticsearchBwcDistro")
         Configuration bwcPlugins = project.configurations.create("${prefix}_elasticsearchBwcPlugins")
+        if (System.getProperty('tests.distribution', 'oss-zip') == 'integ-test-zip') {
+            throw new Exception("tests.distribution=integ-test-zip is not supported")
+        }
         configureDistributionDependency(project, config.distribution, currentDistro, VersionProperties.elasticsearch)
         if (config.numBwcNodes > 0) {
             if (config.bwcVersion == null) {
@@ -533,7 +536,8 @@ class ClusterFormationTasks {
 
     static Task configureInstallModuleTask(String name, Project project, Task setup, NodeInfo node, Project module) {
         if (node.config.distribution != 'integ-test-zip') {
-            throw new GradleException("Module ${module.path} not allowed be installed distributions other than integ-test-zip because they should already have all modules bundled!")
+            project.logger.info("Not installing modules for $name, ${node.config.distribution} already has them")
+            return setup
         }
         if (module.plugins.hasPlugin(PluginBuildPlugin) == false) {
             throw new GradleException("Task ${name} cannot include module ${module.path} which is not an esplugin")
@@ -643,6 +647,9 @@ class ClusterFormationTasks {
             BuildPlugin.requireJavaHome(start, node.javaVersion)
         }
         start.doLast(elasticsearchRunner)
+        start.doFirst {
+            project.logger.info("Starting node in ${node.clusterName} distribution: ${node.config.distribution}")
+        }
         return start
     }
 

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 7.0.0-alpha1
-lucene            = 7.4.0-snapshot-518d303506
+lucene            = 7.4.0
 
 # optional dependencies
 spatial4j         = 0.7

client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/bulk/TransportNoopBulkAction.java

@@ -19,8 +19,8 @@
 package org.elasticsearch.plugin.noop.action.bulk;
 
 import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.DocWriteResponse;
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
@@ -30,21 +30,20 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.shard.ShardId;
-import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 
 public class TransportNoopBulkAction extends HandledTransportAction<BulkRequest, BulkResponse> {
     private static final BulkItemResponse ITEM_RESPONSE = new BulkItemResponse(1, DocWriteRequest.OpType.UPDATE,
         new UpdateResponse(new ShardId("mock", "", 1), "mock_type", "1", 1L, DocWriteResponse.Result.CREATED));
 
     @Inject
-    public TransportNoopBulkAction(Settings settings, ThreadPool threadPool, TransportService transportService,
-                                      ActionFilters actionFilters) {
-        super(settings, NoopBulkAction.NAME, threadPool, transportService, actionFilters, BulkRequest::new);
+    public TransportNoopBulkAction(Settings settings, TransportService transportService, ActionFilters actionFilters) {
+        super(settings, NoopBulkAction.NAME, transportService, actionFilters, BulkRequest::new);
     }
 
     @Override
-    protected void doExecute(BulkRequest request, ActionListener<BulkResponse> listener) {
+    protected void doExecute(Task task, BulkRequest request, ActionListener<BulkResponse> listener) {
         final int itemCount = request.requests().size();
         // simulate at least a realistic amount of data that gets serialized
         BulkItemResponse[] bulkItemResponses = new BulkItemResponse[itemCount];

client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/NoopSearchRequestBuilder.java

@@ -329,7 +329,7 @@ public NoopSearchRequestBuilder addSort(String field, SortOrder order) {
      *
      * @see org.elasticsearch.search.sort.SortBuilders
      */
-    public NoopSearchRequestBuilder addSort(SortBuilder sort) {
+    public NoopSearchRequestBuilder addSort(SortBuilder<?> sort) {
         sourceBuilder().sort(sort);
         return this;
     }
@@ -415,7 +415,7 @@ public NoopSearchRequestBuilder setRescorer(RescorerBuilder<?> rescorer) {
      * @param window   rescore window
      * @return this for chaining
      */
-    public NoopSearchRequestBuilder setRescorer(RescorerBuilder rescorer, int window) {
+    public NoopSearchRequestBuilder setRescorer(RescorerBuilder<?> rescorer, int window) {
         sourceBuilder().clearRescorers();
         return addRescorer(rescorer.windowSize(window));
     }

client/client-benchmark-noop-api-plugin/src/main/java/org/elasticsearch/plugin/noop/action/search/TransportNoopSearchAction.java

@@ -27,27 +27,25 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.aggregations.InternalAggregations;
 import org.elasticsearch.search.internal.InternalSearchResponse;
 import org.elasticsearch.search.profile.SearchProfileShardResults;
 import org.elasticsearch.search.suggest.Suggest;
-import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.tasks.Task;
 import org.elasticsearch.transport.TransportService;
 
 import java.util.Collections;
 
 public class TransportNoopSearchAction extends HandledTransportAction<SearchRequest, SearchResponse> {
     @Inject
-    public TransportNoopSearchAction(Settings settings, ThreadPool threadPool, TransportService transportService,
-                                     ActionFilters actionFilters) {
-        super(settings, NoopSearchAction.NAME, threadPool, transportService, actionFilters,
-              (Writeable.Reader<SearchRequest>) SearchRequest::new);
+    public TransportNoopSearchAction(Settings settings, TransportService transportService, ActionFilters actionFilters) {
+        super(settings, NoopSearchAction.NAME, transportService, actionFilters, (Writeable.Reader<SearchRequest>) SearchRequest::new);
     }
 
     @Override
-    protected void doExecute(SearchRequest request, ActionListener<SearchResponse> listener) {
+    protected void doExecute(Task task, SearchRequest request, ActionListener<SearchResponse> listener) {
         listener.onResponse(new SearchResponse(new InternalSearchResponse(
             new SearchHits(
                 new SearchHit[0], 0L, 0.0f),

client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java

@@ -37,6 +37,8 @@
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeResponse;
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
@@ -188,6 +190,35 @@ public void getMappingsAsync(GetMappingsRequest getMappingsRequest, RequestOptio
             GetMappingsResponse::fromXContent, listener, emptySet());
     }
 
+    /**
+     * Retrieves the field mappings on an index or indices using the Get Field Mapping API.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-get-field-mapping.html">
+     * Get Field Mapping API on elastic.co</a>
+     * @param getFieldMappingsRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public GetFieldMappingsResponse getFieldMapping(GetFieldMappingsRequest getFieldMappingsRequest,
+                                                    RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(getFieldMappingsRequest, RequestConverters::getFieldMapping, options,
+            GetFieldMappingsResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously retrieves the field mappings on an index on indices using the Get Field Mapping API.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-get-field-mapping.html">
+     * Get Field Mapping API on elastic.co</a>
+     * @param getFieldMappingsRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void getFieldMappingAsync(GetFieldMappingsRequest getFieldMappingsRequest, RequestOptions options,
+                                     ActionListener<GetFieldMappingsResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(getFieldMappingsRequest, RequestConverters::getFieldMapping, options,
+            GetFieldMappingsResponse::fromXContent, listener, emptySet());
+    }
+
     /**
      * Updates aliases using the Index Aliases API.
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-aliases.html">

client/rest-high-level/src/main/java/org/elasticsearch/client/IngestClient.java

@@ -24,6 +24,8 @@
 import org.elasticsearch.action.ingest.GetPipelineRequest;
 import org.elasticsearch.action.ingest.GetPipelineResponse;
 import org.elasticsearch.action.ingest.PutPipelineRequest;
+import org.elasticsearch.action.ingest.SimulatePipelineRequest;
+import org.elasticsearch.action.ingest.SimulatePipelineResponse;
 import org.elasticsearch.action.ingest.WritePipelineResponse;
 
 import java.io.IOException;
@@ -125,4 +127,37 @@ public void deletePipelineAsync(DeletePipelineRequest request, RequestOptions op
         restHighLevelClient.performRequestAsyncAndParseEntity( request, RequestConverters::deletePipeline, options,
             WritePipelineResponse::fromXContent, listener, emptySet());
     }
+
+    /**
+     * Simulate a pipeline on a set of documents provided in the request
+     * <p>
+     * See
+     * <a href="https://www.elastic.co/guide/en/elasticsearch/reference/master/simulate-pipeline-api.html">
+     *     Simulate Pipeline API on elastic.co</a>
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public SimulatePipelineResponse simulatePipeline(SimulatePipelineRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity( request, RequestConverters::simulatePipeline, options,
+            SimulatePipelineResponse::fromXContent, emptySet());
+    }
+
+    /**
+     * Asynchronously simulate a pipeline on a set of documents provided in the request
+     * <p>
+     * See
+     * <a href="https://www.elastic.co/guide/en/elasticsearch/reference/master/simulate-pipeline-api.html">
+     *     Simulate Pipeline API on elastic.co</a>
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void simulatePipelineAsync(SimulatePipelineRequest request,
+                                      RequestOptions options,
+                                      ActionListener<SimulatePipelineResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity( request, RequestConverters::simulatePipeline, options,
+            SimulatePipelineResponse::fromXContent, listener, emptySet());
+    }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java

@@ -37,6 +37,7 @@
 import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
 import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryRequest;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
+import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest;
 import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
@@ -50,6 +51,7 @@
 import org.elasticsearch.action.admin.indices.flush.SyncedFlushRequest;
 import org.elasticsearch.action.admin.indices.forcemerge.ForceMergeRequest;
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest;
 import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest;
 import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.elasticsearch.action.admin.indices.open.OpenIndexRequest;
@@ -64,13 +66,15 @@
 import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequest;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.explain.ExplainRequest;
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.MultiGetRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.ingest.DeletePipelineRequest;
-import org.elasticsearch.action.ingest.PutPipelineRequest;
 import org.elasticsearch.action.ingest.GetPipelineRequest;
+import org.elasticsearch.action.ingest.SimulatePipelineRequest;
+import org.elasticsearch.action.ingest.PutPipelineRequest;
 import org.elasticsearch.action.search.ClearScrollRequest;
 import org.elasticsearch.action.search.MultiSearchRequest;
 import org.elasticsearch.action.search.SearchRequest;
@@ -229,6 +233,25 @@ static Request getMappings(GetMappingsRequest getMappingsRequest) throws IOExcep
         return request;
     }
 
+    static Request getFieldMapping(GetFieldMappingsRequest getFieldMappingsRequest) throws IOException {
+        String[] indices = getFieldMappingsRequest.indices() == null ? Strings.EMPTY_ARRAY : getFieldMappingsRequest.indices();
+        String[] types = getFieldMappingsRequest.types() == null ? Strings.EMPTY_ARRAY : getFieldMappingsRequest.types();
+        String[] fields = getFieldMappingsRequest.fields() == null ? Strings.EMPTY_ARRAY : getFieldMappingsRequest.fields();
+
+        String endpoint = new EndpointBuilder().addCommaSeparatedPathParts(indices)
+            .addPathPartAsIs("_mapping").addCommaSeparatedPathParts(types)
+            .addPathPartAsIs("field").addCommaSeparatedPathParts(fields)
+            .build();
+
+        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
+
+        Params parameters = new Params(request);
+        parameters.withIndicesOptions(getFieldMappingsRequest.indicesOptions());
+        parameters.withIncludeDefaults(getFieldMappingsRequest.includeDefaults());
+        parameters.withLocal(getFieldMappingsRequest.local());
+        return request;
+    }
+
     static Request refresh(RefreshRequest refreshRequest) {
         String[] indices = refreshRequest.indices() == null ? Strings.EMPTY_ARRAY : refreshRequest.indices();
         Request request = new Request(HttpPost.METHOD_NAME, endpoint(indices, "_refresh"));
@@ -597,6 +620,19 @@ static Request existsAlias(GetAliasesRequest getAliasesRequest) {
         return request;
     }
 
+    static Request explain(ExplainRequest explainRequest) throws IOException {
+        Request request = new Request(HttpGet.METHOD_NAME,
+            endpoint(explainRequest.index(), explainRequest.type(), explainRequest.id(), "_explain"));
+
+        Params params = new Params(request);
+        params.withStoredFields(explainRequest.storedFields());
+        params.withFetchSourceContext(explainRequest.fetchSourceContext());
+        params.withRouting(explainRequest.routing());
+        params.withPreference(explainRequest.preference());
+        request.setEntity(createEntity(explainRequest, REQUEST_BODY_CONTENT_TYPE));
+        return request;
+    }
+
     static Request fieldCaps(FieldCapabilitiesRequest fieldCapabilitiesRequest) {
         Request request = new Request(HttpGet.METHOD_NAME, endpoint(fieldCapabilitiesRequest.indices(), "_field_caps"));
 
@@ -845,6 +881,19 @@ static Request verifyRepository(VerifyRepositoryRequest verifyRepositoryRequest)
         return request;
     }
 
+    static Request createSnapshot(CreateSnapshotRequest createSnapshotRequest) throws IOException {
+        String endpoint = new EndpointBuilder().addPathPart("_snapshot")
+            .addPathPart(createSnapshotRequest.repository())
+            .addPathPart(createSnapshotRequest.snapshot())
+            .build();
+        Request request = new Request(HttpPut.METHOD_NAME, endpoint);
+        Params params = new Params(request);
+        params.withMasterTimeout(createSnapshotRequest.masterNodeTimeout());
+        params.withWaitForCompletion(createSnapshotRequest.waitForCompletion());
+        request.setEntity(createEntity(createSnapshotRequest, REQUEST_BODY_CONTENT_TYPE));
+        return request;
+    }
+
     static Request deleteSnapshot(DeleteSnapshotRequest deleteSnapshotRequest) {
         String endpoint = new EndpointBuilder().addPathPartAsIs("_snapshot")
             .addPathPart(deleteSnapshotRequest.repository())
@@ -886,6 +935,20 @@ static Request validateQuery(ValidateQueryRequest validateQueryRequest) throws I
         return request;
     }
 
+    static Request simulatePipeline(SimulatePipelineRequest simulatePipelineRequest) throws IOException {
+        EndpointBuilder builder = new EndpointBuilder().addPathPartAsIs("_ingest/pipeline");
+        if (simulatePipelineRequest.getId() != null && !simulatePipelineRequest.getId().isEmpty()) {
+            builder.addPathPart(simulatePipelineRequest.getId());
+        }
+        builder.addPathPartAsIs("_simulate");
+        String endpoint = builder.build();
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
+        Params params = new Params(request);
+        params.putParam("verbose", Boolean.toString(simulatePipelineRequest.isVerbose()));
+        request.setEntity(createEntity(simulatePipelineRequest, REQUEST_BODY_CONTENT_TYPE));
+        return request;
+    }
+
     static Request getAlias(GetAliasesRequest getAliasesRequest) {
         String[] indices = getAliasesRequest.indices() == null ? Strings.EMPTY_ARRAY : getAliasesRequest.indices();
         String[] aliases = getAliasesRequest.aliases() == null ? Strings.EMPTY_ARRAY : getAliasesRequest.aliases();