Merge branch 'master' of https://github.com/elastic/elasticsearch int…

…o 35992_fix

docs/painless/painless-contexts/painless-score-context.asciidoc

@@ -11,8 +11,10 @@ score to documents returned from a query.
         User-defined parameters passed in as part of the query.
 
 `doc` (`Map`, read-only)::
-        Contains the fields of the current document where each field is a
-        `List` of values.
+        Contains the fields of the current document.  For single-valued fields,
+        the value can be accessed via `doc['fieldname'].value`.  For multi-valued
+        fields, this returns the first value; other values can be accessed
+        via `doc['fieldname'].get(index)`
 
 `_score` (`double` read-only)::
         The similarity score of the current document.
@@ -24,4 +26,33 @@ score to documents returned from a query.
 
 *API*
 
-The standard <<painless-api-reference, Painless API>> is available.
+The standard <<painless-api-reference, Painless API>> is available.
+
+*Example*
+
+To run this example, first follow the steps in
+<<painless-context-examples, context examples>>.
+
+The following query finds all unsold seats, with lower 'row' values
+scored higher.
+
+[source,js]
+--------------------------------------------------
+GET /seats/_search
+{
+    "query": {
+        "function_score": {
+            "query": {
+                "match": { "sold": "false" }
+            },
+            "script_score" : {
+                "script" : {
+                  "source": "1.0 / doc['row'].value"
+                }
+            }
+        }
+    }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[setup:seats]

docs/painless/painless-contexts/painless-similarity-context.asciidoc

@@ -15,6 +15,9 @@ documents in a query.
 `params` (`Map`, read-only)::
         User-defined parameters passed in at query-time.
 
+`weight` (`float`, read-only)::
+        The weight as calculated by a <<painless-weight-context,weight script>>
+
 `query.boost` (`float`, read-only)::
         The boost value if provided by the query.  If this is not provided the
         value is `1.0f`.
@@ -37,12 +40,23 @@ documents in a query.
         The total occurrences of the current term in the index.
 
 `doc.length` (`long`, read-only)::
-        The number of tokens the current document has in the current field.
+        The number of tokens the current document has in the current field.  This
+        is decoded from the stored {ref}/norms.html[norms] and may be approximate for
+        long fields
 
 `doc.freq` (`long`, read-only)::
         The number of occurrences of the current term in the current
         document for the current field.
 
+Note that the `query`, `field`, and `term` variables are also available to the
+<<painless-weight-context,weight context>>. They are more efficiently used
+there, as they are constant for all documents.
+
+For queries that contain multiple terms, the script is called once for each
+term with that term's calculated weight, and the results are summed.  Note that some
+terms might have a `doc.freq` value of `0` on a document, for example if a query
+uses synonyms.
+
 *Return*
 
 `double`::

docs/painless/painless-contexts/painless-sort-context.asciidoc

@@ -10,8 +10,10 @@ Use a Painless script to
         User-defined parameters passed in as part of the query.
 
 `doc` (`Map`, read-only)::
-        Contains the fields of the current document where each field is a
-        `List` of values.
+        Contains the fields of the current document.  For single-valued fields,
+        the value can be accessed via `doc['fieldname'].value`.  For multi-valued
+        fields, this returns the first value; other values can be accessed
+        via `doc['fieldname'].get(index)`
 
 `_score` (`double` read-only)::
         The similarity score of the current document.
@@ -23,4 +25,37 @@ Use a Painless script to
 
 *API*
 
-The standard <<painless-api-reference, Painless API>> is available.
+The standard <<painless-api-reference, Painless API>> is available.
+
+*Example*
+
+To run this example, first follow the steps in
+<<painless-context-examples, context examples>>.
+
+To sort results by the length of the `theatre` field, submit the following query:
+
+[source,js]
+----
+GET /_search
+{
+    "query" : {
+        "term" : { "sold" : "true" }
+    },
+    "sort" : {
+        "_script" : {
+            "type" : "number",
+            "script" : {
+                "lang": "painless",
+                "source": "doc['theatre'].value.length() * params.factor",
+                "params" : {
+                    "factor" : 1.1
+                }
+            },
+            "order" : "asc"
+        }
+    }
+}
+
+----
+// CONSOLE
+// TEST[setup:seats]

docs/painless/painless-contexts/painless-weight-context.asciidoc

@@ -3,8 +3,11 @@
 
 Use a Painless script to create a
 {ref}/index-modules-similarity.html[weight] for use in a
-<<painless-similarity-context, similarity script>>.  Weight is used to prevent
-recalculation of constants that remain the same across documents.
+<<painless-similarity-context, similarity script>>.  The weight makes up the
+part of the similarity calculation that is independent of the document being
+scored, and so can be built up front and cached.
+
+Queries that contain multiple terms calculate a separate weight for each term.
 
 *Variables*
 

docs/reference/docs/termvectors.asciidoc

@@ -27,9 +27,6 @@ or by adding the requested fields in the request body (see
 example below). Fields can also be specified with wildcards
 in similar way to the <<query-dsl-multi-match-query,multi match query>>
 
-[WARNING]
-Note that the usage of `/_termvector` is deprecated in 2.0, and replaced by `/_termvectors`.
-
 [float]
 === Return values
 

docs/reference/migration/migrate_7_0/api.asciidoc

@@ -119,3 +119,10 @@ while now an exception is thrown.
 
 The deprecated graph endpoints (those with `/_graph/_explore`) have been
 removed.
+
+
+[float]
+==== Deprecated `_termvector` endpoint removed
+
+The `_termvector` endpoint was deprecated in 2.0 and has now been removed.
+The endpoint `_termvectors` (plural) should be used instead.

docs/reference/migration/migrate_7_0/java.asciidoc

@@ -32,4 +32,10 @@ was moved to `org.elasticsearch.search.aggregations.PipelineAggregationBuilders`
 ==== `Retry.withBackoff` methods with `Settings` removed
 
 The variants of `Retry.withBackoff` that included `Settings` have been removed
-because `Settings` is no longer needed.
+because `Settings` is no longer needed.
+
+[float]
+==== Deprecated method `Client#termVector` removed
+
+The client method `termVector`, deprecated in 2.0, has been removed. The method
+`termVectors` (plural) should be used instead.

docs/reference/migration/migrate_7_0/search.asciidoc

@@ -159,3 +159,14 @@ Negative scores in the Function Score Query are deprecated in 6.x, and are
 not allowed in this version. If a negative score is produced as a result
 of computation (e.g. in `script_score` or `field_value_factor` functions),
 an error will be thrown.
+
+[float]
+==== The filter context has been removed
+
+The `filter` context has been removed from Elasticsearch's query builders,
+the distinction between queries and filters is now decided in Lucene depending
+on whether queries need to access score or not. As a result `bool` queries with
+`should` clauses that don't need to access the score will no longer set their
+`minimum_should_match` to 1. This behavior has been deprecated in the previous
+major version.
+

docs/reference/query-dsl/bool-query.asciidoc

@@ -17,29 +17,14 @@ contribute to the score.
 in <<query-filter-context,filter context>>, meaning that scoring is ignored
 and clauses are considered for caching.
 
-|`should` |The clause (query) should appear in the matching document. If the
-`bool` query is in a <<query-filter-context,query context>> and has a `must` or
-`filter` clause then a document will match the `bool` query even if none of the
-`should` queries match. In this case these clauses are only used to influence
-the score. If the `bool` query is a <<query-filter-context,filter context>>
-or has neither `must` or `filter` then at least one of the `should` queries
-must match a document for it to match the `bool` query. This behavior may be
-explicitly controlled by settings the
-<<query-dsl-minimum-should-match,`minimum_should_match`>> parameter.
+|`should` |The clause (query) should appear in the matching document.
 
 |`must_not` |The clause (query) must not appear in the matching
 documents.  Clauses are executed in <<query-filter-context,filter context>> meaning
 that scoring is ignored and clauses are considered for caching. Because scoring is
 ignored, a score of `0` for all documents is returned.
 |=======================================================================
 
-[IMPORTANT]
-.Bool query in filter context
-========================================================================
-If this query is used in a filter context and it has `should`
-clauses then at least one `should` clause is required to match.
-========================================================================
-
 The `bool` query takes a _more-matches-is-better_ approach, so the score from
 each matching `must` or `should` clause will be added together to provide the
 final `_score` for each document.

docs/reference/search/request/scroll.asciidoc

@@ -125,6 +125,11 @@ TIP: Keeping older segments alive means that more file handles are needed.
 Ensure that you have configured your nodes to have ample free file handles.
 See <<file-descriptors>>.
 
+NOTE: To prevent against issues caused by having too many scrolls open, the
+user is not allowed to open scrolls past a certain limit. By default, the
+maximum number of open scrolls is 500. This limit can be updated with the
+`search.max_open_scroll_context` cluster setting.
+
 You can check how many search contexts are open with the
 <<cluster-nodes-stats,nodes stats API>>:
 

server/src/main/java/org/elasticsearch/client/Client.java

@@ -370,39 +370,6 @@ public interface Client extends ElasticsearchClient, Releasable {
      */
     TermVectorsRequestBuilder prepareTermVectors(String index, String type, String id);
 
-    /**
-     * An action that returns the term vectors for a specific document.
-     *
-     * @param request The term vector request
-     * @return The response future
-     */
-    @Deprecated
-    ActionFuture<TermVectorsResponse> termVector(TermVectorsRequest request);
-
-    /**
-     * An action that returns the term vectors for a specific document.
-     *
-     * @param request The term vector request
-     */
-    @Deprecated
-    void termVector(TermVectorsRequest request, ActionListener<TermVectorsResponse> listener);
-
-    /**
-     * Builder for the term vector request.
-     */
-    @Deprecated
-    TermVectorsRequestBuilder prepareTermVector();
-
-    /**
-     * Builder for the term vector request.
-     *
-     * @param index The index to load the document from
-     * @param type  The type of the document
-     * @param id    The id of the document
-     */
-    @Deprecated
-    TermVectorsRequestBuilder prepareTermVector(String index, String type, String id);
-
     /**
      * Multi get term vectors.
      */

server/src/main/java/org/elasticsearch/client/support/AbstractClient.java

@@ -581,30 +581,6 @@ public TermVectorsRequestBuilder prepareTermVectors(String index, String type, S
         return new TermVectorsRequestBuilder(this, TermVectorsAction.INSTANCE, index, type, id);
     }
 
-    @Deprecated
-    @Override
-    public ActionFuture<TermVectorsResponse> termVector(final TermVectorsRequest request) {
-        return termVectors(request);
-    }
-
-    @Deprecated
-    @Override
-    public void termVector(final TermVectorsRequest request, final ActionListener<TermVectorsResponse> listener) {
-        termVectors(request, listener);
-    }
-
-    @Deprecated
-    @Override
-    public TermVectorsRequestBuilder prepareTermVector() {
-        return prepareTermVectors();
-    }
-
-    @Deprecated
-    @Override
-    public TermVectorsRequestBuilder prepareTermVector(String index, String type, String id) {
-        return prepareTermVectors(index, type, id);
-    }
-
     @Override
     public ActionFuture<MultiTermVectorsResponse> multiTermVectors(final MultiTermVectorsRequest request) {
         return execute(MultiTermVectorsAction.INSTANCE, request);

server/src/main/java/org/elasticsearch/cluster/metadata/AliasValidator.java

@@ -139,6 +139,6 @@ public void validateAliasFilter(String alias, byte[] filter, QueryShardContext q
     private static void validateAliasFilter(XContentParser parser, QueryShardContext queryShardContext) throws IOException {
         QueryBuilder parseInnerQueryBuilder = parseInnerQueryBuilder(parser);
         QueryBuilder queryBuilder = Rewriteable.rewrite(parseInnerQueryBuilder, queryShardContext, true);
-        queryBuilder.toFilter(queryShardContext);
+        queryBuilder.toQuery(queryShardContext);
     }
 }

server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java

@@ -387,6 +387,7 @@ public void apply(Settings value, Settings current, Settings previous) {
                     SearchService.MAX_KEEPALIVE_SETTING,
                     MultiBucketConsumerService.MAX_BUCKET_SETTING,
                     SearchService.LOW_LEVEL_CANCELLATION_SETTING,
+                    SearchService.MAX_OPEN_SCROLL_CONTEXT,
                     Node.WRITE_PORTS_FILE_SETTING,
                     Node.NODE_NAME_SETTING,
                     Node.NODE_DATA_SETTING,

server/src/main/java/org/elasticsearch/index/query/AbstractQueryBuilder.java

@@ -111,19 +111,6 @@ public final Query toQuery(QueryShardContext context) throws IOException {
         return query;
     }
 
-    @Override
-    public final Query toFilter(QueryShardContext context) throws IOException {
-        Query result;
-            final boolean originalIsFilter = context.isFilter();
-            try {
-                context.setIsFilter(true);
-                result = toQuery(context);
-            } finally {
-                context.setIsFilter(originalIsFilter);
-            }
-        return result;
-    }
-
     protected abstract Query doToQuery(QueryShardContext context) throws IOException;
 
     /**

server/src/main/java/org/elasticsearch/index/query/BoolQueryBuilder.java

@@ -384,30 +384,14 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
             return new MatchAllDocsQuery();
         }
 
-        final String minimumShouldMatch;
-        if (context.isFilter() && this.minimumShouldMatch == null && shouldClauses.size() > 0) {
-            minimumShouldMatch = "1";
-        } else {
-            minimumShouldMatch = this.minimumShouldMatch;
-        }
         Query query = Queries.applyMinimumShouldMatch(booleanQuery, minimumShouldMatch);
         return adjustPureNegative ? fixNegativeQueryIfNeeded(query) : query;
     }
 
     private static void addBooleanClauses(QueryShardContext context, BooleanQuery.Builder booleanQueryBuilder,
                                           List<QueryBuilder> clauses, Occur occurs) throws IOException {
         for (QueryBuilder query : clauses) {
-            Query luceneQuery = null;
-            switch (occurs) {
-                case MUST:
-                case SHOULD:
-                    luceneQuery = query.toQuery(context);
-                    break;
-                case FILTER:
-                case MUST_NOT:
-                    luceneQuery = query.toFilter(context);
-                    break;
-            }
+            Query luceneQuery = query.toQuery(context);
             booleanQueryBuilder.add(new BooleanClause(luceneQuery, occurs));
         }
     }

server/src/main/java/org/elasticsearch/index/query/ConstantScoreQueryBuilder.java

@@ -129,7 +129,7 @@ public static ConstantScoreQueryBuilder fromXContent(XContentParser parser) thro
 
     @Override
     protected Query doToQuery(QueryShardContext context) throws IOException {
-        Query innerFilter = filterBuilder.toFilter(context);
+        Query innerFilter = filterBuilder.toQuery(context);
         return new ConstantScoreQuery(innerFilter);
     }