Merge branch 'main' into making-BulkRequest-implement-RefCounted

docs/changelog/102713.yaml

@@ -0,0 +1,5 @@
+pr: 102713
+summary: "ESQL: Add `profile` option"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/102731.yaml

@@ -0,0 +1,5 @@
+pr: 102731
+summary: Add internal inference action for ml models an services
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/102806.yaml

@@ -0,0 +1,5 @@
+pr: 102806
+summary: Support for GET all models and by task type in the `_inference` API
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/102832.yaml

@@ -0,0 +1,5 @@
+pr: 102832
+summary: Disable concurrency for sampler and diversified sampler
+area: Aggregations
+type: enhancement
+issues: []

docs/changelog/102844.yaml

@@ -0,0 +1,5 @@
+pr: 102844
+summary: Skip global ordinals loading if query does not match after rewrite
+area: Aggregations
+type: bug
+issues: []

docs/changelog/102848.yaml

@@ -0,0 +1,5 @@
+pr: 102848
+summary: Decref `SharedBytes.IO` after read is done not before
+area: Snapshot/Restore
+type: bug
+issues: []

docs/reference/mapping/types/geo-shape.asciidoc

@@ -30,6 +30,15 @@ The `geo_shape` mapping maps GeoJSON or WKT geometry objects to the `geo_shape`
 type. To enable it, users must explicitly map fields to the `geo_shape`
 type.
 
+[NOTE]
+=============================================
+In https://datatracker.ietf.org/doc/html/rfc7946[GeoJSON]
+and https://www.ogc.org/standard/sfa/[WKT], and therefore Elasticsearch,
+the correct *coordinate order is longitude, latitude (X, Y)* within coordinate
+arrays. This differs from many Geospatial APIs (e.g., Google Maps) that generally
+use the colloquial latitude, longitude (Y, X).
+=============================================
+
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Option |Description| Default
@@ -142,11 +151,6 @@ specifying only the top left and bottom right points.
 =============================================
 For all types, both the inner `type` and `coordinates` fields are
 required.
-
-In GeoJSON and WKT, and therefore Elasticsearch, the correct *coordinate
-order is longitude, latitude (X, Y)* within coordinate arrays. This
-differs from many Geospatial APIs (e.g., Google Maps) that generally
-use the colloquial latitude, longitude (Y, X).
 =============================================
 
 [[geo-point-type]]

gradle/verification-metadata.xml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<verification-metadata xmlns="https://schema.gradle.org/dependency-verification" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="https://schema.gradle.org/dependency-verification https://schema.gradle.org/dependency-verification/dependency-verification-1.2.xsd">
+<verification-metadata xmlns="https://schema.gradle.org/dependency-verification" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="https://schema.gradle.org/dependency-verification https://schema.gradle.org/dependency-verification/dependency-verification-1.3.xsd">
    <configuration>
       <verify-metadata>false</verify-metadata>
       <verify-signatures>false</verify-signatures>
@@ -1401,19 +1401,19 @@
             <sha256 value="3ea995b55a4068be22989b70cc29a4d788c2d328d1d50613a7a9afd13fdd2d0a" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.opentelemetry" name="opentelemetry-api" version="1.17.0">
-         <artifact name="opentelemetry-api-1.17.0.jar">
-            <sha256 value="dce58a21caef0c6be082ce5ec86ad75f3017744a029a454e068871c110ae029c" origin="Generated by Gradle"/>
+      <component group="io.opentelemetry" name="opentelemetry-api" version="1.31.0">
+         <artifact name="opentelemetry-api-1.31.0.jar">
+            <sha256 value="7de2c7268850a9c1bae4401cf264febb871d811c6be8e5b3fb2cae52886e8ec1" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.opentelemetry" name="opentelemetry-context" version="1.17.0">
-         <artifact name="opentelemetry-context-1.17.0.jar">
-            <sha256 value="ac5b7a7bda673b08d62d4dafe23a105ec4a7d0b7a967cda5597dcca994988227" origin="Generated by Gradle"/>
+      <component group="io.opentelemetry" name="opentelemetry-context" version="1.31.0">
+         <artifact name="opentelemetry-context-1.31.0.jar">
+            <sha256 value="664896a5c34bcda20c95c8f45198a95e8f97a1cd5e5c2923978f42dddada787d" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="io.opentelemetry" name="opentelemetry-semconv" version="1.17.0-alpha">
-         <artifact name="opentelemetry-semconv-1.17.0-alpha.jar">
-            <sha256 value="272f310e8c3e9adad07a2b8910a520ce5c58d1e209661d57b5bd30e8b1614402" origin="Generated by Gradle"/>
+      <component group="io.opentelemetry" name="opentelemetry-semconv" version="1.21.0-alpha">
+         <artifact name="opentelemetry-semconv-1.21.0-alpha.jar">
+            <sha256 value="4a8f41b93eec51e85fa6b48e43de6785b742316fdd9c9baf595adbce6d5de6af" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="io.ous" name="jtoml" version="2.0.0">

modules/apm/build.gradle

@@ -12,12 +12,13 @@ esplugin {
   classname 'org.elasticsearch.telemetry.apm.APM'
 }
 
-def otelVersion = '1.17.0'
+def otelVersion = '1.31.0'
+def otelSemconvVersion = '1.21.0-alpha'
 
 dependencies {
   implementation "io.opentelemetry:opentelemetry-api:${otelVersion}"
   implementation "io.opentelemetry:opentelemetry-context:${otelVersion}"
-  implementation "io.opentelemetry:opentelemetry-semconv:${otelVersion}-alpha"
+  implementation "io.opentelemetry:opentelemetry-semconv:${otelSemconvVersion}"
   runtimeOnly    "co.elastic.apm:elastic-apm-agent:1.44.0"
 }
 

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml

@@ -210,8 +210,10 @@ setup:
 ---
 "Create data stream with failure store":
   - skip:
-      version: " - 8.10.99"
-      reason: "data stream failure stores only creatable in 8.11+"
+      version: all
+      reason: "AwaitsFix https://github.com/elastic/elasticsearch/issues/102873"
+#      version: " - 8.10.99"
+#      reason: "data stream failure stores only creatable in 8.11+"
 
   - do:
       allowed_warnings:

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/30_auto_create_data_stream.yml

@@ -50,8 +50,10 @@
 ---
 "Put index template with failure store":
   - skip:
-      version: " - 8.10.99"
-      reason: "data stream failure stores only creatable in 8.11+"
+      version: all
+      reason: "AwaitsFix https://github.com/elastic/elasticsearch/issues/102873"
+#      version: " - 8.10.99"
+#      reason: "data stream failure stores only creatable in 8.11+"
       features: allowed_warnings
 
   - do:

modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomMustacheFactory.java

@@ -47,6 +47,7 @@ public final class CustomMustacheFactory extends DefaultMustacheFactory {
     static final String X_WWW_FORM_URLENCODED_MEDIA_TYPE = "application/x-www-form-urlencoded";
 
     private static final String DEFAULT_MEDIA_TYPE = JSON_MEDIA_TYPE;
+    private static final boolean DEFAULT_DETECT_MISSING_PARAMS = false;
 
     private static final Map<String, Supplier<Encoder>> ENCODERS = Map.of(
         V7_JSON_MEDIA_TYPE_WITH_CHARSET,
@@ -63,14 +64,30 @@ public final class CustomMustacheFactory extends DefaultMustacheFactory {
 
     private final Encoder encoder;
 
+    /**
+     * Initializes a CustomMustacheFactory object with a specified mediaType.
+     *
+     * @deprecated Use {@link #builder()} instead to retrieve a {@link Builder} object that can be used to create a factory.
+     */
+    @Deprecated
     public CustomMustacheFactory(String mediaType) {
-        super();
-        setObjectHandler(new CustomReflectionObjectHandler());
-        this.encoder = createEncoder(mediaType);
+        this(mediaType, DEFAULT_DETECT_MISSING_PARAMS);
     }
 
+    /**
+     * Default constructor for the factory.
+     *
+     * @deprecated Use {@link #builder()} instead to retrieve a {@link Builder} object that can be used to create a factory.
+     */
+    @Deprecated
     public CustomMustacheFactory() {
-        this(DEFAULT_MEDIA_TYPE);
+        this(DEFAULT_MEDIA_TYPE, DEFAULT_DETECT_MISSING_PARAMS);
+    }
+
+    private CustomMustacheFactory(String mediaType, boolean detectMissingParams) {
+        super();
+        setObjectHandler(new CustomReflectionObjectHandler(detectMissingParams));
+        this.encoder = createEncoder(mediaType);
     }
 
     @Override
@@ -95,6 +112,10 @@ public MustacheVisitor createMustacheVisitor() {
         return new CustomMustacheVisitor(this);
     }
 
+    public static Builder builder() {
+        return new Builder();
+    }
+
     class CustomMustacheVisitor extends DefaultMustacheVisitor {
 
         CustomMustacheVisitor(DefaultMustacheFactory df) {
@@ -360,4 +381,34 @@ public void encode(String s, Writer writer) throws IOException {
             writer.write(URLEncoder.encode(s, StandardCharsets.UTF_8));
         }
     }
+
+    /**
+     * Build a new {@link CustomMustacheFactory} object.
+     */
+    public static class Builder {
+        private String mediaType = DEFAULT_MEDIA_TYPE;
+        private boolean detectMissingParams = DEFAULT_DETECT_MISSING_PARAMS;
+
+        private Builder() {}
+
+        public Builder mediaType(String mediaType) {
+            this.mediaType = mediaType;
+            return this;
+        }
+
+        /**
+         * Sets the behavior for handling missing parameters during template execution.
+         *
+         * @param detectMissingParams If true, an exception is thrown when executing the template with missing parameters.
+         *                            If false, the template gracefully handles missing parameters without throwing an exception.
+         */
+        public Builder detectMissingParams(boolean detectMissingParams) {
+            this.detectMissingParams = detectMissingParams;
+            return this;
+        }
+
+        public CustomMustacheFactory build() {
+            return new CustomMustacheFactory(mediaType, detectMissingParams);
+        }
+    }
 }

modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/CustomReflectionObjectHandler.java

@@ -8,7 +8,15 @@
 
 package org.elasticsearch.script.mustache;
 
+import com.github.mustachejava.Binding;
+import com.github.mustachejava.Code;
+import com.github.mustachejava.ObjectHandler;
+import com.github.mustachejava.TemplateContext;
+import com.github.mustachejava.codes.ValueCode;
+import com.github.mustachejava.reflect.GuardedBinding;
+import com.github.mustachejava.reflect.MissingWrapper;
 import com.github.mustachejava.reflect.ReflectionObjectHandler;
+import com.github.mustachejava.util.Wrapper;
 
 import org.elasticsearch.common.util.CollectionUtils;
 import org.elasticsearch.common.util.Maps;
@@ -19,10 +27,16 @@
 import java.util.AbstractMap;
 import java.util.Collection;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
 final class CustomReflectionObjectHandler extends ReflectionObjectHandler {
+    private final boolean detectMissingParams;
+
+    CustomReflectionObjectHandler(boolean detectMissingParams) {
+        this.detectMissingParams = detectMissingParams;
+    }
 
     @Override
     public Object coerce(Object object) {
@@ -41,6 +55,11 @@ public Object coerce(Object object) {
         }
     }
 
+    @Override
+    public Binding createBinding(String name, TemplateContext tc, Code code) {
+        return detectMissingParams ? new DetectMissingParamsGuardedBinding(this, name, tc, code) : super.createBinding(name, tc, code);
+    }
+
     @Override
     @SuppressWarnings("rawtypes")
     protected AccessibleObject findMember(Class sClass, String name) {
@@ -59,6 +78,23 @@ protected AccessibleObject findMember(Class sClass, String name) {
         return null;
     }
 
+    static class DetectMissingParamsGuardedBinding extends GuardedBinding {
+        private final Code code;
+
+        DetectMissingParamsGuardedBinding(ObjectHandler oh, String name, TemplateContext tc, Code code) {
+            super(oh, name, tc, code);
+            this.code = code;
+        }
+
+        protected synchronized Wrapper getWrapper(String name, List<Object> scopes) {
+            Wrapper wrapper = super.getWrapper(name, scopes);
+            if (wrapper instanceof MissingWrapper && code instanceof ValueCode) {
+                throw new MustacheInvalidParameterException("Parameter [" + name + "] is missing");
+            }
+            return wrapper;
+        }
+    }
+
     static final class ArrayMap extends AbstractMap<Object, Object> implements Iterable<Object> {
 
         private final Object array;

modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheInvalidParameterException.java

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.script.mustache;
+
+import com.github.mustachejava.MustacheException;
+
+public class MustacheInvalidParameterException extends MustacheException {
+    MustacheInvalidParameterException(String message) {
+        super(message, null, null);
+    }
+}

modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MustacheScriptEngine.java

@@ -38,6 +38,10 @@
  * {@link Mustache} object can then be re-used for subsequent executions.
  */
 public final class MustacheScriptEngine implements ScriptEngine {
+    /**
+     * Compiler option to enable detection of missing parameters.
+     */
+    public static final String DETECT_MISSING_PARAMS_OPTION = "detect_missing_params";
     private static final Logger logger = LogManager.getLogger(MustacheScriptEngine.class);
 
     public static final String NAME = "mustache";
@@ -72,10 +76,20 @@ public Set<ScriptContext<?>> getSupportedContexts() {
     }
 
     private static CustomMustacheFactory createMustacheFactory(Map<String, String> options) {
-        if (options == null || options.isEmpty() || options.containsKey(Script.CONTENT_TYPE_OPTION) == false) {
-            return new CustomMustacheFactory();
+        CustomMustacheFactory.Builder builder = CustomMustacheFactory.builder();
+        if (options == null || options.isEmpty()) {
+            return builder.build();
+        }
+
+        if (options.containsKey(Script.CONTENT_TYPE_OPTION)) {
+            builder.mediaType(options.get(Script.CONTENT_TYPE_OPTION));
+        }
+
+        if (options.containsKey(DETECT_MISSING_PARAMS_OPTION)) {
+            builder.detectMissingParams(Boolean.valueOf(options.get(DETECT_MISSING_PARAMS_OPTION)));
         }
-        return new CustomMustacheFactory(options.get(Script.CONTENT_TYPE_OPTION));
+
+        return builder.build();
     }
 
     @Override
@@ -107,10 +121,17 @@ public String execute() {
             try {
                 template.execute(writer, params);
             } catch (Exception e) {
-                logger.error(() -> format("Error running %s", template), e);
+                if (shouldLogException(e)) {
+                    logger.error(() -> format("Error running %s", template), e);
+                }
                 throw new GeneralScriptException("Error running " + template, e);
             }
             return writer.toString();
         }
+
+        public boolean shouldLogException(Throwable e) {
+            return e.getCause() != null && e.getCause() instanceof MustacheInvalidParameterException == false;
+        }
     }
+
 }