Merge pull request #154 from elastic/master

🤖 ESQL: Merge upstream

docs/changelog/87366.yaml

@@ -0,0 +1,5 @@
+pr: 87366
+summary: Improve scalability of NLP models
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/87735.yaml

@@ -0,0 +1,5 @@
+pr: 87735
+summary: Use desired nodes during data tier allocation decisions
+area: Allocation
+type: enhancement
+issues: []

docs/reference/ml/anomaly-detection/functions/ml-time-functions.asciidoc

@@ -2,8 +2,8 @@
 = Time functions
 
 The time functions detect events that happen at unusual times, either of the day
-or of the week. These functions can be used to find unusual patterns of behavior,
-typically associated with suspicious user activity.
+or of the week. These functions can be used to find unusual patterns of 
+behavior, typically associated with suspicious user activity.
 
 The {ml-features} include the following time functions:
 
@@ -77,6 +77,12 @@ its past behavior.
 The `time_of_week` function detects when events occur that are outside normal
 usage patterns. For example, it detects login events on the weekend.
 
+IMPORTANT: The `time_of_week` function models time in epoch seconds modulo the 
+  duration of a week in seconds. It means that the `typical` and `actual` values 
+  are seconds after a whole number of weeks since 1/1/1970 in UTC which is a 
+  Thursday. For example, a value of `475` is 475 seconds after midnight on 
+  Thursday in UTC. 
+
 This function supports the following properties:
 
 * `by_field_name` (optional)
@@ -102,3 +108,5 @@ models when events occur throughout the week for each `eventcode`. It detects
 when a workstation event occurs at an unusual time during the week for that
 `eventcode` compared to other workstations. It detects events for a
 particular workstation that are outside the normal usage pattern.
+
+

libs/core/src/main/java/org/elasticsearch/core/IOUtils.java

@@ -31,8 +31,6 @@
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
 import java.nio.file.attribute.BasicFileAttributes;
-import java.util.Arrays;
-import java.util.Collection;
 import java.util.LinkedHashMap;
 import java.util.Map;
 
@@ -62,7 +60,7 @@ private IOUtils() {
      * @param objects objects to close
      */
     public static void close(final Closeable... objects) throws IOException {
-        close(null, Arrays.asList(objects));
+        close(null, objects);
     }
 
     /**
@@ -82,8 +80,28 @@ public static void close(@Nullable Closeable closeable) throws IOException {
      *
      * @param objects objects to close
      */
-    public static void close(final Exception e, final Closeable... objects) throws IOException {
-        close(e, Arrays.asList(objects));
+    public static void close(final Exception ex, final Closeable... objects) throws IOException {
+        Exception firstException = ex;
+        for (final Closeable object : objects) {
+            try {
+                close(object);
+            } catch (final IOException | RuntimeException e) {
+                firstException = addOrSuppress(firstException, e);
+            }
+        }
+
+        if (firstException != null) {
+            throwRuntimeOrIOException(firstException);
+        }
+    }
+
+    private static void throwRuntimeOrIOException(Exception firstException) throws IOException {
+        if (firstException instanceof IOException) {
+            throw (IOException) firstException;
+        } else {
+            // since we only assigned an IOException or a RuntimeException to ex above, in this case ex must be a RuntimeException
+            throw (RuntimeException) firstException;
+        }
     }
 
     /**
@@ -95,50 +113,38 @@ public static void close(final Exception e, final Closeable... objects) throws I
      * @param objects objects to close
      */
     public static void close(final Iterable<? extends Closeable> objects) throws IOException {
-        close(null, objects);
-    }
-
-    /**
-     * Closes all given {@link Closeable}s. If a non-null exception is passed in, or closing a
-     * stream causes an exception, throws the exception with other {@link RuntimeException} or
-     * {@link IOException} exceptions added as suppressed.
-     *
-     * @param ex existing Exception to add exceptions occurring during close to
-     * @param objects objects to close
-     *
-     * @see #close(Closeable...)
-     */
-    public static void close(final Exception ex, final Iterable<? extends Closeable> objects) throws IOException {
-        Exception firstException = ex;
+        Exception firstException = null;
         for (final Closeable object : objects) {
             try {
                 close(object);
             } catch (final IOException | RuntimeException e) {
-                if (firstException == null) {
-                    firstException = e;
-                } else {
-                    firstException.addSuppressed(e);
-                }
+                firstException = addOrSuppress(firstException, e);
             }
         }
 
         if (firstException != null) {
-            if (firstException instanceof IOException) {
-                throw (IOException) firstException;
-            } else {
-                // since we only assigned an IOException or a RuntimeException to ex above, in this case ex must be a RuntimeException
-                throw (RuntimeException) firstException;
-            }
+            throwRuntimeOrIOException(firstException);
         }
     }
 
+    private static Exception addOrSuppress(Exception firstException, Exception e) {
+        if (firstException == null) {
+            firstException = e;
+        } else {
+            firstException.addSuppressed(e);
+        }
+        return firstException;
+    }
+
     /**
      * Closes all given {@link Closeable}s, suppressing all thrown exceptions. Some of the {@link Closeable}s may be null, they are ignored.
      *
      * @param objects objects to close
      */
     public static void closeWhileHandlingException(final Closeable... objects) {
-        closeWhileHandlingException(Arrays.asList(objects));
+        for (final Closeable object : objects) {
+            closeWhileHandlingException(object);
+        }
     }
 
     /**
@@ -170,15 +176,6 @@ public static void closeWhileHandlingException(final Closeable closeable) {
      * @param files the paths of files to delete
      */
     public static void deleteFilesIgnoringExceptions(final Path... files) {
-        deleteFilesIgnoringExceptions(Arrays.asList(files));
-    }
-
-    /**
-     * Deletes all given files, suppressing all thrown {@link IOException}s. Some of the files may be null, if so they are ignored.
-     *
-     * @param files the paths of files to delete
-     */
-    public static void deleteFilesIgnoringExceptions(final Collection<? extends Path> files) {
         for (final Path name : files) {
             if (name != null) {
                 // noinspection EmptyCatchBlock

libs/core/src/main/java/org/elasticsearch/core/Releasables.java

@@ -10,29 +10,22 @@
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
-import java.util.Arrays;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /** Utility methods to work with {@link Releasable}s. */
 public enum Releasables {
     ;
 
-    private static void close(Iterable<? extends Releasable> releasables, boolean ignoreException) {
+    /** Release the provided {@link Releasable}s. */
+    public static void close(Iterable<? extends Releasable> releasables) {
         try {
             // this does the right thing with respect to add suppressed and not wrapping errors etc.
             IOUtils.close(releasables);
         } catch (IOException e) {
-            if (ignoreException == false) {
-                throw new UncheckedIOException(e);
-            }
+            throw new UncheckedIOException(e);
         }
     }
 
-    /** Release the provided {@link Releasable}s. */
-    public static void close(Iterable<? extends Releasable> releasables) {
-        close(releasables, false);
-    }
-
     /** Release the provided {@link Releasable}. */
     public static void close(@Nullable Releasable releasable) {
         try {
@@ -44,7 +37,7 @@ public static void close(@Nullable Releasable releasable) {
 
     /** Release the provided {@link Releasable}s. */
     public static void close(Releasable... releasables) {
-        close(Arrays.asList(releasables));
+        close(true, releasables);
     }
 
     /** Release the provided {@link Releasable}s expecting no exception to by thrown by any of them. */
@@ -69,17 +62,19 @@ public static void closeExpectNoException(Releasable releasable) {
 
     /** Release the provided {@link Releasable}s, ignoring exceptions. */
     public static void closeWhileHandlingException(Releasable... releasables) {
-        close(Arrays.asList(releasables), true);
+        close(false, releasables);
     }
 
     /** Release the provided {@link Releasable}s, ignoring exceptions if <code>success</code> is {@code false}. */
-    public static void close(boolean success, Iterable<Releasable> releasables) {
-        close(releasables, success == false);
-    }
-
-    /** Release the provided {@link Releasable}s, ignoring exceptions if <code>success</code> is {@code false}. */
-    public static void close(boolean success, Releasable... releasables) {
-        close(success, Arrays.asList(releasables));
+    private static void close(boolean success, Releasable... releasables) {
+        try {
+            // this does the right thing with respect to add suppressed and not wrapping errors etc.
+            IOUtils.close(releasables);
+        } catch (IOException e) {
+            if (success) {
+                throw new UncheckedIOException(e);
+            }
+        }
     }
 
     /** Wrap several releasables into a single one. This is typically useful for use with try-with-resources: for example let's assume

libs/core/src/test/java/org/elasticsearch/core/IOUtilsTests.java

@@ -117,10 +117,6 @@ public void testDeleteFilesIgnoringExceptionsArray() throws IOException {
         runDeleteFilesIgnoringExceptionsTest(Function.identity(), IOUtils::deleteFilesIgnoringExceptions);
     }
 
-    public void testDeleteFilesIgnoringExceptionsIterable() throws IOException {
-        runDeleteFilesIgnoringExceptionsTest((Function<Path[], List<Path>>) Arrays::asList, IOUtils::deleteFilesIgnoringExceptions);
-    }
-
     private <T> void runDeleteFilesIgnoringExceptionsTest(
         final Function<Path[], T> function,
         CheckedConsumer<T, IOException> deleteFilesIgnoringExceptions

rest-api-spec/src/main/resources/rest-api-spec/api/mget.json

@@ -35,6 +35,12 @@
       ]
     },
     "params":{
+      "force_synthetic_source": {
+        "type": "boolean",
+        "description": "Should this request force synthetic _source? Use this to test if the mapping supports synthetic _source and to get a sense of the worst case performance. Fetches with this enabled will be slower the enabling synthetic source natively in the index.",
+        "visibility": "feature_flag",
+        "feature_flag": "es.index_mode_feature_flag_registered"
+      },
       "stored_fields":{
         "type":"list",
         "description":"A comma-separated list of stored fields to return in the response"

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml

@@ -45,3 +45,117 @@ keyword:
   - match:
       docs.1._source:
         kwd: bar
+
+---
+force_synthetic_source_ok:
+  - skip:
+      version: " - 8.3.99"
+      reason: introduced in 8.4.0
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            _source:
+              synthetic: false
+            properties:
+              kwd:
+                type: keyword
+
+  - do:
+      index:
+        index:   test
+        id:      1
+        body:
+          kwd: foo
+
+  - do:
+      index:
+        index:   test
+        id:      2
+        body:
+          kwd: bar
+
+  # When _source is used in the fetch the original _source is perfect
+  - do:
+      mget:
+        index: test
+        body:
+          ids: [1, 2]
+  - match:
+      docs.0._source:
+        kwd: foo
+  - match:
+      docs.1._source:
+        kwd: bar
+
+  # When we force synthetic source dots in field names get turned into objects
+  - do:
+      mget:
+        index: test
+        force_synthetic_source: true
+        body:
+          ids: [ 1, 2 ]
+  - match:
+      docs.0._source:
+        kwd: foo
+  - match:
+      docs.1._source:
+        kwd: bar
+
+---
+force_synthetic_source_bad_mapping:
+  - skip:
+      version: " - 8.3.99"
+      reason: introduced in 8.4.0
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: 1 # Use a single shard to get consistent error messages
+          mappings:
+            _source:
+              synthetic: false
+            properties:
+              text:
+                type: text
+
+  - do:
+      index:
+        index:   test
+        id:      1
+        body:
+          text: foo
+
+  - do:
+      index:
+        index:   test
+        id:      2
+        body:
+          text: bar
+
+  # When _source is used in the fetch the original _source is perfect
+  - do:
+      mget:
+        index: test
+        body:
+          ids: [ 1, 2 ]
+  - match:
+      docs.0._source:
+        text: foo
+  - match:
+      docs.1._source:
+        text: bar
+
+  # Forcing synthetic source fails because the mapping is invalid
+  - do:
+      mget:
+        index: test
+        force_synthetic_source: true
+        body:
+          ids: [ 1, 2 ]
+  - match: {docs.0.error.reason: "field [text] of type [text] doesn't support synthetic source unless it has a sub-field of type [keyword] with doc values enabled and without ignore_above or a normalizer"}
+  - match: {docs.1.error.reason: "field [text] of type [text] doesn't support synthetic source unless it has a sub-field of type [keyword] with doc values enabled and without ignore_above or a normalizer"}