diff --git a/docs/changelog/87366.yaml b/docs/changelog/87366.yaml
new file mode 100644
index 0000000000000..0b2881e7c4778
--- /dev/null
+++ b/docs/changelog/87366.yaml
@@ -0,0 +1,5 @@
+pr: 87366
+summary: Improve scalability of NLP models
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/87735.yaml b/docs/changelog/87735.yaml
new file mode 100644
index 0000000000000..43f296f1411ee
--- /dev/null
+++ b/docs/changelog/87735.yaml
@@ -0,0 +1,5 @@
+pr: 87735
+summary: Use desired nodes during data tier allocation decisions
+area: Allocation
+type: enhancement
+issues: []
diff --git a/docs/reference/ml/anomaly-detection/functions/ml-time-functions.asciidoc b/docs/reference/ml/anomaly-detection/functions/ml-time-functions.asciidoc
index d67dfe084e11e..096fd817ccc4c 100644
--- a/docs/reference/ml/anomaly-detection/functions/ml-time-functions.asciidoc
+++ b/docs/reference/ml/anomaly-detection/functions/ml-time-functions.asciidoc
@@ -2,8 +2,8 @@
= Time functions
The time functions detect events that happen at unusual times, either of the day
-or of the week. These functions can be used to find unusual patterns of behavior,
-typically associated with suspicious user activity.
+or of the week. These functions can be used to find unusual patterns of
+behavior, typically associated with suspicious user activity.
The {ml-features} include the following time functions:
@@ -77,6 +77,12 @@ its past behavior.
The `time_of_week` function detects when events occur that are outside normal
usage patterns. For example, it detects login events on the weekend.
+IMPORTANT: The `time_of_week` function models time in epoch seconds modulo the
+ duration of a week in seconds. It means that the `typical` and `actual` values
+ are seconds after a whole number of weeks since 1/1/1970 in UTC which is a
+ Thursday. For example, a value of `475` is 475 seconds after midnight on
+ Thursday in UTC.
+
This function supports the following properties:
* `by_field_name` (optional)
@@ -102,3 +108,5 @@ models when events occur throughout the week for each `eventcode`. It detects
when a workstation event occurs at an unusual time during the week for that
`eventcode` compared to other workstations. It detects events for a
particular workstation that are outside the normal usage pattern.
+
+
diff --git a/libs/core/src/main/java/org/elasticsearch/core/IOUtils.java b/libs/core/src/main/java/org/elasticsearch/core/IOUtils.java
index 0398418e503bc..fc2b10bf5b480 100644
--- a/libs/core/src/main/java/org/elasticsearch/core/IOUtils.java
+++ b/libs/core/src/main/java/org/elasticsearch/core/IOUtils.java
@@ -31,8 +31,6 @@
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.BasicFileAttributes;
-import java.util.Arrays;
-import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
@@ -62,7 +60,7 @@ private IOUtils() {
* @param objects objects to close
*/
public static void close(final Closeable... objects) throws IOException {
- close(null, Arrays.asList(objects));
+ close(null, objects);
}
/**
@@ -82,8 +80,28 @@ public static void close(@Nullable Closeable closeable) throws IOException {
*
* @param objects objects to close
*/
- public static void close(final Exception e, final Closeable... objects) throws IOException {
- close(e, Arrays.asList(objects));
+ public static void close(final Exception ex, final Closeable... objects) throws IOException {
+ Exception firstException = ex;
+ for (final Closeable object : objects) {
+ try {
+ close(object);
+ } catch (final IOException | RuntimeException e) {
+ firstException = addOrSuppress(firstException, e);
+ }
+ }
+
+ if (firstException != null) {
+ throwRuntimeOrIOException(firstException);
+ }
+ }
+
+ private static void throwRuntimeOrIOException(Exception firstException) throws IOException {
+ if (firstException instanceof IOException) {
+ throw (IOException) firstException;
+ } else {
+ // since we only assigned an IOException or a RuntimeException to ex above, in this case ex must be a RuntimeException
+ throw (RuntimeException) firstException;
+ }
}
/**
@@ -95,50 +113,38 @@ public static void close(final Exception e, final Closeable... objects) throws I
* @param objects objects to close
*/
public static void close(final Iterable extends Closeable> objects) throws IOException {
- close(null, objects);
- }
-
- /**
- * Closes all given {@link Closeable}s. If a non-null exception is passed in, or closing a
- * stream causes an exception, throws the exception with other {@link RuntimeException} or
- * {@link IOException} exceptions added as suppressed.
- *
- * @param ex existing Exception to add exceptions occurring during close to
- * @param objects objects to close
- *
- * @see #close(Closeable...)
- */
- public static void close(final Exception ex, final Iterable extends Closeable> objects) throws IOException {
- Exception firstException = ex;
+ Exception firstException = null;
for (final Closeable object : objects) {
try {
close(object);
} catch (final IOException | RuntimeException e) {
- if (firstException == null) {
- firstException = e;
- } else {
- firstException.addSuppressed(e);
- }
+ firstException = addOrSuppress(firstException, e);
}
}
if (firstException != null) {
- if (firstException instanceof IOException) {
- throw (IOException) firstException;
- } else {
- // since we only assigned an IOException or a RuntimeException to ex above, in this case ex must be a RuntimeException
- throw (RuntimeException) firstException;
- }
+ throwRuntimeOrIOException(firstException);
}
}
+ private static Exception addOrSuppress(Exception firstException, Exception e) {
+ if (firstException == null) {
+ firstException = e;
+ } else {
+ firstException.addSuppressed(e);
+ }
+ return firstException;
+ }
+
/**
* Closes all given {@link Closeable}s, suppressing all thrown exceptions. Some of the {@link Closeable}s may be null, they are ignored.
*
* @param objects objects to close
*/
public static void closeWhileHandlingException(final Closeable... objects) {
- closeWhileHandlingException(Arrays.asList(objects));
+ for (final Closeable object : objects) {
+ closeWhileHandlingException(object);
+ }
}
/**
@@ -170,15 +176,6 @@ public static void closeWhileHandlingException(final Closeable closeable) {
* @param files the paths of files to delete
*/
public static void deleteFilesIgnoringExceptions(final Path... files) {
- deleteFilesIgnoringExceptions(Arrays.asList(files));
- }
-
- /**
- * Deletes all given files, suppressing all thrown {@link IOException}s. Some of the files may be null, if so they are ignored.
- *
- * @param files the paths of files to delete
- */
- public static void deleteFilesIgnoringExceptions(final Collection extends Path> files) {
for (final Path name : files) {
if (name != null) {
// noinspection EmptyCatchBlock
diff --git a/libs/core/src/main/java/org/elasticsearch/core/Releasables.java b/libs/core/src/main/java/org/elasticsearch/core/Releasables.java
index 5cd8794428b96..05e3d154929a7 100644
--- a/libs/core/src/main/java/org/elasticsearch/core/Releasables.java
+++ b/libs/core/src/main/java/org/elasticsearch/core/Releasables.java
@@ -10,29 +10,22 @@
import java.io.IOException;
import java.io.UncheckedIOException;
-import java.util.Arrays;
import java.util.concurrent.atomic.AtomicBoolean;
/** Utility methods to work with {@link Releasable}s. */
public enum Releasables {
;
- private static void close(Iterable extends Releasable> releasables, boolean ignoreException) {
+ /** Release the provided {@link Releasable}s. */
+ public static void close(Iterable extends Releasable> releasables) {
try {
// this does the right thing with respect to add suppressed and not wrapping errors etc.
IOUtils.close(releasables);
} catch (IOException e) {
- if (ignoreException == false) {
- throw new UncheckedIOException(e);
- }
+ throw new UncheckedIOException(e);
}
}
- /** Release the provided {@link Releasable}s. */
- public static void close(Iterable extends Releasable> releasables) {
- close(releasables, false);
- }
-
/** Release the provided {@link Releasable}. */
public static void close(@Nullable Releasable releasable) {
try {
@@ -44,7 +37,7 @@ public static void close(@Nullable Releasable releasable) {
/** Release the provided {@link Releasable}s. */
public static void close(Releasable... releasables) {
- close(Arrays.asList(releasables));
+ close(true, releasables);
}
/** Release the provided {@link Releasable}s expecting no exception to by thrown by any of them. */
@@ -69,17 +62,19 @@ public static void closeExpectNoException(Releasable releasable) {
/** Release the provided {@link Releasable}s, ignoring exceptions. */
public static void closeWhileHandlingException(Releasable... releasables) {
- close(Arrays.asList(releasables), true);
+ close(false, releasables);
}
/** Release the provided {@link Releasable}s, ignoring exceptions if success
is {@code false}. */
- public static void close(boolean success, Iterablesuccess
is {@code false}. */
- public static void close(boolean success, Releasable... releasables) {
- close(success, Arrays.asList(releasables));
+ private static void close(boolean success, Releasable... releasables) {
+ try {
+ // this does the right thing with respect to add suppressed and not wrapping errors etc.
+ IOUtils.close(releasables);
+ } catch (IOException e) {
+ if (success) {
+ throw new UncheckedIOException(e);
+ }
+ }
}
/** Wrap several releasables into a single one. This is typically useful for use with try-with-resources: for example let's assume
diff --git a/libs/core/src/test/java/org/elasticsearch/core/IOUtilsTests.java b/libs/core/src/test/java/org/elasticsearch/core/IOUtilsTests.java
index 719c2e3ef4846..0d7576a56acb0 100644
--- a/libs/core/src/test/java/org/elasticsearch/core/IOUtilsTests.java
+++ b/libs/core/src/test/java/org/elasticsearch/core/IOUtilsTests.java
@@ -117,10 +117,6 @@ public void testDeleteFilesIgnoringExceptionsArray() throws IOException {
runDeleteFilesIgnoringExceptionsTest(Function.identity(), IOUtils::deleteFilesIgnoringExceptions);
}
- public void testDeleteFilesIgnoringExceptionsIterable() throws IOException {
- runDeleteFilesIgnoringExceptionsTest((Function
+ * This concept is useful as it provides more context about future topology changes to the system + * as well as the desired set of nodes in the cluster, allowing it to make better decisions + * about allocation, autoscaling, auto-expand replicas, etc. + *
+ * + *+ * Additionally, settings validation is done during desired nodes updates avoiding boot-looping + * when an invalid setting is provided before the node is started. + *
+ * + *+ * To modify the desired nodes it is necessary to provide the entire collection of nodes that will + * be part of the proposed cluster topology. + *
+ * + *+ * Desired nodes are expected to be part of a lineage defined by the provided {@code historyId}. + * The {@code historyId} is provided by the orchestrator taking care of managing the cluster. + * In order to identify the different proposed desired nodes within the same history, it is + * also expected that the orchestrator provides a monotonically increasing {@code version} + * when it communicates about future topology changes. + * The cluster rejects desired nodes updated with a {@code version} less than or equal + * than the current {@code version} for the same {@code historyId}. + *
+ * + *+ * The {@code historyId} is expected to remain stable during the cluster lifecycle, but it is + * possible that the orchestrator loses its own state and needs to be restored to a + * previous point in time with an older desired nodes {@code version}. In those cases it is + * expected to use new {@code historyId} that would allow starting from a different version. + *
+ * + *+ * Each {@link DesiredNode} part of {@link DesiredNodes} has a {@link DesiredNodeWithStatus.Status} + * depending on whether or not the node has been part of the cluster at some point. + *
+ * + * The two possible statuses {@link DesiredNodeWithStatus.Status} are: + *+ * See {@code JoinTaskExecutor} and {@code TransportUpdateDesiredNodesAction} for more details about + * desired nodes status tracking. + *
+ * + *+ * Finally, each {@link DesiredNode} is expected to provide a way of identifying the node when it joins, + * {@link Node#NODE_EXTERNAL_ID_SETTING} allows providing that identity through settings. + *
+ * + */ public class DesiredNodes implements Writeable, ToXContentObject, Iterable