diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java index 290e407ab664c..e82e53458cb32 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java @@ -678,10 +678,7 @@ public void validate(OpenJobAction.JobParams params, ClusterState clusterState) PersistentTasksCustomMetaData.Assignment assignment = selectLeastLoadedMlNode(params.getJobId(), clusterState, maxConcurrentJobAllocations, fallbackMaxNumberOfOpenJobs, maxMachineMemoryPercent, logger); if (assignment.getExecutorNode() == null) { - String msg = "Could not open job because no suitable nodes were found, allocation explanation [" - + assignment.getExplanation() + "]"; - logger.warn("[{}] {}", params.getJobId(), msg); - throw new ElasticsearchStatusException(msg, RestStatus.TOO_MANY_REQUESTS); + throw makeNoSuitableNodesException(logger, params.getJobId(), assignment.getExplanation()); } } @@ -785,9 +782,9 @@ public boolean test(PersistentTasksCustomMetaData.PersistentTask persistentTa // and this is why this class must only be used when opening a job if (assignment != null && assignment.equals(PersistentTasksCustomMetaData.INITIAL_ASSIGNMENT) == false && assignment.isAssigned() == false) { + OpenJobAction.JobParams params = (OpenJobAction.JobParams) persistentTask.getParams(); // Assignment has failed on the master node despite passing our "fast fail" validation - exception = new ElasticsearchStatusException("Could not open job because no suitable nodes were found, " + - "allocation explanation [" + assignment.getExplanation() + "]", RestStatus.TOO_MANY_REQUESTS); + exception = makeNoSuitableNodesException(logger, params.getJobId(), assignment.getExplanation()); // The persistent task should be cancelled so that the observed outcome is the // same as if the "fast fail" validation on the coordinating node had failed shouldCancel = true; @@ -813,4 +810,12 @@ public boolean test(PersistentTasksCustomMetaData.PersistentTask persistentTa } } } + + static ElasticsearchException makeNoSuitableNodesException(Logger logger, String jobId, String explanation) { + String msg = "Could not open job because no suitable nodes were found, allocation explanation [" + explanation + "]"; + logger.warn("[{}] {}", jobId, msg); + Exception detail = new IllegalStateException(msg); + return new ElasticsearchStatusException("Could not open job because no ML nodes with sufficient capacity were found", + RestStatus.TOO_MANY_REQUESTS, detail); + } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java index 9e8d17e84b44a..83facf4c1f2de 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/BasicDistributedJobsIT.java @@ -366,10 +366,14 @@ public void testMlIndicesNotAvailable() throws Exception { Exception e = expectThrows(ElasticsearchStatusException.class, () -> client().execute(OpenJobAction.INSTANCE, openJobRequest).actionGet()); - assertTrue(e.getMessage(), - e.getMessage().startsWith("Could not open job because no suitable nodes were found, allocation explanation")); - assertTrue(e.getMessage(), e.getMessage().endsWith("because not all primary shards are active for the following indices " - + "[.ml-state,.ml-anomalies-shared]]")); + assertEquals("Could not open job because no ML nodes with sufficient capacity were found", e.getMessage()); + IllegalStateException detail = (IllegalStateException) e.getCause(); + assertNotNull(detail); + String detailedMessage = detail.getMessage(); + assertTrue(detailedMessage, + detailedMessage.startsWith("Could not open job because no suitable nodes were found, allocation explanation")); + assertTrue(detailedMessage, detailedMessage.endsWith("because not all primary shards are active for the following indices " + + "[.ml-state,.ml-anomalies-shared]]")); logger.info("Start data node"); String nonMlNode = internalCluster().startNode(Settings.builder() diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java index f06b73fcd40aa..68ce5da5ca176 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/TooManyJobsIT.java @@ -92,18 +92,22 @@ private void verifyMaxNumberOfJobsLimit(int numNodes, int maxNumberOfJobsPerNode }); logger.info("Opened {}th job", i); } catch (ElasticsearchStatusException e) { - assertTrue(e.getMessage(), - e.getMessage().startsWith("Could not open job because no suitable nodes were found, allocation explanation")); + assertEquals("Could not open job because no ML nodes with sufficient capacity were found", e.getMessage()); + IllegalStateException detail = (IllegalStateException) e.getCause(); + assertNotNull(detail); + String detailedMessage = detail.getMessage(); + assertTrue(detailedMessage, + detailedMessage.startsWith("Could not open job because no suitable nodes were found, allocation explanation")); if (expectMemoryLimitBeforeCountLimit) { int expectedJobsAlreadyOpenOnNode = (i - 1) / numNodes; - assertTrue(e.getMessage(), - e.getMessage().endsWith("because this node has insufficient available memory. Available memory for ML [" + - maxMlMemoryPerNode + "], memory required by existing jobs [" + - (expectedJobsAlreadyOpenOnNode * memoryFootprintPerJob) + - "], estimated memory required for this job [" + memoryFootprintPerJob + "]]")); + assertTrue(detailedMessage, + detailedMessage.endsWith("because this node has insufficient available memory. Available memory for ML [" + + maxMlMemoryPerNode + "], memory required by existing jobs [" + + (expectedJobsAlreadyOpenOnNode * memoryFootprintPerJob) + "], estimated memory required for this job [" + + memoryFootprintPerJob + "]]")); } else { - assertTrue(e.getMessage(), e.getMessage().endsWith("because this node is full. Number of opened jobs [" + - maxNumberOfJobsPerNode + "], xpack.ml.max_open_jobs [" + maxNumberOfJobsPerNode + "]]")); + assertTrue(detailedMessage, detailedMessage.endsWith("because this node is full. Number of opened jobs [" + + maxNumberOfJobsPerNode + "], xpack.ml.max_open_jobs [" + maxNumberOfJobsPerNode + "]]")); } logger.info("good news everybody --> reached maximum number of allowed opened jobs, after trying to open the {}th job", i);