-
Notifications
You must be signed in to change notification settings - Fork 24.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[ML] provide tmp storage for forecasting and possibly any ml native j…
…obs #30399 This implementation lazily (on 1st forecast request) checks for available diskspace and creates a subfolder for storing data outside of Lucene indexes, but as part of the ES data paths. Details: - tmp storage is managed and does not allow allocation if disk space is below a threshold (5GB at the moment) - tmp storage is supposed to be managed by the native component but in case this fails cleanup is provided: - on job close - on process crash - after node crash, on restart - available space is re-checked for every forecast call (the native component has to check again before writing) Note: The 1st path that has enough space is chosen on job open (job close/reopen triggers a new search)
- Loading branch information
Hendrik Muhs
committed
May 18, 2018
1 parent
6bbd1b8
commit 5f16c92
Showing
10 changed files
with
406 additions
and
26 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
123 changes: 123 additions & 0 deletions
123
...plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/NativeStorageProvider.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,123 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License; | ||
| * you may not use this file except in compliance with the Elastic License. | ||
| */ | ||
|
|
||
| package org.elasticsearch.xpack.ml.job.process; | ||
|
|
||
| import org.apache.logging.log4j.Logger; | ||
| import org.elasticsearch.common.logging.Loggers; | ||
| import org.elasticsearch.common.unit.ByteSizeValue; | ||
| import org.elasticsearch.core.internal.io.IOUtils; | ||
| import org.elasticsearch.env.Environment; | ||
|
|
||
| import java.io.IOException; | ||
| import java.nio.file.Files; | ||
| import java.nio.file.Path; | ||
|
|
||
| /** | ||
| * Provide storage for native components. | ||
| */ | ||
| public class NativeStorageProvider { | ||
|
|
||
| private static final Logger LOGGER = Loggers.getLogger(NativeStorageProvider.class); | ||
|
|
||
|
|
||
| private static final String LOCAL_STORAGE_SUBFOLDER = "ml-local-data"; | ||
| private static final String LOCAL_STORAGE_TMP_FOLDER = "tmp"; | ||
|
|
||
| private final Environment environment; | ||
|
|
||
| // do not allow any usage below this threshold | ||
| private final ByteSizeValue minLocalStorageAvailable; | ||
|
|
||
| public NativeStorageProvider(Environment environment, ByteSizeValue minDiskSpaceOffHeap) { | ||
| this.environment = environment; | ||
| this.minLocalStorageAvailable = minDiskSpaceOffHeap; | ||
| } | ||
|
|
||
| /** | ||
| * Removes any temporary storage leftovers. | ||
| * | ||
| * Removes all temp files and folder which might be there as a result of an | ||
| * unclean node shutdown or broken clients. | ||
| * | ||
| * Do not call while there are running jobs. | ||
| * | ||
| * @throws IOException if cleanup fails | ||
| */ | ||
| public void cleanupLocalTmpStorageInCaseOfUncleanShutdown() throws IOException { | ||
| for (Path p : environment.dataFiles()) { | ||
| IOUtils.rm(p.resolve(LOCAL_STORAGE_SUBFOLDER).resolve(LOCAL_STORAGE_TMP_FOLDER)); | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Tries to find local storage for storing temporary data. | ||
| * | ||
| * @param uniqueIdentifier An identifier to be used as sub folder | ||
| * @param requestedSize The maximum size required | ||
| * @return Path for temporary storage if available, null otherwise | ||
| */ | ||
| public Path tryGetLocalTmpStorage(String uniqueIdentifier, ByteSizeValue requestedSize) { | ||
| for (Path path : environment.dataFiles()) { | ||
| try { | ||
| if (getUsableSpace(path) >= requestedSize.getBytes() + minLocalStorageAvailable.getBytes()) { | ||
| Path tmpDirectory = path.resolve(LOCAL_STORAGE_SUBFOLDER).resolve(LOCAL_STORAGE_TMP_FOLDER).resolve(uniqueIdentifier); | ||
| Files.createDirectories(tmpDirectory); | ||
| return tmpDirectory; | ||
| } | ||
| } catch (IOException e) { | ||
| LOGGER.debug("Failed to obtain information about path [{}]: {}", path, e); | ||
| } | ||
|
|
||
| } | ||
| LOGGER.debug("Failed to find native storage for [{}], returning null", uniqueIdentifier); | ||
| return null; | ||
| } | ||
|
|
||
| public boolean localTmpStorageHasEnoughSpace(Path path, ByteSizeValue requestedSize) { | ||
| Path realPath = path.toAbsolutePath(); | ||
| for (Path p : environment.dataFiles()) { | ||
| try { | ||
| if (realPath.startsWith(p.resolve(LOCAL_STORAGE_SUBFOLDER).resolve(LOCAL_STORAGE_TMP_FOLDER))) { | ||
| return getUsableSpace(p) >= requestedSize.getBytes() + minLocalStorageAvailable.getBytes(); | ||
| } | ||
| } catch (IOException e) { | ||
| LOGGER.debug("Failed to optain information about path [{}]: {}", path, e); | ||
| } | ||
| } | ||
|
|
||
| LOGGER.debug("Not enough space left for path [{}]", path); | ||
| return false; | ||
| } | ||
|
|
||
| /** | ||
| * Delete temporary storage, previously allocated | ||
| * | ||
| * @param path | ||
| * Path to temporary storage | ||
| * @throws IOException | ||
| * if path can not be cleaned up | ||
| */ | ||
| public void cleanupLocalTmpStorage(Path path) throws IOException { | ||
| // do not allow to breakout from the tmp storage provided | ||
| Path realPath = path.toAbsolutePath(); | ||
| for (Path p : environment.dataFiles()) { | ||
| if (realPath.startsWith(p.resolve(LOCAL_STORAGE_SUBFOLDER).resolve(LOCAL_STORAGE_TMP_FOLDER))) { | ||
| IOUtils.rm(path); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| long getUsableSpace(Path path) throws IOException { | ||
| long freeSpaceInBytes = Environment.getFileStore(path).getUsableSpace(); | ||
|
|
||
| /* See: https://bugs.openjdk.java.net/browse/JDK-8162520 */ | ||
| if (freeSpaceInBytes < 0) { | ||
| freeSpaceInBytes = Long.MAX_VALUE; | ||
| } | ||
| return freeSpaceInBytes; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.