From effd69ee6492061ef13d7f9cef6420d76458564c Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Fri, 28 Jun 2024 11:31:23 +0200 Subject: [PATCH 1/6] CTX-5921: Swap limit check implemented. --- coretex/cli/modules/node.py | 24 +++++++++++++++++++++++- coretex/statistics.py | 10 ++++++++++ coretex/utils/docker.py | 32 +++++++++++++++++++++++++++++++- 3 files changed, 64 insertions(+), 2 deletions(-) diff --git a/coretex/cli/modules/node.py b/coretex/cli/modules/node.py index 66640fd7..91edbfa6 100644 --- a/coretex/cli/modules/node.py +++ b/coretex/cli/modules/node.py @@ -255,6 +255,23 @@ def promptRam(config: Dict[str, Any], ramLimit: int) -> int: return nodeRam +def promptSwap(config: Dict[str, Any], swapLimit: int) -> int: + nodeSwap: int = clickPrompt( + f"Node SWAP memory limit in GB (Maximum: {swapLimit}GB) (press enter to use default)", + swapLimit, + type = int + ) + + if nodeSwap > swapLimit: + errorEcho( + f"ERROR: SWAP memory limit in Docker Desktop ({swapLimit}GB) is lower than the configured value ({nodeSwap}GB). " + "Please adjust resource limitations in Docker Desktop settings." + ) + return promptRam(config, swapLimit) + + return nodeSwap + + def _configureInitScript() -> str: initScript = clickPrompt("Enter a path to sh script which will be executed before Node starts", config_defaults.DEFAULT_INIT_SCRIPT, type = str) @@ -284,6 +301,7 @@ def checkResourceLimitations() -> None: def isConfigurationValid(config: Dict[str, Any]) -> bool: isValid = True cpuLimit, ramLimit = docker.getResourceLimits() + swapLimit = docker.getDockerSwapLimit() if not isinstance(config["nodeRam"], int): errorEcho(f"Invalid config \"nodeRam\" field type \"{type(config['nodeRam'])}\". Expected: \"int\"") @@ -305,6 +323,9 @@ def isConfigurationValid(config: Dict[str, Any]) -> bool: errorEcho(f"Configuration not valid. RAM limit in Docker Desktop ({ramLimit}GB) is lower than the configured value ({config['nodeRam']}GB)") isValid = False + if config["nodeSwap"] > swapLimit: + errorEcho(f"Configuration not valid. RAM limit in Docker Desktop ({swapLimit}GB) is lower than the configured value ({config['nodeSwap']}GB)") + isValid = False if config["nodeRam"] < config_defaults.MINIMUM_RAM: errorEcho(f"Configuration not valid. Minimum Node RAM requirement ({config_defaults.MINIMUM_RAM}GB) is higher than the configured value ({config['nodeRam']}GB)") @@ -317,6 +338,7 @@ def configureNode(config: Dict[str, Any], verbose: bool) -> None: highlightEcho("[Node Configuration]") cpuLimit, ramLimit = docker.getResourceLimits() + swapLimit = docker.getDockerSwapLimit() config["nodeName"] = clickPrompt("Node name", type = str) @@ -355,8 +377,8 @@ def configureNode(config: Dict[str, Any], verbose: bool) -> None: config["cpuCount"] = promptCpu(config, cpuLimit) config["nodeRam"] = promptRam(config, ramLimit) + config["nodeRam"] = promptSwap(config, swapLimit) - config["nodeSwap"] = clickPrompt("Node swap memory limit in GB, make sure it is larger than mem limit (press enter to use default)", config_defaults.DEFAULT_SWAP_MEMORY, type = int) config["nodeSharedMemory"] = clickPrompt("Node POSIX shared memory limit in GB (press enter to use default)", config_defaults.DEFAULT_SHARED_MEMORY, type = int) config["allowDocker"] = clickPrompt("Allow Node to access system docker? This is a security risk! (Y/n)", config_defaults.DEFAULT_ALLOW_DOCKER, type = bool) config["initScript"] = _configureInitScript() diff --git a/coretex/statistics.py b/coretex/statistics.py index c6013e62..12a8a4fd 100644 --- a/coretex/statistics.py +++ b/coretex/statistics.py @@ -128,6 +128,16 @@ def getSwapUsage() -> float: return psutil.swap_memory().percent +def getTotalSwapMemory() -> int: + """ + Returns + ------- + int -> total swap memory in GB + """ + + return int(psutil.swap_memory().total / (1024 ** 3)) + + def getDiskRead() -> float: """ Returns diff --git a/coretex/utils/docker.py b/coretex/utils/docker.py index 197ebd63..15eeb96d 100644 --- a/coretex/utils/docker.py +++ b/coretex/utils/docker.py @@ -1,8 +1,11 @@ -from typing import Dict, Any, List, Tuple +from typing import Dict, Any, List, Tuple, Optional +from pathlib import Path import json +import platform from .process import command, CommandException +from ..statistics import getTotalSwapMemory def isDockerAvailable() -> None: @@ -147,3 +150,30 @@ def getResourceLimits() -> Tuple[int, int]: jsonOutput = json.loads(output) return jsonOutput["NCPU"], round(jsonOutput["MemTotal"] / (1024 ** 3)) + + +def getDockerConfigPath() -> Optional[Path]: + if platform.system() == "Darwin": + return Path.home().joinpath("Library", "Group Containers", "group.com.docker", "settings.json") + elif platform.system() == "Windows": + return Path.home().joinpath("AppData", "Roaming", "Docker", "settings.json") + elif platform.system() == "Linux": + return Path.home().joinpath(".docker", "desktop", "settings.json") + else: + return None + + +def getDockerSwapLimit() -> int: + configPath = getDockerConfigPath() + + if configPath is None: + return getTotalSwapMemory() + + with configPath.open("r") as configFile: + configJson = json.load(configFile) + + swapLimit = configJson.get("swapMiB") + if not isinstance(swapLimit, int): + raise TypeError(f"Expected \"int\" received \"{type(swapLimit)}\"") + + return int(swapLimit / 1024) From 09e9d3dff4a1f32963bec7896d340892d2682c91 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Fri, 28 Jun 2024 13:27:26 +0200 Subject: [PATCH 2/6] CTX-5921: Added new condition in onBeforeCommandExecute() callback which will check if "command_name" is inside excludeOptions also. --- coretex/cli/main.py | 2 +- coretex/cli/modules/utils.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/coretex/cli/main.py b/coretex/cli/main.py index 296fc1b4..56512e5c 100644 --- a/coretex/cli/main.py +++ b/coretex/cli/main.py @@ -61,7 +61,7 @@ def update() -> None: @click.group(cls = ClickExceptionInterceptor) -@utils.onBeforeCommandExecute(utils.checkLibVersion) +@utils.onBeforeCommandExecute(utils.checkLibVersion, excludeOptions = ["update"]) def cli() -> None: pass diff --git a/coretex/cli/modules/utils.py b/coretex/cli/modules/utils.py index 4894be3b..986fc7f9 100644 --- a/coretex/cli/modules/utils.py +++ b/coretex/cli/modules/utils.py @@ -157,6 +157,9 @@ def onBeforeCommandExecute(fun: Callable[..., Any], excludeOptions: Optional[Lis def decorator(f: Callable[..., Any]) -> Callable[..., Any]: @wraps(f) def wrapper(*args: Any, **kwargs: Any) -> Any: + if click.get_current_context().invoked_subcommand in excludeOptions: + return f(*args, **kwargs) + for key, value in click.get_current_context().params.items(): if key in excludeOptions and value: return f(*args, **kwargs) From 6a6dc6bbd7992b13b5ceb370c0c786a0496c2964 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Fri, 28 Jun 2024 13:28:17 +0200 Subject: [PATCH 3/6] Revert "CTX-5921: Added new condition in onBeforeCommandExecute() callback which will check if "command_name" is inside excludeOptions also." This reverts commit 09e9d3dff4a1f32963bec7896d340892d2682c91. --- coretex/cli/main.py | 2 +- coretex/cli/modules/utils.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/coretex/cli/main.py b/coretex/cli/main.py index 56512e5c..296fc1b4 100644 --- a/coretex/cli/main.py +++ b/coretex/cli/main.py @@ -61,7 +61,7 @@ def update() -> None: @click.group(cls = ClickExceptionInterceptor) -@utils.onBeforeCommandExecute(utils.checkLibVersion, excludeOptions = ["update"]) +@utils.onBeforeCommandExecute(utils.checkLibVersion) def cli() -> None: pass diff --git a/coretex/cli/modules/utils.py b/coretex/cli/modules/utils.py index 986fc7f9..4894be3b 100644 --- a/coretex/cli/modules/utils.py +++ b/coretex/cli/modules/utils.py @@ -157,9 +157,6 @@ def onBeforeCommandExecute(fun: Callable[..., Any], excludeOptions: Optional[Lis def decorator(f: Callable[..., Any]) -> Callable[..., Any]: @wraps(f) def wrapper(*args: Any, **kwargs: Any) -> Any: - if click.get_current_context().invoked_subcommand in excludeOptions: - return f(*args, **kwargs) - for key, value in click.get_current_context().params.items(): if key in excludeOptions and value: return f(*args, **kwargs) From f3bbaddc1af7bd8ca6d5641ad0a3f15b0738ec05 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Mon, 1 Jul 2024 10:10:40 +0200 Subject: [PATCH 4/6] CTX-5921: discussion changes. --- coretex/cli/modules/node.py | 8 ++++---- coretex/utils/docker.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/coretex/cli/modules/node.py b/coretex/cli/modules/node.py index 91edbfa6..310b8eff 100644 --- a/coretex/cli/modules/node.py +++ b/coretex/cli/modules/node.py @@ -258,16 +258,16 @@ def promptRam(config: Dict[str, Any], ramLimit: int) -> int: def promptSwap(config: Dict[str, Any], swapLimit: int) -> int: nodeSwap: int = clickPrompt( f"Node SWAP memory limit in GB (Maximum: {swapLimit}GB) (press enter to use default)", - swapLimit, + min(swapLimit, config.get("nodeRam", 0) * 2), type = int ) if nodeSwap > swapLimit: errorEcho( f"ERROR: SWAP memory limit in Docker Desktop ({swapLimit}GB) is lower than the configured value ({nodeSwap}GB). " - "Please adjust resource limitations in Docker Desktop settings." + f"If you want to use higher value than {swapLimit}GB, you have to change docker limits." ) - return promptRam(config, swapLimit) + return promptSwap(config, swapLimit) return nodeSwap @@ -377,7 +377,7 @@ def configureNode(config: Dict[str, Any], verbose: bool) -> None: config["cpuCount"] = promptCpu(config, cpuLimit) config["nodeRam"] = promptRam(config, ramLimit) - config["nodeRam"] = promptSwap(config, swapLimit) + config["nodeSwap"] = promptSwap(config, swapLimit) config["nodeSharedMemory"] = clickPrompt("Node POSIX shared memory limit in GB (press enter to use default)", config_defaults.DEFAULT_SHARED_MEMORY, type = int) config["allowDocker"] = clickPrompt("Allow Node to access system docker? This is a security risk! (Y/n)", config_defaults.DEFAULT_ALLOW_DOCKER, type = bool) diff --git a/coretex/utils/docker.py b/coretex/utils/docker.py index 15eeb96d..3e4bb879 100644 --- a/coretex/utils/docker.py +++ b/coretex/utils/docker.py @@ -166,7 +166,7 @@ def getDockerConfigPath() -> Optional[Path]: def getDockerSwapLimit() -> int: configPath = getDockerConfigPath() - if configPath is None: + if configPath is None or not configPath.exists(): return getTotalSwapMemory() with configPath.open("r") as configFile: @@ -174,6 +174,6 @@ def getDockerSwapLimit() -> int: swapLimit = configJson.get("swapMiB") if not isinstance(swapLimit, int): - raise TypeError(f"Expected \"int\" received \"{type(swapLimit)}\"") + return getTotalSwapMemory() return int(swapLimit / 1024) From 2c9b8cfda90d710d909c2eb28d401c4760885515 Mon Sep 17 00:00:00 2001 From: Bogdan Tintor Date: Mon, 1 Jul 2024 11:22:48 +0200 Subject: [PATCH 5/6] CTX-5921: Discussion change. --- coretex/cli/modules/node.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/coretex/cli/modules/node.py b/coretex/cli/modules/node.py index 310b8eff..ee52604f 100644 --- a/coretex/cli/modules/node.py +++ b/coretex/cli/modules/node.py @@ -255,10 +255,10 @@ def promptRam(config: Dict[str, Any], ramLimit: int) -> int: return nodeRam -def promptSwap(config: Dict[str, Any], swapLimit: int) -> int: +def promptSwap(nodeRam: int, swapLimit: int) -> int: nodeSwap: int = clickPrompt( f"Node SWAP memory limit in GB (Maximum: {swapLimit}GB) (press enter to use default)", - min(swapLimit, config.get("nodeRam", 0) * 2), + min(swapLimit, nodeRam * 2), type = int ) @@ -267,7 +267,7 @@ def promptSwap(config: Dict[str, Any], swapLimit: int) -> int: f"ERROR: SWAP memory limit in Docker Desktop ({swapLimit}GB) is lower than the configured value ({nodeSwap}GB). " f"If you want to use higher value than {swapLimit}GB, you have to change docker limits." ) - return promptSwap(config, swapLimit) + return promptSwap(nodeRam, swapLimit) return nodeSwap @@ -377,7 +377,7 @@ def configureNode(config: Dict[str, Any], verbose: bool) -> None: config["cpuCount"] = promptCpu(config, cpuLimit) config["nodeRam"] = promptRam(config, ramLimit) - config["nodeSwap"] = promptSwap(config, swapLimit) + config["nodeSwap"] = promptSwap(config.get("nodeRam", 0), swapLimit) config["nodeSharedMemory"] = clickPrompt("Node POSIX shared memory limit in GB (press enter to use default)", config_defaults.DEFAULT_SHARED_MEMORY, type = int) config["allowDocker"] = clickPrompt("Allow Node to access system docker? This is a security risk! (Y/n)", config_defaults.DEFAULT_ALLOW_DOCKER, type = bool) From e0be7c33ac1ebcc3afd7774d1a00095131a9e06b Mon Sep 17 00:00:00 2001 From: bogdant36 <137895499+bogdant36@users.noreply.github.com> Date: Mon, 1 Jul 2024 13:45:02 +0200 Subject: [PATCH 6/6] CTX-5921: Discussion changes. --- coretex/cli/modules/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coretex/cli/modules/node.py b/coretex/cli/modules/node.py index ee52604f..6508c525 100644 --- a/coretex/cli/modules/node.py +++ b/coretex/cli/modules/node.py @@ -377,7 +377,7 @@ def configureNode(config: Dict[str, Any], verbose: bool) -> None: config["cpuCount"] = promptCpu(config, cpuLimit) config["nodeRam"] = promptRam(config, ramLimit) - config["nodeSwap"] = promptSwap(config.get("nodeRam", 0), swapLimit) + config["nodeSwap"] = promptSwap(config["nodeRam"], swapLimit) config["nodeSharedMemory"] = clickPrompt("Node POSIX shared memory limit in GB (press enter to use default)", config_defaults.DEFAULT_SHARED_MEMORY, type = int) config["allowDocker"] = clickPrompt("Allow Node to access system docker? This is a security risk! (Y/n)", config_defaults.DEFAULT_ALLOW_DOCKER, type = bool)