From fe3637002293d542b2e99297050ce9e9b23d2ec2 Mon Sep 17 00:00:00 2001 From: Olawumi Salaam Date: Wed, 29 Jan 2025 08:24:59 +0100 Subject: [PATCH 01/10] Handle BentoML errors & clean up failed models --- ersilia/hub/fetch/fetch.py | 74 ++++++++++++++++++++++++++++---------- 1 file changed, 56 insertions(+), 18 deletions(-) diff --git a/ersilia/hub/fetch/fetch.py b/ersilia/hub/fetch/fetch.py index 0e1d4cd26..0a1352bdb 100644 --- a/ersilia/hub/fetch/fetch.py +++ b/ersilia/hub/fetch/fetch.py @@ -25,6 +25,15 @@ FetchResult = namedtuple("FetchResult", ["fetch_success", "reason"]) +class BentoMLError(Exception): + """ + Raised when an error occurs in BentoML-related processes. + """ + + def __init__(self, message: str): + super().__init__(message) + + class ModelFetcher(ErsiliaBase): """ ModelFetcher is responsible for fetching models from various sources. @@ -186,24 +195,37 @@ def _fetch_from_fastapi(self): @throw_ersilia_exception() def _fetch_from_bentoml(self): self.logger.debug("Fetching using BentoML") - self.check_bentoml() - fetch = importlib.import_module("ersilia.hub.fetch.fetch_bentoml") - mf = fetch.ModelFetcherFromBentoML( - config_json=self.config_json, - credentials_json=self.credentials_json, - overwrite=self.overwrite, - repo_path=self.repo_path, - mode=self.mode, - pip=self.do_pip, - dockerize=self.do_docker, - force_from_github=self.force_from_github, - force_from_s3=self.force_from_s3, - ) - if mf.seems_installable(model_id=self.model_id): - mf.fetch(model_id=self.model_id) - else: - self.logger.debug("Not installable with BentoML") - raise NotInstallableWithBentoML(model_id=self.model_id) + try: + self.check_bentoml() + + fetch = importlib.import_module("ersilia.hub.fetch.fetch_bentoml") + mf = fetch.ModelFetcherFromBentoML( + config_json=self.config_json, + credentials_json=self.credentials_json, + overwrite=self.overwrite, + repo_path=self.repo_path, + mode=self.mode, + pip=self.do_pip, + dockerize=self.do_docker, + force_from_github=self.force_from_github, + force_from_s3=self.force_from_s3, + ) + + # Check if the model can be installed with BentoML + try: + if mf.seems_installable(model_id=self.model_id): + mf.fetch(model_id=self.model_id) + else: + raise NotInstallableWithBentoML(model_id=self.model_id) + + except NotInstallableWithBentoML: + raise + + except NotInstallableWithBentoML: + raise + + except Exception as e: + raise BentoMLError(f"An error occurred during BentoML fetching: {e}") @throw_ersilia_exception() def _fetch_not_from_dockerhub(self, model_id: str): @@ -395,4 +417,20 @@ async def fetch(self, model_id: str) -> bool: fetch_success=True, reason="Model fetched successfully" ) else: + # Handle BentoML-specific errors here + if isinstance(fr.reason, BentoMLError): + self.logger.debug("BentoML fetching failed, deleting artifacts") + do_delete = yes_no_input( + "Do you want to delete the model artifacts? [Y/n]", + default_answer="Y", + ) + if do_delete: + md = ModelFullDeleter(overwrite=False) + md.delete(model_id) + self.logger.info( + f"Model '{model_id}' artifacts successfully deleted." + ) + print( + f"✅ Model '{model_id}' artifacts have been successfully deleted." + ) return fr From 90980a10073e587c9d4a0850dd339161a059b9fa Mon Sep 17 00:00:00 2001 From: Dhanshree Arora Date: Tue, 28 Jan 2025 22:58:12 +0530 Subject: [PATCH 02/10] update version and release date [skip ci] --- CITATION.cff | 2 +- codemeta.json | 4 ++-- ersilia/_static_version.py | 2 +- pyproject.toml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index b2b4fce09..0ef75996d 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -11,7 +11,7 @@ authors: given-names: Miquel orcid: https://orcid.org/0000-0002-9906-6936 title: 'Ersilia Model Hub: a repository of AI/ML models for neglected tropical diseases' -version: 0.1.40 +version: 0.1.41 doi: 10.5281/zenodo.7274645 date-released: '' url: https://github.com/ersilia-os/ersilia diff --git a/codemeta.json b/codemeta.json index 0e2f6d7c9..e4894c1fe 100644 --- a/codemeta.json +++ b/codemeta.json @@ -28,7 +28,7 @@ "givenName": "Miquel" } ], - "codeRepository": "https://github.com/ersilia-os/ersilia/v0.1.40", + "codeRepository": "https://github.com/ersilia-os/ersilia/v0.1.41", "dateCreated": "2021-01-01", "dateModified": "2024-10-01", "datePublished": "2022-10-06", @@ -221,7 +221,7 @@ ], "url": "https://ersilia.io", "downloadUrl": "https://github.com/ersilia-os/ersilia/archive/refs/tags/v0.1.37.tar.gz", - "version": "0.1.40", + "version": "0.1.41", "relatedLink": "https://ersilia.gitbook.io", "developmentStatus": "active", "issueTracker": "https://github.com/ersilia-os/ersilia/issues" diff --git a/ersilia/_static_version.py b/ersilia/_static_version.py index 7624855a8..cd6e2cedd 100644 --- a/ersilia/_static_version.py +++ b/ersilia/_static_version.py @@ -1 +1 @@ -version = "0.1.40" +version = "0.1.41" diff --git a/pyproject.toml b/pyproject.toml index 5f374a497..c1df33646 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "ersilia" -version = "0.1.40" +version = "0.1.41" description = "A hub of AI/ML models for open source drug discovery and global health" license = "GPLv3" authors = ["Ersilia Open Source Initiative "] @@ -52,7 +52,7 @@ numpy = "<=1.26.4" aiofiles = "<=24.1.0" aiohttp = ">=3.10.11" nest_asyncio = "<=1.6.0" -isaura = { version = "0.1.40", optional = true } +isaura = { version = "0.1.41", optional = true } pytest = { version = "^7.4.0", optional = true } pytest-asyncio = { version = "<=0.24.0", optional = true } pytest-benchmark = { version = "<=4.0.0", optional = true } From 40311a7cfaa9ce7cb06e85c459a9ec13db64ccc3 Mon Sep 17 00:00:00 2001 From: Olawumi Salaam Date: Mon, 3 Feb 2025 13:09:32 +0100 Subject: [PATCH 03/10] Handle BentoML errors by encapsulating subprocess calls --- ersilia/core/base.py | 21 +++-- ersilia/hub/content/catalog.py | 63 +++++++------ ersilia/hub/fetch/fetch.py | 139 +++++++++++----------------- ersilia/tools/bentoml/exceptions.py | 8 +- ersilia/utils/terminal.py | 46 +++++---- 5 files changed, 139 insertions(+), 138 deletions(-) diff --git a/ersilia/core/base.py b/ersilia/core/base.py index 260cc1fbe..3a3d00d79 100644 --- a/ersilia/core/base.py +++ b/ersilia/core/base.py @@ -1,11 +1,12 @@ import os -import subprocess from pathlib import Path from .. import logger from ..default import EOS +from ..tools.bentoml.exceptions import BentoMLException from ..utils.config import Config, Credentials from ..utils.paths import resolve_pack_method +from ..utils.terminal import run_command home = str(Path.home()) @@ -70,6 +71,7 @@ def _get_bentoml_location(self, model_id): tag = self._get_latest_bentoml_tag(model_id) path = os.path.join(self._bentoml_dir, model_id) if not os.path.exists(path): + self.logger.debug(f"BentoML path not found: {path}") return None if tag is not None: return os.path.join(path, tag) @@ -80,6 +82,7 @@ def _get_bundle_location(self, model_id): tag = self._get_latest_bundle_tag(model_id) path = os.path.join(self._bundles_dir, model_id) if not os.path.exists(path): + self.logger.debug(f"Bundle path not found: {path}") return None if tag is not None: return os.path.join(path, tag) @@ -90,16 +93,21 @@ def _get_bento_location(self, model_id): bundle_path = self._get_bundle_location(model_id) if resolve_pack_method(bundle_path) != "bentoml": return None - cmd = ["bentoml", "get", "%s:latest" % model_id, "--print-location", "--quiet"] - result = subprocess.run(cmd, stdout=subprocess.PIPE) - result = result.stdout.decode("utf-8").rstrip() - return result + + cmd = ["bentoml", "get", f"{model_id}:latest", "--print-location", "--quiet"] + stdout, stderr, returncode = run_command(cmd, quiet=True) + + if returncode != 0: + self.logger.error(f"BentoML command failed: {stderr}") + raise BentoMLException(f"BentoML error: {stderr}") + return stdout.strip() def _is_ready(self, model_id): """Check whether a model exists in the local computer""" try: self._get_latest_bundle_tag(model_id) - except: + except Exception as e: + self.logger.debug(f"Model {model_id} not ready: {str(e)}") return False path = os.path.join(self._abs_path(self._dest_dir), model_id) if not os.path.exists(path): @@ -108,5 +116,6 @@ def _is_ready(self, model_id): def _has_credentials(self): if self.cred is None: + self.logger.warning("No credentials found.") return False return True diff --git a/ersilia/hub/content/catalog.py b/ersilia/hub/content/catalog.py index 45f363ea4..f6c214ee4 100644 --- a/ersilia/hub/content/catalog.py +++ b/ersilia/hub/content/catalog.py @@ -3,13 +3,13 @@ import csv import json import os -import shutil -import subprocess from ... import ErsiliaBase from ...db.hubdata.interfaces import JsonModelsInterface -from ...default import BENTOML_PATH, MODEL_SOURCE_FILE, TableConstants +from ...default import MODEL_SOURCE_FILE, TableConstants +from ...tools.bentoml.exceptions import BentoMLException from ...utils.identifiers.model import ModelIdentifier +from ...utils.terminal import run_command from .card import ModelCard try: @@ -357,26 +357,37 @@ def bentoml(self) -> CatalogTable: The catalog table containing the models available as BentoServices. """ try: - result = subprocess.run( - ["bentoml", "list"], stdout=subprocess.PIPE, env=os.environ, timeout=10 - ) - except Exception: - shutil.rmtree(BENTOML_PATH) - return None - result = [r for r in result.stdout.decode("utf-8").split("\n") if r] - if len(result) == 1: - return - columns = ["BENTO_SERVICE", "AGE", "APIS", "ARTIFACTS"] - header = result[0] - values = result[1:] - cut_idxs = [] - for col in columns: - cut_idxs += [header.find(col)] - R = [] - for row in values: - r = [] - for i, idx in enumerate(zip(cut_idxs, cut_idxs[1:] + [None])): - r += [row[idx[0] : idx[1]].rstrip()] - R += [[r[0].split(":")[0]] + r] - columns = ["Identifier"] + columns - return CatalogTable(data=R, columns=columns) + stdout, stderr, returncode = run_command(["bentoml", "list"], quiet=True) + if returncode != 0: + raise BentoMLException(f"BentoML list failed: {stderr}") + + # Process stdout to build CatalogTable + output_lines = stdout.split("\n") + if not output_lines or len(output_lines) == 1: + return CatalogTable(data=[], columns=[]) # Return empty table + + # Extract columns and values + columns = ["BENTO_SERVICE", "AGE", "APIS", "ARTIFACTS"] + header = output_lines[0] + values = output_lines[1:] + + # Parse table data + cut_idxs = [header.find(col) for col in columns] + R = [] + for row in values: + r = [] + for i, idx in enumerate(zip(cut_idxs, cut_idxs[1:] + [None])): + r.append( + row[idx[0] : idx[1]].rstrip() + if idx[1] + else row[idx[0] :].rstrip() + ) + R.append([r[0].split(":")[0]] + r) + + return CatalogTable(data=R, columns=["Identifier"] + columns) + + except BentoMLException: + raise + except Exception as e: + self.logger.error(f"Unexpected error: {str(e)}") + raise BentoMLException(f"Failed to fetch BentoML models: {str(e)}") diff --git a/ersilia/hub/fetch/fetch.py b/ersilia/hub/fetch/fetch.py index 0a1352bdb..71bec1c31 100644 --- a/ersilia/hub/fetch/fetch.py +++ b/ersilia/hub/fetch/fetch.py @@ -10,6 +10,7 @@ from ...hub.delete.delete import ModelFullDeleter from ...hub.fetch.actions.template_resolver import TemplateResolver from ...setup.requirements import check_bentoml +from ...tools.bentoml.exceptions import BentoMLException from ...utils.exceptions_utils.fetch_exceptions import ( NotInstallableWithBentoML, NotInstallableWithFastAPI, @@ -25,15 +26,6 @@ FetchResult = namedtuple("FetchResult", ["fetch_success", "reason"]) -class BentoMLError(Exception): - """ - Raised when an error occurs in BentoML-related processes. - """ - - def __init__(self, message: str): - super().__init__(message) - - class ModelFetcher(ErsiliaBase): """ ModelFetcher is responsible for fetching models from various sources. @@ -195,37 +187,26 @@ def _fetch_from_fastapi(self): @throw_ersilia_exception() def _fetch_from_bentoml(self): self.logger.debug("Fetching using BentoML") - try: - self.check_bentoml() - - fetch = importlib.import_module("ersilia.hub.fetch.fetch_bentoml") - mf = fetch.ModelFetcherFromBentoML( - config_json=self.config_json, - credentials_json=self.credentials_json, - overwrite=self.overwrite, - repo_path=self.repo_path, - mode=self.mode, - pip=self.do_pip, - dockerize=self.do_docker, - force_from_github=self.force_from_github, - force_from_s3=self.force_from_s3, - ) - - # Check if the model can be installed with BentoML - try: - if mf.seems_installable(model_id=self.model_id): - mf.fetch(model_id=self.model_id) - else: - raise NotInstallableWithBentoML(model_id=self.model_id) - - except NotInstallableWithBentoML: - raise + self.check_bentoml() - except NotInstallableWithBentoML: - raise + fetch = importlib.import_module("ersilia.hub.fetch.fetch_bentoml") + mf = fetch.ModelFetcherFromBentoML( + config_json=self.config_json, + credentials_json=self.credentials_json, + overwrite=self.overwrite, + repo_path=self.repo_path, + mode=self.mode, + pip=self.do_pip, + dockerize=self.do_docker, + force_from_github=self.force_from_github, + force_from_s3=self.force_from_s3, + ) - except Exception as e: - raise BentoMLError(f"An error occurred during BentoML fetching: {e}") + # Check if the model can be installed with BentoML + if mf.seems_installable(model_id=self.model_id): + mf.fetch(model_id=self.model_id) + else: + raise NotInstallableWithBentoML(model_id=self.model_id) @throw_ersilia_exception() def _fetch_not_from_dockerhub(self, model_id: str): @@ -385,52 +366,40 @@ async def fetch(self, model_id: str) -> bool: fetcher = ModelFetcher(config_json=config) success = await fetcher.fetch(model_id="eosxxxx") """ - fr = await self._fetch(model_id) - if fr.fetch_success: + try: + fr = await self._fetch(model_id) + if not fr.fetch_success: + return fr + + self._standard_csv_example(model_id) + self.logger.debug("Writing model source to file") + model_source_file = os.path.join( + self._model_path(model_id), MODEL_SOURCE_FILE + ) try: - self._standard_csv_example(model_id) - except StandardModelExampleError: - self.logger.debug("Standard model example failed, deleting artifacts") - do_delete = yes_no_input( - "Do you want to delete the model artifacts? [Y/n]", - default_answer="Y", - ) - if do_delete: - md = ModelFullDeleter(overwrite=False) - md.delete(model_id) - return FetchResult( - fetch_success=False, - reason="Could not successfully run a standard example from the model.", - ) - else: - self.logger.debug("Writing model source to file") - model_source_file = os.path.join( - self._model_path(model_id), MODEL_SOURCE_FILE - ) - try: - os.makedirs(self._model_path(model_id), exist_ok=True) - except OSError as error: - self.logger.error(f"Error during folder creation: {error}") - with open(model_source_file, "w") as f: - f.write(self.model_source) - return FetchResult( - fetch_success=True, reason="Model fetched successfully" - ) - else: - # Handle BentoML-specific errors here - if isinstance(fr.reason, BentoMLError): - self.logger.debug("BentoML fetching failed, deleting artifacts") - do_delete = yes_no_input( - "Do you want to delete the model artifacts? [Y/n]", - default_answer="Y", + os.makedirs(self._model_path(model_id), exist_ok=True) + except OSError as error: + self.logger.error(f"Error during folder creation: {error}") + with open(model_source_file, "w") as f: + f.write(self.model_source) + + return FetchResult(fetch_success=True, reason="Model fetched successfully") + + except (StandardModelExampleError, BentoMLException) as err: + self.logger.debug(f"{type(err).__name__} occurred: {str(err)}") + do_delete = yes_no_input( + "Do you want to delete the model artifacts? [Y/n]", + default_answer="Y", + ) + if do_delete: + md = ModelFullDeleter(overwrite=False) + md.delete(model_id) + self.logger.info( + f"✅ Model '{model_id}' artifacts have been successfully deleted." ) - if do_delete: - md = ModelFullDeleter(overwrite=False) - md.delete(model_id) - self.logger.info( - f"Model '{model_id}' artifacts successfully deleted." - ) - print( - f"✅ Model '{model_id}' artifacts have been successfully deleted." - ) - return fr + + reason = ( + str(err) if str(err) else "An unknown error occurred during fetching." + ) + return FetchResult(fetch_success=False, reason=reason) + return fr diff --git a/ersilia/tools/bentoml/exceptions.py b/ersilia/tools/bentoml/exceptions.py index c9889f2a2..426b0b5ef 100644 --- a/ersilia/tools/bentoml/exceptions.py +++ b/ersilia/tools/bentoml/exceptions.py @@ -1,9 +1,15 @@ class BentoMLException(Exception): """ Exception raised for errors in the BentoML tool. + + Parameters + ---------- + message : str + A custom error message describing the issue. """ - pass + def __init__(self, message: str): + super().__init__(message) class BentoMLConfigException(Exception): diff --git a/ersilia/utils/terminal.py b/ersilia/utils/terminal.py index 356fb50fa..4c2e75cec 100644 --- a/ersilia/utils/terminal.py +++ b/ersilia/utils/terminal.py @@ -3,6 +3,7 @@ import os import shutil import subprocess +import sys try: from inputimeout import TimeoutOccurred, inputimeout @@ -38,35 +39,40 @@ def is_quiet(): def run_command(cmd, quiet=None): """ - Run a shell command. + Run a shell command and return stdout, stderr, and return code. Parameters ---------- cmd : str or list The command to run. quiet : bool, optional - Whether to run the command in quiet mode. Default is None. + Whether to run the command in quiet mode. Defaults to `is_quiet()`. """ if quiet is None: quiet = is_quiet() - if type(cmd) == str: - if quiet: - with open(os.devnull, "w") as fp: - subprocess.Popen( - cmd, stdout=fp, stderr=fp, shell=True, env=os.environ - ).wait() - else: - subprocess.Popen(cmd, shell=True, env=os.environ).wait() - else: - if quiet: - subprocess.check_call( - cmd, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - env=os.environ, - ) - else: - subprocess.check_call(cmd, env=os.environ) + # Run the command and capture outputs + result = subprocess.run( + cmd, + shell=isinstance(cmd, str), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=os.environ, + ) + + # Decode outputs + stdout = result.stdout.strip() + stderr = result.stderr.strip() + returncode = result.returncode + + # Log outputs if not in quiet mode + if not quiet: + if stdout: + print(stdout) + if stderr: + print(stderr, file=sys.stderr) + + return stdout, stderr, returncode def run_command_check_output(cmd): From 6dc331e5e84f1c8d89efddc2df5d3f89263d68eb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Feb 2025 19:28:49 +0530 Subject: [PATCH 04/10] Bump abatilo/actions-poetry from 3.0.1 to 4.0.0 (#1530) * upgrade loguru (#1525) * Bump abatilo/actions-poetry from 3.0.1 to 4.0.0 Bumps [abatilo/actions-poetry](https://github.com/abatilo/actions-poetry) from 3.0.1 to 4.0.0. - [Release notes](https://github.com/abatilo/actions-poetry/releases) - [Changelog](https://github.com/abatilo/actions-poetry/blob/master/.releaserc) - [Commits](https://github.com/abatilo/actions-poetry/compare/v3.0.1...v4.0.0) --- updated-dependencies: - dependency-name: abatilo/actions-poetry dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: Dhanshree Arora Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/publish.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index df460b632..48f97cbab 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -121,7 +121,7 @@ jobs: python-version: '3.8' - name: Python Poetry Action - uses: abatilo/actions-poetry@v3.0.1 + uses: abatilo/actions-poetry@v4.0.0 - name: Build and publish env: From 46a07850b347a9c787905cabd1b00580f3da9032 Mon Sep 17 00:00:00 2001 From: Olawumi Salaam Date: Thu, 6 Feb 2025 19:39:50 +0100 Subject: [PATCH 05/10] Fix JSONCodeError --- ersilia/serve/services.py | 47 ++++++-- ersilia/setup/requirements/bentoml.py | 167 +++++++++++++++++--------- 2 files changed, 145 insertions(+), 69 deletions(-) diff --git a/ersilia/serve/services.py b/ersilia/serve/services.py index a8b0c11c3..70f7a82a7 100644 --- a/ersilia/serve/services.py +++ b/ersilia/serve/services.py @@ -21,8 +21,10 @@ PACK_METHOD_FASTAPI, PACKMODE_FILE, ) +from ..setup.requirements.bentoml import BentoMLRequirement from ..setup.requirements.conda import CondaRequirement from ..setup.requirements.docker import DockerRequirement +from ..tools.bentoml.exceptions import BentoMLException from ..utils.conda import SimpleConda, StandaloneConda from ..utils.docker import SimpleDocker, model_image_version_reader from ..utils.exceptions_utils.serve_exceptions import ( @@ -87,19 +89,44 @@ def _get_info_from_bento(self): self.logger.debug( "Getting info from BentoML and storing in {0}".format(tmp_file) ) - run_command(cmd) - with open(tmp_file, "r") as f: - info = json.load(f) - self.logger.debug("Info {0}".format(info)) - return info + # Check command success first + result = run_command(cmd) + if result.returncode != 0: + raise BentoMLException(f"BentoML info failed: {result.stderr}") + + # Handle JSON parsing errors here + try: + with open(tmp_file, "r") as f: + return json.load(f) + except json.JSONDecodeError as e: + self.logger.error(f"Invalid BentoML output: {e}") + raise BentoMLException("Corrupted BentoML installation detected") from e def _get_apis_from_bento(self): self.logger.debug("Getting APIs from Bento") - info = self._get_info_from_bento() - apis_list = [] - for item in info["apis"]: - apis_list += [item["name"]] - return apis_list + bento_requirement = BentoMLRequirement() + + try: + info = self._get_info_from_bento() + except BentoMLException as e: + # Handle both command failures and JSON errors here + if "Corrupted BentoML installation" in str(e): + self.logger.warning("Attempting BentoML cleanup...") + try: + bento_requirement._cleanup_corrupted_bentoml() + self.logger.info("Retrying API fetch after cleanup") + info = self._get_info_from_bento() # Retry + except Exception as cleanup_error: + raise BentoMLException( + f"Cleanup failed: {cleanup_error}" + ) from cleanup_error + else: + raise # Re-raise unrelated errors + + try: + return [item["name"] for item in info["apis"]] + except KeyError as e: + raise BentoMLException(f"Invalid API format: {e}") from e def _get_apis_from_fastapi(self): bundle_path = self._model_path(self.model_id) diff --git a/ersilia/setup/requirements/bentoml.py b/ersilia/setup/requirements/bentoml.py index 8a0dde9a0..2f5f16090 100644 --- a/ersilia/setup/requirements/bentoml.py +++ b/ersilia/setup/requirements/bentoml.py @@ -1,80 +1,129 @@ -import os -import subprocess import sys +from threading import Lock +from typing import Optional -from ...default import EOS +from packaging import version +from packaging.version import InvalidVersion + +from ...tools.bentoml.exceptions import BentoMLException +from ...utils.logging import logger +from ...utils.terminal import run_command class BentoMLRequirement(object): - """ - A class to handle the installation and version checking of BentoML for Ersilia. - - Methods - ------- - is_installed() - Checks if BentoML is installed. - is_bentoml_ersilia_version() - Checks if the installed BentoML version is the Ersilia version. - install() - Installs the Ersilia version of BentoML. - """ + """Handles installation and version checking of BentoML for Ersilia.""" + + _lock = Lock() def __init__(self): - pass + self.logger = logger def is_installed(self) -> bool: - """ - Checks if BentoML is installed. - - Returns - ------- - bool - True if BentoML is installed, False otherwise. - """ + """Checks if BentoML is installed.""" try: import bentoml # noqa: F401 return True except ImportError: + self.logger.debug("BentoML is not installed") return False - def is_bentoml_ersilia_version(self) -> bool: - """ - Checks if the installed BentoML version is the Ersilia version. - - Returns - ------- - bool - True if the installed BentoML version is the Ersilia version, False otherwise. - """ - if not self.is_installed(): - return False + def _get_bentoml_version(self) -> Optional[str]: + """Get BentoML version using run_command""" + result = run_command([sys.executable, "-m", "bentoml", "--version"]) - version_file = os.path.join(EOS, "bentomlversion.txt") + if result.returncode != 0: + self.logger.error(f"BentoML version check failed: {result.stderr}") + return None - if not os.path.exists(version_file): - cmd = "bentoml --version > {0}".format(version_file) - subprocess.Popen(cmd, shell=True).wait() + version_str = result.stdout.strip() - with open(version_file, "r") as f: - text = f.read() - if "0.11.0" in text: - return True - else: + try: + return version.parse(version_str).public + except InvalidVersion: + self.logger.error(f"Invalid BentoML version detected: {version_str}") + return None + + def is_bentoml_ersilia_version(self) -> bool: + """Checks if the installed BentoML version is the Ersilia version.""" + version_str = self._get_bentoml_version() + if not version_str: + return False + try: + return version.parse(version_str) == version.parse("0.11.0") + except InvalidVersion: + self.logger.error(f"Invalid BentoML version detected: {version_str}") return False - def install(self) -> None: - """ - Installs the Ersilia version of BentoML. - - This method installs the BentoML package from the Ersilia GitHub repository. - - Returns - ------- - None - """ - print("Installing bentoml (the ersilia version)") - cmd = "{0} -m pip install -U git+https://github.com/ersilia-os/bentoml-ersilia.git".format( - sys.executable - ) - subprocess.Popen(cmd, shell=True).wait() + def _cleanup_corrupted_bentoml(self) -> None: + """Forcefully uninstall BentoML and reinstall the correct version.""" + with self._lock: + self.logger.info("Cleaning up corrupted BentoML installation...") + + if self.is_installed(): + result = run_command( + [sys.executable, "-m", "pip", "uninstall", "bentoml", "-y"] + ) + if result.returncode != 0: + raise BentoMLException(f"Force uninstall failed: {result.stderr}") + + # ✅ Ensure reinstallation after cleanup + self.logger.info("Reinstalling Ersilia-compatible BentoML after cleanup...") + self.install(retries=1) # Only allow 1 retry to avoid infinite loops + + def install(self, retries: int = 3) -> None: + """Installs the Ersilia version of BentoML with error handling.""" + with self._lock: + if retries <= 0: + self.logger.critical( + "Final installation attempt failed. Manual intervention required." + ) + raise BentoMLException("Installation failed after multiple attempts.") + + try: + # 1. Uninstall if wrong version exists + if self.is_installed() and not self.is_bentoml_ersilia_version(): + self.logger.info("Uninstalling incompatible BentoML version...") + uninstall_result = run_command( + [sys.executable, "-m", "pip", "uninstall", "bentoml", "-y"] + ) + if uninstall_result.returncode != 0: + raise BentoMLException( + f"Uninstall failed: {uninstall_result.stderr}" + ) + + # 2. Install specific version + self.logger.info("Installing Ersilia-compatible BentoML...") + install_result = run_command( + [ + sys.executable, + "-m", + "pip", + "install", + "-U", + "git+https://github.com/ersilia-os/bentoml-ersilia.git", + ] + ) + if install_result.returncode != 0: + raise BentoMLException(f"Install failed: {install_result.stderr}") + + # 3. Post-install verification + if not self.is_bentoml_ersilia_version(): + raise BentoMLException("Installed version verification failed") + + self.logger.info("BentoML was installed and verified successfully.") + + except BentoMLException as e: + self.logger.error(f"Install attempt failed: {str(e)}") + self.logger.warning(f"Attempts remaining: {retries-1}") + + try: + self._cleanup_corrupted_bentoml() + except Exception as cleanup_error: + self.logger.critical(f"Cleanup failed: {str(cleanup_error)}") + raise BentoMLException( + "Aborting due to failed cleanup" + ) from cleanup_error + + # Recursive retry with counter + self.install(retries=retries - 1) From 44ba16b0bdb5c604ae09abeb478dc294a560ade0 Mon Sep 17 00:00:00 2001 From: Olawumi Salaam Date: Wed, 12 Feb 2025 09:07:44 +0100 Subject: [PATCH 06/10] Fix BentoML error handling --- ersilia/serve/services.py | 4 +- ersilia/setup/requirements/__init__.py | 2 +- .../setup/requirements/bentoml_requirement.py | 131 ++++++++++++++++++ ersilia/utils/terminal.py | 28 ++-- 4 files changed, 152 insertions(+), 13 deletions(-) create mode 100644 ersilia/setup/requirements/bentoml_requirement.py diff --git a/ersilia/serve/services.py b/ersilia/serve/services.py index 70f7a82a7..52190b179 100644 --- a/ersilia/serve/services.py +++ b/ersilia/serve/services.py @@ -21,7 +21,9 @@ PACK_METHOD_FASTAPI, PACKMODE_FILE, ) -from ..setup.requirements.bentoml import BentoMLRequirement +from ..setup.requirements.bentoml_requirement import BentoMLRequirement + +# from ..setup.requirements.bentoml import BentoMLRequirement from ..setup.requirements.conda import CondaRequirement from ..setup.requirements.docker import DockerRequirement from ..tools.bentoml.exceptions import BentoMLException diff --git a/ersilia/setup/requirements/__init__.py b/ersilia/setup/requirements/__init__.py index 10bfb9350..9b39903ba 100644 --- a/ersilia/setup/requirements/__init__.py +++ b/ersilia/setup/requirements/__init__.py @@ -1,4 +1,4 @@ -from .bentoml import BentoMLRequirement +from .bentoml_requirement import BentoMLRequirement def check_bentoml(): diff --git a/ersilia/setup/requirements/bentoml_requirement.py b/ersilia/setup/requirements/bentoml_requirement.py new file mode 100644 index 000000000..a813c9132 --- /dev/null +++ b/ersilia/setup/requirements/bentoml_requirement.py @@ -0,0 +1,131 @@ +import sys +from threading import Lock +from typing import Optional + +from packaging import version +from packaging.version import InvalidVersion + +from ...tools.bentoml.exceptions import BentoMLException +from ...utils.logging import logger +from ...utils.terminal import run_command + + +class BentoMLRequirement(object): + """Handles installation and version checking of BentoML for Ersilia.""" + + _lock = Lock() + + def __init__(self): + self.logger = logger + + def is_installed(self) -> bool: + """Checks if BentoML is installed.""" + try: + import bentoml # noqa: F401 + + return True + except ImportError: + self.logger.debug("BentoML is not installed") + return False + + def _get_bentoml_version(self) -> Optional[str]: + """Get BentoML version using run_command""" + result = run_command([sys.executable, "-m", "bentoml", "--version"]) + + if result.returncode != 0: + self.logger.error(f"BentoML version check failed: {result.stderr}") + return None + + # Extract version from "python -m bentoml, version 0.11.0" + version_str = result.stdout.split("version")[-1].strip() + return version_str + + # try: + # return version.parse(version_str).public + # except InvalidVersion: + # self.logger.error(f"Invalid BentoML version detected: {version_str}") + # return None + + def is_bentoml_ersilia_version(self) -> bool: + """Checks if the installed BentoML version is the Ersilia version.""" + version_str = self._get_bentoml_version() + if not version_str: + return False + try: + return version.parse(version_str) == version.parse("0.11.0") + except InvalidVersion: + self.logger.error(f"Invalid BentoML version detected: {version_str}") + return False + + def _cleanup_corrupted_bentoml(self) -> None: + """Forcefully uninstall BentoML and reinstall the correct version.""" + with self._lock: + self.logger.info("Cleaning up corrupted BentoML installation...") + + if self.is_installed(): + result = run_command( + [sys.executable, "-m", "pip", "uninstall", "bentoml", "-y"] + ) + if result.returncode != 0: + raise BentoMLException(f"Force uninstall failed: {result.stderr}") + + # ✅ Ensure reinstallation after cleanup + self.logger.info("Reinstalling Ersilia-compatible BentoML after cleanup...") + self.install(retries=1) # Only allow 1 retry to avoid infinite loops + + def install(self, retries: int = 3) -> None: + """Installs the Ersilia version of BentoML with error handling.""" + with self._lock: + if retries <= 0: + self.logger.critical( + "Final installation attempt failed. Manual intervention required." + ) + raise BentoMLException("Installation failed after multiple attempts.") + + try: + # 1. Uninstall if wrong version exists + if self.is_installed() and not self.is_bentoml_ersilia_version(): + self.logger.info("Uninstalling incompatible BentoML...") + uninstall_result = run_command( + [sys.executable, "-m", "pip", "uninstall", "bentoml", "-y"] + ) + if uninstall_result.returncode != 0: + raise BentoMLException( + f"Uninstall failed: {uninstall_result.stderr}" + ) + + # 2. Install specific version + self.logger.info("Installing Ersilia-compatible BentoML...") + install_result = run_command( + [ + sys.executable, + "-m", + "pip", + "install", + "-U", + "git+https://github.com/ersilia-os/bentoml-ersilia.git", + ] + ) + if install_result.returncode != 0: + raise BentoMLException(f"Install failed: {install_result.stderr}") + + # 3. Post-install verification + if not self.is_bentoml_ersilia_version(): + raise BentoMLException("Installed version verification failed") + + self.logger.info("BentoML was installed and verified successfully.") + + except BentoMLException as e: + self.logger.error(f"Install attempt failed: {str(e)}") + self.logger.warning(f"Attempts remaining: {retries-1}") + + try: + self._cleanup_corrupted_bentoml() + except Exception as cleanup_error: + self.logger.critical(f"Cleanup failed: {str(cleanup_error)}") + raise BentoMLException( + "Aborting due to failed cleanup" + ) from cleanup_error + + # Recursive retry with counter + self.install(retries=retries - 1) diff --git a/ersilia/utils/terminal.py b/ersilia/utils/terminal.py index 4c2e75cec..8628e2ba3 100644 --- a/ersilia/utils/terminal.py +++ b/ersilia/utils/terminal.py @@ -11,6 +11,8 @@ inputimeout = None TimeoutOccurred = None +from collections import namedtuple + from ..default import OUTPUT_DATASTRUCTURE, VERBOSE_FILE from ..utils.logging import make_temp_dir from ..utils.session import get_session_dir @@ -39,17 +41,19 @@ def is_quiet(): def run_command(cmd, quiet=None): """ - Run a shell command and return stdout, stderr, and return code. + Run a shell command and return a named tuple with stdout, stderr, and return code. Parameters ---------- cmd : str or list The command to run. quiet : bool, optional - Whether to run the command in quiet mode. Defaults to `is_quiet()`. + Whether to run the command in quiet mode. Defaults to is_quiet(). """ + # You must define or import is_quiet() somewhere in your code. if quiet is None: quiet = is_quiet() + # Run the command and capture outputs result = subprocess.run( cmd, @@ -60,19 +64,21 @@ def run_command(cmd, quiet=None): env=os.environ, ) - # Decode outputs - stdout = result.stdout.strip() - stderr = result.stderr.strip() - returncode = result.returncode + CommandResult = namedtuple("CommandResult", ["returncode", "stdout", "stderr"]) + stdout_str = result.stdout.strip() + stderr_str = result.stderr.strip() + output = CommandResult( + returncode=result.returncode, stdout=stdout_str, stderr=stderr_str + ) # Log outputs if not in quiet mode if not quiet: - if stdout: - print(stdout) - if stderr: - print(stderr, file=sys.stderr) + if stdout_str: + print(stdout_str) + if stderr_str: + print(stderr_str, file=sys.stderr) - return stdout, stderr, returncode + return output def run_command_check_output(cmd): From 19267e130c849b688f7bbfdce9b2d52c973c3cce Mon Sep 17 00:00:00 2001 From: Olawumi Salaam Date: Wed, 12 Feb 2025 09:23:36 +0100 Subject: [PATCH 07/10] Remove old bentoml.py after renaming to bentoml_requirement.py --- ersilia/setup/requirements/bentoml.py | 129 -------------------------- 1 file changed, 129 deletions(-) delete mode 100644 ersilia/setup/requirements/bentoml.py diff --git a/ersilia/setup/requirements/bentoml.py b/ersilia/setup/requirements/bentoml.py deleted file mode 100644 index 2f5f16090..000000000 --- a/ersilia/setup/requirements/bentoml.py +++ /dev/null @@ -1,129 +0,0 @@ -import sys -from threading import Lock -from typing import Optional - -from packaging import version -from packaging.version import InvalidVersion - -from ...tools.bentoml.exceptions import BentoMLException -from ...utils.logging import logger -from ...utils.terminal import run_command - - -class BentoMLRequirement(object): - """Handles installation and version checking of BentoML for Ersilia.""" - - _lock = Lock() - - def __init__(self): - self.logger = logger - - def is_installed(self) -> bool: - """Checks if BentoML is installed.""" - try: - import bentoml # noqa: F401 - - return True - except ImportError: - self.logger.debug("BentoML is not installed") - return False - - def _get_bentoml_version(self) -> Optional[str]: - """Get BentoML version using run_command""" - result = run_command([sys.executable, "-m", "bentoml", "--version"]) - - if result.returncode != 0: - self.logger.error(f"BentoML version check failed: {result.stderr}") - return None - - version_str = result.stdout.strip() - - try: - return version.parse(version_str).public - except InvalidVersion: - self.logger.error(f"Invalid BentoML version detected: {version_str}") - return None - - def is_bentoml_ersilia_version(self) -> bool: - """Checks if the installed BentoML version is the Ersilia version.""" - version_str = self._get_bentoml_version() - if not version_str: - return False - try: - return version.parse(version_str) == version.parse("0.11.0") - except InvalidVersion: - self.logger.error(f"Invalid BentoML version detected: {version_str}") - return False - - def _cleanup_corrupted_bentoml(self) -> None: - """Forcefully uninstall BentoML and reinstall the correct version.""" - with self._lock: - self.logger.info("Cleaning up corrupted BentoML installation...") - - if self.is_installed(): - result = run_command( - [sys.executable, "-m", "pip", "uninstall", "bentoml", "-y"] - ) - if result.returncode != 0: - raise BentoMLException(f"Force uninstall failed: {result.stderr}") - - # ✅ Ensure reinstallation after cleanup - self.logger.info("Reinstalling Ersilia-compatible BentoML after cleanup...") - self.install(retries=1) # Only allow 1 retry to avoid infinite loops - - def install(self, retries: int = 3) -> None: - """Installs the Ersilia version of BentoML with error handling.""" - with self._lock: - if retries <= 0: - self.logger.critical( - "Final installation attempt failed. Manual intervention required." - ) - raise BentoMLException("Installation failed after multiple attempts.") - - try: - # 1. Uninstall if wrong version exists - if self.is_installed() and not self.is_bentoml_ersilia_version(): - self.logger.info("Uninstalling incompatible BentoML version...") - uninstall_result = run_command( - [sys.executable, "-m", "pip", "uninstall", "bentoml", "-y"] - ) - if uninstall_result.returncode != 0: - raise BentoMLException( - f"Uninstall failed: {uninstall_result.stderr}" - ) - - # 2. Install specific version - self.logger.info("Installing Ersilia-compatible BentoML...") - install_result = run_command( - [ - sys.executable, - "-m", - "pip", - "install", - "-U", - "git+https://github.com/ersilia-os/bentoml-ersilia.git", - ] - ) - if install_result.returncode != 0: - raise BentoMLException(f"Install failed: {install_result.stderr}") - - # 3. Post-install verification - if not self.is_bentoml_ersilia_version(): - raise BentoMLException("Installed version verification failed") - - self.logger.info("BentoML was installed and verified successfully.") - - except BentoMLException as e: - self.logger.error(f"Install attempt failed: {str(e)}") - self.logger.warning(f"Attempts remaining: {retries-1}") - - try: - self._cleanup_corrupted_bentoml() - except Exception as cleanup_error: - self.logger.critical(f"Cleanup failed: {str(cleanup_error)}") - raise BentoMLException( - "Aborting due to failed cleanup" - ) from cleanup_error - - # Recursive retry with counter - self.install(retries=retries - 1) From 685e0c9f8b10531f23db79edbacf4c6e1ac70b34 Mon Sep 17 00:00:00 2001 From: Olawumi Salaam Date: Wed, 12 Feb 2025 17:18:29 +0100 Subject: [PATCH 08/10] Update BentoML configuration to auto-install custom fork when missing --- ersilia/tools/bentoml/configuration/__init__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ersilia/tools/bentoml/configuration/__init__.py b/ersilia/tools/bentoml/configuration/__init__.py index 8d2d60b01..bd2712f27 100644 --- a/ersilia/tools/bentoml/configuration/__init__.py +++ b/ersilia/tools/bentoml/configuration/__init__.py @@ -17,7 +17,15 @@ from functools import lru_cache from pathlib import Path -from bentoml import __version__ +try: + from bentoml import __version__ +except ModuleNotFoundError: + from ersilia.setup.requirements.bentoml_requirement import BentoMLRequirement + + req = BentoMLRequirement() + req.install() + from bentoml import __version__ + from bentoml import _version as version_mod from bentoml.configuration.configparser import BentoMLConfigParser from bentoml.exceptions import BentoMLConfigException From 92df254fb36b37f8ebaac6dea57a9036ba92cfbd Mon Sep 17 00:00:00 2001 From: Olawumi Salaam Date: Thu, 13 Feb 2025 15:01:05 +0100 Subject: [PATCH 09/10] fix: module shadowing --- ersilia/utils/terminal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ersilia/utils/terminal.py b/ersilia/utils/terminal.py index 8628e2ba3..f33ba01dc 100644 --- a/ersilia/utils/terminal.py +++ b/ersilia/utils/terminal.py @@ -50,7 +50,7 @@ def run_command(cmd, quiet=None): quiet : bool, optional Whether to run the command in quiet mode. Defaults to is_quiet(). """ - # You must define or import is_quiet() somewhere in your code. + if quiet is None: quiet = is_quiet() From cb669f8e4c16a8357e7b7abfd5af0506a60a1736 Mon Sep 17 00:00:00 2001 From: Olawumi Salaam Date: Fri, 14 Feb 2025 11:56:44 +0100 Subject: [PATCH 10/10] fix: BentoML issues --- .../setup/requirements/bentoml_requirement.py | 56 ++++++++----------- 1 file changed, 22 insertions(+), 34 deletions(-) diff --git a/ersilia/setup/requirements/bentoml_requirement.py b/ersilia/setup/requirements/bentoml_requirement.py index a813c9132..066762222 100644 --- a/ersilia/setup/requirements/bentoml_requirement.py +++ b/ersilia/setup/requirements/bentoml_requirement.py @@ -1,13 +1,12 @@ +import os +import subprocess import sys from threading import Lock from typing import Optional -from packaging import version -from packaging.version import InvalidVersion - +from ...default import EOS from ...tools.bentoml.exceptions import BentoMLException from ...utils.logging import logger -from ...utils.terminal import run_command class BentoMLRequirement(object): @@ -29,33 +28,28 @@ def is_installed(self) -> bool: return False def _get_bentoml_version(self) -> Optional[str]: - """Get BentoML version using run_command""" - result = run_command([sys.executable, "-m", "bentoml", "--version"]) + """Get BentoML version using subprocess.Popen""" + version_file = os.path.join(EOS, "bentomlversion.txt") - if result.returncode != 0: - self.logger.error(f"BentoML version check failed: {result.stderr}") - return None + if not os.path.exists(version_file): + cmd = "bentoml --version > {0}".format(version_file) + subprocess.Popen(cmd, shell=True).wait() - # Extract version from "python -m bentoml, version 0.11.0" - version_str = result.stdout.split("version")[-1].strip() - return version_str + with open(version_file, "r") as f: + version_str = f.read().strip() - # try: - # return version.parse(version_str).public - # except InvalidVersion: - # self.logger.error(f"Invalid BentoML version detected: {version_str}") - # return None + version_str = version_str.split("version")[-1].strip() + return version_str def is_bentoml_ersilia_version(self) -> bool: """Checks if the installed BentoML version is the Ersilia version.""" version_str = self._get_bentoml_version() + print("We got version", version_str) if not version_str: return False - try: - return version.parse(version_str) == version.parse("0.11.0") - except InvalidVersion: - self.logger.error(f"Invalid BentoML version detected: {version_str}") - return False + if "0.11.0" in version_str: + return True + return False def _cleanup_corrupted_bentoml(self) -> None: """Forcefully uninstall BentoML and reinstall the correct version.""" @@ -63,7 +57,7 @@ def _cleanup_corrupted_bentoml(self) -> None: self.logger.info("Cleaning up corrupted BentoML installation...") if self.is_installed(): - result = run_command( + result = subprocess.Popen( [sys.executable, "-m", "pip", "uninstall", "bentoml", "-y"] ) if result.returncode != 0: @@ -86,7 +80,7 @@ def install(self, retries: int = 3) -> None: # 1. Uninstall if wrong version exists if self.is_installed() and not self.is_bentoml_ersilia_version(): self.logger.info("Uninstalling incompatible BentoML...") - uninstall_result = run_command( + uninstall_result = subprocess.Popen( [sys.executable, "-m", "pip", "uninstall", "bentoml", "-y"] ) if uninstall_result.returncode != 0: @@ -96,17 +90,11 @@ def install(self, retries: int = 3) -> None: # 2. Install specific version self.logger.info("Installing Ersilia-compatible BentoML...") - install_result = run_command( - [ - sys.executable, - "-m", - "pip", - "install", - "-U", - "git+https://github.com/ersilia-os/bentoml-ersilia.git", - ] + cmd = "{0} -m pip install -U git+https://github.com/ersilia-os/bentoml-ersilia.git".format( + sys.executable ) - if install_result.returncode != 0: + install_result = subprocess.Popen(cmd, shell=True).wait() + if install_result != 0: raise BentoMLException(f"Install failed: {install_result.stderr}") # 3. Post-install verification