diff --git a/.env-devel b/.env-devel index 3aa61838001..f7fcf3a6e3e 100644 --- a/.env-devel +++ b/.env-devel @@ -60,7 +60,7 @@ S3_BUCKET_NAME=simcore S3_ENDPOINT=172.17.0.1:9001 S3_SECRET_KEY=12345678 S3_SECURE=0 -R_CLONE_S3_PROVIDER=MINIO +R_CLONE_PROVIDER=MINIO SCICRUNCH_API_BASE_URL=https://scicrunch.org/api/1 SCICRUNCH_API_KEY=REPLACE_ME_with_valid_api_key diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index 958c854cc44..5db3402d809 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -1036,6 +1036,8 @@ jobs: driver: docker - name: setup docker-compose run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }} + - name: install rclone in CI + run: sudo ./ci/github/helpers/install_rclone.bash - name: setup python environment uses: actions/setup-python@v3 with: @@ -1343,7 +1345,7 @@ jobs: - name: setup docker-compose run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }} - name: setup rclone docker volume plugin - run: sudo ./ci/github/helpers/install_rclone.bash + run: sudo ./ci/github/helpers/install_rclone_docker_volume_plugin.bash - name: setup python environment uses: actions/setup-python@v3 with: @@ -1410,6 +1412,8 @@ jobs: driver: docker - name: setup docker-compose run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }} + - name: install rclone in CI + run: sudo ./ci/github/helpers/install_rclone.bash - name: setup python environment uses: actions/setup-python@v3 with: diff --git a/api/specs/storage/openapi.yaml b/api/specs/storage/openapi.yaml index 48515a82a4d..7505f9b149a 100644 --- a/api/specs/storage/openapi.yaml +++ b/api/specs/storage/openapi.yaml @@ -257,11 +257,11 @@ paths: required: true schema: type: string - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/FileMetaData" + - name: user_id + in: query + required: true + schema: + type: string responses: "200": description: "Returns file metadata" diff --git a/ci/github/helpers/install_rclone.bash b/ci/github/helpers/install_rclone.bash index 3bc38a27db9..66f6a3ec83a 100755 --- a/ci/github/helpers/install_rclone.bash +++ b/ci/github/helpers/install_rclone.bash @@ -10,11 +10,8 @@ set -o pipefail # don't hide errors within pipes IFS=$'\n\t' -# Installation instructions from https://rclone.org/docker/ - -apt-get -y install fuse=2.9.9-3 -mkdir -p /var/lib/docker-plugins/rclone/config -mkdir -p /var/lib/docker-plugins/rclone/cache -docker plugin install rclone/docker-volume-rclone:amd64-1.57.0 args="-v" --alias rclone --grant-all-permissions -docker plugin list -docker plugin inspect rclone +R_CLONE_VERSION="1.58.0" +curl --silent --location --remote-name "https://downloads.rclone.org/v${R_CLONE_VERSION}/rclone-v${R_CLONE_VERSION}-linux-amd64.deb" +dpkg --install "rclone-v${R_CLONE_VERSION}-linux-amd64.deb" +rm "rclone-v${R_CLONE_VERSION}-linux-amd64.deb" +rclone --version diff --git a/ci/github/helpers/install_rclone_docker_volume_plugin.bash b/ci/github/helpers/install_rclone_docker_volume_plugin.bash new file mode 100755 index 00000000000..23f1968b027 --- /dev/null +++ b/ci/github/helpers/install_rclone_docker_volume_plugin.bash @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Installs the latest version of rclone plugin +# + +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes +IFS=$'\n\t' + + +# Installation instructions from https://rclone.org/docker/ + +apt-get -y install fuse=2.9.9-3 +mkdir --parents /var/lib/docker-plugins/rclone/config +mkdir --parents /var/lib/docker-plugins/rclone/cache +docker plugin install rclone/docker-volume-rclone:amd64-1.57.0 args="-v" --alias rclone --grant-all-permissions +docker plugin list +docker plugin inspect rclone diff --git a/ci/github/unit-testing/director-v2.bash b/ci/github/unit-testing/director-v2.bash index f57902e0a12..896aac6a0fd 100755 --- a/ci/github/unit-testing/director-v2.bash +++ b/ci/github/unit-testing/director-v2.bash @@ -18,14 +18,13 @@ test() { --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \ -v -m "not travis" services/director-v2/tests/unit --ignore=services/director-v2/tests/unit/with_dbs \ --asyncio-mode=auto \ - --ignore=services/director-v2/tests/unit/with_swarm # these tests cannot be run in parallel pytest --log-format="%(asctime)s %(levelname)s %(message)s" \ --log-date-format="%Y-%m-%d %H:%M:%S" \ --cov=simcore_service_director_v2 --durations=10 --cov-append \ --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \ --asyncio-mode=auto \ - -v -m "not travis" services/director-v2/tests/unit/with_swarm services/director-v2/tests/unit/with_dbs + -v -m "not travis" services/director-v2/tests/unit/with_dbs } # Check if the function exists (bash specific) diff --git a/packages/pytest-simcore/src/pytest_simcore/minio_service.py b/packages/pytest-simcore/src/pytest_simcore/minio_service.py index c3c6d62bd1d..3d1963b75ad 100644 --- a/packages/pytest-simcore/src/pytest_simcore/minio_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/minio_service.py @@ -3,7 +3,7 @@ # pylint: disable=unused-variable import logging -from typing import Dict, Iterator +from typing import Any, Iterator import pytest from _pytest.monkeypatch import MonkeyPatch @@ -43,8 +43,8 @@ def _ensure_remove_bucket(client: Minio, bucket_name: str): @pytest.fixture(scope="module") def minio_config( - docker_stack: Dict, testing_environ_vars: Dict, monkeypatch_module: MonkeyPatch -) -> Dict[str, str]: + docker_stack: dict, testing_environ_vars: dict, monkeypatch_module: MonkeyPatch +) -> dict[str, Any]: assert "pytest-ops_minio" in docker_stack["services"] config = { @@ -68,7 +68,7 @@ def minio_config( @pytest.fixture(scope="module") -def minio_service(minio_config: Dict[str, str]) -> Iterator[Minio]: +def minio_service(minio_config: dict[str, str]) -> Iterator[Minio]: client = Minio(**minio_config["client"]) @@ -99,7 +99,7 @@ def minio_service(minio_config: Dict[str, str]) -> Iterator[Minio]: @pytest.fixture(scope="module") -def bucket(minio_config: Dict[str, str], minio_service: Minio) -> Iterator[str]: +def bucket(minio_config: dict[str, str], minio_service: Minio) -> Iterator[str]: bucket_name = minio_config["bucket_name"] _ensure_remove_bucket(minio_service, bucket_name) diff --git a/packages/service-library/src/servicelib/aiohttp/dev_error_logger.py b/packages/service-library/src/servicelib/aiohttp/dev_error_logger.py new file mode 100644 index 00000000000..99d5fa72d59 --- /dev/null +++ b/packages/service-library/src/servicelib/aiohttp/dev_error_logger.py @@ -0,0 +1,36 @@ + +from aiohttp.web import Application, middleware, Request, HTTPError +from servicelib.aiohttp.typing_extension import Handler, Middleware +import logging +import traceback + +logger = logging.getLogger(__name__) + +_SEP = "|||" + + +def _middleware_factory() -> Middleware: + @middleware + async def middleware_handler(request: Request, handler: Handler): + try: + return await handler(request) + except HTTPError as err: + fields = { + "Body": err.body, + "Status": err.status, + "Reason": err.reason, + "Headers": err.headers, + "Traceback": "\n".join(traceback.format_tb(err.__traceback__)), + } + formatted_error = "".join( + [f"\n{_SEP}{k}{_SEP}\n{v}" for k, v in fields.items()] + ) + logger.debug("Error serialized to client:%s", formatted_error) + raise err + + return middleware_handler + + +def setup_dev_error_logger(app: Application) -> None: + logger.info("Setting up dev_error_logger") + app.middlewares.append(_middleware_factory()) diff --git a/packages/settings-library/src/settings_library/r_clone.py b/packages/settings-library/src/settings_library/r_clone.py new file mode 100644 index 00000000000..53d3757b5cf --- /dev/null +++ b/packages/settings-library/src/settings_library/r_clone.py @@ -0,0 +1,16 @@ +from enum import Enum + +from pydantic import Field +from .base import BaseCustomSettings +from .s3 import S3Settings + + +class S3Provider(str, Enum): + AWS = "AWS" + CEPH = "CEPH" + MINIO = "MINIO" + + +class RCloneSettings(BaseCustomSettings): + R_CLONE_S3: S3Settings = Field(auto_default_from_env=True) + R_CLONE_PROVIDER: S3Provider diff --git a/packages/settings-library/src/settings_library/s3.py b/packages/settings-library/src/settings_library/s3.py index 91733e441ed..b9c70f4f0c5 100644 --- a/packages/settings-library/src/settings_library/s3.py +++ b/packages/settings-library/src/settings_library/s3.py @@ -1,12 +1,22 @@ from typing import Optional from .base import BaseCustomSettings +from pydantic import validator class S3Settings(BaseCustomSettings): - S3_ENDPOINT: str = "minio:9000" - S3_ACCESS_KEY: str = "12345678" - S3_SECRET_KEY: str = "12345678" + S3_ENDPOINT: str + S3_ACCESS_KEY: str + S3_SECRET_KEY: str S3_ACCESS_TOKEN: Optional[str] = None - S3_BUCKET_NAME: str = "simcore" + S3_BUCKET_NAME: str S3_SECURE: bool = False + S3_REGION: str = "us-east-1" + + @validator("S3_ENDPOINT", pre=True) + @classmethod + def ensure_scheme(cls, v: str, values) -> str: + if not v.startswith("http"): + scheme = "https" if values.get("S3_SECURE") else "http" + return f"{scheme}://{v}" + return v diff --git a/packages/settings-library/src/settings_library/utils_r_clone.py b/packages/settings-library/src/settings_library/utils_r_clone.py new file mode 100644 index 00000000000..7c488d735a6 --- /dev/null +++ b/packages/settings-library/src/settings_library/utils_r_clone.py @@ -0,0 +1,47 @@ +import configparser +from copy import deepcopy +from io import StringIO +from typing import Dict + +from .r_clone import RCloneSettings, S3Provider + +_COMMON_ENTRIES: Dict[str, str] = { + "type": "s3", + "access_key_id": "{access_key}", + "secret_access_key": "{secret_key}", + "region": "{aws_region}", + "acl": "private", +} + +_PROVIDER_ENTRIES: Dict[S3Provider, Dict[str, str]] = { + # NOTE: # AWS_SESSION_TOKEN should be required for STS + S3Provider.AWS: {"provider": "AWS"}, + S3Provider.CEPH: {"provider": "Ceph", "endpoint": "{endpoint}"}, + S3Provider.MINIO: {"provider": "Minio", "endpoint": "{endpoint}"}, +} + + +def _format_config(entries: Dict[str, str]) -> str: + config = configparser.ConfigParser() + config["dst"] = entries + with StringIO() as string_io: + config.write(string_io) + string_io.seek(0) + return string_io.read() + + +def get_r_clone_config(r_clone_settings: RCloneSettings) -> str: + provider = r_clone_settings.R_CLONE_PROVIDER + entries = deepcopy(_COMMON_ENTRIES) + entries.update(_PROVIDER_ENTRIES[provider]) + + r_clone_config_template = _format_config(entries=entries) + + # replace entries in template + r_clone_config = r_clone_config_template.format( + endpoint=r_clone_settings.R_CLONE_S3.S3_ENDPOINT, + access_key=r_clone_settings.R_CLONE_S3.S3_ACCESS_KEY, + secret_key=r_clone_settings.R_CLONE_S3.S3_SECRET_KEY, + aws_region=r_clone_settings.R_CLONE_S3.S3_REGION, + ) + return r_clone_config diff --git a/packages/settings-library/tests/test_utils_r_clone.py b/packages/settings-library/tests/test_utils_r_clone.py new file mode 100644 index 00000000000..b367fd57b4d --- /dev/null +++ b/packages/settings-library/tests/test_utils_r_clone.py @@ -0,0 +1,28 @@ +# pylint: disable=redefined-outer-name + +import pytest +from settings_library.r_clone import RCloneSettings, S3Provider +from settings_library.utils_r_clone import _COMMON_ENTRIES, get_r_clone_config + + +@pytest.fixture(params=list(S3Provider)) +def r_clone_settings(request, monkeypatch) -> RCloneSettings: + monkeypatch.setenv("R_CLONE_PROVIDER", request.param) + monkeypatch.setenv("S3_ENDPOINT", "endpoint") + monkeypatch.setenv("S3_ACCESS_KEY", "access_key") + monkeypatch.setenv("S3_SECRET_KEY", "secret_key") + monkeypatch.setenv("S3_BUCKET_NAME", "bucket_name") + monkeypatch.setenv("S3_SECURE", False) + return RCloneSettings() + + +def test_r_clone_config_template_replacement(r_clone_settings: RCloneSettings) -> None: + r_clone_config = get_r_clone_config(r_clone_settings) + print(r_clone_config) + + assert "{endpoint}" not in r_clone_config + assert "{access_key}" not in r_clone_config + assert "{secret_key}" not in r_clone_config + + for key in _COMMON_ENTRIES.keys(): + assert key in r_clone_config diff --git a/packages/simcore-sdk/requirements/_base.in b/packages/simcore-sdk/requirements/_base.in index 43f1be7de6d..569451217e2 100644 --- a/packages/simcore-sdk/requirements/_base.in +++ b/packages/simcore-sdk/requirements/_base.in @@ -4,8 +4,10 @@ --constraint ../../../requirements/constraints.txt --requirement ../../../packages/postgres-database/requirements/_base.in --requirement ../../../packages/service-library/requirements/_base.in +--requirement ../../../packages/settings-library/requirements/_base.in --requirement ../../../packages/models-library/requirements/_base.in +aiocache aiofiles aiohttp aiopg[sa] diff --git a/packages/simcore-sdk/requirements/_base.txt b/packages/simcore-sdk/requirements/_base.txt index 2777f0c7761..a523d7fcca8 100644 --- a/packages/simcore-sdk/requirements/_base.txt +++ b/packages/simcore-sdk/requirements/_base.txt @@ -4,6 +4,8 @@ # # pip-compile --output-file=requirements/_base.txt --strip-extras requirements/_base.in # +aiocache==0.11.1 + # via -r requirements/_base.in aiodebug==2.3.0 # via -r requirements/../../../packages/service-library/requirements/_base.in aiofiles==0.8.0 @@ -15,6 +17,7 @@ aiohttp==3.8.1 # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/_base.in aiopg==1.3.3 @@ -34,6 +37,8 @@ attrs==20.3.0 # jsonschema charset-normalizer==2.0.12 # via aiohttp +click==8.1.2 + # via typer dnspython==2.2.1 # via email-validator email-validator==1.1.3 @@ -73,13 +78,15 @@ pydantic==1.9.0 # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/_base.in pyinstrument==4.1.1 # via -r requirements/../../../packages/service-library/requirements/_base.in -pyparsing==3.0.7 +pyparsing==3.0.8 # via packaging pyrsistent==0.18.1 # via jsonschema @@ -89,15 +96,17 @@ pyyaml==5.4.1 # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/./constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in six==1.16.0 # via jsonschema -sqlalchemy==1.4.32 +sqlalchemy==1.4.35 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_base.in # aiopg @@ -106,9 +115,11 @@ tenacity==8.0.1 # via # -r requirements/../../../packages/service-library/requirements/_base.in # -r requirements/_base.in -tqdm==4.63.1 +tqdm==4.64.0 # via -r requirements/_base.in -typing-extensions==4.1.1 +typer==0.4.1 + # via -r requirements/../../../packages/settings-library/requirements/_base.in +typing-extensions==4.2.0 # via # aiodebug # pydantic diff --git a/packages/simcore-sdk/requirements/_test.in b/packages/simcore-sdk/requirements/_test.in index 5fd198d4ba5..9bea2458f75 100644 --- a/packages/simcore-sdk/requirements/_test.in +++ b/packages/simcore-sdk/requirements/_test.in @@ -21,6 +21,7 @@ pytest-xdist pytest-lazy-fixture # mockups/fixtures +aioboto3 aioresponses alembic click diff --git a/packages/simcore-sdk/requirements/_test.txt b/packages/simcore-sdk/requirements/_test.txt index 25cebdaffeb..c6ba67c5b79 100644 --- a/packages/simcore-sdk/requirements/_test.txt +++ b/packages/simcore-sdk/requirements/_test.txt @@ -4,12 +4,19 @@ # # pip-compile --output-file=requirements/_test.txt --strip-extras requirements/_test.in # +aioboto3==9.5.0 + # via -r requirements/_test.in +aiobotocore==2.2.0 + # via aioboto3 aiohttp==3.8.1 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt + # aiobotocore # aioresponses # pytest-aiohttp +aioitertools==0.10.0 + # via aiobotocore aioresponses==0.7.3 # via -r requirements/_test.in aiosignal==1.2.0 @@ -20,7 +27,7 @@ alembic==1.7.7 # via # -c requirements/_base.txt # -r requirements/_test.in -astroid==2.11.2 +astroid==2.11.3 # via pylint async-timeout==4.0.2 # via @@ -31,6 +38,13 @@ attrs==20.3.0 # -c requirements/_base.txt # aiohttp # pytest +boto3==1.21.21 + # via aiobotocore +botocore==1.24.21 + # via + # aiobotocore + # boto3 + # s3transfer certifi==2021.10.8 # via # minio @@ -40,8 +54,10 @@ charset-normalizer==2.0.12 # -c requirements/_base.txt # aiohttp # requests -click==8.0.4 - # via -r requirements/_test.in +click==8.1.2 + # via + # -c requirements/_base.txt + # -r requirements/_test.in coverage==6.3.2 # via # -r requirements/_test.in @@ -57,7 +73,7 @@ docopt==0.6.2 # via coveralls execnet==1.9.0 # via pytest-xdist -faker==13.3.3 +faker==13.3.5 # via -r requirements/_test.in frozenlist==1.3.0 # via @@ -68,7 +84,7 @@ greenlet==1.1.2 # via # -c requirements/_base.txt # sqlalchemy -icdiff==2.0.4 +icdiff==2.0.5 # via pytest-icdiff idna==3.3 # via @@ -79,6 +95,10 @@ iniconfig==1.1.1 # via pytest isort==5.10.1 # via pylint +jmespath==1.0.0 + # via + # boto3 + # botocore lazy-object-proxy==1.7.1 # via astroid mako==1.2.0 @@ -105,7 +125,7 @@ packaging==21.3 # -c requirements/_base.txt # pytest # pytest-sugar -platformdirs==2.5.1 +platformdirs==2.5.2 # via pylint pluggy==1.0.0 # via pytest @@ -119,9 +139,9 @@ py==1.11.0 # via # pytest # pytest-forked -pylint==2.13.2 +pylint==2.13.7 # via -r requirements/_test.in -pyparsing==3.0.7 +pyparsing==3.0.8 # via # -c requirements/_base.txt # packaging @@ -161,7 +181,9 @@ pytest-sugar==0.9.4 pytest-xdist==2.5.0 # via -r requirements/_test.in python-dateutil==2.8.2 - # via faker + # via + # botocore + # faker python-dotenv==0.20.0 # via -r requirements/_test.in requests==2.27.1 @@ -169,11 +191,13 @@ requests==2.27.1 # -r requirements/_test.in # coveralls # docker +s3transfer==0.5.2 + # via boto3 six==1.16.0 # via # -c requirements/_base.txt # python-dateutil -sqlalchemy==1.4.32 +sqlalchemy==1.4.35 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -185,20 +209,24 @@ tomli==2.0.1 # coverage # pylint # pytest -typing-extensions==4.1.1 +typing-extensions==4.2.0 # via # -c requirements/_base.txt + # aioitertools # astroid # pylint urllib3==1.26.9 # via # -c requirements/../../../requirements/constraints.txt + # botocore # minio # requests -websocket-client==1.3.1 +websocket-client==1.3.2 # via docker wrapt==1.14.0 - # via astroid + # via + # aiobotocore + # astroid yarl==1.7.2 # via # -c requirements/_base.txt diff --git a/packages/simcore-sdk/requirements/_tools.txt b/packages/simcore-sdk/requirements/_tools.txt index b52c3ba2620..017b0e4f7e4 100644 --- a/packages/simcore-sdk/requirements/_tools.txt +++ b/packages/simcore-sdk/requirements/_tools.txt @@ -10,8 +10,9 @@ bump2version==1.0.1 # via -r requirements/../../../requirements/devenv.txt cfgv==3.3.1 # via pre-commit -click==8.0.4 +click==8.1.2 # via + # -c requirements/_base.txt # -c requirements/_test.txt # black # pip-tools @@ -33,14 +34,14 @@ pathspec==0.9.0 # via black pep517==0.12.0 # via pip-tools -pip-tools==6.5.1 +pip-tools==6.6.0 # via -r requirements/../../../requirements/devenv.txt -platformdirs==2.5.1 +platformdirs==2.5.2 # via # -c requirements/_test.txt # black # virtualenv -pre-commit==2.17.0 +pre-commit==2.18.1 # via -r requirements/../../../requirements/devenv.txt pyyaml==5.4.1 # via @@ -59,12 +60,12 @@ tomli==2.0.1 # -c requirements/_test.txt # black # pep517 -typing-extensions==4.1.1 +typing-extensions==4.2.0 # via # -c requirements/_base.txt # -c requirements/_test.txt # black -virtualenv==20.14.0 +virtualenv==20.14.1 # via pre-commit wheel==0.37.1 # via pip-tools diff --git a/packages/simcore-sdk/requirements/ci.txt b/packages/simcore-sdk/requirements/ci.txt index 12f77debb43..5b661d543e3 100644 --- a/packages/simcore-sdk/requirements/ci.txt +++ b/packages/simcore-sdk/requirements/ci.txt @@ -14,6 +14,7 @@ ../postgres-database ../pytest-simcore/ ../models-library/ +../settings-library/ # FIXME: these dependencies should be removed ../service-library/ diff --git a/packages/simcore-sdk/requirements/dev.txt b/packages/simcore-sdk/requirements/dev.txt index 8d1be3ce9ad..b67f43d8690 100644 --- a/packages/simcore-sdk/requirements/dev.txt +++ b/packages/simcore-sdk/requirements/dev.txt @@ -16,6 +16,7 @@ --editable ../postgres-database --editable ../models-library/ +--editable ../settings-library/ # FIXME: these dependencies should be removed --editable ../service-library/ diff --git a/packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py b/packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py index 725f4c50dca..ce530b7ccb2 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py @@ -5,6 +5,7 @@ from typing import Optional, Union from servicelib.archiving_utils import archive_dir, unarchive_dir +from settings_library.r_clone import RCloneSettings from ..node_ports_common import filemanager @@ -24,6 +25,7 @@ async def _push_file( node_uuid: str, file_path: Path, rename_to: Optional[str], + r_clone_settings: Optional[RCloneSettings] = None, ): store_id = "0" # this is for simcore.s3 s3_object = _create_s3_object( @@ -36,6 +38,7 @@ async def _push_file( store_name=None, s3_object=s3_object, local_file_path=file_path, + r_clone_settings=r_clone_settings, ) log.info("%s successfuly uploaded", file_path) @@ -46,6 +49,7 @@ async def push( node_uuid: str, file_or_folder: Path, rename_to: Optional[str] = None, + r_clone_settings: Optional[RCloneSettings] = None, ): if file_or_folder.is_file(): return await _push_file( @@ -64,7 +68,9 @@ async def push( compress=False, # disabling compression for faster speeds store_relative_path=True, ) - return await _push_file(user_id, project_id, node_uuid, archive_file_path, None) + return await _push_file( + user_id, project_id, node_uuid, archive_file_path, None, r_clone_settings + ) async def _pull_file( diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/constants.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/constants.py new file mode 100644 index 00000000000..342cd410c05 --- /dev/null +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/constants.py @@ -0,0 +1,4 @@ +from typing import Final + +SIMCORE_LOCATION: Final[str] = "0" +ETag = str diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py index de411796527..950473996c2 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py @@ -8,11 +8,15 @@ import aiofiles from aiohttp import ClientPayloadError, ClientSession from pydantic.networks import AnyUrl +from settings_library.r_clone import RCloneSettings from tqdm import tqdm from yarl import URL from ..node_ports_common.client_session_manager import ClientSessionContextManager +from ..node_ports_common.storage_client import update_file_meta_data from . import exceptions, storage_client +from .constants import SIMCORE_LOCATION, ETag +from .r_clone import is_r_clone_available, sync_local_to_s3 log = logging.getLogger(__name__) @@ -92,9 +96,6 @@ async def _download_link_to_file(session: ClientSession, url: URL, file_path: Pa raise exceptions.TransferError(url) from exc -ETag = str - - async def _upload_file_to_link( session: ClientSession, url: URL, file_path: Path ) -> ETag: @@ -258,6 +259,7 @@ async def upload_file( s3_object: str, local_file_path: Path, client_session: Optional[ClientSession] = None, + r_clone_settings: Optional[RCloneSettings] = None, ) -> Tuple[str, str]: """Uploads a file to S3 @@ -287,7 +289,33 @@ async def upload_file( if not upload_link: raise exceptions.S3InvalidPathError(s3_object) - e_tag = await _upload_file_to_link(session, upload_link, local_file_path) + if ( + await is_r_clone_available(r_clone_settings) + and store_id == SIMCORE_LOCATION + ): + await sync_local_to_s3( + session=session, + r_clone_settings=r_clone_settings, + s3_object=s3_object, + local_file_path=local_file_path, + user_id=user_id, + store_id=store_id, + ) + else: + try: + await _upload_file_to_link(session, upload_link, local_file_path) + except exceptions.S3TransferError as err: + await delete_file( + user_id=user_id, + store_id=store_id, + s3_object=s3_object, + client_session=session, + ) + raise err + + e_tag = await update_file_meta_data( + session=session, s3_object=s3_object, user_id=user_id + ) return store_id, e_tag diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone.py new file mode 100644 index 00000000000..237339d663b --- /dev/null +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/r_clone.py @@ -0,0 +1,136 @@ +import asyncio +import logging +import re +import shlex +from contextlib import asynccontextmanager +from pathlib import Path +from typing import AsyncGenerator, Optional + +from aiocache import cached +from aiofiles import tempfile +from aiohttp import ClientSession +from models_library.users import UserID +from pydantic.errors import PydanticErrorMixin +from settings_library.r_clone import RCloneSettings +from settings_library.utils_r_clone import get_r_clone_config + +from .constants import SIMCORE_LOCATION +from .storage_client import LinkType, delete_file, get_upload_file_link + +logger = logging.getLogger(__name__) + + +class _CommandFailedException(PydanticErrorMixin, RuntimeError): + msg_template: str = "Command {command} finished with exception:\n{stdout}" + + +@asynccontextmanager +async def _config_file(config: str) -> AsyncGenerator[str, None]: + async with tempfile.NamedTemporaryFile("w") as f: + await f.write(config) + await f.flush() + yield f.name + + +async def _async_command(*cmd: str, cwd: Optional[str] = None) -> str: + str_cmd = " ".join(cmd) + proc = await asyncio.create_subprocess_shell( + str_cmd, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + cwd=cwd, + ) + + stdout, _ = await proc.communicate() + decoded_stdout = stdout.decode() + if proc.returncode != 0: + raise _CommandFailedException(command=str_cmd, stdout=decoded_stdout) + + logger.debug("'%s' result:\n%s", str_cmd, decoded_stdout) + return decoded_stdout + + +@cached() +async def is_r_clone_available(r_clone_settings: Optional[RCloneSettings]) -> bool: + """returns: True if the `rclone` cli is installed and a configuration is provided""" + if r_clone_settings is None: + return False + try: + await _async_command("rclone", "--version") + return True + except _CommandFailedException: + return False + + +async def sync_local_to_s3( + session: ClientSession, + r_clone_settings: RCloneSettings, + s3_object: str, + local_file_path: Path, + user_id: UserID, + store_id: str, +) -> None: + """NOTE: only works with simcore location""" + assert store_id == SIMCORE_LOCATION # nosec + + s3_link = await get_upload_file_link( + session=session, + file_id=s3_object, + location_id=store_id, + user_id=user_id, + link_type=LinkType.S3, + ) + s3_path = re.sub(r"^s3://", "", s3_link) + logger.debug(" %s; %s", f"{s3_link=}", f"{s3_path=}") + + r_clone_config_file_content = get_r_clone_config(r_clone_settings) + async with _config_file(r_clone_config_file_content) as config_file_name: + source_path = local_file_path + destination_path = Path(s3_path) + file_name = local_file_path.name + # FIXME: capture progress and connect progressbars or some event to inform the UI + + # rclone only acts upon directories, so to target a specific file + # we must run the command from the file's directory. See below + # example for further details: + # + # local_file_path=`/tmp/pytest-of-silenthk/pytest-80/test_sync_local_to_s30/filee3e70682-c209-4cac-a29f-6fbed82c07cd.txt` + # s3_path=`simcore/00000000-0000-0000-0000-000000000001/00000000-0000-0000-0000-000000000002/filee3e70682-c209-4cac-a29f-6fbed82c07cd.txt` + # + # rclone + # --config + # /tmp/tmpd_1rtmss + # sync + # '/tmp/pytest-of-silenthk/pytest-80/test_sync_local_to_s30' + # 'dst:simcore/00000000-0000-0000-0000-000000000001/00000000-0000-0000-0000-000000000002' + # --progress + # --copy-links + # --include + # 'filee3e70682-c209-4cac-a29f-6fbed82c07cd.txt' + r_clone_command = ( + "rclone", + "--config", + config_file_name, + "sync", + shlex.quote(f"{source_path.parent}"), + shlex.quote(f"dst:{destination_path.parent}"), + "--progress", + "--copy-links", + "--include", + shlex.quote(f"{file_name}"), + ) + + try: + await _async_command(*r_clone_command, cwd=f"{source_path.parent}") + except Exception as e: + logger.warning( + "There was an error while uploading %s. Removing metadata", s3_object + ) + await delete_file( + session=session, + file_id=s3_object, + location_id=store_id, + user_id=user_id, + ) + raise e diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/storage_client.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/storage_client.py index d4620b27f94..0c4a6406d44 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_common/storage_client.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_common/storage_client.py @@ -2,9 +2,9 @@ from functools import wraps from json import JSONDecodeError from typing import Any, Callable, Dict -from urllib.parse import quote +from urllib.parse import quote, quote_plus -from aiohttp import ClientSession +from aiohttp import ClientSession, web from aiohttp.client_exceptions import ClientConnectionError, ClientResponseError from models_library.api_schemas_storage import ( FileLocationArray, @@ -16,6 +16,7 @@ from pydantic.networks import AnyUrl from . import config, exceptions +from .constants import ETag def handle_client_exception(handler: Callable): @@ -186,3 +187,18 @@ async def delete_file( params={"user_id": f"{user_id}"}, ) as response: response.raise_for_status() + + +@handle_client_exception +async def update_file_meta_data( + session: ClientSession, s3_object: str, user_id: UserID +) -> ETag: + url = f"{_base_url()}/locations/0/files/{quote_plus(s3_object)}/metadata" + result = await session.patch(url, params=dict(user_id=user_id)) + if result.status != web.HTTPOk.status_code: + raise exceptions.StorageInvalidCall( + f"Could not fetch metadata: status={result.status} {await result.text()}" + ) + + response = await result.json() + return response["data"]["entity_tag"] diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py index 56efd8ebb00..d79ecabde9b 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/__init__.py @@ -1,6 +1,8 @@ import logging from typing import Optional +from settings_library.r_clone import RCloneSettings + from ..node_ports_common import config as node_config from ..node_ports_common import exceptions from ..node_ports_common.dbmanager import DBManager @@ -18,6 +20,7 @@ async def ports( node_uuid: str, *, db_manager: Optional[DBManager] = None, + r_clone_settings: Optional[RCloneSettings] = None ) -> Nodeports: log.debug("creating node_ports_v2 object using provided dbmanager: %s", db_manager) # FIXME: warning every dbmanager create a new db engine! @@ -31,6 +34,7 @@ async def ports( project_id=project_id, node_uuid=node_uuid, auto_update=True, + r_clone_settings=r_clone_settings, ) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py index 2c3d3156ff7..9078080b78a 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py @@ -6,6 +6,7 @@ from pydantic import BaseModel, Field, ValidationError from pydantic.error_wrappers import flatten_errors from servicelib.utils import logged_gather +from settings_library.r_clone import RCloneSettings from simcore_sdk.node_ports_common.storage_client import LinkType from ..node_ports_common.dbmanager import DBManager @@ -29,6 +30,7 @@ class Nodeports(BaseModel): [DBManager, int, str, str], Coroutine[Any, Any, Type["Nodeports"]] ] auto_update: bool = False + r_clone_settings: Optional[RCloneSettings] = None class Config: arbitrary_types_allowed = True diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py index 8acc7a57c14..0f6c7479b29 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py @@ -314,6 +314,7 @@ async def _set(self, new_concrete_value: ItemConcreteValue) -> None: user_id=self._node_ports.user_id, project_id=self._node_ports.project_id, node_id=self._node_ports.node_uuid, + r_clone_settings=self._node_ports.r_clone_settings, ) else: new_value = converted_value diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py index 872145e00b8..03347b70bf5 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py @@ -5,6 +5,7 @@ from pydantic import AnyUrl from pydantic.tools import parse_obj_as +from settings_library.r_clone import RCloneSettings from simcore_sdk.node_ports_common.storage_client import LinkType from yarl import URL @@ -149,15 +150,18 @@ async def push_file_to_store( user_id: int, project_id: str, node_id: str, + r_clone_settings: Optional[RCloneSettings] = None, ) -> FileLink: log.debug("file path %s will be uploaded to s3", file) s3_object = data_items_utils.encode_file_id(file, project_id, node_id) + store_id, e_tag = await filemanager.upload_file( user_id=user_id, store_id=None, store_name=config.STORE, s3_object=s3_object, local_file_path=file, + r_clone_settings=r_clone_settings, ) log.debug("file path %s uploaded, received ETag %s", file, e_tag) return FileLink(store=store_id, path=s3_object, e_tag=e_tag) diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py index 7e69cc6f341..d6f8670b53c 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/serialization_v2.py @@ -1,12 +1,13 @@ import json import logging from pprint import pformat -from typing import Any, Dict, Set +from typing import Any, Dict, Optional, Set import pydantic from models_library.projects_nodes import NodeID from models_library.utils.nodes import compute_node_hash from packaging import version +from settings_library.r_clone import RCloneSettings from ..node_ports_common.dbmanager import DBManager from ..node_ports_common.exceptions import InvalidProtocolError @@ -32,6 +33,7 @@ async def load( project_id: str, node_uuid: str, auto_update: bool = False, + r_clone_settings: Optional[RCloneSettings] = None, ) -> Nodeports: """creates a nodeport object from a row from comp_tasks""" log.debug( @@ -88,6 +90,7 @@ async def load( save_to_db_cb=dump, node_port_creator_cb=load, auto_update=auto_update, + r_clone_settings=r_clone_settings, ) log.debug( "created node_ports_v2 object %s", diff --git a/packages/simcore-sdk/tests/integration/conftest.py b/packages/simcore-sdk/tests/integration/conftest.py index 64d28c789a2..d7460fc8a93 100644 --- a/packages/simcore-sdk/tests/integration/conftest.py +++ b/packages/simcore-sdk/tests/integration/conftest.py @@ -5,7 +5,7 @@ import json from pathlib import Path -from typing import Any, Callable, Dict, Iterable, List, Tuple +from typing import Any, Awaitable, Callable, Dict, Iterable, List, Tuple from urllib.parse import quote_plus from uuid import uuid4 @@ -14,11 +14,13 @@ import sqlalchemy as sa from aiohttp import ClientSession from pytest_simcore.helpers.rawdata_fakers import random_project, random_user +from settings_library.r_clone import RCloneSettings, S3Provider from simcore_postgres_database.models.comp_pipeline import comp_pipeline from simcore_postgres_database.models.comp_tasks import comp_tasks from simcore_postgres_database.models.projects import projects from simcore_postgres_database.models.users import users from simcore_sdk.node_ports import node_config +from simcore_sdk.node_ports_common.r_clone import is_r_clone_available from yarl import URL @@ -326,3 +328,68 @@ def _create(project_id: str, node_uuid: str, **overrides) -> str: comp_tasks.c.task_id.in_(created_task_ids) ) ) + + +def _set_configuration( + task: Callable[..., str], + project_id: str, + node_id: str, + json_configuration: str, +) -> Dict[str, Any]: + json_configuration = json_configuration.replace("SIMCORE_NODE_UUID", str(node_id)) + configuration = json.loads(json_configuration) + task(project_id, node_id, **configuration) + return configuration + + +def _assign_config( + config_dict: dict, port_type: str, entries: List[Tuple[str, str, Any]] +): + if entries is None: + return + for entry in entries: + config_dict["schema"][port_type].update( + { + entry[0]: { + "label": "some label", + "description": "some description", + "displayOrder": 2, + "type": entry[1], + } + } + ) + if not entry[2] is None: + config_dict[port_type].update({entry[0]: entry[2]}) + + +@pytest.fixture +async def r_clone_settings_factory( + minio_config: Dict[str, Any], storage_service: URL +) -> Awaitable[RCloneSettings]: + async def _factory() -> RCloneSettings: + client = minio_config["client"] + settings = RCloneSettings.parse_obj( + dict( + R_CLONE_S3=dict( + S3_ENDPOINT=client["endpoint"], + S3_ACCESS_KEY=client["access_key"], + S3_SECRET_KEY=client["secret_key"], + S3_BUCKET_NAME=minio_config["bucket_name"], + S3_SECURE=client["secure"], + ), + R_CLONE_PROVIDER=S3Provider.MINIO, + ) + ) + if not await is_r_clone_available(settings): + pytest.skip("rclone not installed") + + return settings + + return _factory() + + +@pytest.fixture +async def r_clone_settings( + r_clone_settings_factory: Awaitable[RCloneSettings], +) -> RCloneSettings: + return await r_clone_settings_factory diff --git a/packages/simcore-sdk/tests/integration/test_node_data_data_manager.py b/packages/simcore-sdk/tests/integration/test_node_data_data_manager_.py similarity index 100% rename from packages/simcore-sdk/tests/integration/test_node_data_data_manager.py rename to packages/simcore-sdk/tests/integration/test_node_data_data_manager_.py diff --git a/packages/simcore-sdk/tests/integration/test_node_ports_v2_nodeports2.py b/packages/simcore-sdk/tests/integration/test_node_ports_v2_nodeports2.py index ca11df54d56..30cbd396011 100644 --- a/packages/simcore-sdk/tests/integration/test_node_ports_v2_nodeports2.py +++ b/packages/simcore-sdk/tests/integration/test_node_ports_v2_nodeports2.py @@ -11,12 +11,13 @@ import threading from asyncio import gather from pathlib import Path -from typing import Any, Callable, Dict, Iterable, Type, Union +from typing import Any, Awaitable, Callable, Dict, Iterable, Optional, Type, Union from uuid import uuid4 import np_helpers import pytest import sqlalchemy as sa +from settings_library.r_clone import RCloneSettings from simcore_sdk import node_ports_v2 from simcore_sdk.node_ports_common.exceptions import UnboundPortError from simcore_sdk.node_ports_v2 import exceptions @@ -30,7 +31,10 @@ "storage", ] -pytest_simcore_ops_services_selection = ["minio", "adminer"] +pytest_simcore_ops_services_selection = [ + "minio", + "adminer", +] async def _check_port_valid( @@ -128,16 +132,29 @@ def config_value_symlink_path(symlink_path: Path) -> Dict[str, Any]: return {"store": "0", "path": symlink_path} +@pytest.fixture(params=[True, False]) +async def option_r_clone_settings( + request, r_clone_settings_factory: Awaitable[RCloneSettings] +) -> Optional[RCloneSettings]: + if request.param: + return await r_clone_settings_factory + return None + + async def test_default_configuration( user_id: int, project_id: str, node_uuid: str, default_configuration: Dict[str, Any], + option_r_clone_settings: Optional[RCloneSettings], ): config_dict = default_configuration await check_config_valid( await node_ports_v2.ports( - user_id=user_id, project_id=project_id, node_uuid=node_uuid + user_id=user_id, + project_id=project_id, + node_uuid=node_uuid, + r_clone_settings=option_r_clone_settings, ), config_dict, ) @@ -148,10 +165,14 @@ async def test_invalid_ports( project_id: str, node_uuid: str, create_special_configuration: Callable, + option_r_clone_settings: Optional[RCloneSettings], ): config_dict, _, _ = create_special_configuration() PORTS = await node_ports_v2.ports( - user_id=user_id, project_id=project_id, node_uuid=node_uuid + user_id=user_id, + project_id=project_id, + node_uuid=node_uuid, + r_clone_settings=option_r_clone_settings, ) await check_config_valid(PORTS, config_dict) @@ -185,7 +206,8 @@ async def test_port_value_accessors( item_type: str, item_value: ItemConcreteValue, item_pytype: Type, -): + option_r_clone_settings: Optional[RCloneSettings], +): # pylint: disable=W0613, W0621 item_key = "some_key" config_dict, _, _ = create_special_configuration( inputs=[(item_key, item_type, item_value)], @@ -193,7 +215,10 @@ async def test_port_value_accessors( ) PORTS = await node_ports_v2.ports( - user_id=user_id, project_id=project_id, node_uuid=node_uuid + user_id=user_id, + project_id=project_id, + node_uuid=node_uuid, + r_clone_settings=option_r_clone_settings, ) await check_config_valid(PORTS, config_dict) @@ -237,6 +262,7 @@ async def test_port_file_accessors( project_id: str, node_uuid: str, e_tag: str, + option_r_clone_settings: Optional[RCloneSettings], ): # pylint: disable=W0613, W0621 config_value["path"] = f"{project_id}/{node_uuid}/{Path(config_value['path']).name}" @@ -250,7 +276,10 @@ async def test_port_file_accessors( assert _node_uuid == node_uuid PORTS = await node_ports_v2.ports( - user_id=user_id, project_id=project_id, node_uuid=node_uuid + user_id=user_id, + project_id=project_id, + node_uuid=node_uuid, + r_clone_settings=option_r_clone_settings, ) await check_config_valid(PORTS, config_dict) assert await (await PORTS.outputs)["out_34"].get() is None # check emptyness @@ -294,10 +323,14 @@ async def test_adding_new_ports( node_uuid: str, create_special_configuration: Callable, postgres_db: sa.engine.Engine, + option_r_clone_settings: Optional[RCloneSettings], ): config_dict, project_id, node_uuid = create_special_configuration() PORTS = await node_ports_v2.ports( - user_id=user_id, project_id=project_id, node_uuid=node_uuid + user_id=user_id, + project_id=project_id, + node_uuid=node_uuid, + r_clone_settings=option_r_clone_settings, ) await check_config_valid(PORTS, config_dict) @@ -341,13 +374,17 @@ async def test_removing_ports( node_uuid: str, create_special_configuration: Callable, postgres_db: sa.engine.Engine, + option_r_clone_settings: Optional[RCloneSettings], ): config_dict, project_id, node_uuid = create_special_configuration( inputs=[("in_14", "integer", 15), ("in_17", "boolean", False)], outputs=[("out_123", "string", "blahblah"), ("out_2", "number", -12.3)], ) # pylint: disable=W0612 PORTS = await node_ports_v2.ports( - user_id=user_id, project_id=project_id, node_uuid=node_uuid + user_id=user_id, + project_id=project_id, + node_uuid=node_uuid, + r_clone_settings=option_r_clone_settings, ) await check_config_valid(PORTS, config_dict) # let's remove the first input @@ -391,6 +428,7 @@ async def test_get_value_from_previous_node( item_type: str, item_value: ItemConcreteValue, item_pytype: Type, + option_r_clone_settings: Optional[RCloneSettings], ): config_dict, _, _ = create_2nodes_configuration( prev_node_inputs=None, @@ -403,7 +441,10 @@ async def test_get_value_from_previous_node( ) PORTS = await node_ports_v2.ports( - user_id=user_id, project_id=project_id, node_uuid=node_uuid + user_id=user_id, + project_id=project_id, + node_uuid=node_uuid, + r_clone_settings=option_r_clone_settings, ) await check_config_valid(PORTS, config_dict) @@ -431,6 +472,7 @@ async def test_get_file_from_previous_node( item_type: str, item_value: str, item_pytype: Type, + option_r_clone_settings: Optional[RCloneSettings], ): config_dict, _, _ = create_2nodes_configuration( prev_node_inputs=None, @@ -444,7 +486,10 @@ async def test_get_file_from_previous_node( node_id=node_uuid, ) PORTS = await node_ports_v2.ports( - user_id=user_id, project_id=project_id, node_uuid=node_uuid + user_id=user_id, + project_id=project_id, + node_uuid=node_uuid, + r_clone_settings=option_r_clone_settings, ) await check_config_valid(PORTS, config_dict) file_path = await (await PORTS.inputs)["in_15"].get() @@ -483,6 +528,7 @@ async def test_get_file_from_previous_node_with_mapping_of_same_key_name( item_value: str, item_alias: str, item_pytype: Type, + option_r_clone_settings: Optional[RCloneSettings], ): config_dict, _, this_node_uuid = create_2nodes_configuration( prev_node_inputs=None, @@ -494,7 +540,10 @@ async def test_get_file_from_previous_node_with_mapping_of_same_key_name( node_id=node_uuid, ) PORTS = await node_ports_v2.ports( - user_id=user_id, project_id=project_id, node_uuid=node_uuid + user_id=user_id, + project_id=project_id, + node_uuid=node_uuid, + r_clone_settings=option_r_clone_settings, ) await check_config_valid(PORTS, config_dict) # add a filetokeymap @@ -540,6 +589,7 @@ async def test_file_mapping( item_value: str, item_alias: str, item_pytype: Type, + option_r_clone_settings: Optional[RCloneSettings], ): config_dict, project_id, node_uuid = create_special_configuration( inputs=[("in_1", item_type, await create_store_link(item_value))], @@ -548,7 +598,10 @@ async def test_file_mapping( node_id=node_uuid, ) PORTS = await node_ports_v2.ports( - user_id=user_id, project_id=project_id, node_uuid=node_uuid + user_id=user_id, + project_id=project_id, + node_uuid=node_uuid, + r_clone_settings=option_r_clone_settings, ) await check_config_valid(PORTS, config_dict) # add a filetokeymap @@ -621,6 +674,7 @@ async def test_regression_concurrent_port_update_fails( int_item_value: int, parallel_int_item_value: int, port_count: int, + option_r_clone_settings: Optional[RCloneSettings], ) -> None: """ when using `await PORTS.outputs` test will fail @@ -631,7 +685,10 @@ async def test_regression_concurrent_port_update_fails( config_dict, _, _ = create_special_configuration(inputs=[], outputs=outputs) PORTS = await node_ports_v2.ports( - user_id=user_id, project_id=project_id, node_uuid=node_uuid + user_id=user_id, + project_id=project_id, + node_uuid=node_uuid, + r_clone_settings=option_r_clone_settings, ) await check_config_valid(PORTS, config_dict) @@ -668,13 +725,17 @@ async def test_batch_update_inputs_outputs( node_uuid: str, create_special_configuration: Callable, port_count: int, + option_r_clone_settings: Optional[RCloneSettings], ) -> None: outputs = [(f"value_out_{i}", "integer", None) for i in range(port_count)] inputs = [(f"value_in_{i}", "integer", None) for i in range(port_count)] config_dict, _, _ = create_special_configuration(inputs=inputs, outputs=outputs) PORTS = await node_ports_v2.ports( - user_id=user_id, project_id=project_id, node_uuid=node_uuid + user_id=user_id, + project_id=project_id, + node_uuid=node_uuid, + r_clone_settings=option_r_clone_settings, ) await check_config_valid(PORTS, config_dict) diff --git a/packages/simcore-sdk/tests/integration/test_node_ports_v2_r_clone_.py b/packages/simcore-sdk/tests/integration/test_node_ports_v2_r_clone_.py new file mode 100644 index 00000000000..d7345dada3f --- /dev/null +++ b/packages/simcore-sdk/tests/integration/test_node_ports_v2_r_clone_.py @@ -0,0 +1,210 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument + +import asyncio +import shutil +from contextlib import asynccontextmanager +from pathlib import Path +from typing import AsyncGenerator, AsyncIterable, Final, Iterator, Optional +from uuid import uuid4 + +import aioboto3 +import pytest +import sqlalchemy as sa +from _pytest.fixtures import FixtureRequest +from aiohttp import ClientSession +from faker import Faker +from pytest_mock.plugin import MockerFixture +from settings_library.r_clone import RCloneSettings +from simcore_postgres_database.models.file_meta_data import file_meta_data +from simcore_sdk.node_ports_common import r_clone +from simcore_sdk.node_ports_common.constants import SIMCORE_LOCATION + +pytest_simcore_core_services_selection = [ + "migration", + "postgres", + "storage", +] + +pytest_simcore_ops_services_selection = [ + "minio", + "adminer", +] + + +WAIT_FOR_S3_BACKEND_TO_UPDATE: Final[float] = 1.0 + + +class _TestException(Exception): + pass + + +# FIXTURES + + +@pytest.fixture( + params=[ + f"{uuid4()}.bin", + "some funky name.txt", + "öä$äö2-34 no extension", + ] +) +def file_name(request: FixtureRequest) -> str: + return request.param + + +@pytest.fixture +def upload_file_dir(tmp_path: Path) -> Iterator[Path]: + assert tmp_path.is_dir() + yield tmp_path + shutil.rmtree(tmp_path) + + +@pytest.fixture +def file_to_upload(upload_file_dir: Path, file_name: str, faker: Faker) -> Path: + # generate file with data + file_path = upload_file_dir / file_name + file_path.write_text(faker.paragraph(nb_sentences=5)) + return file_path + + +@pytest.fixture +def local_file_for_download(upload_file_dir: Path, file_name: str) -> Path: + local_file_path = upload_file_dir / f"__local__{file_name}" + return local_file_path + + +@pytest.fixture +def s3_object(project_id: str, node_uuid: str, file_name: str) -> str: + s3_path = Path(project_id) / node_uuid / file_name + return f"{s3_path}" + + +@pytest.fixture +async def cleanup_s3( + r_clone_settings: RCloneSettings, s3_object: str +) -> AsyncIterable[None]: + yield + async with _get_s3_object(r_clone_settings, s3_object) as s3_object: + await s3_object.delete() + + +@pytest.fixture +def raise_error_after_upload( + mocker: MockerFixture, postgres_db: sa.engine.Engine, s3_object: str +) -> None: + handler = r_clone._async_command # pylint: disable=protected-access + + async def _mock_async_command(*cmd: str, cwd: Optional[str] = None) -> str: + await handler(*cmd, cwd=cwd) + assert _is_file_present(postgres_db=postgres_db, s3_object=s3_object) is True + + raise _TestException() + + mocker.patch( + "simcore_sdk.node_ports_common.r_clone._async_command", + side_effect=_mock_async_command, + ) + + +@pytest.fixture +async def client_session(filemanager_cfg: None) -> AsyncIterable[ClientSession]: + async with ClientSession() as session: + yield session + + +# UTILS + + +@asynccontextmanager +async def _get_s3_object( + r_clone_settings: RCloneSettings, s3_path: str +) -> AsyncGenerator["aioboto3.resources.factory.s3.Object", None]: + session = aioboto3.Session( + aws_access_key_id=r_clone_settings.R_CLONE_S3.S3_ACCESS_KEY, + aws_secret_access_key=r_clone_settings.R_CLONE_S3.S3_SECRET_KEY, + ) + async with session.resource( + "s3", endpoint_url=r_clone_settings.R_CLONE_S3.S3_ENDPOINT + ) as s3: + s3_object = await s3.Object( + bucket_name=r_clone_settings.R_CLONE_S3.S3_BUCKET_NAME, + key=s3_path.lstrip(r_clone_settings.R_CLONE_S3.S3_BUCKET_NAME), + ) + yield s3_object + + +async def _download_s3_object( + r_clone_settings: RCloneSettings, s3_path: str, local_path: Path +): + await asyncio.sleep(WAIT_FOR_S3_BACKEND_TO_UPDATE) + async with _get_s3_object(r_clone_settings, s3_path) as s3_object: + await s3_object.download_file(f"{local_path}") + + +def _is_file_present(postgres_db: sa.engine.Engine, s3_object: str) -> bool: + with postgres_db.begin() as conn: + result = conn.execute( + file_meta_data.select().where(file_meta_data.c.file_uuid == s3_object) + ) + result_list = list(result) + result_len = len(result_list) + assert result_len <= 1, result_list + return result_len == 1 + + +# TESTS + + +async def test_sync_local_to_s3( + r_clone_settings: RCloneSettings, + s3_object: str, + file_to_upload: Path, + local_file_for_download: Path, + user_id: int, + postgres_db: sa.engine.Engine, + client_session: ClientSession, + cleanup_s3: None, +) -> None: + + await r_clone.sync_local_to_s3( + session=client_session, + r_clone_settings=r_clone_settings, + s3_object=s3_object, + local_file_path=file_to_upload, + user_id=user_id, + store_id=SIMCORE_LOCATION, + ) + + await _download_s3_object( + r_clone_settings=r_clone_settings, + s3_path=s3_object, + local_path=local_file_for_download, + ) + + # check same file contents after upload and download + assert file_to_upload.read_text() == local_file_for_download.read_text() + + assert _is_file_present(postgres_db=postgres_db, s3_object=s3_object) is True + + +async def test_sync_local_to_s3_cleanup_on_error( + r_clone_settings: RCloneSettings, + s3_object: str, + file_to_upload: Path, + user_id: int, + postgres_db: sa.engine.Engine, + client_session: ClientSession, + cleanup_s3: None, + raise_error_after_upload: None, +) -> None: + with pytest.raises(_TestException): + await r_clone.sync_local_to_s3( + session=client_session, + r_clone_settings=r_clone_settings, + s3_object=s3_object, + local_file_path=file_to_upload, + user_id=user_id, + store_id=SIMCORE_LOCATION, + ) + assert _is_file_present(postgres_db=postgres_db, s3_object=s3_object) is False diff --git a/packages/simcore-sdk/tests/unit/test_node_data_data_manager.py b/packages/simcore-sdk/tests/unit/test_node_data_data_manager.py index f1067c68315..8721233166c 100644 --- a/packages/simcore-sdk/tests/unit/test_node_data_data_manager.py +++ b/packages/simcore-sdk/tests/unit/test_node_data_data_manager.py @@ -74,6 +74,7 @@ async def test_push_folder( mock_temporary_directory.assert_called_once() mock_filemanager.upload_file.assert_called_once_with( local_file_path=(test_compression_folder / "{}.zip".format(test_folder.stem)), + r_clone_settings=None, s3_object=f"{project_id}/{node_uuid}/{test_folder.stem}.zip", store_id="0", store_name=None, @@ -119,6 +120,7 @@ async def test_push_file( await data_manager.push(user_id, project_id, node_uuid, file_path) mock_temporary_directory.assert_not_called() mock_filemanager.upload_file.assert_called_once_with( + r_clone_settings=None, local_file_path=file_path, s3_object=f"{project_id}/{node_uuid}/{file_path.name}", store_id="0", diff --git a/packages/simcore-sdk/tests/unit/test_node_ports_v2_port.py b/packages/simcore-sdk/tests/unit/test_node_ports_v2_port.py index 3865911e29b..82b81cc38a3 100644 --- a/packages/simcore-sdk/tests/unit/test_node_ports_v2_port.py +++ b/packages/simcore-sdk/tests/unit/test_node_ports_v2_port.py @@ -607,6 +607,7 @@ class FakeNodePorts: user_id: int project_id: str node_uuid: str + r_clone_settings: Optional[Any] = None @staticmethod async def get(key): diff --git a/packages/simcore-sdk/tests/unit/test_node_ports_v2_r_clone.py b/packages/simcore-sdk/tests/unit/test_node_ports_v2_r_clone.py new file mode 100644 index 00000000000..74267ab3463 --- /dev/null +++ b/packages/simcore-sdk/tests/unit/test_node_ports_v2_r_clone.py @@ -0,0 +1,100 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=protected-access +# pylint: disable=unused-argument + +import subprocess +from pathlib import Path +from typing import Iterable, List, Optional +from unittest.mock import Mock + +import pytest +from pytest_mock.plugin import MockerFixture +from faker import Faker +from pytest import MonkeyPatch +from settings_library.r_clone import S3Provider +from simcore_sdk.node_ports_common import r_clone +from simcore_sdk.node_ports_common.r_clone import RCloneSettings + + +@pytest.fixture(params=list(S3Provider)) +def s3_provider(request) -> S3Provider: + return request.param + + +@pytest.fixture +def r_clone_settings( + monkeypatch: MonkeyPatch, s3_provider: S3Provider +) -> RCloneSettings: + monkeypatch.setenv("R_CLONE_PROVIDER", s3_provider.value) + monkeypatch.setenv("S3_ENDPOINT", "endpoint") + monkeypatch.setenv("S3_ACCESS_KEY", "access_key") + monkeypatch.setenv("S3_SECRET_KEY", "secret_key") + monkeypatch.setenv("S3_BUCKET_NAME", "bucket_name") + monkeypatch.setenv("S3_SECURE", "false") + return RCloneSettings() + + +@pytest.fixture +def skip_if_r_clone_is_missing() -> None: + try: + subprocess.check_output(["rclone", "--version"]) + except Exception: # pylint: disable=broad-except + pytest.skip("rclone is not installed") + + +@pytest.fixture +def mock_async_command(mocker: MockerFixture) -> Iterable[Mock]: + mock = Mock() + + original_async_command = r_clone._async_command + + async def _mock_async_command(*cmd: str, cwd: Optional[str] = None) -> str: + mock() + return await original_async_command(*cmd, cwd=cwd) + + mocker.patch( + "simcore_sdk.node_ports_common.r_clone._async_command", + side_effect=_mock_async_command, + ) + + yield mock + + +async def test_is_r_clone_available_cached( + r_clone_settings: RCloneSettings, + mock_async_command: Mock, + skip_if_r_clone_is_missing: None, +) -> None: + for _ in range(3): + result = await r_clone.is_r_clone_available(r_clone_settings) + assert type(result) is bool + assert mock_async_command.call_count == 1 + + assert await r_clone.is_r_clone_available(None) is False + + +async def test__config_file(faker: Faker) -> None: + text_to_write = faker.text() + async with r_clone._config_file(text_to_write) as file_name: + assert text_to_write == Path(file_name).read_text() + assert Path(file_name).exists() is False + + +async def test__async_command_ok() -> None: + await r_clone._async_command("ls", "-la") + + +@pytest.mark.parametrize( + "cmd", + [ + ("__i_do_not_exist__",), + ("ls_", "-lah"), + ], +) +async def test__async_command_error(cmd: List[str]) -> None: + with pytest.raises(r_clone._CommandFailedException) as exe_info: + await r_clone._async_command(*cmd) + assert ( + f"{exe_info.value}" + == f"Command {' '.join(cmd)} finished with exception:\n/bin/sh: 1: {cmd[0]}: not found\n" + ) diff --git a/packages/simcore-sdk/tests/unit/test_storage_client.py b/packages/simcore-sdk/tests/unit/test_storage_client.py index c022f6d9dbf..03429963ae9 100644 --- a/packages/simcore-sdk/tests/unit/test_storage_client.py +++ b/packages/simcore-sdk/tests/unit/test_storage_client.py @@ -140,4 +140,8 @@ async def test_invalid_calls( **{invalid_keyword: None}, **additional_kwargs, } + if ( # pylint: disable=comparison-with-callable + fct_call == get_upload_file_link + ): + kwargs["link_type"] = LinkType.S3 await fct_call(session=session, **kwargs) diff --git a/services/dask-sidecar/src/simcore_service_dask_sidecar/file_utils.py b/services/dask-sidecar/src/simcore_service_dask_sidecar/file_utils.py index e71abbe0b5a..374948c7703 100644 --- a/services/dask-sidecar/src/simcore_service_dask_sidecar/file_utils.py +++ b/services/dask-sidecar/src/simcore_service_dask_sidecar/file_utils.py @@ -65,9 +65,7 @@ def _s3fs_settings_from_s3_settings(s3_settings: S3Settings) -> S3FsSettingsDict "secret": s3_settings.S3_SECRET_KEY, "token": s3_settings.S3_ACCESS_TOKEN, "use_ssl": s3_settings.S3_SECURE, - "client_kwargs": { - "endpoint_url": f"http{'s' if s3_settings.S3_SECURE else ''}://{s3_settings.S3_ENDPOINT}" - }, + "client_kwargs": {"endpoint_url": s3_settings.S3_ENDPOINT}, } diff --git a/services/director-v2/.env-devel b/services/director-v2/.env-devel index 6cc203f9c68..ad703f474cf 100644 --- a/services/director-v2/.env-devel +++ b/services/director-v2/.env-devel @@ -50,7 +50,7 @@ S3_ACCESS_KEY=12345678 S3_SECRET_KEY=12345678 S3_BUCKET_NAME=simcore S3_SECURE=0 -R_CLONE_S3_PROVIDER=MINIO +R_CLONE_PROVIDER=MINIO TRACING_ENABLED=True TRACING_ZIPKIN_ENDPOINT=http://jaeger:9411 diff --git a/services/director-v2/requirements/_base.txt b/services/director-v2/requirements/_base.txt index 3d553f653b3..1fdfc9c4cac 100644 --- a/services/director-v2/requirements/_base.txt +++ b/services/director-v2/requirements/_base.txt @@ -7,7 +7,9 @@ aio-pika==6.8.0 # via -r requirements/_base.in aiocache==0.11.1 - # via -r requirements/_base.in + # via + # -r requirements/../../../packages/simcore-sdk/requirements/_base.in + # -r requirements/_base.in aiodebug==2.3.0 # via # -c requirements/../../../packages/service-library/requirements/./_base.in @@ -15,7 +17,7 @@ aiodebug==2.3.0 # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in aiodocker==0.19.1 # via -r requirements/_base.in -aiofiles==0.5.0 +aiofiles==0.8.0 # via # -c requirements/../../../packages/service-library/requirements/./_base.in # -r requirements/../../../packages/service-library/requirements/_base.in @@ -33,6 +35,7 @@ aiohttp==3.8.1 # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/simcore-sdk/requirements/_base.in @@ -169,6 +172,7 @@ jinja2==2.11.3 # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt @@ -247,6 +251,7 @@ pydantic==1.9.0 # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/dask-task-models-library/requirements/../../../packages/models-library/requirements/_base.in @@ -256,6 +261,7 @@ pydantic==1.9.0 # -r requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/simcore-sdk/requirements/_base.in # -r requirements/_base.in # fastapi @@ -292,6 +298,7 @@ pyyaml==5.4.1 # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/./constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in @@ -334,6 +341,7 @@ sqlalchemy==1.4.31 # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_base.in @@ -372,7 +380,9 @@ tornado==6.1 tqdm==4.62.3 # via -r requirements/../../../packages/simcore-sdk/requirements/_base.in typer==0.4.1 - # via -r requirements/../../../packages/settings-library/requirements/_base.in + # via + # -r requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/_base.in typing-extensions==4.1.1 # via # aiodebug @@ -393,6 +403,7 @@ urllib3==1.26.7 # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt diff --git a/services/director-v2/src/simcore_service_director_v2/core/settings.py b/services/director-v2/src/simcore_service_director_v2/core/settings.py index b7d2f027e4a..544e4ca03ec 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/settings.py +++ b/services/director-v2/src/simcore_service_director_v2/core/settings.py @@ -34,8 +34,8 @@ from settings_library.docker_registry import RegistrySettings from settings_library.http_client_request import ClientRequestSettings from settings_library.postgres import PostgresSettings +from settings_library.r_clone import RCloneSettings from settings_library.rabbit import RabbitSettings -from settings_library.s3 import S3Settings from settings_library.tracing import TracingSettings from settings_library.utils_logging import MixinLoggingSettings from simcore_postgres_database.models.clusters import ClusterType @@ -64,12 +64,6 @@ ) -class S3Provider(str, Enum): - AWS = "AWS" - CEPH = "CEPH" - MINIO = "MINIO" - - class VFSCacheMode(str, Enum): OFF = "off" MINIMAL = "minimal" @@ -77,9 +71,7 @@ class VFSCacheMode(str, Enum): FULL = "full" -class RCloneSettings(S3Settings): - R_CLONE_S3_PROVIDER: S3Provider - +class RCloneSettings(RCloneSettings): # pylint: disable=function-redefined R_CLONE_DIR_CACHE_TIME_SECONDS: PositiveInt = Field( 10, description="time to cache directory entries for", @@ -106,13 +98,6 @@ def enforce_r_clone_requirement(cls, v, values) -> PositiveInt: ) return v - @cached_property - def endpoint(self) -> str: - if not self.S3_ENDPOINT.startswith("http"): - scheme = "https" if self.S3_SECURE else "http" - return f"{scheme}://{self.S3_ENDPOINT}" - return self.S3_ENDPOINT - class StorageSettings(BaseCustomSettings): STORAGE_HOST: str = "storage" diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py index 0364d6d8614..83ff2614510 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py @@ -25,6 +25,9 @@ def _get_environment_variables( ) -> Dict[str, str]: registry_settings = app_settings.DIRECTOR_V2_DOCKER_REGISTRY rabbit_settings = app_settings.DIRECTOR_V2_RABBITMQ + r_clone_settings = ( + app_settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR.DYNAMIC_SIDECAR_R_CLONE_SETTINGS + ) state_exclude = [] if scheduler_data.paths_mapping.state_exclude is not None: @@ -60,6 +63,12 @@ def _get_environment_variables( "RABBIT_USER": f"{rabbit_settings.RABBIT_USER}", "RABBIT_PASSWORD": f"{rabbit_settings.RABBIT_PASSWORD.get_secret_value()}", "RABBIT_CHANNELS": json_dumps(rabbit_settings.RABBIT_CHANNELS), + "S3_ENDPOINT": r_clone_settings.R_CLONE_S3.S3_ENDPOINT, + "S3_ACCESS_KEY": r_clone_settings.R_CLONE_S3.S3_ACCESS_KEY, + "S3_SECRET_KEY": r_clone_settings.R_CLONE_S3.S3_SECRET_KEY, + "S3_BUCKET_NAME": r_clone_settings.R_CLONE_S3.S3_BUCKET_NAME, + "S3_SECURE": f"{r_clone_settings.R_CLONE_S3.S3_SECURE}", + "R_CLONE_PROVIDER": r_clone_settings.R_CLONE_PROVIDER, } diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes_resolver.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes_resolver.py index b1212066737..825b012cbd8 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes_resolver.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes_resolver.py @@ -4,8 +4,9 @@ from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID +from settings_library.r_clone import S3Provider -from ...core.settings import RCloneSettings, S3Provider +from ...core.settings import RCloneSettings from .errors import DynamicSidecarError @@ -20,10 +21,10 @@ def _get_s3_volume_driver_config( "Name": "rclone", "Options": { "type": "s3", - "s3-access_key_id": r_clone_settings.S3_ACCESS_KEY, - "s3-secret_access_key": r_clone_settings.S3_SECRET_KEY, + "s3-access_key_id": r_clone_settings.R_CLONE_S3.S3_ACCESS_KEY, + "s3-secret_access_key": r_clone_settings.R_CLONE_S3.S3_SECRET_KEY, "s3-endpoint": r_clone_settings.endpoint, - "path": f"{r_clone_settings.S3_BUCKET_NAME}/{project_id}/{node_uuid}/{storage_directory_name}", + "path": f"{r_clone_settings.R_CLONE_S3.S3_BUCKET_NAME}/{project_id}/{node_uuid}/{storage_directory_name}", "allow-other": "true", "vfs-cache-mode": r_clone_settings.R_CLONE_VFS_CACHE_MODE.value, # Directly connected to how much time it takes for @@ -37,19 +38,19 @@ def _get_s3_volume_driver_config( extra_options = None - if r_clone_settings.R_CLONE_S3_PROVIDER == S3Provider.MINIO: + if r_clone_settings.R_CLONE_PROVIDER == S3Provider.MINIO: extra_options = { "s3-provider": "Minio", "s3-region": "us-east-1", "s3-location_constraint": "", "s3-server_side_encryption": "", } - elif r_clone_settings.R_CLONE_S3_PROVIDER == S3Provider.CEPH: + elif r_clone_settings.R_CLONE_PROVIDER == S3Provider.CEPH: extra_options = { "s3-provider": "Ceph", "s3-acl": "private", } - elif r_clone_settings.R_CLONE_S3_PROVIDER == S3Provider.AWS: + elif r_clone_settings.R_CLONE_PROVIDER == S3Provider.AWS: extra_options = { "s3-provider": "AWS", "s3-region": "us-east-1", diff --git a/services/director-v2/tests/conftest.py b/services/director-v2/tests/conftest.py index 7c7313c7246..0e06bfb8bd6 100644 --- a/services/director-v2/tests/conftest.py +++ b/services/director-v2/tests/conftest.py @@ -103,7 +103,7 @@ def mock_env(monkeypatch: MonkeyPatch, dynamic_sidecar_docker_image: str) -> Non monkeypatch.setenv("REGISTRY_PW", "test") monkeypatch.setenv("REGISTRY_SSL", "false") - monkeypatch.setenv("R_CLONE_S3_PROVIDER", "MINIO") + monkeypatch.setenv("R_CLONE_PROVIDER", "MINIO") monkeypatch.setenv("POSTGRES_HOST", "mocked_host") monkeypatch.setenv("POSTGRES_USER", "mocked_user") diff --git a/services/director-v2/tests/integration/01/test_computation_api.py b/services/director-v2/tests/integration/01/test_computation_api.py index da5795b430b..be0930cf3a8 100644 --- a/services/director-v2/tests/integration/01/test_computation_api.py +++ b/services/director-v2/tests/integration/01/test_computation_api.py @@ -70,7 +70,7 @@ def mock_env( monkeypatch.setenv("SIMCORE_SERVICES_NETWORK_NAME", "test_swarm_network_name") monkeypatch.setenv("SWARM_STACK_NAME", "test_mocked_stack_name") monkeypatch.setenv("TRAEFIK_SIMCORE_ZONE", "test_mocked_simcore_zone") - monkeypatch.setenv("R_CLONE_S3_PROVIDER", "MINIO") + monkeypatch.setenv("R_CLONE_PROVIDER", "MINIO") @pytest.fixture() diff --git a/services/director-v2/tests/integration/02/test_dynamic_services_routes.py b/services/director-v2/tests/integration/02/test_dynamic_services_routes.py index ff88388bdf1..a5b0731a5c7 100644 --- a/services/director-v2/tests/integration/02/test_dynamic_services_routes.py +++ b/services/director-v2/tests/integration/02/test_dynamic_services_routes.py @@ -124,7 +124,12 @@ async def test_client( monkeypatch.setenv("POSTGRES_PASSWORD", "mocked_password") monkeypatch.setenv("POSTGRES_DB", "mocked_db") monkeypatch.setenv("DIRECTOR_V2_POSTGRES_ENABLED", "false") - monkeypatch.setenv("R_CLONE_S3_PROVIDER", "MINIO") + monkeypatch.setenv("R_CLONE_PROVIDER", "MINIO") + monkeypatch.setenv("S3_ENDPOINT", "endpoint") + monkeypatch.setenv("S3_ACCESS_KEY", "access_key") + monkeypatch.setenv("S3_SECRET_KEY", "secret_key") + monkeypatch.setenv("S3_BUCKET_NAME", "bucket_name") + monkeypatch.setenv("S3_SECURE", "false") # patch host for dynamic-sidecar, not reachable via localhost # the dynamic-sidecar (running inside a container) will use diff --git a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py index b36c859482b..5db46afcec2 100644 --- a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py +++ b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py @@ -301,6 +301,8 @@ def mock_env( dev_features_enabled: str, rabbit_service: RabbitSettings, dask_scheduler_service: str, + minio_config: Dict[str, Any], + storage_service: URL, ) -> None: # Works as below line in docker.compose.yml # ${DOCKER_REGISTRY:-itisfoundation}/dynamic-sidecar:${DOCKER_IMAGE_TAG:-latest} @@ -328,7 +330,12 @@ def mock_env( # this address to reach the rabbit service monkeypatch.setenv("RABBIT_HOST", f"{get_localhost_ip()}") monkeypatch.setenv("POSTGRES_HOST", f"{get_localhost_ip()}") - monkeypatch.setenv("R_CLONE_S3_PROVIDER", "MINIO") + monkeypatch.setenv("R_CLONE_PROVIDER", "MINIO") + monkeypatch.setenv("S3_ENDPOINT", minio_config["client"]["endpoint"]) + monkeypatch.setenv("S3_ACCESS_KEY", minio_config["client"]["access_key"]) + monkeypatch.setenv("S3_SECRET_KEY", minio_config["client"]["secret_key"]) + monkeypatch.setenv("S3_BUCKET_NAME", minio_config["bucket_name"]) + monkeypatch.setenv("S3_SECURE", minio_config["client"]["secure"]) monkeypatch.setenv("DIRECTOR_V2_DEV_FEATURES_ENABLED", dev_features_enabled) monkeypatch.setenv("DIRECTOR_V2_TRACING", "null") monkeypatch.setenv( @@ -611,7 +618,9 @@ async def _fetch_data_via_aioboto( aws_access_key_id=r_clone_settings.S3_ACCESS_KEY, aws_secret_access_key=r_clone_settings.S3_SECRET_KEY, ) - async with session.resource("s3", endpoint_url=r_clone_settings.endpoint) as s3: + async with session.resource( + "s3", endpoint_url=r_clone_settings.R_CLONE_S3.S3_ENDPOINT + ) as s3: bucket = await s3.Bucket(r_clone_settings.S3_BUCKET_NAME) async for s3_object in bucket.objects.all(): key_path = f"{project_id}/{node_id}/{DY_SERVICES_R_CLONE_DIR_NAME}/" diff --git a/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py b/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py index 6ffc57dd64b..f2c9bc5d106 100644 --- a/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py +++ b/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py @@ -128,6 +128,8 @@ def _assemble_node_data(spec: Dict, label: str) -> Dict[str, str]: @pytest.fixture async def director_v2_client( minimal_configuration: None, + minio_config: Dict[str, Any], + storage_service: URL, network_name: str, monkeypatch, ) -> AsyncIterable[httpx.AsyncClient]: @@ -154,7 +156,12 @@ async def director_v2_client( monkeypatch.setenv("POSTGRES_HOST", f"{get_localhost_ip()}") monkeypatch.setenv("COMPUTATIONAL_BACKEND_DASK_CLIENT_ENABLED", "false") monkeypatch.setenv("COMPUTATIONAL_BACKEND_ENABLED", "false") - monkeypatch.setenv("R_CLONE_S3_PROVIDER", "MINIO") + monkeypatch.setenv("R_CLONE_PROVIDER", "MINIO") + monkeypatch.setenv("S3_ENDPOINT", minio_config["client"]["endpoint"]) + monkeypatch.setenv("S3_ACCESS_KEY", minio_config["client"]["access_key"]) + monkeypatch.setenv("S3_SECRET_KEY", minio_config["client"]["secret_key"]) + monkeypatch.setenv("S3_BUCKET_NAME", minio_config["bucket_name"]) + monkeypatch.setenv("S3_SECURE", minio_config["client"]["secure"]) # patch host for dynamic-sidecar, not reachable via localhost # the dynamic-sidecar (running inside a container) will use diff --git a/services/director-v2/tests/unit/test_core_settings.py b/services/director-v2/tests/unit/test_core_settings.py index b6d21c655d5..d1550a03f1a 100644 --- a/services/director-v2/tests/unit/test_core_settings.py +++ b/services/director-v2/tests/unit/test_core_settings.py @@ -9,12 +9,12 @@ from models_library.basic_types import LogLevel from pydantic import ValidationError from pytest import FixtureRequest +from settings_library.r_clone import S3Provider from simcore_service_director_v2.core.settings import ( AppSettings, BootModeEnum, DynamicSidecarSettings, RCloneSettings, - S3Provider, ) @@ -35,7 +35,7 @@ def test_supported_backends_did_not_change() -> None: "endpoint, is_secure", [ ("localhost", False), - ("s3_aws", True), + ("s3_aws", False), ("https://ceph.home", True), ("http://local.dev", False), ], @@ -43,19 +43,22 @@ def test_supported_backends_did_not_change() -> None: def test_expected_s3_endpoint( endpoint: str, is_secure: bool, monkeypatch: MonkeyPatch ) -> None: - monkeypatch.setenv("R_CLONE_S3_PROVIDER", "MINIO") + monkeypatch.setenv("R_CLONE_PROVIDER", "MINIO") monkeypatch.setenv("S3_ENDPOINT", endpoint) monkeypatch.setenv("S3_SECURE", "true" if is_secure else "false") + monkeypatch.setenv("S3_ACCESS_KEY", "access_key") + monkeypatch.setenv("S3_SECRET_KEY", "secret_key") + monkeypatch.setenv("S3_BUCKET_NAME", "bucket_name") r_clone_settings = RCloneSettings() scheme = "https" if is_secure else "http" - assert r_clone_settings.endpoint.startswith(f"{scheme}://") - assert r_clone_settings.endpoint.endswith(endpoint) + assert r_clone_settings.R_CLONE_S3.S3_ENDPOINT.startswith(f"{scheme}://") + assert r_clone_settings.R_CLONE_S3.S3_ENDPOINT.endswith(endpoint) def test_enforce_r_clone_requirement(monkeypatch: MonkeyPatch) -> None: - monkeypatch.setenv("R_CLONE_S3_PROVIDER", "MINIO") + monkeypatch.setenv("R_CLONE_PROVIDER", "MINIO") monkeypatch.setenv("R_CLONE_POLL_INTERVAL_SECONDS", "11") with pytest.raises(ValueError): RCloneSettings() diff --git a/services/director-v2/tests/unit/with_swarm/test_modules_dynamic_sidecar_docker_api.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_api.py similarity index 97% rename from services/director-v2/tests/unit/with_swarm/test_modules_dynamic_sidecar_docker_api.py rename to services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_api.py index 0ff86e81f49..69b3c6684ee 100644 --- a/services/director-v2/tests/unit/with_swarm/test_modules_dynamic_sidecar_docker_api.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_api.py @@ -59,7 +59,12 @@ def dynamic_sidecar_settings( monkeypatch.setenv("TRAEFIK_SIMCORE_ZONE", "test_traefik_zone") monkeypatch.setenv("SWARM_STACK_NAME", "test_swarm_name") monkeypatch.setenv("SIMCORE_SERVICES_NETWORK_NAME", "test_network_name") - monkeypatch.setenv("R_CLONE_S3_PROVIDER", "MINIO") + monkeypatch.setenv("R_CLONE_PROVIDER", "MINIO") + monkeypatch.setenv("S3_ENDPOINT", "endpoint") + monkeypatch.setenv("S3_ACCESS_KEY", "access_key") + monkeypatch.setenv("S3_SECRET_KEY", "secret_key") + monkeypatch.setenv("S3_BUCKET_NAME", "bucket_name") + monkeypatch.setenv("S3_SECURE", "false") return DynamicSidecarSettings.create_from_envs() @@ -399,7 +404,12 @@ def test_valid_network_names( monkeypatch.setenv("SIMCORE_SERVICES_NETWORK_NAME", simcore_services_network_name) monkeypatch.setenv("TRAEFIK_SIMCORE_ZONE", "test_traefik_zone") monkeypatch.setenv("SWARM_STACK_NAME", "test_swarm_name") - monkeypatch.setenv("R_CLONE_S3_PROVIDER", "MINIO") + monkeypatch.setenv("R_CLONE_PROVIDER", "MINIO") + monkeypatch.setenv("S3_ENDPOINT", "endpoint") + monkeypatch.setenv("S3_ACCESS_KEY", "access_key") + monkeypatch.setenv("S3_SECRET_KEY", "secret_key") + monkeypatch.setenv("S3_BUCKET_NAME", "bucket_name") + monkeypatch.setenv("S3_SECURE", "false") dynamic_sidecar_settings = DynamicSidecarSettings.create_from_envs() assert dynamic_sidecar_settings diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs.py index 6d375becd14..2f514b1c3f0 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs.py @@ -37,7 +37,12 @@ def mocked_env(monkeypatch: MonkeyPatch) -> Iterator[Dict[str, str]]: "SIMCORE_SERVICES_NETWORK_NAME": "simcore_services_network_name", "TRAEFIK_SIMCORE_ZONE": "test_traefik_zone", "SWARM_STACK_NAME": "test_swarm_name", - "R_CLONE_S3_PROVIDER": "MINIO", + "R_CLONE_PROVIDER": "MINIO", + "S3_ENDPOINT": "endpoint", + "S3_ACCESS_KEY": "access_key", + "S3_SECRET_KEY": "secret_key", + "S3_BUCKET_NAME": "bucket_name", + "S3_SECURE": "false", } with monkeypatch.context() as m: diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs_sidecar.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs_sidecar.py index 853773d666b..aec75c45d27 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs_sidecar.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs_sidecar.py @@ -22,7 +22,12 @@ "SIMCORE_SERVICES_NETWORK_NAME": "simcore_services_network_name", "TRAEFIK_SIMCORE_ZONE": "test_traefik_zone", "SWARM_STACK_NAME": "test_swarm_name", - "R_CLONE_S3_PROVIDER": "MINIO", + "R_CLONE_PROVIDER": "MINIO", + "S3_ENDPOINT": "endpoint", + "S3_ACCESS_KEY": "s3_access_key", + "S3_SECRET_KEY": "s3_secret_key", + "S3_BUCKET_NAME": "bucket_name", + "S3_SECURE": "false", } EXPECTED_DYNAMIC_SIDECAR_ENV_VAR_NAMES = { @@ -53,6 +58,12 @@ "REGISTRY_USER", "SIMCORE_HOST_NAME", "STORAGE_ENDPOINT", + "R_CLONE_PROVIDER", + "S3_ENDPOINT", + "S3_ACCESS_KEY", + "S3_SECRET_KEY", + "S3_BUCKET_NAME", + "S3_SECURE", } diff --git a/services/director-v2/tests/unit/with_swarm/test_modules_dynamic_sidecar_scheduler.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py similarity index 98% rename from services/director-v2/tests/unit/with_swarm/test_modules_dynamic_sidecar_scheduler.py rename to services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py index 720ed315665..0c40b0e530d 100644 --- a/services/director-v2/tests/unit/with_swarm/test_modules_dynamic_sidecar_scheduler.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py @@ -196,7 +196,12 @@ def dynamic_sidecar_settings( "DIRECTOR_V2_DYNAMIC_SCHEDULER_INTERVAL_SECONDS", str(TEST_SCHEDULER_INTERVAL_SECONDS), ) - monkeypatch.setenv("R_CLONE_S3_PROVIDER", "MINIO") + monkeypatch.setenv("R_CLONE_PROVIDER", "MINIO") + monkeypatch.setenv("S3_ENDPOINT", "endpoint") + monkeypatch.setenv("S3_ACCESS_KEY", "access_key") + monkeypatch.setenv("S3_SECRET_KEY", "secret_key") + monkeypatch.setenv("S3_BUCKET_NAME", "bucket_name") + monkeypatch.setenv("S3_SECURE", "false") app_settings = AppSettings.create_from_envs() return app_settings diff --git a/services/director-v2/tests/unit/with_swarm/test_routes_dynamic_services.py b/services/director-v2/tests/unit/test_routes_dynamic_services.py similarity index 98% rename from services/director-v2/tests/unit/with_swarm/test_routes_dynamic_services.py rename to services/director-v2/tests/unit/test_routes_dynamic_services.py index 2a47a38afeb..8641b9df887 100644 --- a/services/director-v2/tests/unit/with_swarm/test_routes_dynamic_services.py +++ b/services/director-v2/tests/unit/test_routes_dynamic_services.py @@ -97,7 +97,12 @@ def mock_env(monkeypatch: MonkeyPatch, docker_swarm: None) -> None: monkeypatch.setenv("SC_BOOT_MODE", "production") - monkeypatch.setenv("R_CLONE_S3_PROVIDER", "MINIO") + monkeypatch.setenv("R_CLONE_PROVIDER", "MINIO") + monkeypatch.setenv("S3_ENDPOINT", "endpoint") + monkeypatch.setenv("S3_ACCESS_KEY", "access_key") + monkeypatch.setenv("S3_SECRET_KEY", "secret_key") + monkeypatch.setenv("S3_BUCKET_NAME", "bucket_name") + monkeypatch.setenv("S3_SECURE", "false") @pytest.fixture diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_comp_scheduler_dask_scheduler.py b/services/director-v2/tests/unit/with_dbs/test_modules_comp_scheduler_dask_scheduler.py index 2fcc1f150e2..9adf6cc9607 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_comp_scheduler_dask_scheduler.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_comp_scheduler_dask_scheduler.py @@ -80,7 +80,12 @@ def minimal_dask_scheduler_config( monkeypatch.setenv("DIRECTOR_V2_POSTGRES_ENABLED", "1") monkeypatch.setenv("COMPUTATIONAL_BACKEND_DASK_CLIENT_ENABLED", "1") monkeypatch.setenv("COMPUTATIONAL_BACKEND_ENABLED", "1") - monkeypatch.setenv("R_CLONE_S3_PROVIDER", "MINIO") + monkeypatch.setenv("R_CLONE_PROVIDER", "MINIO") + monkeypatch.setenv("S3_ENDPOINT", "endpoint") + monkeypatch.setenv("S3_ACCESS_KEY", "access_key") + monkeypatch.setenv("S3_SECRET_KEY", "secret_key") + monkeypatch.setenv("S3_BUCKET_NAME", "bucket_name") + monkeypatch.setenv("S3_SECURE", "false") @pytest.fixture diff --git a/services/director-v2/tests/unit/with_dbs/test_route_clusters.py b/services/director-v2/tests/unit/with_dbs/test_route_clusters.py index 28e1908686c..b76f1cdf54e 100644 --- a/services/director-v2/tests/unit/with_dbs/test_route_clusters.py +++ b/services/director-v2/tests/unit/with_dbs/test_route_clusters.py @@ -49,7 +49,12 @@ def clusters_config( ): monkeypatch.setenv("DIRECTOR_V2_POSTGRES_ENABLED", "1") monkeypatch.setenv("COMPUTATIONAL_BACKEND_DASK_CLIENT_ENABLED", "1") - monkeypatch.setenv("R_CLONE_S3_PROVIDER", "MINIO") + monkeypatch.setenv("R_CLONE_PROVIDER", "MINIO") + monkeypatch.setenv("S3_ENDPOINT", "endpoint") + monkeypatch.setenv("S3_ACCESS_KEY", "access_key") + monkeypatch.setenv("S3_SECRET_KEY", "secret_key") + monkeypatch.setenv("S3_BUCKET_NAME", "bucket_name") + monkeypatch.setenv("S3_SECURE", "false") @pytest.fixture diff --git a/services/director-v2/tests/unit/with_dbs/test_route_clusters_details.py b/services/director-v2/tests/unit/with_dbs/test_route_clusters_details.py index 2e355ecb899..a0581828d8e 100644 --- a/services/director-v2/tests/unit/with_dbs/test_route_clusters_details.py +++ b/services/director-v2/tests/unit/with_dbs/test_route_clusters_details.py @@ -37,7 +37,12 @@ def clusters_config( ): monkeypatch.setenv("DIRECTOR_V2_POSTGRES_ENABLED", "1") monkeypatch.setenv("COMPUTATIONAL_BACKEND_DASK_CLIENT_ENABLED", "1") - monkeypatch.setenv("R_CLONE_S3_PROVIDER", "MINIO") + monkeypatch.setenv("R_CLONE_PROVIDER", "MINIO") + monkeypatch.setenv("S3_ENDPOINT", "endpoint") + monkeypatch.setenv("S3_ACCESS_KEY", "access_key") + monkeypatch.setenv("S3_SECRET_KEY", "secret_key") + monkeypatch.setenv("S3_BUCKET_NAME", "bucket_name") + monkeypatch.setenv("S3_SECURE", "false") @pytest.fixture diff --git a/services/director-v2/tests/unit/with_dbs/test_route_computations.py b/services/director-v2/tests/unit/with_dbs/test_route_computations.py index 533ccdc6bd1..3d35401461b 100644 --- a/services/director-v2/tests/unit/with_dbs/test_route_computations.py +++ b/services/director-v2/tests/unit/with_dbs/test_route_computations.py @@ -36,6 +36,12 @@ def minimal_configuration( ): monkeypatch.setenv("DIRECTOR_V2_DYNAMIC_SIDECAR_ENABLED", "false") monkeypatch.setenv("DIRECTOR_V2_POSTGRES_ENABLED", "1") + monkeypatch.setenv("R_CLONE_PROVIDER", "MINIO") + monkeypatch.setenv("S3_ENDPOINT", "endpoint") + monkeypatch.setenv("S3_ACCESS_KEY", "access_key") + monkeypatch.setenv("S3_SECRET_KEY", "secret_key") + monkeypatch.setenv("S3_BUCKET_NAME", "bucket_name") + monkeypatch.setenv("S3_SECURE", "false") async def test_get_computation_from_empty_project( diff --git a/services/director-v2/tests/unit/with_dbs/test_utils_dask.py b/services/director-v2/tests/unit/with_dbs/test_utils_dask.py index 5f87d172cb6..88b9412c4b0 100644 --- a/services/director-v2/tests/unit/with_dbs/test_utils_dask.py +++ b/services/director-v2/tests/unit/with_dbs/test_utils_dask.py @@ -241,7 +241,12 @@ def app_with_db( postgres_host_config: Dict[str, str], ): monkeypatch.setenv("DIRECTOR_V2_POSTGRES_ENABLED", "1") - monkeypatch.setenv("R_CLONE_S3_PROVIDER", "MINIO") + monkeypatch.setenv("R_CLONE_PROVIDER", "MINIO") + monkeypatch.setenv("S3_ENDPOINT", "endpoint") + monkeypatch.setenv("S3_ACCESS_KEY", "access_key") + monkeypatch.setenv("S3_SECRET_KEY", "secret_key") + monkeypatch.setenv("S3_BUCKET_NAME", "bucket_name") + monkeypatch.setenv("S3_SECURE", "false") async def test_compute_input_data( diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 49466337bdd..4c5d04765f8 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -115,7 +115,7 @@ services: - S3_BUCKET_NAME=${S3_BUCKET_NAME} - S3_ENDPOINT=${S3_ENDPOINT} - S3_SECRET_KEY=${S3_SECRET_KEY} - - R_CLONE_S3_PROVIDER=${R_CLONE_S3_PROVIDER} + - R_CLONE_PROVIDER=${R_CLONE_PROVIDER} - MONITORING_ENABLED=${MONITORING_ENABLED:-True} - SIMCORE_SERVICES_NETWORK_NAME=interactive_services_subnet - TRACING_THRIFT_COMPACT_ENDPOINT=${TRACING_THRIFT_COMPACT_ENDPOINT} diff --git a/services/dynamic-sidecar/Dockerfile b/services/dynamic-sidecar/Dockerfile index e72a26d30d3..9e83fdcb946 100644 --- a/services/dynamic-sidecar/Dockerfile +++ b/services/dynamic-sidecar/Dockerfile @@ -16,6 +16,7 @@ RUN set -eux && \ gosu \ curl \ libmagic1 \ + curl \ && \ rm -rf /var/lib/apt/lists/* && \ # verify that the binary works @@ -47,6 +48,12 @@ ENV PATH="${VIRTUAL_ENV}/bin:$PATH" # volumes between itself and the spawned containers ENV DY_VOLUMES="/dy-volumes" +# rclone installation +ARG R_CLONE_VERSION="1.58.0" +RUN curl --silent --location --remote-name "https://downloads.rclone.org/v${R_CLONE_VERSION}/rclone-v${R_CLONE_VERSION}-linux-amd64.deb" && \ + dpkg --install "rclone-v${R_CLONE_VERSION}-linux-amd64.deb" && \ + rm "rclone-v${R_CLONE_VERSION}-linux-amd64.deb" && \ + rclone --version # -------------------------- Build stage ------------------- # Installs build/package management tools and third party dependencies diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py index 031de75520d..ffe50483bd3 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/api/containers.py @@ -39,6 +39,7 @@ ) from ..models.domains.shared_store import SharedStore from ..models.schemas.application_health import ApplicationHealth +from ..modules.directory_watcher import directory_watcher_disabled from ..modules.mounted_fs import MountedVolumes logger = logging.getLogger(__name__) @@ -64,12 +65,13 @@ async def _task_docker_compose_up( "docker-compose --project-name {project} --file {file_path} " "up --no-build --detach" ) - finished_without_errors, stdout = await write_file_and_run_command( - settings=settings, - file_content=shared_store.compose_spec, - command=command, - command_timeout=None, - ) + with directory_watcher_disabled(app): + finished_without_errors, stdout = await write_file_and_run_command( + settings=settings, + file_content=shared_store.compose_spec, + command=command, + command_timeout=None, + ) message = f"Finished {command} with output\n{stdout}" if finished_without_errors: diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py index f2efbdc27c9..6d306feceee 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/settings.py @@ -10,6 +10,7 @@ from pydantic import Field, PositiveInt, validator from settings_library.base import BaseCustomSettings from settings_library.docker_registry import RegistrySettings +from settings_library.r_clone import RCloneSettings from settings_library.rabbit import RabbitSettings @@ -93,6 +94,7 @@ def match_logging_level(cls, v: str) -> str: REGISTRY_SETTINGS: RegistrySettings = Field(auto_default_from_env=True) RABBIT_SETTINGS: Optional[RabbitSettings] = Field(auto_default_from_env=True) + DY_SIDECAR_R_CLONE_SETTINGS: RCloneSettings = Field(auto_default_from_env=True) @property def is_development_mode(self) -> bool: diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/data_manager.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/data_manager.py index 0582acc355b..7a8873b2e0e 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/data_manager.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/data_manager.py @@ -76,5 +76,6 @@ async def upload_path_if_exists(path: Path, state_exclude: List[str]) -> None: project_id=str(settings.DY_SIDECAR_PROJECT_ID), node_uuid=str(settings.DY_SIDECAR_NODE_ID), file_or_folder=path, + r_clone_settings=settings.DY_SIDECAR_R_CLONE_SETTINGS, ) logger.info("Finished upload of %s", path) diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/directory_watcher.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/directory_watcher.py index 87be7e6cdf2..4760cefe648 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/directory_watcher.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/directory_watcher.py @@ -3,10 +3,11 @@ import time from asyncio import AbstractEventLoop from collections import deque +from contextlib import contextmanager from functools import wraps from os import name from pathlib import Path -from typing import Any, Awaitable, Callable, Deque, Optional +from typing import Any, Awaitable, Callable, Deque, Generator, Optional from fastapi import FastAPI from servicelib.utils import logged_gather @@ -215,8 +216,18 @@ def enable_directory_watcher(app: FastAPI) -> None: app.state.dir_watcher.enable_event_propagation() +@contextmanager +def directory_watcher_disabled(app: FastAPI) -> Generator[None, None, None]: + disable_directory_watcher(app) + try: + yield None + finally: + enable_directory_watcher(app) + + __all__ = [ "disable_directory_watcher", "enable_directory_watcher", + "directory_watcher_disabled", "setup_directory_watcher", ] diff --git a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/nodeports.py b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/nodeports.py index 8842b01a4e2..4a7686116f3 100644 --- a/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/nodeports.py +++ b/services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/modules/nodeports.py @@ -60,6 +60,7 @@ async def upload_outputs(outputs_path: Path, port_keys: List[str]) -> None: user_id=settings.DY_SIDECAR_USER_ID, project_id=str(settings.DY_SIDECAR_PROJECT_ID), node_uuid=str(settings.DY_SIDECAR_NODE_ID), + r_clone_settings=settings.DY_SIDECAR_R_CLONE_SETTINGS, ) # let's gather the tasks @@ -180,6 +181,7 @@ async def download_target_ports( user_id=settings.DY_SIDECAR_USER_ID, project_id=str(settings.DY_SIDECAR_PROJECT_ID), node_uuid=str(settings.DY_SIDECAR_NODE_ID), + r_clone_settings=settings.DY_SIDECAR_R_CLONE_SETTINGS, ) data = {} diff --git a/services/dynamic-sidecar/tests/conftest.py b/services/dynamic-sidecar/tests/conftest.py index 2464e02019a..e35c18e6b22 100644 --- a/services/dynamic-sidecar/tests/conftest.py +++ b/services/dynamic-sidecar/tests/conftest.py @@ -101,6 +101,13 @@ def mock_environment( ) monkeypatch_module.setenv("RABBIT_SETTINGS", "null") + monkeypatch_module.setenv("S3_ENDPOINT", "endpoint") + monkeypatch_module.setenv("S3_ACCESS_KEY", "access_key") + monkeypatch_module.setenv("S3_SECRET_KEY", "secret_key") + monkeypatch_module.setenv("S3_BUCKET_NAME", "bucket_name") + monkeypatch_module.setenv("S3_SECURE", "false") + monkeypatch_module.setenv("R_CLONE_PROVIDER", "MINIO") + monkeypatch_module.setattr(mounted_fs, "DY_VOLUMES", mock_dy_volumes) diff --git a/services/dynamic-sidecar/tests/unit/test_core_docker_logs.py b/services/dynamic-sidecar/tests/unit/test_core_docker_logs.py index afb8e8c0329..a57b8d5a7a1 100644 --- a/services/dynamic-sidecar/tests/unit/test_core_docker_logs.py +++ b/services/dynamic-sidecar/tests/unit/test_core_docker_logs.py @@ -57,6 +57,13 @@ def app( monkeypatch_module.setattr(mounted_fs, "DY_VOLUMES", mock_dy_volumes) + monkeypatch_module.setenv("S3_ENDPOINT", "endpoint") + monkeypatch_module.setenv("S3_ACCESS_KEY", "access_key") + monkeypatch_module.setenv("S3_SECRET_KEY", "secret_key") + monkeypatch_module.setenv("S3_BUCKET_NAME", "bucket_name") + monkeypatch_module.setenv("S3_SECURE", "false") + monkeypatch_module.setenv("R_CLONE_PROVIDER", "MINIO") + yield assemble_application() diff --git a/services/dynamic-sidecar/tests/unit/test_core_rabbitmq.py b/services/dynamic-sidecar/tests/unit/test_core_rabbitmq.py index 16867aa021f..693219732ae 100644 --- a/services/dynamic-sidecar/tests/unit/test_core_rabbitmq.py +++ b/services/dynamic-sidecar/tests/unit/test_core_rabbitmq.py @@ -106,6 +106,13 @@ def mock_environment( monkeypatch_module.setattr(mounted_fs, "DY_VOLUMES", mock_dy_volumes) + monkeypatch_module.setenv("S3_ENDPOINT", "endpoint") + monkeypatch_module.setenv("S3_ACCESS_KEY", "access_key") + monkeypatch_module.setenv("S3_SECRET_KEY", "secret_key") + monkeypatch_module.setenv("S3_BUCKET_NAME", "bucket_name") + monkeypatch_module.setenv("S3_SECURE", "false") + monkeypatch_module.setenv("R_CLONE_PROVIDER", "MINIO") + @pytest.fixture def app(mock_environment: None) -> FastAPI: diff --git a/services/dynamic-sidecar/tests/unit/test_core_settings.py b/services/dynamic-sidecar/tests/unit/test_core_settings.py index 8ee9737d2b3..9ec3a9988b4 100644 --- a/services/dynamic-sidecar/tests/unit/test_core_settings.py +++ b/services/dynamic-sidecar/tests/unit/test_core_settings.py @@ -35,6 +35,13 @@ def mocked_non_request_settings(tmp_dir: Path, monkeypatch: MonkeyPatch) -> None monkeypatch.setenv("DY_SIDECAR_PROJECT_ID", f"{uuid.uuid4()}") monkeypatch.setenv("DY_SIDECAR_NODE_ID", f"{uuid.uuid4()}") + monkeypatch.setenv("S3_ENDPOINT", "endpoint") + monkeypatch.setenv("S3_ACCESS_KEY", "access_key") + monkeypatch.setenv("S3_SECRET_KEY", "secret_key") + monkeypatch.setenv("S3_BUCKET_NAME", "bucket_name") + monkeypatch.setenv("S3_SECURE", "false") + monkeypatch.setenv("R_CLONE_PROVIDER", "MINIO") + def test_non_request_dynamic_sidecar_settings( mocked_non_request_settings: None, diff --git a/services/storage/src/simcore_service_storage/access_layer.py b/services/storage/src/simcore_service_storage/access_layer.py index c453708aa20..bde85f59f4d 100644 --- a/services/storage/src/simcore_service_storage/access_layer.py +++ b/services/storage/src/simcore_service_storage/access_layer.py @@ -70,7 +70,7 @@ def none(cls) -> "AccessRights": class AccessLayerError(Exception): - """ Base class for access-layer related errors """ + """Base class for access-layer related errors""" class InvalidFileIdentifier(AccessLayerError): @@ -282,6 +282,6 @@ async def get_file_access_rights( async def get_readable_project_ids(conn: SAConnection, user_id: int) -> List[ProjectID]: - """ Returns a list of projects where user has granted read-access """ + """Returns a list of projects where user has granted read-access""" projects_access_rights = await list_projects_access_rights(conn, int(user_id)) return [pid for pid, access in projects_access_rights.items() if access.read] diff --git a/services/storage/src/simcore_service_storage/api/v0/openapi.yaml b/services/storage/src/simcore_service_storage/api/v0/openapi.yaml index ac294cbc75d..8a544996743 100644 --- a/services/storage/src/simcore_service_storage/api/v0/openapi.yaml +++ b/services/storage/src/simcore_service_storage/api/v0/openapi.yaml @@ -251,11 +251,11 @@ paths: required: true schema: type: string - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/FileMetaData' + - name: user_id + in: query + required: true + schema: + type: string responses: '200': description: Returns file metadata diff --git a/services/storage/src/simcore_service_storage/application.py b/services/storage/src/simcore_service_storage/application.py index 51b0668c424..896c78810ea 100644 --- a/services/storage/src/simcore_service_storage/application.py +++ b/services/storage/src/simcore_service_storage/application.py @@ -7,6 +7,7 @@ from aiohttp import web from servicelib.aiohttp.application import APP_CONFIG_KEY, create_safe_application +from servicelib.aiohttp.dev_error_logger import setup_dev_error_logger from servicelib.aiohttp.monitoring import setup_monitoring from servicelib.aiohttp.tracing import setup_tracing @@ -45,6 +46,9 @@ def create(settings: Settings) -> web.Application: setup_dsm(app) # core subsystem. Needs s3 and db setups done setup_rest(app) # lastly, we expose API to the world + if settings.LOG_LEVEL == "DEBUG": + setup_dev_error_logger(app) + if settings.STORAGE_MONITORING_ENABLED: setup_monitoring(app, app_name, version=f"{version}") diff --git a/services/storage/src/simcore_service_storage/dsm.py b/services/storage/src/simcore_service_storage/dsm.py index c478a2b54fa..5e821eaee3b 100644 --- a/services/storage/src/simcore_service_storage/dsm.py +++ b/services/storage/src/simcore_service_storage/dsm.py @@ -491,15 +491,34 @@ async def try_update_database_from_storage( ) async def auto_update_database_from_storage_task( self, file_uuid: str, bucket_name: str, object_name: str - ): + ) -> Optional[FileMetaDataEx]: return await self.try_update_database_from_storage( file_uuid, bucket_name, object_name, silence_exception=True ) - async def upload_link(self, user_id: str, file_uuid: str, as_presigned_link: bool): + async def update_metadata( + self, file_uuid: str, user_id: int + ) -> Optional[FileMetaDataEx]: + async with self.engine.acquire() as conn: + can: Optional[AccessRights] = await get_file_access_rights( + conn, int(user_id), file_uuid + ) + if not can.write: + raise web.HTTPForbidden( + reason=f"User {user_id} was not allowed to upload file {file_uuid}" + ) + + bucket_name = self.simcore_bucket_name + object_name = file_uuid + return await self.auto_update_database_from_storage_task( + file_uuid=file_uuid, + bucket_name=bucket_name, + object_name=object_name, + ) + + async def _generate_metadata_for_link(self, user_id: str, file_uuid: str): """ - Creates pre-signed upload link and updates metadata table when - link is used and upload is successfuly completed + Updates metadata table when link is used and upload is successfuly completed SEE _metadata_file_updater """ @@ -509,11 +528,8 @@ async def upload_link(self, user_id: str, file_uuid: str, as_presigned_link: boo conn, int(user_id), file_uuid ) if not can.write: - logger.debug( - "User %s was not allowed to upload file %s", user_id, file_uuid - ) raise web.HTTPForbidden( - reason=f"User does not have enough access rights to upload file {file_uuid}" + reason=f"User {user_id} does not have enough access rights to upload file {file_uuid}" ) @retry(**postgres_service_retry_policy_kwargs) @@ -536,6 +552,14 @@ async def _init_metadata() -> Tuple[int, str]: await _init_metadata() + async def upload_link( + self, user_id: str, file_uuid: str, as_presigned_link: bool + ) -> AnyUrl: + """returns: a presigned upload link + + NOTE: updates metadata once the upload is concluded""" + await self._generate_metadata_for_link(user_id=user_id, file_uuid=file_uuid) + bucket_name = self.simcore_bucket_name object_name = file_uuid @@ -548,12 +572,10 @@ async def _init_metadata() -> Tuple[int, str]: object_name=object_name, ) ) - link = parse_obj_as( - AnyUrl, f"s3://{bucket_name}/{urllib.parse.quote( object_name)}" - ) + link = f"s3://{bucket_name}/{urllib.parse.quote( object_name)}" if as_presigned_link: link = self.s3_client.create_presigned_put_url(bucket_name, object_name) - return f"{link}" + return parse_obj_as(AnyUrl, f"{link}") async def download_link_s3( self, file_uuid: str, user_id: int, as_presigned_link: bool @@ -569,11 +591,8 @@ async def download_link_s3( # If write permission would be required, then shared projects as views cannot # recover data in nodes (e.g. jupyter cannot pull work data) # - logger.debug( - "User %s was not allowed to download file %s", user_id, file_uuid - ) raise web.HTTPForbidden( - reason=f"User does not have enough rights to download {file_uuid}" + reason=f"User {user_id} does not have enough rights to download file {file_uuid}" ) bucket_name = self.simcore_bucket_name @@ -761,23 +780,13 @@ async def deep_copy_project_simcore_s3( conn, int(user_id), project_id=dest_folder ) if not source_access_rights.read: - logger.debug( - "User %s was not allowed to read from project %s", - user_id, - source_folder, - ) raise web.HTTPForbidden( - reason=f"User does not have enough access rights to read from project '{source_folder}'" + reason=f"User {user_id} does not have enough access rights to read from project '{source_folder}'" ) if not dest_access_rights.write: - logger.debug( - "User %s was not allowed to write to project %s", - user_id, - dest_folder, - ) raise web.HTTPForbidden( - reason=f"User does not have enough access rights to write to project '{dest_folder}'" + reason=f"User {user_id} does not have enough access rights to write to project '{dest_folder}'" ) # build up naming map based on labels @@ -936,13 +945,8 @@ async def delete_file(self, user_id: str, location: str, file_uuid: str): conn, int(user_id), file_uuid ) if not can.delete: - logger.debug( - "User %s was not allowed to delete file %s", - user_id, - file_uuid, - ) raise web.HTTPForbidden( - reason=f"User '{user_id}' does not have enough access rights to delete file {file_uuid}" + reason=f"User {user_id} does not have enough access rights to delete file {file_uuid}" ) query = sa.select( @@ -987,13 +991,8 @@ async def delete_project_simcore_s3( conn, int(user_id), project_id ) if not can.delete: - logger.debug( - "User %s was not allowed to delete project %s", - user_id, - project_id, - ) raise web.HTTPForbidden( - reason=f"User does not have delete access for {project_id}" + reason=f"User {user_id} does not have delete access for {project_id}" ) delete_me = file_meta_data.delete().where( diff --git a/services/storage/src/simcore_service_storage/handlers.py b/services/storage/src/simcore_service_storage/handlers.py index a3f13b38c0b..89e20028ce7 100644 --- a/services/storage/src/simcore_service_storage/handlers.py +++ b/services/storage/src/simcore_service_storage/handlers.py @@ -1,8 +1,9 @@ import asyncio import json import logging +import urllib.parse from contextlib import contextmanager -from typing import Any, Dict +from typing import Any, Dict, Optional import attr from aiohttp import web @@ -19,6 +20,7 @@ from .constants import APP_DSM_KEY, DATCORE_STR, SIMCORE_S3_ID, SIMCORE_S3_STR from .db_tokens import get_api_token_and_secret from .dsm import DataStorageManager, DatCoreApiToken +from .models import FileMetaDataEx from .settings import Settings log = logging.getLogger(__name__) @@ -66,7 +68,7 @@ def handle_storage_errors(): except InvalidFileIdentifier as err: raise web.HTTPUnprocessableEntity( - reason=f"{err.identifier} is an invalid file identifier" + reason=f"{err} is an invalid file identifier" ) from err @@ -271,7 +273,7 @@ async def _go(): return {"error": None, "data": sync_results} -# DISABLED: @routes.patch(f"/{api_vtag}/locations/{{location_id}}/files/{{fileId}}/metadata") # type: ignore +@routes.patch(f"/{api_vtag}/locations/{{location_id}}/files/{{fileId}}/metadata") # type: ignore async def update_file_meta_data(request: web.Request): params, query, body = await extract_and_validate(request) @@ -279,17 +281,22 @@ async def update_file_meta_data(request: web.Request): assert query, "query %s" % query # nosec assert not body, "body %s" % body # nosec - assert params["location_id"] # nosec - assert params["fileId"] # nosec - assert query["user_id"] # nosec - with handle_storage_errors(): - location_id = params["location_id"] - _user_id = query["user_id"] - _file_uuid = params["fileId"] + file_uuid = urllib.parse.unquote_plus(params["fileId"]) + user_id = query["user_id"] dsm = await _prepare_storage_manager(params, query, request) - _location = dsm.location_from_id(location_id) + + data: Optional[FileMetaDataEx] = await dsm.update_metadata( + file_uuid=file_uuid, user_id=user_id + ) + if data is None: + raise web.HTTPNotFound(reason=f"Could not update metadata for {file_uuid}") + + return { + "error": None, + "data": {**attr.asdict(data.fmd), "parent_id": data.parent_id}, + } @routes.get(f"/{api_vtag}/locations/{{location_id}}/files/{{fileId}}") # type: ignore @@ -310,6 +317,11 @@ async def download_file(request: web.Request): user_id = query["user_id"] file_uuid = params["fileId"] + if int(location_id) != SIMCORE_S3_ID: + raise web.HTTPPreconditionFailed( + reason=f"Only allowed to fetch s3 link for '{SIMCORE_S3_STR}'" + ) + dsm = await _prepare_storage_manager(params, query, request) location = dsm.location_from_id(location_id) if location == SIMCORE_S3_STR: diff --git a/services/storage/src/simcore_service_storage/rest_models.py b/services/storage/src/simcore_service_storage/rest_models.py index 3eadb58d635..a35710ceaa7 100644 --- a/services/storage/src/simcore_service_storage/rest_models.py +++ b/services/storage/src/simcore_service_storage/rest_models.py @@ -2,6 +2,7 @@ """ from datetime import datetime + from pydantic import BaseModel diff --git a/services/storage/src/simcore_service_storage/s3.py b/services/storage/src/simcore_service_storage/s3.py index daef20d7315..60adba38e9f 100644 --- a/services/storage/src/simcore_service_storage/s3.py +++ b/services/storage/src/simcore_service_storage/s3.py @@ -6,6 +6,7 @@ from aiohttp import web from tenacity import before_sleep_log, retry, stop_after_attempt, wait_fixed +from pydantic import AnyUrl, parse_obj_as from .constants import APP_CONFIG_KEY, APP_S3_KEY from .s3wrapper.s3_client import MinioClientWrapper @@ -54,6 +55,14 @@ async def do_create_bucket(): log.debug("tear-down %s.setup.cleanup_ctx", __name__) +def _minio_client_endpint(s3_endpoint: str) -> str: + # Minio client adds http and https based on the secure paramenter + # provided at construction time, already including the schema + # will cause issues, encoding url to HOST:PORT + url = parse_obj_as(AnyUrl, s3_endpoint) + return f"{url.host}:{url.port}" + + def setup_s3(app: web.Application): """minio/s3 service setup""" @@ -67,7 +76,7 @@ def setup_s3(app: web.Application): cfg = app[APP_CONFIG_KEY] s3_client = MinioClientWrapper( - cfg.STORAGE_S3.S3_ENDPOINT, + _minio_client_endpint(cfg.STORAGE_S3.S3_ENDPOINT), cfg.STORAGE_S3.S3_ACCESS_KEY, cfg.STORAGE_S3.S3_SECRET_KEY, secure=cfg.STORAGE_S3.S3_SECURE, diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index c7efb204957..ac392b284eb 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -2730,15 +2730,7 @@ paths: content: application/json: schema: - type: object - required: - - data - properties: - data: - $ref: '#/paths/~1projects~1%7Bproject_id%7D~1nodes~1%7Bnode_id%7D~1resources/put/requestBody/content/application~1json/schema' - error: - nullable: true - default: null + $ref: '#/paths/~1projects~1%7Bproject_id%7D~1nodes~1%7Bnode_id%7D~1resources/put/responses/200/content/application~1json/schema' default: $ref: '#/components/responses/DefaultErrorResponse' put: @@ -2774,7 +2766,15 @@ paths: content: application/json: schema: - $ref: '#/paths/~1projects~1%7Bproject_id%7D~1nodes~1%7Bnode_id%7D~1resources/get/responses/200/content/application~1json/schema' + type: object + required: + - data + properties: + data: + $ref: '#/paths/~1projects~1%7Bproject_id%7D~1nodes~1%7Bnode_id%7D~1resources/put/requestBody/content/application~1json/schema' + error: + nullable: true + default: null default: $ref: '#/components/responses/DefaultErrorResponse' '/nodes/{nodeInstanceUUID}/outputUi/{outputKey}': @@ -4712,7 +4712,7 @@ paths: content: application/json: schema: - $ref: '#/paths/~1projects~1%7Bproject_id%7D~1nodes~1%7Bnode_id%7D~1resources/get/responses/200/content/application~1json/schema' + $ref: '#/paths/~1projects~1%7Bproject_id%7D~1nodes~1%7Bnode_id%7D~1resources/put/responses/200/content/application~1json/schema' default: $ref: '#/components/responses/DefaultErrorResponse' '/clusters:ping':