From 2c56e369f95056378b660418e79cdb294c27abdd Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Fri, 23 Apr 2021 00:11:04 +0200 Subject: [PATCH 01/27] moved s3wrapper inside storage --- packages/s3wrapper/requirements/Makefile | 6 - packages/s3wrapper/requirements/_base.in | 7 - packages/s3wrapper/requirements/_base.txt | 25 --- packages/s3wrapper/requirements/_test.in | 23 --- packages/s3wrapper/requirements/_test.txt | 147 ------------------ packages/s3wrapper/requirements/_tools.in | 5 - packages/s3wrapper/requirements/_tools.txt | 92 ----------- packages/s3wrapper/requirements/dev.txt | 14 -- packages/s3wrapper/setup.py | 33 ---- packages/s3wrapper/tests/__init__.py | 0 packages/s3wrapper/tests/conftest.py | 9 -- packages/s3wrapper/tests/docker-compose.yml | 10 -- services/storage/requirements/_base.in | 3 +- services/storage/requirements/_base.txt | 11 +- services/storage/requirements/ci.txt | 1 - services/storage/requirements/dev.txt | 1 - services/storage/requirements/prod.txt | 1 - services/storage/setup.py | 1 - .../src/simcore_service_storage/dsm.py | 2 +- .../storage/src/simcore_service_storage/s3.py | 2 +- .../s3wrapper/__init__.py | 0 .../s3wrapper/s3_client.py | 23 ++- services/storage/tests/conftest.py | 2 +- .../storage}/tests/test_minio.py | 10 +- .../storage}/tests/test_s3_client.py | 69 +++++--- 25 files changed, 72 insertions(+), 425 deletions(-) delete mode 100644 packages/s3wrapper/requirements/Makefile delete mode 100644 packages/s3wrapper/requirements/_base.in delete mode 100644 packages/s3wrapper/requirements/_base.txt delete mode 100644 packages/s3wrapper/requirements/_test.in delete mode 100644 packages/s3wrapper/requirements/_test.txt delete mode 100644 packages/s3wrapper/requirements/_tools.in delete mode 100644 packages/s3wrapper/requirements/_tools.txt delete mode 100644 packages/s3wrapper/requirements/dev.txt delete mode 100644 packages/s3wrapper/setup.py delete mode 100644 packages/s3wrapper/tests/__init__.py delete mode 100644 packages/s3wrapper/tests/conftest.py delete mode 100644 packages/s3wrapper/tests/docker-compose.yml rename {packages/s3wrapper/src => services/storage/src/simcore_service_storage}/s3wrapper/__init__.py (100%) rename {packages/s3wrapper/src => services/storage/src/simcore_service_storage}/s3wrapper/s3_client.py (94%) rename {packages/s3wrapper => services/storage}/tests/test_minio.py (91%) rename {packages/s3wrapper => services/storage}/tests/test_s3_client.py (87%) diff --git a/packages/s3wrapper/requirements/Makefile b/packages/s3wrapper/requirements/Makefile deleted file mode 100644 index 3f25442b790..00000000000 --- a/packages/s3wrapper/requirements/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# -# Targets to pip-compile requirements -# -include ../../../requirements/base.Makefile - -# Add here any extra explicit dependency: e.g. _migration.txt: _base.txt diff --git a/packages/s3wrapper/requirements/_base.in b/packages/s3wrapper/requirements/_base.in deleted file mode 100644 index 775bc9a2fc4..00000000000 --- a/packages/s3wrapper/requirements/_base.in +++ /dev/null @@ -1,7 +0,0 @@ -# -# Specifies third-party dependencies for 's3wrapper' -# --c ../../../requirements/constraints.txt - -urllib3 -minio diff --git a/packages/s3wrapper/requirements/_base.txt b/packages/s3wrapper/requirements/_base.txt deleted file mode 100644 index dd77bc481c0..00000000000 --- a/packages/s3wrapper/requirements/_base.txt +++ /dev/null @@ -1,25 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/_base.txt requirements/_base.in -# -certifi==2020.12.5 - # via minio -configparser==5.0.2 - # via minio -minio==6.0.2 - # via - # -c requirements/../../../requirements/constraints.txt - # -r requirements/_base.in -python-dateutil==2.8.1 - # via minio -pytz==2021.1 - # via minio -six==1.15.0 - # via python-dateutil -urllib3==1.26.4 - # via - # -c requirements/../../../requirements/constraints.txt - # -r requirements/_base.in - # minio diff --git a/packages/s3wrapper/requirements/_test.in b/packages/s3wrapper/requirements/_test.in deleted file mode 100644 index 49b4f6d0640..00000000000 --- a/packages/s3wrapper/requirements/_test.in +++ /dev/null @@ -1,23 +0,0 @@ -# -# Specifies dependencies required to run 's3wrapper' -# --c ../../../requirements/constraints.txt - -# Adds base AS CONSTRAINT specs, not requirement. -# - Resulting _text.txt is a frozen list of EXTRA packages for testing, besides _base.txt -# --c _base.txt - -# testing -coverage -pytest -pytest-cov -pytest-docker -pytest-runner - -# mockups/fixtures -requests - -# tools for CI -pylint -coveralls diff --git a/packages/s3wrapper/requirements/_test.txt b/packages/s3wrapper/requirements/_test.txt deleted file mode 100644 index 9b6150926f4..00000000000 --- a/packages/s3wrapper/requirements/_test.txt +++ /dev/null @@ -1,147 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/_test.txt requirements/_test.in -# -astroid==2.5.3 - # via pylint -attrs==20.3.0 - # via - # jsonschema - # pytest - # pytest-docker -bcrypt==3.2.0 - # via paramiko -cached-property==1.5.2 - # via docker-compose -certifi==2020.12.5 - # via - # -c requirements/_base.txt - # requests -cffi==1.14.5 - # via - # bcrypt - # cryptography - # pynacl -chardet==4.0.0 - # via requests -coverage==5.5 - # via - # -r requirements/_test.in - # coveralls - # pytest-cov -coveralls==3.0.1 - # via -r requirements/_test.in -cryptography==3.4.7 - # via - # -c requirements/../../../requirements/constraints.txt - # paramiko -distro==1.5.0 - # via docker-compose -docker-compose==1.29.1 - # via - # -c requirements/../../../requirements/constraints.txt - # pytest-docker -docker[ssh]==5.0.0 - # via docker-compose -dockerpty==0.4.1 - # via docker-compose -docopt==0.6.2 - # via - # coveralls - # docker-compose -idna==2.10 - # via - # -c requirements/../../../requirements/constraints.txt - # requests -importlib-metadata==4.0.1 - # via - # jsonschema - # pluggy - # pytest -iniconfig==1.1.1 - # via pytest -isort==5.8.0 - # via pylint -jsonschema==3.2.0 - # via docker-compose -lazy-object-proxy==1.6.0 - # via astroid -mccabe==0.6.1 - # via pylint -packaging==20.9 - # via pytest -paramiko==2.7.2 - # via docker -pluggy==0.13.1 - # via pytest -py==1.10.0 - # via pytest -pycparser==2.20 - # via cffi -pylint==2.7.4 - # via -r requirements/_test.in -pynacl==1.4.0 - # via paramiko -pyparsing==2.4.7 - # via packaging -pyrsistent==0.17.3 - # via jsonschema -pytest-cov==2.11.1 - # via -r requirements/_test.in -pytest-docker==0.10.1 - # via -r requirements/_test.in -pytest-runner==5.3.0 - # via -r requirements/_test.in -pytest==6.2.3 - # via - # -r requirements/_test.in - # pytest-cov - # pytest-docker -python-dotenv==0.17.0 - # via docker-compose -pyyaml==5.4.1 - # via - # -c requirements/../../../requirements/constraints.txt - # docker-compose -requests==2.25.1 - # via - # -r requirements/_test.in - # coveralls - # docker - # docker-compose -six==1.15.0 - # via - # -c requirements/_base.txt - # bcrypt - # dockerpty - # jsonschema - # pynacl - # websocket-client -texttable==1.6.3 - # via docker-compose -toml==0.10.2 - # via - # pylint - # pytest -typed-ast==1.4.3 - # via astroid -typing-extensions==3.7.4.3 - # via importlib-metadata -urllib3==1.26.4 - # via - # -c requirements/../../../requirements/constraints.txt - # -c requirements/_base.txt - # requests -websocket-client==0.58.0 - # via - # docker - # docker-compose -wrapt==1.12.1 - # via astroid -zipp==3.4.1 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/packages/s3wrapper/requirements/_tools.in b/packages/s3wrapper/requirements/_tools.in deleted file mode 100644 index 9d152d594d2..00000000000 --- a/packages/s3wrapper/requirements/_tools.in +++ /dev/null @@ -1,5 +0,0 @@ --c ../../../requirements/constraints.txt --c _base.txt --c _test.txt - --r ../../../requirements/devenv.txt diff --git a/packages/s3wrapper/requirements/_tools.txt b/packages/s3wrapper/requirements/_tools.txt deleted file mode 100644 index 92ee1df4663..00000000000 --- a/packages/s3wrapper/requirements/_tools.txt +++ /dev/null @@ -1,92 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/_tools.txt requirements/_tools.in -# -appdirs==1.4.4 - # via - # black - # virtualenv -black==20.8b1 - # via -r requirements/../../../requirements/devenv.txt -bump2version==1.0.1 - # via -r requirements/../../../requirements/devenv.txt -cfgv==3.2.0 - # via pre-commit -click==7.1.2 - # via - # black - # pip-tools -dataclasses==0.8 - # via black -distlib==0.3.1 - # via virtualenv -filelock==3.0.12 - # via virtualenv -identify==2.2.4 - # via pre-commit -importlib-metadata==4.0.1 - # via - # -c requirements/_test.txt - # pep517 - # pre-commit - # virtualenv -importlib-resources==5.1.2 - # via - # pre-commit - # virtualenv -isort==5.8.0 - # via - # -c requirements/_test.txt - # -r requirements/../../../requirements/devenv.txt -mypy-extensions==0.4.3 - # via black -nodeenv==1.6.0 - # via pre-commit -pathspec==0.8.1 - # via black -pep517==0.10.0 - # via pip-tools -pip-tools==6.1.0 - # via -r requirements/../../../requirements/devenv.txt -pre-commit==2.12.1 - # via -r requirements/../../../requirements/devenv.txt -pyyaml==5.4.1 - # via - # -c requirements/../../../requirements/constraints.txt - # -c requirements/_test.txt - # pre-commit -regex==2021.4.4 - # via black -six==1.15.0 - # via - # -c requirements/_base.txt - # -c requirements/_test.txt - # virtualenv -toml==0.10.2 - # via - # -c requirements/_test.txt - # black - # pep517 - # pre-commit -typed-ast==1.4.3 - # via - # -c requirements/_test.txt - # black -typing-extensions==3.7.4.3 - # via - # -c requirements/_test.txt - # black - # importlib-metadata -virtualenv==20.4.4 - # via pre-commit -zipp==3.4.1 - # via - # -c requirements/_test.txt - # importlib-metadata - # importlib-resources - # pep517 - -# The following packages are considered to be unsafe in a requirements file: -# pip diff --git a/packages/s3wrapper/requirements/dev.txt b/packages/s3wrapper/requirements/dev.txt deleted file mode 100644 index 7634b7285fd..00000000000 --- a/packages/s3wrapper/requirements/dev.txt +++ /dev/null @@ -1,14 +0,0 @@ -# Shortcut to install all packages needed to develop 's3wrapper' -# -# - As ci.txt but with current and repo packages in develop (edit) mode -# -# Usage: -# pip install -r requirements/dev.txt -# - -# installs base + tests requirements --r _base.txt --r _test.txt - -# installs current package --e . diff --git a/packages/s3wrapper/setup.py b/packages/s3wrapper/setup.py deleted file mode 100644 index 250715cb871..00000000000 --- a/packages/s3wrapper/setup.py +++ /dev/null @@ -1,33 +0,0 @@ -import re -import sys -from pathlib import Path - -from setuptools import setup - -here = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent - - -def read_reqs(reqs_path: Path): - return re.findall( - r"(^[^#\n-][\w\[,\]]+[-~>=<.\w]*)", reqs_path.read_text(), re.MULTILINE - ) - - -install_requirements = read_reqs( - here / "requirements" / "_base.in" -) # WEAK requirements -test_requirements = read_reqs( - here / "requirements" / "_test.txt" -) # STRONG requirements - -setup( - name="s3wrapper", - version="0.1.0", - package_dir={"": "src"}, - packages=["s3wrapper"], - python_requires=">=3.6", - install_requires=install_requirements, - tests_require=test_requirements, - extras_require={"test": test_requirements}, - test_suite="tests", -) diff --git a/packages/s3wrapper/tests/__init__.py b/packages/s3wrapper/tests/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/packages/s3wrapper/tests/conftest.py b/packages/s3wrapper/tests/conftest.py deleted file mode 100644 index 515c8f6e53f..00000000000 --- a/packages/s3wrapper/tests/conftest.py +++ /dev/null @@ -1,9 +0,0 @@ -import pytest -import os - -# pylint:disable=unused-argument - -@pytest.fixture(scope='session') -def docker_compose_file(pytestconfig): - my_path = os.path.join(os.path.dirname(__file__), 'docker-compose.yml') - return my_path diff --git a/packages/s3wrapper/tests/docker-compose.yml b/packages/s3wrapper/tests/docker-compose.yml deleted file mode 100644 index d0a46aaa572..00000000000 --- a/packages/s3wrapper/tests/docker-compose.yml +++ /dev/null @@ -1,10 +0,0 @@ -version: "3.8" -services: - minio: - image: minio/minio - environment: - - MINIO_ACCESS_KEY=12345678 - - MINIO_SECRET_KEY=12345678 - ports: - - "9001:9000" - command: server /data diff --git a/services/storage/requirements/_base.in b/services/storage/requirements/_base.in index e77a92e93b5..6f67515bbd5 100644 --- a/services/storage/requirements/_base.in +++ b/services/storage/requirements/_base.in @@ -1,6 +1,5 @@ -c ../../../requirements/constraints.txt --r ../../../packages/s3wrapper/requirements/_base.in -r ../../../packages/postgres-database/requirements/_base.in -r ../../../packages/service-library/requirements/_base.in -r ../../../packages/models-library/requirements/_base.in @@ -21,3 +20,5 @@ aiopg[sa] tenacity semantic_version click +minio +urllib3 diff --git a/services/storage/requirements/_base.txt b/services/storage/requirements/_base.txt index 9a876e2bf90..e91eb20c46e 100644 --- a/services/storage/requirements/_base.txt +++ b/services/storage/requirements/_base.txt @@ -29,7 +29,6 @@ aiozipkin==0.7.1 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in @@ -119,11 +118,10 @@ minio==6.0.0 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt - # -r requirements/../../../packages/s3wrapper/requirements/_base.in -multidict==5.0.0 + # -r requirements/_base.in +multidict==5.1.0 # via # aiohttp # yarl @@ -161,7 +159,6 @@ pyyaml==5.3.1 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in @@ -189,7 +186,6 @@ sqlalchemy[postgresql_psycopg2binary]==1.3.20 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_base.in @@ -216,10 +212,9 @@ urllib3==1.25.11 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt - # -r requirements/../../../packages/s3wrapper/requirements/_base.in + # -r requirements/_base.in # botocore # minio # requests diff --git a/services/storage/requirements/ci.txt b/services/storage/requirements/ci.txt index 822d88ed2a2..d04970edace 100644 --- a/services/storage/requirements/ci.txt +++ b/services/storage/requirements/ci.txt @@ -11,7 +11,6 @@ -r _test.txt # installs this repo's packages -../../packages/s3wrapper/ ../../packages/postgres-database/ ../../packages/service-library/ ../../packages/models-library/ diff --git a/services/storage/requirements/dev.txt b/services/storage/requirements/dev.txt index d2a827ff30c..b2e5c4ac9fe 100644 --- a/services/storage/requirements/dev.txt +++ b/services/storage/requirements/dev.txt @@ -12,7 +12,6 @@ -r _tools.txt # installs this repo's packages --e ../../packages/s3wrapper/ -e ../../packages/postgres-database/ -e ../../packages/service-library -e ../../packages/models-library diff --git a/services/storage/requirements/prod.txt b/services/storage/requirements/prod.txt index 2de63f3f375..ff28edf7af0 100644 --- a/services/storage/requirements/prod.txt +++ b/services/storage/requirements/prod.txt @@ -10,7 +10,6 @@ -r _base.txt # installs this repo's packages -../../packages/s3wrapper/ ../../packages/postgres-database/ ../../packages/models-library/ ../../packages/service-library diff --git a/services/storage/setup.py b/services/storage/setup.py index 881f8c3a553..0e2cf63657f 100644 --- a/services/storage/setup.py +++ b/services/storage/setup.py @@ -20,7 +20,6 @@ def read_reqs(reqs_path: Path): install_requirements = read_reqs(here / "requirements" / "_base.txt") + [ - "s3wrapper==0.1.0", "simcore-postgres-database", "simcore-service-library", "simcore-models-library", diff --git a/services/storage/src/simcore_service_storage/dsm.py b/services/storage/src/simcore_service_storage/dsm.py index edaedfac605..9ca77ad77b1 100644 --- a/services/storage/src/simcore_service_storage/dsm.py +++ b/services/storage/src/simcore_service_storage/dsm.py @@ -20,7 +20,6 @@ from aiopg.sa import Engine from aiopg.sa.result import RowProxy from blackfynn.base import UnauthorizedException -from s3wrapper.s3_client import S3Client from servicelib.aiopg_utils import DBAPIError, PostgresRetryPolicyUponOperation from servicelib.client_session import get_client_session from servicelib.utils import fire_and_forget_task @@ -44,6 +43,7 @@ projects, ) from .s3 import get_config_s3 +from .s3wrapper.s3_client import S3Client from .settings import ( APP_CONFIG_KEY, APP_DB_ENGINE_KEY, diff --git a/services/storage/src/simcore_service_storage/s3.py b/services/storage/src/simcore_service_storage/s3.py index f54bc7d5a81..9875f58ded6 100644 --- a/services/storage/src/simcore_service_storage/s3.py +++ b/services/storage/src/simcore_service_storage/s3.py @@ -6,9 +6,9 @@ from typing import Dict from aiohttp import web -from s3wrapper.s3_client import S3Client from tenacity import before_sleep_log, retry, stop_after_attempt, wait_fixed +from .s3wrapper.s3_client import S3Client from .settings import APP_CONFIG_KEY, APP_S3_KEY from .utils import RETRY_COUNT, RETRY_WAIT_SECS diff --git a/packages/s3wrapper/src/s3wrapper/__init__.py b/services/storage/src/simcore_service_storage/s3wrapper/__init__.py similarity index 100% rename from packages/s3wrapper/src/s3wrapper/__init__.py rename to services/storage/src/simcore_service_storage/s3wrapper/__init__.py diff --git a/packages/s3wrapper/src/s3wrapper/s3_client.py b/services/storage/src/simcore_service_storage/s3wrapper/s3_client.py similarity index 94% rename from packages/s3wrapper/src/s3wrapper/s3_client.py rename to services/storage/src/simcore_service_storage/s3wrapper/s3_client.py index 7f483e92533..41b3b1521ec 100644 --- a/packages/s3wrapper/src/s3wrapper/s3_client.py +++ b/services/storage/src/simcore_service_storage/s3wrapper/s3_client.py @@ -7,9 +7,9 @@ log = logging.getLogger(__name__) + class S3Client: - """ Wrapper around minio - """ + """Wrapper around minio""" def __init__( self, @@ -81,16 +81,16 @@ def list_buckets(self): return [] def upload_file(self, bucket_name, object_name, filepath, metadata=None): - """ Note + """Note - metadata are special, you need to use the - 'X-Amz-Meta' standard, i.e: - - key and value must be strings - - and the keys are case insensitive: + metadata are special, you need to use the + 'X-Amz-Meta' standard, i.e: + - key and value must be strings + - and the keys are case insensitive: - key1 -- > Key1 - key_one --> Key_one - key-one --> Key-One + key1 -- > Key1 + key_one --> Key_one + key-one --> Key-One """ try: @@ -158,8 +158,7 @@ def remove_objects(self, bucket_name, objects): return True def exists_object(self, bucket_name, object_name, recursive=False): - """ This seems to be pretty heavy, should be used with care - """ + """This seems to be pretty heavy, should be used with care""" try: objects = self.list_objects(bucket_name, recursive=recursive) for obj in objects: diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 4e34b618ec7..988de856797 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -25,6 +25,7 @@ from simcore_service_storage.datcore_wrapper import DatcoreWrapper from simcore_service_storage.dsm import DataStorageManager, DatCoreApiToken from simcore_service_storage.models import FileMetaData +from simcore_service_storage.s3wrapper.s3_client import S3Client from simcore_service_storage.settings import SIMCORE_S3_STR from tests.utils import ( ACCESS_KEY, @@ -213,7 +214,6 @@ def minio_service(docker_services, docker_ip): @pytest.fixture(scope="module") def s3_client(minio_service): - from s3wrapper.s3_client import S3Client s3_client = S3Client( endpoint=minio_service["endpoint"], diff --git a/packages/s3wrapper/tests/test_minio.py b/services/storage/tests/test_minio.py similarity index 91% rename from packages/s3wrapper/tests/test_minio.py rename to services/storage/tests/test_minio.py index 8387c571896..9f33e14b467 100644 --- a/packages/s3wrapper/tests/test_minio.py +++ b/services/storage/tests/test_minio.py @@ -1,11 +1,12 @@ -import requests import pytest +import requests # pylint:disable=unused-import from pytest_docker import docker_ip, docker_services # pylint:disable=redefined-outer-name + def is_responsive(url, code=200): """Check if something responds to ``url``.""" try: @@ -14,17 +15,18 @@ def is_responsive(url, code=200): return True except requests.exceptions.RequestException as _e: pass - + return False + @pytest.mark.enable_travis def test_minio(docker_ip, docker_services): """wait for minio to be up""" # Build URL to service listening on random port. - url = 'http://%s:%d/' % ( + url = "http://%s:%d/" % ( docker_ip, - docker_services.port_for('minio', 9000), + docker_services.port_for("minio", 9000), ) # Wait until service is responsive. docker_services.wait_until_responsive( diff --git a/packages/s3wrapper/tests/test_s3_client.py b/services/storage/tests/test_s3_client.py similarity index 87% rename from packages/s3wrapper/tests/test_s3_client.py rename to services/storage/tests/test_s3_client.py index 5c799bb626c..2a9aafcbbda 100644 --- a/packages/s3wrapper/tests/test_s3_client.py +++ b/services/storage/tests/test_s3_client.py @@ -1,22 +1,20 @@ import filecmp import os -import uuid - -import pytest -import urllib import time +import urllib +import uuid from datetime import timedelta -from s3wrapper.s3_client import S3Client - +import pytest import requests # pylint:disable=unused-import from pytest_docker import docker_ip, docker_services - +from simcore_service_storage.s3wrapper.s3_client import S3Client # pylint:disable=redefined-outer-name + def is_responsive(url, code=200): """Check if something responds to ``url``.""" try: @@ -27,14 +25,15 @@ def is_responsive(url, code=200): pass return False + @pytest.fixture(scope="module") def s3_client(docker_ip, docker_services): """wait for minio to be up""" # Build URL to service listening on random port. - url = 'http://%s:%d/' % ( + url = "http://%s:%d/" % ( docker_ip, - docker_services.port_for('minio', 9000), + docker_services.port_for("minio", 9000), ) # Wait until service is responsive. @@ -48,36 +47,46 @@ def s3_client(docker_ip, docker_services): response = requests.get(url) assert response.status_code == 403 - endpoint = '{ip}:{port}'.format(ip=docker_ip, port=docker_services.port_for('minio', 9000)) + endpoint = "{ip}:{port}".format( + ip=docker_ip, port=docker_services.port_for("minio", 9000) + ) access_key = "12345678" secret_key = "12345678" secure = False s3_client = S3Client(endpoint, access_key, secret_key, secure) return s3_client + @pytest.fixture() def bucket(s3_client, request): bucket_name = "simcore-test" s3_client.create_bucket(bucket_name, delete_contents_if_exists=True) + def fin(): s3_client.remove_bucket(bucket_name, delete_contents=True) + request.addfinalizer(fin) return bucket_name + @pytest.fixture(scope="function") def text_files(tmpdir_factory): def _create_files(N): filepaths = [] for _i in range(N): name = str(uuid.uuid4()) - filepath = os.path.normpath(str(tmpdir_factory.mktemp('data').join(name + ".txt"))) - with open(filepath, 'w') as fout: + filepath = os.path.normpath( + str(tmpdir_factory.mktemp("data").join(name + ".txt")) + ) + with open(filepath, "w") as fout: fout.write("Hello world\n") filepaths.append(filepath) return filepaths + return _create_files + @pytest.mark.enable_travis def test_create_remove_bucket(s3_client): bucket_name = "simcore-test" @@ -86,6 +95,7 @@ def test_create_remove_bucket(s3_client): s3_client.remove_bucket(bucket_name, delete_contents=True) assert not s3_client.exists_bucket(bucket_name) + @pytest.mark.enable_travis def test_create_remove_bucket_with_contents(s3_client, text_files): bucket_name = "simcore-test" @@ -99,6 +109,7 @@ def test_create_remove_bucket_with_contents(s3_client, text_files): s3_client.remove_bucket(bucket_name, delete_contents=True) assert not s3_client.exists_bucket(bucket_name) + @pytest.mark.enable_travis def test_file_upload_download(s3_client, bucket, text_files): filepath = text_files(1)[0] @@ -108,21 +119,23 @@ def test_file_upload_download(s3_client, bucket, text_files): assert s3_client.download_file(bucket, object_name, filepath2) assert filecmp.cmp(filepath2, filepath) + @pytest.mark.enable_travis def test_file_upload_meta_data(s3_client, bucket, text_files): filepath = text_files(1)[0] object_name = "1" _id = uuid.uuid4() - metadata = {'user' : 'guidon', 'node_id' : str(_id), 'boom-boom' : str(42.0)} + metadata = {"user": "guidon", "node_id": str(_id), "boom-boom": str(42.0)} assert s3_client.upload_file(bucket, object_name, filepath, metadata=metadata) metadata2 = s3_client.get_metadata(bucket, object_name) - assert metadata2["User"] == 'guidon' + assert metadata2["User"] == "guidon" assert metadata2["Node_id"] == str(_id) assert metadata2["Boom-Boom"] == str(42.0) + @pytest.mark.enable_travis def test_sub_folders(s3_client, bucket, text_files): bucket_sub_folder = str(uuid.uuid4()) @@ -133,18 +146,21 @@ def test_sub_folders(s3_client, bucket, text_files): assert s3_client.upload_file(bucket, object_name, f) counter += 1 + @pytest.mark.enable_travis def test_search(s3_client, bucket, text_files): - metadata = [{'User' : 'alpha'}, {'User' : 'beta' }, {'User' : 'gamma'}] + metadata = [{"User": "alpha"}, {"User": "beta"}, {"User": "gamma"}] for i in range(3): - bucket_sub_folder = "Folder"+ str(i+1) + bucket_sub_folder = "Folder" + str(i + 1) filepaths = text_files(3) counter = 0 for f in filepaths: object_name = bucket_sub_folder + "/" + "Data" + str(counter) - assert s3_client.upload_file(bucket, object_name, f, metadata=metadata[counter]) + assert s3_client.upload_file( + bucket, object_name, f, metadata=metadata[counter] + ) counter += 1 query = "DATA1" @@ -162,14 +178,15 @@ def test_search(s3_client, bucket, text_files): results = s3_client.search(bucket, query, recursive=True, include_metadata=False) assert len(results) == 9 + @pytest.mark.enable_travis def test_presigned_put(s3_client, bucket, text_files): filepath = text_files(1)[0] object_name = "my_file" url = s3_client.create_presigned_put_url(bucket, object_name) - with open(filepath, 'rb') as fp: + with open(filepath, "rb") as fp: d = fp.read() - req = urllib.request.Request(url, data=d, method='PUT') + req = urllib.request.Request(url, data=d, method="PUT") with urllib.request.urlopen(req) as _f: pass @@ -177,6 +194,7 @@ def test_presigned_put(s3_client, bucket, text_files): assert s3_client.download_file(bucket, object_name, filepath2) assert filecmp.cmp(filepath2, filepath) + @pytest.mark.enable_travis def test_presigned_put_expired(s3_client, bucket, text_files): filepath = text_files(1)[0] @@ -184,9 +202,9 @@ def test_presigned_put_expired(s3_client, bucket, text_files): url = s3_client.create_presigned_put_url(bucket, object_name, timedelta(seconds=1)) time.sleep(2) failed = False - with open(filepath, 'rb') as fp: + with open(filepath, "rb") as fp: d = fp.read() - req = urllib.request.Request(url, data=d, method='PUT') + req = urllib.request.Request(url, data=d, method="PUT") try: urllib.request.urlopen(req) except urllib.error.HTTPError as _ex: @@ -205,6 +223,7 @@ def test_presigned_get(s3_client, bucket, text_files): assert filecmp.cmp(filepath2, filepath) + @pytest.mark.enable_travis def test_presigned_get_expired(s3_client, bucket, text_files): filepath = text_files(1)[0] @@ -221,6 +240,7 @@ def test_presigned_get_expired(s3_client, bucket, text_files): assert failed + @pytest.mark.enable_travis def test_object_exists(s3_client, bucket, text_files): files = text_files(2) @@ -234,6 +254,7 @@ def test_object_exists(s3_client, bucket, text_files): assert not s3_client.exists_object(bucket, object_name, False) assert s3_client.exists_object(bucket, object_name, True) + @pytest.mark.enable_travis def test_copy_object(s3_client, bucket, text_files): files = text_files(1) @@ -245,6 +266,7 @@ def test_copy_object(s3_client, bucket, text_files): assert s3_client.copy_object(bucket, copied_object, bucket + "/" + object_name) assert s3_client.exists_object(bucket, copied_object, False) + def test_list_objects(s3_client, bucket, text_files): files = text_files(2) file1 = files[0] @@ -268,4 +290,7 @@ def test_list_objects(s3_client, bucket, text_files): listed_objects = s3_client.list_objects(bucket, recursive=True) for s3_obj in listed_objects: - assert s3_obj.object_name == "level1/level2/1" or s3_obj.object_name == "level2/level2/2" + assert ( + s3_obj.object_name == "level1/level2/1" + or s3_obj.object_name == "level2/level2/2" + ) From dcf49fe301776a74fb2af96eb8efb3137c95e4e1 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 26 Apr 2021 11:57:05 +0200 Subject: [PATCH 02/27] removes s3wrapper dependency from simcore-skd --- packages/simcore-sdk/requirements/_base.in | 1 - packages/simcore-sdk/requirements/_base.txt | 29 --------------------- packages/simcore-sdk/requirements/_test.txt | 9 ++----- packages/simcore-sdk/requirements/ci.txt | 1 - packages/simcore-sdk/requirements/dev.txt | 1 - packages/simcore-sdk/setup.py | 1 - 6 files changed, 2 insertions(+), 40 deletions(-) diff --git a/packages/simcore-sdk/requirements/_base.in b/packages/simcore-sdk/requirements/_base.in index a2322eefab0..9bf85f5bde5 100644 --- a/packages/simcore-sdk/requirements/_base.in +++ b/packages/simcore-sdk/requirements/_base.in @@ -3,7 +3,6 @@ # -c ../../../requirements/constraints.txt -r ../../../packages/postgres-database/requirements/_base.in --r ../../../packages/s3wrapper/requirements/_base.in -r ../../../packages/service-library/requirements/_base.in -r ../../../packages/models-library/requirements/_base.in diff --git a/packages/simcore-sdk/requirements/_base.txt b/packages/simcore-sdk/requirements/_base.txt index 57e231639a6..c3297750170 100644 --- a/packages/simcore-sdk/requirements/_base.txt +++ b/packages/simcore-sdk/requirements/_base.txt @@ -21,7 +21,6 @@ aiozipkin==0.7.1 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in @@ -36,12 +35,8 @@ attrs==20.3.0 # aiohttp # jsonschema # openapi-core -certifi==2020.12.5 - # via minio chardet==4.0.0 # via aiohttp -configparser==5.0.2 - # via minio dataclasses==0.8 # via pydantic decorator==4.4.2 @@ -75,14 +70,6 @@ lazy-object-proxy==1.4.3 # via # -r requirements/../../../packages/service-library/requirements/_base.in # openapi-core -minio==6.0.2 - # via - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt - # -r requirements/../../../packages/s3wrapper/requirements/_base.in multidict==5.1.0 # via # aiohttp @@ -114,15 +101,10 @@ pyparsing==2.4.7 # via packaging pyrsistent==0.17.3 # via jsonschema -python-dateutil==2.8.1 - # via minio -pytz==2021.1 - # via minio pyyaml==5.4.1 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in @@ -135,13 +117,11 @@ six==1.15.0 # openapi-core # openapi-schema-validator # openapi-spec-validator - # python-dateutil # tenacity sqlalchemy[postgresql_psycopg2binary]==1.3.24 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_base.in @@ -169,15 +149,6 @@ typing-extensions==3.7.4.3 # yarl ujson==4.0.2 # via -r requirements/../../../packages/service-library/requirements/_base.in -urllib3==1.26.4 - # via - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt - # -r requirements/../../../packages/s3wrapper/requirements/_base.in - # minio werkzeug==1.0.1 # via -r requirements/../../../packages/service-library/requirements/_base.in yarl==1.6.3 diff --git a/packages/simcore-sdk/requirements/_test.txt b/packages/simcore-sdk/requirements/_test.txt index 377b85b8578..ff4faf408e8 100644 --- a/packages/simcore-sdk/requirements/_test.txt +++ b/packages/simcore-sdk/requirements/_test.txt @@ -27,9 +27,7 @@ attrs==20.3.0 # aiohttp # pytest certifi==2020.12.5 - # via - # -c requirements/_base.txt - # requests + # via requests chardet==4.0.0 # via # -c requirements/_base.txt @@ -140,7 +138,6 @@ pytest==6.2.3 # pytest-xdist python-dateutil==2.8.1 # via - # -c requirements/_base.txt # alembic # faker python-dotenv==0.17.0 @@ -178,9 +175,7 @@ typing-extensions==3.7.4.3 # importlib-metadata # yarl urllib3==1.26.4 - # via - # -c requirements/_base.txt - # requests + # via requests websocket-client==0.58.0 # via docker wrapt==1.12.1 diff --git a/packages/simcore-sdk/requirements/ci.txt b/packages/simcore-sdk/requirements/ci.txt index c8ebd862753..29a00179eae 100644 --- a/packages/simcore-sdk/requirements/ci.txt +++ b/packages/simcore-sdk/requirements/ci.txt @@ -12,7 +12,6 @@ # installs this repo's packages ../postgres-database/[migration] -../s3wrapper/ ../service-library/ ../pytest-simcore/ ../models-library/ diff --git a/packages/simcore-sdk/requirements/dev.txt b/packages/simcore-sdk/requirements/dev.txt index 372192e1269..f31724f61f7 100644 --- a/packages/simcore-sdk/requirements/dev.txt +++ b/packages/simcore-sdk/requirements/dev.txt @@ -13,7 +13,6 @@ # installs this repo's packages -e ../postgres-database/[migration] --e ../s3wrapper/ -e ../service-library/ -e ../pytest-simcore/ -e ../models-library/ diff --git a/packages/simcore-sdk/setup.py b/packages/simcore-sdk/setup.py index 63b6e846f14..92589626549 100644 --- a/packages/simcore-sdk/setup.py +++ b/packages/simcore-sdk/setup.py @@ -18,7 +18,6 @@ def read_reqs(reqs_path: Path): "simcore-postgres-database[migration]", "simcore-service-library", "simcore-models-library", - "s3wrapper", "simcore-service-storage-sdk", ] From da97eb3d3e7ffec6e1fd4167aa1f53bee6e094ff Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 26 Apr 2021 12:27:21 +0200 Subject: [PATCH 03/27] removes s3wrapper dependency from web/server --- services/web/server/requirements/ci.txt | 1 - services/web/server/requirements/dev.txt | 1 - services/web/server/requirements/prod.txt | 1 - services/web/server/setup.py | 1 - 4 files changed, 4 deletions(-) diff --git a/services/web/server/requirements/ci.txt b/services/web/server/requirements/ci.txt index 7ba1907d1e0..88e0bb3b1b4 100644 --- a/services/web/server/requirements/ci.txt +++ b/services/web/server/requirements/ci.txt @@ -11,7 +11,6 @@ -r _test.txt # installs this repo's packages -../../../packages/s3wrapper/ ../../../packages/models-library/ ../../../packages/postgres-database/ ../../../packages/simcore-sdk/ diff --git a/services/web/server/requirements/dev.txt b/services/web/server/requirements/dev.txt index 55bf91d539c..8e19896a1ad 100644 --- a/services/web/server/requirements/dev.txt +++ b/services/web/server/requirements/dev.txt @@ -12,7 +12,6 @@ -r _tools.txt # installs this repo's packages --e ../../../packages/s3wrapper/ -e ../../../packages/models-library/ -e ../../../packages/postgres-database/ -e ../../../packages/simcore-sdk/ diff --git a/services/web/server/requirements/prod.txt b/services/web/server/requirements/prod.txt index e66a0feff0a..7e0c40af111 100644 --- a/services/web/server/requirements/prod.txt +++ b/services/web/server/requirements/prod.txt @@ -10,7 +10,6 @@ -r _base.txt # installs this repo's packages -../../../packages/s3wrapper/ ../../../packages/models-library/ ../../../packages/postgres-database/ ../../../packages/simcore-sdk/ diff --git a/services/web/server/setup.py b/services/web/server/setup.py index fb910bba526..1096d8d421b 100644 --- a/services/web/server/setup.py +++ b/services/web/server/setup.py @@ -16,7 +16,6 @@ def read_reqs(reqs_path: Path): # ----------------------------------------------------------------- # Hard requirements on third-parties and latest for in-repo packages install_requirements = read_reqs(current_dir / "requirements" / "_base.txt") + [ - "s3wrapper", "simcore-models-library", "simcore-postgres-database", "simcore-sdk", From 2ae297e6a77818e2b16f8c0e7e143a34b63d8cdb Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 26 Apr 2021 12:29:58 +0200 Subject: [PATCH 04/27] removes s3wrapper dependency from swarm-deploy --- tests/swarm-deploy/requirements/_test.in | 1 - tests/swarm-deploy/requirements/_test.txt | 30 ----------------------- tests/swarm-deploy/requirements/ci.txt | 1 - tests/swarm-deploy/requirements/dev.txt | 1 - 4 files changed, 33 deletions(-) diff --git a/tests/swarm-deploy/requirements/_test.in b/tests/swarm-deploy/requirements/_test.in index b345d404a0d..99490bb3171 100644 --- a/tests/swarm-deploy/requirements/_test.in +++ b/tests/swarm-deploy/requirements/_test.in @@ -1,6 +1,5 @@ -c ../../../requirements/constraints.txt --r ../../../packages/s3wrapper/requirements/_base.in -r ../../../packages/postgres-database/requirements/_migration.txt -r ../../../packages/simcore-sdk/requirements/_base.in -r ../../../packages/service-library/requirements/_base.in diff --git a/tests/swarm-deploy/requirements/_test.txt b/tests/swarm-deploy/requirements/_test.txt index 126f1f44c82..8f81265ca97 100644 --- a/tests/swarm-deploy/requirements/_test.txt +++ b/tests/swarm-deploy/requirements/_test.txt @@ -29,11 +29,9 @@ aiormq==3.3.1 aiozipkin==0.7.1 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -59,7 +57,6 @@ attrs==20.3.0 certifi==2020.12.5 # via # -r requirements/../../../packages/postgres-database/requirements/_migration.txt - # minio # requests chardet==4.0.0 # via @@ -70,8 +67,6 @@ click==7.1.2 # via # -r requirements/../../../packages/postgres-database/requirements/_migration.txt # -r requirements/_test.in -configparser==5.0.2 - # via minio coverage==5.5 # via # -r requirements/_test.in @@ -132,19 +127,6 @@ markupsafe==1.1.1 # via # -r requirements/../../../packages/postgres-database/requirements/_migration.txt # mako -minio==6.0.2 - # via - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt - # -r requirements/../../../packages/s3wrapper/requirements/_base.in - # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/_base.in multidict==5.1.0 # via # aiohttp @@ -217,23 +199,18 @@ python-dateutil==2.8.1 # via # -r requirements/../../../packages/postgres-database/requirements/_migration.txt # alembic - # minio python-dotenv==0.17.0 # via -r requirements/_test.in python-editor==1.0.4 # via # -r requirements/../../../packages/postgres-database/requirements/_migration.txt # alembic -pytz==2021.1 - # via minio pyyaml==5.4.1 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -260,11 +237,9 @@ six==1.15.0 sqlalchemy[postgresql_psycopg2binary]==1.3.24 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -309,18 +284,13 @@ ujson==4.0.2 urllib3==1.26.4 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_migration.txt - # -r requirements/../../../packages/s3wrapper/requirements/_base.in - # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/_base.in - # minio # requests websocket-client==0.58.0 # via diff --git a/tests/swarm-deploy/requirements/ci.txt b/tests/swarm-deploy/requirements/ci.txt index e6bfcf36193..7dd9c73ea46 100644 --- a/tests/swarm-deploy/requirements/ci.txt +++ b/tests/swarm-deploy/requirements/ci.txt @@ -10,7 +10,6 @@ -r _test.txt # installs this repo's packages -../../packages/s3wrapper/ ../../packages/postgres-database/[migration] ../../packages/simcore-sdk/ ../../packages/service-library/ diff --git a/tests/swarm-deploy/requirements/dev.txt b/tests/swarm-deploy/requirements/dev.txt index f438fa50b51..d2181d9183b 100644 --- a/tests/swarm-deploy/requirements/dev.txt +++ b/tests/swarm-deploy/requirements/dev.txt @@ -12,7 +12,6 @@ -r tools.txt # installs this repo's packages --e ../../packages/s3wrapper/ -e ../../packages/postgres-database/ -e ../../packages/simcore-sdk/ -e ../../packages/service-library/ From cd87470ab7ece873c4f390016cc0897fa6a8a267 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 26 Apr 2021 12:34:50 +0200 Subject: [PATCH 05/27] removes s3wrapper dependency from sidecar --- services/sidecar/requirements/_base.in | 1 - services/sidecar/requirements/_base.txt | 3 -- services/sidecar/requirements/_packages.in | 1 - services/sidecar/requirements/_packages.txt | 32 --------------------- services/sidecar/requirements/_test.txt | 6 +--- services/sidecar/requirements/ci.txt | 1 - services/sidecar/requirements/dev.txt | 1 - services/sidecar/requirements/prod.txt | 1 - services/sidecar/setup.py | 1 - 9 files changed, 1 insertion(+), 46 deletions(-) diff --git a/services/sidecar/requirements/_base.in b/services/sidecar/requirements/_base.in index fab6e675f5f..4b9dbd6ab45 100644 --- a/services/sidecar/requirements/_base.in +++ b/services/sidecar/requirements/_base.in @@ -11,7 +11,6 @@ # -c ../../../packages/models-library/requirements/_base.in -c ../../../packages/postgres-database/requirements/_base.in --c ../../../packages/s3wrapper/requirements/_base.in -c ../../../packages/service-library/requirements/_base.in diff --git a/services/sidecar/requirements/_base.txt b/services/sidecar/requirements/_base.txt index b604a20eaa1..366f3c14682 100644 --- a/services/sidecar/requirements/_base.txt +++ b/services/sidecar/requirements/_base.txt @@ -107,7 +107,6 @@ sqlalchemy[postgresql_psycopg2binary]==1.3.19 # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/_base.in - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/_base.in # -c requirements/../../../requirements/constraints.txt @@ -126,8 +125,6 @@ urllib3==1.25.11 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/_base.in # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/_base.in diff --git a/services/sidecar/requirements/_packages.in b/services/sidecar/requirements/_packages.in index 266b3e9c285..1cb39faa9b5 100644 --- a/services/sidecar/requirements/_packages.in +++ b/services/sidecar/requirements/_packages.in @@ -6,5 +6,4 @@ -r ../../../packages/models-library/requirements/_base.in -r ../../../packages/postgres-database/requirements/_base.in --r ../../../packages/s3wrapper/requirements/_base.in -r ../../../packages/service-library/requirements/_base.in diff --git a/services/sidecar/requirements/_packages.txt b/services/sidecar/requirements/_packages.txt index 2effc8506df..98b02e0769b 100644 --- a/services/sidecar/requirements/_packages.txt +++ b/services/sidecar/requirements/_packages.txt @@ -19,7 +19,6 @@ aiozipkin==0.7.1 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in @@ -35,14 +34,10 @@ attrs==20.2.0 # aiohttp # jsonschema # openapi-core -certifi==2020.6.20 - # via minio chardet==3.0.4 # via # -c requirements/_base.txt # aiohttp -configparser==5.0.0 - # via minio dataclasses==0.7 # via # -c requirements/_base.txt @@ -82,14 +77,6 @@ lazy-object-proxy==1.4.3 # via # -r requirements/../../../packages/service-library/requirements/_base.in # openapi-core -minio==6.0.0 - # via - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt - # -r requirements/../../../packages/s3wrapper/requirements/_base.in multidict==4.7.6 # via # -c requirements/_base.txt @@ -114,17 +101,10 @@ pydantic[email]==1.7.2 # -r requirements/../../../packages/service-library/requirements/_base.in pyrsistent==0.17.3 # via jsonschema -python-dateutil==2.8.1 - # via minio -pytz==2020.1 - # via - # -c requirements/_base.txt - # minio pyyaml==5.3.1 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in @@ -136,13 +116,11 @@ six==1.15.0 # jsonschema # openapi-core # openapi-spec-validator - # python-dateutil # tenacity sqlalchemy[postgresql_psycopg2binary]==1.3.19 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -164,16 +142,6 @@ typing-extensions==3.7.4.3 # yarl ujson==3.2.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -urllib3==1.25.11 - # via - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt - # -c requirements/_base.txt - # -r requirements/../../../packages/s3wrapper/requirements/_base.in - # minio werkzeug==1.0.1 # via -r requirements/../../../packages/service-library/requirements/_base.in yarl==1.5.1 diff --git a/services/sidecar/requirements/_test.txt b/services/sidecar/requirements/_test.txt index 0f13db325a1..f71b9a6c954 100644 --- a/services/sidecar/requirements/_test.txt +++ b/services/sidecar/requirements/_test.txt @@ -31,9 +31,7 @@ attrs==20.2.0 # aiohttp # pytest certifi==2020.6.20 - # via - # -c requirements/_packages.txt - # requests + # via requests chardet==3.0.4 # via # -c requirements/_base.txt @@ -144,7 +142,6 @@ pytest==6.2.3 # pytest-sugar python-dateutil==2.8.1 # via - # -c requirements/_packages.txt # alembic # faker python-dotenv==0.17.0 @@ -189,7 +186,6 @@ urllib3==1.25.11 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt - # -c requirements/_packages.txt # requests websocket-client==0.58.0 # via docker diff --git a/services/sidecar/requirements/ci.txt b/services/sidecar/requirements/ci.txt index a9272de2df8..f46cf54bc34 100644 --- a/services/sidecar/requirements/ci.txt +++ b/services/sidecar/requirements/ci.txt @@ -15,7 +15,6 @@ ../../packages/models-library/ ../../packages/postgres-database/ ../../packages/pytest-simcore/ -../../packages/s3wrapper/ ../../packages/service-library/ ../../packages/simcore-sdk/ ../../services/storage/client-sdk/python/ diff --git a/services/sidecar/requirements/dev.txt b/services/sidecar/requirements/dev.txt index d904068128c..5b3c7992244 100644 --- a/services/sidecar/requirements/dev.txt +++ b/services/sidecar/requirements/dev.txt @@ -16,7 +16,6 @@ -e ../../packages/models-library/ -e ../../packages/postgres-database/ -e ../../packages/pytest-simcore/ --e ../../packages/s3wrapper/ -e ../../packages/service-library/ -e ../../packages/simcore-sdk/ -e ../../services/storage/client-sdk/python/ diff --git a/services/sidecar/requirements/prod.txt b/services/sidecar/requirements/prod.txt index a239271cfbf..1a3909a3782 100644 --- a/services/sidecar/requirements/prod.txt +++ b/services/sidecar/requirements/prod.txt @@ -13,7 +13,6 @@ # installs this repo's packages ../../packages/models-library/ ../../packages/postgres-database/ -../../packages/s3wrapper/ ../../packages/service-library/ ../../packages/simcore-sdk/ ../../services/storage/client-sdk/python/ diff --git a/services/sidecar/setup.py b/services/sidecar/setup.py index 4bdab22da30..ce57988a1c7 100644 --- a/services/sidecar/setup.py +++ b/services/sidecar/setup.py @@ -23,7 +23,6 @@ def read_reqs(reqs_path: Path): version = (current_dir / "VERSION").read_text().strip() install_requirements = read_reqs(current_dir / "requirements" / "_base.txt") + [ - "s3wrapper==0.1.0", "simcore-postgres-database", "simcore-sdk", "simcore-service-library", From fced709a6d91a4e2f3fb9ca96ae51dc98f1c6466 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 26 Apr 2021 12:37:57 +0200 Subject: [PATCH 06/27] removes s3wrapper dependency from api-server --- services/api-server/requirements/_base.txt | 29 ---------------------- services/api-server/requirements/_test.txt | 1 - 2 files changed, 30 deletions(-) diff --git a/services/api-server/requirements/_base.txt b/services/api-server/requirements/_base.txt index 37cf9a421d6..6819021fb41 100644 --- a/services/api-server/requirements/_base.txt +++ b/services/api-server/requirements/_base.txt @@ -31,7 +31,6 @@ aiozipkin==0.7.1 # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -63,7 +62,6 @@ attrs==20.3.0 certifi==2020.12.5 # via # httpx - # minio # requests cffi==1.14.4 # via cryptography @@ -74,8 +72,6 @@ chardet==3.0.4 # requests click==7.1.2 # via uvicorn -configparser==5.0.2 - # via minio contextvars==2.4 # via sniffio cryptography==3.4.5 @@ -85,7 +81,6 @@ cryptography==3.4.5 # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -125,7 +120,6 @@ httpx==0.14.3 # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -172,18 +166,6 @@ lazy-object-proxy==1.4.3 # openapi-core markupsafe==1.1.1 # via jinja2 -minio==6.0.2 - # via - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt - # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/_base.in multidict==5.1.0 # via # aiohttp @@ -234,16 +216,12 @@ pyparsing==2.4.7 # via packaging pyrsistent==0.17.3 # via jsonschema -python-dateutil==2.8.1 - # via minio python-dotenv==0.15.0 # via # pydantic # uvicorn python-multipart==0.0.5 # via fastapi -pytz==2021.1 - # via minio pyyaml==5.4.1 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -251,7 +229,6 @@ pyyaml==5.4.1 # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -277,7 +254,6 @@ six==1.15.0 # openapi-core # openapi-schema-validator # openapi-spec-validator - # python-dateutil # python-multipart # tenacity sniffio==1.2.0 @@ -291,7 +267,6 @@ sqlalchemy[postgresql_psycopg2binary]==1.3.22 # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -338,12 +313,9 @@ urllib3==1.26.3 # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt - # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/_base.in - # minio # requests uvicorn[standard]==0.13.3 # via fastapi @@ -354,7 +326,6 @@ uvloop==0.14.0 ; python_version < "3.7" # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt diff --git a/services/api-server/requirements/_test.txt b/services/api-server/requirements/_test.txt index 8fab363e42f..65d1f54be23 100644 --- a/services/api-server/requirements/_test.txt +++ b/services/api-server/requirements/_test.txt @@ -190,7 +190,6 @@ pytest==6.2.3 # pytest-mock python-dateutil==2.8.1 # via - # -c requirements/_base.txt # alembic # faker python-dotenv==0.15.0 From 08e5e1ddb50c3f2d4757c0b2b011cf9ab0ce46c1 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 26 Apr 2021 12:56:35 +0200 Subject: [PATCH 07/27] removes s3wrapper dependency from pytest-simcore --- .vscode/settings.template.json | 1 - .../src/pytest_simcore/minio_service.py | 53 +++++++++++++------ 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/.vscode/settings.template.json b/.vscode/settings.template.json index 13c745cda91..5c5aa8944ed 100644 --- a/.vscode/settings.template.json +++ b/.vscode/settings.template.json @@ -33,7 +33,6 @@ "./packages/postgres-database/src", "./packages/postgres-database/tests", "./packages/pytest-simcore/src", - "./packages/s3wrapper/src", "./packages/service-integration/src", "./packages/service-library/src", "./packages/simcore-sdk/src", diff --git a/packages/pytest-simcore/src/pytest_simcore/minio_service.py b/packages/pytest-simcore/src/pytest_simcore/minio_service.py index e68b6ee32b1..6226ea880c7 100644 --- a/packages/pytest-simcore/src/pytest_simcore/minio_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/minio_service.py @@ -1,16 +1,16 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + import logging import os from copy import deepcopy - -# pylint:disable=unused-variable -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name from distutils.util import strtobool -from typing import Dict +from typing import Dict, Iterator import pytest import tenacity -from s3wrapper.s3_client import S3Client +from minio import Minio from .helpers.utils_docker import get_ip, get_service_published_port @@ -44,15 +44,25 @@ def minio_config(docker_stack: Dict, devel_environ: Dict) -> Dict[str, str]: @pytest.fixture(scope="module") -def minio_service(minio_config: Dict[str, str]) -> S3Client: +def minio_service(minio_config: Dict[str, str]) -> Iterator[Minio]: assert wait_till_minio_responsive(minio_config) - client = S3Client(**minio_config["client"]) - assert client.create_bucket(minio_config["bucket_name"]) + client = Minio(**minio_config["client"]) + + bucket_name = minio_config["bucket_name"] + assert not client.bucket_exists(bucket_name) + + client.make_bucket(bucket_name) yield client - assert client.remove_bucket(minio_config["bucket_name"], delete_contents=True) + assert client.bucket_exists(bucket_name) + # remove content + objs = client.list_objects(bucket_name, prefix=None, recursive=True) + errors = client.remove_objects(bucket_name, [o.object_name for o in objs]) + assert not list(errors) + # remove bucket + client.remove_bucket(bucket_name) @tenacity.retry( @@ -63,18 +73,31 @@ def minio_service(minio_config: Dict[str, str]) -> S3Client: ) def wait_till_minio_responsive(minio_config: Dict[str, str]) -> bool: """Check if something responds to ``url`` """ - client = S3Client(**minio_config["client"]) - if client.create_bucket("pytest"): + client = Minio(**minio_config["client"]) + if client.make_bucket("pytest"): client.remove_bucket("pytest") return True raise Exception(f"Minio not responding to {minio_config}") @pytest.fixture(scope="module") -def bucket(minio_config: Dict[str, str], minio_service: S3Client) -> str: +def bucket(minio_config: Dict[str, str], minio_service: Minio) -> str: bucket_name = minio_config["bucket_name"] - minio_service.create_bucket(bucket_name, delete_contents_if_exists=True) + + def safe_delete(): + if minio_service.bucket_exists(bucket_name): + # remove content + objs = minio_service.list_objects(bucket_name, prefix=None, recursive=True) + errors = minio_service.remove_objects( + bucket_name, [o.object_name for o in objs] + ) + assert not list(errors) + # remove bucket + minio_service.remove_bucket(bucket_name) + + safe_delete() + minio_service.make_bucket(bucket_name) yield bucket_name - minio_service.remove_bucket(bucket_name, delete_contents=True) + safe_delete() From d8380929b201cad9e599b9cf2d6d5c59c8a9a333 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 26 Apr 2021 12:57:02 +0200 Subject: [PATCH 08/27] minor fix test in storage --- .../src/pytest_simcore/simcore_storage_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py index 03921e63c4f..d15cebc9369 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py @@ -8,11 +8,11 @@ import aiohttp import pytest import tenacity -from s3wrapper.s3_client import S3Client from servicelib.minio_utils import MinioRetryPolicyUponInitialization from yarl import URL from .helpers.utils_docker import get_service_published_port +from .s3wrapper.s3_client import S3Client @pytest.fixture(scope="module") From 358d4b5f8691ca85c3ba9b373faf39da59b1eaca Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 26 Apr 2021 13:07:02 +0200 Subject: [PATCH 09/27] adds minio in tests reqs for simcore-sdk --- packages/simcore-sdk/requirements/_test.in | 1 + packages/simcore-sdk/requirements/_test.txt | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/simcore-sdk/requirements/_test.in b/packages/simcore-sdk/requirements/_test.in index e13c8cd7fcf..b19baac13c6 100644 --- a/packages/simcore-sdk/requirements/_test.in +++ b/packages/simcore-sdk/requirements/_test.in @@ -26,6 +26,7 @@ requests docker python-dotenv faker +minio # tools for CI pylint diff --git a/packages/simcore-sdk/requirements/_test.txt b/packages/simcore-sdk/requirements/_test.txt index ff4faf408e8..009a208229a 100644 --- a/packages/simcore-sdk/requirements/_test.txt +++ b/packages/simcore-sdk/requirements/_test.txt @@ -27,7 +27,9 @@ attrs==20.3.0 # aiohttp # pytest certifi==2020.12.5 - # via requests + # via + # minio + # requests chardet==4.0.0 # via # -c requirements/_base.txt @@ -79,6 +81,8 @@ markupsafe==1.1.1 # via mako mccabe==0.6.1 # via pylint +minio==7.0.3 + # via -r requirements/_test.in multidict==5.1.0 # via # -c requirements/_base.txt @@ -175,7 +179,9 @@ typing-extensions==3.7.4.3 # importlib-metadata # yarl urllib3==1.26.4 - # via requests + # via + # minio + # requests websocket-client==0.58.0 # via docker wrapt==1.12.1 From 777572030c42444f56f21f5394184792eaa0feaa Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 26 Apr 2021 13:12:49 +0200 Subject: [PATCH 10/27] Fixes minio in pytest-simcore --- .../src/pytest_simcore/simcore_storage_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py index d15cebc9369..96e1c311d31 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py @@ -8,11 +8,11 @@ import aiohttp import pytest import tenacity +from minio import Minio from servicelib.minio_utils import MinioRetryPolicyUponInitialization from yarl import URL from .helpers.utils_docker import get_service_published_port -from .s3wrapper.s3_client import S3Client @pytest.fixture(scope="module") @@ -33,7 +33,7 @@ def storage_endpoint(docker_stack: Dict, devel_environ: Dict) -> URL: @pytest.fixture(scope="function") async def storage_service( - minio_service: S3Client, storage_endpoint: URL, docker_stack: Dict + minio_service: Minio, storage_endpoint: URL, docker_stack: Dict ) -> URL: await wait_till_storage_responsive(storage_endpoint) From 00502d2684158fbe3a43eaf62dedb936a70db29e Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 26 Apr 2021 13:13:20 +0200 Subject: [PATCH 11/27] adds minio in tests reqs for sidecar --- services/sidecar/requirements/_test.in | 1 + services/sidecar/requirements/_test.txt | 18 ++++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/services/sidecar/requirements/_test.in b/services/sidecar/requirements/_test.in index 16316a981f0..7c570bef861 100644 --- a/services/sidecar/requirements/_test.in +++ b/services/sidecar/requirements/_test.in @@ -26,6 +26,7 @@ aiopg docker python-dotenv faker +minio # tools for CI pylint diff --git a/services/sidecar/requirements/_test.txt b/services/sidecar/requirements/_test.txt index f71b9a6c954..ba17ad3f1e0 100644 --- a/services/sidecar/requirements/_test.txt +++ b/services/sidecar/requirements/_test.txt @@ -9,7 +9,7 @@ aiohttp==3.7.3 # -c requirements/_base.txt # -c requirements/_packages.txt # pytest-aiohttp -aiopg[sa]==1.2.1 +aiopg==1.2.1 # via # -c requirements/_base.txt # -c requirements/_packages.txt @@ -31,13 +31,17 @@ attrs==20.2.0 # aiohttp # pytest certifi==2020.6.20 - # via requests + # via + # minio + # requests chardet==3.0.4 # via # -c requirements/_base.txt # -c requirements/_packages.txt # aiohttp # requests +configparser==5.0.2 + # via minio coverage==5.5 # via # -r requirements/_test.in @@ -85,6 +89,10 @@ markupsafe==1.1.1 # via mako mccabe==0.6.1 # via pylint +minio==6.0.2 + # via + # -c requirements/../../../requirements/constraints.txt + # -r requirements/_test.in multidict==4.7.6 # via # -c requirements/_base.txt @@ -144,10 +152,15 @@ python-dateutil==2.8.1 # via # alembic # faker + # minio python-dotenv==0.17.0 # via -r requirements/_test.in python-editor==1.0.4 # via alembic +pytz==2020.1 + # via + # -c requirements/_base.txt + # minio requests==2.25.1 # via # coveralls @@ -186,6 +199,7 @@ urllib3==1.25.11 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt + # minio # requests websocket-client==0.58.0 # via docker From 726a1c973cf7dfcf9b0ce75b58f9135c313daca3 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 26 Apr 2021 13:14:17 +0200 Subject: [PATCH 12/27] adds minio in tests reqs for swarm-deploy --- tests/swarm-deploy/requirements/_test.in | 1 + tests/swarm-deploy/requirements/_test.txt | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/tests/swarm-deploy/requirements/_test.in b/tests/swarm-deploy/requirements/_test.in index 99490bb3171..a5f927b8708 100644 --- a/tests/swarm-deploy/requirements/_test.in +++ b/tests/swarm-deploy/requirements/_test.in @@ -20,3 +20,4 @@ pytest-sugar python-dotenv pyyaml tenacity +minio diff --git a/tests/swarm-deploy/requirements/_test.txt b/tests/swarm-deploy/requirements/_test.txt index 8f81265ca97..f03a37dffd7 100644 --- a/tests/swarm-deploy/requirements/_test.txt +++ b/tests/swarm-deploy/requirements/_test.txt @@ -57,6 +57,7 @@ attrs==20.3.0 certifi==2020.12.5 # via # -r requirements/../../../packages/postgres-database/requirements/_migration.txt + # minio # requests chardet==4.0.0 # via @@ -67,6 +68,8 @@ click==7.1.2 # via # -r requirements/../../../packages/postgres-database/requirements/_migration.txt # -r requirements/_test.in +configparser==5.0.2 + # via minio coverage==5.5 # via # -r requirements/_test.in @@ -127,6 +130,16 @@ markupsafe==1.1.1 # via # -r requirements/../../../packages/postgres-database/requirements/_migration.txt # mako +minio==6.0.2 + # via + # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt + # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt + # -c requirements/../../../requirements/constraints.txt + # -r requirements/_test.in multidict==5.1.0 # via # aiohttp @@ -199,12 +212,15 @@ python-dateutil==2.8.1 # via # -r requirements/../../../packages/postgres-database/requirements/_migration.txt # alembic + # minio python-dotenv==0.17.0 # via -r requirements/_test.in python-editor==1.0.4 # via # -r requirements/../../../packages/postgres-database/requirements/_migration.txt # alembic +pytz==2021.1 + # via minio pyyaml==5.4.1 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -291,6 +307,7 @@ urllib3==1.26.4 # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_migration.txt + # minio # requests websocket-client==0.58.0 # via From 8eefd5db2570aea51c61ab3f114cc23613f03bb9 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 26 Apr 2021 15:03:57 +0200 Subject: [PATCH 13/27] fixes reqs --- services/storage/requirements/_test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/storage/requirements/_test.txt b/services/storage/requirements/_test.txt index 91b988fd93b..55c102c9176 100644 --- a/services/storage/requirements/_test.txt +++ b/services/storage/requirements/_test.txt @@ -100,7 +100,7 @@ lazy-object-proxy==1.4.3 # astroid mccabe==0.6.1 # via pylint -multidict==5.0.0 +multidict==5.1.0 # via # -c requirements/_base.txt # aiohttp From eaa4cc39c599c6ede2300e34317036c7b8c40be1 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 26 Apr 2021 16:42:49 +0200 Subject: [PATCH 14/27] minor --- .../pytest-simcore/src/pytest_simcore/environment_configs.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/pytest-simcore/src/pytest_simcore/environment_configs.py b/packages/pytest-simcore/src/pytest_simcore/environment_configs.py index 7fd65124f16..c92a88e4a8f 100644 --- a/packages/pytest-simcore/src/pytest_simcore/environment_configs.py +++ b/packages/pytest-simcore/src/pytest_simcore/environment_configs.py @@ -1,4 +1,7 @@ # pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + from copy import deepcopy from pathlib import Path From a1f3b78fa05c9f3575c04ffefb59037749d5811f Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 26 Apr 2021 16:46:43 +0200 Subject: [PATCH 15/27] Cleanup pytest_simcore.minio_service and some extra monkeypatch fixtures --- .../src/pytest_simcore/minio_service.py | 54 +++++++++---------- .../src/pytest_simcore/monkeypatch_extra.py | 26 +++++++++ 2 files changed, 50 insertions(+), 30 deletions(-) create mode 100644 packages/pytest-simcore/src/pytest_simcore/monkeypatch_extra.py diff --git a/packages/pytest-simcore/src/pytest_simcore/minio_service.py b/packages/pytest-simcore/src/pytest_simcore/minio_service.py index 6226ea880c7..b2dd1e2eddd 100644 --- a/packages/pytest-simcore/src/pytest_simcore/minio_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/minio_service.py @@ -3,8 +3,6 @@ # pylint: disable=unused-variable import logging -import os -from copy import deepcopy from distutils.util import strtobool from typing import Dict, Iterator @@ -17,8 +15,21 @@ log = logging.getLogger(__name__) +def _remove_bucket(client: Minio, bucket_name: str): + if client.bucket_exists(bucket_name): + # remove content + objs = client.list_objects(bucket_name, prefix=None, recursive=True) + errors = client.remove_objects(bucket_name, [o.object_name for o in objs]) + assert not list(errors) + # remove bucket + client.remove_bucket(bucket_name) + assert not client.bucket_exists(bucket_name) + + @pytest.fixture(scope="module") -def minio_config(docker_stack: Dict, devel_environ: Dict) -> Dict[str, str]: +def minio_config( + docker_stack: Dict, devel_environ: Dict, monkeypatch_module +) -> Dict[str, str]: assert "ops_minio" in docker_stack["services"] config = { @@ -32,15 +43,13 @@ def minio_config(docker_stack: Dict, devel_environ: Dict) -> Dict[str, str]: } # nodeports takes its configuration from env variables - old_environ = deepcopy(os.environ) for key, value in config["client"].items(): - os.environ[f"S3_{key.upper()}"] = str(value) - os.environ["S3_SECURE"] = devel_environ["S3_SECURE"] - os.environ["S3_BUCKET_NAME"] = config["bucket_name"] + monkeypatch_module.setenv(f"S3_{key.upper()}", str(value)) + + monkeypatch_module.setenv("S3_SECURE", devel_environ["S3_SECURE"]) + monkeypatch_module.setenv("S3_BUCKET_NAME", config["bucket_name"]) - yield config - # restore environ - os.environ = old_environ + return config @pytest.fixture(scope="module") @@ -50,19 +59,15 @@ def minio_service(minio_config: Dict[str, str]) -> Iterator[Minio]: client = Minio(**minio_config["client"]) bucket_name = minio_config["bucket_name"] - assert not client.bucket_exists(bucket_name) + assert not client.bucket_exists(bucket_name) client.make_bucket(bucket_name) + assert client.bucket_exists(bucket_name) yield client assert client.bucket_exists(bucket_name) - # remove content - objs = client.list_objects(bucket_name, prefix=None, recursive=True) - errors = client.remove_objects(bucket_name, [o.object_name for o in objs]) - assert not list(errors) - # remove bucket - client.remove_bucket(bucket_name) + _remove_bucket(client, bucket_name) @tenacity.retry( @@ -84,20 +89,9 @@ def wait_till_minio_responsive(minio_config: Dict[str, str]) -> bool: def bucket(minio_config: Dict[str, str], minio_service: Minio) -> str: bucket_name = minio_config["bucket_name"] - def safe_delete(): - if minio_service.bucket_exists(bucket_name): - # remove content - objs = minio_service.list_objects(bucket_name, prefix=None, recursive=True) - errors = minio_service.remove_objects( - bucket_name, [o.object_name for o in objs] - ) - assert not list(errors) - # remove bucket - minio_service.remove_bucket(bucket_name) - - safe_delete() + _remove_bucket(minio_service, bucket_name) minio_service.make_bucket(bucket_name) yield bucket_name - safe_delete() + _remove_bucket(minio_service, bucket_name) diff --git a/packages/pytest-simcore/src/pytest_simcore/monkeypatch_extra.py b/packages/pytest-simcore/src/pytest_simcore/monkeypatch_extra.py new file mode 100644 index 00000000000..0877702d785 --- /dev/null +++ b/packages/pytest-simcore/src/pytest_simcore/monkeypatch_extra.py @@ -0,0 +1,26 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + +import pytest + +# Some extras to overcome https://github.com/pytest-dev/pytest/issues/363 + + +@pytest.fixture(scope="session") +def monkeypatch_session(request): + from _pytest.monkeypatch import MonkeyPatch + + mpatch_session = MonkeyPatch() + yield mpatch_session + mpatch_session.undo() + + +@pytest.fixture(scope="module") +def monkeypatch_module(request): + from _pytest.monkeypatch import MonkeyPatch + + mpatch_module = MonkeyPatch() + yield mpatch_module + + mpatch_module.undo() From e965e814706141ec04810da4327fd2bfa9c2b2e4 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 26 Apr 2021 16:47:11 +0200 Subject: [PATCH 16/27] moved to a subfolder former s3wrapper tests --- services/storage/tests/s3wrapper/__init__.py | 0 .../tests/{ => s3wrapper}/test_minio.py | 0 .../tests/{ => s3wrapper}/test_s3_client.py | 22 +++++++++---------- 3 files changed, 10 insertions(+), 12 deletions(-) create mode 100644 services/storage/tests/s3wrapper/__init__.py rename services/storage/tests/{ => s3wrapper}/test_minio.py (100%) rename services/storage/tests/{ => s3wrapper}/test_s3_client.py (95%) diff --git a/services/storage/tests/s3wrapper/__init__.py b/services/storage/tests/s3wrapper/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/storage/tests/test_minio.py b/services/storage/tests/s3wrapper/test_minio.py similarity index 100% rename from services/storage/tests/test_minio.py rename to services/storage/tests/s3wrapper/test_minio.py diff --git a/services/storage/tests/test_s3_client.py b/services/storage/tests/s3wrapper/test_s3_client.py similarity index 95% rename from services/storage/tests/test_s3_client.py rename to services/storage/tests/s3wrapper/test_s3_client.py index 2a9aafcbbda..dad1ecc8cb1 100644 --- a/services/storage/tests/test_s3_client.py +++ b/services/storage/tests/s3wrapper/test_s3_client.py @@ -1,3 +1,7 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + import filecmp import os import time @@ -7,13 +11,8 @@ import pytest import requests - -# pylint:disable=unused-import -from pytest_docker import docker_ip, docker_services from simcore_service_storage.s3wrapper.s3_client import S3Client -# pylint:disable=redefined-outer-name - def is_responsive(url, code=200): """Check if something responds to ``url``.""" @@ -50,10 +49,9 @@ def s3_client(docker_ip, docker_services): endpoint = "{ip}:{port}".format( ip=docker_ip, port=docker_services.port_for("minio", 9000) ) - access_key = "12345678" - secret_key = "12345678" - secure = False - s3_client = S3Client(endpoint, access_key, secret_key, secure) + s3_client = S3Client( + endpoint, access_key="12345678", secret_key="12345678", secure=False + ) return s3_client @@ -131,9 +129,9 @@ def test_file_upload_meta_data(s3_client, bucket, text_files): metadata2 = s3_client.get_metadata(bucket, object_name) - assert metadata2["User"] == "guidon" - assert metadata2["Node_id"] == str(_id) - assert metadata2["Boom-Boom"] == str(42.0) + assert metadata2["user"] == "guidon" + assert metadata2["node_id"] == str(_id) + assert metadata2["boom-boom"] == str(42.0) @pytest.mark.enable_travis From 26645d85ba57a1629e5c102f6570f20314982661 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 26 Apr 2021 17:27:05 +0200 Subject: [PATCH 17/27] adds missing pytest-plugin --- packages/simcore-sdk/tests/conftest.py | 7 ++++--- services/sidecar/tests/conftest.py | 11 ++++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/simcore-sdk/tests/conftest.py b/packages/simcore-sdk/tests/conftest.py index bc2461990ec..268d755d711 100644 --- a/packages/simcore-sdk/tests/conftest.py +++ b/packages/simcore-sdk/tests/conftest.py @@ -16,13 +16,14 @@ # FIXTURES pytest_plugins = [ - "pytest_simcore.repository_paths", "pytest_simcore.docker_compose", "pytest_simcore.docker_swarm", - "pytest_simcore.postgres_service", "pytest_simcore.minio_service", - "pytest_simcore.simcore_storage_service", + "pytest_simcore.monkeypatch_extra", + "pytest_simcore.postgres_service", + "pytest_simcore.repository_paths", "pytest_simcore.services_api_mocks_for_aiohttp_clients", + "pytest_simcore.simcore_storage_service", ] diff --git a/services/sidecar/tests/conftest.py b/services/sidecar/tests/conftest.py index 1ff888eaa38..a13121b9d66 100644 --- a/services/sidecar/tests/conftest.py +++ b/services/sidecar/tests/conftest.py @@ -1,6 +1,7 @@ -# pylint:disable=unused-variable -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + import sys from pathlib import Path @@ -8,11 +9,11 @@ import simcore_service_sidecar pytest_plugins = [ - "pytest_simcore.repository_paths", "pytest_simcore.docker_compose", "pytest_simcore.docker_swarm", + "pytest_simcore.monkeypatch_extra" "pytest_simcore.postgres_service", "pytest_simcore.rabbit_service", - "pytest_simcore.postgres_service", + "pytest_simcore.repository_paths", ] current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent From 27da19b275267eea822437258672c85deaed5b92 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 26 Apr 2021 17:30:20 +0200 Subject: [PATCH 18/27] fixes healtcheck in mino --- .../src/pytest_simcore/minio_service.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/minio_service.py b/packages/pytest-simcore/src/pytest_simcore/minio_service.py index b2dd1e2eddd..65e2a0d0939 100644 --- a/packages/pytest-simcore/src/pytest_simcore/minio_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/minio_service.py @@ -73,16 +73,16 @@ def minio_service(minio_config: Dict[str, str]) -> Iterator[Minio]: @tenacity.retry( wait=tenacity.wait_fixed(5), stop=tenacity.stop_after_attempt(60), - before_sleep=tenacity.before_sleep_log(log, logging.INFO), + before_sleep=tenacity.before_sleep_log(log, logging.WARNING), reraise=True, ) def wait_till_minio_responsive(minio_config: Dict[str, str]) -> bool: - """Check if something responds to ``url`` """ client = Minio(**minio_config["client"]) - if client.make_bucket("pytest"): - client.remove_bucket("pytest") - return True - raise Exception(f"Minio not responding to {minio_config}") + # TODO: improve as https://docs.min.io/docs/minio-monitoring-guide.html + if not client.bucket_exists("pytest"): + client.make_bucket("pytest") + client.remove_bucket("pytest") + return True @pytest.fixture(scope="module") From 42837221b48072a3588e50a3affeb30686680411 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 26 Apr 2021 18:24:35 +0200 Subject: [PATCH 19/27] Fixes minio deprecated functionality --- .../src/pytest_simcore/minio_service.py | 22 ++-- .../src/simcore_service_storage/dsm.py | 11 +- .../s3wrapper/s3_client.py | 101 +++++++++--------- .../storage/tests/s3wrapper/test_s3_client.py | 2 +- 4 files changed, 71 insertions(+), 65 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/minio_service.py b/packages/pytest-simcore/src/pytest_simcore/minio_service.py index 65e2a0d0939..e28117fc8d0 100644 --- a/packages/pytest-simcore/src/pytest_simcore/minio_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/minio_service.py @@ -9,17 +9,21 @@ import pytest import tenacity from minio import Minio +from minio.deleteobjects import DeleteObject from .helpers.utils_docker import get_ip, get_service_published_port log = logging.getLogger(__name__) -def _remove_bucket(client: Minio, bucket_name: str): +def _ensure_rm_bucket(client: Minio, bucket_name: str): if client.bucket_exists(bucket_name): # remove content - objs = client.list_objects(bucket_name, prefix=None, recursive=True) - errors = client.remove_objects(bucket_name, [o.object_name for o in objs]) + delete_object_list = map( + lambda x: DeleteObject(x.object_name), + client.list_objects(bucket_name, prefix=None, recursive=True), + ) + errors = client.remove_objects(bucket_name, delete_object_list) assert not list(errors) # remove bucket client.remove_bucket(bucket_name) @@ -60,14 +64,16 @@ def minio_service(minio_config: Dict[str, str]) -> Iterator[Minio]: bucket_name = minio_config["bucket_name"] - assert not client.bucket_exists(bucket_name) + # makes sure there is not an old copy, e.g. leftover from failure + _ensure_rm_bucket(client, bucket_name) + client.make_bucket(bucket_name) assert client.bucket_exists(bucket_name) yield client - assert client.bucket_exists(bucket_name) - _remove_bucket(client, bucket_name) + # can fail because minio is down + _ensure_rm_bucket(client, bucket_name) @tenacity.retry( @@ -89,9 +95,9 @@ def wait_till_minio_responsive(minio_config: Dict[str, str]) -> bool: def bucket(minio_config: Dict[str, str], minio_service: Minio) -> str: bucket_name = minio_config["bucket_name"] - _remove_bucket(minio_service, bucket_name) + _ensure_rm_bucket(minio_service, bucket_name) minio_service.make_bucket(bucket_name) yield bucket_name - _remove_bucket(minio_service, bucket_name) + _ensure_rm_bucket(minio_service, bucket_name) diff --git a/services/storage/src/simcore_service_storage/dsm.py b/services/storage/src/simcore_service_storage/dsm.py index 9ca77ad77b1..06926f3c3bd 100644 --- a/services/storage/src/simcore_service_storage/dsm.py +++ b/services/storage/src/simcore_service_storage/dsm.py @@ -577,16 +577,13 @@ async def download_link_datcore(self, user_id: str, file_id: str) -> Dict[str, s async def copy_file_s3_s3(self, user_id: str, dest_uuid: str, source_uuid: str): # FIXME: operation MUST be atomic + # FIXME: This is not async! # source is s3, location is s3 - to_bucket_name = self.simcore_bucket_name - to_object_name = dest_uuid - from_bucket = self.simcore_bucket_name - from_object_name = source_uuid - from_bucket_object_name = os.path.join(from_bucket, from_object_name) - # FIXME: This is not async! self.s3_client.copy_object( - to_bucket_name, to_object_name, from_bucket_object_name + bucket_name=self.simcore_bucket_name, + to_object_name=dest_uuid, + from_object_name=source_uuid, ) # update db diff --git a/services/storage/src/simcore_service_storage/s3wrapper/s3_client.py b/services/storage/src/simcore_service_storage/s3wrapper/s3_client.py index 41b3b1521ec..c89cb4ab43a 100644 --- a/services/storage/src/simcore_service_storage/s3wrapper/s3_client.py +++ b/services/storage/src/simcore_service_storage/s3wrapper/s3_client.py @@ -2,8 +2,10 @@ import re from datetime import timedelta -from minio import CopyConditions, Minio -from minio.error import ResponseError +from minio import Minio +from minio.commonconfig import CopySource +from minio.deleteobjects import DeleteObject +from minio.error import MinioException log = logging.getLogger(__name__) @@ -29,16 +31,16 @@ def __init__( self.client = Minio( endpoint, access_key=access_key, secret_key=secret_key, secure=secure ) - except ResponseError as _err: - logging.exception("Could not create minio client") + except MinioException as _err: + log.exception("Could not create minio client") def __remove_objects_recursively(self, bucket_name): - objs = self.list_objects(bucket_name, recursive=True) - to_del = [] - for obj in objs: - to_del.append(obj.object_name) + delete_object_list = map( + lambda x: DeleteObject(x.object_name), + self.list_objects(bucket_name, prefix=None, recursive=True), + ) - self.remove_objects(bucket_name, to_del) + self.remove_objects(bucket_name, delete_object_list) def create_bucket(self, bucket_name, delete_contents_if_exists=False): try: @@ -47,8 +49,8 @@ def create_bucket(self, bucket_name, delete_contents_if_exists=False): elif delete_contents_if_exists: return self.__remove_objects_recursively(bucket_name) - except ResponseError as _err: - logging.exception("Could not create bucket") + except MinioException as _err: + log.exception("Could not create bucket") return False # it probably already exists and is return True @@ -59,24 +61,24 @@ def remove_bucket(self, bucket_name, delete_contents=False): if delete_contents: self.__remove_objects_recursively(bucket_name) self.client.remove_bucket(bucket_name) - except ResponseError as _err: - logging.exception("Could not remove bucket") + except MinioException as _err: + log.exception("Could not remove bucket") return False return True def exists_bucket(self, bucket_name): try: return self.client.bucket_exists(bucket_name) - except ResponseError as _err: - logging.exception("Could not check bucket for existence") + except MinioException as _err: + log.exception("Could not check bucket for existence") return False def list_buckets(self): try: return self.client.list_buckets() - except ResponseError as _err: - logging.exception("Could not list bucket") + except MinioException as _err: + log.exception("Could not list bucket") return [] @@ -101,16 +103,16 @@ def upload_file(self, bucket_name, object_name, filepath, metadata=None): self.client.fput_object( bucket_name, object_name, filepath, metadata=_metadata ) - except ResponseError as _err: - logging.exception("Could not upload file") + except MinioException as _err: + log.exception("Could not upload file") return False return True def download_file(self, bucket_name, object_name, filepath): try: self.client.fget_object(bucket_name, object_name, filepath) - except ResponseError as _err: - logging.exception("Could not download file") + except MinioException as _err: + log.exception("Could not download file") return False return True @@ -124,8 +126,8 @@ def get_metadata(self, bucket_name, object_name): metadata[_key] = _metadata[key] return metadata - except ResponseError as _err: - logging.exception("Could not get metadata") + except MinioException as _err: + log.exception("Could not get metadata") return {} @@ -134,16 +136,16 @@ def list_objects(self, bucket_name, prefix=None, recursive=False): return self.client.list_objects( bucket_name, prefix=prefix, recursive=recursive ) - except ResponseError as _err: - logging.exception("Could not list objects") + except MinioException as _err: + log.exception("Could not list objects") return [] def list_objects_v2(self, bucket_name, recursive=False): try: return self.client.list_objects_v2(bucket_name, recursive=recursive) - except ResponseError as _err: - logging.exception("Could not list objects") + except MinioException as _err: + log.exception("Could not list objects") return [] @@ -152,8 +154,8 @@ def remove_objects(self, bucket_name, objects): for del_err in self.client.remove_objects(bucket_name, objects): msg = "Deletion Error: {}".format(del_err) log.debug(msg) - except ResponseError as _err: - logging.exception("Could remove objects") + except MinioException as _err: + log.exception("Could remove objects") return False return True @@ -164,8 +166,8 @@ def exists_object(self, bucket_name, object_name, recursive=False): for obj in objects: if obj.object_name == object_name: return True - except ResponseError as _err: - logging.exception("Could check object for existence") + except MinioException as _err: + log.exception("Could check object for existence") return False return False @@ -198,8 +200,8 @@ def create_presigned_put_url(self, bucket_name, object_name, dt=timedelta(days=3 bucket_name, object_name, expires=dt ) - except ResponseError as _err: - logging.exception("Could create presigned put url") + except MinioException as _err: + log.exception("Could create presigned put url") return "" @@ -209,22 +211,23 @@ def create_presigned_get_url(self, bucket_name, object_name, dt=timedelta(days=3 bucket_name, object_name, expires=dt ) - except ResponseError as _err: - logging.exception("Could create presigned get url") + except MinioException as _err: + log.exception("Could create presigned get url") return "" - def copy_object(self, to_bucket_name, to_object_name, from_bucket_object_name): - try: - ret = self.client.copy_object( - to_bucket_name, - to_object_name, - from_bucket_object_name, - CopyConditions(), - ) - print(ret) - return True - except ResponseError as _err: - logging.exception("Could not copy") - - return False + def copy_object(self, bucket_name, to_object_name, from_object_name): + """ Copies from_object_name -> to_object_name within the same bucket """ + result = self.client.copy_object( + bucket_name=bucket_name, + object_name=to_object_name, + source=CopySource(bucket_name, from_object_name), + ) + log.debug( + "%s:%s -> %s:%s: %s", + bucket_name, + from_object_name, + bucket_name, + to_object_name, + result, + ) diff --git a/services/storage/tests/s3wrapper/test_s3_client.py b/services/storage/tests/s3wrapper/test_s3_client.py index dad1ecc8cb1..bd770773e2d 100644 --- a/services/storage/tests/s3wrapper/test_s3_client.py +++ b/services/storage/tests/s3wrapper/test_s3_client.py @@ -261,7 +261,7 @@ def test_copy_object(s3_client, bucket, text_files): assert s3_client.upload_file(bucket, object_name, file) assert s3_client.exists_object(bucket, object_name, False) copied_object = "copy" - assert s3_client.copy_object(bucket, copied_object, bucket + "/" + object_name) + assert s3_client.copy_object(bucket, copied_object, object_name) assert s3_client.exists_object(bucket, copied_object, False) From 54907b6724dadc012e00d1c3983e366b53e41df3 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 26 Apr 2021 18:28:45 +0200 Subject: [PATCH 20/27] typo --- services/sidecar/tests/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/sidecar/tests/conftest.py b/services/sidecar/tests/conftest.py index a13121b9d66..93b16284de7 100644 --- a/services/sidecar/tests/conftest.py +++ b/services/sidecar/tests/conftest.py @@ -11,7 +11,8 @@ pytest_plugins = [ "pytest_simcore.docker_compose", "pytest_simcore.docker_swarm", - "pytest_simcore.monkeypatch_extra" "pytest_simcore.postgres_service", + "pytest_simcore.monkeypatch_extra", + "pytest_simcore.postgres_service", "pytest_simcore.rabbit_service", "pytest_simcore.repository_paths", ] From d1a213237327e19a3c0ebe45b1a774712ff1b129 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 26 Apr 2021 18:55:49 +0200 Subject: [PATCH 21/27] git fixes simcore-sdk requirements constraints --- packages/simcore-sdk/requirements/_test.in | 2 +- packages/simcore-sdk/requirements/_test.txt | 13 +++++++++++-- packages/simcore-sdk/requirements/_tools.in | 2 ++ packages/simcore-sdk/requirements/_tools.txt | 1 + 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/packages/simcore-sdk/requirements/_test.in b/packages/simcore-sdk/requirements/_test.in index b19baac13c6..1d1370eb305 100644 --- a/packages/simcore-sdk/requirements/_test.in +++ b/packages/simcore-sdk/requirements/_test.in @@ -1,7 +1,7 @@ # # Specifies dependencies required to run 'simcore-sdk' # - +-c ../../../requirements/constraints.txt # Adds base AS CONSTRAINT specs, not requirement. # - Resulting _text.txt is a frozen list of EXTRA packages for testing, besides _base.txt # diff --git a/packages/simcore-sdk/requirements/_test.txt b/packages/simcore-sdk/requirements/_test.txt index 009a208229a..7c8174a2b86 100644 --- a/packages/simcore-sdk/requirements/_test.txt +++ b/packages/simcore-sdk/requirements/_test.txt @@ -35,6 +35,8 @@ chardet==4.0.0 # -c requirements/_base.txt # aiohttp # requests +configparser==5.0.2 + # via minio coverage==5.5 # via # -r requirements/_test.in @@ -81,8 +83,10 @@ markupsafe==1.1.1 # via mako mccabe==0.6.1 # via pylint -minio==7.0.3 - # via -r requirements/_test.in +minio==6.0.2 + # via + # -c requirements/../../../requirements/constraints.txt + # -r requirements/_test.in multidict==5.1.0 # via # -c requirements/_base.txt @@ -144,10 +148,13 @@ python-dateutil==2.8.1 # via # alembic # faker + # minio python-dotenv==0.17.0 # via -r requirements/_test.in python-editor==1.0.4 # via alembic +pytz==2021.1 + # via minio requests==2.25.1 # via # -r requirements/_test.in @@ -160,6 +167,7 @@ six==1.15.0 # websocket-client sqlalchemy[postgresql_psycopg2binary]==1.3.24 # via + # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # alembic termcolor==1.1.0 @@ -180,6 +188,7 @@ typing-extensions==3.7.4.3 # yarl urllib3==1.26.4 # via + # -c requirements/../../../requirements/constraints.txt # minio # requests websocket-client==0.58.0 diff --git a/packages/simcore-sdk/requirements/_tools.in b/packages/simcore-sdk/requirements/_tools.in index 61b62e5647c..92dd68e474f 100644 --- a/packages/simcore-sdk/requirements/_tools.in +++ b/packages/simcore-sdk/requirements/_tools.in @@ -1,3 +1,5 @@ +-c ../../../requirements/constraints.txt + -c _base.txt -c _test.txt diff --git a/packages/simcore-sdk/requirements/_tools.txt b/packages/simcore-sdk/requirements/_tools.txt index 6f19871e7e1..110b5c828d8 100644 --- a/packages/simcore-sdk/requirements/_tools.txt +++ b/packages/simcore-sdk/requirements/_tools.txt @@ -57,6 +57,7 @@ pre-commit==2.12.1 # via -r requirements/../../../requirements/devenv.txt pyyaml==5.4.1 # via + # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # pre-commit regex==2021.4.4 From a0fc117a79c10f45655be0e0f56efe4a91deaa2f Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 26 Apr 2021 18:56:01 +0200 Subject: [PATCH 22/27] minor comment --- requirements/constraints.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 4b59c04904a..7bd384bac36 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -23,7 +23,9 @@ httpx<0.15.0 # [https://github.com/encode/httpx/releases/tag/0.15.0] due to PR#1285 [https://github.com/encode/httpx/pull/1285] respx<0.13.0 -# Breaking changes. TODO: review ASAP + +# limited by aiobotocore in storage. In addition, pytest_simcore.minio_service uses minio +# Version 7 changes API! Has Breaking changes TODO: review ASAP minio<7.0.0 From 73bd57b054ce8146316400395e925d8e9fca37de Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 26 Apr 2021 18:57:25 +0200 Subject: [PATCH 23/27] Revert "Fixes minio deprecated functionality" This reverts commit 42837221b48072a3588e50a3affeb30686680411. --- .../src/pytest_simcore/minio_service.py | 22 ++-- .../src/simcore_service_storage/dsm.py | 11 +- .../s3wrapper/s3_client.py | 101 +++++++++--------- .../storage/tests/s3wrapper/test_s3_client.py | 2 +- 4 files changed, 65 insertions(+), 71 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/minio_service.py b/packages/pytest-simcore/src/pytest_simcore/minio_service.py index e28117fc8d0..65e2a0d0939 100644 --- a/packages/pytest-simcore/src/pytest_simcore/minio_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/minio_service.py @@ -9,21 +9,17 @@ import pytest import tenacity from minio import Minio -from minio.deleteobjects import DeleteObject from .helpers.utils_docker import get_ip, get_service_published_port log = logging.getLogger(__name__) -def _ensure_rm_bucket(client: Minio, bucket_name: str): +def _remove_bucket(client: Minio, bucket_name: str): if client.bucket_exists(bucket_name): # remove content - delete_object_list = map( - lambda x: DeleteObject(x.object_name), - client.list_objects(bucket_name, prefix=None, recursive=True), - ) - errors = client.remove_objects(bucket_name, delete_object_list) + objs = client.list_objects(bucket_name, prefix=None, recursive=True) + errors = client.remove_objects(bucket_name, [o.object_name for o in objs]) assert not list(errors) # remove bucket client.remove_bucket(bucket_name) @@ -64,16 +60,14 @@ def minio_service(minio_config: Dict[str, str]) -> Iterator[Minio]: bucket_name = minio_config["bucket_name"] - # makes sure there is not an old copy, e.g. leftover from failure - _ensure_rm_bucket(client, bucket_name) - + assert not client.bucket_exists(bucket_name) client.make_bucket(bucket_name) assert client.bucket_exists(bucket_name) yield client - # can fail because minio is down - _ensure_rm_bucket(client, bucket_name) + assert client.bucket_exists(bucket_name) + _remove_bucket(client, bucket_name) @tenacity.retry( @@ -95,9 +89,9 @@ def wait_till_minio_responsive(minio_config: Dict[str, str]) -> bool: def bucket(minio_config: Dict[str, str], minio_service: Minio) -> str: bucket_name = minio_config["bucket_name"] - _ensure_rm_bucket(minio_service, bucket_name) + _remove_bucket(minio_service, bucket_name) minio_service.make_bucket(bucket_name) yield bucket_name - _ensure_rm_bucket(minio_service, bucket_name) + _remove_bucket(minio_service, bucket_name) diff --git a/services/storage/src/simcore_service_storage/dsm.py b/services/storage/src/simcore_service_storage/dsm.py index 06926f3c3bd..9ca77ad77b1 100644 --- a/services/storage/src/simcore_service_storage/dsm.py +++ b/services/storage/src/simcore_service_storage/dsm.py @@ -577,13 +577,16 @@ async def download_link_datcore(self, user_id: str, file_id: str) -> Dict[str, s async def copy_file_s3_s3(self, user_id: str, dest_uuid: str, source_uuid: str): # FIXME: operation MUST be atomic - # FIXME: This is not async! # source is s3, location is s3 + to_bucket_name = self.simcore_bucket_name + to_object_name = dest_uuid + from_bucket = self.simcore_bucket_name + from_object_name = source_uuid + from_bucket_object_name = os.path.join(from_bucket, from_object_name) + # FIXME: This is not async! self.s3_client.copy_object( - bucket_name=self.simcore_bucket_name, - to_object_name=dest_uuid, - from_object_name=source_uuid, + to_bucket_name, to_object_name, from_bucket_object_name ) # update db diff --git a/services/storage/src/simcore_service_storage/s3wrapper/s3_client.py b/services/storage/src/simcore_service_storage/s3wrapper/s3_client.py index c89cb4ab43a..41b3b1521ec 100644 --- a/services/storage/src/simcore_service_storage/s3wrapper/s3_client.py +++ b/services/storage/src/simcore_service_storage/s3wrapper/s3_client.py @@ -2,10 +2,8 @@ import re from datetime import timedelta -from minio import Minio -from minio.commonconfig import CopySource -from minio.deleteobjects import DeleteObject -from minio.error import MinioException +from minio import CopyConditions, Minio +from minio.error import ResponseError log = logging.getLogger(__name__) @@ -31,16 +29,16 @@ def __init__( self.client = Minio( endpoint, access_key=access_key, secret_key=secret_key, secure=secure ) - except MinioException as _err: - log.exception("Could not create minio client") + except ResponseError as _err: + logging.exception("Could not create minio client") def __remove_objects_recursively(self, bucket_name): - delete_object_list = map( - lambda x: DeleteObject(x.object_name), - self.list_objects(bucket_name, prefix=None, recursive=True), - ) + objs = self.list_objects(bucket_name, recursive=True) + to_del = [] + for obj in objs: + to_del.append(obj.object_name) - self.remove_objects(bucket_name, delete_object_list) + self.remove_objects(bucket_name, to_del) def create_bucket(self, bucket_name, delete_contents_if_exists=False): try: @@ -49,8 +47,8 @@ def create_bucket(self, bucket_name, delete_contents_if_exists=False): elif delete_contents_if_exists: return self.__remove_objects_recursively(bucket_name) - except MinioException as _err: - log.exception("Could not create bucket") + except ResponseError as _err: + logging.exception("Could not create bucket") return False # it probably already exists and is return True @@ -61,24 +59,24 @@ def remove_bucket(self, bucket_name, delete_contents=False): if delete_contents: self.__remove_objects_recursively(bucket_name) self.client.remove_bucket(bucket_name) - except MinioException as _err: - log.exception("Could not remove bucket") + except ResponseError as _err: + logging.exception("Could not remove bucket") return False return True def exists_bucket(self, bucket_name): try: return self.client.bucket_exists(bucket_name) - except MinioException as _err: - log.exception("Could not check bucket for existence") + except ResponseError as _err: + logging.exception("Could not check bucket for existence") return False def list_buckets(self): try: return self.client.list_buckets() - except MinioException as _err: - log.exception("Could not list bucket") + except ResponseError as _err: + logging.exception("Could not list bucket") return [] @@ -103,16 +101,16 @@ def upload_file(self, bucket_name, object_name, filepath, metadata=None): self.client.fput_object( bucket_name, object_name, filepath, metadata=_metadata ) - except MinioException as _err: - log.exception("Could not upload file") + except ResponseError as _err: + logging.exception("Could not upload file") return False return True def download_file(self, bucket_name, object_name, filepath): try: self.client.fget_object(bucket_name, object_name, filepath) - except MinioException as _err: - log.exception("Could not download file") + except ResponseError as _err: + logging.exception("Could not download file") return False return True @@ -126,8 +124,8 @@ def get_metadata(self, bucket_name, object_name): metadata[_key] = _metadata[key] return metadata - except MinioException as _err: - log.exception("Could not get metadata") + except ResponseError as _err: + logging.exception("Could not get metadata") return {} @@ -136,16 +134,16 @@ def list_objects(self, bucket_name, prefix=None, recursive=False): return self.client.list_objects( bucket_name, prefix=prefix, recursive=recursive ) - except MinioException as _err: - log.exception("Could not list objects") + except ResponseError as _err: + logging.exception("Could not list objects") return [] def list_objects_v2(self, bucket_name, recursive=False): try: return self.client.list_objects_v2(bucket_name, recursive=recursive) - except MinioException as _err: - log.exception("Could not list objects") + except ResponseError as _err: + logging.exception("Could not list objects") return [] @@ -154,8 +152,8 @@ def remove_objects(self, bucket_name, objects): for del_err in self.client.remove_objects(bucket_name, objects): msg = "Deletion Error: {}".format(del_err) log.debug(msg) - except MinioException as _err: - log.exception("Could remove objects") + except ResponseError as _err: + logging.exception("Could remove objects") return False return True @@ -166,8 +164,8 @@ def exists_object(self, bucket_name, object_name, recursive=False): for obj in objects: if obj.object_name == object_name: return True - except MinioException as _err: - log.exception("Could check object for existence") + except ResponseError as _err: + logging.exception("Could check object for existence") return False return False @@ -200,8 +198,8 @@ def create_presigned_put_url(self, bucket_name, object_name, dt=timedelta(days=3 bucket_name, object_name, expires=dt ) - except MinioException as _err: - log.exception("Could create presigned put url") + except ResponseError as _err: + logging.exception("Could create presigned put url") return "" @@ -211,23 +209,22 @@ def create_presigned_get_url(self, bucket_name, object_name, dt=timedelta(days=3 bucket_name, object_name, expires=dt ) - except MinioException as _err: - log.exception("Could create presigned get url") + except ResponseError as _err: + logging.exception("Could create presigned get url") return "" - def copy_object(self, bucket_name, to_object_name, from_object_name): - """ Copies from_object_name -> to_object_name within the same bucket """ - result = self.client.copy_object( - bucket_name=bucket_name, - object_name=to_object_name, - source=CopySource(bucket_name, from_object_name), - ) - log.debug( - "%s:%s -> %s:%s: %s", - bucket_name, - from_object_name, - bucket_name, - to_object_name, - result, - ) + def copy_object(self, to_bucket_name, to_object_name, from_bucket_object_name): + try: + ret = self.client.copy_object( + to_bucket_name, + to_object_name, + from_bucket_object_name, + CopyConditions(), + ) + print(ret) + return True + except ResponseError as _err: + logging.exception("Could not copy") + + return False diff --git a/services/storage/tests/s3wrapper/test_s3_client.py b/services/storage/tests/s3wrapper/test_s3_client.py index bd770773e2d..dad1ecc8cb1 100644 --- a/services/storage/tests/s3wrapper/test_s3_client.py +++ b/services/storage/tests/s3wrapper/test_s3_client.py @@ -261,7 +261,7 @@ def test_copy_object(s3_client, bucket, text_files): assert s3_client.upload_file(bucket, object_name, file) assert s3_client.exists_object(bucket, object_name, False) copied_object = "copy" - assert s3_client.copy_object(bucket, copied_object, object_name) + assert s3_client.copy_object(bucket, copied_object, bucket + "/" + object_name) assert s3_client.exists_object(bucket, copied_object, False) From cfaa8b73da9bb62a5150ea4fc8efbbbd7f7f4329 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 26 Apr 2021 18:57:38 +0200 Subject: [PATCH 24/27] upgardes minio in storage --- services/storage/requirements/_base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/storage/requirements/_base.txt b/services/storage/requirements/_base.txt index e91eb20c46e..f180d68162b 100644 --- a/services/storage/requirements/_base.txt +++ b/services/storage/requirements/_base.txt @@ -114,7 +114,7 @@ markupsafe==1.1.1 # via # aiohttp-swagger # jinja2 -minio==6.0.0 +minio==6.0.2 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt From 42979a6caa36e64642b2aa11fdafba61442d03ff Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Mon, 26 Apr 2021 20:10:46 +0200 Subject: [PATCH 25/27] fixes cleanup and refactors retry --- .../src/pytest_simcore/minio_service.py | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/minio_service.py b/packages/pytest-simcore/src/pytest_simcore/minio_service.py index 65e2a0d0939..e12c4bdb34e 100644 --- a/packages/pytest-simcore/src/pytest_simcore/minio_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/minio_service.py @@ -9,13 +9,14 @@ import pytest import tenacity from minio import Minio +from tenacity import Retrying from .helpers.utils_docker import get_ip, get_service_published_port log = logging.getLogger(__name__) -def _remove_bucket(client: Minio, bucket_name: str): +def _ensure_remove_bucket(client: Minio, bucket_name: str): if client.bucket_exists(bucket_name): # remove content objs = client.list_objects(bucket_name, prefix=None, recursive=True) @@ -54,44 +55,42 @@ def minio_config( @pytest.fixture(scope="module") def minio_service(minio_config: Dict[str, str]) -> Iterator[Minio]: - assert wait_till_minio_responsive(minio_config) client = Minio(**minio_config["client"]) + for attempt in Retrying( + wait=tenacity.wait_fixed(5), + stop=tenacity.stop_after_attempt(60), + before_sleep=tenacity.before_sleep_log(log, logging.WARNING), + reraise=True, + ): + with attempt: + # TODO: improve as https://docs.min.io/docs/minio-monitoring-guide.html + if not client.bucket_exists("pytest"): + client.make_bucket("pytest") + client.remove_bucket("pytest") + bucket_name = minio_config["bucket_name"] - assert not client.bucket_exists(bucket_name) + # cleans up in case a failing tests left this bucket + _ensure_remove_bucket(client, bucket_name) + client.make_bucket(bucket_name) assert client.bucket_exists(bucket_name) yield client - assert client.bucket_exists(bucket_name) - _remove_bucket(client, bucket_name) - - -@tenacity.retry( - wait=tenacity.wait_fixed(5), - stop=tenacity.stop_after_attempt(60), - before_sleep=tenacity.before_sleep_log(log, logging.WARNING), - reraise=True, -) -def wait_till_minio_responsive(minio_config: Dict[str, str]) -> bool: - client = Minio(**minio_config["client"]) - # TODO: improve as https://docs.min.io/docs/minio-monitoring-guide.html - if not client.bucket_exists("pytest"): - client.make_bucket("pytest") - client.remove_bucket("pytest") - return True + # cleanup upon tear-down + _ensure_remove_bucket(client, bucket_name) @pytest.fixture(scope="module") def bucket(minio_config: Dict[str, str], minio_service: Minio) -> str: bucket_name = minio_config["bucket_name"] - _remove_bucket(minio_service, bucket_name) + _ensure_remove_bucket(minio_service, bucket_name) minio_service.make_bucket(bucket_name) yield bucket_name - _remove_bucket(minio_service, bucket_name) + _ensure_remove_bucket(minio_service, bucket_name) From 14e315e38fcce810e03f1f7da0b50a567575bcaf Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 27 Apr 2021 10:22:17 +0200 Subject: [PATCH 26/27] review @sanderegg: renamed s3client --- services/storage/src/simcore_service_storage/dsm.py | 4 ++-- services/storage/src/simcore_service_storage/s3.py | 4 ++-- .../src/simcore_service_storage/s3wrapper/s3_client.py | 2 +- services/storage/tests/conftest.py | 4 ++-- services/storage/tests/s3wrapper/test_s3_client.py | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/services/storage/src/simcore_service_storage/dsm.py b/services/storage/src/simcore_service_storage/dsm.py index 9ca77ad77b1..3d3245b75aa 100644 --- a/services/storage/src/simcore_service_storage/dsm.py +++ b/services/storage/src/simcore_service_storage/dsm.py @@ -43,7 +43,7 @@ projects, ) from .s3 import get_config_s3 -from .s3wrapper.s3_client import S3Client +from .s3wrapper.s3_client import MinioClientWrapper from .settings import ( APP_CONFIG_KEY, APP_DB_ENGINE_KEY, @@ -141,7 +141,7 @@ class DataStorageManager: https://docs.minio.io/docs/minio-bucket-notification-guide.html """ - s3_client: S3Client + s3_client: MinioClientWrapper engine: Engine loop: object pool: ThreadPoolExecutor diff --git a/services/storage/src/simcore_service_storage/s3.py b/services/storage/src/simcore_service_storage/s3.py index 9875f58ded6..c4fb1781f7f 100644 --- a/services/storage/src/simcore_service_storage/s3.py +++ b/services/storage/src/simcore_service_storage/s3.py @@ -8,7 +8,7 @@ from aiohttp import web from tenacity import before_sleep_log, retry, stop_after_attempt, wait_fixed -from .s3wrapper.s3_client import S3Client +from .s3wrapper.s3_client import MinioClientWrapper from .settings import APP_CONFIG_KEY, APP_S3_KEY from .utils import RETRY_COUNT, RETRY_WAIT_SECS @@ -76,7 +76,7 @@ def setup(app: web.Application): s3_secret_key = s3_cfg["secret_key"] s3_secure = s3_cfg["secure"] - s3_client = S3Client( + s3_client = MinioClientWrapper( s3_endpoint, s3_access_key, s3_secret_key, secure=s3_secure == 1 ) app[APP_S3_KEY] = s3_client diff --git a/services/storage/src/simcore_service_storage/s3wrapper/s3_client.py b/services/storage/src/simcore_service_storage/s3wrapper/s3_client.py index 41b3b1521ec..e456849140d 100644 --- a/services/storage/src/simcore_service_storage/s3wrapper/s3_client.py +++ b/services/storage/src/simcore_service_storage/s3wrapper/s3_client.py @@ -8,7 +8,7 @@ log = logging.getLogger(__name__) -class S3Client: +class MinioClientWrapper: """Wrapper around minio""" def __init__( diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 988de856797..a9683bf675c 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -25,7 +25,7 @@ from simcore_service_storage.datcore_wrapper import DatcoreWrapper from simcore_service_storage.dsm import DataStorageManager, DatCoreApiToken from simcore_service_storage.models import FileMetaData -from simcore_service_storage.s3wrapper.s3_client import S3Client +from simcore_service_storage.s3wrapper.s3_client import MinioClientWrapper from simcore_service_storage.settings import SIMCORE_S3_STR from tests.utils import ( ACCESS_KEY, @@ -215,7 +215,7 @@ def minio_service(docker_services, docker_ip): @pytest.fixture(scope="module") def s3_client(minio_service): - s3_client = S3Client( + s3_client = MinioClientWrapper( endpoint=minio_service["endpoint"], access_key=minio_service["access_key"], secret_key=minio_service["secret_key"], diff --git a/services/storage/tests/s3wrapper/test_s3_client.py b/services/storage/tests/s3wrapper/test_s3_client.py index dad1ecc8cb1..2b171abb1d2 100644 --- a/services/storage/tests/s3wrapper/test_s3_client.py +++ b/services/storage/tests/s3wrapper/test_s3_client.py @@ -11,7 +11,7 @@ import pytest import requests -from simcore_service_storage.s3wrapper.s3_client import S3Client +from simcore_service_storage.s3wrapper.s3_client import MinioClientWrapper def is_responsive(url, code=200): @@ -49,7 +49,7 @@ def s3_client(docker_ip, docker_services): endpoint = "{ip}:{port}".format( ip=docker_ip, port=docker_services.port_for("minio", 9000) ) - s3_client = S3Client( + s3_client = MinioClientWrapper( endpoint, access_key="12345678", secret_key="12345678", secure=False ) return s3_client From b6950c06133ffd740a53b0aea3a3724b6bafc405 Mon Sep 17 00:00:00 2001 From: Pedro Crespo <32402063+pcrespov@users.noreply.github.com> Date: Tue, 27 Apr 2021 10:26:08 +0200 Subject: [PATCH 27/27] review @sanderegg: added annotations --- .../src/pytest_simcore/monkeypatch_extra.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/monkeypatch_extra.py b/packages/pytest-simcore/src/pytest_simcore/monkeypatch_extra.py index 0877702d785..ae63d68b130 100644 --- a/packages/pytest-simcore/src/pytest_simcore/monkeypatch_extra.py +++ b/packages/pytest-simcore/src/pytest_simcore/monkeypatch_extra.py @@ -2,14 +2,18 @@ # pylint: disable=unused-argument # pylint: disable=unused-variable +from typing import Iterator + import pytest +from _pytest.fixtures import FixtureRequest +from _pytest.monkeypatch import MonkeyPatch # Some extras to overcome https://github.com/pytest-dev/pytest/issues/363 @pytest.fixture(scope="session") -def monkeypatch_session(request): - from _pytest.monkeypatch import MonkeyPatch +def monkeypatch_session(request: FixtureRequest) -> Iterator[MonkeyPatch]: + assert request.scope == "session" mpatch_session = MonkeyPatch() yield mpatch_session @@ -17,10 +21,9 @@ def monkeypatch_session(request): @pytest.fixture(scope="module") -def monkeypatch_module(request): - from _pytest.monkeypatch import MonkeyPatch +def monkeypatch_module(request: FixtureRequest) -> Iterator[MonkeyPatch]: + assert request.scope == "module" mpatch_module = MonkeyPatch() yield mpatch_module - mpatch_module.undo()