diff --git a/.vscode/settings.template.json b/.vscode/settings.template.json index 13c745cda91..5c5aa8944ed 100644 --- a/.vscode/settings.template.json +++ b/.vscode/settings.template.json @@ -33,7 +33,6 @@ "./packages/postgres-database/src", "./packages/postgres-database/tests", "./packages/pytest-simcore/src", - "./packages/s3wrapper/src", "./packages/service-integration/src", "./packages/service-library/src", "./packages/simcore-sdk/src", diff --git a/packages/pytest-simcore/src/pytest_simcore/environment_configs.py b/packages/pytest-simcore/src/pytest_simcore/environment_configs.py index 7fd65124f16..c92a88e4a8f 100644 --- a/packages/pytest-simcore/src/pytest_simcore/environment_configs.py +++ b/packages/pytest-simcore/src/pytest_simcore/environment_configs.py @@ -1,4 +1,7 @@ # pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + from copy import deepcopy from pathlib import Path diff --git a/packages/pytest-simcore/src/pytest_simcore/minio_service.py b/packages/pytest-simcore/src/pytest_simcore/minio_service.py index e68b6ee32b1..e12c4bdb34e 100644 --- a/packages/pytest-simcore/src/pytest_simcore/minio_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/minio_service.py @@ -1,24 +1,36 @@ -import logging -import os -from copy import deepcopy +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable -# pylint:disable=unused-variable -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name +import logging from distutils.util import strtobool -from typing import Dict +from typing import Dict, Iterator import pytest import tenacity -from s3wrapper.s3_client import S3Client +from minio import Minio +from tenacity import Retrying from .helpers.utils_docker import get_ip, get_service_published_port log = logging.getLogger(__name__) +def _ensure_remove_bucket(client: Minio, bucket_name: str): + if client.bucket_exists(bucket_name): + # remove content + objs = client.list_objects(bucket_name, prefix=None, recursive=True) + errors = client.remove_objects(bucket_name, [o.object_name for o in objs]) + assert not list(errors) + # remove bucket + client.remove_bucket(bucket_name) + assert not client.bucket_exists(bucket_name) + + @pytest.fixture(scope="module") -def minio_config(docker_stack: Dict, devel_environ: Dict) -> Dict[str, str]: +def minio_config( + docker_stack: Dict, devel_environ: Dict, monkeypatch_module +) -> Dict[str, str]: assert "ops_minio" in docker_stack["services"] config = { @@ -32,49 +44,53 @@ def minio_config(docker_stack: Dict, devel_environ: Dict) -> Dict[str, str]: } # nodeports takes its configuration from env variables - old_environ = deepcopy(os.environ) for key, value in config["client"].items(): - os.environ[f"S3_{key.upper()}"] = str(value) - os.environ["S3_SECURE"] = devel_environ["S3_SECURE"] - os.environ["S3_BUCKET_NAME"] = config["bucket_name"] + monkeypatch_module.setenv(f"S3_{key.upper()}", str(value)) + + monkeypatch_module.setenv("S3_SECURE", devel_environ["S3_SECURE"]) + monkeypatch_module.setenv("S3_BUCKET_NAME", config["bucket_name"]) - yield config - # restore environ - os.environ = old_environ + return config @pytest.fixture(scope="module") -def minio_service(minio_config: Dict[str, str]) -> S3Client: - assert wait_till_minio_responsive(minio_config) +def minio_service(minio_config: Dict[str, str]) -> Iterator[Minio]: + + client = Minio(**minio_config["client"]) + + for attempt in Retrying( + wait=tenacity.wait_fixed(5), + stop=tenacity.stop_after_attempt(60), + before_sleep=tenacity.before_sleep_log(log, logging.WARNING), + reraise=True, + ): + with attempt: + # TODO: improve as https://docs.min.io/docs/minio-monitoring-guide.html + if not client.bucket_exists("pytest"): + client.make_bucket("pytest") + client.remove_bucket("pytest") - client = S3Client(**minio_config["client"]) - assert client.create_bucket(minio_config["bucket_name"]) + bucket_name = minio_config["bucket_name"] - yield client + # cleans up in case a failing tests left this bucket + _ensure_remove_bucket(client, bucket_name) - assert client.remove_bucket(minio_config["bucket_name"], delete_contents=True) + client.make_bucket(bucket_name) + assert client.bucket_exists(bucket_name) + yield client -@tenacity.retry( - wait=tenacity.wait_fixed(5), - stop=tenacity.stop_after_attempt(60), - before_sleep=tenacity.before_sleep_log(log, logging.INFO), - reraise=True, -) -def wait_till_minio_responsive(minio_config: Dict[str, str]) -> bool: - """Check if something responds to ``url`` """ - client = S3Client(**minio_config["client"]) - if client.create_bucket("pytest"): - client.remove_bucket("pytest") - return True - raise Exception(f"Minio not responding to {minio_config}") + # cleanup upon tear-down + _ensure_remove_bucket(client, bucket_name) @pytest.fixture(scope="module") -def bucket(minio_config: Dict[str, str], minio_service: S3Client) -> str: +def bucket(minio_config: Dict[str, str], minio_service: Minio) -> str: bucket_name = minio_config["bucket_name"] - minio_service.create_bucket(bucket_name, delete_contents_if_exists=True) + + _ensure_remove_bucket(minio_service, bucket_name) + minio_service.make_bucket(bucket_name) yield bucket_name - minio_service.remove_bucket(bucket_name, delete_contents=True) + _ensure_remove_bucket(minio_service, bucket_name) diff --git a/packages/pytest-simcore/src/pytest_simcore/monkeypatch_extra.py b/packages/pytest-simcore/src/pytest_simcore/monkeypatch_extra.py new file mode 100644 index 00000000000..ae63d68b130 --- /dev/null +++ b/packages/pytest-simcore/src/pytest_simcore/monkeypatch_extra.py @@ -0,0 +1,29 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + +from typing import Iterator + +import pytest +from _pytest.fixtures import FixtureRequest +from _pytest.monkeypatch import MonkeyPatch + +# Some extras to overcome https://github.com/pytest-dev/pytest/issues/363 + + +@pytest.fixture(scope="session") +def monkeypatch_session(request: FixtureRequest) -> Iterator[MonkeyPatch]: + assert request.scope == "session" + + mpatch_session = MonkeyPatch() + yield mpatch_session + mpatch_session.undo() + + +@pytest.fixture(scope="module") +def monkeypatch_module(request: FixtureRequest) -> Iterator[MonkeyPatch]: + assert request.scope == "module" + + mpatch_module = MonkeyPatch() + yield mpatch_module + mpatch_module.undo() diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py index 03921e63c4f..96e1c311d31 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py @@ -8,7 +8,7 @@ import aiohttp import pytest import tenacity -from s3wrapper.s3_client import S3Client +from minio import Minio from servicelib.minio_utils import MinioRetryPolicyUponInitialization from yarl import URL @@ -33,7 +33,7 @@ def storage_endpoint(docker_stack: Dict, devel_environ: Dict) -> URL: @pytest.fixture(scope="function") async def storage_service( - minio_service: S3Client, storage_endpoint: URL, docker_stack: Dict + minio_service: Minio, storage_endpoint: URL, docker_stack: Dict ) -> URL: await wait_till_storage_responsive(storage_endpoint) diff --git a/packages/s3wrapper/requirements/Makefile b/packages/s3wrapper/requirements/Makefile deleted file mode 100644 index 3f25442b790..00000000000 --- a/packages/s3wrapper/requirements/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# -# Targets to pip-compile requirements -# -include ../../../requirements/base.Makefile - -# Add here any extra explicit dependency: e.g. _migration.txt: _base.txt diff --git a/packages/s3wrapper/requirements/_base.in b/packages/s3wrapper/requirements/_base.in deleted file mode 100644 index 775bc9a2fc4..00000000000 --- a/packages/s3wrapper/requirements/_base.in +++ /dev/null @@ -1,7 +0,0 @@ -# -# Specifies third-party dependencies for 's3wrapper' -# --c ../../../requirements/constraints.txt - -urllib3 -minio diff --git a/packages/s3wrapper/requirements/_base.txt b/packages/s3wrapper/requirements/_base.txt deleted file mode 100644 index dd77bc481c0..00000000000 --- a/packages/s3wrapper/requirements/_base.txt +++ /dev/null @@ -1,25 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/_base.txt requirements/_base.in -# -certifi==2020.12.5 - # via minio -configparser==5.0.2 - # via minio -minio==6.0.2 - # via - # -c requirements/../../../requirements/constraints.txt - # -r requirements/_base.in -python-dateutil==2.8.1 - # via minio -pytz==2021.1 - # via minio -six==1.15.0 - # via python-dateutil -urllib3==1.26.4 - # via - # -c requirements/../../../requirements/constraints.txt - # -r requirements/_base.in - # minio diff --git a/packages/s3wrapper/requirements/_test.in b/packages/s3wrapper/requirements/_test.in deleted file mode 100644 index 49b4f6d0640..00000000000 --- a/packages/s3wrapper/requirements/_test.in +++ /dev/null @@ -1,23 +0,0 @@ -# -# Specifies dependencies required to run 's3wrapper' -# --c ../../../requirements/constraints.txt - -# Adds base AS CONSTRAINT specs, not requirement. -# - Resulting _text.txt is a frozen list of EXTRA packages for testing, besides _base.txt -# --c _base.txt - -# testing -coverage -pytest -pytest-cov -pytest-docker -pytest-runner - -# mockups/fixtures -requests - -# tools for CI -pylint -coveralls diff --git a/packages/s3wrapper/requirements/_test.txt b/packages/s3wrapper/requirements/_test.txt deleted file mode 100644 index 9b6150926f4..00000000000 --- a/packages/s3wrapper/requirements/_test.txt +++ /dev/null @@ -1,147 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/_test.txt requirements/_test.in -# -astroid==2.5.3 - # via pylint -attrs==20.3.0 - # via - # jsonschema - # pytest - # pytest-docker -bcrypt==3.2.0 - # via paramiko -cached-property==1.5.2 - # via docker-compose -certifi==2020.12.5 - # via - # -c requirements/_base.txt - # requests -cffi==1.14.5 - # via - # bcrypt - # cryptography - # pynacl -chardet==4.0.0 - # via requests -coverage==5.5 - # via - # -r requirements/_test.in - # coveralls - # pytest-cov -coveralls==3.0.1 - # via -r requirements/_test.in -cryptography==3.4.7 - # via - # -c requirements/../../../requirements/constraints.txt - # paramiko -distro==1.5.0 - # via docker-compose -docker-compose==1.29.1 - # via - # -c requirements/../../../requirements/constraints.txt - # pytest-docker -docker[ssh]==5.0.0 - # via docker-compose -dockerpty==0.4.1 - # via docker-compose -docopt==0.6.2 - # via - # coveralls - # docker-compose -idna==2.10 - # via - # -c requirements/../../../requirements/constraints.txt - # requests -importlib-metadata==4.0.1 - # via - # jsonschema - # pluggy - # pytest -iniconfig==1.1.1 - # via pytest -isort==5.8.0 - # via pylint -jsonschema==3.2.0 - # via docker-compose -lazy-object-proxy==1.6.0 - # via astroid -mccabe==0.6.1 - # via pylint -packaging==20.9 - # via pytest -paramiko==2.7.2 - # via docker -pluggy==0.13.1 - # via pytest -py==1.10.0 - # via pytest -pycparser==2.20 - # via cffi -pylint==2.7.4 - # via -r requirements/_test.in -pynacl==1.4.0 - # via paramiko -pyparsing==2.4.7 - # via packaging -pyrsistent==0.17.3 - # via jsonschema -pytest-cov==2.11.1 - # via -r requirements/_test.in -pytest-docker==0.10.1 - # via -r requirements/_test.in -pytest-runner==5.3.0 - # via -r requirements/_test.in -pytest==6.2.3 - # via - # -r requirements/_test.in - # pytest-cov - # pytest-docker -python-dotenv==0.17.0 - # via docker-compose -pyyaml==5.4.1 - # via - # -c requirements/../../../requirements/constraints.txt - # docker-compose -requests==2.25.1 - # via - # -r requirements/_test.in - # coveralls - # docker - # docker-compose -six==1.15.0 - # via - # -c requirements/_base.txt - # bcrypt - # dockerpty - # jsonschema - # pynacl - # websocket-client -texttable==1.6.3 - # via docker-compose -toml==0.10.2 - # via - # pylint - # pytest -typed-ast==1.4.3 - # via astroid -typing-extensions==3.7.4.3 - # via importlib-metadata -urllib3==1.26.4 - # via - # -c requirements/../../../requirements/constraints.txt - # -c requirements/_base.txt - # requests -websocket-client==0.58.0 - # via - # docker - # docker-compose -wrapt==1.12.1 - # via astroid -zipp==3.4.1 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/packages/s3wrapper/requirements/_tools.in b/packages/s3wrapper/requirements/_tools.in deleted file mode 100644 index 9d152d594d2..00000000000 --- a/packages/s3wrapper/requirements/_tools.in +++ /dev/null @@ -1,5 +0,0 @@ --c ../../../requirements/constraints.txt --c _base.txt --c _test.txt - --r ../../../requirements/devenv.txt diff --git a/packages/s3wrapper/requirements/_tools.txt b/packages/s3wrapper/requirements/_tools.txt deleted file mode 100644 index 92ee1df4663..00000000000 --- a/packages/s3wrapper/requirements/_tools.txt +++ /dev/null @@ -1,92 +0,0 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile --output-file=requirements/_tools.txt requirements/_tools.in -# -appdirs==1.4.4 - # via - # black - # virtualenv -black==20.8b1 - # via -r requirements/../../../requirements/devenv.txt -bump2version==1.0.1 - # via -r requirements/../../../requirements/devenv.txt -cfgv==3.2.0 - # via pre-commit -click==7.1.2 - # via - # black - # pip-tools -dataclasses==0.8 - # via black -distlib==0.3.1 - # via virtualenv -filelock==3.0.12 - # via virtualenv -identify==2.2.4 - # via pre-commit -importlib-metadata==4.0.1 - # via - # -c requirements/_test.txt - # pep517 - # pre-commit - # virtualenv -importlib-resources==5.1.2 - # via - # pre-commit - # virtualenv -isort==5.8.0 - # via - # -c requirements/_test.txt - # -r requirements/../../../requirements/devenv.txt -mypy-extensions==0.4.3 - # via black -nodeenv==1.6.0 - # via pre-commit -pathspec==0.8.1 - # via black -pep517==0.10.0 - # via pip-tools -pip-tools==6.1.0 - # via -r requirements/../../../requirements/devenv.txt -pre-commit==2.12.1 - # via -r requirements/../../../requirements/devenv.txt -pyyaml==5.4.1 - # via - # -c requirements/../../../requirements/constraints.txt - # -c requirements/_test.txt - # pre-commit -regex==2021.4.4 - # via black -six==1.15.0 - # via - # -c requirements/_base.txt - # -c requirements/_test.txt - # virtualenv -toml==0.10.2 - # via - # -c requirements/_test.txt - # black - # pep517 - # pre-commit -typed-ast==1.4.3 - # via - # -c requirements/_test.txt - # black -typing-extensions==3.7.4.3 - # via - # -c requirements/_test.txt - # black - # importlib-metadata -virtualenv==20.4.4 - # via pre-commit -zipp==3.4.1 - # via - # -c requirements/_test.txt - # importlib-metadata - # importlib-resources - # pep517 - -# The following packages are considered to be unsafe in a requirements file: -# pip diff --git a/packages/s3wrapper/requirements/dev.txt b/packages/s3wrapper/requirements/dev.txt deleted file mode 100644 index 7634b7285fd..00000000000 --- a/packages/s3wrapper/requirements/dev.txt +++ /dev/null @@ -1,14 +0,0 @@ -# Shortcut to install all packages needed to develop 's3wrapper' -# -# - As ci.txt but with current and repo packages in develop (edit) mode -# -# Usage: -# pip install -r requirements/dev.txt -# - -# installs base + tests requirements --r _base.txt --r _test.txt - -# installs current package --e . diff --git a/packages/s3wrapper/setup.py b/packages/s3wrapper/setup.py deleted file mode 100644 index 250715cb871..00000000000 --- a/packages/s3wrapper/setup.py +++ /dev/null @@ -1,33 +0,0 @@ -import re -import sys -from pathlib import Path - -from setuptools import setup - -here = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent - - -def read_reqs(reqs_path: Path): - return re.findall( - r"(^[^#\n-][\w\[,\]]+[-~>=<.\w]*)", reqs_path.read_text(), re.MULTILINE - ) - - -install_requirements = read_reqs( - here / "requirements" / "_base.in" -) # WEAK requirements -test_requirements = read_reqs( - here / "requirements" / "_test.txt" -) # STRONG requirements - -setup( - name="s3wrapper", - version="0.1.0", - package_dir={"": "src"}, - packages=["s3wrapper"], - python_requires=">=3.6", - install_requires=install_requirements, - tests_require=test_requirements, - extras_require={"test": test_requirements}, - test_suite="tests", -) diff --git a/packages/s3wrapper/tests/conftest.py b/packages/s3wrapper/tests/conftest.py deleted file mode 100644 index 515c8f6e53f..00000000000 --- a/packages/s3wrapper/tests/conftest.py +++ /dev/null @@ -1,9 +0,0 @@ -import pytest -import os - -# pylint:disable=unused-argument - -@pytest.fixture(scope='session') -def docker_compose_file(pytestconfig): - my_path = os.path.join(os.path.dirname(__file__), 'docker-compose.yml') - return my_path diff --git a/packages/s3wrapper/tests/docker-compose.yml b/packages/s3wrapper/tests/docker-compose.yml deleted file mode 100644 index d0a46aaa572..00000000000 --- a/packages/s3wrapper/tests/docker-compose.yml +++ /dev/null @@ -1,10 +0,0 @@ -version: "3.8" -services: - minio: - image: minio/minio - environment: - - MINIO_ACCESS_KEY=12345678 - - MINIO_SECRET_KEY=12345678 - ports: - - "9001:9000" - command: server /data diff --git a/packages/simcore-sdk/requirements/_base.in b/packages/simcore-sdk/requirements/_base.in index a2322eefab0..9bf85f5bde5 100644 --- a/packages/simcore-sdk/requirements/_base.in +++ b/packages/simcore-sdk/requirements/_base.in @@ -3,7 +3,6 @@ # -c ../../../requirements/constraints.txt -r ../../../packages/postgres-database/requirements/_base.in --r ../../../packages/s3wrapper/requirements/_base.in -r ../../../packages/service-library/requirements/_base.in -r ../../../packages/models-library/requirements/_base.in diff --git a/packages/simcore-sdk/requirements/_base.txt b/packages/simcore-sdk/requirements/_base.txt index 57e231639a6..c3297750170 100644 --- a/packages/simcore-sdk/requirements/_base.txt +++ b/packages/simcore-sdk/requirements/_base.txt @@ -21,7 +21,6 @@ aiozipkin==0.7.1 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in @@ -36,12 +35,8 @@ attrs==20.3.0 # aiohttp # jsonschema # openapi-core -certifi==2020.12.5 - # via minio chardet==4.0.0 # via aiohttp -configparser==5.0.2 - # via minio dataclasses==0.8 # via pydantic decorator==4.4.2 @@ -75,14 +70,6 @@ lazy-object-proxy==1.4.3 # via # -r requirements/../../../packages/service-library/requirements/_base.in # openapi-core -minio==6.0.2 - # via - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt - # -r requirements/../../../packages/s3wrapper/requirements/_base.in multidict==5.1.0 # via # aiohttp @@ -114,15 +101,10 @@ pyparsing==2.4.7 # via packaging pyrsistent==0.17.3 # via jsonschema -python-dateutil==2.8.1 - # via minio -pytz==2021.1 - # via minio pyyaml==5.4.1 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in @@ -135,13 +117,11 @@ six==1.15.0 # openapi-core # openapi-schema-validator # openapi-spec-validator - # python-dateutil # tenacity sqlalchemy[postgresql_psycopg2binary]==1.3.24 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_base.in @@ -169,15 +149,6 @@ typing-extensions==3.7.4.3 # yarl ujson==4.0.2 # via -r requirements/../../../packages/service-library/requirements/_base.in -urllib3==1.26.4 - # via - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt - # -r requirements/../../../packages/s3wrapper/requirements/_base.in - # minio werkzeug==1.0.1 # via -r requirements/../../../packages/service-library/requirements/_base.in yarl==1.6.3 diff --git a/packages/simcore-sdk/requirements/_test.in b/packages/simcore-sdk/requirements/_test.in index e13c8cd7fcf..1d1370eb305 100644 --- a/packages/simcore-sdk/requirements/_test.in +++ b/packages/simcore-sdk/requirements/_test.in @@ -1,7 +1,7 @@ # # Specifies dependencies required to run 'simcore-sdk' # - +-c ../../../requirements/constraints.txt # Adds base AS CONSTRAINT specs, not requirement. # - Resulting _text.txt is a frozen list of EXTRA packages for testing, besides _base.txt # @@ -26,6 +26,7 @@ requests docker python-dotenv faker +minio # tools for CI pylint diff --git a/packages/simcore-sdk/requirements/_test.txt b/packages/simcore-sdk/requirements/_test.txt index 377b85b8578..7c8174a2b86 100644 --- a/packages/simcore-sdk/requirements/_test.txt +++ b/packages/simcore-sdk/requirements/_test.txt @@ -28,13 +28,15 @@ attrs==20.3.0 # pytest certifi==2020.12.5 # via - # -c requirements/_base.txt + # minio # requests chardet==4.0.0 # via # -c requirements/_base.txt # aiohttp # requests +configparser==5.0.2 + # via minio coverage==5.5 # via # -r requirements/_test.in @@ -81,6 +83,10 @@ markupsafe==1.1.1 # via mako mccabe==0.6.1 # via pylint +minio==6.0.2 + # via + # -c requirements/../../../requirements/constraints.txt + # -r requirements/_test.in multidict==5.1.0 # via # -c requirements/_base.txt @@ -140,13 +146,15 @@ pytest==6.2.3 # pytest-xdist python-dateutil==2.8.1 # via - # -c requirements/_base.txt # alembic # faker + # minio python-dotenv==0.17.0 # via -r requirements/_test.in python-editor==1.0.4 # via alembic +pytz==2021.1 + # via minio requests==2.25.1 # via # -r requirements/_test.in @@ -159,6 +167,7 @@ six==1.15.0 # websocket-client sqlalchemy[postgresql_psycopg2binary]==1.3.24 # via + # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # alembic termcolor==1.1.0 @@ -179,7 +188,8 @@ typing-extensions==3.7.4.3 # yarl urllib3==1.26.4 # via - # -c requirements/_base.txt + # -c requirements/../../../requirements/constraints.txt + # minio # requests websocket-client==0.58.0 # via docker diff --git a/packages/simcore-sdk/requirements/_tools.in b/packages/simcore-sdk/requirements/_tools.in index 61b62e5647c..92dd68e474f 100644 --- a/packages/simcore-sdk/requirements/_tools.in +++ b/packages/simcore-sdk/requirements/_tools.in @@ -1,3 +1,5 @@ +-c ../../../requirements/constraints.txt + -c _base.txt -c _test.txt diff --git a/packages/simcore-sdk/requirements/_tools.txt b/packages/simcore-sdk/requirements/_tools.txt index 6f19871e7e1..110b5c828d8 100644 --- a/packages/simcore-sdk/requirements/_tools.txt +++ b/packages/simcore-sdk/requirements/_tools.txt @@ -57,6 +57,7 @@ pre-commit==2.12.1 # via -r requirements/../../../requirements/devenv.txt pyyaml==5.4.1 # via + # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # pre-commit regex==2021.4.4 diff --git a/packages/simcore-sdk/requirements/ci.txt b/packages/simcore-sdk/requirements/ci.txt index c8ebd862753..29a00179eae 100644 --- a/packages/simcore-sdk/requirements/ci.txt +++ b/packages/simcore-sdk/requirements/ci.txt @@ -12,7 +12,6 @@ # installs this repo's packages ../postgres-database/[migration] -../s3wrapper/ ../service-library/ ../pytest-simcore/ ../models-library/ diff --git a/packages/simcore-sdk/requirements/dev.txt b/packages/simcore-sdk/requirements/dev.txt index 372192e1269..f31724f61f7 100644 --- a/packages/simcore-sdk/requirements/dev.txt +++ b/packages/simcore-sdk/requirements/dev.txt @@ -13,7 +13,6 @@ # installs this repo's packages -e ../postgres-database/[migration] --e ../s3wrapper/ -e ../service-library/ -e ../pytest-simcore/ -e ../models-library/ diff --git a/packages/simcore-sdk/setup.py b/packages/simcore-sdk/setup.py index 63b6e846f14..92589626549 100644 --- a/packages/simcore-sdk/setup.py +++ b/packages/simcore-sdk/setup.py @@ -18,7 +18,6 @@ def read_reqs(reqs_path: Path): "simcore-postgres-database[migration]", "simcore-service-library", "simcore-models-library", - "s3wrapper", "simcore-service-storage-sdk", ] diff --git a/packages/simcore-sdk/tests/conftest.py b/packages/simcore-sdk/tests/conftest.py index bc2461990ec..268d755d711 100644 --- a/packages/simcore-sdk/tests/conftest.py +++ b/packages/simcore-sdk/tests/conftest.py @@ -16,13 +16,14 @@ # FIXTURES pytest_plugins = [ - "pytest_simcore.repository_paths", "pytest_simcore.docker_compose", "pytest_simcore.docker_swarm", - "pytest_simcore.postgres_service", "pytest_simcore.minio_service", - "pytest_simcore.simcore_storage_service", + "pytest_simcore.monkeypatch_extra", + "pytest_simcore.postgres_service", + "pytest_simcore.repository_paths", "pytest_simcore.services_api_mocks_for_aiohttp_clients", + "pytest_simcore.simcore_storage_service", ] diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 4b59c04904a..7bd384bac36 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -23,7 +23,9 @@ httpx<0.15.0 # [https://github.com/encode/httpx/releases/tag/0.15.0] due to PR#1285 [https://github.com/encode/httpx/pull/1285] respx<0.13.0 -# Breaking changes. TODO: review ASAP + +# limited by aiobotocore in storage. In addition, pytest_simcore.minio_service uses minio +# Version 7 changes API! Has Breaking changes TODO: review ASAP minio<7.0.0 diff --git a/services/api-server/requirements/_base.txt b/services/api-server/requirements/_base.txt index 37cf9a421d6..6819021fb41 100644 --- a/services/api-server/requirements/_base.txt +++ b/services/api-server/requirements/_base.txt @@ -31,7 +31,6 @@ aiozipkin==0.7.1 # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -63,7 +62,6 @@ attrs==20.3.0 certifi==2020.12.5 # via # httpx - # minio # requests cffi==1.14.4 # via cryptography @@ -74,8 +72,6 @@ chardet==3.0.4 # requests click==7.1.2 # via uvicorn -configparser==5.0.2 - # via minio contextvars==2.4 # via sniffio cryptography==3.4.5 @@ -85,7 +81,6 @@ cryptography==3.4.5 # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -125,7 +120,6 @@ httpx==0.14.3 # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -172,18 +166,6 @@ lazy-object-proxy==1.4.3 # openapi-core markupsafe==1.1.1 # via jinja2 -minio==6.0.2 - # via - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt - # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/_base.in multidict==5.1.0 # via # aiohttp @@ -234,16 +216,12 @@ pyparsing==2.4.7 # via packaging pyrsistent==0.17.3 # via jsonschema -python-dateutil==2.8.1 - # via minio python-dotenv==0.15.0 # via # pydantic # uvicorn python-multipart==0.0.5 # via fastapi -pytz==2021.1 - # via minio pyyaml==5.4.1 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -251,7 +229,6 @@ pyyaml==5.4.1 # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -277,7 +254,6 @@ six==1.15.0 # openapi-core # openapi-schema-validator # openapi-spec-validator - # python-dateutil # python-multipart # tenacity sniffio==1.2.0 @@ -291,7 +267,6 @@ sqlalchemy[postgresql_psycopg2binary]==1.3.22 # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -338,12 +313,9 @@ urllib3==1.26.3 # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt - # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/_base.in - # minio # requests uvicorn[standard]==0.13.3 # via fastapi @@ -354,7 +326,6 @@ uvloop==0.14.0 ; python_version < "3.7" # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt diff --git a/services/api-server/requirements/_test.txt b/services/api-server/requirements/_test.txt index 8fab363e42f..65d1f54be23 100644 --- a/services/api-server/requirements/_test.txt +++ b/services/api-server/requirements/_test.txt @@ -190,7 +190,6 @@ pytest==6.2.3 # pytest-mock python-dateutil==2.8.1 # via - # -c requirements/_base.txt # alembic # faker python-dotenv==0.15.0 diff --git a/services/sidecar/requirements/_base.in b/services/sidecar/requirements/_base.in index fab6e675f5f..4b9dbd6ab45 100644 --- a/services/sidecar/requirements/_base.in +++ b/services/sidecar/requirements/_base.in @@ -11,7 +11,6 @@ # -c ../../../packages/models-library/requirements/_base.in -c ../../../packages/postgres-database/requirements/_base.in --c ../../../packages/s3wrapper/requirements/_base.in -c ../../../packages/service-library/requirements/_base.in diff --git a/services/sidecar/requirements/_base.txt b/services/sidecar/requirements/_base.txt index b604a20eaa1..366f3c14682 100644 --- a/services/sidecar/requirements/_base.txt +++ b/services/sidecar/requirements/_base.txt @@ -107,7 +107,6 @@ sqlalchemy[postgresql_psycopg2binary]==1.3.19 # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/_base.in - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/_base.in # -c requirements/../../../requirements/constraints.txt @@ -126,8 +125,6 @@ urllib3==1.25.11 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/_base.in # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/_base.in diff --git a/services/sidecar/requirements/_packages.in b/services/sidecar/requirements/_packages.in index 266b3e9c285..1cb39faa9b5 100644 --- a/services/sidecar/requirements/_packages.in +++ b/services/sidecar/requirements/_packages.in @@ -6,5 +6,4 @@ -r ../../../packages/models-library/requirements/_base.in -r ../../../packages/postgres-database/requirements/_base.in --r ../../../packages/s3wrapper/requirements/_base.in -r ../../../packages/service-library/requirements/_base.in diff --git a/services/sidecar/requirements/_packages.txt b/services/sidecar/requirements/_packages.txt index 2effc8506df..98b02e0769b 100644 --- a/services/sidecar/requirements/_packages.txt +++ b/services/sidecar/requirements/_packages.txt @@ -19,7 +19,6 @@ aiozipkin==0.7.1 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in @@ -35,14 +34,10 @@ attrs==20.2.0 # aiohttp # jsonschema # openapi-core -certifi==2020.6.20 - # via minio chardet==3.0.4 # via # -c requirements/_base.txt # aiohttp -configparser==5.0.0 - # via minio dataclasses==0.7 # via # -c requirements/_base.txt @@ -82,14 +77,6 @@ lazy-object-proxy==1.4.3 # via # -r requirements/../../../packages/service-library/requirements/_base.in # openapi-core -minio==6.0.0 - # via - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt - # -r requirements/../../../packages/s3wrapper/requirements/_base.in multidict==4.7.6 # via # -c requirements/_base.txt @@ -114,17 +101,10 @@ pydantic[email]==1.7.2 # -r requirements/../../../packages/service-library/requirements/_base.in pyrsistent==0.17.3 # via jsonschema -python-dateutil==2.8.1 - # via minio -pytz==2020.1 - # via - # -c requirements/_base.txt - # minio pyyaml==5.3.1 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in @@ -136,13 +116,11 @@ six==1.15.0 # jsonschema # openapi-core # openapi-spec-validator - # python-dateutil # tenacity sqlalchemy[postgresql_psycopg2binary]==1.3.19 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -164,16 +142,6 @@ typing-extensions==3.7.4.3 # yarl ujson==3.2.0 # via -r requirements/../../../packages/service-library/requirements/_base.in -urllib3==1.25.11 - # via - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt - # -c requirements/_base.txt - # -r requirements/../../../packages/s3wrapper/requirements/_base.in - # minio werkzeug==1.0.1 # via -r requirements/../../../packages/service-library/requirements/_base.in yarl==1.5.1 diff --git a/services/sidecar/requirements/_test.in b/services/sidecar/requirements/_test.in index 16316a981f0..7c570bef861 100644 --- a/services/sidecar/requirements/_test.in +++ b/services/sidecar/requirements/_test.in @@ -26,6 +26,7 @@ aiopg docker python-dotenv faker +minio # tools for CI pylint diff --git a/services/sidecar/requirements/_test.txt b/services/sidecar/requirements/_test.txt index 0f13db325a1..ba17ad3f1e0 100644 --- a/services/sidecar/requirements/_test.txt +++ b/services/sidecar/requirements/_test.txt @@ -9,7 +9,7 @@ aiohttp==3.7.3 # -c requirements/_base.txt # -c requirements/_packages.txt # pytest-aiohttp -aiopg[sa]==1.2.1 +aiopg==1.2.1 # via # -c requirements/_base.txt # -c requirements/_packages.txt @@ -32,7 +32,7 @@ attrs==20.2.0 # pytest certifi==2020.6.20 # via - # -c requirements/_packages.txt + # minio # requests chardet==3.0.4 # via @@ -40,6 +40,8 @@ chardet==3.0.4 # -c requirements/_packages.txt # aiohttp # requests +configparser==5.0.2 + # via minio coverage==5.5 # via # -r requirements/_test.in @@ -87,6 +89,10 @@ markupsafe==1.1.1 # via mako mccabe==0.6.1 # via pylint +minio==6.0.2 + # via + # -c requirements/../../../requirements/constraints.txt + # -r requirements/_test.in multidict==4.7.6 # via # -c requirements/_base.txt @@ -144,13 +150,17 @@ pytest==6.2.3 # pytest-sugar python-dateutil==2.8.1 # via - # -c requirements/_packages.txt # alembic # faker + # minio python-dotenv==0.17.0 # via -r requirements/_test.in python-editor==1.0.4 # via alembic +pytz==2020.1 + # via + # -c requirements/_base.txt + # minio requests==2.25.1 # via # coveralls @@ -189,7 +199,7 @@ urllib3==1.25.11 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt - # -c requirements/_packages.txt + # minio # requests websocket-client==0.58.0 # via docker diff --git a/services/sidecar/requirements/ci.txt b/services/sidecar/requirements/ci.txt index a9272de2df8..f46cf54bc34 100644 --- a/services/sidecar/requirements/ci.txt +++ b/services/sidecar/requirements/ci.txt @@ -15,7 +15,6 @@ ../../packages/models-library/ ../../packages/postgres-database/ ../../packages/pytest-simcore/ -../../packages/s3wrapper/ ../../packages/service-library/ ../../packages/simcore-sdk/ ../../services/storage/client-sdk/python/ diff --git a/services/sidecar/requirements/dev.txt b/services/sidecar/requirements/dev.txt index d904068128c..5b3c7992244 100644 --- a/services/sidecar/requirements/dev.txt +++ b/services/sidecar/requirements/dev.txt @@ -16,7 +16,6 @@ -e ../../packages/models-library/ -e ../../packages/postgres-database/ -e ../../packages/pytest-simcore/ --e ../../packages/s3wrapper/ -e ../../packages/service-library/ -e ../../packages/simcore-sdk/ -e ../../services/storage/client-sdk/python/ diff --git a/services/sidecar/requirements/prod.txt b/services/sidecar/requirements/prod.txt index a239271cfbf..1a3909a3782 100644 --- a/services/sidecar/requirements/prod.txt +++ b/services/sidecar/requirements/prod.txt @@ -13,7 +13,6 @@ # installs this repo's packages ../../packages/models-library/ ../../packages/postgres-database/ -../../packages/s3wrapper/ ../../packages/service-library/ ../../packages/simcore-sdk/ ../../services/storage/client-sdk/python/ diff --git a/services/sidecar/setup.py b/services/sidecar/setup.py index 4bdab22da30..ce57988a1c7 100644 --- a/services/sidecar/setup.py +++ b/services/sidecar/setup.py @@ -23,7 +23,6 @@ def read_reqs(reqs_path: Path): version = (current_dir / "VERSION").read_text().strip() install_requirements = read_reqs(current_dir / "requirements" / "_base.txt") + [ - "s3wrapper==0.1.0", "simcore-postgres-database", "simcore-sdk", "simcore-service-library", diff --git a/services/sidecar/tests/conftest.py b/services/sidecar/tests/conftest.py index 1ff888eaa38..93b16284de7 100644 --- a/services/sidecar/tests/conftest.py +++ b/services/sidecar/tests/conftest.py @@ -1,6 +1,7 @@ -# pylint:disable=unused-variable -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + import sys from pathlib import Path @@ -8,11 +9,12 @@ import simcore_service_sidecar pytest_plugins = [ - "pytest_simcore.repository_paths", "pytest_simcore.docker_compose", "pytest_simcore.docker_swarm", - "pytest_simcore.rabbit_service", + "pytest_simcore.monkeypatch_extra", "pytest_simcore.postgres_service", + "pytest_simcore.rabbit_service", + "pytest_simcore.repository_paths", ] current_dir = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent diff --git a/services/storage/requirements/_base.in b/services/storage/requirements/_base.in index e77a92e93b5..6f67515bbd5 100644 --- a/services/storage/requirements/_base.in +++ b/services/storage/requirements/_base.in @@ -1,6 +1,5 @@ -c ../../../requirements/constraints.txt --r ../../../packages/s3wrapper/requirements/_base.in -r ../../../packages/postgres-database/requirements/_base.in -r ../../../packages/service-library/requirements/_base.in -r ../../../packages/models-library/requirements/_base.in @@ -21,3 +20,5 @@ aiopg[sa] tenacity semantic_version click +minio +urllib3 diff --git a/services/storage/requirements/_base.txt b/services/storage/requirements/_base.txt index 9a876e2bf90..f180d68162b 100644 --- a/services/storage/requirements/_base.txt +++ b/services/storage/requirements/_base.txt @@ -29,7 +29,6 @@ aiozipkin==0.7.1 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in @@ -115,15 +114,14 @@ markupsafe==1.1.1 # via # aiohttp-swagger # jinja2 -minio==6.0.0 +minio==6.0.2 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt - # -r requirements/../../../packages/s3wrapper/requirements/_base.in -multidict==5.0.0 + # -r requirements/_base.in +multidict==5.1.0 # via # aiohttp # yarl @@ -161,7 +159,6 @@ pyyaml==5.3.1 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_base.in @@ -189,7 +186,6 @@ sqlalchemy[postgresql_psycopg2binary]==1.3.20 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_base.in @@ -216,10 +212,9 @@ urllib3==1.25.11 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt - # -r requirements/../../../packages/s3wrapper/requirements/_base.in + # -r requirements/_base.in # botocore # minio # requests diff --git a/services/storage/requirements/_test.txt b/services/storage/requirements/_test.txt index 91b988fd93b..55c102c9176 100644 --- a/services/storage/requirements/_test.txt +++ b/services/storage/requirements/_test.txt @@ -100,7 +100,7 @@ lazy-object-proxy==1.4.3 # astroid mccabe==0.6.1 # via pylint -multidict==5.0.0 +multidict==5.1.0 # via # -c requirements/_base.txt # aiohttp diff --git a/services/storage/requirements/ci.txt b/services/storage/requirements/ci.txt index 822d88ed2a2..d04970edace 100644 --- a/services/storage/requirements/ci.txt +++ b/services/storage/requirements/ci.txt @@ -11,7 +11,6 @@ -r _test.txt # installs this repo's packages -../../packages/s3wrapper/ ../../packages/postgres-database/ ../../packages/service-library/ ../../packages/models-library/ diff --git a/services/storage/requirements/dev.txt b/services/storage/requirements/dev.txt index d2a827ff30c..b2e5c4ac9fe 100644 --- a/services/storage/requirements/dev.txt +++ b/services/storage/requirements/dev.txt @@ -12,7 +12,6 @@ -r _tools.txt # installs this repo's packages --e ../../packages/s3wrapper/ -e ../../packages/postgres-database/ -e ../../packages/service-library -e ../../packages/models-library diff --git a/services/storage/requirements/prod.txt b/services/storage/requirements/prod.txt index 2de63f3f375..ff28edf7af0 100644 --- a/services/storage/requirements/prod.txt +++ b/services/storage/requirements/prod.txt @@ -10,7 +10,6 @@ -r _base.txt # installs this repo's packages -../../packages/s3wrapper/ ../../packages/postgres-database/ ../../packages/models-library/ ../../packages/service-library diff --git a/services/storage/setup.py b/services/storage/setup.py index 881f8c3a553..0e2cf63657f 100644 --- a/services/storage/setup.py +++ b/services/storage/setup.py @@ -20,7 +20,6 @@ def read_reqs(reqs_path: Path): install_requirements = read_reqs(here / "requirements" / "_base.txt") + [ - "s3wrapper==0.1.0", "simcore-postgres-database", "simcore-service-library", "simcore-models-library", diff --git a/services/storage/src/simcore_service_storage/dsm.py b/services/storage/src/simcore_service_storage/dsm.py index edaedfac605..3d3245b75aa 100644 --- a/services/storage/src/simcore_service_storage/dsm.py +++ b/services/storage/src/simcore_service_storage/dsm.py @@ -20,7 +20,6 @@ from aiopg.sa import Engine from aiopg.sa.result import RowProxy from blackfynn.base import UnauthorizedException -from s3wrapper.s3_client import S3Client from servicelib.aiopg_utils import DBAPIError, PostgresRetryPolicyUponOperation from servicelib.client_session import get_client_session from servicelib.utils import fire_and_forget_task @@ -44,6 +43,7 @@ projects, ) from .s3 import get_config_s3 +from .s3wrapper.s3_client import MinioClientWrapper from .settings import ( APP_CONFIG_KEY, APP_DB_ENGINE_KEY, @@ -141,7 +141,7 @@ class DataStorageManager: https://docs.minio.io/docs/minio-bucket-notification-guide.html """ - s3_client: S3Client + s3_client: MinioClientWrapper engine: Engine loop: object pool: ThreadPoolExecutor diff --git a/services/storage/src/simcore_service_storage/s3.py b/services/storage/src/simcore_service_storage/s3.py index f54bc7d5a81..c4fb1781f7f 100644 --- a/services/storage/src/simcore_service_storage/s3.py +++ b/services/storage/src/simcore_service_storage/s3.py @@ -6,9 +6,9 @@ from typing import Dict from aiohttp import web -from s3wrapper.s3_client import S3Client from tenacity import before_sleep_log, retry, stop_after_attempt, wait_fixed +from .s3wrapper.s3_client import MinioClientWrapper from .settings import APP_CONFIG_KEY, APP_S3_KEY from .utils import RETRY_COUNT, RETRY_WAIT_SECS @@ -76,7 +76,7 @@ def setup(app: web.Application): s3_secret_key = s3_cfg["secret_key"] s3_secure = s3_cfg["secure"] - s3_client = S3Client( + s3_client = MinioClientWrapper( s3_endpoint, s3_access_key, s3_secret_key, secure=s3_secure == 1 ) app[APP_S3_KEY] = s3_client diff --git a/packages/s3wrapper/src/s3wrapper/__init__.py b/services/storage/src/simcore_service_storage/s3wrapper/__init__.py similarity index 100% rename from packages/s3wrapper/src/s3wrapper/__init__.py rename to services/storage/src/simcore_service_storage/s3wrapper/__init__.py diff --git a/packages/s3wrapper/src/s3wrapper/s3_client.py b/services/storage/src/simcore_service_storage/s3wrapper/s3_client.py similarity index 93% rename from packages/s3wrapper/src/s3wrapper/s3_client.py rename to services/storage/src/simcore_service_storage/s3wrapper/s3_client.py index 7f483e92533..e456849140d 100644 --- a/packages/s3wrapper/src/s3wrapper/s3_client.py +++ b/services/storage/src/simcore_service_storage/s3wrapper/s3_client.py @@ -7,9 +7,9 @@ log = logging.getLogger(__name__) -class S3Client: - """ Wrapper around minio - """ + +class MinioClientWrapper: + """Wrapper around minio""" def __init__( self, @@ -81,16 +81,16 @@ def list_buckets(self): return [] def upload_file(self, bucket_name, object_name, filepath, metadata=None): - """ Note + """Note - metadata are special, you need to use the - 'X-Amz-Meta' standard, i.e: - - key and value must be strings - - and the keys are case insensitive: + metadata are special, you need to use the + 'X-Amz-Meta' standard, i.e: + - key and value must be strings + - and the keys are case insensitive: - key1 -- > Key1 - key_one --> Key_one - key-one --> Key-One + key1 -- > Key1 + key_one --> Key_one + key-one --> Key-One """ try: @@ -158,8 +158,7 @@ def remove_objects(self, bucket_name, objects): return True def exists_object(self, bucket_name, object_name, recursive=False): - """ This seems to be pretty heavy, should be used with care - """ + """This seems to be pretty heavy, should be used with care""" try: objects = self.list_objects(bucket_name, recursive=recursive) for obj in objects: diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index 4e34b618ec7..a9683bf675c 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -25,6 +25,7 @@ from simcore_service_storage.datcore_wrapper import DatcoreWrapper from simcore_service_storage.dsm import DataStorageManager, DatCoreApiToken from simcore_service_storage.models import FileMetaData +from simcore_service_storage.s3wrapper.s3_client import MinioClientWrapper from simcore_service_storage.settings import SIMCORE_S3_STR from tests.utils import ( ACCESS_KEY, @@ -213,9 +214,8 @@ def minio_service(docker_services, docker_ip): @pytest.fixture(scope="module") def s3_client(minio_service): - from s3wrapper.s3_client import S3Client - s3_client = S3Client( + s3_client = MinioClientWrapper( endpoint=minio_service["endpoint"], access_key=minio_service["access_key"], secret_key=minio_service["secret_key"], diff --git a/packages/s3wrapper/tests/__init__.py b/services/storage/tests/s3wrapper/__init__.py similarity index 100% rename from packages/s3wrapper/tests/__init__.py rename to services/storage/tests/s3wrapper/__init__.py diff --git a/packages/s3wrapper/tests/test_minio.py b/services/storage/tests/s3wrapper/test_minio.py similarity index 91% rename from packages/s3wrapper/tests/test_minio.py rename to services/storage/tests/s3wrapper/test_minio.py index 8387c571896..9f33e14b467 100644 --- a/packages/s3wrapper/tests/test_minio.py +++ b/services/storage/tests/s3wrapper/test_minio.py @@ -1,11 +1,12 @@ -import requests import pytest +import requests # pylint:disable=unused-import from pytest_docker import docker_ip, docker_services # pylint:disable=redefined-outer-name + def is_responsive(url, code=200): """Check if something responds to ``url``.""" try: @@ -14,17 +15,18 @@ def is_responsive(url, code=200): return True except requests.exceptions.RequestException as _e: pass - + return False + @pytest.mark.enable_travis def test_minio(docker_ip, docker_services): """wait for minio to be up""" # Build URL to service listening on random port. - url = 'http://%s:%d/' % ( + url = "http://%s:%d/" % ( docker_ip, - docker_services.port_for('minio', 9000), + docker_services.port_for("minio", 9000), ) # Wait until service is responsive. docker_services.wait_until_responsive( diff --git a/packages/s3wrapper/tests/test_s3_client.py b/services/storage/tests/s3wrapper/test_s3_client.py similarity index 84% rename from packages/s3wrapper/tests/test_s3_client.py rename to services/storage/tests/s3wrapper/test_s3_client.py index 5c799bb626c..2b171abb1d2 100644 --- a/packages/s3wrapper/tests/test_s3_client.py +++ b/services/storage/tests/s3wrapper/test_s3_client.py @@ -1,22 +1,19 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + import filecmp import os -import uuid - -import pytest -import urllib import time +import urllib +import uuid from datetime import timedelta -from s3wrapper.s3_client import S3Client - +import pytest import requests - -# pylint:disable=unused-import -from pytest_docker import docker_ip, docker_services +from simcore_service_storage.s3wrapper.s3_client import MinioClientWrapper -# pylint:disable=redefined-outer-name - def is_responsive(url, code=200): """Check if something responds to ``url``.""" try: @@ -27,14 +24,15 @@ def is_responsive(url, code=200): pass return False + @pytest.fixture(scope="module") def s3_client(docker_ip, docker_services): """wait for minio to be up""" # Build URL to service listening on random port. - url = 'http://%s:%d/' % ( + url = "http://%s:%d/" % ( docker_ip, - docker_services.port_for('minio', 9000), + docker_services.port_for("minio", 9000), ) # Wait until service is responsive. @@ -48,36 +46,45 @@ def s3_client(docker_ip, docker_services): response = requests.get(url) assert response.status_code == 403 - endpoint = '{ip}:{port}'.format(ip=docker_ip, port=docker_services.port_for('minio', 9000)) - access_key = "12345678" - secret_key = "12345678" - secure = False - s3_client = S3Client(endpoint, access_key, secret_key, secure) + endpoint = "{ip}:{port}".format( + ip=docker_ip, port=docker_services.port_for("minio", 9000) + ) + s3_client = MinioClientWrapper( + endpoint, access_key="12345678", secret_key="12345678", secure=False + ) return s3_client + @pytest.fixture() def bucket(s3_client, request): bucket_name = "simcore-test" s3_client.create_bucket(bucket_name, delete_contents_if_exists=True) + def fin(): s3_client.remove_bucket(bucket_name, delete_contents=True) + request.addfinalizer(fin) return bucket_name + @pytest.fixture(scope="function") def text_files(tmpdir_factory): def _create_files(N): filepaths = [] for _i in range(N): name = str(uuid.uuid4()) - filepath = os.path.normpath(str(tmpdir_factory.mktemp('data').join(name + ".txt"))) - with open(filepath, 'w') as fout: + filepath = os.path.normpath( + str(tmpdir_factory.mktemp("data").join(name + ".txt")) + ) + with open(filepath, "w") as fout: fout.write("Hello world\n") filepaths.append(filepath) return filepaths + return _create_files + @pytest.mark.enable_travis def test_create_remove_bucket(s3_client): bucket_name = "simcore-test" @@ -86,6 +93,7 @@ def test_create_remove_bucket(s3_client): s3_client.remove_bucket(bucket_name, delete_contents=True) assert not s3_client.exists_bucket(bucket_name) + @pytest.mark.enable_travis def test_create_remove_bucket_with_contents(s3_client, text_files): bucket_name = "simcore-test" @@ -99,6 +107,7 @@ def test_create_remove_bucket_with_contents(s3_client, text_files): s3_client.remove_bucket(bucket_name, delete_contents=True) assert not s3_client.exists_bucket(bucket_name) + @pytest.mark.enable_travis def test_file_upload_download(s3_client, bucket, text_files): filepath = text_files(1)[0] @@ -108,20 +117,22 @@ def test_file_upload_download(s3_client, bucket, text_files): assert s3_client.download_file(bucket, object_name, filepath2) assert filecmp.cmp(filepath2, filepath) + @pytest.mark.enable_travis def test_file_upload_meta_data(s3_client, bucket, text_files): filepath = text_files(1)[0] object_name = "1" _id = uuid.uuid4() - metadata = {'user' : 'guidon', 'node_id' : str(_id), 'boom-boom' : str(42.0)} + metadata = {"user": "guidon", "node_id": str(_id), "boom-boom": str(42.0)} assert s3_client.upload_file(bucket, object_name, filepath, metadata=metadata) metadata2 = s3_client.get_metadata(bucket, object_name) - assert metadata2["User"] == 'guidon' - assert metadata2["Node_id"] == str(_id) - assert metadata2["Boom-Boom"] == str(42.0) + assert metadata2["user"] == "guidon" + assert metadata2["node_id"] == str(_id) + assert metadata2["boom-boom"] == str(42.0) + @pytest.mark.enable_travis def test_sub_folders(s3_client, bucket, text_files): @@ -133,18 +144,21 @@ def test_sub_folders(s3_client, bucket, text_files): assert s3_client.upload_file(bucket, object_name, f) counter += 1 + @pytest.mark.enable_travis def test_search(s3_client, bucket, text_files): - metadata = [{'User' : 'alpha'}, {'User' : 'beta' }, {'User' : 'gamma'}] + metadata = [{"User": "alpha"}, {"User": "beta"}, {"User": "gamma"}] for i in range(3): - bucket_sub_folder = "Folder"+ str(i+1) + bucket_sub_folder = "Folder" + str(i + 1) filepaths = text_files(3) counter = 0 for f in filepaths: object_name = bucket_sub_folder + "/" + "Data" + str(counter) - assert s3_client.upload_file(bucket, object_name, f, metadata=metadata[counter]) + assert s3_client.upload_file( + bucket, object_name, f, metadata=metadata[counter] + ) counter += 1 query = "DATA1" @@ -162,14 +176,15 @@ def test_search(s3_client, bucket, text_files): results = s3_client.search(bucket, query, recursive=True, include_metadata=False) assert len(results) == 9 + @pytest.mark.enable_travis def test_presigned_put(s3_client, bucket, text_files): filepath = text_files(1)[0] object_name = "my_file" url = s3_client.create_presigned_put_url(bucket, object_name) - with open(filepath, 'rb') as fp: + with open(filepath, "rb") as fp: d = fp.read() - req = urllib.request.Request(url, data=d, method='PUT') + req = urllib.request.Request(url, data=d, method="PUT") with urllib.request.urlopen(req) as _f: pass @@ -177,6 +192,7 @@ def test_presigned_put(s3_client, bucket, text_files): assert s3_client.download_file(bucket, object_name, filepath2) assert filecmp.cmp(filepath2, filepath) + @pytest.mark.enable_travis def test_presigned_put_expired(s3_client, bucket, text_files): filepath = text_files(1)[0] @@ -184,9 +200,9 @@ def test_presigned_put_expired(s3_client, bucket, text_files): url = s3_client.create_presigned_put_url(bucket, object_name, timedelta(seconds=1)) time.sleep(2) failed = False - with open(filepath, 'rb') as fp: + with open(filepath, "rb") as fp: d = fp.read() - req = urllib.request.Request(url, data=d, method='PUT') + req = urllib.request.Request(url, data=d, method="PUT") try: urllib.request.urlopen(req) except urllib.error.HTTPError as _ex: @@ -205,6 +221,7 @@ def test_presigned_get(s3_client, bucket, text_files): assert filecmp.cmp(filepath2, filepath) + @pytest.mark.enable_travis def test_presigned_get_expired(s3_client, bucket, text_files): filepath = text_files(1)[0] @@ -221,6 +238,7 @@ def test_presigned_get_expired(s3_client, bucket, text_files): assert failed + @pytest.mark.enable_travis def test_object_exists(s3_client, bucket, text_files): files = text_files(2) @@ -234,6 +252,7 @@ def test_object_exists(s3_client, bucket, text_files): assert not s3_client.exists_object(bucket, object_name, False) assert s3_client.exists_object(bucket, object_name, True) + @pytest.mark.enable_travis def test_copy_object(s3_client, bucket, text_files): files = text_files(1) @@ -245,6 +264,7 @@ def test_copy_object(s3_client, bucket, text_files): assert s3_client.copy_object(bucket, copied_object, bucket + "/" + object_name) assert s3_client.exists_object(bucket, copied_object, False) + def test_list_objects(s3_client, bucket, text_files): files = text_files(2) file1 = files[0] @@ -268,4 +288,7 @@ def test_list_objects(s3_client, bucket, text_files): listed_objects = s3_client.list_objects(bucket, recursive=True) for s3_obj in listed_objects: - assert s3_obj.object_name == "level1/level2/1" or s3_obj.object_name == "level2/level2/2" + assert ( + s3_obj.object_name == "level1/level2/1" + or s3_obj.object_name == "level2/level2/2" + ) diff --git a/services/web/server/requirements/ci.txt b/services/web/server/requirements/ci.txt index 7ba1907d1e0..88e0bb3b1b4 100644 --- a/services/web/server/requirements/ci.txt +++ b/services/web/server/requirements/ci.txt @@ -11,7 +11,6 @@ -r _test.txt # installs this repo's packages -../../../packages/s3wrapper/ ../../../packages/models-library/ ../../../packages/postgres-database/ ../../../packages/simcore-sdk/ diff --git a/services/web/server/requirements/dev.txt b/services/web/server/requirements/dev.txt index 55bf91d539c..8e19896a1ad 100644 --- a/services/web/server/requirements/dev.txt +++ b/services/web/server/requirements/dev.txt @@ -12,7 +12,6 @@ -r _tools.txt # installs this repo's packages --e ../../../packages/s3wrapper/ -e ../../../packages/models-library/ -e ../../../packages/postgres-database/ -e ../../../packages/simcore-sdk/ diff --git a/services/web/server/requirements/prod.txt b/services/web/server/requirements/prod.txt index e66a0feff0a..7e0c40af111 100644 --- a/services/web/server/requirements/prod.txt +++ b/services/web/server/requirements/prod.txt @@ -10,7 +10,6 @@ -r _base.txt # installs this repo's packages -../../../packages/s3wrapper/ ../../../packages/models-library/ ../../../packages/postgres-database/ ../../../packages/simcore-sdk/ diff --git a/services/web/server/setup.py b/services/web/server/setup.py index fb910bba526..1096d8d421b 100644 --- a/services/web/server/setup.py +++ b/services/web/server/setup.py @@ -16,7 +16,6 @@ def read_reqs(reqs_path: Path): # ----------------------------------------------------------------- # Hard requirements on third-parties and latest for in-repo packages install_requirements = read_reqs(current_dir / "requirements" / "_base.txt") + [ - "s3wrapper", "simcore-models-library", "simcore-postgres-database", "simcore-sdk", diff --git a/tests/swarm-deploy/requirements/_test.in b/tests/swarm-deploy/requirements/_test.in index b345d404a0d..a5f927b8708 100644 --- a/tests/swarm-deploy/requirements/_test.in +++ b/tests/swarm-deploy/requirements/_test.in @@ -1,6 +1,5 @@ -c ../../../requirements/constraints.txt --r ../../../packages/s3wrapper/requirements/_base.in -r ../../../packages/postgres-database/requirements/_migration.txt -r ../../../packages/simcore-sdk/requirements/_base.in -r ../../../packages/service-library/requirements/_base.in @@ -21,3 +20,4 @@ pytest-sugar python-dotenv pyyaml tenacity +minio diff --git a/tests/swarm-deploy/requirements/_test.txt b/tests/swarm-deploy/requirements/_test.txt index 126f1f44c82..f03a37dffd7 100644 --- a/tests/swarm-deploy/requirements/_test.txt +++ b/tests/swarm-deploy/requirements/_test.txt @@ -29,11 +29,9 @@ aiormq==3.3.1 aiozipkin==0.7.1 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -135,16 +133,13 @@ markupsafe==1.1.1 minio==6.0.2 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt - # -r requirements/../../../packages/s3wrapper/requirements/_base.in - # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/_base.in + # -r requirements/_test.in multidict==5.1.0 # via # aiohttp @@ -229,11 +224,9 @@ pytz==2021.1 pyyaml==5.4.1 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -260,11 +253,9 @@ six==1.15.0 sqlalchemy[postgresql_psycopg2binary]==1.3.24 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -309,17 +300,13 @@ ujson==4.0.2 urllib3==1.26.4 # via # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_migration.txt - # -r requirements/../../../packages/s3wrapper/requirements/_base.in - # -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/s3wrapper/requirements/_base.in # minio # requests websocket-client==0.58.0 diff --git a/tests/swarm-deploy/requirements/ci.txt b/tests/swarm-deploy/requirements/ci.txt index e6bfcf36193..7dd9c73ea46 100644 --- a/tests/swarm-deploy/requirements/ci.txt +++ b/tests/swarm-deploy/requirements/ci.txt @@ -10,7 +10,6 @@ -r _test.txt # installs this repo's packages -../../packages/s3wrapper/ ../../packages/postgres-database/[migration] ../../packages/simcore-sdk/ ../../packages/service-library/ diff --git a/tests/swarm-deploy/requirements/dev.txt b/tests/swarm-deploy/requirements/dev.txt index f438fa50b51..d2181d9183b 100644 --- a/tests/swarm-deploy/requirements/dev.txt +++ b/tests/swarm-deploy/requirements/dev.txt @@ -12,7 +12,6 @@ -r tools.txt # installs this repo's packages --e ../../packages/s3wrapper/ -e ../../packages/postgres-database/ -e ../../packages/simcore-sdk/ -e ../../packages/service-library/