Refactoring storage: removed s3wrapper

.vscode/settings.template.json

@@ -33,7 +33,6 @@
     "./packages/postgres-database/src",
     "./packages/postgres-database/tests",
     "./packages/pytest-simcore/src",
-    "./packages/s3wrapper/src",
     "./packages/service-integration/src",
     "./packages/service-library/src",
     "./packages/simcore-sdk/src",

packages/pytest-simcore/src/pytest_simcore/environment_configs.py

@@ -1,4 +1,7 @@
 # pylint: disable=redefined-outer-name
+# pylint: disable=unused-argument
+# pylint: disable=unused-variable
+
 
 from copy import deepcopy
 from pathlib import Path

packages/pytest-simcore/src/pytest_simcore/minio_service.py

@@ -1,24 +1,36 @@
-import logging
-import os
-from copy import deepcopy
+# pylint: disable=redefined-outer-name
+# pylint: disable=unused-argument
+# pylint: disable=unused-variable
 
-# pylint:disable=unused-variable
-# pylint:disable=unused-argument
-# pylint:disable=redefined-outer-name
+import logging
 from distutils.util import strtobool
-from typing import Dict
+from typing import Dict, Iterator
 
 import pytest
 import tenacity
-from s3wrapper.s3_client import S3Client
+from minio import Minio
+from tenacity import Retrying
 
 from .helpers.utils_docker import get_ip, get_service_published_port
 
 log = logging.getLogger(__name__)
 
 
+def _ensure_remove_bucket(client: Minio, bucket_name: str):
+    if client.bucket_exists(bucket_name):
+        # remove content
+        objs = client.list_objects(bucket_name, prefix=None, recursive=True)
+        errors = client.remove_objects(bucket_name, [o.object_name for o in objs])
+        assert not list(errors)
+        # remove bucket
+        client.remove_bucket(bucket_name)
+    assert not client.bucket_exists(bucket_name)
+
+
 @pytest.fixture(scope="module")
-def minio_config(docker_stack: Dict, devel_environ: Dict) -> Dict[str, str]:
+def minio_config(
+    docker_stack: Dict, devel_environ: Dict, monkeypatch_module
+) -> Dict[str, str]:
     assert "ops_minio" in docker_stack["services"]
 
     config = {
@@ -32,49 +44,53 @@ def minio_config(docker_stack: Dict, devel_environ: Dict) -> Dict[str, str]:
     }
 
     # nodeports takes its configuration from env variables
-    old_environ = deepcopy(os.environ)
     for key, value in config["client"].items():
-        os.environ[f"S3_{key.upper()}"] = str(value)
-    os.environ["S3_SECURE"] = devel_environ["S3_SECURE"]
-    os.environ["S3_BUCKET_NAME"] = config["bucket_name"]
+        monkeypatch_module.setenv(f"S3_{key.upper()}", str(value))
+
+    monkeypatch_module.setenv("S3_SECURE", devel_environ["S3_SECURE"])
+    monkeypatch_module.setenv("S3_BUCKET_NAME", config["bucket_name"])
 
-    yield config
-    # restore environ
-    os.environ = old_environ
+    return config
 
 
 @pytest.fixture(scope="module")
-def minio_service(minio_config: Dict[str, str]) -> S3Client:
-    assert wait_till_minio_responsive(minio_config)
+def minio_service(minio_config: Dict[str, str]) -> Iterator[Minio]:
+
+    client = Minio(**minio_config["client"])
+
+    for attempt in Retrying(
+        wait=tenacity.wait_fixed(5),
+        stop=tenacity.stop_after_attempt(60),
+        before_sleep=tenacity.before_sleep_log(log, logging.WARNING),
+        reraise=True,
+    ):
+        with attempt:
+            # TODO: improve as https://docs.min.io/docs/minio-monitoring-guide.html
+            if not client.bucket_exists("pytest"):
+                client.make_bucket("pytest")
+            client.remove_bucket("pytest")
 
-    client = S3Client(**minio_config["client"])
-    assert client.create_bucket(minio_config["bucket_name"])
+    bucket_name = minio_config["bucket_name"]
 
-    yield client
+    # cleans up in case a failing tests left this bucket
+    _ensure_remove_bucket(client, bucket_name)
 
-    assert client.remove_bucket(minio_config["bucket_name"], delete_contents=True)
+    client.make_bucket(bucket_name)
+    assert client.bucket_exists(bucket_name)
 
+    yield client
 
-@tenacity.retry(
-    wait=tenacity.wait_fixed(5),
-    stop=tenacity.stop_after_attempt(60),
-    before_sleep=tenacity.before_sleep_log(log, logging.INFO),
-    reraise=True,
-)
-def wait_till_minio_responsive(minio_config: Dict[str, str]) -> bool:
-    """Check if something responds to ``url`` """
-    client = S3Client(**minio_config["client"])
-    if client.create_bucket("pytest"):
-        client.remove_bucket("pytest")
-        return True
-    raise Exception(f"Minio not responding to {minio_config}")
+    # cleanup upon tear-down
+    _ensure_remove_bucket(client, bucket_name)
 
 
 @pytest.fixture(scope="module")
-def bucket(minio_config: Dict[str, str], minio_service: S3Client) -> str:
+def bucket(minio_config: Dict[str, str], minio_service: Minio) -> str:
     bucket_name = minio_config["bucket_name"]
-    minio_service.create_bucket(bucket_name, delete_contents_if_exists=True)
+
+    _ensure_remove_bucket(minio_service, bucket_name)
+    minio_service.make_bucket(bucket_name)
 
     yield bucket_name
 
-    minio_service.remove_bucket(bucket_name, delete_contents=True)
+    _ensure_remove_bucket(minio_service, bucket_name)

packages/pytest-simcore/src/pytest_simcore/monkeypatch_extra.py

@@ -0,0 +1,29 @@
+# pylint: disable=redefined-outer-name
+# pylint: disable=unused-argument
+# pylint: disable=unused-variable
+
+from typing import Iterator
+
+import pytest
+from _pytest.fixtures import FixtureRequest
+from _pytest.monkeypatch import MonkeyPatch
+
+# Some extras to overcome https://github.com/pytest-dev/pytest/issues/363
+
+
+@pytest.fixture(scope="session")
+def monkeypatch_session(request: FixtureRequest) -> Iterator[MonkeyPatch]:
+    assert request.scope == "session"
+
+    mpatch_session = MonkeyPatch()
+    yield mpatch_session
+    mpatch_session.undo()
+
+
+@pytest.fixture(scope="module")
+def monkeypatch_module(request: FixtureRequest) -> Iterator[MonkeyPatch]:
+    assert request.scope == "module"
+
+    mpatch_module = MonkeyPatch()
+    yield mpatch_module
+    mpatch_module.undo()

packages/pytest-simcore/src/pytest_simcore/simcore_storage_service.py

@@ -8,7 +8,7 @@
 import aiohttp
 import pytest
 import tenacity
-from s3wrapper.s3_client import S3Client
+from minio import Minio
 from servicelib.minio_utils import MinioRetryPolicyUponInitialization
 from yarl import URL
 
@@ -33,7 +33,7 @@ def storage_endpoint(docker_stack: Dict, devel_environ: Dict) -> URL:
 
 @pytest.fixture(scope="function")
 async def storage_service(
-    minio_service: S3Client, storage_endpoint: URL, docker_stack: Dict
+    minio_service: Minio, storage_endpoint: URL, docker_stack: Dict
 ) -> URL:
     await wait_till_storage_responsive(storage_endpoint)