✨♻️ (⚠️ devops) Remove 5gb limit when uploading data via nodeports

.env-devel

@@ -60,7 +60,7 @@ S3_BUCKET_NAME=simcore
 S3_ENDPOINT=172.17.0.1:9001
 S3_SECRET_KEY=12345678
 S3_SECURE=0
-R_CLONE_S3_PROVIDER=MINIO
+R_CLONE_PROVIDER=MINIO
 
 SCICRUNCH_API_BASE_URL=https://scicrunch.org/api/1
 SCICRUNCH_API_KEY=REPLACE_ME_with_valid_api_key

api/specs/storage/openapi.yaml

@@ -257,11 +257,11 @@ paths:
           required: true
           schema:
             type: string
-      requestBody:
-        content:
-          application/json:
-            schema:
-              $ref: "#/components/schemas/FileMetaData"
+        - name: user_id
+          in: query
+          required: true
+          schema:
+            type: string
       responses:
         "200":
           description: "Returns file metadata"
@@ -271,6 +271,30 @@ paths:
                 $ref: "#/components/schemas/FileMetaEnvelope"
         default:
           $ref: "#/components/responses/DefaultErrorResponse"
+    delete:
+      summary: Removes a file's meta data entry
+      operationId: delete_file_meta_data
+      parameters:
+        - name: fileId
+          in: path
+          required: true
+          schema:
+            type: string
+        - name: location_id
+          in: path
+          required: true
+          schema:
+            type: string
+        - name: user_id
+          in: query
+          required: true
+          schema:
+            type: string
+      responses:
+        "204":
+          description: "Removes the file meta data entry from the database"
+        default:
+          $ref: "#/components/responses/DefaultErrorResponse"
 
   /locations/{location_id}/files/{fileId}:
     get:

ci/github/unit-testing/director-v2.bash

@@ -18,14 +18,13 @@ test() {
     --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
     -v -m "not travis" services/director-v2/tests/unit --ignore=services/director-v2/tests/unit/with_dbs \
     --asyncio-mode=auto \
-    --ignore=services/director-v2/tests/unit/with_swarm
   # these tests cannot be run in parallel
   pytest --log-format="%(asctime)s %(levelname)s %(message)s" \
     --log-date-format="%Y-%m-%d %H:%M:%S" \
     --cov=simcore_service_director_v2 --durations=10 --cov-append \
     --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
     --asyncio-mode=auto \
-    -v -m "not travis" services/director-v2/tests/unit/with_swarm services/director-v2/tests/unit/with_dbs
+    -v -m "not travis" services/director-v2/tests/unit/with_dbs
 }
 
 # Check if the function exists (bash specific)

packages/pytest-simcore/src/pytest_simcore/minio_service.py

@@ -3,7 +3,7 @@
 # pylint: disable=unused-variable
 
 import logging
-from typing import Dict, Iterator
+from typing import Any, Dict, Iterator
 
 import pytest
 from _pytest.monkeypatch import MonkeyPatch
@@ -44,7 +44,7 @@ def _ensure_remove_bucket(client: Minio, bucket_name: str):
 @pytest.fixture(scope="module")
 def minio_config(
     docker_stack: Dict, testing_environ_vars: Dict, monkeypatch_module: MonkeyPatch
-) -> Dict[str, str]:
+) -> Dict[str, Any]:
     assert "pytest-ops_minio" in docker_stack["services"]
 
     config = {

packages/settings-library/src/settings_library/r_clone.py

@@ -0,0 +1,17 @@
+from enum import Enum
+
+from pydantic import Field
+from .base import BaseCustomSettings
+from .s3 import S3Settings
+
+
+class S3Provider(str, Enum):
+    AWS = "AWS"
+    CEPH = "CEPH"
+    MINIO = "MINIO"
+
+
+class RCloneSettings(BaseCustomSettings):
+    R_CLONE_S3: S3Settings = Field(auto_default_from_env=True)
+    R_CLONE_PROVIDER: S3Provider
+    R_CLONE_REGION: str = Field("us-east-1", description="S3 region to use")

packages/settings-library/src/settings_library/s3.py

@@ -1,12 +1,20 @@
 from typing import Optional
 
 from .base import BaseCustomSettings
+from functools import cached_property
 
 
 class S3Settings(BaseCustomSettings):
-    S3_ENDPOINT: str = "minio:9000"
-    S3_ACCESS_KEY: str = "12345678"
-    S3_SECRET_KEY: str = "12345678"
+    S3_ENDPOINT: str
+    S3_ACCESS_KEY: str
+    S3_SECRET_KEY: str
     S3_ACCESS_TOKEN: Optional[str] = None
-    S3_BUCKET_NAME: str = "simcore"
+    S3_BUCKET_NAME: str
     S3_SECURE: bool = False
+
+    @cached_property
+    def endpoint(self) -> str:
+        if not self.S3_ENDPOINT.startswith("http"):
+            scheme = "https" if self.S3_SECURE else "http"
+            return f"{scheme}://{self.S3_ENDPOINT}"
+        return self.S3_ENDPOINT

packages/settings-library/src/settings_library/utils_r_clone.py

@@ -0,0 +1,47 @@
+import configparser
+from copy import deepcopy
+from io import StringIO
+from typing import Dict
+
+from .r_clone import RCloneSettings, S3Provider
+
+_COMMON_ENTRIES: Dict[str, str] = {
+    "type": "s3",
+    "access_key_id": "{access_key}",
+    "secret_access_key": "{secret_key}",
+    "region": "{aws_region}",
+    "acl": "private",
+}
+
+_PROVIDER_ENTRIES: Dict[S3Provider, Dict[str, str]] = {
+    # NOTE: # AWS_SESSION_TOKEN should be required for STS
+    S3Provider.AWS: {"provider": "AWS"},
+    S3Provider.CEPH: {"provider": "Ceph", "endpoint": "{endpoint}"},
+    S3Provider.MINIO: {"provider": "Minio", "endpoint": "{endpoint}"},
+}
+
+
+def _format_config(entries: Dict[str, str]) -> str:
+    config = configparser.ConfigParser()
+    config["dst"] = entries
+    with StringIO() as string_io:
+        config.write(string_io)
+        string_io.seek(0)
+        return string_io.read()
+
+
+def get_r_clone_config(r_clone_settings: RCloneSettings) -> str:
+    provider = r_clone_settings.R_CLONE_PROVIDER
+    entries = deepcopy(_COMMON_ENTRIES)
+    entries.update(_PROVIDER_ENTRIES[provider])
+
+    r_clone_config_template = _format_config(entries=entries)
+
+    # replace entries in template
+    r_clone_config = r_clone_config_template.format(
+        endpoint=r_clone_settings.R_CLONE_S3.endpoint,
+        access_key=r_clone_settings.R_CLONE_S3.S3_ACCESS_KEY,
+        secret_key=r_clone_settings.R_CLONE_S3.S3_SECRET_KEY,
+        aws_region=r_clone_settings.R_CLONE_REGION,
+    )
+    return r_clone_config

packages/settings-library/tests/test_utils_r_clone.py

@@ -0,0 +1,28 @@
+# pylint: disable=redefined-outer-name
+
+import pytest
+from settings_library.r_clone import RCloneSettings, S3Provider
+from settings_library.utils_r_clone import _COMMON_ENTRIES, get_r_clone_config
+
+
+@pytest.fixture(params=list(S3Provider))
+def r_clone_settings(request, monkeypatch) -> RCloneSettings:
+    monkeypatch.setenv("R_CLONE_PROVIDER", request.param)
+    monkeypatch.setenv("S3_ENDPOINT", "endpoint")
+    monkeypatch.setenv("S3_ACCESS_KEY", "access_key")
+    monkeypatch.setenv("S3_SECRET_KEY", "secret_key")
+    monkeypatch.setenv("S3_BUCKET_NAME", "bucket_name")
+    monkeypatch.setenv("S3_SECURE", False)
+    return RCloneSettings()
+
+
+def test_r_clone_config_template_replacement(r_clone_settings: RCloneSettings) -> None:
+    r_clone_config = get_r_clone_config(r_clone_settings)
+    print(r_clone_config)
+
+    assert "{endpoint}" not in r_clone_config
+    assert "{access_key}" not in r_clone_config
+    assert "{secret_key}" not in r_clone_config
+
+    for key in _COMMON_ENTRIES.keys():
+        assert key in r_clone_config

packages/simcore-sdk/requirements/_base.in

@@ -4,8 +4,10 @@
 --constraint ../../../requirements/constraints.txt
 --requirement ../../../packages/postgres-database/requirements/_base.in
 --requirement ../../../packages/service-library/requirements/_base.in
+--requirement ../../../packages/settings-library/requirements/_base.in
 --requirement ../../../packages/models-library/requirements/_base.in
 
+aiocache
 aiofiles
 aiohttp
 aiopg[sa]

packages/simcore-sdk/requirements/_base.txt

@@ -4,6 +4,8 @@
 #
 #    pip-compile --output-file=requirements/_base.txt --strip-extras requirements/_base.in
 #
+aiocache==0.11.1
+    # via -r requirements/_base.in
 aiodebug==2.3.0
     # via -r requirements/../../../packages/service-library/requirements/_base.in
 aiofiles==0.8.0
@@ -15,6 +17,7 @@ aiohttp==3.8.1
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   -r requirements/_base.in
 aiopg==1.3.3
@@ -34,6 +37,8 @@ attrs==20.3.0
     #   jsonschema
 charset-normalizer==2.0.12
     # via aiohttp
+click==8.1.2
+    # via typer
 dnspython==2.2.1
     # via email-validator
 email-validator==1.1.3
@@ -71,13 +76,15 @@ pydantic==1.9.0
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   -r requirements/../../../packages/models-library/requirements/_base.in
     #   -r requirements/../../../packages/service-library/requirements/_base.in
+    #   -r requirements/../../../packages/settings-library/requirements/_base.in
     #   -r requirements/_base.in
 pyinstrument==4.1.1
     # via -r requirements/../../../packages/service-library/requirements/_base.in
-pyparsing==3.0.7
+pyparsing==3.0.8
     # via packaging
 pyrsistent==0.18.1
     # via jsonschema
@@ -87,15 +94,17 @@ pyyaml==5.4.1
     #   -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/./constraints.txt
+    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   -r requirements/../../../packages/service-library/requirements/_base.in
 six==1.16.0
     # via jsonschema
-sqlalchemy==1.4.32
+sqlalchemy==1.4.35
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   -r requirements/../../../packages/postgres-database/requirements/_base.in
     #   aiopg
@@ -104,9 +113,11 @@ tenacity==8.0.1
     # via
     #   -r requirements/../../../packages/service-library/requirements/_base.in
     #   -r requirements/_base.in
-tqdm==4.63.1
+tqdm==4.64.0
     # via -r requirements/_base.in
-typing-extensions==4.1.1
+typer==0.4.1
+    # via -r requirements/../../../packages/settings-library/requirements/_base.in
+typing-extensions==4.2.0
     # via
     #   aiodebug
     #   pydantic

packages/simcore-sdk/requirements/_test.in

@@ -21,6 +21,7 @@ pytest-xdist
 pytest-lazy-fixture
 
 # mockups/fixtures
+aioboto3
 aioresponses
 alembic
 click