WIP ♻️ storage tests refactoring

ci/github/unit-testing/storage.bash

@@ -14,7 +14,7 @@ install() {
 test() {
     pytest --cov=simcore_service_storage --durations=10 --cov-append \
           --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
-          -v -m "not travis" services/storage/tests
+          -v -m "not travis" services/storage/tests/unit
 }
 
 # Check if the function exists (bash specific)

packages/models-library/tests/test_package.py

@@ -2,7 +2,6 @@
 # pylint:disable=unused-argument
 # pylint:disable=redefined-outer-name
 
-
 from pytest_simcore.helpers.utils_pylint import (
     assert_no_pdb_in_code,
     assert_pylint_is_passing,

packages/pytest-simcore/src/pytest_simcore/helpers/utils_projects.py

@@ -6,17 +6,24 @@
 import json
 import re
 import uuid as uuidlib
+from datetime import datetime
 from typing import Dict
-from simcore_service_webserver.utils import now_str
 
 from aiohttp import web
-
 from simcore_service_webserver.projects.projects_db import (
     APP_PROJECT_DBAPI,
     DB_EXCLUSIVE_COLUMNS,
 )
 from simcore_service_webserver.resources import resources
 
+# FIXME: remove dependencies with simcore_service_webserver
+# TODO:  and merge with utils_projects_extra.py
+
+
+def now_str() -> str:
+    return "{}Z".format(datetime.utcnow().isoformat(timespec="milliseconds"))
+
+
 fake_template_resources = [
     "data/" + name
     for name in resources.listdir("data")

services/storage/tests/helpers/utils_project.py → packages/pytest-simcore/src/pytest_simcore/helpers/utils_projects_extra.py

@@ -1,9 +1,12 @@
+# FIXME: temporary solution until simcore_service_webserver dependency is removed from utils_projects2.py
+
+
 import uuid as uuidlib
 from copy import deepcopy
-from typing import Dict, Tuple
+from typing import Any, Dict, Tuple
 
 
-def clone_project_data(project: Dict) -> Tuple[Dict, Dict]:
+def clone_project_data(project: Dict[str, Any]) -> Tuple[Dict, Dict]:
     project_copy = deepcopy(project)
 
     # Update project id

packages/pytest-simcore/src/pytest_simcore/repository_paths.py

@@ -34,6 +34,13 @@ def osparc_simcore_root_dir(request) -> Path:
     return root_dir
 
 
+@pytest.fixture(scope="session")
+def osparc_simcore_api_specs_dir(osparc_simcore_root_dir) -> Path:
+    dirpath = osparc_simcore_root_dir / "api" / "specs"
+    assert dirpath.exists()
+    return dirpath
+
+
 @pytest.fixture(scope="session")
 def osparc_simcore_services_dir(osparc_simcore_root_dir) -> Path:
     """ Path to osparc-simcore/services folder """

scripts/common-service.Makefile

@@ -32,6 +32,11 @@ export APP_VERSION
 # COMMON TASKS
 #
 
+.env: .env-devel ## creates .env file from defaults in .env-devel
+	$(if $(wildcard $@), \
+	@echo "WARNING #####  $< is newer than $@ ####"; diff -uN $@ $<; false;,\
+	@echo "INFO $@ does not exist, cloning $< as $@"; cp $< $@)
+
 
 .PHONY: install-dev install-prod install-ci
 
@@ -45,7 +50,7 @@ install-dev install-prod install-ci: _check_venv_active ## install app in develo
 ## pip 21.3 will remove support for this functionality. You can find discussion regarding this at https://github.com/pypa/pip/issues/7555.
 
 
-.PHONY: test-dev-unit test-ci-unit test-dev-integration test-ci-integration test-dev
+.PHONY: test-dev-unit test-ci-unit test-dev-integration test-ci-integration test-dev test-ci
 
 test-dev-unit test-ci-unit: _check_venv_active
 	# Targets tests/unit folder
@@ -58,7 +63,9 @@ test-dev-integration test-ci-integration:
 	make --no-print-directory _run-$(subst -integration,,$@) target=$(CURDIR)/tests/integration
 
 
-test-dev: test-dev-unit test-dev-integration ## runs unit and integration tests for development (e.g. w/ pdb)
+test-dev: test-dev-unit test-dev-integration ## runs unit and integration tests for development (w/ pdb, fail-first)
+
+test-ci: test-ci-unit test-ci-integration ## runs unit and integration tests for CI (w/ coverage, timing)
 
 test-ci: test-ci-unit test-ci-integration ## runs unit and integration tests for CI
 

scripts/common.Makefile

@@ -150,6 +150,9 @@ version-minor: ## commits version with backwards-compatible API addition or chan
 version-major: ## commits version with backwards-INcompatible addition or changes
 	$(_bumpversion)
 
+.PHONY: reqs requirements
+reqs requirements: ## compiles pip requirements (.in -> .txt). Usage 'make reqs upgrade=$package'
+	@$(MAKE_C) requirements reqs
 
 #
 # SUBTASKS

services/storage/Makefile

@@ -17,8 +17,6 @@ tests: ## runs unit tests
 
 
 # DEVELOPMENT ########
-.env:
-	cp .env-devel $@
 
 run-devel: .env
 	# start app (within $<) in prod mode

services/storage/requirements/_base.in

@@ -16,11 +16,6 @@ aiofiles
 aiopg[sa]
 
 tenacity
-semantic_version
 typer
 minio
 urllib3
-
-#
-# FIXME: pip copiling these requirements wil fail due to incompatible contraints on urllib3 (ongoing in weekely mainteinance)
-#

services/storage/requirements/_base.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile
+# This file is autogenerated by pip-compile with python 3.8
 # To update, run:
 #
 #    pip-compile --output-file=requirements/_base.txt requirements/_base.in
@@ -10,8 +10,6 @@ aiodebug==1.1.2
     # via -r requirements/../../../packages/service-library/requirements/_base.in
 aiofiles==0.7.0
     # via -r requirements/_base.in
-aiohttp-swagger[performance]==1.0.15
-    # via -r requirements/_base.in
 aiohttp==3.7.4.post0
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
@@ -24,6 +22,8 @@ aiohttp==3.7.4.post0
     #   aiobotocore
     #   aiohttp-swagger
     #   aiozipkin
+aiohttp-swagger[performance]==1.0.15
+    # via -r requirements/_base.in
 aioitertools==0.7.1
     # via aiobotocore
 aiopg[sa]==1.3.0
@@ -60,8 +60,6 @@ dnspython==2.1.0
     # via email-validator
 email-validator==1.1.3
     # via pydantic
-fastapi==0.65.2
-    # via -r requirements/../../../packages/service-library/requirements/_base.in
 idna==2.10
     # via
     #   -r requirements/../../../packages/models-library/requirements/_base.in
@@ -125,11 +123,10 @@ pydantic[email]==1.8.2
     #   -r requirements/../../../packages/models-library/requirements/_base.in
     #   -r requirements/../../../packages/service-library/requirements/_base.in
     #   -r requirements/../../../packages/settings-library/requirements/_base.in
-    #   fastapi
-pyinstrument-cext==0.2.4
-    # via pyinstrument
 pyinstrument==3.4.2
     # via -r requirements/../../../packages/service-library/requirements/_base.in
+pyinstrument-cext==0.2.4
+    # via pyinstrument
 pyrsistent==0.18.0
     # via jsonschema
 python-dateutil==2.8.1
@@ -144,8 +141,6 @@ pyyaml==5.4.1
     #   -r requirements/../../../packages/service-library/requirements/_base.in
     #   aiohttp-swagger
     #   openapi-spec-validator
-semantic-version==2.8.5
-    # via -r requirements/_base.in
 six==1.16.0
     # via
     #   isodate
@@ -165,8 +160,6 @@ sqlalchemy[postgresql_psycopg2binary]==1.3.24
     #   -r requirements/../../../packages/postgres-database/requirements/_base.in
     #   -r requirements/../../../packages/service-library/requirements/_base.in
     #   aiopg
-starlette==0.14.2
-    # via fastapi
 strict-rfc3339==0.7
     # via openapi-core
 tenacity==7.0.0

services/storage/requirements/_test.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile
+# This file is autogenerated by pip-compile with python 3.8
 # To update, run:
 #
 #    pip-compile --output-file=requirements/_test.txt requirements/_test.in
@@ -54,12 +54,12 @@ cryptography==3.4.7
     #   paramiko
 distro==1.5.0
     # via docker-compose
+docker[ssh]==5.0.0
+    # via docker-compose
 docker-compose==1.29.1
     # via
     #   -c requirements/../../../requirements/constraints.txt
     #   pytest-docker
-docker[ssh]==5.0.0
-    # via docker-compose
 dockerpty==0.4.1
     # via docker-compose
 docopt==0.6.2
@@ -120,6 +120,15 @@ pyrsistent==0.18.0
     # via
     #   -c requirements/_base.txt
     #   jsonschema
+pytest==6.2.4
+    # via
+    #   -r requirements/_test.in
+    #   pytest-aiohttp
+    #   pytest-cov
+    #   pytest-docker
+    #   pytest-instafail
+    #   pytest-mock
+    #   pytest-sugar
 pytest-aiohttp==0.3.0
     # via -r requirements/_test.in
 pytest-cov==2.12.1
@@ -134,15 +143,6 @@ pytest-runner==5.3.1
     # via -r requirements/_test.in
 pytest-sugar==0.9.4
     # via -r requirements/_test.in
-pytest==6.2.4
-    # via
-    #   -r requirements/_test.in
-    #   pytest-aiohttp
-    #   pytest-cov
-    #   pytest-docker
-    #   pytest-instafail
-    #   pytest-mock
-    #   pytest-sugar
 python-dateutil==2.8.1
     # via
     #   -c requirements/_base.txt

services/storage/requirements/_tools.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile
+# This file is autogenerated by pip-compile with python 3.8
 # To update, run:
 #
 #    pip-compile --output-file=requirements/_tools.txt requirements/_tools.in

services/storage/src/simcore_service_storage/abc_storage.py

@@ -0,0 +1,44 @@
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional
+
+from .models import DatasetMetaData, FileMetaDataEx
+
+
+def DataStorageInterface(ABC):
+    """An interface to any data storage system plugged to this service"""
+
+    @abstractmethod
+    async def is_responsive(self, session_id: str, **options) -> bool:
+        """Returns True if service is available for a given session"""
+        raise NotImplementedError
+
+    # FILES resource ---
+
+    @abstractmethod
+    async def list_files(
+        self, session_id: str, *, fields_filter: Optional[Dict[str, Any]] = None
+    ) -> List[FileMetaDataEx]:
+        """Lists all files of a given session"""
+        # TODO: pagination
+        raise NotImplementedError
+
+    @abstractmethod
+    async def get_file(self, session_id: str, file_id: str) -> Optional[FileMetaDataEx]:
+        raise NotImplementedError
+
+    @abstractmethod
+    async def delete_file(self, session_id: str, file_id: str) -> None:
+        raise NotImplementedError
+
+    # DATASET resource ---
+    #  - a group of files and its metadata?
+
+    @abstractmethod
+    async def list_datasets(self, session_id: str) -> List[DatasetMetaData]:
+        raise NotImplementedError
+
+    @abstractmethod
+    async def get_dataset(
+        self, session_id: str, dataset_id: str
+    ) -> List[FileMetaDataEx]:
+        raise NotImplementedError

services/storage/src/simcore_service_storage/access_layer.py

@@ -70,7 +70,7 @@ def none(cls) -> "AccessRights":
 
 
 class AccessLayerError(Exception):
-    """ Base class for access-layer related errors """
+    """Base class for access-layer related errors"""
 
 
 class InvalidFileIdentifier(AccessLayerError):
@@ -282,6 +282,6 @@ async def get_file_access_rights(
 
 
 async def get_readable_project_ids(conn: SAConnection, user_id: int) -> List[ProjectID]:
-    """ Returns a list of projects where user has granted read-access """
+    """Returns a list of projects where user has granted read-access"""
     projects_access_rights = await list_projects_access_rights(conn, int(user_id))
     return [pid for pid, access in projects_access_rights.items() if access.read]