From df388da9bd174e57e0f1c44c89bb14c1274a1025 Mon Sep 17 00:00:00 2001 From: Jennings Zhang Date: Mon, 5 Jun 2023 11:08:38 -0400 Subject: [PATCH] Plain filesystem storage (#516) * Minimize public interface of SwiftManager * Add support for FilesystemStorage and SwiftStorage --- .gitignore | 1 + chris_backend/config/settings/local.py | 15 +- chris_backend/config/settings/production.py | 32 ++- chris_backend/core/storage/__init__.py | 22 ++ chris_backend/core/storage/helpers.py | 64 +++++ chris_backend/core/storage/plain_fs.py | 57 ++++ chris_backend/core/storage/storagemanager.py | 70 +++++ .../core/{ => storage}/swiftmanager.py | 75 ++--- chris_backend/feeds/tests/test_views.py | 5 +- chris_backend/filebrowser/tests/test_views.py | 6 +- chris_backend/pacsfiles/serializers.py | 5 +- .../pacsfiles/tests/test_serializers.py | 18 +- chris_backend/pacsfiles/tests/test_views.py | 5 +- chris_backend/pipelines/tests/test_views.py | 5 +- chris_backend/pipelines/views.py | 6 +- chris_backend/plugininstances/serializers.py | 5 +- .../plugininstances/services/manager.py | 6 +- .../plugininstances/tests/test_manager.py | 5 +- .../plugininstances/tests/test_serializers.py | 35 ++- .../plugininstances/tests/test_views.py | 8 +- chris_backend/servicefiles/serializers.py | 5 +- .../servicefiles/tests/test_serializers.py | 15 +- .../servicefiles/tests/test_views.py | 6 +- .../uploadedfiles/tests/test_views.py | 10 +- chris_backend/uploadedfiles/views.py | 10 +- chris_backend/users/serializers.py | 5 +- chris_backend/users/tests/test_serializers.py | 13 +- chris_backend/users/tests/test_views.py | 14 +- docker-compose_noswift.yml | 257 ++++++++++++++++++ 29 files changed, 621 insertions(+), 159 deletions(-) create mode 100755 chris_backend/core/storage/__init__.py create mode 100755 chris_backend/core/storage/helpers.py create mode 100755 chris_backend/core/storage/plain_fs.py create mode 100755 chris_backend/core/storage/storagemanager.py rename chris_backend/core/{ => storage}/swiftmanager.py (67%) create mode 100755 docker-compose_noswift.yml diff --git a/.gitignore b/.gitignore index e323bb3e..9489d2b4 100755 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ dc.out celerybeat.pid swarm/prod/secrets/ kubernetes/prod/base/secrets/ +venv diff --git a/chris_backend/config/settings/local.py b/chris_backend/config/settings/local.py index 1b291e27..1b9f7d43 100755 --- a/chris_backend/config/settings/local.py +++ b/chris_backend/config/settings/local.py @@ -11,7 +11,7 @@ import ldap from django_auth_ldap.config import LDAPSearch from .common import * # noqa -from core.swiftmanager import SwiftManager +from core.storage import verify_storage_connection # Normally you should not import ANYTHING from Django directly # into your settings, but ImproperlyConfigured is an exception. @@ -81,6 +81,12 @@ 'propagate': False # required to avoid double logging with root logger } +# Storage Settings +# +# To use local storage: +# DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage' +# MEDIA_ROOT = '/var/chris' +# # Swift service settings DEFAULT_FILE_STORAGE = 'swift.storage.SwiftStorage' SWIFT_AUTH_URL = 'http://swift_service:8080/auth/v1.0' @@ -91,7 +97,12 @@ 'key': SWIFT_KEY, 'authurl': SWIFT_AUTH_URL} try: - SwiftManager(SWIFT_CONTAINER_NAME, SWIFT_CONNECTION_PARAMS).create_container() + verify_storage_connection( + DEFAULT_FILE_STORAGE=DEFAULT_FILE_STORAGE, + MEDIA_ROOT=globals().get('MEDIA_ROOT', None), + SWIFT_CONTAINER_NAME=SWIFT_CONTAINER_NAME, + SWIFT_CONNECTION_PARAMS=SWIFT_CONNECTION_PARAMS + ) except Exception as e: raise ImproperlyConfigured(str(e)) diff --git a/chris_backend/config/settings/production.py b/chris_backend/config/settings/production.py index 7c3f1a91..4dbeb0e1 100755 --- a/chris_backend/config/settings/production.py +++ b/chris_backend/config/settings/production.py @@ -8,7 +8,7 @@ from django_auth_ldap.config import LDAPSearch from .common import * # noqa from environs import Env, EnvValidationError -from core.swiftmanager import SwiftManager +from core.storage import verify_storage_connection # Normally you should not import ANYTHING from Django directly # into your settings, but ImproperlyConfigured is an exception. @@ -55,16 +55,28 @@ def get_secret(setting, secret_type=env): # SWIFT SERVICE CONFIGURATION # ------------------------------------------------------------------------------ -DEFAULT_FILE_STORAGE = 'swift.storage.SwiftStorage' -SWIFT_AUTH_URL = get_secret('SWIFT_AUTH_URL') -SWIFT_USERNAME = get_secret('SWIFT_USERNAME') -SWIFT_KEY = get_secret('SWIFT_KEY') -SWIFT_CONTAINER_NAME = get_secret('SWIFT_CONTAINER_NAME') -SWIFT_CONNECTION_PARAMS = {'user': SWIFT_USERNAME, - 'key': SWIFT_KEY, - 'authurl': SWIFT_AUTH_URL} +DEFAULT_FILE_STORAGE = get_secret('DEFAULT_FILE_STORAGE') + +if DEFAULT_FILE_STORAGE == 'django.core.files.storage.FileSystemStorage': + MEDIA_ROOT = get_secret('MEDIA_ROOT') + verify_storage = lambda: verify_storage_connection(DEFAULT_FILE_STORAGE=DEFAULT_FILE_STORAGE, MEDIA_ROOT=MEDIA_ROOT) +elif DEFAULT_FILE_STORAGE == 'swift.storage.SwiftStorage': + SWIFT_AUTH_URL = get_secret('SWIFT_AUTH_URL') + SWIFT_USERNAME = get_secret('SWIFT_USERNAME') + SWIFT_KEY = get_secret('SWIFT_KEY') + SWIFT_CONTAINER_NAME = get_secret('SWIFT_CONTAINER_NAME') + SWIFT_CONNECTION_PARAMS = {'user': SWIFT_USERNAME, + 'key': SWIFT_KEY, + 'authurl': SWIFT_AUTH_URL} + verify_storage = lambda: verify_storage_connection( + SWIFT_CONTAINER_NAME=SWIFT_CONTAINER_NAME, + SWIFT_CONNECTION_PARAMS=SWIFT_CONNECTION_PARAMS + ) +else: + verify_storage = lambda: verify_storage_connection() + try: - SwiftManager(SWIFT_CONTAINER_NAME, SWIFT_CONNECTION_PARAMS).create_container() + verify_storage() except Exception as e: raise ImproperlyConfigured(str(e)) diff --git a/chris_backend/core/storage/__init__.py b/chris_backend/core/storage/__init__.py new file mode 100755 index 00000000..e9ddc72d --- /dev/null +++ b/chris_backend/core/storage/__init__.py @@ -0,0 +1,22 @@ +""" +A module for interfacing with file storage backends. + +File storage backends are "services" which store arbitrary data identified by path-like strings. +Examples include OpenStack Swift object storage, AWS S3, Nooba on OpenShift, or of course, +a literal UNIX-y filesystem. + +ChRIS files are immutable, so file storage services can be optimized for WORM +(write-once, read-many) workloads. + +Note to developers: historically, *ChRIS* was tightly-coupled to OpenStack Swift, hence +variable and function names use Swift terminology. +""" +from typing import Dict + +from .storagemanager import StorageManager +from .swiftmanager import SwiftManager +from .plain_fs import FilesystemManager +from .helpers import connect_storage, verify_storage_connection + + +__all__ = ['StorageManager', 'SwiftManager', 'FilesystemManager', 'connect_storage', 'verify_storage_connection'] diff --git a/chris_backend/core/storage/helpers.py b/chris_backend/core/storage/helpers.py new file mode 100755 index 00000000..a752babe --- /dev/null +++ b/chris_backend/core/storage/helpers.py @@ -0,0 +1,64 @@ +from typing import Dict, Any, ContextManager +from tempfile import TemporaryDirectory +import unittest.mock +from contextlib import contextmanager + +from core.storage.storagemanager import StorageManager +from core.storage.swiftmanager import SwiftManager +from core.storage.plain_fs import FilesystemManager + + +def connect_storage(settings) -> StorageManager: + """ + :param settings: django.conf.settings object + :returns: a manager for the storage configured by settings + """ + storage_name = __get_storage_name(settings) + if storage_name == 'SwiftStorage': + return SwiftManager(settings.SWIFT_CONTAINER_NAME, settings.SWIFT_CONNECTION_PARAMS) + elif storage_name == 'FileSystemStorage': + return FilesystemManager(settings.MEDIA_ROOT) + raise ValueError(f'Unsupported storage system: {storage_name}') + + +def verify_storage_connection(**kwargs) -> None: + """ + Create a ``StorageManager`` for the given settings. Raises an exception if the connection + or configuration is wrong. + + If the connection works, then ``StorageManager.create_container`` is called. + """ + settings = _DummySettings(kwargs) + storage_manager = connect_storage(settings) + storage_manager.create_container() + + +@contextmanager +def mock_storage(target_settings) -> ContextManager[FilesystemManager]: + """ + For testing only. + + Uses ``unittest.mock.patch`` to configure a given settings object to use a temporary directory + for ChRIS files storage. + + :param target_settings: a django.conf settings object + :returns: a FilesystemManager for the temporary directory + """ + with TemporaryDirectory() as tmp_dir: + settings = { + 'DEFAULT_FILE_STORAGE': 'fake.FileSystemStorage', + 'MEDIA_ROOT': tmp_dir + } + with unittest.mock.patch.multiple(target_settings, **settings): + yield FilesystemManager(tmp_dir) + + +class _DummySettings: + + def __init__(self, settings_dict: Dict[str, str]): + for k, v in settings_dict.items(): + setattr(self, k, v) + + +def __get_storage_name(settings: Any) -> str: + return settings.DEFAULT_FILE_STORAGE.rsplit('.', maxsplit=1)[-1] diff --git a/chris_backend/core/storage/plain_fs.py b/chris_backend/core/storage/plain_fs.py new file mode 100755 index 00000000..5e936dac --- /dev/null +++ b/chris_backend/core/storage/plain_fs.py @@ -0,0 +1,57 @@ +from pathlib import Path +from typing import Union, List, AnyStr, Optional + +from core.storage.storagemanager import StorageManager + + +class FilesystemManager(StorageManager): + """ + The simplest manager, something everyone has, the one you can trust... + + ``FilesystemManager`` is for storing files on disk as-is, no magic involved. + More technically, ``FilesystemManager`` methods adapt method calls of ``pathlib`` to the ``StoreManager`` interface. + + This code can be used as a reference for how to implement ``StorageManager`` + for other file storage services. + """ + + def __init__(self, base: Union[str, Path]): + self.__base = Path(base) + + def create_container(self) -> None: + self.__base.mkdir(exist_ok=True, parents=True) + + def ls(self, path_prefix: str) -> List[str]: + all_paths = (self.__base / path_prefix).rglob('*') + return [str(p) for p in all_paths if p.is_file()] + + def path_exists(self, path: str) -> bool: + return (self.__base / path).exists() + + def obj_exists(self, file_path: str) -> bool: + return (self.__base / file_path).is_file() + + def upload_obj(self, file_path: str, contents: AnyStr, content_type: Optional[str] = None): + dst = (self.__base / file_path) + dst.parent.mkdir(exist_ok=True, parents=True) + + if self.__is_textual(content_type): + dst.write_text(contents) + else: + dst.write_bytes(contents) + + @staticmethod + def __is_textual(media_type: Optional[str]) -> bool: + """ + :returns: True if given media type is a text-based media type. + """ + return media_type is not None and media_type.split('/', maxsplit=1)[0] == 'text' + + def download_obj(self, file_path: str) -> AnyStr: + return (self.__base / file_path).read_bytes() + + def copy_obj(self, src: str, dst: str) -> None: + (self.__base / src).link_to(self.__base / dst) + + def delete_obj(self, file_path: str) -> None: + (self.__base / file_path).unlink() diff --git a/chris_backend/core/storage/storagemanager.py b/chris_backend/core/storage/storagemanager.py new file mode 100755 index 00000000..b6bc4b14 --- /dev/null +++ b/chris_backend/core/storage/storagemanager.py @@ -0,0 +1,70 @@ +import abc +from typing import List, AnyStr, Optional + + +class StorageManager(abc.ABC): + """ + ``StorageManager`` provides an interface between ChRIS and its file storage backend. + + ``StorageManager`` methods implement helper functions for browsing stored files and retrieving + file data. These functions are analogous to ``ls``, ``stat``, and ``cat`` commands. + """ + + @abc.abstractmethod + def create_container(self) -> None: + """ + Create the container where all ChRIS file data is to be stored. + + For Swift, a container is... a container. For S3, a container is a bucket. + + For a plain filesystem, a container is simply the top-level/parent directory. + """ + ... + + def ls(self, path_prefix: str) -> List[str]: + """ + :returns: a list of all files under a given path prefix. + """ + ... + + def path_exists(self, path: str) -> bool: + """ + :returns: True if path exists (whether it be a directory OR file) + """ + ... + + def obj_exists(self, file_path: str) -> bool: + """ + :returns: True if given path is an existing file + """ + ... + + def upload_obj(self, file_path: str, contents: AnyStr, content_type: Optional[str] = None): + """ + Upload file data to the storage service. + + :param file_path: file path to upload to + :param contents: file data + :param content_type: optional media type, e.g. "text/plain" + """ + ... + + def download_obj(self, file_path: str) -> AnyStr: + """ + Download file data from the storage service. + """ + ... + + def copy_obj(self, src: str, dst: str) -> None: + """ + Copy data to a new path. + + Instead of a copy, implementations may create links or shallow copies for efficiency. + """ + ... + + def delete_obj(self, file_path: str) -> None: + """ + Delete data from the given path. + """ + ... diff --git a/chris_backend/core/swiftmanager.py b/chris_backend/core/storage/swiftmanager.py similarity index 67% rename from chris_backend/core/swiftmanager.py rename to chris_backend/core/storage/swiftmanager.py index e87c22fd..3ad362e6 100755 --- a/chris_backend/core/swiftmanager.py +++ b/chris_backend/core/storage/swiftmanager.py @@ -9,11 +9,12 @@ from swiftclient import Connection from swiftclient.exceptions import ClientException +from core.storage.storagemanager import StorageManager logger = logging.getLogger(__name__) -class SwiftManager(object): +class SwiftManager(StorageManager): def __init__(self, container_name, conn_params): self.container_name = container_name @@ -22,7 +23,7 @@ def __init__(self, container_name, conn_params): # swift storage connection object self._conn = None - def get_connection(self): + def __get_connection(self): """ Connect to swift storage and return the connection object. """ @@ -43,22 +44,29 @@ def create_container(self): """ Create the storage container. """ - conn = self.get_connection() + conn = self.__get_connection() try: conn.put_container(self.container_name) except ClientException as e: logger.error(str(e)) raise - def ls(self, path, **kwargs): + def ls(self, path): """ Return a list of objects in the swift storage with the provided path as a prefix. """ - b_full_listing = kwargs.get('full_listing', True) + return self._ls(path, b_full_listing=True) + + def _ls(self, path, b_full_listing: bool): + """ + Note to developers: the body of ``_ls`` was originally the body of ``self.ls``, + though it's been renamed to ``_ls`` so that ``self.ls``'s signature could be + changed. ``self.ls`` originally accepted ``**kwargs`` but that is no longer the case. + """ l_ls = [] # listing of names to return if path: - conn = self.get_connection() + conn = self.__get_connection() for i in range(5): try: # get the full list of objects in Swift storage with given prefix @@ -79,13 +87,13 @@ def path_exists(self, path): """ Return True/False if passed path exists in swift storage. """ - return len(self.ls(path, full_listing=False)) > 0 + return len(self._ls(path, b_full_listing=False)) > 0 def obj_exists(self, obj_path): """ Return True/False if passed object exists in swift storage. """ - conn = self.get_connection() + conn = self.__get_connection() for i in range(5): try: conn.head_object(self.container_name, obj_path) @@ -100,17 +108,17 @@ def obj_exists(self, obj_path): else: return True - def upload_obj(self, swift_path, contents, **kwargs): + def upload_obj(self, swift_path, contents, content_type=None): """ Upload an object (a file contents) into swift storage. """ - conn = self.get_connection() + conn = self.__get_connection() for i in range(5): try: conn.put_object(self.container_name, swift_path, contents=contents, - **kwargs) + content_type=content_type) except ClientException as e: logger.error(str(e)) if i == 4: @@ -119,15 +127,14 @@ def upload_obj(self, swift_path, contents, **kwargs): else: break - def download_obj(self, obj_path, **kwargs): + def download_obj(self, obj_path): """ Download an object from swift storage. """ - conn = self.get_connection() + conn = self.__get_connection() for i in range(5): try: - resp_headers, obj_contents = conn.get_object(self.container_name, - obj_path, **kwargs) + resp_headers, obj_contents = conn.get_object(self.container_name, obj_path) except ClientException as e: logger.error(str(e)) if i == 4: @@ -136,15 +143,15 @@ def download_obj(self, obj_path, **kwargs): else: return obj_contents - def copy_obj(self, obj_path, dest_path, **kwargs): + def copy_obj(self, obj_path, dest_path): """ Copy an object to a new destination in swift storage. """ - conn = self.get_connection() + conn = self.__get_connection() dest = os.path.join('/' + self.container_name, dest_path.lstrip('/')) for i in range(5): try: - conn.copy_object(self.container_name, obj_path, dest, **kwargs) + conn.copy_object(self.container_name, obj_path, dest) except ClientException as e: logger.error(str(e)) if i == 4: @@ -157,7 +164,7 @@ def delete_obj(self, obj_path): """ Delete an object from swift storage. """ - conn = self.get_connection() + conn = self.__get_connection() for i in range(5): try: conn.delete_object(self.container_name, obj_path) @@ -168,33 +175,3 @@ def delete_obj(self, obj_path): time.sleep(0.4) else: break - - def upload_files(self, local_dir, swift_prefix='', **kwargs): - """ - Upload all the files within a local directory recursively to swift storage. - - By default, the location in swift storage will map 1:1 to the location of - files in the local filesytem. This location can be remapped by using the - . For example, assume a local directory /home/user/project/data/ - with the following files: - - '/home/user/project/data/file1', - '/home/user/project/data/dir1/file_d1', - '/home/user/project/data/dir2/file_d2' - - and we want to upload everything in that directory to object storage, at location - '/storage'. In this case, swift_prefix='/storage' results in a new list - - '/storage/file1', - '/storage/dir1/file_d1', - '/storage/dir2/file_d2' - """ - # upload all files down the - for root, dirs, files in os.walk(local_dir): - swift_base = root.replace(local_dir, swift_prefix, 1) if swift_prefix else root - for filename in files: - swift_path = os.path.join(swift_base, filename) - if not self.obj_exists(swift_path): - local_file_path = os.path.join(root, filename) - with open(local_file_path, 'rb') as f: - self.upload_obj(swift_path, f.read(), **kwargs) diff --git a/chris_backend/feeds/tests/test_views.py b/chris_backend/feeds/tests/test_views.py index c354fc40..40593919 100755 --- a/chris_backend/feeds/tests/test_views.py +++ b/chris_backend/feeds/tests/test_views.py @@ -12,7 +12,7 @@ from plugins.models import PluginMeta, Plugin, ComputeResource from plugininstances.models import PluginInstance, PluginInstanceFile from feeds.models import Note, Tag, Tagging, Feed, Comment -from core.swiftmanager import SwiftManager +from core.storage import connect_storage COMPUTE_RESOURCE_URL = settings.COMPUTE_RESOURCE_URL @@ -836,8 +836,7 @@ def setUp(self): # create two files in the DB "already uploaded" to the server from two different # plugin instances that write to the same feed - self.swift_manager = SwiftManager(settings.SWIFT_CONTAINER_NAME, - settings.SWIFT_CONNECTION_PARAMS) + self.swift_manager = connect_storage(settings) plg_inst = PluginInstance.objects.all()[0] self.path1 = 'tests/file1.txt' with io.StringIO("test file1") as file1: diff --git a/chris_backend/filebrowser/tests/test_views.py b/chris_backend/filebrowser/tests/test_views.py index eafbe8b1..b367d6e2 100755 --- a/chris_backend/filebrowser/tests/test_views.py +++ b/chris_backend/filebrowser/tests/test_views.py @@ -11,7 +11,7 @@ from rest_framework import status -from core.swiftmanager import SwiftManager +from core.storage import connect_storage from uploadedfiles.models import UploadedFile from plugins.models import PluginMeta, Plugin, ComputeResource from plugininstances.models import PluginInstance, PluginInstanceFile @@ -151,9 +151,7 @@ def setUp(self): super(FileBrowserPathFileListViewTests, self).setUp() # create a file in the DB "already uploaded" to the server) - self.swift_manager = SwiftManager(settings.SWIFT_CONTAINER_NAME, - settings.SWIFT_CONNECTION_PARAMS) - + self.swift_manager = connect_storage(settings) # upload file to Swift storage self.upload_path = f'{self.username}/uploads/file2.txt' with io.StringIO("test file") as file1: diff --git a/chris_backend/pacsfiles/serializers.py b/chris_backend/pacsfiles/serializers.py index 20fff2d0..432fe1cc 100755 --- a/chris_backend/pacsfiles/serializers.py +++ b/chris_backend/pacsfiles/serializers.py @@ -6,7 +6,7 @@ from collectionjson.fields import ItemLinkField from core.utils import get_file_resource_link -from core.swiftmanager import SwiftManager +from core.storage import connect_storage from .models import PACS, PACSFile @@ -76,8 +76,7 @@ def validate_path(self, path): raise serializers.ValidationError( ["File path must start with 'SERVICES/PACS/'."]) # verify that the file is indeed already in Swift - swift_manager = SwiftManager(settings.SWIFT_CONTAINER_NAME, - settings.SWIFT_CONNECTION_PARAMS) + swift_manager = connect_storage(settings) try: swift_path_exists = swift_manager.obj_exists(path) except Exception as e: diff --git a/chris_backend/pacsfiles/tests/test_serializers.py b/chris_backend/pacsfiles/tests/test_serializers.py index 8f3419df..5abf38bf 100755 --- a/chris_backend/pacsfiles/tests/test_serializers.py +++ b/chris_backend/pacsfiles/tests/test_serializers.py @@ -2,7 +2,6 @@ import logging import time import io -from unittest import mock from django.test import TestCase, tag from django.conf import settings @@ -10,7 +9,7 @@ from pacsfiles.models import PACS, PACSFile from pacsfiles.serializers import PACSFileSerializer -from pacsfiles.serializers import SwiftManager +from core.storage.helpers import connect_storage, mock_storage class PACSFileSerializerTests(TestCase): @@ -44,11 +43,17 @@ def test_validate_path_failure_does_not_exist(self): pacsfiles_serializer = PACSFileSerializer() path = 'SERVICES/PACS/MyPACS/123456-Jorge/brain/brain_mri/file1.dcm' - with mock.patch.object(SwiftManager, 'obj_exists', - return_value=False) as obj_exists_mock: + with mock_storage('pacsfiles.serializers.settings') as storage_manager: with self.assertRaises(serializers.ValidationError): pacsfiles_serializer.validate_path(path) - obj_exists_mock.assert_called_with(path.strip(' ').strip('/')) + expected_fname = path.strip(' ').strip('/') + + expected_errmsg = r'Could not find this path\.' + with self.assertRaisesRegex(serializers.ValidationError, expected_errmsg): + pacsfiles_serializer.validate_path(path) + + storage_manager.upload_obj(expected_fname, b'example data') + pacsfiles_serializer.validate_path(path) # expect to not raise an exception @tag('integration') def test_integration_validate_path_failure_does_not_exist(self): @@ -68,8 +73,7 @@ def test_integration_validate_path_success(self): """ pacsfiles_serializer = PACSFileSerializer() path = 'SERVICES/PACS/MyPACS/123456-crazy/brain_crazy_study/SAG_T1_MPRAGE/file1.dcm' - swift_manager = SwiftManager(settings.SWIFT_CONTAINER_NAME, - settings.SWIFT_CONNECTION_PARAMS) + swift_manager = connect_storage(settings) # upload file to Swift storage with io.StringIO("test file") as file1: swift_manager.upload_obj(path, file1.read(), content_type='text/plain') diff --git a/chris_backend/pacsfiles/tests/test_views.py b/chris_backend/pacsfiles/tests/test_views.py index 33e4f317..a54b2d99 100755 --- a/chris_backend/pacsfiles/tests/test_views.py +++ b/chris_backend/pacsfiles/tests/test_views.py @@ -11,7 +11,7 @@ from rest_framework import status -from core.swiftmanager import SwiftManager +from core.storage import connect_storage from pacsfiles.models import PACS, PACSFile from pacsfiles import views @@ -32,8 +32,7 @@ def setUp(self): User.objects.create_user(username=self.username, password=self.password) # create a PACS file in the DB "already registered" to the server) - self.swift_manager = SwiftManager(settings.SWIFT_CONTAINER_NAME, - settings.SWIFT_CONNECTION_PARAMS) + self.swift_manager = connect_storage(settings) # upload file to Swift storage self.path = 'SERVICES/PACS/MyPACS/123456-crazy/brain_crazy_study/SAG_T1_MPRAGE/file1.dcm' with io.StringIO("test file") as file1: diff --git a/chris_backend/pipelines/tests/test_views.py b/chris_backend/pipelines/tests/test_views.py index ba480e1f..4706003b 100755 --- a/chris_backend/pipelines/tests/test_views.py +++ b/chris_backend/pipelines/tests/test_views.py @@ -16,7 +16,7 @@ from plugins.models import DefaultFloatParameter, DefaultIntParameter from pipelines.models import Pipeline, PluginPiping, DefaultPipingStrParameter -from core.swiftmanager import SwiftManager +from core.storage import connect_storage COMPUTE_RESOURCE_URL = settings.COMPUTE_RESOURCE_URL @@ -342,8 +342,7 @@ def test_integration_pipelinesourcefile_create_success(self): self.assertEqual(response.status_code, status.HTTP_201_CREATED) fpath = f'PIPELINES/{self.username}/test_pipeline0000001.yaml' # delete file from Swift storage - swift_manager = SwiftManager(settings.SWIFT_CONTAINER_NAME, - settings.SWIFT_CONNECTION_PARAMS) + swift_manager = connect_storage(settings) swift_manager.delete_obj(fpath) def test_pipelinesourcefile_create_failure_unauthenticated(self): diff --git a/chris_backend/pipelines/views.py b/chris_backend/pipelines/views.py index d1deaed5..8457125b 100755 --- a/chris_backend/pipelines/views.py +++ b/chris_backend/pipelines/views.py @@ -6,7 +6,7 @@ from rest_framework import generics, permissions from rest_framework.reverse import reverse -from core.swiftmanager import SwiftManager +from core.storage import connect_storage from core.renderers import BinaryFileRenderer from collectionjson import services from plugins.serializers import PluginSerializer @@ -135,8 +135,8 @@ def perform_destroy(self, instance): swift_path = instance.source_file.fname.name instance.delete() if swift_path: - swift_manager = SwiftManager(settings.SWIFT_CONTAINER_NAME, - settings.SWIFT_CONNECTION_PARAMS) + swift_manager = connect_storage(settings) + try: swift_manager.delete_obj(swift_path) except Exception as e: diff --git a/chris_backend/plugininstances/serializers.py b/chris_backend/plugininstances/serializers.py index a8645727..5f4dc1a0 100755 --- a/chris_backend/plugininstances/serializers.py +++ b/chris_backend/plugininstances/serializers.py @@ -9,7 +9,7 @@ from collectionjson.fields import ItemLinkField from core.utils import get_file_resource_link -from core.swiftmanager import SwiftManager +from core.storage import connect_storage from plugins.models import TYPES, Plugin from feeds.models import Feed @@ -343,8 +343,7 @@ def validate_paths(user, string): Custom function to check that a user is allowed to access the provided object storage paths. """ - swift_manager = SwiftManager(settings.SWIFT_CONTAINER_NAME, - settings.SWIFT_CONNECTION_PARAMS) + swift_manager = connect_storage(settings) path_list = [s.strip() for s in string.split(',')] for path in path_list: path_parts = pathlib.Path(path).parts diff --git a/chris_backend/plugininstances/services/manager.py b/chris_backend/plugininstances/services/manager.py index 59575519..777d8336 100755 --- a/chris_backend/plugininstances/services/manager.py +++ b/chris_backend/plugininstances/services/manager.py @@ -56,7 +56,8 @@ from django.conf import settings from django.db.utils import IntegrityError -from core.swiftmanager import SwiftManager, ClientException +from core.storage import connect_storage +from swiftclient.exceptions import ClientException from core.utils import json_zip2str from core.models import ChrisInstance from plugininstances.models import PluginInstance, PluginInstanceFile, PluginInstanceLock @@ -84,8 +85,7 @@ def __init__(self, plugin_instance): cr = self.c_plugin_inst.compute_resource self.pfcon_client = pfcon.Client(cr.compute_url, cr.compute_auth_token) - self.swift_manager = SwiftManager(settings.SWIFT_CONTAINER_NAME, - settings.SWIFT_CONNECTION_PARAMS) + self.swift_manager = connect_storage(settings) def _refresh_compute_resource_auth_token(self): """ diff --git a/chris_backend/plugininstances/tests/test_manager.py b/chris_backend/plugininstances/tests/test_manager.py index 92a16752..5befb846 100755 --- a/chris_backend/plugininstances/tests/test_manager.py +++ b/chris_backend/plugininstances/tests/test_manager.py @@ -9,7 +9,7 @@ from django.contrib.auth.models import User from django.conf import settings -from core.swiftmanager import SwiftManager +from core.storage import connect_storage from plugins.models import PluginMeta, Plugin from plugins.models import PluginParameter from plugininstances.models import PluginInstance, PathParameter, ComputeResource @@ -25,8 +25,7 @@ def setUp(self): # avoid cluttered console output (for instance logging all the http requests) logging.disable(logging.WARNING) - self.swift_manager = SwiftManager(settings.SWIFT_CONTAINER_NAME, - settings.SWIFT_CONNECTION_PARAMS) + self.swift_manager = connect_storage(settings) self.plugin_fs_name = "simplefsapp" self.username = 'foo' diff --git a/chris_backend/plugininstances/tests/test_serializers.py b/chris_backend/plugininstances/tests/test_serializers.py index 27758ba0..94577f26 100755 --- a/chris_backend/plugininstances/tests/test_serializers.py +++ b/chris_backend/plugininstances/tests/test_serializers.py @@ -9,7 +9,7 @@ from plugins.models import PluginMeta, Plugin, PluginParameter, ComputeResource from plugininstances.models import PluginInstance -from plugininstances.serializers import SwiftManager +from core.storage.helpers import mock_storage from plugininstances.serializers import PluginInstanceSerializer from plugininstances.serializers import (PathParameterSerializer, UnextpathParameterSerializer) @@ -317,11 +317,9 @@ def test_validate_value_fail_path_does_not_exist(self): """ user = User.objects.get(username=self.username) path_parm_serializer = PathParameterSerializer(user=user) - with mock.patch.object(SwiftManager, 'path_exists', - return_value=False) as path_exists_mock: + with mock_storage('plugininstances.serializers.settings'): with self.assertRaises(serializers.ValidationError): path_parm_serializer.validate_value(self.username) - path_exists_mock.assert_called_with(self.username) def test_validate_value_success(self): """ @@ -338,8 +336,16 @@ def test_validate_value_success(self): path_parm_serializer = PathParameterSerializer(user=user) value = "{}, {}/feed_{} ".format(self.username, self.other_username, pl_inst.feed.id) - with mock.patch.object(SwiftManager, 'path_exists', - return_value=True) as path_exists_mock: + with mock_storage('plugininstances.serializers.settings') as storage_manager: + storage_manager.upload_obj( + f'{self.username}/uploads/dummy_data.txt', + b'dummy data' + ) + storage_manager.upload_obj( + f'{self.other_username}/feed_{pl_inst.feed.id}/' + f'{pl_inst.plugin.meta.name}_{pl_inst.id}/data/dummy_data.txt', + b'dummy data' + ) returned_value = path_parm_serializer.validate_value(value) self.assertEqual(returned_value, "{},{}/feed_{}".format(self.username, self.other_username, @@ -412,11 +418,9 @@ def test_validate_value_fail_path_does_not_exist(self): """ user = User.objects.get(username=self.username) path_parm_serializer = UnextpathParameterSerializer(user=user) - with mock.patch.object(SwiftManager, 'path_exists', - return_value=False) as path_exists_mock: + with mock_storage('plugininstances.serializers.settings'): with self.assertRaises(serializers.ValidationError): path_parm_serializer.validate_value(self.username) - path_exists_mock.assert_called_with(self.username) def test_validate_value_success(self): """ @@ -433,8 +437,17 @@ def test_validate_value_success(self): path_parm_serializer = UnextpathParameterSerializer(user=user) value = "{}, {}/feed_{} ".format(self.username, self.other_username, pl_inst.feed.id) - with mock.patch.object(SwiftManager, 'path_exists', - return_value=True) as path_exists_mock: + with mock_storage('plugininstances.serializers.settings') as storage_manager: + storage_manager.upload_obj( + f'{self.username}/uploads/dummy_data.txt', + b'dummy data' + ) + storage_manager.upload_obj( + f'{self.other_username}/feed_{pl_inst.feed.id}/' + f'{pl_inst.plugin.meta.name}_{pl_inst.id}/data/dummy_data.txt', + b'dummy data' + ) + returned_value = path_parm_serializer.validate_value(value) self.assertEqual(returned_value, "{},{}/feed_{}".format(self.username, self.other_username, diff --git a/chris_backend/plugininstances/tests/test_views.py b/chris_backend/plugininstances/tests/test_views.py index 9ab98c95..fceb5e0f 100755 --- a/chris_backend/plugininstances/tests/test_views.py +++ b/chris_backend/plugininstances/tests/test_views.py @@ -16,7 +16,7 @@ from core.celery import app as celery_app from core.celery import task_routes -from core.swiftmanager import SwiftManager +from core.storage import connect_storage from plugins.models import PluginMeta, Plugin, PluginParameter, ComputeResource from plugininstances.models import PluginInstance, PluginInstanceFile from plugininstances.models import PathParameter, FloatParameter @@ -96,8 +96,7 @@ def tearDownClass(cls): def setUp(self): - self.swift_manager = SwiftManager(settings.SWIFT_CONTAINER_NAME, - settings.SWIFT_CONNECTION_PARAMS) + self.swift_manager = connect_storage(settings) self.chris_username = 'chris' self.chris_password = 'chris12' self.username = 'foo' @@ -946,8 +945,7 @@ def setUp(self): plugin=plugin, owner=user, compute_resource=plugin.compute_resources.all()[0]) # create a plugin instance file associated to the plugin instance - self.swift_manager = SwiftManager(settings.SWIFT_CONTAINER_NAME, - settings.SWIFT_CONNECTION_PARAMS) + self.swift_manager = connect_storage(settings) # upload file to Swift storage self.path = 'tests/file1.txt' with io.StringIO("test file") as file1: diff --git a/chris_backend/servicefiles/serializers.py b/chris_backend/servicefiles/serializers.py index bc2ab52d..c4121fe1 100755 --- a/chris_backend/servicefiles/serializers.py +++ b/chris_backend/servicefiles/serializers.py @@ -6,7 +6,7 @@ from collectionjson.fields import ItemLinkField from core.utils import get_file_resource_link -from core.swiftmanager import SwiftManager +from core.storage import connect_storage from .models import Service, ServiceFile from .models import REGISTERED_SERVICES @@ -85,8 +85,7 @@ def validate(self, data): error_msg = "File path must start with '%s'." % prefix raise serializers.ValidationError([error_msg]) # verify that the file is indeed already in Swift - swift_manager = SwiftManager(settings.SWIFT_CONTAINER_NAME, - settings.SWIFT_CONNECTION_PARAMS) + swift_manager = connect_storage(settings) try: swift_path_exists = swift_manager.obj_exists(path) except Exception as e: diff --git a/chris_backend/servicefiles/tests/test_serializers.py b/chris_backend/servicefiles/tests/test_serializers.py index 4a175c9d..4c38fb1f 100755 --- a/chris_backend/servicefiles/tests/test_serializers.py +++ b/chris_backend/servicefiles/tests/test_serializers.py @@ -10,8 +10,7 @@ from servicefiles.models import Service, ServiceFile from servicefiles.serializers import ServiceFileSerializer -from servicefiles.serializers import SwiftManager - +from core.storage.helpers import connect_storage, mock_storage class ServiceFileSerializerTests(TestCase): @@ -52,13 +51,12 @@ def test_validate_updates_validated_data(self): path = 'SERVICES/MyService/123456-crazy/brain_crazy_study/brain_crazy_mri/file1.dcm' data = {'service_name': 'MyService', 'path': path} servicefiles_serializer = ServiceFileSerializer() - with mock.patch.object(SwiftManager, 'obj_exists', - return_value=True) as obj_exists_mock: + with mock_storage('servicefiles.serializers.settings') as storage_manager: + storage_manager.upload_obj(path, b'dummy data') new_data = servicefiles_serializer.validate(data) self.assertIn('service', new_data) self.assertNotIn('service_name', new_data) self.assertEqual(new_data.get('path'), path.strip(' ').strip('/')) - obj_exists_mock.assert_called_with(new_data.get('path')) def test_validate_failure_path_does_not_start_with_SERVICES_PACS(self): """ @@ -79,11 +77,9 @@ def test_validate_failure_path_does_not_exist(self): path = 'SERVICES/MyService/123456-crazy/brain_crazy_study/brain_crazy_mri/file1.dcm' data = {'service_name': 'MyService', 'path': path} servicefiles_serializer = ServiceFileSerializer() - with mock.patch.object(SwiftManager, 'obj_exists', - return_value=False) as obj_exists_mock: + with mock_storage('servicefiles.serializers.settings'): with self.assertRaises(serializers.ValidationError): servicefiles_serializer.validate(data) - obj_exists_mock.assert_called_with(path.strip(' ').strip('/')) @tag('integration') def test_integration_validate_path_failure_does_not_exist(self): @@ -106,8 +102,7 @@ def test_integration_validate_path_success(self): data = {'service_name': 'MyService', 'path': path} servicefiles_serializer = ServiceFileSerializer() - swift_manager = SwiftManager(settings.SWIFT_CONTAINER_NAME, - settings.SWIFT_CONNECTION_PARAMS) + swift_manager = connect_storage(settings) # upload file to Swift storage with io.StringIO("test file") as file1: swift_manager.upload_obj(path, file1.read(), content_type='text/plain') diff --git a/chris_backend/servicefiles/tests/test_views.py b/chris_backend/servicefiles/tests/test_views.py index 06c71420..eeac5af6 100755 --- a/chris_backend/servicefiles/tests/test_views.py +++ b/chris_backend/servicefiles/tests/test_views.py @@ -11,7 +11,7 @@ from rest_framework import status -from core.swiftmanager import SwiftManager +from core.storage import connect_storage from servicefiles.models import Service, ServiceFile from servicefiles import views @@ -35,8 +35,8 @@ def setUp(self): service.save() # create a service file in the DB "already registered" to the server) - self.swift_manager = SwiftManager(settings.SWIFT_CONTAINER_NAME, - settings.SWIFT_CONNECTION_PARAMS) + self.swift_manager = connect_storage(settings) + # upload file to Swift storage self.path = 'SERVICES/MyService/123456-crazy/brain_crazy_study/brain_crazy_mri/file1.dcm' with io.StringIO("test file") as file1: diff --git a/chris_backend/uploadedfiles/tests/test_views.py b/chris_backend/uploadedfiles/tests/test_views.py index 74ad6b8d..b595bd70 100755 --- a/chris_backend/uploadedfiles/tests/test_views.py +++ b/chris_backend/uploadedfiles/tests/test_views.py @@ -11,7 +11,7 @@ from rest_framework import status -from core.swiftmanager import SwiftManager +from core.storage.helpers import connect_storage, mock_storage from uploadedfiles.models import UploadedFile, uploaded_file_path from uploadedfiles import views @@ -42,8 +42,7 @@ def setUp(self): password=self.password) # create a file in the DB "already uploaded" to the server) - self.swift_manager = SwiftManager(settings.SWIFT_CONTAINER_NAME, - settings.SWIFT_CONNECTION_PARAMS) + self.swift_manager = connect_storage(settings) # upload file to Swift storage self.upload_path = f'{self.username}/uploads/file1.txt' with io.StringIO("test file") as file1: @@ -142,10 +141,9 @@ def test_uploadedfile_update_failure_access_denied(self): def test_uploadedfile_delete_success(self): self.client.login(username=self.username, password=self.password) swift_path = self.uploadedfile.fname.name - mocked_method = 'uploadedfiles.views.SwiftManager.delete_obj' - with mock.patch(mocked_method) as delete_obj_mock: + with mock_storage('uploadedfiles.views.settings') as storage_manager: response = self.client.delete(self.read_update_delete_url) - delete_obj_mock.assert_called_with(swift_path) + self.assertFalse(storage_manager.obj_exists(swift_path)) self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT) self.assertEqual(UploadedFile.objects.count(), 0) diff --git a/chris_backend/uploadedfiles/views.py b/chris_backend/uploadedfiles/views.py index ada41754..a8d9ca0a 100755 --- a/chris_backend/uploadedfiles/views.py +++ b/chris_backend/uploadedfiles/views.py @@ -8,7 +8,7 @@ from collectionjson import services from core.renderers import BinaryFileRenderer -from core.swiftmanager import SwiftManager +from core.storage import connect_storage from .models import UploadedFile, UploadedFileFilter from .serializers import UploadedFileSerializer @@ -95,8 +95,8 @@ def perform_update(self, serializer): user_file = self.get_object() old_swift_path = user_file.fname.name serializer.save() - swift_manager = SwiftManager(settings.SWIFT_CONTAINER_NAME, - settings.SWIFT_CONNECTION_PARAMS) + swift_manager = connect_storage(settings) + try: swift_manager.delete_obj(old_swift_path) except Exception as e: @@ -108,8 +108,8 @@ def perform_destroy(self, instance): """ swift_path = instance.fname.name instance.delete() - swift_manager = SwiftManager(settings.SWIFT_CONTAINER_NAME, - settings.SWIFT_CONNECTION_PARAMS) + swift_manager = connect_storage(settings) + try: swift_manager.delete_obj(swift_path) except Exception as e: diff --git a/chris_backend/users/serializers.py b/chris_backend/users/serializers.py index 0b4cbb67..08f57369 100755 --- a/chris_backend/users/serializers.py +++ b/chris_backend/users/serializers.py @@ -7,7 +7,7 @@ from rest_framework import serializers from rest_framework.validators import UniqueValidator -from core.swiftmanager import SwiftManager +from core.storage import connect_storage from uploadedfiles.models import UploadedFile @@ -38,8 +38,7 @@ def create(self, validated_data): email = validated_data.get('email') password = validated_data.get('password') user = User.objects.create_user(username, email, password) - swift_manager = SwiftManager(settings.SWIFT_CONTAINER_NAME, - settings.SWIFT_CONNECTION_PARAMS) + swift_manager = connect_storage(settings) welcome_file_path = '%s/uploads/welcome.txt' % username try: with io.StringIO('Welcome to ChRIS!') as f: diff --git a/chris_backend/users/tests/test_serializers.py b/chris_backend/users/tests/test_serializers.py index a66b2c62..9b690efa 100755 --- a/chris_backend/users/tests/test_serializers.py +++ b/chris_backend/users/tests/test_serializers.py @@ -1,15 +1,13 @@ import logging - -from unittest import mock - from django.test import TestCase from rest_framework import serializers from uploadedfiles.models import UploadedFile -from users.serializers import UserSerializer, SwiftManager +from users.serializers import UserSerializer +from core.storage.helpers import mock_storage class UserSerializerTests(TestCase): @@ -37,9 +35,7 @@ def test_create(self): user_serializer = UserSerializer() validated_data = {'username': self.username, 'password': self.password, 'email': self.email} - with mock.patch.object(SwiftManager, 'upload_obj', - return_value=None) as upload_obj_mock: - + with mock_storage('users.serializers.settings') as storage_manager: user = user_serializer.create(validated_data) self.assertEqual(user.username, self.username) @@ -50,8 +46,7 @@ def test_create(self): welcome_file_path = '%s/uploads/welcome.txt' % self.username welcome_file = UploadedFile.objects.get(owner=user) self.assertEqual(welcome_file.fname.name, welcome_file_path) - upload_obj_mock.assert_called_with(welcome_file_path, mock.ANY, - content_type='text/plain') + self.assertTrue(storage_manager.obj_exists(welcome_file_path)) def test_validate_username(self): """ diff --git a/chris_backend/users/tests/test_views.py b/chris_backend/users/tests/test_views.py index 57338cca..feb887a4 100755 --- a/chris_backend/users/tests/test_views.py +++ b/chris_backend/users/tests/test_views.py @@ -1,7 +1,6 @@ import logging import json -from unittest import mock from django.test import TestCase, tag from django.contrib.auth.models import User @@ -11,7 +10,8 @@ from rest_framework import status from uploadedfiles.models import UploadedFile -from users.serializers import SwiftManager +from core.storage.helpers import mock_storage, connect_storage + class UserViewTests(TestCase): @@ -47,8 +47,7 @@ def setUp(self): {"name": "email", "value": self.email}]}}) def test_user_create_success(self): - with mock.patch.object(SwiftManager, 'upload_obj', - return_value=None) as upload_obj_mock: + with mock_storage('users.serializers.settings') as storage_manager: response = self.client.post(self.create_url, data=self.post, content_type=self.content_type) self.assertEqual(response.status_code, status.HTTP_201_CREATED) @@ -56,8 +55,7 @@ def test_user_create_success(self): self.assertEqual(response.data["email"], self.email) welcome_file_path = '%s/uploads/welcome.txt' % self.username - upload_obj_mock.assert_called_with(welcome_file_path, mock.ANY, - content_type='text/plain') + self.assertTrue(storage_manager.obj_exists(welcome_file_path)) @tag('integration') def test_integration_user_create_success(self): @@ -73,8 +71,8 @@ def test_integration_user_create_success(self): self.assertEqual(welcome_file.fname.name, welcome_file_path) # delete welcome file - swift_manager = SwiftManager(settings.SWIFT_CONTAINER_NAME, - settings.SWIFT_CONNECTION_PARAMS) + swift_manager = connect_storage(settings) + swift_manager.delete_obj(welcome_file_path) def test_user_create_failure_already_exists(self): diff --git a/docker-compose_noswift.yml b/docker-compose_noswift.yml new file mode 100755 index 00000000..ccbfe250 --- /dev/null +++ b/docker-compose_noswift.yml @@ -0,0 +1,257 @@ +# An alternative docker-compose_dev.yml which uses a local volume to store files +# instead of Swift Object Storage. + +version: '3.9' + +services: + chrisomatic: + image: ghcr.io/fnndsc/chrisomatic:0.4.1 + volumes: + - "./chrisomatic:/etc/chrisomatic:ro" + - "/var/run/docker.sock:/var/run/docker.sock:rw" + working_dir: /etc/chrisomatic + userns_mode: host + networks: + - local + depends_on: + - chris_dev + - chris_store + profiles: + - tools + + permission_workaround: + image: alpine + command: chmod 777 /tmp/chris + volumes: + - chris_files:/tmp/chris + + chris_dev: + image: ${CHRISREPO}/chris:dev + build: + context: . + args: + ENVIRONMENT: local + stdin_open: true # docker run -i + tty: true # docker run -t + volumes: + - ./chris_backend:/home/localuser/chris_backend:z + - chris_files:/var/chris + environment: + - DJANGO_SETTINGS_MODULE=config.settings.local + - DJANGO_DB_MIGRATE=on + - DJANGO_COLLECTSTATIC=off + command: python manage.py runserver 0.0.0.0:8000 + ports: + - "8000:8000" + depends_on: + - chris_dev_db + - swift_service + - queue + - chris_store + networks: + local: + aliases: + - chrisdev.local + remote: # bc special automated tests worker runs within CUBE, not needed in prod + minikube: + extra_hosts: + - "${PFCONDNS:-lhost}:${PFCONIP:-127.0.0.1}" # used only for kubernetes, not needed in prod + labels: + name: "ChRIS_ultron_backEnd" + role: "Backend development server" + org.chrisproject.role: "ChRIS ultron backEnd" + + worker: + image: ${CHRISREPO}/chris:dev + build: + context: . + args: + ENVIRONMENT: local + volumes: + - ./chris_backend:/home/localuser/chris_backend:z + - chris_files:/tmp/chris + environment: + - DJANGO_SETTINGS_MODULE=config.settings.local + - DJANGO_DB_MIGRATE=off + - DJANGO_COLLECTSTATIC=off + - CELERY_RDB_HOST=0.0.0.0 + - CELERY_RDB_PORT=6900 + command: celery -A core worker -c 3 -l DEBUG -Q main1,main2 + ports: + - "6900-6905:6900-6905" + depends_on: + - chris_dev_db + - swift_service + - queue + # service also depends on pfcon service defined in swarm/docker-compose_remote.yml + networks: + - local + - remote + # When the remote ancillary service pfcon is deployed using kubernetes it can not + # use (connect to) an external docker overlay network: remote. In that case we + # instead use extra_hosts to let the worker know pfcon's IP address. The required + # shell variables referenced here must then be set like this: PFCONDNS=pfcon.remote, + # PFCONIP= and REMOTENETWORK=false + + # if you are using minikube to run the kubernetes cluster, Set the environment variable in make.sh + # MINIKUBENETWORK to true and the HOSTIP will be minikube's ip, to get this, you can run the + # command 'minikube ip' on the terminal + extra_hosts: + - "${PFCONDNS:-lhost}:${PFCONIP:-127.0.0.1}" + labels: + name: "ChRIS_ultron_backEnd Asynchronous Tasks Worker" + role: "Backend development async task worker" + + worker_periodic: + image: ${CHRISREPO}/chris:dev + build: + context: . + args: + ENVIRONMENT: local + volumes: + - ./chris_backend:/home/localuser/chris_backend:z + - chris_files:/tmp/chris + + environment: + - DJANGO_SETTINGS_MODULE=config.settings.local + - DJANGO_DB_MIGRATE=off + - DJANGO_COLLECTSTATIC=off + command: celery -A core worker -c 1 -l DEBUG -Q periodic + depends_on: + - chris_dev_db + - queue + networks: + - local + labels: + name: "ChRIS_ultron_backEnd Periodic Task Worker" + role: "Backend development periodic task worker" + + scheduler: + image: ${CHRISREPO}/chris:dev + build: + context: . + args: + ENVIRONMENT: local + volumes: + - ./chris_backend:/home/localuser/chris_backend:z + - chris_files:/tmp/chris + environment: + - DJANGO_SETTINGS_MODULE=config.settings.local + - DJANGO_DB_MIGRATE=off + - DJANGO_COLLECTSTATIC=off + command: celery -A core beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler + depends_on: + - chris_dev_db + - queue + # restart until Django DB migrations are ready + deploy: + restart_policy: + condition: on-failure + networks: + - local + labels: + name: "ChRIS_ultron_backEnd Periodic Tasks Scheduler" + role: "Backend development periodic tasks scheduler" + + chris_dev_db: + image: postgres:13 + volumes: + - chris_dev_db_data:/var/lib/postgresql/data + environment: + - POSTGRES_DB=chris_dev + - POSTGRES_USER=chris + - POSTGRES_PASSWORD=Chris1234 + networks: + - local + labels: + name: "ChRIS_ultron_backEnd PostgreSQL Database" + role: "Backend development database" + + queue: + image: rabbitmq:3 + hostname: 'queue' + volumes: + - queue_data:/var/lib/rabbitmq + networks: + - local + labels: + name: "ChRIS_ultron_backEnd Asynchronous Task Queue" + role: "Backend development async task queue" + + chris_store: + image: ${STOREREPO}/chris_store + environment: + - DJANGO_SETTINGS_MODULE=config.settings.production + - DJANGO_DB_MIGRATE=on + - DJANGO_ALLOWED_HOSTS=* + - DJANGO_SECRET_KEY="w1kxu^l=@pnsf!5piqz6!!5kdcdpo79y6jebbp+2244yjm*#+k" + - DJANGO_CORS_ALLOW_ALL_ORIGINS=true + - DJANGO_CORS_ALLOWED_ORIGINS=https://babymri.org + - DJANGO_SECURE_PROXY_SSL_HEADER= + - DJANGO_USE_X_FORWARDED_HOST=false + - DATABASE_HOST=chris_store_db + - DATABASE_PORT=5432 + - SWIFT_AUTH_URL=http://swift_service:8080/auth/v1.0 + - POSTGRES_DB=chris_store + - POSTGRES_USER=chris + - POSTGRES_PASSWORD=Chris1234 + - SWIFT_USERNAME=chris:chris1234 + - SWIFT_KEY=testing + - SWIFT_CONTAINER_NAME=store_users + ports: + - "8010:8010" + depends_on: + - chris_store_db + - swift_service + networks: + local: + aliases: + - chris-store.local + labels: + name: "ChRIS_store" + role: "Chris store service" + + chris_store_db: + image: postgres:13 + volumes: + - chris_store_db_data:/var/lib/postgresql/data + environment: + - POSTGRES_DB=chris_store + - POSTGRES_USER=chris + - POSTGRES_PASSWORD=Chris1234 + networks: + - local + labels: + name: "ChRIS_store PostgreSQL Database" + role: "Chris store database" + + swift_service: + image: ${SWIFTREPO}/docker-swift-onlyone + init: true + volumes: + - swift_storage_dev:/srv + environment: + - SWIFT_USERNAME=chris:chris1234 + - SWIFT_KEY=testing + ports: + - "8080:8080" + networks: + - local + labels: + name: "Swift" + role: "Swift object storage service" + + +networks: + local: + remote: + external: ${REMOTENETWORK:-true} + minikube: + external: ${MINIKUBENETWORK:-false} + +volumes: + chris_dev_db_data: + chris_store_db_data: + queue_data: + swift_storage_dev: + chris_files: