From 9e8245387f06db75e431938e9e91ae5ffd8fa3c8 Mon Sep 17 00:00:00 2001 From: "John T. Wodder II" Date: Tue, 27 Apr 2021 12:09:09 -0400 Subject: [PATCH] Remove Girder code from API --- README.md | 12 ---- dandiapi/api/girder.py | 100 --------------------------------- dandiapi/api/tests/girder.py | 77 ------------------------- dandiapi/settings.py | 4 -- dev/.env.docker-compose | 2 - dev/.env.docker-compose-native | 2 - 6 files changed, 197 deletions(-) delete mode 100644 dandiapi/api/girder.py delete mode 100644 dandiapi/api/tests/girder.py diff --git a/README.md b/README.md index 28978c93f..6948758c8 100644 --- a/README.md +++ b/README.md @@ -89,18 +89,6 @@ Useful sub-commands include: To automatically reformat all code to comply with some (but not all) of the style checks, run `tox -e format`. -## Connecting to `dandiarchive` -`dandiarchive` is another application and will need to be setup and run separately. -1. Login to the `dandiarchive` Girder client using the `publish` admin account. If -you followed the README it will be located at `http://localhost:8080/`. - * **NOTE**: the username of the Girder admin account used here must be `publish` for publishing to work properly. If an admin account with that username doesn't exist, it must be created. -2. Navigate to account settings, click on the 'API keys' tab, and generate an API key. -3. Save this API key and the Girder client URL in environment variables named `DJANGO_DANDI_GIRDER_API_KEY` - and `DJANGO_DANDI_GIRDER_API_URL`. -4. Run `dandi-publish` as described above. - -**NOTE**: `dandiarchive` also needs to be configured to connect to `dandi-publish`. See its README for instructions. - ## API Authentication Read-only API endpoints (i.e. `GET`, `HEAD`) do not require any authentication. All other endpoints require token authentication diff --git a/dandiapi/api/girder.py b/dandiapi/api/girder.py deleted file mode 100644 index 290798bcd..000000000 --- a/dandiapi/api/girder.py +++ /dev/null @@ -1,100 +0,0 @@ -import contextlib -from dataclasses import dataclass -from typing import Any, Dict, Iterator, List - -from django.conf import settings -from httpx import Client - - -class GirderError(Exception): - pass - - -@dataclass -class GirderFile: - girder_id: str - path: str - metadata: Dict[str, Any] - size: int - - -class GirderClient(Client): - def __init__(self, authenticate=False, **kwargs): - girder_api_url = settings.DANDI_GIRDER_API_URL - girder_api_key = settings.DANDI_GIRDER_API_KEY - if not girder_api_url.endswith('/'): - girder_api_url += '/' - - kwargs.setdefault('base_url', girder_api_url) - super().__init__(**kwargs) - - if not authenticate: - return - - # Fetch the token using the API key - resp = self.post('api_key/token', params={'key': girder_api_key}) - if resp.status_code != 200: - raise GirderError('Failed to authenticate with Girder') - token = resp.json()['authToken']['token'] - - # Include the token header for all subsequent requests - self.headers = {'Girder-Token': token} - - def get_json(self, *args, **kwargs) -> Any: - resp = self.get(*args, **kwargs) - resp.raise_for_status() - return resp.json() - - def get_folder(self, folder_id: str) -> Dict: - return self.get_json(f'folder/{folder_id}') - - def get_subfolders(self, folder_id: str) -> List[Dict]: - return self.get_json( - 'folder', params={'parentId': folder_id, 'parentType': 'folder', 'limit': 0} - ) - - def get_items(self, folder_id: str) -> List[Dict]: - return self.get_json('item', params={'folderId': folder_id, 'limit': 0}) - - def get_item_files(self, item_id: str) -> List[Dict]: - return self.get_json(f'item/{item_id}/files') - - @contextlib.contextmanager - def iter_file_content(self, file_id: str) -> Iterator[bytes]: - with self.stream('GET', f'file/{file_id}/download') as resp: - resp.raise_for_status() - yield resp.iter_bytes() - - @contextlib.contextmanager - def dandiset_lock(self, dandiset_identifier: str) -> None: - resp = self.post(f'dandi/{dandiset_identifier}/lock') - if resp.status_code != 200: - raise GirderError(f'Failed to lock dandiset {dandiset_identifier}') - try: - yield - finally: - resp = self.post(f'dandi/{dandiset_identifier}/unlock') - if resp.status_code != 200: - raise GirderError(f'Failed to unlock dandiset {dandiset_identifier}') - - def files_in_folder(self, folder_id: str, current_path: str = '/') -> Iterator[GirderFile]: - for item in self.get_items(folder_id): - file_list = self.get_item_files(item['_id']) - if len(file_list) != 1: - raise GirderError(f'Found {len(file_list)} files in item {item["_id"]}') - - f = file_list[0] - if f['size'] == 0: - raise GirderError(f'Found empty file {f["_id"]}') - - yield GirderFile( - girder_id=f['_id'], - # Use item name instead of file name, since it's more likely to reflect an - # explicit rename operation in Girder - path=f'{current_path}{item["name"]}', - metadata=item['meta'], - size=f['size'], - ) - - for subfolder in self.get_subfolders(folder_id): - yield from self.files_in_folder(subfolder['_id'], f'{current_path}{subfolder["name"]}/') diff --git a/dandiapi/api/tests/girder.py b/dandiapi/api/tests/girder.py deleted file mode 100644 index 025fdac0e..000000000 --- a/dandiapi/api/tests/girder.py +++ /dev/null @@ -1,77 +0,0 @@ -import contextlib -from typing import Any, Dict, Iterator, List - -import factory - -from dandiapi.api.girder import GirderClient, GirderFile - - -class _GirderClientFolderFactory(factory.DictFactory): - _id = factory.Faker('hexify', text='^' * 24) - name = factory.Faker('bothify', text='sub-????????##') - # A basic folder has a 'meta' key with no content - meta = factory.Dict({}) - - -class _GirderClientDraftFolderFactory(_GirderClientFolderFactory): - name = factory.Faker('numerify', text='#' * 6) - meta = factory.Dict( - { - 'dandiset': factory.Dict( - {'name': factory.Faker('sentence'), 'description': factory.Faker('paragraph')} - ) - } - ) - - -class _GirderClientItemFactory(factory.DictFactory): - _id = factory.Faker('hexify', text='^' * 24) - name = factory.Faker('file_name', extension='nwb') - meta = factory.Dict({}) - - -class _GirderClientFileFactory(factory.DictFactory): - _id = factory.Faker('hexify', text='^' * 24) - size = factory.Faker('random_int', min=10, max=100) - - -class MockGirderClient(GirderClient): - def __init__(self, authenticate: bool = False, **kwargs) -> None: - super().__init__(authenticate=False, **kwargs) - - def get_json(self, *args, **kwargs) -> Any: - raise NotImplementedError - - def get_folder(self, folder_id: str) -> Dict: - if folder_id == 'magic_draft_folder_id': - return _GirderClientDraftFolderFactory() - else: - return _GirderClientFolderFactory() - - def get_subfolders(self, folder_id: str) -> List[Dict]: - return _GirderClientFolderFactory.build_batch(1) - - def get_items(self, folder_id: str) -> List[Dict]: - return _GirderClientItemFactory.build_batch(1) - - def get_item_files(self, item_id: str) -> List[Dict]: - file_size = len(b'Fake DANDI file content.Part 2.') - return _GirderClientFileFactory.build_batch(1, size=file_size) - - @contextlib.contextmanager - def iter_file_content(self, file_id: str) -> Iterator[bytes]: - yield iter([b'Fake DANDI file content.', b'Part 2.']) - - @contextlib.contextmanager - def dandiset_lock(self, dandiset_identifier: str) -> None: - yield - - -class GirderFileFactory(factory.Factory): - class Meta: - model = GirderFile - - girder_id = factory.Faker('hexify', text='^' * 24) - path = factory.Faker('file_path', extension='nwb') - metadata = factory.Dict({}) - size = factory.Faker('random_int', min=10, max=100) diff --git a/dandiapi/settings.py b/dandiapi/settings.py index 44c72e86b..3b7291458 100644 --- a/dandiapi/settings.py +++ b/dandiapi/settings.py @@ -47,8 +47,6 @@ def before_binding(configuration: Type[ComposedConfiguration]): DANDI_DANDISETS_BUCKET_NAME = values.Value(environ_required=True) DANDI_DANDISETS_BUCKET_PREFIX = values.Value(default='', environ=True) - DANDI_GIRDER_API_URL = values.URLValue(environ_required=True) - DANDI_GIRDER_API_KEY = values.Value(environ_required=True) DANDI_SCHEMA_VERSION = values.Value(environ_required=True) DANDI_DOI_API_URL = values.URLValue(environ=True) @@ -70,8 +68,6 @@ class TestingConfiguration(DandiMixin, TestingBaseConfiguration): DANDI_DANDISETS_BUCKET_NAME = 'test-dandiapi-dandisets' DANDI_DANDISETS_BUCKET_PREFIX = 'test-prefix/' - DANDI_GIRDER_API_KEY = 'testkey' - DANDI_GIRDER_API_URL = 'http://girder.test/api/v1' class ProductionConfiguration(DandiMixin, ProductionBaseConfiguration): diff --git a/dev/.env.docker-compose b/dev/.env.docker-compose index fe2cc85cf..be791f332 100644 --- a/dev/.env.docker-compose +++ b/dev/.env.docker-compose @@ -7,6 +7,4 @@ DJANGO_MINIO_STORAGE_SECRET_KEY=minioSecretKey DJANGO_STORAGE_BUCKET_NAME=django-storage DJANGO_MINIO_STORAGE_MEDIA_URL=http://localhost:9000/django-storage DJANGO_DANDI_DANDISETS_BUCKET_NAME=dandi-dandisets -DJANGO_DANDI_GIRDER_API_URL=https://girder.dandiarchive.org/api/v1 -DJANGO_DANDI_GIRDER_API_KEY=girderkey DJANGO_DANDI_SCHEMA_VERSION=0.3.0 diff --git a/dev/.env.docker-compose-native b/dev/.env.docker-compose-native index b6b6f5878..41333f5ae 100644 --- a/dev/.env.docker-compose-native +++ b/dev/.env.docker-compose-native @@ -6,6 +6,4 @@ DJANGO_MINIO_STORAGE_ACCESS_KEY=minioAccessKey DJANGO_MINIO_STORAGE_SECRET_KEY=minioSecretKey DJANGO_STORAGE_BUCKET_NAME=django-storage DJANGO_DANDI_DANDISETS_BUCKET_NAME=dandi-dandisets -DJANGO_DANDI_GIRDER_API_URL=https://girder.dandiarchive.org/api/v1 -DJANGO_DANDI_GIRDER_API_KEY=girderkey DJANGO_DANDI_SCHEMA_VERSION=0.3.0