Remove Girder support

DEVELOPMENT.md

@@ -73,13 +73,7 @@ in the simplest scenario.  Additional options might come handy, such as
   `clear` to have cache `clear()`ed before use.
 
 - `DANDI_INSTANCEHOST` -- defaults to `localhost`. Point to host/IP which hosts
-  a local instance of dandiarchive. Typically,
-  `DANDI_REUSE_LOCAL_DOCKER_TESTS_API_KEY`
-  should also be set.
-
-- `DANDI_REUSE_LOCAL_DOCKER_TESTS_API_KEY` -- make the
- `local_docker_compose*` fixtures use the given API key for
- `DANDI_INSTANCEHOST` instead of spinning up a new environment with a new key.
+  a local instance of dandiarchive.
 
 - `DANDI_TESTS_PERSIST_DOCKER_COMPOSE` -- When set, the tests will reuse the
   same Docker containers across test runs instead of creating & destroying a

dandi/cli/base.py

@@ -101,10 +101,6 @@ def map_to_click_exceptions(f):
     @click.pass_obj
     @wraps(f)
     def wrapper(obj, *args, **kwargs):
-        import girder_client as gcl
-
-        from ..girder import get_HttpError_response
-
         try:
             return f(*args, **kwargs)
         # Prints global Usage: useless in majority of cases.
@@ -113,14 +109,7 @@ def wrapper(obj, *args, **kwargs):
         # except ValueError as e:
         #     raise click.UsageError(str(e))
         except Exception as e:
-            if isinstance(e, gcl.HttpError):
-                resp = get_HttpError_response(e)
-                if resp is None:
-                    e_str = str(e)
-                else:
-                    e_str = resp.get("message", str(e))
-            else:
-                e_str = str(e)
+            e_str = str(e)
             lgr.debug("Caught exception %s", e_str)
             if not map_to_click_exceptions._do_map:
                 raise

dandi/cli/cmd_ls.py

@@ -57,7 +57,6 @@ def ls(paths, schema, metadata, fields=None, format="auto", recursive=False, job
     # TODO: more logical ordering in case of fields = None
     from .formatter import JSONFormatter, PYOUTFormatter, YAMLFormatter
     from ..consts import metadata_all_fields
-    from ..dandiapi import DandiAPIClient
 
     # TODO: avoid
     from ..support.pyout import PYOUT_SHORT_NAMES_rev
@@ -117,10 +116,7 @@ def assets_gen():
                     else:
                         dandiset_id = None
                     if recursive and assets:
-                        if isinstance(client, DandiAPIClient) and metadata in (
-                            "all",
-                            "assets",
-                        ):
+                        if metadata in ("all", "assets"):
                             for a in assets:
                                 if "metadata" not in a:
                                     a["metadata"] = client.get_asset(

dandi/cli/cmd_upload.py

@@ -6,7 +6,6 @@
     instance_option,
     map_to_click_exceptions,
 )
-from ..consts import collection_drafts
 
 
 class IntColonInt(click.ParamType):
@@ -64,13 +63,6 @@ def get_metavar(self, param):
 #
 # TODO: should always go to dandi for now
 @instance_option()
-# TODO: should always go into 'drafts' (consts.collection_drafts)
-@devel_option(
-    "-c", "--girder-collection", help="For development: Girder collection to upload to"
-)
-# TODO: figure out folder for the dandiset
-@devel_option("--girder-top-folder", help="For development: Girder top folder")
-#
 @devel_option(
     "--fake-data",
     help="For development: fake file content (filename will be stored instead of actual load)",
@@ -98,8 +90,6 @@ def upload(
     validation="require",
     dandiset_path=None,
     # Development options should come as kwargs
-    girder_collection=collection_drafts,
-    girder_top_folder=None,
     dandi_instance="dandi",
     fake_data=False,  # TODO: not implemented, prune?
     allow_any_path=False,
@@ -132,8 +122,6 @@ def upload(
         existing=existing,
         validation=validation,
         dandiset_path=dandiset_path,
-        girder_collection=girder_collection,
-        girder_top_folder=girder_top_folder,
         dandi_instance=dandi_instance,
         fake_data=fake_data,
         allow_any_path=allow_any_path,

dandi/cli/command.py

@@ -144,11 +144,10 @@ def main(ctx, log_level, pdb=False):
 from .cmd_download import download  # noqa: E402
 from .cmd_ls import ls  # noqa: E402
 from .cmd_organize import organize  # noqa: E402
-from .cmd_register import register  # noqa: E402
 from .cmd_upload import upload  # noqa: E402
 from .cmd_validate import validate  # noqa: E402
 
-__all_commands__ = (ls, organize, upload, download, validate, register, digest, delete)
+__all_commands__ = (ls, organize, upload, download, validate, digest, delete)
 
 for cmd in __all_commands__:
     main.add_command(cmd)

dandi/consts.py

@@ -66,12 +66,6 @@
 
 metadata_all_fields = metadata_nwb_fields + metadata_dandiset_fields
 
-# checksums and other digests to compute on the files to upload
-# Order matters - observed compute time from shorter to longer
-# Those are not to be included in metadata reported for a local file,
-# but will be available for files in the archive
-metadata_digests = ("sha1", "md5", "sha512", "sha256")
-
 dandiset_metadata_file = "dandiset.yaml"
 dandiset_identifier_regex = "^[0-9]{6}$"
 
@@ -84,18 +78,7 @@
 instancehost = os.environ.get("DANDI_INSTANCEHOST", "localhost")
 
 known_instances = {
-    "local-girder-only": dandi_instance(
-        0, f"http://{instancehost}:8080", None, None, None
-    ),  # just pure girder
     # Redirector: TODO https://github.com/dandi/dandiarchive/issues/139
-    "local-docker": dandi_instance(
-        0,
-        f"http://{instancehost}:8080",
-        f"http://{instancehost}:8085",
-        None,
-        f"http://{instancehost}:9000",  # ATM it is minio, not sure where /api etc
-        # may be https://github.com/dandi/dandi-publish/pull/71 would help
-    ),
     "local-docker-tests": dandi_instance(
         0,
         f"http://{instancehost}:8081",
@@ -124,9 +107,6 @@
 # to map back url: name
 known_instances_rev = {vv: k for k, v in known_instances.items() for vv in v if vv}
 
-collection_drafts = "drafts"
-collection_releases = "releases"
-
 file_operation_modes = [
     "dry",
     "simulate",

dandi/dandiapi.py

@@ -10,8 +10,8 @@
 import tenacity
 
 from .consts import MAX_CHUNK_SIZE, known_instances_rev
-from .girder import keyring_lookup
 from . import get_logger
+from .keyring import keyring_lookup
 from .utils import USER_AGENT, try_multiple
 
 lgr = get_logger()

dandi/dandiarchive.py

@@ -16,7 +16,7 @@
 from .consts import known_instances, known_instances_rev
 from .dandiapi import DandiAPIClient
 from .exceptions import FailedToConnectError, NotFoundError, UnknownURLError
-from . import get_logger, girder
+from . import get_logger
 from .utils import get_instance
 
 lgr = get_logger()
@@ -37,89 +37,44 @@ def navigate_url(url):
       `client` will have established a session for the duration of the context
     """
     server_type, server_url, asset_type, asset_id = parse_dandi_url(url)
-
     # We could later try to "dandi_authenticate" if run into permission issues.
     # May be it could be not just boolean but the "id" to be used?
-    kwargs = {}
-    if server_type == "girder":
-        asset_id, asset_type, client, server_type = _map_to_girder(url)
-        args = asset_id, asset_type
-    elif server_type == "api":
-        client = DandiAPIClient(server_url)
-        if asset_id["version"] is None:
-            r = client.get(f"/dandisets/{asset_id['dandiset_id']}/")
-            if "draft_version" in r:
-                asset_id["version"] = r["draft_version"]["version"]
-                published_version = r["most_recent_published_version"]
-                if published_version:
-                    asset_id["version"] = published_version["version"]
-            else:
-                # TODO: remove `if` after https://github.com/dandi/dandi-api/pull/219
-                # is merged/deployed
-                asset_id["version"] = r["most_recent_version"]["version"]
-        args = (asset_id["dandiset_id"], asset_id["version"])
-        kwargs["include_metadata"] = True
+    assert server_type == "api"
+    client = DandiAPIClient(server_url)
+    if asset_id["version"] is None:
+        r = client.get(f"/dandisets/{asset_id['dandiset_id']}/")
+        if "draft_version" in r:
+            asset_id["version"] = r["draft_version"]["version"]
+            published_version = r["most_recent_published_version"]
+            if published_version:
+                asset_id["version"] = published_version["version"]
+        else:
+            # TODO: remove `if` after https://github.com/dandi/dandi-api/pull/219
+            # is merged/deployed
+            asset_id["version"] = r["most_recent_version"]["version"]
+    args = (asset_id["dandiset_id"], asset_id["version"])
+    with client.session():
         if asset_id.get("location") or asset_id.get("asset_id"):
-            with client.session():
-                dandiset = client.get_dandiset(*args)
-                if asset_type == "folder":
-                    assets = client.get_dandiset_assets(
-                        *args, path=asset_id["location"]
-                    )
-                elif asset_type == "item":
-                    if "location" in asset_id:
-                        asset = client.get_asset_bypath(*args, asset_id["location"])
-                        assets = [asset] if asset is not None else []
-                    else:
-                        assets = [client.get_asset(*args, asset_id["asset_id"])]
+            dandiset = client.get_dandiset(*args)
+            if asset_type == "folder":
+                assets = client.get_dandiset_assets(*args, path=asset_id["location"])
+            elif asset_type == "item":
+                if "location" in asset_id:
+                    asset = client.get_asset_bypath(*args, asset_id["location"])
+                    assets = [asset] if asset is not None else []
                 else:
-                    raise NotImplementedError(
-                        f"Do not know how to handle asset type {asset_type} with location"
-                    )
-                yield (client, dandiset, assets)
-            return
-    else:
-        raise NotImplementedError(
-            f"Download from server of type {server_type} is not yet implemented"
-        )
-
-    with client.session():
-        dandiset, assets = client.get_dandiset_and_assets(*args, **kwargs)
+                    assets = [client.get_asset(*args, asset_id["asset_id"])]
+            else:
+                raise NotImplementedError(
+                    f"Do not know how to handle asset type {asset_type} with location"
+                )
+        else:
+            dandiset, assets = client.get_dandiset_and_assets(
+                *args, include_metadata=True
+            )
         yield client, dandiset, assets
 
 
-def _map_to_girder(url):
-    """
-    discover girder_id for a draft dataset
-    """
-    # This is a duplicate call from above but it is cheap, so decided to just redo
-    # it here instead of passing all the variables + url
-    server_type, server_url, asset_type, asset_id = parse_dandi_url(url)
-    server_url = known_instances[known_instances_rev[server_url.rstrip("/")]].girder
-    client = girder.get_client(server_url, authenticate=False, progressbars=True)
-    # TODO: RF if https://github.com/dandi/dandiarchive/issues/316 gets addressed
-    # A hybrid UI case not yet adjusted for drafts API.
-    # TODO: remove whenever it is gone in an unknown version
-    if asset_id.get("folder_id"):
-        asset_type = "folder"
-        asset_id = [asset_id.get("folder_id")]
-    else:
-        girder_path = "{}/{}".format("drafts", asset_id["dandiset_id"])
-        asset_type = "folder"
-        if asset_id.get("location"):
-            # Not implemented by UI ATM but might come
-            girder_path = "{}/{}".format(girder_path, asset_id["location"])
-        try:
-            girder_rec = girder.lookup(client, girder_path)
-        except BaseException:
-            lgr.warning(f"Failed to lookup girder information for {girder_path}")
-            girder_rec = None
-        if not girder_rec:
-            raise RuntimeError(f"Cannot download from {url}")
-        asset_id = girder_rec.get("_id")
-    return asset_id, asset_type, client, "girder"
-
-
 class _dandi_url_parser:
     # Defining as a class with all the attributes to not leak all the variables
     # etc into module space, and later we might end up with classes for those
@@ -137,7 +92,6 @@ class _dandi_url_parser:
         #   - 'pass' - would continue with original url if no redirect happen
         #   - 'only' - would interrupt if no redirection happens
         # - server_type:
-        #   - 'girder' - underlying requests should go to girder server
         #   - 'api' - the "new" API service
         #   - 'redirect' - use redirector's server-info
         # - rewrite:
@@ -254,7 +208,7 @@ class _dandi_url_parser:
     # We might need to remap some assert_types
     map_asset_types = {"dandiset": "folder"}
     # And lets create our mapping into girder instances from known_instances:
-    map_to = {"girder": {}, "api": {}}
+    map_to = {}
     for (
         metadata_version,
         girder,  # noqa: F402
@@ -263,11 +217,8 @@ class _dandi_url_parser:
         api,
     ) in known_instances.values():
         for h in (gui, redirector):
-            if h:
-                if girder:
-                    map_to["girder"][h] = girder
-                if api:
-                    map_to["api"][h] = api
+            if h and api:
+                map_to[h] = api
 
     @classmethod
     def parse(cls, url, *, map_instance=True):
@@ -382,18 +333,15 @@ def parse(cls, url, *, map_instance=True):
             except KeyError:
                 raise UnknownURLError(f"{url} does not map to a known instance")
             instance = get_instance(instance_name)
-            if instance.metadata_version == 0:
-                server_type = "girder"
-                server = instance.girder
-            elif instance.metadata_version == 1:
+            if instance.metadata_version == 1:
                 server_type = "api"
                 server = instance.api
             else:
                 raise RuntimeError(
                     f"Unknown instance metadata_version: {instance.metadata_version}"
                 )
         else:
-            server = cls.map_to[server_type].get(url_server.rstrip("/"), url_server)
+            server = cls.map_to.get(url_server.rstrip("/"), url_server)
 
         if not server.endswith("/"):
             server += "/"  # we expected '/' to be there so let it be
@@ -411,8 +359,6 @@ def parse(cls, url, *, map_instance=True):
             location = location.lstrip("/")
         if not (asset_type == "dandiset" and dandiset_id):
             raise ValueError(f"{url} does not point to a dandiset")
-        if server_type == "girder" and not version:
-            version = "draft"
         # Let's just return a structured record for the requested asset
         asset_ids = {"dandiset_id": dandiset_id, "version": version}
         # if location is not degenerate -- it would be a folder or a file