diff --git a/dandiapi/api/asset_paths.py b/dandiapi/api/asset_paths.py index ffa2a434a..2a2fcb69c 100644 --- a/dandiapi/api/asset_paths.py +++ b/dandiapi/api/asset_paths.py @@ -6,6 +6,7 @@ from django.db.models import Count, F, QuerySet from dandiapi.api.models import Asset, AssetPath, AssetPathRelation, Version +from dandiapi.zarr.models import ZarrArchive #################################################################### # Dandiset and version deletion will cascade to asset path deletion. @@ -154,3 +155,19 @@ def add_version_asset_paths(version: Version): """Add every asset from a version.""" for asset in version.assets.iterator(): add_asset_paths(asset, version) + + +def add_zarr_paths(zarr: ZarrArchive): + """Add all asset paths that are associated with a zarr.""" + # Only act on draft assets/versions + for asset in zarr.assets.filter(published=False).iterator(): + for version in asset.versions.filter(version='draft').iterator(): + add_asset_paths(asset, version) + + +def delete_zarr_paths(zarr: ZarrArchive): + """Remove all asset paths that are associated with a zarr.""" + # Only act on draft assets/versions + for asset in zarr.assets.filter(published=False).iterator(): + for version in asset.versions.filter(version='draft').iterator(): + delete_asset_paths(asset, version) diff --git a/dandiapi/api/tests/test_asset.py b/dandiapi/api/tests/test_asset.py index 1798d25a4..b40367bad 100644 --- a/dandiapi/api/tests/test_asset.py +++ b/dandiapi/api/tests/test_asset.py @@ -13,6 +13,7 @@ from dandiapi.api.models import Asset, AssetBlob, EmbargoedAssetBlob, Version from dandiapi.api.models.asset_paths import AssetPath from dandiapi.api.models.dandiset import Dandiset +from dandiapi.zarr.tasks import ingest_zarr_archive from .fuzzy import HTTP_URL_RE, TIMESTAMP_RE, URN_RE, UTC_ISO_TIMESTAMP_RE, UUID_RE @@ -902,12 +903,21 @@ def test_asset_rest_update_embargo(api_client, user, draft_version, asset, embar @pytest.mark.django_db -def test_asset_rest_update_zarr(api_client, user, draft_version, asset, zarr_archive): +def test_asset_rest_update_zarr( + api_client, user, draft_version, draft_asset_factory, zarr_archive, zarr_upload_file_factory +): assign_perm('owner', user, draft_version.dandiset) api_client.force_authenticate(user=user) + + asset = draft_asset_factory(blob=None, embargoed_blob=None, zarr=zarr_archive) draft_version.assets.add(asset) add_asset_paths(asset=asset, version=draft_version) + # Upload file and perform ingest + zarr_upload_file_factory(zarr_archive=zarr_archive) + ingest_zarr_archive(zarr_archive.zarr_id) + zarr_archive.refresh_from_db() + new_path = 'test/asset/rest/update.txt' new_metadata = { 'encodingFormat': 'application/x-zarr', @@ -916,7 +926,6 @@ def test_asset_rest_update_zarr(api_client, user, draft_version, asset, zarr_arc 'num': 123, 'list': ['a', 'b', 'c'], } - resp = api_client.put( f'/api/dandisets/{draft_version.dandiset.identifier}/' f'versions/{draft_version.version}/assets/{asset.asset_id}/', @@ -929,7 +938,7 @@ def test_asset_rest_update_zarr(api_client, user, draft_version, asset, zarr_arc 'path': new_path, 'size': zarr_archive.size, 'blob': None, - 'zarr': zarr_archive.zarr_id, + 'zarr': str(zarr_archive.zarr_id), 'created': TIMESTAMP_RE, 'modified': TIMESTAMP_RE, 'metadata': new_asset.metadata, @@ -1060,6 +1069,31 @@ def test_asset_rest_delete(api_client, user, draft_version, asset): assert draft_version.status == Version.Status.PENDING +@pytest.mark.django_db +def test_asset_rest_delete_zarr( + api_client, user, draft_version, draft_asset_factory, zarr_archive, zarr_upload_file_factory +): + asset = draft_asset_factory(blob=None, embargoed_blob=None, zarr=zarr_archive) + assign_perm('owner', user, draft_version.dandiset) + draft_version.assets.add(asset) + + # Add paths + add_asset_paths(asset=asset, version=draft_version) + + # Upload zarr file and perform ingest + zarr_upload_file_factory(zarr_archive=zarr_archive) + ingest_zarr_archive(zarr_archive.zarr_id) + zarr_archive.refresh_from_db() + + # Delete + api_client.force_authenticate(user=user) + resp = api_client.delete( + f'/api/dandisets/{draft_version.dandiset.identifier}/' + f'versions/{draft_version.version}/assets/{asset.asset_id}/' + ) + assert resp.status_code == 204 + + @pytest.mark.django_db def test_asset_rest_delete_not_an_owner(api_client, user, version, asset): api_client.force_authenticate(user=user) diff --git a/dandiapi/zarr/tasks/__init__.py b/dandiapi/zarr/tasks/__init__.py index 332fa56e2..097521af3 100644 --- a/dandiapi/zarr/tasks/__init__.py +++ b/dandiapi/zarr/tasks/__init__.py @@ -10,6 +10,7 @@ from django.db import transaction from django.db.transaction import atomic +from dandiapi.api.asset_paths import add_zarr_paths, delete_zarr_paths from dandiapi.api.storage import get_boto_client, yield_files from dandiapi.zarr.checksums import ( ZarrChecksum, @@ -119,6 +120,9 @@ def ingest_zarr_archive( with transaction.atomic(): zarr = ZarrArchive.objects.select_for_update().get(zarr_id=zarr_id) + # Remove all asset paths associated with this zarr before ingest + delete_zarr_paths(zarr) + # Reset before compute if not no_size: zarr.size = 0 @@ -178,6 +182,9 @@ def ingest_zarr_archive( for asset in zarr.assets.iterator(): asset.save() + # Add asset paths after ingest is finished + add_zarr_paths(zarr) + def ingest_dandiset_zarrs(dandiset_id: int, **kwargs): for zarr in ZarrArchive.objects.filter(dandiset__id=dandiset_id):