Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update zarr checksums #120

Merged
merged 6 commits into from
Mar 16, 2022
Merged

Update zarr checksums #120

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
cea25c7
Rename path to name in zarr checksum
dchiquito Mar 2, 2022
a117865
Add size to zarr checksums
dchiquito Mar 2, 2022
2bd30c3
Remove unnecessary list comprehension
dchiquito Mar 2, 2022
38efd13
Enhance zarr checksum digests
dchiquito Mar 2, 2022
20f54c7
Merge branch 'master' into update-zarr-checksums
satra Mar 11, 2022
02972c5
Add new zarr checksum format to model
dchiquito Mar 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 46 additions & 42 deletions dandischema/digests/tests/test_zarr.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@

def test_zarr_checksum_sort_order():
# The a < b in the path should take precedence over z > y in the md5
a = ZarrChecksum(path="1/2/3/a/z", md5="z")
b = ZarrChecksum(path="1/2/3/b/z", md5="y")
a = ZarrChecksum(name="a", md5="z", size=1)
b = ZarrChecksum(name="b", md5="y", size=1)
assert sorted([b, a]) == [a, b]


Expand All @@ -24,16 +24,16 @@ def test_zarr_checksum_sort_order():
def test_zarr_checkums_is_empty():
assert ZarrChecksums(directories=[], files=[]).is_empty
assert not ZarrChecksums(
directories=[ZarrChecksum(md5="md5", path="path")], files=[]
directories=[ZarrChecksum(md5="md5", name="name", size=1)], files=[]
).is_empty
assert not ZarrChecksums(
directories=[], files=[ZarrChecksum(md5="md5", path="path")]
directories=[], files=[ZarrChecksum(md5="md5", name="name", size=1)]
).is_empty


a = ZarrChecksum(path="a", md5="a")
b = ZarrChecksum(path="b", md5="b")
c = ZarrChecksum(path="c", md5="c")
a = ZarrChecksum(name="a", md5="a", size=1)
b = ZarrChecksum(name="b", md5="b", size=1)
c = ZarrChecksum(name="c", md5="c", size=1)


@pytest.mark.parametrize(
Expand Down Expand Up @@ -113,35 +113,35 @@ def test_zarr_checkums_remove_checksums(
[
([], [], "481a2f77ab786a0f45aafd5db0971caa"),
(
[ZarrChecksum(path="foo/bar", md5="a")],
[ZarrChecksum(name="bar", md5="a", size=1)],
[],
"cdcfdfca3622e20df03219273872549e",
"677dddd9af150be166c461acdef1b025",
),
(
[],
[ZarrChecksum(path="foo/bar", md5="a")],
"243aca82c6872222747183dd738b6fcb",
[ZarrChecksum(name="bar", md5="a", size=1)],
"aa776d184c64cbd6a5956ab0af012830",
),
(
[
ZarrChecksum(path="foo/bar", md5="a"),
ZarrChecksum(path="foo/baz", md5="b"),
ZarrChecksum(name="bar", md5="a", size=1),
ZarrChecksum(name="baz", md5="b", size=1),
],
[],
"785295076ae9156b363e442ef6d485e0",
"c8a9b1dd53bb43ec6e5d379c29a1f1dd",
),
(
[],
[
ZarrChecksum(path="foo/bar", md5="a"),
ZarrChecksum(path="foo/baz", md5="b"),
ZarrChecksum(name="bar", md5="a", size=1),
ZarrChecksum(name="baz", md5="b", size=1),
],
"ebca8bb8e716237e0f71657d1045930f",
"f45aa3833a2129628a38e421f74ff792",
),
(
[ZarrChecksum(path="foo/baz", md5="a")],
[ZarrChecksum(path="foo/bar", md5="b")],
"9c34644ba03b7e9f58ebd1caef4215ad",
[ZarrChecksum(name="baz", md5="a", size=1)],
[ZarrChecksum(name="bar", md5="b", size=1)],
"bc0a0e85a0205eb3cb5f163f173774e5",
),
],
)
Expand All @@ -160,11 +160,13 @@ def test_zarr_checksum_serializer_aggregate_checksum(
def test_zarr_checksum_serializer_generate_listing():
serializer = ZarrJSONChecksumSerializer()
checksums = ZarrChecksums(
files=[ZarrChecksum(path="foo/bar", md5="a")],
directories=[ZarrChecksum(path="foo/baz", md5="b")],
files=[ZarrChecksum(name="bar", md5="a", size=1)],
directories=[ZarrChecksum(name="baz", md5="b", size=2)],
)
assert serializer.generate_listing(checksums) == ZarrChecksumListing(
checksums=checksums, md5="23076057c0da63f8ab50d0a108db332c"
checksums=checksums,
md5="c20479b1afe558a919eac450028a706e",
size=3,
)


Expand All @@ -174,61 +176,63 @@ def test_zarr_serialize():
serializer.serialize(
ZarrChecksumListing(
checksums=ZarrChecksums(
files=[ZarrChecksum(path="foo/bar", md5="a")],
directories=[ZarrChecksum(path="bar/foo", md5="b")],
files=[ZarrChecksum(name="bar", md5="a", size=1)],
directories=[ZarrChecksum(name="foo", md5="b", size=2)],
),
md5="c",
size=3,
)
)
== '{"checksums":{"directories":[{"md5":"b","path":"bar/foo"}],"files":[{"md5":"a","path":"foo/bar"}]},"md5":"c"}' # noqa: E501
== '{"checksums":{"directories":[{"md5":"b","name":"foo","size":2}],"files":[{"md5":"a","name":"bar","size":1}]},"md5":"c","size":3}' # noqa: E501
)


def test_zarr_deserialize():
serializer = ZarrJSONChecksumSerializer()
assert serializer.deserialize(
'{"checksums":{"directories":[{"md5":"b","path":"bar/foo"}],"files":[{"md5":"a","path":"foo/bar"}]},"md5":"c"}' # noqa: E501
'{"checksums":{"directories":[{"md5":"b","name":"foo","size":2}],"files":[{"md5":"a","name":"bar","size":1}]},"md5":"c","size":3}' # noqa: E501
) == ZarrChecksumListing(
checksums=ZarrChecksums(
files=[ZarrChecksum(path="foo/bar", md5="a")],
directories=[ZarrChecksum(path="bar/foo", md5="b")],
files=[ZarrChecksum(name="bar", md5="a", size=1)],
directories=[ZarrChecksum(name="foo", md5="b", size=2)],
),
md5="c",
size=3,
)


@pytest.mark.parametrize(
"files,directories,checksum",
[
(
{"foo/bar": "a"},
{"bar": ("a", 1)},
{},
"cdcfdfca3622e20df03219273872549e",
"677dddd9af150be166c461acdef1b025",
),
(
{},
{"foo/bar": "a"},
"243aca82c6872222747183dd738b6fcb",
{"bar": ("a", 1)},
"aa776d184c64cbd6a5956ab0af012830",
),
(
{"foo/bar": "a", "foo/baz": "b"},
{"bar": ("a", 1), "baz": ("b", 2)},
{},
"785295076ae9156b363e442ef6d485e0",
"66c03ae00824e6be1283cc370969f6ea",
),
(
{},
{"foo/bar": "a", "foo/baz": "b"},
"ebca8bb8e716237e0f71657d1045930f",
{"bar": ("a", 1), "baz": ("b", 2)},
"6969470da4b829f0a8b665ac78350abd",
),
(
{},
{"foo/baz": "b", "foo/bar": "a"},
"ebca8bb8e716237e0f71657d1045930f",
{"baz": ("b", 1), "bar": ("a", 2)},
"25f351bbdcfb33f7706f7ef1e80cb010",
),
(
{"foo/baz": "a"},
{"foo/bar": "b"},
"9c34644ba03b7e9f58ebd1caef4215ad",
{"baz": ("a", 1)},
{"bar": ("b", 2)},
"a9540738019a48e6392c942217f7526d",
),
],
)
Expand Down
36 changes: 24 additions & 12 deletions dandischema/digests/zarr.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

from functools import total_ordering
import hashlib
from typing import Dict, List, Optional
from typing import Dict, List, Optional, Tuple

import pydantic

Expand All @@ -15,15 +15,16 @@ class ZarrChecksum(pydantic.BaseModel):
"""
A checksum for a single file/directory in a zarr file.

Every file and directory in a zarr archive has a path and a MD5 hash.
Every file and directory in a zarr archive has a name and a MD5 hash.
"""

md5: str
path: str
name: str
size: int
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if to simplify transition there could be a proper deprecation cycle which would support path, and decompose it into name and size as needed. Should be simple right? Wdyt?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think that's necessary. As soon as this is released and incorporated in dandi-archive and dandi-cli, we can just recompute checksums on all zarrs and path will be updated to name.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

well, since there is no way to announce that dandischema (next release) breaks dandi-cli << (whatever would become compatible with new interface) people would end up with a dandi-cli which might puke an exception if they manage to upgrade dandischema but not dandi-cli. unlikely but possible. deprecation cycle allows to amortize perturbation and avoid such problems...
oh well -- since python is dynamic and zarr isn't primary target yet for users, I am ok to proceed without deprecation cycle at hopefully low or none count of affected users.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we make the next dandischema release a new minor version, it would be incompatible with dandi-cli's dandischema ~= 0.5.1 requirement and thus help to prevent that from happening (but there would still be problems with checksum failures if people use the old client to upload Zarrs).

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ah, right, nice.
@jwodder Could you prep a PR for dandi-cli to accompany this one?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@yarikoptic PR: dandi/dandi-cli#933


# To make ZarrChecksums sortable
def __lt__(self, other: ZarrChecksum):
return self.path < other.path
return self.name < other.name


class ZarrChecksums(pydantic.BaseModel):
Expand All @@ -43,7 +44,7 @@ def is_empty(self):
def _index(self, checksums: List[ZarrChecksum], checksum: ZarrChecksum):
# O(n) performance, consider using the bisect module or an ordered dict for optimization
for i in range(0, len(checksums)):
if checksums[i].path == checksum.path:
if checksums[i].name == checksum.name:
return i
raise ValueError("Not found")

Expand All @@ -66,13 +67,13 @@ def add_directory_checksums(self, checksums: List[ZarrChecksum]):
self.directories.append(new_checksum)
self.directories = sorted(self.directories)

def remove_checksums(self, paths: List[str]):
"""Remove a list of paths from the listing."""
def remove_checksums(self, names: List[str]):
"""Remove a list of names from the listing."""
self.files = sorted(
filter(lambda checksum: checksum.path not in paths, self.files)
filter(lambda checksum: checksum.name not in names, self.files)
)
self.directories = sorted(
filter(lambda checksum: checksum.path not in paths, self.directories)
filter(lambda checksum: checksum.name not in names, self.directories)
)


Expand All @@ -85,6 +86,7 @@ class ZarrChecksumListing(pydantic.BaseModel):

checksums: ZarrChecksums
md5: str
size: int


class ZarrJSONChecksumSerializer:
Expand Down Expand Up @@ -125,9 +127,13 @@ def generate_listing(
files=sorted(files) if files is not None else [],
directories=sorted(directories) if directories is not None else [],
)
size = sum(file.size for file in checksums.files) + sum(
directory.size for directory in checksums.directories
)
dchiquito marked this conversation as resolved.
Show resolved Hide resolved
dchiquito marked this conversation as resolved.
Show resolved Hide resolved
return ZarrChecksumListing(
checksums=checksums,
md5=self.aggregate_checksum(checksums),
size=size,
)


Expand All @@ -137,14 +143,20 @@ def generate_listing(
EMPTY_CHECKSUM = ZarrJSONChecksumSerializer().generate_listing(ZarrChecksums()).md5


def get_checksum(files: Dict[str, str], directories: Dict[str, str]) -> str:
def get_checksum(
files: Dict[str, Tuple[str, int]], directories: Dict[str, Tuple[str, int]]
) -> str:
"""Calculate the checksum of a directory."""
if not files and not directories:
raise ValueError("Cannot compute a Zarr checksum for an empty directory")
checksum_listing = ZarrJSONChecksumSerializer().generate_listing(
files=[ZarrChecksum(md5=md5, path=path) for path, md5 in files.items()],
files=[
ZarrChecksum(md5=md5, name=name, size=size)
for name, (md5, size) in files.items()
],
directories=[
ZarrChecksum(md5=md5, path=path) for path, md5 in directories.items()
ZarrChecksum(md5=md5, name=name, size=size)
for name, (md5, size) in directories.items()
],
)
return checksum_listing.md5