Skip to content

Commit

Permalink
Merge branch 'main' into doc/3.0-updates
Browse files Browse the repository at this point in the history
  • Loading branch information
jhamman authored Dec 18, 2024
2 parents 2271548 + 4455726 commit f6a9883
Show file tree
Hide file tree
Showing 3 changed files with 85 additions and 51 deletions.
1 change: 1 addition & 0 deletions docs/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@
"release": "developers/release.html",
"roadmap": "developers/roadmap.html",
"installation": "user-guide/installation.html",
"license": "https://github.com/zarr-developers/zarr-python/blob/main/LICENSE.txt"
}

# The language for content autogenerated by Sphinx. Refer to documentation
Expand Down
2 changes: 2 additions & 0 deletions src/zarr/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,8 @@ def get_codec_class(key: str, reload_config: bool = False) -> type[Codec]:

config_entry = config.get("codecs", {}).get(key)
if config_entry is None:
if len(codec_classes) == 1:
return next(iter(codec_classes.values()))
warnings.warn(
f"Codec '{key}' not configured in config. Selecting any implementation.", stacklevel=2
)
Expand Down
133 changes: 82 additions & 51 deletions tests/test_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,7 @@ class MockEnvCodecPipeline(CodecPipeline):
assert get_pipeline_class(reload_config=True) == MockEnvCodecPipeline


@pytest.mark.filterwarnings("error")
@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"])
def test_config_codec_implementation(store: Store) -> None:
# has default value
Expand All @@ -156,24 +157,29 @@ async def _encode_single(
) -> CodecOutput | None:
_mock.call()

config.set({"codecs.blosc": fully_qualified_name(MockBloscCodec)})
register_codec("blosc", MockBloscCodec)
assert get_codec_class("blosc") == MockBloscCodec

# test if codec is used
arr = Array.create(
store=store,
shape=(100,),
chunks=(10,),
zarr_format=3,
dtype="i4",
codecs=[BytesCodec(), {"name": "blosc", "configuration": {}}],
)
arr[:] = range(100)
_mock.call.assert_called()
with config.set({"codecs.blosc": fully_qualified_name(MockBloscCodec)}):
assert get_codec_class("blosc") == MockBloscCodec

# test if codec is used
arr = Array.create(
store=store,
shape=(100,),
chunks=(10,),
zarr_format=3,
dtype="i4",
codecs=[BytesCodec(), {"name": "blosc", "configuration": {}}],
)
arr[:] = range(100)
_mock.call.assert_called()

# test set codec with environment variable
class NewBloscCodec(BloscCodec):
pass

with mock.patch.dict(os.environ, {"ZARR_CODECS__BLOSC": fully_qualified_name(BloscCodec)}):
assert get_codec_class("blosc", reload_config=True) == BloscCodec
register_codec("blosc", NewBloscCodec)
with mock.patch.dict(os.environ, {"ZARR_CODECS__BLOSC": fully_qualified_name(NewBloscCodec)}):
assert get_codec_class("blosc", reload_config=True) == NewBloscCodec


@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"])
Expand All @@ -183,18 +189,17 @@ def test_config_ndbuffer_implementation(store: Store) -> None:

# set custom ndbuffer with TestNDArrayLike implementation
register_ndbuffer(NDBufferUsingTestNDArrayLike)
config.set({"ndbuffer": fully_qualified_name(NDBufferUsingTestNDArrayLike)})
assert get_ndbuffer_class() == NDBufferUsingTestNDArrayLike
arr = Array.create(
store=store,
shape=(100,),
chunks=(10,),
zarr_format=3,
dtype="i4",
)
got = arr[:]
print(type(got))
assert isinstance(got, TestNDArrayLike)
with config.set({"ndbuffer": fully_qualified_name(NDBufferUsingTestNDArrayLike)}):
assert get_ndbuffer_class() == NDBufferUsingTestNDArrayLike
arr = Array.create(
store=store,
shape=(100,),
chunks=(10,),
zarr_format=3,
dtype="i4",
)
got = arr[:]
assert isinstance(got, TestNDArrayLike)


def test_config_buffer_implementation() -> None:
Expand All @@ -208,27 +213,53 @@ def test_config_buffer_implementation() -> None:
arr[:] = np.arange(100)

register_buffer(TestBuffer)
config.set({"buffer": fully_qualified_name(TestBuffer)})
assert get_buffer_class() == TestBuffer

# no error using TestBuffer
data = np.arange(100)
arr[:] = np.arange(100)
assert np.array_equal(arr[:], data)

data2d = np.arange(1000).reshape(100, 10)
arr_sharding = zeros(
shape=(100, 10),
store=StoreExpectingTestBuffer(),
codecs=[ShardingCodec(chunk_shape=(10, 10))],
)
arr_sharding[:] = data2d
assert np.array_equal(arr_sharding[:], data2d)
with config.set({"buffer": fully_qualified_name(TestBuffer)}):
assert get_buffer_class() == TestBuffer

arr_Crc32c = zeros(
shape=(100, 10),
store=StoreExpectingTestBuffer(),
codecs=[BytesCodec(), Crc32cCodec()],
)
arr_Crc32c[:] = data2d
assert np.array_equal(arr_Crc32c[:], data2d)
# no error using TestBuffer
data = np.arange(100)
arr[:] = np.arange(100)
assert np.array_equal(arr[:], data)

data2d = np.arange(1000).reshape(100, 10)
arr_sharding = zeros(
shape=(100, 10),
store=StoreExpectingTestBuffer(),
codecs=[ShardingCodec(chunk_shape=(10, 10))],
)
arr_sharding[:] = data2d
assert np.array_equal(arr_sharding[:], data2d)

arr_Crc32c = zeros(
shape=(100, 10),
store=StoreExpectingTestBuffer(),
codecs=[BytesCodec(), Crc32cCodec()],
)
arr_Crc32c[:] = data2d
assert np.array_equal(arr_Crc32c[:], data2d)


@pytest.mark.filterwarnings("error")
def test_warning_on_missing_codec_config() -> None:
class NewCodec(BytesCodec):
pass

class NewCodec2(BytesCodec):
pass

# error if codec is not registered
with pytest.raises(KeyError):
get_codec_class("missing_codec")

# no warning if only one implementation is available
register_codec("new_codec", NewCodec)
get_codec_class("new_codec")

# warning because multiple implementations are available but none is selected in the config
register_codec("new_codec", NewCodec2)
with pytest.warns(UserWarning):
get_codec_class("new_codec")

# no warning if multiple implementations are available and one is selected in the config
with config.set({"codecs.new_codec": fully_qualified_name(NewCodec)}):
get_codec_class("new_codec")

0 comments on commit f6a9883

Please sign in to comment.