Bump zarrs to 0.16.0 (#7)

LDeakin · Jul 28, 2024 · a2356ce · a2356ce
1 parent 09fb8b1
commit a2356ce
Show file tree

Hide file tree

Showing 24 changed files with 107 additions and 89 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+ - Bump `zarrs` to 0.16.0
+
 ## [0.5.3] - 2024-07-24
 
 ### Added

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "zarrs_tools"
-version = "0.5.3"
+version = "0.5.4"
 authors = ["Lachlan Deakin <ljdgit@gmail.com>"]
 edition = "2021"
 rust-version = "1.75"
@@ -48,7 +48,7 @@ sysinfo = "0.30.6"
 tempfile = "3.10.1"
 thiserror = "1.0.57"
 tokio = { version = "1.34.0", features = ["macros", "rt-multi-thread"] }
-zarrs = { version = "0.15.0" }
+zarrs = { version = "0.16.0" }
 
 [[bin]]
 name = "zarrs_benchmark_read_sync"

diff --git a/README.md b/README.md
@@ -9,8 +9,8 @@ Various tools for creating and manipulating [Zarr v3](https://zarr.dev) data wit
 A changelog can be found [here](https://github.com/LDeakin/zarrs_tools/blob/main/CHANGELOG.md).
 
 ## Tools
-All tools support input and output [Zarr V3](https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html) data.
-Some tools additionally support input of a [V3 compatible subset](https://docs.rs/zarrs/latest/zarrs/#arrays-zarr-v3-and-zarr-v2) of [Zarr V2](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html).
+All tools support input and output of [Zarr V3](https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html) data.
+Some tools additionally support input of a [V3 compatible subset](https://docs.rs/zarrs/latest/zarrs/#implementation-status) of [Zarr V2](https://zarr-specs.readthedocs.io/en/latest/v2/v2.0.html).
 
 - [zarrs_reencode](https://github.com/LDeakin/zarrs_tools/blob/main/docs/zarrs_reencode.md): reencode an array. Manipulate the chunk size, shard size, codecs, fill value, chunk key encoding separator, and attributes.
 - [zarrs_filter](https://github.com/LDeakin/zarrs_tools/blob/main/docs/zarrs_filter.md) (feature `filter`): apply simple image filters (transformations) to an array.

diff --git a/src/bin/zarrs_benchmark_read_async.rs b/src/bin/zarrs_benchmark_read_async.rs
@@ -83,7 +83,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
         let array_shape = array.shape().to_vec();
         let array_subset = ArraySubset::new_with_shape(array_shape.to_vec());
         let array_data = array.async_retrieve_array_subset(&array_subset).await?;
-        bytes_decoded += array_data.len();
+        bytes_decoded += array_data.size();
     } else {
         // Calculate chunk/codec concurrency
         let chunk_representation =
@@ -120,7 +120,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
                 async move {
                     array
                         .async_retrieve_chunk_opt(&chunk_indices, &codec_options)
-                        .map(|bytes| bytes.map(|bytes| bytes.len()))
+                        .map(|bytes| bytes.map(|bytes| bytes.size()))
                         .await
                 }
             })

diff --git a/src/bin/zarrs_benchmark_read_async_as_sync.rs b/src/bin/zarrs_benchmark_read_async_as_sync.rs
@@ -93,7 +93,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     let bytes_decoded = Mutex::new(0);
     if args.read_all {
         let subset = ArraySubset::new_with_shape(array.shape().to_vec());
-        *bytes_decoded.lock().unwrap() += array.retrieve_array_subset(&subset)?.len();
+        *bytes_decoded.lock().unwrap() += array.retrieve_array_subset(&subset)?.size();
     } else {
         let chunk_representation =
             array.chunk_array_representation(&vec![0; array.chunk_grid().dimensionality()])?;
@@ -134,7 +134,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                 let bytes = array
                     .retrieve_chunk_opt(&chunk_indices, &codec_options)
                     .unwrap();
-                *bytes_decoded.lock().unwrap() += bytes.len();
+                *bytes_decoded.lock().unwrap() += bytes.size();
             }
         );
     }

diff --git a/src/bin/zarrs_benchmark_read_sync.rs b/src/bin/zarrs_benchmark_read_sync.rs
@@ -62,7 +62,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     let bytes_decoded = Mutex::new(0);
     if args.read_all {
         let subset = ArraySubset::new_with_shape(array.shape().to_vec());
-        *bytes_decoded.lock().unwrap() += array.retrieve_array_subset(&subset)?.len();
+        *bytes_decoded.lock().unwrap() += array.retrieve_array_subset(&subset)?.size();
     } else {
         let chunk_representation =
             array.chunk_array_representation(&vec![0; array.chunk_grid().dimensionality()])?;
@@ -103,7 +103,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                 let bytes = array
                     .retrieve_chunk_opt(&chunk_indices, &codec_options)
                     .unwrap();
-                *bytes_decoded.lock().unwrap() += bytes.len();
+                *bytes_decoded.lock().unwrap() += bytes.size();
             }
         );
     }

diff --git a/src/bin/zarrs_binary2zarr.rs b/src/bin/zarrs_binary2zarr.rs
@@ -81,7 +81,10 @@ fn stdin_to_array(
     endianness: Option<Endianness>,
     concurrent_chunks: Option<usize>,
 ) -> usize {
-    let data_type_size = array.data_type().size();
+    let data_type_size = array
+        .data_type()
+        .fixed_size()
+        .expect("data type should be fixed size");
     let dimensionality = array.chunk_grid().dimensionality();
     let array_shape = array.shape();
     let array_shape_n = *array_shape.first().unwrap();
@@ -159,7 +162,7 @@ fn stdin_to_array(
         }
 
         array
-            .store_array_subset_opt(&array_subset, &subset_bytes, &codec_options)
+            .store_array_subset_opt(&array_subset, subset_bytes, &codec_options)
             .unwrap();
     };
     iter_concurrent_limit!(concurrent_chunks, 0..n_blocks, for_each, op);

diff --git a/src/bin/zarrs_ncvar2zarr.rs b/src/bin/zarrs_ncvar2zarr.rs
@@ -94,7 +94,14 @@ fn ncfiles_to_array<TStore: ReadableWritableStorageTraits + ?Sized + 'static>(
         let mut start = vec![0u64; array.chunk_grid().dimensionality()];
         start[concat_dim] = offsets[idx];
         let array_subset = ArraySubset::new_with_start_shape(start, dim_sizes_u64.clone()).unwrap();
-        let mut buf = vec![0u8; array.data_type().size() * array_subset.num_elements_usize()];
+        let mut buf = vec![
+            0u8;
+            array
+                .data_type()
+                .fixed_size()
+                .expect("data type should be fixed size")
+                * array_subset.num_elements_usize()
+        ];
         // println!("{array_subset:?} {dim_sizes:?} {}", buf.len());
         nc_var
             .get_raw_values(
@@ -110,15 +117,15 @@ fn ncfiles_to_array<TStore: ReadableWritableStorageTraits + ?Sized + 'static>(
 
         if validate {
             array
-                .store_array_subset_opt(&array_subset, &buf, &codec_options)
+                .store_array_subset_opt(&array_subset, buf.clone(), &codec_options)
                 .unwrap();
             let buf_validate = array
                 .retrieve_array_subset_opt(&array_subset, &codec_options)
                 .unwrap();
-            assert!(buf == buf_validate);
+            assert!(buf == buf_validate.into_fixed().unwrap().into_owned());
         } else {
             array
-                .store_array_subset_opt(&array_subset, &buf, &codec_options)
+                .store_array_subset_opt(&array_subset, buf, &codec_options)
                 .unwrap();
         }
     };

diff --git a/src/bin/zarrs_ome.rs b/src/bin/zarrs_ome.rs
@@ -16,7 +16,7 @@ use ome_zarr_metadata::v0_5_dev::{
 };
 use rayon::iter::{IntoParallelIterator, ParallelIterator};
 use zarrs::{
-    array::{Array, ArrayCodecTraits, ArrayMetadata, ChunkRepresentation},
+    array::{Array, ArrayCodecTraits, ArrayMetadata, ChunkRepresentation, Element, ElementOwned},
     array_subset::ArraySubset,
     group::{Group, GroupMetadata, GroupMetadataV3},
     storage::{store::FilesystemStore, StorePrefix, WritableStorageTraits},
@@ -180,7 +180,7 @@ fn apply_chunk_discrete<T>(
     progress: &Progress,
 ) -> Result<(), FilterError>
 where
-    T: bytemuck::Pod + Copy + Send + Sync + Eq + PartialEq + Hash + AsPrimitive<T>,
+    T: Element + ElementOwned + Copy + Send + Sync + Eq + PartialEq + Hash + AsPrimitive<T>,
 {
     let output_subset = array_output.chunk_subset_bounded(chunk_indices).unwrap();
     let downsample_input_subset =
@@ -191,7 +191,7 @@ where
         downsample_filter.apply_ndarray_discrete(input_chunk, progress)
     };
     progress.write(|| {
-        array_output.store_array_subset_ndarray::<T, _, _>(output_subset.start(), output_chunk)
+        array_output.store_array_subset_ndarray::<T, _>(output_subset.start(), output_chunk)
     })?;
     Ok(())
 }
@@ -204,7 +204,7 @@ fn apply_chunk_continuous<T>(
     progress: &Progress,
 ) -> Result<(), FilterError>
 where
-    T: bytemuck::Pod + Copy + Send + Sync + AsPrimitive<f64> + std::iter::Sum,
+    T: Element + ElementOwned + Copy + Send + Sync + AsPrimitive<f64> + std::iter::Sum,
     f64: AsPrimitive<T>,
 {
     let output_subset = array_output.chunk_subset_bounded(chunk_indices).unwrap();
@@ -216,7 +216,7 @@ where
         downsample_filter.apply_ndarray_continuous(input_chunk, progress)
     };
     progress.write(|| {
-        array_output.store_array_subset_ndarray::<T, _, _>(output_subset.start(), output_chunk)
+        array_output.store_array_subset_ndarray::<T, _>(output_subset.start(), output_chunk)
     })?;
     Ok(())
 }
@@ -230,7 +230,7 @@ fn apply_chunk_continuous_gaussian<T>(
     progress: &Progress,
 ) -> Result<(), FilterError>
 where
-    T: bytemuck::Pod + Copy + Send + Sync + AsPrimitive<f32> + std::iter::Sum,
+    T: Element + ElementOwned + Copy + Send + Sync + AsPrimitive<f32> + std::iter::Sum,
     f64: AsPrimitive<T>,
 {
     let output_subset = array_output.chunk_subset_bounded(chunk_indices).unwrap();
@@ -253,7 +253,7 @@ where
     };
     let output_chunk = downsample_filter.apply_ndarray_continuous(gaussian_chunk, progress);
     progress.write(|| {
-        array_output.store_array_subset_ndarray::<T, _, _>(output_subset.start(), output_chunk)
+        array_output.store_array_subset_ndarray::<T, _>(output_subset.start(), output_chunk)
     })?;
     Ok(())
 }

diff --git a/src/filter/filters/clamp.rs b/src/filter/filters/clamp.rs
@@ -102,7 +102,7 @@ impl FilterTraits for Clamp {
         chunk_input: &zarrs::array::ChunkRepresentation,
         chunk_output: &zarrs::array::ChunkRepresentation,
     ) -> usize {
-        chunk_input.size_usize() + chunk_output.size_usize()
+        chunk_input.fixed_element_size().unwrap() + chunk_output.fixed_element_size().unwrap()
     }
 
     fn apply(

diff --git a/src/filter/filters/crop.rs b/src/filter/filters/crop.rs
@@ -3,7 +3,7 @@ use num_traits::AsPrimitive;
 use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator};
 use serde::{Deserialize, Serialize};
 use zarrs::{
-    array::{data_type::UnsupportedDataTypeError, Array, DataType},
+    array::{data_type::UnsupportedDataTypeError, Array, DataType, Element, ElementOwned},
     array_subset::ArraySubset,
     storage::store::FilesystemStore,
 };
@@ -84,7 +84,7 @@ impl Crop {
     ) -> Result<(), FilterError> {
         let (input_subset, output_subset) = self.get_input_output_subset(output, chunk_indices);
         let output_bytes = progress.read(|| input.retrieve_array_subset(&input_subset))?;
-        progress.write(|| output.store_array_subset(&output_subset, &output_bytes))?;
+        progress.write(|| output.store_array_subset(&output_subset, output_bytes))?;
         progress.next();
         Ok(())
     }
@@ -97,8 +97,8 @@ impl Crop {
         progress: &Progress,
     ) -> Result<(), FilterError>
     where
-        TIn: bytemuck::Pod + Send + Sync + AsPrimitive<TOut>,
-        TOut: bytemuck::Pod + Send + Sync,
+        TIn: ElementOwned + Send + Sync + AsPrimitive<TOut>,
+        TOut: Element + Send + Sync + Copy + 'static,
     {
         let (input_subset, output_subset) = self.get_input_output_subset(output, chunk_indices);
 
@@ -154,7 +154,7 @@ impl FilterTraits for Crop {
         _chunk_input: &zarrs::array::ChunkRepresentation,
         chunk_output: &zarrs::array::ChunkRepresentation,
     ) -> usize {
-        chunk_output.size_usize()
+        chunk_output.fixed_element_size().unwrap()
     }
 
     fn output_shape(&self, _input: &Array<FilesystemStore>) -> Option<Vec<u64>> {

diff --git a/src/filter/filters/downsample.rs b/src/filter/filters/downsample.rs
@@ -153,9 +153,9 @@ impl FilterTraits for Downsample {
         chunk_output: &zarrs::array::ChunkRepresentation,
     ) -> usize {
         debug_assert_eq!(_chunk_input.data_type(), chunk_output.data_type());
-        let input = chunk_output.size_usize()
+        let input = chunk_output.fixed_element_size().unwrap()
             * usize::try_from(self.stride.iter().product::<u64>()).unwrap();
-        let output = chunk_output.size_usize();
+        let output = chunk_output.fixed_element_size().unwrap();
         input + output
     }
 
@@ -207,7 +207,7 @@ impl FilterTraits for Downsample {
                             self.apply_ndarray_continuous(input_array, &progress)
                         };
                         progress.write(|| {
-                            output.store_array_subset_ndarray::<$t_out, _, _>(
+                            output.store_array_subset_ndarray::<$t_out, _>(
                                 output_subset.start(),
                                 output_array,
                             )
@@ -220,7 +220,7 @@ impl FilterTraits for Downsample {
                             .read(|| input.retrieve_array_subset_ndarray::<$t_in>(&input_subset))?;
                         let output_array = self.apply_ndarray_continuous(input_array, &progress);
                         progress.write(|| {
-                            output.store_array_subset_ndarray::<$t_out, _, _>(
+                            output.store_array_subset_ndarray::<$t_out, _>(
                                 output_subset.start(),
                                 output_array,
                             )

diff --git a/src/filter/filters/equal.rs b/src/filter/filters/equal.rs
@@ -3,7 +3,10 @@ use num_traits::AsPrimitive;
 use rayon::iter::{IntoParallelIterator, ParallelIterator};
 use serde::{Deserialize, Serialize};
 use zarrs::{
-    array::{data_type::UnsupportedDataTypeError, Array, DataType, FillValue, FillValueMetadata},
+    array::{
+        data_type::UnsupportedDataTypeError, Array, DataType, Element, ElementOwned, FillValue,
+        FillValueMetadata,
+    },
     array_subset::ArraySubset,
     storage::store::FilesystemStore,
 };
@@ -64,8 +67,8 @@ impl Equal {
         equal: &TIn,
     ) -> Result<Vec<TOut>, FilterError>
     where
-        TIn: bytemuck::Pod + Copy + Send + Sync + PartialEq,
-        TOut: bytemuck::Pod + Send + Sync,
+        TIn: ElementOwned + Copy + Send + Sync + PartialEq,
+        TOut: Element + Send + Sync + Copy + 'static,
         bool: AsPrimitive<TOut>,
     {
         let output_elements = input_elements
@@ -114,7 +117,7 @@ impl FilterTraits for Equal {
         chunk_input: &zarrs::array::ChunkRepresentation,
         chunk_output: &zarrs::array::ChunkRepresentation,
     ) -> usize {
-        chunk_input.size_usize() + chunk_output.size_usize()
+        chunk_input.fixed_element_size().unwrap() + chunk_output.fixed_element_size().unwrap()
     }
 
     fn output_data_type(&self, _input: &Array<FilesystemStore>) -> Option<(DataType, FillValue)> {

diff --git a/src/filter/filters/gaussian.rs b/src/filter/filters/gaussian.rs
@@ -5,7 +5,7 @@ use num_traits::AsPrimitive;
 use rayon::iter::{IntoParallelIterator, ParallelIterator};
 use serde::{Deserialize, Serialize};
 use zarrs::{
-    array::{data_type::UnsupportedDataTypeError, Array, DataType},
+    array::{data_type::UnsupportedDataTypeError, Array, DataType, Element, ElementOwned},
     array_subset::ArraySubset,
     storage::store::FilesystemStore,
 };
@@ -77,8 +77,8 @@ impl Gaussian {
         progress: &Progress,
     ) -> Result<(), FilterError>
     where
-        TIn: bytemuck::Pod + Send + Sync + AsPrimitive<f32>,
-        TOut: bytemuck::Pod + Send + Sync,
+        TIn: ElementOwned + Send + Sync + AsPrimitive<f32>,
+        TOut: Element + Send + Sync + Copy + 'static,
         f32: AsPrimitive<TOut>,
     {
         let subset_output = output.chunk_subset_bounded(chunk_indices).unwrap();
@@ -98,7 +98,7 @@ impl Gaussian {
 
         progress.write(|| {
             output
-                .store_array_subset_ndarray::<TOut, _, _>(subset_output.start(), output_array)
+                .store_array_subset_ndarray::<TOut, _>(subset_output.start(), output_array)
                 .unwrap()
         });
 
@@ -160,8 +160,10 @@ impl FilterTraits for Gaussian {
         )
         .unwrap();
         let num_output_elements = chunk_output.num_elements_usize();
-        num_input_elements * (chunk_input.data_type().size() + core::mem::size_of::<f32>() * 2)
-            + num_output_elements * (core::mem::size_of::<f32>() + chunk_output.data_type().size())
+        num_input_elements
+            * (chunk_input.data_type().fixed_size().unwrap() + core::mem::size_of::<f32>() * 2)
+            + num_output_elements
+                * (core::mem::size_of::<f32>() + chunk_output.data_type().fixed_size().unwrap())
     }
 
     fn apply(