From ac80497123e7f764bd052f6c21cf0dd8f0a15661 Mon Sep 17 00:00:00 2001 From: Lachlan Deakin Date: Wed, 31 Jul 2024 10:40:56 +1000 Subject: [PATCH] Add HTTP read support to `zarrs_reencode` Bump opendal to 0.48 --- CHANGELOG.md | 4 + Cargo.lock | 173 ++++++++++-------- Cargo.toml | 11 +- src/bin/zarrs_benchmark_read_async.rs | 6 +- src/bin/zarrs_benchmark_read_async_as_sync.rs | 6 +- src/bin/zarrs_reencode.rs | 53 +++++- src/lib.rs | 9 +- 7 files changed, 160 insertions(+), 102 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7565f6f..d00fa55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + - Add http read support to `zarrs_reencode` + - Bump `opendal` to 0.48 + ## [0.5.4] - 2024-07-30 ### Changed diff --git a/Cargo.lock b/Cargo.lock index 6e0b70c..67f0b48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -43,9 +43,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.14" +version = "0.6.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" +checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" dependencies = [ "anstyle", "anstyle-parse", @@ -58,33 +58,33 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" +checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" [[package]] name = "anstyle-parse" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" +checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391" +checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" dependencies = [ "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" +checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" dependencies = [ "anstyle", "windows-sys 0.52.0", @@ -230,9 +230,9 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytemuck" -version = "1.16.1" +version = "1.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b236fc92302c97ed75b38da1f4917b5cdda4984745740f153a5d3059e48d725e" +checksum = "102087e286b4677862ea56cf8fc58bb2cdfa8725c40ffb80fe3a008eb7f2fc83" dependencies = [ "bytemuck_derive", ] @@ -248,6 +248,12 @@ dependencies = [ "syn 2.0.72", ] +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.6.1" @@ -256,9 +262,9 @@ checksum = "a12916984aab3fa6e39d655a33e09c0071eb36d6ab3aea5c2d78551f1df6d952" [[package]] name = "cc" -version = "1.1.6" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aba8f4e9906c7ce3c73463f62a7f0c65183ada1a2d47e397cc8810827f9694f" +checksum = "26a5c3fd7bfa1ce3897a3a3501d362b2d87b7f2583ebcb4a949ec25911025cbc" dependencies = [ "jobserver", "libc", @@ -285,9 +291,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.9" +version = "4.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64acc1846d54c1fe936a78dc189c34e28d3f5afc348403f28ecf53660b9b8462" +checksum = "35723e6a11662c2afb578bcf0b88bf6ea8e21282a953428f240574fcc3a2b5b3" dependencies = [ "clap_builder", "clap_derive", @@ -295,9 +301,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.9" +version = "4.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8393d67ba2e7bfaf28a23458e4e2b543cc73a99595511eb207fdb8aede942" +checksum = "49eb96cbfa7cfa35017b7cd548c75b14c3118c98b423041d70562665e07fb0fa" dependencies = [ "anstream", "anstyle", @@ -307,9 +313,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.8" +version = "4.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bac35c6dafb060fd4d275d9a4ffae97917c13a6327903a8be2153cd964f7085" +checksum = "5d029b67f89d30bbb547c89fd5161293c0aec155fc691d7924b64550662db93e" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -319,9 +325,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" +checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" [[package]] name = "cmake" @@ -334,9 +340,9 @@ dependencies = [ [[package]] name = "colorchoice" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" [[package]] name = "concurrent-queue" @@ -443,9 +449,9 @@ dependencies = [ [[package]] name = "curl-sys" -version = "0.4.73+curl-8.8.0" +version = "0.4.74+curl-8.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "450ab250ecf17227c39afb9a2dd9261dc0035cb80f2612472fc0c4aac2dcb84d" +checksum = "8af10b986114528fcdc4b63b6f5f021b7057618411046a4de2ba0f0149a097bf" dependencies = [ "cc", "libc", @@ -973,9 +979,9 @@ checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "is_terminal_polyfill" -version = "1.70.0" +version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" [[package]] name = "itertools" @@ -994,9 +1000,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jobserver" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" dependencies = [ "libc", ] @@ -1093,9 +1099,9 @@ dependencies = [ [[package]] name = "matrixmultiply" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7574c1cf36da4798ab73da5b215bbf444f50718207754cb522201d78d1cd0ff2" +checksum = "9380b911e3e96d10c1f415da0876389aaf1b56759054eeb0de7df940c456ba1a" dependencies = [ "autocfg", "rawpointer", @@ -1134,13 +1140,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.11" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4" dependencies = [ + "hermit-abi", "libc", "wasi", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -1299,16 +1306,6 @@ dependencies = [ "libm", ] -[[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi", - "libc", -] - [[package]] name = "number_prefix" version = "0.4.0" @@ -1317,9 +1314,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.36.1" +version = "0.36.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "081b846d1d56ddfc18fdf1a922e4f6e07a11768ea1b92dec44e42b72712ccfce" +checksum = "3f203fa8daa7bb185f760ae12bd8e097f63d17041dcdcaf675ac54cdf863170e" dependencies = [ "memchr", ] @@ -1340,7 +1337,7 @@ dependencies = [ "itertools", "parking_lot", "percent-encoding", - "quick-xml 0.36.0", + "quick-xml", "rand", "reqwest", "ring", @@ -1372,9 +1369,9 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "opendal" -version = "0.47.3" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cac4826fe3d5482a49b92955b0f6b06ce45b46ec84484176588209bfbf996870" +checksum = "615d41187deea0ea7fab5b48e9afef6ae8fc742fdcfa248846ee3d92ff71e986" dependencies = [ "anyhow", "async-trait", @@ -1390,7 +1387,7 @@ dependencies = [ "md-5", "once_cell", "percent-encoding", - "quick-xml 0.31.0", + "quick-xml", "reqwest", "serde", "serde_json", @@ -1513,9 +1510,13 @@ checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265" [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "2288c0e17cc8d342c712bb43a257a80ebffce59cdb33d5000d8348f3ec02528b" +dependencies = [ + "zerocopy", + "zerocopy-derive", +] [[package]] name = "proc-macro2" @@ -1528,19 +1529,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.31.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" -dependencies = [ - "memchr", - "serde", -] - -[[package]] -name = "quick-xml" -version = "0.36.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4091e032efecb09d7b1f711f487b85ab925632a842627e3200fb088382cde32c" +checksum = "96a05e2e8efddfa51a84ca47cec303fac86c8541b686d37cac5efc0e094417bc" dependencies = [ "memchr", "serde", @@ -1800,9 +1791,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.11" +version = "0.23.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4828ea528154ae444e5a642dbb7d5623354030dc9822b83fd9bb79683c7399d0" +checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" dependencies = [ "once_cell", "ring", @@ -1933,12 +1924,13 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.120" +version = "1.0.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +checksum = "4ab380d7d9f22ef3f21ad3e6c1ebe8e4fc7a2000ccba2e4d71fc96f15b2cb609" dependencies = [ "indexmap", "itoa", + "memchr", "ryu", "serde", ] @@ -2133,26 +2125,25 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.38.1" +version = "1.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb2caba9f80616f438e09748d5acda951967e1ea58508ef53d9c6402485a46df" +checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1" dependencies = [ "backtrace", "bytes", "libc", "mio", - "num_cpus", "pin-project-lite", "socket2", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "tokio-macros" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", @@ -2321,9 +2312,9 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "walkdir" @@ -2650,9 +2641,9 @@ dependencies = [ [[package]] name = "zarrs" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3e13209785de81409deb78dce3bf405cb7725fba163b2817c2615ce78e2a273" +checksum = "78dd3f58d07d622c8a7c7bbb88e2354724357996a19aa30b61717906d6db952c" dependencies = [ "async-lock", "async-recursion", @@ -2688,8 +2679,9 @@ dependencies = [ [[package]] name = "zarrs_tools" -version = "0.5.4" +version = "0.5.5" dependencies = [ + "anyhow", "approx", "async-scoped", "bytemuck", @@ -2716,6 +2708,27 @@ dependencies = [ "zarrs", ] +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + [[package]] name = "zeroize" version = "1.8.1" diff --git a/Cargo.toml b/Cargo.toml index ceedec4..550944c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "zarrs_tools" -version = "0.5.4" +version = "0.5.5" authors = ["Lachlan Deakin "] edition = "2021" rust-version = "1.75" @@ -16,7 +16,7 @@ exclude = [ ] [features] -async = ["zarrs/async", "zarrs/object_store", "zarrs/opendal", "dep:object_store", "dep:opendal"] +async = ["zarrs/object_store", "dep:object_store"] benchmark = [] binary2zarr = [] filter = [] @@ -25,6 +25,7 @@ ncvar2zarr = ["dep:netcdf"] ome = ["dep:ome_zarr_metadata"] [dependencies] +anyhow = "1.0.86" approx = "0.5.1" async-scoped = { version = "0.9.0", features = ["use-tokio"] } bytemuck = "1.16.0" @@ -37,9 +38,9 @@ itertools = "0.13.0" ndarray = { version = "0.15.6", features = ["rayon", "approx-0_5"] } netcdf = { version = "0.9.0", optional = true, features = ["static"] } num-traits = "0.2.18" -object_store = { version = "0.10.1", optional = true, features = ["http"] } +object_store = { version = "0.10.1", optional = true, features = ["http"] } # , "aws" ome_zarr_metadata = { version = "0.1.0", optional = true } -opendal = { version = "0.47", optional = true, default-features = false, features = ["services-fs", "services-http"] } +opendal = { version = "0.48", default-features = false, features = ["services-fs", "services-http"] } # , "services-s3" rayon = "1.10.0" rayon_iter_concurrent_limit = "0.2.0" serde = { version = "1.0.197", features = ["derive"] } @@ -48,7 +49,7 @@ sysinfo = "0.30.6" tempfile = "3.10.1" thiserror = "1.0.57" tokio = { version = "1.34.0", features = ["macros", "rt-multi-thread"] } -zarrs = { version = "0.16.0" } +zarrs = { version = "0.16.0", features = ["async", "opendal"] } [[bin]] name = "zarrs_benchmark_read_sync" diff --git a/src/bin/zarrs_benchmark_read_async.rs b/src/bin/zarrs_benchmark_read_async.rs index 8346ff4..ccda3d4 100644 --- a/src/bin/zarrs_benchmark_read_async.rs +++ b/src/bin/zarrs_benchmark_read_async.rs @@ -47,8 +47,7 @@ async fn main() -> Result<(), Box> { let storage: AsyncReadableStorage = if args.path.starts_with("http") { // opendal - let mut builder = opendal::services::Http::default(); - builder.endpoint(&args.path); + let builder = opendal::services::Http::default().endpoint(&args.path); let operator = opendal::Operator::new(builder)?.finish(); Arc::new(store::AsyncOpendalStore::new(operator)) @@ -61,8 +60,7 @@ async fn main() -> Result<(), Box> { // Arc::new(store::AsyncObjectStore::new(store)) } else { // opendal - let mut builder = opendal::services::Fs::default(); - builder.root(&args.path); + let builder = opendal::services::Fs::default().root(&args.path); let operator = opendal::Operator::new(builder)?.finish(); Arc::new(store::AsyncOpendalStore::new(operator)) diff --git a/src/bin/zarrs_benchmark_read_async_as_sync.rs b/src/bin/zarrs_benchmark_read_async_as_sync.rs index 29171ad..4a6d7a1 100644 --- a/src/bin/zarrs_benchmark_read_async_as_sync.rs +++ b/src/bin/zarrs_benchmark_read_async_as_sync.rs @@ -55,8 +55,7 @@ fn main() -> Result<(), Box> { let storage: AsyncReadableStorage = if args.path.starts_with("http") { // opendal - let mut builder = opendal::services::Http::default(); - builder.endpoint(&args.path); + let builder = opendal::services::Http::default().endpoint(&args.path); let operator = opendal::Operator::new(builder)?.finish(); Arc::new(store::AsyncOpendalStore::new(operator)) @@ -69,8 +68,7 @@ fn main() -> Result<(), Box> { // Arc::new(store::AsyncObjectStore::new(store)) } else { // opendal - let mut builder = opendal::services::Fs::default(); - builder.root(&args.path); + let builder = opendal::services::Fs::default().root(&args.path); let operator = opendal::Operator::new(builder)?.finish(); Arc::new(store::AsyncOpendalStore::new(operator)) diff --git a/src/bin/zarrs_reencode.rs b/src/bin/zarrs_reencode.rs index da57365..7b15b5d 100644 --- a/src/bin/zarrs_reencode.rs +++ b/src/bin/zarrs_reencode.rs @@ -1,8 +1,14 @@ +use core::f32; use std::sync::Arc; use clap::Parser; use indicatif::{ProgressBar, ProgressStyle}; -use zarrs::storage::{ListableStorageTraits, StorePrefix, WritableStorageTraits}; +use zarrs::storage::{ + storage_adapter::async_to_sync::{AsyncToSyncBlockOn, AsyncToSyncStorageAdapter}, + store::AsyncOpendalStore, + AsyncReadableListableStorage, ListableStorageTraits, ReadableListableStorage, StorePrefix, + WritableStorageTraits, +}; use zarrs_tools::{ do_reencode, get_array_builder_reencode, progress::{ProgressCallback, ProgressStats}, @@ -78,13 +84,45 @@ fn progress_callback(stats: ProgressStats, bar: &ProgressBar) { } } -fn main() -> Result<(), Box> { +struct TokioBlockOn(tokio::runtime::Runtime); + +impl AsyncToSyncBlockOn for TokioBlockOn { + fn block_on(&self, future: F) -> F::Output { + self.0.block_on(future) + } +} + +fn get_storage(path: &str) -> anyhow::Result { + if path.starts_with("http://") || path.starts_with("https://") { + let builder = opendal::services::Http::default().endpoint(path); + let operator = opendal::Operator::new(builder)?.finish(); + let storage: AsyncReadableListableStorage = Arc::new(AsyncOpendalStore::new(operator)); + let block_on = TokioBlockOn(tokio::runtime::Runtime::new()?); + Ok(Arc::new(AsyncToSyncStorageAdapter::new(storage, block_on))) + // } else if path.starts_with("s3://") { + // let endpoint = ""; + // let bucket = ""; + // let root = ""; + // let region = "auto"; + // let builder = opendal::services::S3::default() + // .endpoint(&endpoint) + // .region(®ion) + // .root(path) + // .allow_anonymous() + // .bucket(&bucket); + // let operator = opendal::Operator::new(builder)?.finish(); + // Arc::new(AsyncOpendalStore::new(operator)) + } else { + Ok(Arc::new(zarrs::storage::store::FilesystemStore::new(path)?)) + } +} + +fn main() -> anyhow::Result<()> { let args = Args::parse(); zarrs::config::global_config_mut().set_validate_checksums(!args.ignore_checksums); - let storage_in = - Arc::new(zarrs::storage::store::FilesystemStore::new(args.path_in.clone()).unwrap()); + let storage_in = get_storage(&args.path_in)?; let array_in = zarrs::array::Array::open(storage_in.clone(), "/").unwrap(); if args.verbose { println!( @@ -114,8 +152,11 @@ fn main() -> Result<(), Box> { )?; bar.set_style(bar_style_finish()); bar.finish_and_clear(); - let size_in = storage_in.size().unwrap() as f32; - let size_out = storage_out.size().unwrap() as f32; + let size_in = storage_in + .size() + .map(|size| size as f32) + .unwrap_or(f32::NAN); + let size_out = storage_out.size().unwrap_or_default() as f32; let bytes_decoded = bytes_decoded as f32; println!( "Reencode {} to {}\n\tread: ~{:.2}ms @ {:.2}GB/s\n\twrite: ~{:.2}ms @ {:.2}GB/s\n\ttotal: {:.2}ms\n\tsize: {:.2}MB to {:.2}MB ({:.2}MB uncompressed)", diff --git a/src/lib.rs b/src/lib.rs index 4055f9b..f404fa7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,7 +21,7 @@ use zarrs::{ array_subset::ArraySubset, config::global_config, metadata::Metadata, - storage::{store::FilesystemStore, ReadableWritableStorageTraits}, + storage::{ReadableStorageTraits, ReadableWritableStorageTraits}, }; pub mod filter; @@ -586,8 +586,11 @@ pub fn get_array_builder_reencode( array_builder } -pub fn do_reencode( - array_in: &Array, +pub fn do_reencode< + TStorageIn: ReadableStorageTraits + ?Sized + 'static, + TStorageOut: ReadableWritableStorageTraits + ?Sized + 'static, +>( + array_in: &Array, array_out: &Array, validate: bool, concurrent_chunks: Option,