This library provides an OCaml implementation of the Zarr version 3 storage format specification for chunked & compressed multi-dimensional arrays, designed for use in parallel computing.
- Supports creating n-dimensional Zarr arrays and chunking them along any dimension.
- Compresses chunks using a variety of supported compression codecs.
- Supports indexing operations to read/write views of a Zarr array.
- Supports storing arrays in-memory or the local filesystem. It is also extensible, allowing users to easily create and use their own custom storage backends. See the example implementing a Zip file store for more details.
- Supports both synchronous and asynchronous I/O via Lwt and Eio. The user can easily use their own scheduler of choice. See the example implementing a filesystem store that uses the Picos concurrency library for non-blocking I/O.
- Leverages the strong type system of Ocaml to create a type-safe API; making it impossible to create, read or write malformed arrays.
- Supports organizing arrays into heirarchies via groups.
API documentation can be found here. The full specification of the storage format can be found there.
The library comes in several flavors dependending on the synchronous/asynchronous backend of choice. To install the synchronous API, use
$ opam install zarr-sync
To install zarr with an asynchronous API powered by Lwt
or Eio
, use
$ opam install zarr-lwt
$ opam install zarr-eio
To install the development version using the latest git commit, do
# for zarr-sync
opam pin add zarr-sync git+https://github.com/zoj613/zarr-ml
# for zarr-lwt
opam pin add zarr-lwt git+https://github.com/zoj613/zarr-ml
# for zarr-eio
opam pin add zarr-eio git+https://github.com/zoj613/zarr-ml
Below is a demonstration of the library's API for synchronous reads/writes.
A similar example using the Lwt
-backed Asynchronous API can be found here
open Zarr
open Zarr.Codecs
open Zarr.Indexing
open Zarr_sync.Storage
open Deferred.Infix (* opens infix operators >>= and >>| for monadic bind & map *)
let store = FilesystemStore.create "testdata.zarr";;
let group_node = Node.Group.of_path "/some/group";;
FilesystemStore.Group.create store group_node;;
let array_node = Node.Array.(group_node / "name");;
(* creates an array with char data type and fill value '?' *)
FilesystemStore.Array.create
~codecs:[`Transpose [|2; 0; 1|]; `Bytes BE; `Gzip L2]
~shape:[|100; 100; 50|]
~chunks:[|10; 15; 20|]
Ndarray.Char
'?'
array_node
store;;
let slice = [|R [|0; 20|]; I 10; R [||]|];;
let x = FilesystemStore.Array.read store array_node slice Ndarray.Char;;
(* Do some computation on the array slice *)
let x' = Zarr.Ndarray.map (fun _ -> Random.int 256 |> Char.chr) x;;
FilesystemStore.Array.write store array_node slice x';;
let y = FilesystemStore.Array.read store array_node slice Ndarray.Char;;
assert (Ndarray.equal x' y);;
let config =
{chunk_shape = [|5; 3; 5|]
;codecs = [`Transpose [|2; 0; 1|]; `Bytes LE; `Zstd (0, true)]
;index_codecs = [`Bytes BE; `Crc32c]
;index_location = Start};;
let shard_node = Node.Array.(group_node / "another");;
FilesystemStore.Array.create
~codecs:[`ShardingIndexed config]
~shape:[|100; 100; 50|]
~chunks:[|10; 15; 20|]
Ndarray.Complex32
Complex.zero
shard_node
store;;
let a, g = FilesystemStore.hierarchy store;;
List.map Node.Array.to_path a;;
(*- : string list = ["/some/group/name"; "/some/group/another"] *)
List.map Node.Group.to_path g;;
(*- : string list = ["/"; "/some"; "/some/group"] *)
FilesystemStore.Array.reshape store array_node [|25; 32; 10|];;
let meta = FilesystemStore.Group.metadata store group_node;;
Metadata.Group.show meta;; (* pretty prints the contents of the metadata *)
FilesystemStore.Array.exists store shard_node;;
FilesystemStore.Group.exists store group_node;;
let a, g = FilesystemStore.Group.children store group_node;;
List.map Node.Array.to_path a;;
(*- : string list = ["/some/group/name"; "/some/group/another"] *)
List.map Node.Group.to_path g;;
(*- : string list = [] *)
FilesystemStore.Group.delete store group_node;;
FilesystemStore.clear store;; (* clears the store *)
FilesystemStore.Group.rename store group_node "new_name";;
FilesystemStore.Array.rename store anode "new_name";;