From 76462fbe2177d40713bb400f6580f6e3cbc09a04 Mon Sep 17 00:00:00 2001
From: Misko <misko@cs.toronto.edu>
Date: Sun, 31 Mar 2024 10:09:27 -0700
Subject: [PATCH] move to lmdb backend

---
 spf/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/spf/utils.py b/spf/utils.py
index aa7550d5..0ebe87a2 100644
--- a/spf/utils.py
+++ b/spf/utils.py
@@ -39,8 +39,9 @@ def zarr_new_dataset(
     chunk_size=512,  # tested , blosc1 / chunk_size=512 / buffer_size (2^18~20) = seems pretty good
     compressor=None,
 ):
+    store = zarr.LMDBStore(filename, map_size=2**38, writemap=True, map_async=True)
     z = zarr.open(
-        filename,
+        store=store,
         mode="w",
     )
     if compressor is None: