Oxidize ZipStorage (#1909)

Expose the (read-only for now) Rust ZipStorage and use it instead of the regular (read-write) ZipStorage.

if writing is needed, fall back to current ZipStorage (as a stopgap while the Rust one doesn't support writing)

This PR also removes Python 3.7 support (which we actually dropped in #1839), and adds Python 3.9 and Python 3.10 to the setup.cfg classifiers.

* init rust zipstorage
* unify into ZipStorage
* benchmark that works with previous ZipStorage
* benchmark for loading small data from ZipStorage
* ouroboros seems to work
* peakmem benchmark for zipstorage
* add docker target in nix

Makefile

@@ -32,6 +32,7 @@ include/sourmash.h: src/core/src/lib.rs \
                     src/core/src/ffi/nodegraph.rs \
                     src/core/src/ffi/index/mod.rs \
                     src/core/src/ffi/index/revindex.rs \
+                    src/core/src/ffi/storage.rs \
                     src/core/src/errors.rs
 	cd src/core && \
 	RUSTC_BOOTSTRAP=1 cbindgen -c cbindgen.toml . -o ../../$@

benchmarks/benchmarks.py

@@ -1,6 +1,10 @@
+import os
 import random
+from pathlib import Path
+from tempfile import NamedTemporaryFile
 
 
+from sourmash.sbt_storage import ZipStorage
 from sourmash.minhash import MinHash
 
 
@@ -139,3 +143,60 @@ class PeakmemMinAbundanceSuite(PeakmemMinHashSuite):
     def setup(self):
         PeakmemMinHashSuite.setup(self)
         self.mh = MinHash(500, 21, track_abundance=True)
+
+####################
+
+class TimeZipStorageSuite:
+
+    def setup(self):
+        import zipfile
+        self.zipfile = NamedTemporaryFile()
+
+        with zipfile.ZipFile(self.zipfile, mode='w',
+                          compression=zipfile.ZIP_STORED) as storage:
+            for i in range(100_000):
+                # just so we have lots of entries
+                storage.writestr(str(i), b"0")
+            # one big-ish entry
+            storage.writestr("sig1", b"9" * 1_000_000)
+
+    def time_load_from_zipstorage(self):
+        with ZipStorage(self.zipfile.name) as storage:
+            for i in range(20):
+                storage.load("sig1")
+
+    def time_load_small_from_zipstorage(self):
+        with ZipStorage(self.zipfile.name) as storage:
+            for i in range(20):
+                storage.load("99999")
+
+    def teardown(self):
+        self.zipfile.close()
+
+
+class PeakmemZipStorageSuite:
+    def setup(self):
+        import zipfile
+        self.zipfile = NamedTemporaryFile()
+
+        with zipfile.ZipFile(self.zipfile, mode='w',
+                          compression=zipfile.ZIP_STORED) as storage:
+            for i in range(100_000):
+                # just so we have lots of entries
+                storage.writestr(str(i), b"0")
+            # one big-ish entry
+            storage.writestr("sig1", b"9" * 1_000_000)
+
+
+    def peakmem_load_from_zipstorage(self):
+        with ZipStorage(self.zipfile.name) as storage:
+            for i in range(20):
+                storage.load("sig1")
+
+    def peakmem_load_small_from_zipstorage(self):
+        with ZipStorage(self.zipfile.name) as storage:
+            for i in range(20):
+                storage.load("99999")
+
+    def teardown(self):
+        self.zipfile.close()

doc/environment.yml

@@ -3,4 +3,4 @@ channels:
   - defaults
 dependencies:
   - rust
-  - python =3.7
+  - python =3.8

flake.nix

@@ -77,6 +77,20 @@
             DYLD_LIBRARY_PATH = "${self.packages.${system}.lib}/lib";
             NO_BUILD = "1";
           };
+          docker =
+            let
+              bin = self.defaultPackage.${system};
+            in
+            pkgs.dockerTools.buildLayeredImage {
+              name = bin.pname;
+              tag = bin.version;
+              contents = [ bin ];
+
+              config = {
+                Cmd = [ "/bin/sourmash" ];
+                WorkingDir = "/";
+              };
+            };
         };
 
         defaultPackage = self.packages.${system}.sourmash;

include/sourmash.h

@@ -58,6 +58,8 @@ typedef struct SourmashSearchResult SourmashSearchResult;
 
 typedef struct SourmashSignature SourmashSignature;
 
+typedef struct SourmashZipStorage SourmashZipStorage;
+
 /**
  * Represents a string.
  */
@@ -456,4 +458,23 @@ SourmashStr sourmash_str_from_cstr(const char *s);
 
 char sourmash_translate_codon(const char *codon);
 
+SourmashStr **zipstorage_filenames(const SourmashZipStorage *ptr, uintptr_t *size);
+
+void zipstorage_free(SourmashZipStorage *ptr);
+
+SourmashStr **zipstorage_list_sbts(const SourmashZipStorage *ptr, uintptr_t *size);
+
+const uint8_t *zipstorage_load(const SourmashZipStorage *ptr,
+                               const char *path_ptr,
+                               uintptr_t insize,
+                               uintptr_t *size);
+
+SourmashZipStorage *zipstorage_new(const char *ptr, uintptr_t insize);
+
+SourmashStr zipstorage_path(const SourmashZipStorage *ptr);
+
+void zipstorage_set_subdir(SourmashZipStorage *ptr, const char *path_ptr, uintptr_t insize);
+
+SourmashStr zipstorage_subdir(const SourmashZipStorage *ptr);
+
 #endif /* SOURMASH_H_INCLUDED */

setup.cfg

@@ -20,8 +20,9 @@ classifiers =
     Operating System :: POSIX :: Linux
     Operating System :: MacOS :: MacOS X
     Programming Language :: Rust
-    Programming Language :: Python :: 3.7
     Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
     Topic :: Scientific/Engineering :: Bio-Informatics
 project_urls = 
     Documentation = https://sourmash.readthedocs.io
@@ -42,7 +43,7 @@ install_requires =
     scipy
     deprecation>=2.0.6
     cachetools>=4,<5
-python_requires = >=3.7
+python_requires = >=3.8
 
 [bdist_wheel]
 universal = 1

src/core/Cargo.toml

@@ -47,6 +47,7 @@ twox-hash = "1.6.0"
 vec-collections = "0.3.4"
 piz = "0.4.0"
 memmap2 = "0.5.0"
+ouroboros = "0.15.0"
 
 [dev-dependencies]
 assert_matches = "1.3.0"

src/core/src/ffi/mod.rs

@@ -12,6 +12,7 @@ pub mod index;
 pub mod minhash;
 pub mod nodegraph;
 pub mod signature;
+pub mod storage;
 
 use std::ffi::CStr;
 use std::os::raw::c_char;