Merge branch 'latest' into add-dist-est

.ci/install_cargo.sh

@@ -2,3 +2,4 @@
 curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable
 export PATH="$HOME/.cargo/bin:$PATH"
 rustc -V
+rustup target add aarch64-apple-darwin

.github/workflows/build_wheel.yml

@@ -20,6 +20,7 @@ jobs:
           linux-ppc64le,
           linux-s390x,
           macos-x86_64,
+          macos-arm64,
         ]
         include:
           - build: linux-x86_64
@@ -41,7 +42,11 @@ jobs:
           - build: macos-x86_64
             os: macos-latest
             arch: x86_64
-            macos_target: 'MACOSX_DEPLOYMENT_TARGET=10.11'
+            macos_target: 'MACOSX_DEPLOYMENT_TARGET=10.11 CARGO_BUILD_TARGET=x86_64-apple-darwin'
+          - build: macos-arm64
+            os: macos-latest
+            arch: arm64
+            macos_target: 'MACOSX_DEPLOYMENT_TARGET=11 CARGO_BUILD_TARGET=aarch64-apple-darwin'
       fail-fast: false
 
     steps:
@@ -63,12 +68,7 @@ jobs:
       - name: Build wheels
         uses: pypa/cibuildwheel@v2.2.2
         env:
-          CIBW_BUILD: "cp39-*"
-          CIBW_SKIP: "*-win32 *-manylinux_i686 *-musllinux_ppc64le *-musllinux_s390x"
-          CIBW_BEFORE_BUILD: 'source .ci/install_cargo.sh'
-          CIBW_ENVIRONMENT: 'PATH="$HOME/.cargo/bin:$PATH"'
           CIBW_ENVIRONMENT_MACOS: ${{ matrix.macos_target }}
-          CIBW_BUILD_VERBOSITY: 3
           CIBW_ARCHS_LINUX: ${{ matrix.arch }}
           CIBW_ARCHS_MACOS: ${{ matrix.arch }}
 

Makefile

@@ -32,6 +32,7 @@ include/sourmash.h: src/core/src/lib.rs \
                     src/core/src/ffi/nodegraph.rs \
                     src/core/src/ffi/index/mod.rs \
                     src/core/src/ffi/index/revindex.rs \
+                    src/core/src/ffi/storage.rs \
                     src/core/src/errors.rs
 	cd src/core && \
 	RUSTC_BOOTSTRAP=1 cbindgen -c cbindgen.toml . -o ../../$@

asv.conf.json

@@ -6,7 +6,6 @@
     "branches": ["latest"],
     "dvcs": "git",
     "environment_type": "virtualenv",
-    "pythons": ["3.10"],
     "env_dir": ".asv/env",
     "results_dir": ".asv/results",
     "html_dir": ".asv/html",

benchmarks/benchmarks.py

@@ -1,6 +1,10 @@
+import os
 import random
+from pathlib import Path
+from tempfile import NamedTemporaryFile
 
 
+from sourmash.sbt_storage import ZipStorage
 from sourmash.minhash import MinHash
 
 
@@ -139,3 +143,60 @@ class PeakmemMinAbundanceSuite(PeakmemMinHashSuite):
     def setup(self):
         PeakmemMinHashSuite.setup(self)
         self.mh = MinHash(500, 21, track_abundance=True)
+
+####################
+
+class TimeZipStorageSuite:
+
+    def setup(self):
+        import zipfile
+        self.zipfile = NamedTemporaryFile()
+
+        with zipfile.ZipFile(self.zipfile, mode='w',
+                          compression=zipfile.ZIP_STORED) as storage:
+            for i in range(100_000):
+                # just so we have lots of entries
+                storage.writestr(str(i), b"0")
+            # one big-ish entry
+            storage.writestr("sig1", b"9" * 1_000_000)
+
+    def time_load_from_zipstorage(self):
+        with ZipStorage(self.zipfile.name) as storage:
+            for i in range(20):
+                storage.load("sig1")
+
+    def time_load_small_from_zipstorage(self):
+        with ZipStorage(self.zipfile.name) as storage:
+            for i in range(20):
+                storage.load("99999")
+
+    def teardown(self):
+        self.zipfile.close()
+
+
+class PeakmemZipStorageSuite:
+    def setup(self):
+        import zipfile
+        self.zipfile = NamedTemporaryFile()
+
+        with zipfile.ZipFile(self.zipfile, mode='w',
+                          compression=zipfile.ZIP_STORED) as storage:
+            for i in range(100_000):
+                # just so we have lots of entries
+                storage.writestr(str(i), b"0")
+            # one big-ish entry
+            storage.writestr("sig1", b"9" * 1_000_000)
+
+
+    def peakmem_load_from_zipstorage(self):
+        with ZipStorage(self.zipfile.name) as storage:
+            for i in range(20):
+                storage.load("sig1")
+
+    def peakmem_load_small_from_zipstorage(self):
+        with ZipStorage(self.zipfile.name) as storage:
+            for i in range(20):
+                storage.load("99999")
+
+    def teardown(self):
+        self.zipfile.close()

doc/environment.yml

@@ -3,4 +3,4 @@ channels:
   - defaults
 dependencies:
   - rust
-  - python =3.7
+  - python =3.8

flake.nix

@@ -77,6 +77,20 @@
             DYLD_LIBRARY_PATH = "${self.packages.${system}.lib}/lib";
             NO_BUILD = "1";
           };
+          docker =
+            let
+              bin = self.defaultPackage.${system};
+            in
+            pkgs.dockerTools.buildLayeredImage {
+              name = bin.pname;
+              tag = bin.version;
+              contents = [ bin ];
+
+              config = {
+                Cmd = [ "/bin/sourmash" ];
+                WorkingDir = "/";
+              };
+            };
         };
 
         defaultPackage = self.packages.${system}.sourmash;

include/sourmash.h

@@ -58,6 +58,8 @@ typedef struct SourmashSearchResult SourmashSearchResult;
 
 typedef struct SourmashSignature SourmashSignature;
 
+typedef struct SourmashZipStorage SourmashZipStorage;
+
 /**
  * Represents a string.
  */
@@ -456,4 +458,23 @@ SourmashStr sourmash_str_from_cstr(const char *s);
 
 char sourmash_translate_codon(const char *codon);
 
+SourmashStr **zipstorage_filenames(const SourmashZipStorage *ptr, uintptr_t *size);
+
+void zipstorage_free(SourmashZipStorage *ptr);
+
+SourmashStr **zipstorage_list_sbts(const SourmashZipStorage *ptr, uintptr_t *size);
+
+const uint8_t *zipstorage_load(const SourmashZipStorage *ptr,
+                               const char *path_ptr,
+                               uintptr_t insize,
+                               uintptr_t *size);
+
+SourmashZipStorage *zipstorage_new(const char *ptr, uintptr_t insize);
+
+SourmashStr zipstorage_path(const SourmashZipStorage *ptr);
+
+void zipstorage_set_subdir(SourmashZipStorage *ptr, const char *path_ptr, uintptr_t insize);
+
+SourmashStr zipstorage_subdir(const SourmashZipStorage *ptr);
+
 #endif /* SOURMASH_H_INCLUDED */

pyproject.toml

@@ -19,3 +19,10 @@ include_trailing_comma = true
 force_grid_wrap = 0
 line_length = 88
 known_first_party = ["sourmash"]
+
+[tool.cibuildwheel]
+build = "cp39-*"
+skip = "*-win32 *-manylinux_i686 *-musllinux_ppc64le *-musllinux_s390x"
+before-build = "source .ci/install_cargo.sh"
+environment = { PATH="$HOME/.cargo/bin:$PATH" }
+build-verbosity = 3

setup.cfg

@@ -20,8 +20,9 @@ classifiers =
     Operating System :: POSIX :: Linux
     Operating System :: MacOS :: MacOS X
     Programming Language :: Rust
-    Programming Language :: Python :: 3.7
     Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
     Topic :: Scientific/Engineering :: Bio-Informatics
 project_urls = 
     Documentation = https://sourmash.readthedocs.io
@@ -42,7 +43,7 @@ install_requires =
     scipy
     deprecation>=2.0.6
     cachetools>=4,<5
-python_requires = >=3.7
+python_requires = >=3.8
 
 [bdist_wheel]
 universal = 1