Merge branch 'latest' of https://github.com/sourmash-bio/sourmash int…

…o update/lca_summarize_out

.github/workflows/asv.yml

@@ -22,10 +22,10 @@ jobs:
         git checkout latest
         git checkout -
 
-    - name: Set up Python 3.7
+    - name: Set up Python 3.10
       uses: actions/setup-python@v1
       with:
-        python-version: 3.7
+        python-version: "3.10"
 
     - uses: actions-rs/toolchain@v1
       with:

.github/workflows/hypothesis.yml

@@ -13,10 +13,10 @@ jobs:
     steps:
       - uses: actions/checkout@v1
 
-      - name: Set up Python 3.7
+      - name: Set up Python 3.10
         uses: actions/setup-python@v1
         with:
-          python-version: "3.7"
+          python-version: "3.10"
 
       - name: Install dependencies
         run: |

.github/workflows/khmer.yml

@@ -13,10 +13,10 @@ jobs:
     steps:
       - uses: actions/checkout@v1
 
-      - name: Set up Python 3.7
+      - name: Set up Python 3.10
         uses: actions/setup-python@v1
         with:
-          python-version: "3.7"
+          python-version: "3.10"
 
       - name: Install dependencies
         run: |

.github/workflows/python.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-18.04, macos-latest]
-        py: [3.9, 3.8, 3.7]
+        py: ["3.10", "3.9", "3.8"]
       fail-fast: false
 
     steps:

Makefile

@@ -30,6 +30,8 @@ include/sourmash.h: src/core/src/lib.rs \
                     src/core/src/ffi/minhash.rs \
                     src/core/src/ffi/signature.rs \
                     src/core/src/ffi/nodegraph.rs \
+                    src/core/src/ffi/index/mod.rs \
+                    src/core/src/ffi/index/revindex.rs \
                     src/core/src/errors.rs
 	cd src/core && \
 	RUSTC_BOOTSTRAP=1 cbindgen -c cbindgen.toml . -o ../../$@

asv.conf.json

@@ -6,7 +6,7 @@
     "branches": ["latest"],
     "dvcs": "git",
     "environment_type": "virtualenv",
-    "pythons": ["3.7"],
+    "pythons": ["3.10"],
     "env_dir": ".asv/env",
     "results_dir": ".asv/results",
     "html_dir": ".asv/html",

doc/developer.md

@@ -6,7 +6,7 @@ You can get the latest development branch with:
 ```
 git clone https://github.com/sourmash-bio/sourmash.git
 ```
-sourmash runs under Python 3.7 and later.
+sourmash runs under Python 3.8 and later.
 
 We recommend using `conda` or `Nix` for setting up an environment for developing
 new features, running tests and code quality checks.

doc/requirements.md

@@ -11,7 +11,7 @@ in a second or so on a rather slow 2016 Mac laptop.
 MinHash sketches and signatures are quite small on disk.
 
 sourmash should run with no modification on Linux and Mac OS X,
-under Python 3.7 and later.  Please see [the development repository README][0]
+under Python 3.8 and later.  Please see [the development repository README][0]
 for
 information on source code, tests, and continuous integration.
 

doc/support.md

@@ -103,8 +103,8 @@ and our intent is that it will support as-yet unreleased versions of Python 3.x
 For future versions of sourmash, we plan to follow the
 [Numpy NEP 29](https://numpy.org/neps/nep-0029-deprecation_policy.html)
 proposal for Python version support. For example, this
-would mean that we would drop support for Python 3.7 on December 26,
-2021.
+would mean that we would drop support for Python 3.8 on April 14,
+2023.
 
 ### Rust API
 

include/sourmash.h

@@ -52,6 +52,10 @@ typedef struct SourmashKmerMinHash SourmashKmerMinHash;
 
 typedef struct SourmashNodegraph SourmashNodegraph;
 
+typedef struct SourmashRevIndex SourmashRevIndex;
+
+typedef struct SourmashSearchResult SourmashSearchResult;
+
 typedef struct SourmashSignature SourmashSignature;
 
 /**
@@ -302,6 +306,51 @@ SourmashNodegraph *nodegraph_with_tables(uintptr_t ksize,
                                          uintptr_t starting_size,
                                          uintptr_t n_tables);
 
+void revindex_free(SourmashRevIndex *ptr);
+
+const SourmashSearchResult *const *revindex_gather(const SourmashRevIndex *ptr,
+                                                   const SourmashSignature *sig_ptr,
+                                                   double threshold,
+                                                   bool _do_containment,
+                                                   bool _ignore_abundance,
+                                                   uintptr_t *size);
+
+uint64_t revindex_len(const SourmashRevIndex *ptr);
+
+SourmashRevIndex *revindex_new_with_paths(const SourmashStr *const *search_sigs_ptr,
+                                          uintptr_t insigs,
+                                          const SourmashKmerMinHash *template_ptr,
+                                          uintptr_t threshold,
+                                          const SourmashKmerMinHash *const *queries_ptr,
+                                          uintptr_t inqueries,
+                                          bool keep_sigs);
+
+SourmashRevIndex *revindex_new_with_sigs(const SourmashSignature *const *search_sigs_ptr,
+                                         uintptr_t insigs,
+                                         const SourmashKmerMinHash *template_ptr,
+                                         uintptr_t threshold,
+                                         const SourmashKmerMinHash *const *queries_ptr,
+                                         uintptr_t inqueries);
+
+uint64_t revindex_scaled(const SourmashRevIndex *ptr);
+
+const SourmashSearchResult *const *revindex_search(const SourmashRevIndex *ptr,
+                                                   const SourmashSignature *sig_ptr,
+                                                   double threshold,
+                                                   bool do_containment,
+                                                   bool _ignore_abundance,
+                                                   uintptr_t *size);
+
+SourmashSignature **revindex_signatures(const SourmashRevIndex *ptr, uintptr_t *size);
+
+SourmashStr searchresult_filename(const SourmashSearchResult *ptr);
+
+void searchresult_free(SourmashSearchResult *ptr);
+
+double searchresult_score(const SourmashSearchResult *ptr);
+
+SourmashSignature *searchresult_signature(const SourmashSearchResult *ptr);
+
 void signature_add_protein(SourmashSignature *ptr, const char *sequence);
 
 void signature_add_sequence(SourmashSignature *ptr, const char *sequence, bool force);

setup.py

@@ -8,7 +8,10 @@
 
 
 def build_native(spec):
-    cmd = ["cargo", "build", "--manifest-path", "src/core/Cargo.toml", "--lib"]
+    cmd = ["cargo", "build",
+           "--manifest-path", "src/core/Cargo.toml",
+            # "--features", "parallel",
+           "--lib"]
 
     target = "debug"
     if not DEBUG_BUILD:

shell.nix

@@ -17,9 +17,9 @@ in
 
       git
       stdenv.cc.cc.lib
-      (python38.withPackages(ps: with ps; [ virtualenv tox setuptools ]))
+      (python310.withPackages(ps: with ps; [ virtualenv tox setuptools ]))
       (python39.withPackages(ps: with ps; [ virtualenv setuptools ]))
-      (python37.withPackages(ps: with ps; [ virtualenv setuptools ]))
+      (python38.withPackages(ps: with ps; [ virtualenv setuptools ]))
 
       rust-cbindgen
 

src/core/CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+Added:
+
+- An inverted index, codename Greyhound (#1238)
+
 ## [0.11.0] - 2021-07-07
 
 Added:

src/core/Cargo.toml

@@ -26,6 +26,7 @@ az = "1.0.0"
 bytecount = "0.6.0"
 byteorder = "1.4.3"
 cfg-if = "1.0"
+counter = "0.5.2"
 finch = { version = "0.4.1", optional = true }
 fixedbitset = "0.4.0"
 getset = "0.1.1"
@@ -42,6 +43,10 @@ serde_json = "1.0.53"
 primal-check = "0.3.1"
 thiserror = "1.0"
 typed-builder = "0.9.0"
+twox-hash = "1.6.0"
+vec-collections = "0.3.4"
+piz = "0.4.0"
+memmap2 = "0.5.0"
 
 [dev-dependencies]
 assert_matches = "1.3.0"

src/core/cbindgen.toml

@@ -8,6 +8,7 @@ clean = true
 
 [parse.expand]
 crates = ["sourmash"]
+features = []
 
 [enum]
 rename_variants = "QualifiedScreamingSnakeCase"