Replacing C++ with Rust

- Update build system and CI - Add Rust install instructions to docs - Remove dependency on Cython (replaced with cffi and milksnake) - Move _minhash.pyx to _minhash.py, and remove Cython bits - Add convenience functions and classes to work with Rust layer - Remove third-party/ directory
sourmash-bio · Dec 17, 2019 · 320b021 · 320b021
1 parent 93b9b90
commit 320b021
Show file tree

Hide file tree

Showing 35 changed files with 982 additions and 1,755 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -6,6 +6,5 @@ omit =
     doc/conf.py
     setup.py
     tests/*
-    third-party/smhasher/MurmurHash3.cc
     .tox/*
     benchmarks/*
diff --git a/.gitignore b/.gitignore
@@ -13,14 +13,9 @@ dist
 build
 sourmash.egg-info
 .ipynb_checkpoints
-_minhash.so
 .cache
 *.so
 .coverage
-sourmash_lib/_minhash.cpp
-sourmash/_minhash.cpp
-.asv/
-.eggs/
 .pytest_cache
 .python-version
 sourmash/version.py
@@ -30,6 +25,9 @@ sourmash/_lowlevel*.py
 .env
 Pipfile
 Pipfile.lock
-ocf/target/
 target/
 Cargo.lock
+.eggs
+.asv
+pkg/
+wasm-pack.log
diff --git a/.travis.yml b/.travis.yml
@@ -7,16 +7,18 @@ cache:
     - "$HOME/.cache/pip"
     - "$HOME/.cargo"
     - "target"
-    - ".tox"
 
 branches:
   only:
   - master
   - "/^v.*$/"
 
-script: tox
+script: tox -vv
 
-install: pip install tox-travis
+install:
+  - source .travis/install_cargo.sh
+
+before_script: pip install tox-travis
 
 jobs:
   allow_failures:
@@ -28,12 +30,14 @@ jobs:
 
     - &test
       stage: test
-      python: 2.7
+      python: 3.6
     - <<: *test
       os: osx
+      osx_image: xcode10.1
+      python: 3.7
       language: generic
       env:
-        - TOXENV=py36
+        - TOXENV=py37
     - <<: *test
       python: 3.7
       name: integration (ipfs/redis)
@@ -45,7 +49,7 @@ jobs:
         - redis-server
         - docker
     - <<: *test
-      python: 3.6
+      python: 2.7
     - <<: *test
       python: 3.5
 
@@ -55,12 +59,16 @@ jobs:
       services:
         - docker
       env:
-        - PIP=pip
+        - CIBW_BUILD='cp37-*'
         - CIBW_SKIP='*-manylinux_i686'
-      install: skip
+        - CIBW_BEFORE_BUILD='source .travis/install_cargo.sh'
+        - CIBW_ENVIRONMENT='PATH="$HOME/.cargo/bin:$PATH"'
+        - CIBW_ENVIRONMENT_MACOS='MACOSX_DEPLOYMENT_TARGET=10.11'
+      before_script: skip
       script:
-        - sudo $PIP install cibuildwheel==1.0.0
-        - cibuildwheel --output-dir wheelhouse
+        - python -m pip install -U pip setuptools
+        - python -m pip install cibuildwheel==1.1.0
+        - python -m cibuildwheel --output-dir wheelhouse
       deploy:
         provider: releases
         api_key:
@@ -73,12 +81,7 @@ jobs:
     - <<: *wheel
       os: osx
       osx_image: xcode10.1
-      language: generic
-      before_script:
-        - sudo $PIP install -U pip setuptools
-      env:
-        - PIP=pip2
-        - CIBW_ENVIRONMENT_MACOS='MACOSX_DEPLOYMENT_TARGET=10.11'
+      language: shell
 
 stages:
   - check

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,10 +1,16 @@
 include LICENSE Makefile Dockerfile LICENSE Makefile README.md requirements.txt
 include index.ipynb
+include sourmash VERSION
 recursive-include sourmash_lib *
 recursive-include sourmash *
-recursive-include third-party *.cc *.h
-exclude tests/*
+recursive-include src *.rs
+recursive-include benches *.rs
+include Cargo.toml
+include include/sourmash.h
+prune .eggs
+global-exclude *.rlib
 global-exclude *.orig
 global-exclude *.pyc
 global-exclude *.so
 prune tests/test-data/
+global-exclude *.git/
diff --git a/Makefile b/Makefile
@@ -1,10 +1,13 @@
 PYTHON ?= python
 
-all:
-	$(PYTHON) setup.py build_ext -i
+all: build
 
 .PHONY:
 
+build:
+	$(PYTHON) setup.py build_ext -i
+	cargo build
+
 clean:
 	$(PYTHON) setup.py clean --all
 	rm -f sourmash/*.so
@@ -19,6 +22,7 @@ dist: FORCE
 test: all
 	pip install -e '.[test]'
 	$(PYTHON) -m pytest
+	cargo test
 
 doc: .PHONY
 	cd doc && make html
@@ -29,12 +33,12 @@ include/sourmash.h: src/lib.rs src/ffi/minhash.rs src/ffi/signature.rs src/ffi/n
 	rustup override set stable
 
 coverage: all
-	$(PYTHON) setup.py clean --all
-	SOURMASH_COVERAGE=1 $(PYTHON) setup.py build_ext -i
+	$(PYTHON) setup.py build_ext -i
 	$(PYTHON) -m pytest --cov=. --cov-report term-missing
 
 benchmark:
 	asv continuous master `git rev-parse HEAD`
+	cargo bench
 
 check:
 	cargo build

diff --git a/README.md b/README.md
@@ -1,9 +1,19 @@
+<meta charset="utf-8"/>
+
 # sourmash
 
 [![Documentation](https://readthedocs.org/projects/sourmash/badge/?version=latest)](http://sourmash.readthedocs.io/en/latest/)
 [![Build Status](https://travis-ci.com/dib-lab/sourmash.svg?branch=master)](https://travis-ci.com/dib-lab/sourmash)
+<a href="https://pypi.org/project/sourmash/"><img alt="PyPI" src="https://badge.fury.io/py/sourmash.svg"></a>
 [![codecov](https://codecov.io/gh/dib-lab/sourmash/branch/master/graph/badge.svg)](https://codecov.io/gh/dib-lab/sourmash)
 [![DOI](http://joss.theoj.org/papers/10.21105/joss.00027/status.svg)](http://joss.theoj.org/papers/10.21105/joss.00027)
+<a href="https://github.com/dib-lab/sourmash/blob/master/LICENSE"><img alt="License: 3-Clause BSD" src="https://img.shields.io/badge/License-BSD%203--Clause-blue.svg"></a>
+
+🦀
+[![](http://meritbadge.herokuapp.com/sourmash)](https://crates.io/crates/sourmash)
+[![Rust API Documentation on docs.rs](https://docs.rs/sourmash/badge.svg)](https://docs.rs/sourmash)
+
+---
 
 Compute MinHash signatures for nucleotide (DNA/RNA) and protein sequences.
 
@@ -13,7 +23,7 @@ Usage:
     sourmash compare *.sig -o distances
     sourmash plot distances
 
-Sourmash 1.0 is [published on JOSS](https://doi.org/10.21105/joss.00027); please cite that paper if you use sourmash (`doi: 10.21105/joss.00027`):.
+sourmash 1.0 is [published on JOSS](https://doi.org/10.21105/joss.00027); please cite that paper if you use sourmash (`doi: 10.21105/joss.00027`):.
 
 ----
 
@@ -48,9 +58,10 @@ A quickstart tutorial [is available](https://sourmash.readthedocs.io/en/latest/t
 ### Requirements
 
 sourmash runs under both Python 2.7.x and Python 3.5+.  The base
-requirements are screed and ijson, together with a C++ development
-environment and the CPython development headers and libraries (for the
-C++ extension).
+requirements are screed and ijson, together with a Rust environment (for the
+extension code). We suggest using `rustup` to install the Rust environment:
+
+    curl https://sh.rustup.rs -sSf | sh
 
 The comparison code (`sourmash compare`) uses numpy, and the plotting
 code uses matplotlib and scipy, but most of the code is usable without

diff --git a/doc/developer.md b/doc/developer.md
@@ -7,7 +7,13 @@ You can get the latest development master branch with:
 ```
 git clone https://github.com/dib-lab/sourmash.git
 ```
-To install all of the necessary dependencies, do:
+sourmash runs under both Python 2.7.x and Python 3.5+.  The base
+requirements are screed and ijson, together with a Rust environment (for the
+extension code). We suggest using `rustup` to install the Rust environment:
+
+    curl https://sh.rustup.rs -sSf | sh
+
+To install all of the necessary Python dependencies, do:
 ```
 pip install -r requirements.txt
 ```
@@ -25,13 +31,6 @@ pip install -e .
 
 We use [Travis][0] for continuous integration.
 
-Code coverage calculation is enabled (on Linux only) by running
-`make coverage`.  This recompiles the C++ extension without
-optimization and with coverage configured.  See `setup.py` for
-more information on this; the environment variable
-`SOURMASH_COVERAGE` controls whether the C++ extension is
-compiled with code coverage analysis enabled.
-
 Code coverage can be viewed interactively at [codecov.io][1].
 
 [0]:https://travis-ci.org/dib-lab/sourmash

diff --git a/netlify.toml b/netlify.toml
@@ -0,0 +1,12 @@
+# Configuration for pull request documentation previews via Netlify
+
+[build]
+publish = "_build/html"
+base = "doc"
+command = '''
+    cd .. && \
+    curl https://sh.rustup.rs -sSf | sh -s -- -y && \
+    source $HOME/.cargo/env && \
+    pip install -e .[doc] && \
+    make doc
+'''
diff --git a/setup.py b/setup.py
@@ -1,11 +1,32 @@
 from __future__ import print_function
-import sys
-from setuptools import setup, find_packages
-from setuptools import Extension
 import os
+from setuptools import setup, find_packages
+import sys
+
+
+DEBUG_BUILD = os.environ.get("SOURMASH_DEBUG") == "1"
+
+
+def build_native(spec):
+    cmd = ["cargo", "build", "--lib"]
+
+    target = "debug"
+    if not DEBUG_BUILD:
+        cmd.append("--release")
+        target = "release"
+
+    build = spec.add_external_build(cmd=cmd, path=".")
+
+    rtld_flags = ["NOW"]
+    if sys.platform == "darwin":
+        rtld_flags.append("NODELETE")
+    spec.add_cffi_module(
+        module_path="sourmash._lowlevel",
+        dylib=lambda: build.find_dylib("sourmash", in_path="target/%s" % target),
+        header_filename=lambda: build.find_header("sourmash.h", in_path="include"),
+        rtld_flags=rtld_flags,
+    )
 
-EXTRA_COMPILE_ARGS = ['-std=c++11', '-pedantic']
-EXTRA_LINK_ARGS=[]
 
 CLASSIFIERS = [
     "Environment :: Console",
@@ -15,7 +36,7 @@
     "Natural Language :: English",
     "Operating System :: POSIX :: Linux",
     "Operating System :: MacOS :: MacOS X",
-    "Programming Language :: C++",
+    "Programming Language :: Rust",
     "Programming Language :: Python :: 2.7",
     "Programming Language :: Python :: 3.5",
     "Programming Language :: Python :: 3.6",
@@ -24,24 +45,10 @@
 
 CLASSIFIERS.append("Development Status :: 5 - Production/Stable")
 
-if sys.platform == 'darwin':              # Mac OS X?
-    # force 64bit only builds
-    EXTRA_COMPILE_ARGS.extend(['-arch', 'x86_64', '-mmacosx-version-min=10.7',
-                               '-stdlib=libc++'])
-
-else:                                     # ...likely Linux
-   if os.environ.get('SOURMASH_COVERAGE'):
-      print('Turning on coverage analysis.')
-      EXTRA_COMPILE_ARGS.extend(['-g', '--coverage', '-lgcov'])
-      EXTRA_LINK_ARGS.extend(['--coverage', '-lgcov'])
-   else:
-      EXTRA_COMPILE_ARGS.append('-O3')
-
-with open('README.md', 'r') as readme:
+with open("README.md", "r") as readme:
     LONG_DESCRIPTION = readme.read()
 
-SETUP_METADATA = \
-               {
+SETUP_METADATA = {
     "name": "sourmash",
     "description": "tools for comparing DNA sequences with MinHash sketches",
     "long_description": LONG_DESCRIPTION,
@@ -55,34 +62,29 @@
         'sourmash = sourmash.__main__:main'
         ]
     },
-    "ext_modules": [Extension("sourmash._minhash",
-                               sources=["sourmash/_minhash.pyx",
-                                        "third-party/smhasher/MurmurHash3.cc"],
-                               depends=["sourmash/kmer_min_hash.hh"],
-                               include_dirs=["./sourmash",
-                                             "./third-party/smhasher/"],
-                               language="c++",
-                               extra_compile_args=EXTRA_COMPILE_ARGS,
-                               extra_link_args=EXTRA_LINK_ARGS)],
     "install_requires": ["screed>=0.9", "ijson>=2.5.1", "khmer>=2.1", 'numpy',
+                         "cffi",
                          'matplotlib', 'scipy', "deprecation>=2.0.6"],
-    "setup_requires": ['Cython>=0.25.2', "setuptools>=38.6.0",
-                       'setuptools_scm', 'setuptools_scm_git_archive'],
+    "setup_requires": [
+        "setuptools>=38.6.0",
+        "milksnake",
+        "setuptools_scm",
+        "setuptools_scm_git_archive",
+    ],
     "use_scm_version": {"write_to": "sourmash/version.py"},
+    "zip_safe": False,
+    "platforms": "any",
     "extras_require": {
         'test' : ['pytest', 'pytest-cov'],
         'demo' : ['jupyter', 'jupyter_client', 'ipython'],
         'doc' : ['sphinx', 'recommonmark', 'alabaster',
                  "sphinxcontrib-napoleon", "nbsphinx"],
         '10x': ['bam2fasta==1.0.1'],
         'storage': ["ipfshttpclient", "redis"]
-        },
-    "include_package_data": True,
-    "package_data": {
-        "sourmash": ['*.pxd']
     },
-    "classifiers": CLASSIFIERS
-    }
+    "include_package_data": True,
+    "classifiers": CLASSIFIERS,
+    "milksnake_tasks": [build_native],
+}
 
 setup(**SETUP_METADATA)
-