Remove sucpp and depend on unifrac-binaries, instead (#139)

* Remove files that were moved to unifrac-binaries * Get ssu binaries from conda * Change compile to use the unifrac-binaries from conda * Remove sucpp, now in unnifrac-binaries repo * Minor edits to make flake8 happy and update github tests * Fix MacOs path * Fix typo * Add additional python test * Remoove sucpp lines * Update docs * Add one more python test
biocore · Jan 7, 2022 · 077fca4 · 077fca4
1 parent 3f4f412
commit 077fca4
Show file tree

Hide file tree

Showing 45 changed files with 38 additions and 14,997 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -54,78 +54,57 @@ jobs:
         conda config --add channels bioconda
         if [[ "$(uname -s)" == "Linux" ]];
         then
-          conda install --yes -c conda-forge -c bioconda gxx_linux-64=7.5.0
+          conda install --yes -c conda-forge -c bioconda gxx_linux-64
         else
-          conda install --yes -c conda-forge -c bioconda clangxx_osx-64=10.0.0
+          conda install --yes -c conda-forge -c bioconda clangxx_osx-64
         fi 
-        conda install --yes -c conda-forge -c bioconda cython "hdf5>=1.8.17" biom-format numpy "h5py>=2.7.0" "scikit-bio>=0.5.1" flake8 nose
-        conda install --yes -c conda-forge -c bioconda mkl-include lz4 hdf5-static
+        conda install --yes -c conda-forge -c bioconda cython "hdf5>=1.8.17" biom-format numpy "h5py>=2.7.0" "scikit-bio>=0.5.1" unifrac-binaries flake8 nose
         echo "$(uname -s)"
         if [[ "$(uname -s)" == "Linux" ]];
         then
           which x86_64-conda-linux-gnu-gcc
           x86_64-conda-linux-gnu-gcc -v
           x86_64-conda-linux-gnu-g++ -v
         else
-          conda install --yes liblapacke
           which clang
           clang -v
         fi
         which h5c++
-        if [[ "$(uname -s)" == "Linux" ]];
-        then
-          ./scripts/install_hpc_sdk.sh
-          source setup_nv_h5.sh
-        fi
-        pushd sucpp
-        make test
-        make main
-        make api
-        make capi_test
-        if [[ "$(uname -s)" == "Linux" ]]; 
-        then
-          rm -f ~/.R/Makevars
-          conda install -c conda-forge r-base
-          # the r-base package has a broken dependency
-          ln -s $CONDA_PREFIX/lib/libreadline.so $CONDA_PREFIX/lib/libreadline.so.6
-          R -e 'install.packages("Rcpp", repos="http://lib.stat.cmu.edu/R/CRAN/")'
-          make rapi_test
-        fi
-        popd
         pip install -e .   
             
     - name: Tests
       shell: bash -l {0}
       run: |
         conda activate unifrac
-        pushd sucpp
-        export UNIFRAC_GPU_INFO=Y
-        ./test_su
-        ./test_api
-        ./test_ska
-        popd
         nosetests
         flake8 unifrac setup.py
         
     - name: Sanity checks
       shell: bash -l {0}
       run: |
         conda activate unifrac
-        ./sucpp/ssu -i unifrac/tests/data/crawford.biom -t unifrac/tests/data/crawford.tre -o ci/test.dm -m unweighted
+        set -e
+        ssu -i unifrac/tests/data/crawford.biom -t unifrac/tests/data/crawford.tre -o ci/test.dm -m unweighted
         python -c "import skbio; dm = skbio.DistanceMatrix.read('ci/test.dm')"
+        pushd unifrac/tests
+        python -c "import unifrac; unifrac.unweighted_to_file('data/crawford.biom','data/crawford.tre','../../ci/test.dm.h5')"
+        python -c "import unifrac,skbio; dm_u=unifrac.unweighted('data/crawford.biom','data/crawford.tre');  dm = skbio.DistanceMatrix.read('../../ci/test.dm'); t=abs(dm_u.data-dm.data).max(); print(t); assert t < 0.1"
+        popd
+        python -c "import h5py,skbio; f_u=h5py.File('ci/test.dm.h5','r'); dm_u=skbio.stats.distance.DistanceMatrix(f_u['matrix'][:,:],f_u['order'][:])"
+        python -c "import h5py,skbio; dm = skbio.DistanceMatrix.read('ci/test.dm'); f_u=h5py.File('ci/test.dm.h5','r'); dm_u=skbio.stats.distance.DistanceMatrix(f_u['matrix'][:,:],f_u['order'][:]); t=abs(dm_u.data-dm.data).max(); print(t); assert t < 0.1"
         if [[ "$(uname -s)" == "Linux" ]]; 
         then
           MD5=md5sum
         else
           MD5='md5 -r'
         fi
-        ./sucpp/ssu -i unifrac/tests/data/crawford.biom -t unifrac/tests/data/crawford.tre -o ci/test.dm.start0.stop3 -m unweighted --mode partial --start 0 --stop 3
-        ./sucpp/ssu -i unifrac/tests/data/crawford.biom -t unifrac/tests/data/crawford.tre -o ci/test.dm.start3.stop5 -m unweighted --mode partial --start 3 --stop 5
-        ./sucpp/ssu -i unifrac/tests/data/crawford.biom -t unifrac/tests/data/crawford.tre -o ci/test.dm.partial --mode merge-partial --partial-pattern "ci/test.dm.start*"
+        ssu -i unifrac/tests/data/crawford.biom -t unifrac/tests/data/crawford.tre -o ci/test.dm.start0.stop3 -m unweighted --mode partial --start 0 --stop 3
+        ssu -i unifrac/tests/data/crawford.biom -t unifrac/tests/data/crawford.tre -o ci/test.dm.start3.stop5 -m unweighted --mode partial --start 3 --stop 5
+        ssu -i unifrac/tests/data/crawford.biom -t unifrac/tests/data/crawford.tre -o ci/test.dm.partial --mode merge-partial --partial-pattern "ci/test.dm.start*"
         exp=$($MD5 ci/test.dm | awk '{ print $1 }')
         obs=$($MD5 ci/test.dm.partial | awk '{ print $1 }')
         python -c "assert '${obs}' == '${exp}'"
-        ./sucpp/faithpd -i unifrac/tests/data/crawford.biom -t unifrac/tests/data/crawford.tre -o ci/test.faith.obs
+        faithpd -i unifrac/tests/data/crawford.biom -t unifrac/tests/data/crawford.tre -o ci/test.faith.obs
         tail -n +2 ci/test.faith.obs > ci/test.faith.header-removed.obs
         exp1=$($MD5 unifrac/tests/data/test.faith.exp | awk '{ print $1 }')
         obs1=$($MD5 ci/test.faith.header-removed.obs | awk '{ print $1 }')

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,13 +1,5 @@
 graft unifrac
 
-include sucpp/*.biom
-include sucpp/*.tre
-
-include sucpp/*.hpp
-include sucpp/*.cpp
-include sucpp/*.sh
-include sucpp/Makefile
-
 global-exclude *.pyc
 global-exclude *.pyo
 global-exclude .git

diff --git a/README.md b/README.md
@@ -48,12 +48,6 @@ This library can also be installed via a combination of `conda-forge` and `bioco
 conda create --name unifrac -c conda-forge -c bioconda unifrac
 ```
 
-Note: Only the CPU version of the binaries is currently available in conda. 
-The GPU version must either be [locally compiled using freely-available NVIDIA HPC SDK](docs/compile_gpu.README.md) 
-or obtained [from a github branch](https://github.com/sfiligoi/unifrac/blob/v0.20.2-docs/docs/install_gpu.README.md).
-
-Note: If you desire a fully optimized the binaries for your CPU, you can [compile them locally](docs/compile_cpu.README.md).
-
 ## Install (pip)
 
 ```
@@ -62,18 +56,17 @@ pip install unifrac
 
 ## Install (native)
 
-To install, first the binary needs to be compiled. This assumes that the HDF5 
-toolchain and libraries are available. More information about how to setup the
-stack can be found [here](https://support.hdfgroup.org/HDF5/Tutor/compile.html). 
+To install, first the cython wrappers must be compiled. It also needs
+the libssu library to be present.
 
-Assuming `h5c++` is in your path, the following should work:
+Assuming the compiler is in your path, the following should work:
 
     pip install -e . 
 
-**Note**: if you are using `conda` we recommend installing HDF5 using the
-`conda-forge` channel, for example:
+**Note**: if you are using `conda` we recommend installing the compiler and 
+libssu using the `biooconda` channel, for example:
 
-    conda install -c conda-forge hdf5
+    conda install -c conda-forge -c bioconda gxx_linux-64 unifrac-binaries
         
 # Examples of use
 

diff --git a/docs/compile_cpu.README.md b/docs/compile_cpu.README.md
diff --git a/docs/compile_gpu.README.md b/docs/compile_gpu.README.md