diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8d30050b..02f010cb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -54,12 +54,11 @@ jobs: conda config --add channels bioconda if [[ "$(uname -s)" == "Linux" ]]; then - conda install --yes -c conda-forge -c bioconda gxx_linux-64=7.5.0 + conda install --yes -c conda-forge -c bioconda gxx_linux-64 else - conda install --yes -c conda-forge -c bioconda clangxx_osx-64=10.0.0 + conda install --yes -c conda-forge -c bioconda clangxx_osx-64 fi - conda install --yes -c conda-forge -c bioconda cython "hdf5>=1.8.17" biom-format numpy "h5py>=2.7.0" "scikit-bio>=0.5.1" flake8 nose - conda install --yes -c conda-forge -c bioconda mkl-include lz4 hdf5-static + conda install --yes -c conda-forge -c bioconda cython "hdf5>=1.8.17" biom-format numpy "h5py>=2.7.0" "scikit-bio>=0.5.1" unifrac-binaries flake8 nose echo "$(uname -s)" if [[ "$(uname -s)" == "Linux" ]]; then @@ -67,43 +66,16 @@ jobs: x86_64-conda-linux-gnu-gcc -v x86_64-conda-linux-gnu-g++ -v else - conda install --yes liblapacke which clang clang -v fi which h5c++ - if [[ "$(uname -s)" == "Linux" ]]; - then - ./scripts/install_hpc_sdk.sh - source setup_nv_h5.sh - fi - pushd sucpp - make test - make main - make api - make capi_test - if [[ "$(uname -s)" == "Linux" ]]; - then - rm -f ~/.R/Makevars - conda install -c conda-forge r-base - # the r-base package has a broken dependency - ln -s $CONDA_PREFIX/lib/libreadline.so $CONDA_PREFIX/lib/libreadline.so.6 - R -e 'install.packages("Rcpp", repos="http://lib.stat.cmu.edu/R/CRAN/")' - make rapi_test - fi - popd pip install -e . - name: Tests shell: bash -l {0} run: | conda activate unifrac - pushd sucpp - export UNIFRAC_GPU_INFO=Y - ./test_su - ./test_api - ./test_ska - popd nosetests flake8 unifrac setup.py @@ -111,21 +83,28 @@ jobs: shell: bash -l {0} run: | conda activate unifrac - ./sucpp/ssu -i unifrac/tests/data/crawford.biom -t unifrac/tests/data/crawford.tre -o ci/test.dm -m unweighted + set -e + ssu -i unifrac/tests/data/crawford.biom -t unifrac/tests/data/crawford.tre -o ci/test.dm -m unweighted python -c "import skbio; dm = skbio.DistanceMatrix.read('ci/test.dm')" + pushd unifrac/tests + python -c "import unifrac; unifrac.unweighted_to_file('data/crawford.biom','data/crawford.tre','../../ci/test.dm.h5')" + python -c "import unifrac,skbio; dm_u=unifrac.unweighted('data/crawford.biom','data/crawford.tre'); dm = skbio.DistanceMatrix.read('../../ci/test.dm'); t=abs(dm_u.data-dm.data).max(); print(t); assert t < 0.1" + popd + python -c "import h5py,skbio; f_u=h5py.File('ci/test.dm.h5','r'); dm_u=skbio.stats.distance.DistanceMatrix(f_u['matrix'][:,:],f_u['order'][:])" + python -c "import h5py,skbio; dm = skbio.DistanceMatrix.read('ci/test.dm'); f_u=h5py.File('ci/test.dm.h5','r'); dm_u=skbio.stats.distance.DistanceMatrix(f_u['matrix'][:,:],f_u['order'][:]); t=abs(dm_u.data-dm.data).max(); print(t); assert t < 0.1" if [[ "$(uname -s)" == "Linux" ]]; then MD5=md5sum else MD5='md5 -r' fi - ./sucpp/ssu -i unifrac/tests/data/crawford.biom -t unifrac/tests/data/crawford.tre -o ci/test.dm.start0.stop3 -m unweighted --mode partial --start 0 --stop 3 - ./sucpp/ssu -i unifrac/tests/data/crawford.biom -t unifrac/tests/data/crawford.tre -o ci/test.dm.start3.stop5 -m unweighted --mode partial --start 3 --stop 5 - ./sucpp/ssu -i unifrac/tests/data/crawford.biom -t unifrac/tests/data/crawford.tre -o ci/test.dm.partial --mode merge-partial --partial-pattern "ci/test.dm.start*" + ssu -i unifrac/tests/data/crawford.biom -t unifrac/tests/data/crawford.tre -o ci/test.dm.start0.stop3 -m unweighted --mode partial --start 0 --stop 3 + ssu -i unifrac/tests/data/crawford.biom -t unifrac/tests/data/crawford.tre -o ci/test.dm.start3.stop5 -m unweighted --mode partial --start 3 --stop 5 + ssu -i unifrac/tests/data/crawford.biom -t unifrac/tests/data/crawford.tre -o ci/test.dm.partial --mode merge-partial --partial-pattern "ci/test.dm.start*" exp=$($MD5 ci/test.dm | awk '{ print $1 }') obs=$($MD5 ci/test.dm.partial | awk '{ print $1 }') python -c "assert '${obs}' == '${exp}'" - ./sucpp/faithpd -i unifrac/tests/data/crawford.biom -t unifrac/tests/data/crawford.tre -o ci/test.faith.obs + faithpd -i unifrac/tests/data/crawford.biom -t unifrac/tests/data/crawford.tre -o ci/test.faith.obs tail -n +2 ci/test.faith.obs > ci/test.faith.header-removed.obs exp1=$($MD5 unifrac/tests/data/test.faith.exp | awk '{ print $1 }') obs1=$($MD5 ci/test.faith.header-removed.obs | awk '{ print $1 }') diff --git a/MANIFEST.in b/MANIFEST.in index e49bfc07..7ae68283 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,13 +1,5 @@ graft unifrac -include sucpp/*.biom -include sucpp/*.tre - -include sucpp/*.hpp -include sucpp/*.cpp -include sucpp/*.sh -include sucpp/Makefile - global-exclude *.pyc global-exclude *.pyo global-exclude .git diff --git a/README.md b/README.md index 02811012..30cecdc2 100644 --- a/README.md +++ b/README.md @@ -48,12 +48,6 @@ This library can also be installed via a combination of `conda-forge` and `bioco conda create --name unifrac -c conda-forge -c bioconda unifrac ``` -Note: Only the CPU version of the binaries is currently available in conda. -The GPU version must either be [locally compiled using freely-available NVIDIA HPC SDK](docs/compile_gpu.README.md) -or obtained [from a github branch](https://github.com/sfiligoi/unifrac/blob/v0.20.2-docs/docs/install_gpu.README.md). - -Note: If you desire a fully optimized the binaries for your CPU, you can [compile them locally](docs/compile_cpu.README.md). - ## Install (pip) ``` @@ -62,18 +56,17 @@ pip install unifrac ## Install (native) -To install, first the binary needs to be compiled. This assumes that the HDF5 -toolchain and libraries are available. More information about how to setup the -stack can be found [here](https://support.hdfgroup.org/HDF5/Tutor/compile.html). +To install, first the cython wrappers must be compiled. It also needs +the libssu library to be present. -Assuming `h5c++` is in your path, the following should work: +Assuming the compiler is in your path, the following should work: pip install -e . -**Note**: if you are using `conda` we recommend installing HDF5 using the -`conda-forge` channel, for example: +**Note**: if you are using `conda` we recommend installing the compiler and +libssu using the `biooconda` channel, for example: - conda install -c conda-forge hdf5 + conda install -c conda-forge -c bioconda gxx_linux-64 unifrac-binaries # Examples of use diff --git a/docs/compile_cpu.README.md b/docs/compile_cpu.README.md deleted file mode 100644 index 79d38331..00000000 --- a/docs/compile_cpu.README.md +++ /dev/null @@ -1,96 +0,0 @@ -# Compiling an optimized version of UniFrac - -The binaries provided through conda should work on all major platforms. -However, you may be able to get slightly faster results by locally compiling your own binaries. - -Note: UniFrac supports only Linux and MacOS builds. -One can however run on Windows systems, too, using [WSL2](https://docs.microsoft.com/en-us/windows/wsl/install-win10). - - -## Anaconda - -UniFrac has several dependencies, which we assume come via [Anaconda](https://www.anaconda.com/products/individual). - -The instructions below has been tested with version 2021.05. - -In case you have never used Anaconda below, here are the installation instruction: - -### On Linux: -``` -wget https://repo.anaconda.com/archive/Anaconda3-2021.05-Linux-x86_64.sh -chmod a+x Anaconda3-2021.05-Linux-x86_64.sh -./Anaconda3-2021.05-Linux-x86_64.sh -#log out and back in -``` - -### On MacOS: -``` -curl -o Anaconda3-2021.05-MacOSX-x86_64.sh https://repo.anaconda.com/archive/Anaconda3-2021.05-MacOSX-x86_64.sh -chmod a+x Anaconda3-2021.05-MacOSX-x86_64.sh -./Anaconda3-2021.05-MacOSX-x86_64.sh -#log out and back in -``` - - -## Create a dedicated environment - -While it is possible to build your own UniFrac binaries in any Anaconda environment, we assume a dedicated one in this document. -We call it **unifrac-cpu**. - -Note: If you decide to change the used environment, you will have to make the appropriate changes to the commands below. - -To create our **unifrac-cpu** with all the needed dependencies, run: - -``` -# create and activate unifrac-gpu Anaconda environment -conda create --name unifrac-cpu -c conda-forge -c bioconda unifrac -conda activate unifrac-cpu -``` - -On linux, you will need the conda provided gcc compiler: -``` -# Linux only -conda install -c conda-forge -c bioconda gxx_linux-64=9.3 -``` - -On MacOs, you will need the conda provided clang compiler: -``` -# MacOS only -conda install -c conda-forge -c bioconda clangxx_osx-64=10.0.0 -# Also add library which is not imported automatically -conda install -c conda-forge -c bioconda liblapacke -``` - -Finally, on both Linux and MacOS, add two additinal compile-only dependencies: -``` -conda install -c conda-forge -c bioconda hdf5-static mkl-include -``` - -*Note:* On MacOS, the conda-provided clang is not compatible with XCode 12. - If you have it installed, you must either remove it or [downgrade to XCode 11](https://developer.apple.com/download/more/?=command%20line%20tools). - -## Compile UniFrac - -Assuming you are in the environment above, fetch the UniFrac source code and perform the build. -The binaries will be automatically put in the conda path. - -``` -# save original version of binaries -mkdir $CONDA_PREFIX/bin/org -mkdir $CONDA_PREFIX/lib/org - -mv $CONDA_PREFIX/bin/ssu $CONDA_PREFIX/bin/org/ -mv $CONDA_PREFIX/bin/faithpd $CONDA_PREFIX/bin/org/ -mv $CONDA_PREFIX/lib/libssu*.so $CONDA_PREFIX/lib/org/ - -(cd $CONDA_PREFIX/lib/python*/site-packages && mv unifrac unifrac.org) - -git clone https://github.com/biocore/unifrac.git -(cd unifrac/ && export USE_CYTHON=True && python setup.py build && python setup.py install) -``` - -And you are all done. -The UniFrac binary and libraries in the Anaconda environment are now the locally compiled ones. - -Note: If you do not want the cutting edge UniFrac from git, you can get a tarball of the released versions. These instructions have been tested with version [0.20.2](https://codeload.github.com/biocore/unifrac/tar.gz/0.20.2). - diff --git a/docs/compile_gpu.README.md b/docs/compile_gpu.README.md deleted file mode 100644 index e0690499..00000000 --- a/docs/compile_gpu.README.md +++ /dev/null @@ -1,103 +0,0 @@ -# Compiling a GPU-enabled version of UniFrac - -Note: The GPU-enabled version is currenlty only supported on Linux systems. -One can however run on Windows systems, too, using [CUDA-enabled WSL2](https://docs.nvidia.com/cuda/wsl-user-guide/index.html). - - -## System toools and libraries - -While we will install most of the tools we need, there these instrcutions assume you have installed the following packages -``` -wget make git glibc-devel -``` - -## Anaconda - -UniFrac has several dependencies, which we assume come via [Anaconda](https://www.anaconda.com/products/individual). - -The instructions below has been tested with version [2021.05](https://repo.anaconda.com/archive/Anaconda3-2021.05-Linux-x86_64.sh). - -In case you have never used Anaconda below, here are the installation instruction: - -``` -wget https://repo.anaconda.com/archive/Anaconda3-2021.05-Linux-x86_64.sh -chmod a+x Anaconda3-2021.05-Linux-x86_64.sh -./Anaconda3-2021.05-Linux-x86_64.sh -#log out and back in -``` - -## Create a dedicated environment - -While it is possible to build a GPU-enabled UniFrac in any Anaconda environment, we assume a dedicated one in this document. -We call it **unifrac-gpu**. - -Note: If you decide to change the used environment, you will have to make the appropriate changes to the scripts below. - -To create our **unifrac-gpu** with all the needed dependencies, run: - -``` -# create and activate unifrac-gpu Anaconda environment -conda create --name unifrac-gpu -c conda-forge -c bioconda unifrac -conda activate unifrac-gpu -conda install -c conda-forge -c bioconda gxx_linux-64=7.5.0 -conda install -c conda-forge -c bioconda hdf5-static mkl-include -``` - -## Installing the NVIDIA HPC SDK - -Currently, the only supported GPU-enabled compiler is the freely available [NVIDIA HPC SDK](https://developer.nvidia.com/hpc-sdk). - -Note that internally the NVIDIA HPC SDK relies on GCC, which makes it possible for the resulting objects to link with the libraries provided through Anaconda. - -We provide a helper script to install it, so you should fetch the code from github first, and then run that script. -You are encouraged to look into the script and improved as needed (e.g. updating the NVIDIA SDK version). -``` -git clone https://github.com/biocore/unifrac.git -# the following will install and configure the NVIDIA SDK -./unifrac/scripts/install_hpc_sdk.sh -``` - -For convenience, we also create a setup script that will be used to properly setup the environment anytime needed. - - -## Compiling the GPU-enabled UniFrac with the NVIDIA HPC SDK - -In order to compile UniFrac, you will need both the Anaconda dependencies, the NVIDIA HPC SDK and the UniFrac source code. - -``` -source setup_nv_h5.sh - -# save original version of binaries -mkdir $CONDA_PREFIX/bin/org -mkdir $CONDA_PREFIX/lib/org - -mv $CONDA_PREFIX/bin/ssu $CONDA_PREFIX/bin/org/ -mv $CONDA_PREFIX/bin/faithpd $CONDA_PREFIX/bin/org/ -mv $CONDA_PREFIX/lib/libssu*.so $CONDA_PREFIX/lib/org/ - -(cd $CONDA_PREFIX/lib/python*/site-packages && mv unifrac unifrac.org) - -(cd unifrac/ && export USE_CYTHON=True && python setup.py build && python setup.py install) -``` - -And you are all done. -The UniFrac binary and libraries in the Anaconda environment are now the GPU-enabled ones. - -*Note:* The produced binary has both GPU and CPU unifrac code paths. -The GPU path will be used if a GPU is detected, and use the CPU code path else. -You can forecefully disable the GPU with: -``` -export UNIFRAC_USE_GPU=N -``` - -If you want to know which path is being used, enabe info messages with: -``` -export UNIFRAC_GPU_INFO=Y -``` - -## Compiling an older version of Unifrac for GPUs - -If you do not want the cutting edge UniFrac from git, you will have to use version-specific instructions: -* [0.20.1 GPU compile instructions](https://github.com/sfiligoi/unifrac/blob/v0.20.1-docs/docs/compile_gpu.README.txt) -* [0.20.2 GPU compile instructions](https://github.com/sfiligoi/unifrac/blob/v0.20.2-docs/docs/compile_gpu.README.md) - diff --git a/scripts/install_hpc_sdk.sh b/scripts/install_hpc_sdk.sh deleted file mode 100755 index 53fadfe0..00000000 --- a/scripts/install_hpc_sdk.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/bin/bash - -# -# This is a helper script for installing the NVIDIA HPC SDK -# needed to compile a GPU-enabled version of unifrac. -# -# Note: The script currently assumes Linux_x86_64 platform. -# - -# Create GCC symbolic links -# since NVIDIA HPC SDK does not use the env variables -if [ "x${GCC}" == "x" ]; then - echo "ERROR: GCC not defined" - exit 1 -fi - -# usually $CONDA_PREFIX/bin/x86_64-conda_cos6-linux-gnu- -EXE_PREFIX=`echo "$GCC" |sed 's/gcc$//g'` - -echo "GCC pointing to ${EXE_PREFIX}gcc" -ls -l ${EXE_PREFIX}gcc - -mkdir conda_nv_bins -(cd conda_nv_bins && for f in \ - ar as c++ cc cpp g++ gcc ld nm ranlib strip; \ - do \ - ln -s ${EXE_PREFIX}${f} ${f}; \ - done ) - -export PATH=$PWD/conda_nv_bins:$PATH - -# Install the NVIDIA HPC SDK - -# This link may need to be updated, as new compiler versions are released -# Note: Verified that it works with v21.7 -wget -q https://developer.download.nvidia.com/hpc-sdk/21.7/nvhpc_2021_217_Linux_x86_64_cuda_multi.tar.gz -tar xpzf nvhpc_*.tar.gz -rm -f nvhpc_*.tar.gz - -# must patch the install scripts to find the right gcc -sed -i -e "s#PATH=/#PATH=$PWD/conda_nv_bins:/#g" \ - nvhpc_*/install_components/install -sed -i -e "s#PATH=/#PATH=$PWD/conda_nv_bins:/#g" \ - nvhpc_*/install_components/*/*/compilers/bin/makelocalrc -sed -i -e "s#PATH=/#PATH=$PWD/conda_nv_bins:/#g" \ - nvhpc_*/install_components/install_cuda - - -export NVHPC_INSTALL_DIR=$PWD/hpc_sdk -export NVHPC_SILENT=true - -(cd nvhpc_*; ./install) - -# create helper scripts -mkdir setup_scripts -cat > setup_scripts/setup_nv_hpc_bins.sh << EOF -PATH=$PWD/conda_nv_bins:`ls -d $PWD/hpc_sdk/*/202*/compilers/bin`:\$PATH - -# pgc++ does not define it, but gcc libraries expect it -# also remove the existing conda flags, which are not compatible -export CPPFLAGS=-D__GCC_ATOMIC_TEST_AND_SET_TRUEVAL=0 -export CXXFLAGS=\${CPPFLAGS} -export CFLAGS=\${CPPFLAGS} - -unset DEBUG_CPPFLAGS -unset DEBUG_CXXFLAGS -unset DEBUG_CFLAGS - -EOF - -# h5c++ patch -mkdir conda_h5 -cp $CONDA_PREFIX/bin/h5c++ conda_h5/ - -# This works on linux with gcc .. -sed -i \ - "s#x86_64-conda.*-linux-gnu-c++#pgc++ -I`ls -d $PWD/hpc_sdk/*/202*/compilers/include`#g" \ - conda_h5/h5c++ -sed -i \ - 's#H5BLD_CXXFLAGS=".*"#H5BLD_CXXFLAGS=" -fvisibility-inlines-hidden -std=c++17 -fPIC -O2 -I${includedir}"#g' \ - conda_h5/h5c++ -sed -i \ - 's#H5BLD_CPPFLAGS=".*"#H5BLD_CPPFLAGS=" -I${includedir} -DNDEBUG -D_FORTIFY_SOURCE=2 -O2"#g' \ - conda_h5/h5c++ -sed -i \ - 's#H5BLD_LDFLAGS=".*"#H5BLD_LDFLAGS=" -L${prefix}/x86_64-conda-linux-gnu/sysroot/usr/lib64/ -L${libdir} -Wl,-O2 -Wl,--sort-common -Wl,--as-needed -Wl,-z,relro -Wl,-z,now -Wl,--disable-new-dtags -Wl,-rpath,\\\\\\$ORIGIN/../x86_64-conda-linux-gnu/sysroot/usr/lib64/ -Wl,-rpath,\\\\\\$ORIGIN/../lib -Wl,-rpath,${prefix}/x86_64-conda-linux-gnu/sysroot/usr/lib64/ -Wl,-rpath,${libdir}"#g' \ - conda_h5/h5c++ - -cat > setup_nv_h5.sh << EOF -source $PWD/setup_scripts/setup_nv_hpc_bins.sh - -PATH=${PWD}/conda_h5:\$PATH -EOF - -echo "Setup script avaiabile in $PWD/setup_nv_h5.sh" diff --git a/setup.py b/setup.py index 4bc532b9..7c5d149f 100644 --- a/setup.py +++ b/setup.py @@ -15,10 +15,6 @@ import sys -SUCPP = os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'sucpp/') - - PREFIX = os.environ.get('PREFIX', "") base = ["cython >= 0.26", "biom-format", "numpy", "h5py >= 2.7.0", @@ -36,22 +32,23 @@ def compile_ssu(): """Clean and compile the SSU binary""" # clean the target - subprocess.call(['make', 'clean'], cwd=SUCPP) - - cmd = ['make', 'test'] - ret = subprocess.call(cmd, cwd=SUCPP) + cmd = ["rm", "-f", "unifrac/task_parameters.hpp", "unifrac/api.hpp"] + ret = subprocess.call(cmd) if ret != 0: - raise Exception('Error compiling ssu!') + raise Exception('Error removing temp unifrac files!') - cmd = ['make', 'main'] - ret = subprocess.call(cmd, cwd=SUCPP) + # link to files from conda + cmd = ["ln", "-s", os.environ.get('CONDA_PREFIX') + + "/include/unifrac/task_parameters.hpp", "unifrac/"] + ret = subprocess.call(cmd) if ret != 0: - raise Exception('Error compiling ssu!') + raise Exception('Error removing linking unifrac files!') - cmd = ['make', 'api'] - ret = subprocess.call(cmd, cwd=SUCPP) + cmd = ["ln", "-s", os.environ.get('CONDA_PREFIX') + + "/include/unifrac/api.hpp", "unifrac/"] + ret = subprocess.call(cmd) if ret != 0: - raise Exception('Error compiling ssu!') + raise Exception('Error removing linking unifrac files!') class build_ext(build_ext_orig): @@ -63,13 +60,11 @@ def run(self): def run_compile_ssu(self): self.execute(compile_ssu, [], 'Compiling SSU') - if PREFIX: - self.copy_file(os.path.join(SUCPP, 'libssu.so'), - os.path.join(PREFIX, 'lib/')) if sys.platform == "darwin": - LINK_ARGS = ['-Wl,sucpp/libssu.so'] + LINK_ARGS = ['-Wl,' + os.environ.get('CONDA_PREFIX') + + '/lib/libssu.so'] else: LINK_ARGS = [] @@ -81,12 +76,10 @@ def run_compile_ssu(self): USE_CYTHON = os.environ.get('USE_CYTHON', True) ext = '.pyx' if USE_CYTHON else '.cpp' extensions = [Extension("unifrac._api", - sources=["unifrac/_api" + ext, - "sucpp/api.cpp"], + sources=["unifrac/_api" + ext], language="c++", extra_link_args=LINK_ARGS, include_dirs=([np.get_include()] + - ['sucpp/'] + CONDA_INCLUDES), libraries=['ssu'])] @@ -99,7 +92,7 @@ def run_compile_ssu(self): setup( name="unifrac", - version="0.20.3", + version="1.0.0", packages=find_packages(), author="Daniel McDonald", license='BSD-3-Clause', diff --git a/sucpp/Makefile b/sucpp/Makefile deleted file mode 100644 index 2c429169..00000000 --- a/sucpp/Makefile +++ /dev/null @@ -1,143 +0,0 @@ -CXX := h5c++ - -PLATFORM := $(shell uname -s) -COMPILER := $(shell ($(CXX) -v 2>&1) | tr A-Z a-z ) - -ifdef DEBUG - ifneq (,$(findstring pgi,$(COMPILER))) - OPT = -g - else - OPT = -O0 -DDEBUG=1 --debug -g -ggdb - endif -else - ifneq (,$(findstring pgi,$(COMPILER))) - OPT = -fast - else - ifneq (,$(findstring gcc,$(COMPILER))) - OPT = -O4 - TGTFLAGS = -fwhole-program - else - OPT = -O3 - endif - endif -endif - -ifeq ($(PREFIX),) - PREFIX := $(CONDA_PREFIX) -endif - -ifeq ($(PLATFORM),Darwin) - AVX2 := $(shell sysctl -a | grep -c AVX2) - LDDFLAGS = -dynamiclib -install_name @rpath/libssu.so -else - AVX2 := $(shell grep "^flags" /proc/cpuinfo | head -n 1 | grep -c avx2) - LDDFLAGS = -shared -endif - -EXEFLAGS = - -ifneq (,$(findstring pgi,$(COMPILER))) - MPFLAG = -mp -else - MPFLAG = -fopenmp -endif - -LDDFLAGS += $(MPFLAG) -CPPFLAGS += $(MPFLAG) - -UNIFRAC_FILES = unifrac_internal.o unifrac_cmp_cpu.o - -ifndef NOGPU - ifneq (,$(findstring pgi,$(COMPILER))) - CPPFLAGS += -DUNIFRAC_ENABLE_ACC=1 - UNIFRAC_FILES += unifrac_cmp_acc.o - ACCCPPFLAGS += -acc - ifeq ($(PERFORMING_CONDA_BUILD),True) - ACCCPPFLAGS += -ta=tesla:ccall - else - ACCCPPFLAGS += -ta=tesla - endif - # optional info - ACCCPPFLAGS += -Minfo=accel - LDDFLAGS += -shlib -acc -Bstatic_pgi - EXEFLAGS += -acc -Bstatic_pgi - endif -endif - -ifneq (,$(findstring pgi,$(COMPILER))) - ifeq ($(PERFORMING_CONDA_BUILD),True) - CPPFLAGS += -tp=px - endif -else - ifeq ($(PERFORMING_CONDA_BUILD),True) - CPPFLAGS += -mtune=generic - else - CPPFLAGS += -mfma -march=native - endif -endif - -ifeq (,$(findstring pgi,$(COMPILER))) - # basically, not gcc - CPPFLAGS += -Wextra -Wno-unused-parameter -endif - -ifeq ($(PLATFORM),Darwin) - BLASLIB=-llapacke -lcblas -else - BLASLIB=-lcblas -endif - - -CPPFLAGS += -Wall -std=c++11 -pedantic -I. $(OPT) -fPIC -L$(CONDA_PREFIX)/lib - -test: tree.o test_su.cpp biom.o unifrac.o skbio_alt.o api.o $(UNIFRAC_FILES) - $(CXX) $(CPPFLAGS) $(EXEFLAGS) test_su.cpp -o test_su tree.o biom.o $(UNIFRAC_FILES) unifrac.o skbio_alt.o api.o -llz4 $(BLASLIB) -lpthread - $(CXX) $(CPPFLAGS) $(EXEFLAGS) test_ska.cpp -o test_ska skbio_alt.o tree.o biom.o $(UNIFRAC_FILES) unifrac.o api.o -llz4 $(BLASLIB) -lpthread - $(CXX) $(CPPFLAGS) $(EXEFLAGS) test_api.cpp -o test_api tree.o biom.o $(UNIFRAC_FILES) unifrac.o skbio_alt.o api.o -llz4 $(BLASLIB) -lpthread - -main: tree.o biom.o unifrac.o cmd.o skbio_alt.o api.o $(UNIFRAC_FILES) - $(CXX) $(CPPFLAGS) $(EXEFLAGS) su.cpp -o ssu tree.o biom.o $(UNIFRAC_FILES) unifrac.o cmd.o skbio_alt.o api.o -lhdf5_cpp -llz4 $(BLASLIB) -lpthread - $(CXX) $(CPPFLAGS) $(EXEFLAGS) faithpd.cpp -o faithpd tree.o biom.o $(UNIFRAC_FILES) unifrac.o cmd.o skbio_alt.o api.o -lhdf5_cpp -llz4 $(BLASLIB) -lpthread - cp ssu ${PREFIX}/bin/ - cp faithpd ${PREFIX}/bin/ - -rapi_test: main - mkdir -p ~/.R - if [ -e ~/.R/Makevars ] ; \ - then \ - echo "WARNING: OVERWRITING ~/.R/Makevars" ; \ - echo "The original Makevars file has been copied to ~/.R/Makevars" ;\ - cp ~/.R/Makevars Makevars-original ; \ - fi; - echo CXX1X=h5c++ > ~/.R/Makevars - echo CXX=h5c++ >> ~/.R/Makevars - echo CC=h5c++ >> ~/.R/Makevars - echo LDFLAGS= -llz4 $(BLASLIB) >> ~/.R/Makevars - rm -f *.o - Rscript R_interface/rapi_test.R - rm -f *.o - -api: tree.o biom.o unifrac.o cmd.o skbio_alt.o api.o $(UNIFRAC_FILES) - $(CXX) $(LDDFLAGS) -o libssu.so tree.o biom.o $(UNIFRAC_FILES) unifrac.o cmd.o skbio_alt.o api.o -lc -lhdf5_cpp -llz4 $(BLASLIB) -L$(PREFIX)/lib - cp libssu.so ${PREFIX}/lib/ - -capi_test: api - gcc -std=c99 capi_test.c -lssu -L${PREFIX}/lib -Wl,-rpath,${PREFIX}/lib -o capi_test - export LD_LIBRARY_PATH="${PREFIX}/lib":"./capi_test" - -api.o: api.cpp api.hpp unifrac.hpp skbio_alt.hpp biom.hpp tree.hpp - $(CXX) $(CPPFLAGS) api.cpp -c -o api.o -fPIC - -unifrac_cmp_cpu.o: unifrac_cmp.cpp unifrac_cmp.hpp unifrac_internal.hpp unifrac.hpp unifrac_task.cpp unifrac_task.hpp biom_interface.hpp tree.hpp - $(CXX) $(CPPFLAGS) -c $< -o $@ - -unifrac_cmp_acc.o: unifrac_cmp.cpp unifrac_cmp.hpp unifrac_internal.hpp unifrac.hpp unifrac_task.cpp unifrac_task.hpp biom_interface.hpp tree.hpp - $(CXX) $(CPPFLAGS) $(ACCCPPFLAGS) -c $< -o $@ - - -%.o: %.cpp %.hpp - $(CXX) $(CPPFLAGS) -c $< -o $@ - -clean: - -rm -f *.o ssu - diff --git a/sucpp/R_interface/README.md b/sucpp/R_interface/README.md deleted file mode 100644 index 3fb7e2d5..00000000 --- a/sucpp/R_interface/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# R interface for Strided State Unifrac - -This provides an R interface for Unweighted Unifrac. This interface works using -R's Rcpp library. To load this, in R use `library(Rcpp)` and -`sourceCpp("su_R.cpp")`. The Unifrac method takes in three arguments: a file -path to an HDF5 formatted BIOM table, a filepath to a newick formatted tree -file, and the number of threads to be used It is expected that the observations -described in the BIOM table correspond to a subset of the tips of the input -tree. The method returns a list containing an `int` `n_samples`, denoting the -number of samples in the table, a `boolean` `is_upper_triangle`, denoting -whether Unifrac generated a square matrix and if it has returned the upper -triangle , an `int` `cf_size`, denoting the size of the condensed form of the -matrix, and `c_form`, an array representation of the condensed form of the -matrix, obtained by taking the upper triangle. - -```R -> library(Rcpp) -> sourceCpp("su_R.cpp") -> table = "../test.biom" -> tree = "../test.tre" -> nthreads = 2 -> unif = unifrac(table, tree, nthreads) -> unif -$n_samples -[1] 6 - -$is_sqaure -[1] TRUE - -$cf_size -[1] 15 - -$c_form - [1] 0.2000000 0.5714286 0.6000000 0.5000000 0.2000000 0.4285714 0.6666667 - [8] 0.6000000 0.3333333 0.7142857 0.8571429 0.4285714 0.3333333 0.4000000 -[15] 0.6000000 - -``` - diff --git a/sucpp/R_interface/rapi_test.R b/sucpp/R_interface/rapi_test.R deleted file mode 100644 index c560ba90..00000000 --- a/sucpp/R_interface/rapi_test.R +++ /dev/null @@ -1,52 +0,0 @@ -library(Rcpp) - -equals <- function(x, y, msg){ - if (x!=y) - stop(msg) - - -} -aboutEquals <- function(x, y, msg){ - if((x-y)>0.005) - stop(msg) - - -} -source = "su_R.cpp" -sourceCpp(source) -table = "test.biom" -tree = "test.tre" -nthreads = 1 - -print('Testing UniFrac..') -unif = unifrac(table, tree, nthreads) - -exp = c(0.2000000, 0.5714286, 0.6000000, 0.5000000, 0.2000000, - 0.4285714, 0.6666667, 0.6000000, 0.3333333, 0.7142857, - 0.8571429, 0.4285714, 0.3333333, 0.4000000, 0.6000000) - -equals(unif["n_samples"][[1]], 6, "n_samples != 6") -equals(unif["cf_size"][[1]], 15, "cf_size != 15") -equals(unif["is_upper_triangle"][[1]], TRUE, "is_upper_triagnle != TRUE") - - -for ( i in 1:15){ - aboutEquals(unif["c_form"][[1]][i], exp[i], "Output not as expected") -} -print('Success.') - -print('Testing Faith PD..') - -faith = faith_pd(table, tree) - -exp = c(4, 5, 6, 3, 2, 5) - -equals(faith["n_samples"][[1]], 6, "n_samples != 6") -for ( i in 1:6){ - aboutEquals(faith["faith_pd"][[1]][i], exp[i], "Output not as expected") -} - -print('Success.') - -print('All tests pass') - diff --git a/sucpp/R_interface/test.biom b/sucpp/R_interface/test.biom deleted file mode 100644 index b3c019bf..00000000 Binary files a/sucpp/R_interface/test.biom and /dev/null differ diff --git a/sucpp/R_interface/test.tre b/sucpp/R_interface/test.tre deleted file mode 100644 index 1ba14a5b..00000000 --- a/sucpp/R_interface/test.tre +++ /dev/null @@ -1 +0,0 @@ -(GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1):1,(GG_OTU_5:1,GG_OTU_4:1):1); diff --git a/sucpp/affinity.hpp b/sucpp/affinity.hpp deleted file mode 100644 index 55d62c98..00000000 --- a/sucpp/affinity.hpp +++ /dev/null @@ -1,125 +0,0 @@ -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include - -#ifdef __LINUX__ -#include -#endif - - -#ifdef __APPLE__ -#include -#include -#include -#include -#include - -// OSX code adapted from -// http://yyshen.github.io/2015/01/18/binding_threads_to_cores_osx.html -// these macros and methods don't exist on OSX - -#define SYSCTL_CORE_COUNT "machdep.cpu.core_count" - -typedef struct cpu_set { - uint32_t count; -} cpu_set_t; - -static inline void -CPU_ZERO(cpu_set_t *cs) { cs->count = 0; } - -static inline void -CPU_SET(int num, cpu_set_t *cs) { cs->count |= (1 << num); } - -static inline int -CPU_ISSET(int num, cpu_set_t *cs) { return (cs->count & (1 << num)); } - -static inline int -CPU_COUNT(cpu_set_t *cs) { return __builtin_popcount(cs->count); } - -#define CPU_SETSIZE 32 - -static int sched_getaffinity(pid_t pid, size_t cpu_size, cpu_set_t *cpu_set) -{ - int32_t core_count = 0; - size_t len = sizeof(core_count); - int ret = sysctlbyname(SYSCTL_CORE_COUNT, &core_count, &len, 0, 0); - if (ret) { - return -1; - } - cpu_set->count = 0; - for (int i = 0; i < core_count; i++) { - cpu_set->count |= (1 << i); - } - - return 0; -} - -static int pthread_setaffinity_np(pthread_t thread, size_t cpu_size, - cpu_set_t *cpu_set) -{ - thread_port_t mach_thread; - int core = 0; - - for (core = 0; core < 8 * cpu_size; core++) { - if (CPU_ISSET(core, cpu_set)) break; - } - thread_affinity_policy_data_t policy = { core }; - mach_thread = pthread_mach_thread_np(thread); - thread_policy_set(mach_thread, THREAD_AFFINITY_POLICY, - (thread_policy_t)&policy, 1); - return 0; -} - -#endif - -#define handle_error_en(en, msg) \ - do { errno = en; perror(msg); exit(EXIT_FAILURE); } while (0) - -static int bind_to_core(int core) { - /* bind the calling thread to the requested core - * - * The use of this method is for better NUMA utilization. The - * default NUMA policy is local, where memory is allocated on the NUMA node - * relative to the core if possible. The intention with this method is to - * bind to a core first, and then allocate memory. A beneficial side effect - * is that threads should not hop between cores either. - * - * This method is cgroup safe. - */ - // https://stackoverflow.com/a/11583550/19741 - // http://blog.saliya.org/2015/07/get-and-set-process-affinity-in-c.html - pthread_t thread = pthread_self(); - pid_t pid = getpid(); - - cpu_set_t current_set, new_set; - int j, ret; - - CPU_ZERO(¤t_set); - CPU_ZERO(&new_set); - - ret = sched_getaffinity(pid, sizeof(current_set), ¤t_set); - - // find which core in our cpu_set corresponds to the callers - // request - int target = -1; - for(j = 0; j < CPU_SETSIZE; j++) { - if(CPU_ISSET(j, ¤t_set)) { - target++; - } - if(target == core) - break; - } - - if(target != core) { - fprintf(stderr, "Unable to bind this thread to core %d. Are sufficient processors available?", thread); - return -1; - } - - CPU_SET(j, &new_set); - int serr = pthread_setaffinity_np(thread, sizeof(new_set), &new_set); - return serr; -} diff --git a/sucpp/api.cpp b/sucpp/api.cpp deleted file mode 100644 index 25f70ad5..00000000 --- a/sucpp/api.cpp +++ /dev/null @@ -1,1505 +0,0 @@ -#include "api.hpp" -#include "biom.hpp" -#include "tree.hpp" -#include "unifrac.hpp" -#include "skbio_alt.hpp" -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#define MMAP_FD_MASK 0x0fff -#define MMAP_FLAG 0x1000 - -/* O_NOATIME is defined at fcntl.h when supported */ -#ifndef O_NOATIME -#define O_NOATIME 0 -#endif - - -#define CHECK_FILE(filename, err) if(!is_file_exists(filename)) { \ - return err; \ - } - -#define SET_METHOD(requested_method, err) Method method; \ - if(std::strcmp(requested_method, "unweighted") == 0) \ - method = unweighted; \ - else if(std::strcmp(requested_method, "weighted_normalized") == 0) \ - method = weighted_normalized; \ - else if(std::strcmp(requested_method, "weighted_unnormalized") == 0) \ - method = weighted_unnormalized; \ - else if(std::strcmp(requested_method, "generalized") == 0) \ - method = generalized; \ - else if(std::strcmp(requested_method, "unweighted_fp32") == 0) \ - method = unweighted_fp32; \ - else if(std::strcmp(requested_method, "weighted_normalized_fp32") == 0) \ - method = weighted_normalized_fp32; \ - else if(std::strcmp(requested_method, "weighted_unnormalized_fp32") == 0) \ - method = weighted_unnormalized_fp32; \ - else if(std::strcmp(requested_method, "generalized_fp32") == 0) \ - method = generalized_fp32; \ - else { \ - return err; \ - } - -#define PARSE_SYNC_TREE_TABLE(tree_filename, table_filename) std::ifstream ifs(tree_filename); \ - std::string content = std::string(std::istreambuf_iterator(ifs), \ - std::istreambuf_iterator()); \ - su::BPTree tree = su::BPTree(content); \ - su::biom table = su::biom(biom_filename); \ - if(table.n_samples <= 0 | table.n_obs <= 0) { \ - return table_empty; \ - } \ - std::string bad_id = su::test_table_ids_are_subset_of_tree(table, tree); \ - if(bad_id != "") { \ - return table_and_tree_do_not_overlap; \ - } \ - std::unordered_set to_keep(table.obs_ids.begin(), \ - table.obs_ids.end()); \ - su::BPTree tree_sheared = tree.shear(to_keep).collapse(); - - -using namespace su; -using namespace std; - -// https://stackoverflow.com/a/19841704/19741 -bool is_file_exists(const char *fileName) { - std::ifstream infile(fileName); - return infile.good(); -} - - -void destroy_stripes(vector &dm_stripes, vector &dm_stripes_total, unsigned int n_samples, - unsigned int stripe_start, unsigned int stripe_stop) { - unsigned int n_rotations = (n_samples + 1) / 2; - - if(stripe_stop == 0) { - for(unsigned int i = 0; i < n_rotations; i++) { - free(dm_stripes[i]); - if(dm_stripes_total[i] != NULL) - free(dm_stripes_total[i]); - } - } else { - // if a stripe_stop is specified, and if we're in the stripe window, do not free - // dm_stripes. this is done as the pointers in dm_stripes are assigned to the partial_mat_t - // and subsequently freed in destroy_partial_mat. but, we do need to free dm_stripes_total - // if appropriate - for(unsigned int i = stripe_start; i < stripe_stop; i++) { - if(dm_stripes_total[i] != NULL) - free(dm_stripes_total[i]); - } - } -} - - -void initialize_mat(mat_t* &result, biom &table, bool is_upper_triangle) { - result = (mat_t*)malloc(sizeof(mat)); - result->n_samples = table.n_samples; - - result->cf_size = su::comb_2(table.n_samples); - result->is_upper_triangle = is_upper_triangle; - result->sample_ids = (char**)malloc(sizeof(char*) * result->n_samples); - result->condensed_form = (double*)malloc(sizeof(double) * su::comb_2(table.n_samples)); - - for(unsigned int i = 0; i < result->n_samples; i++) { - size_t len = table.sample_ids[i].length(); - result->sample_ids[i] = (char*)malloc(sizeof(char) * len + 1); - table.sample_ids[i].copy(result->sample_ids[i], len); - result->sample_ids[i][len] = '\0'; - } -} - -void initialize_results_vec(r_vec* &result, biom& table){ - // Stores results for Faith PD - result = (r_vec*)malloc(sizeof(results_vec)); - result->n_samples = table.n_samples; - result->values = (double*)malloc(sizeof(double) * result->n_samples); - result->sample_ids = (char**)malloc(sizeof(char*) * result->n_samples); - - for(unsigned int i = 0; i < result->n_samples; i++) { - size_t len = table.sample_ids[i].length(); - result->sample_ids[i] = (char*)malloc(sizeof(char) * len + 1); - table.sample_ids[i].copy(result->sample_ids[i], len); - result->sample_ids[i][len] = '\0'; - result->values[i] = 0; - } - -} - -void initialize_mat_no_biom(mat_t* &result, char** sample_ids, unsigned int n_samples, bool is_upper_triangle) { - result = (mat_t*)malloc(sizeof(mat)); - result->n_samples = n_samples; - - result->cf_size = su::comb_2(n_samples); - result->is_upper_triangle = is_upper_triangle; - result->sample_ids = (char**)malloc(sizeof(char*) * result->n_samples); - result->condensed_form = (double*)malloc(sizeof(double) * su::comb_2(n_samples)); - - for(unsigned int i = 0; i < n_samples; i++) { - result->sample_ids[i] = strdup(sample_ids[i]); - } -} - -template -void initialize_mat_full_no_biom_T(TMat* &result, const char* const * sample_ids, unsigned int n_samples, - const char *mmap_dir /* if NULL or "", use malloc */) { - result = (TMat*)malloc(sizeof(mat)); - result->n_samples = n_samples; - - uint64_t n_samples_64 = result->n_samples; // force 64bit to avoit overflow problems - - result->sample_ids = (char**)malloc(sizeof(char*) * n_samples_64); - result->flags=0; - - if (mmap_dir!=NULL) { - if (mmap_dir[0]==0) mmap_dir = NULL; // easier to have a simple test going on - } - - uint64_t msize = sizeof(TReal) * n_samples_64 * n_samples_64; - if (mmap_dir==NULL) { - result->matrix = (TReal*)malloc(msize); - } else { - std::string mmap_template(mmap_dir); - mmap_template+="/su_mmap_XXXXXX"; - // note: mkostemp will update mmap_template in place - int fd=mkostemp((char *) mmap_template.c_str(), O_NOATIME ); - if (fd<0) { - result->matrix = NULL; - // leave error handling to the caller - } else { - // remove the file name, so it will be destroyed on close - unlink(mmap_template.c_str()); - // make it big enough - ftruncate(fd,msize); - // now can be used, just like a malloc-ed buffer - result->matrix = (TReal*)mmap(NULL, msize,PROT_READ|PROT_WRITE, MAP_SHARED|MAP_NORESERVE, fd, 0); - result->flags=(uint32_t(fd) & MMAP_FD_MASK) | MMAP_FLAG; - } - } - - for(unsigned int i = 0; i < n_samples; i++) { - result->sample_ids[i] = strdup(sample_ids[i]); - } -} - -void initialize_partial_mat(partial_mat_t* &result, biom &table, std::vector &dm_stripes, - unsigned int stripe_start, unsigned int stripe_stop, bool is_upper_triangle) { - result = (partial_mat_t*)malloc(sizeof(partial_mat)); - result->n_samples = table.n_samples; - - result->sample_ids = (char**)malloc(sizeof(char*) * result->n_samples); - for(unsigned int i = 0; i < result->n_samples; i++) { - size_t len = table.sample_ids[i].length(); - result->sample_ids[i] = (char*)malloc(sizeof(char) * len + 1); - table.sample_ids[i].copy(result->sample_ids[i], len); - result->sample_ids[i][len] = '\0'; - } - - result->stripes = (double**)malloc(sizeof(double*) * (stripe_stop - stripe_start)); - result->stripe_start = stripe_start; - result->stripe_stop = stripe_stop; - result->is_upper_triangle = is_upper_triangle; - result->stripe_total = dm_stripes.size(); - - for(unsigned int i = stripe_start; i < stripe_stop; i++) { - result->stripes[i - stripe_start] = dm_stripes[i]; - } -} - -void destroy_results_vec(r_vec** result) { - // for Faith PD - for(unsigned int i = 0; i < (*result)->n_samples; i++) { - free((*result)->sample_ids[i]); - }; - free((*result)->sample_ids); - free((*result)->values); - free(*result); -} - -void destroy_mat(mat_t** result) { - for(unsigned int i = 0; i < (*result)->n_samples; i++) { - free((*result)->sample_ids[i]); - }; - free((*result)->sample_ids); - if (((*result)->condensed_form)!=NULL) { - free((*result)->condensed_form); - } - free(*result); -} - -template -inline void destroy_mat_full_T(TMat** result) { - for(uint32_t i = 0; i < (*result)->n_samples; i++) { - free((*result)->sample_ids[i]); - }; - free((*result)->sample_ids); - if (((*result)->matrix)!=NULL) { - if (((*result)->flags & MMAP_FLAG) == 0) { - free((*result)->matrix); - } else { - uint64_t n_samples = (*result)->n_samples; - munmap((*result)->matrix, sizeof(TReal)*n_samples*n_samples); - - int fd = (*result)->flags & MMAP_FD_MASK; - close(fd); - } - (*result)->matrix=NULL; - } - free(*result); -} - - -void destroy_mat_full_fp64(mat_full_fp64_t** result) { - destroy_mat_full_T(result); -} - -void destroy_mat_full_fp32(mat_full_fp32_t** result) { - destroy_mat_full_T(result); -} - -void destroy_partial_mat(partial_mat_t** result) { - for(unsigned int i = 0; i < (*result)->n_samples; i++) { - if((*result)->sample_ids[i] != NULL) - free((*result)->sample_ids[i]); - }; - if((*result)->sample_ids != NULL) - free((*result)->sample_ids); - - unsigned int n_stripes = (*result)->stripe_stop - (*result)->stripe_start; - for(unsigned int i = 0; i < n_stripes; i++) - if((*result)->stripes[i] != NULL) - free((*result)->stripes[i]); - if((*result)->stripes != NULL) - free((*result)->stripes); - - free(*result); -} - -void destroy_partial_dyn_mat(partial_dyn_mat_t** result) { - for(unsigned int i = 0; i < (*result)->n_samples; i++) { - if((*result)->sample_ids[i] != NULL) - free((*result)->sample_ids[i]); - }; - if((*result)->sample_ids != NULL) - free((*result)->sample_ids); - - unsigned int n_stripes = (*result)->stripe_stop - (*result)->stripe_start; - for(unsigned int i = 0; i < n_stripes; i++) - if((*result)->stripes[i] != NULL) - free((*result)->stripes[i]); - if((*result)->stripes != NULL) - free((*result)->stripes); - if((*result)->offsets != NULL) - free((*result)->offsets); - if((*result)->filename != NULL) - free((*result)->filename); - - free(*result); -} - - -void set_tasks(std::vector &tasks, - double alpha, - unsigned int n_samples, - unsigned int stripe_start, - unsigned int stripe_stop, - bool bypass_tips, - unsigned int nthreads) { - - // compute from start to the max possible stripe if stop doesn't make sense - if(stripe_stop <= stripe_start) - stripe_stop = (n_samples + 1) / 2; - - /* chunking strategy is to balance as much as possible. eg if there are 15 stripes - * and 4 threads, our goal is to assign 4 stripes to 3 threads, and 3 stripes to one thread. - * - * we use the remaining the chunksize for bins which cannot be full maximally - */ - unsigned int fullchunk = ((stripe_stop - stripe_start) + nthreads - 1) / nthreads; // this computes the ceiling - unsigned int smallchunk = (stripe_stop - stripe_start) / nthreads; - - unsigned int n_fullbins = (stripe_stop - stripe_start) % nthreads; - if(n_fullbins == 0) - n_fullbins = nthreads; - - unsigned int start = stripe_start; - - for(unsigned int tid = 0; tid < nthreads; tid++) { - tasks[tid].tid = tid; - tasks[tid].start = start; // stripe start - tasks[tid].bypass_tips = bypass_tips; - - if(tid < n_fullbins) { - tasks[tid].stop = start + fullchunk; // stripe end - start = start + fullchunk; - } else { - tasks[tid].stop = start + smallchunk; // stripe end - start = start + smallchunk; - } - - tasks[tid].n_samples = n_samples; - tasks[tid].g_unifrac_alpha = alpha; - } -} - -compute_status partial(const char* biom_filename, const char* tree_filename, - const char* unifrac_method, bool variance_adjust, double alpha, bool bypass_tips, - unsigned int nthreads, unsigned int stripe_start, unsigned int stripe_stop, - partial_mat_t** result) { - - CHECK_FILE(biom_filename, table_missing) - CHECK_FILE(tree_filename, tree_missing) - SET_METHOD(unifrac_method, unknown_method) - PARSE_SYNC_TREE_TABLE(tree_filename, table_filename) - - // we resize to the largest number of possible stripes even if only computing - // partial, however we do not allocate arrays for non-computed stripes so - // there is a little memory waste here but should be on the order of - // 8 bytes * N samples per vector. - std::vector dm_stripes((table.n_samples + 1) / 2); - std::vector dm_stripes_total((table.n_samples + 1) / 2); - - if(nthreads > dm_stripes.size()) { - fprintf(stderr, "More threads were requested than stripes. Using %d threads.\n", dm_stripes.size()); - nthreads = dm_stripes.size(); - } - - std::vector tasks(nthreads); - std::vector threads(nthreads); - - if(((table.n_samples + 1) / 2) < stripe_stop) { - fprintf(stderr, "Stopping stripe is out-of-bounds, max %d\n", (table.n_samples + 1) / 2); - exit(EXIT_FAILURE); - } - - set_tasks(tasks, alpha, table.n_samples, stripe_start, stripe_stop, bypass_tips, nthreads); - su::process_stripes(table, tree_sheared, method, variance_adjust, dm_stripes, dm_stripes_total, threads, tasks); - - initialize_partial_mat(*result, table, dm_stripes, stripe_start, stripe_stop, true); // true -> is_upper_triangle - destroy_stripes(dm_stripes, dm_stripes_total, table.n_samples, stripe_start, stripe_stop); - - return okay; -} - -compute_status faith_pd_one_off(const char* biom_filename, const char* tree_filename, - r_vec** result){ - CHECK_FILE(biom_filename, table_missing) - CHECK_FILE(tree_filename, tree_missing) - PARSE_SYNC_TREE_TABLE(tree_filename, table_filename) - - initialize_results_vec(*result, table); - - // compute faithpd - su::faith_pd(table, tree_sheared, std::ref((*result)->values)); - - return okay; -} - -compute_status one_off(const char* biom_filename, const char* tree_filename, - const char* unifrac_method, bool variance_adjust, double alpha, - bool bypass_tips, unsigned int nthreads, mat_t** result) { - - CHECK_FILE(biom_filename, table_missing) - CHECK_FILE(tree_filename, tree_missing) - SET_METHOD(unifrac_method, unknown_method) - PARSE_SYNC_TREE_TABLE(tree_filename, table_filename) - - const unsigned int stripe_stop = (table.n_samples + 1) / 2; - std::vector dm_stripes(stripe_stop); - std::vector dm_stripes_total(stripe_stop); - - if(nthreads > dm_stripes.size()) { - fprintf(stderr, "More threads were requested than stripes. Using %d threads.\n", dm_stripes.size()); - nthreads = dm_stripes.size(); - } - - std::vector tasks(nthreads); - std::vector threads(nthreads); - - set_tasks(tasks, alpha, table.n_samples, 0, stripe_stop, bypass_tips, nthreads); - su::process_stripes(table, tree_sheared, method, variance_adjust, dm_stripes, dm_stripes_total, threads, tasks); - - initialize_mat(*result, table, true); // true -> is_upper_triangle - for(unsigned int tid = 0; tid < threads.size(); tid++) { - threads[tid] = std::thread(su::stripes_to_condensed_form, - std::ref(dm_stripes), - table.n_samples, - std::ref((*result)->condensed_form), - tasks[tid].start, - tasks[tid].stop); - } - for(unsigned int tid = 0; tid < threads.size(); tid++) { - threads[tid].join(); - } - - destroy_stripes(dm_stripes, dm_stripes_total, table.n_samples, 0, 0); - - return okay; -} - -// TMat mat_full_fp32_t -template -compute_status one_off_matrix_T(const char* biom_filename, const char* tree_filename, - const char* unifrac_method, bool variance_adjust, double alpha, - bool bypass_tips, unsigned int nthreads, - const char *mmap_dir, - TMat** result) { - if (mmap_dir!=NULL) { - if (mmap_dir[0]==0) mmap_dir = NULL; // easier to have a simple test going on - } - - CHECK_FILE(biom_filename, table_missing) - CHECK_FILE(tree_filename, tree_missing) - SET_METHOD(unifrac_method, unknown_method) - PARSE_SYNC_TREE_TABLE(tree_filename, table_filename) - - const unsigned int stripe_stop = (table.n_samples + 1) / 2; - partial_mat_t *partial_mat = NULL; - - { - std::vector dm_stripes(stripe_stop); - std::vector dm_stripes_total(stripe_stop); - - std::vector tasks(nthreads); - std::vector threads(nthreads); - - set_tasks(tasks, alpha, table.n_samples, 0, stripe_stop, bypass_tips, nthreads); - su::process_stripes(table, tree_sheared, method, variance_adjust, dm_stripes, dm_stripes_total, threads, tasks); - - initialize_partial_mat(partial_mat, table, dm_stripes, 0, stripe_stop, true); // true -> is_upper_triangle - if ((partial_mat==NULL) || (partial_mat->stripes==NULL) || (partial_mat->sample_ids==NULL) ) { - fprintf(stderr, "Memory allocation error! (initialize_partial_mat)\n"); - exit(EXIT_FAILURE); - } - destroy_stripes(dm_stripes, dm_stripes_total, table.n_samples, 0, stripe_stop); - } - - initialize_mat_full_no_biom_T(*result, partial_mat->sample_ids, partial_mat->n_samples,mmap_dir); - - if (((*result)==NULL) || ((*result)->matrix==NULL) || ((*result)->sample_ids==NULL) ) { - fprintf(stderr, "Memory allocation error! (initialize_mat)\n"); - exit(EXIT_FAILURE); - } - - - { - MemoryStripes ps(partial_mat->stripes); - const uint32_t tile_size = (mmap_dir==NULL) ? \ - (128/sizeof(TReal)) : /* keep it small for memory access, to fit in chip cache */ \ - (4096/sizeof(TReal)); /* make it larger for mmap, as the limiting factor is swapping */ - su::stripes_to_matrix_T(ps, partial_mat->n_samples, partial_mat->stripe_total, (*result)->matrix, tile_size); - } - destroy_partial_mat(&partial_mat); - - return okay; -} - - -compute_status one_off_matrix(const char* biom_filename, const char* tree_filename, - const char* unifrac_method, bool variance_adjust, double alpha, - bool bypass_tips, unsigned int nthreads, - const char *mmap_dir, - mat_full_fp64_t** result) { - return one_off_matrix_T(biom_filename,tree_filename,unifrac_method,variance_adjust,alpha,bypass_tips,nthreads,mmap_dir,result); -} - -compute_status one_off_matrix_fp32(const char* biom_filename, const char* tree_filename, - const char* unifrac_method, bool variance_adjust, double alpha, - bool bypass_tips, unsigned int nthreads, - const char *mmap_dir, - mat_full_fp32_t** result) { - return one_off_matrix_T(biom_filename,tree_filename,unifrac_method,variance_adjust,alpha,bypass_tips,nthreads,mmap_dir,result); -} - -inline compute_status is_fp64(const std::string &method_string, const std::string &format_string, bool &fp64) { - if (format_string == "hdf5_fp32") { - fp64 = false; - } else if (format_string == "hdf5_fp64") { - fp64 = true; - } else if (format_string == "hdf5") { - if ((method_string=="unweighted_fp32") || (method_string=="weighted_normalized_fp32") || (method_string=="weighted_unnormalized_fp32") || (method_string=="generalized_fp32")) { - fp64 = false; - } else if ((method_string=="unweighted") || (method_string=="weighted_normalized") || (method_string=="weighted_unnormalized") || (method_string=="generalized")) { - fp64 = true; - } else { - return unknown_method; - } - } else { - return unknown_method; - } - - return okay; -} - - -compute_status unifrac_to_file(const char* biom_filename, const char* tree_filename, const char* out_filename, - const char* unifrac_method, bool variance_adjust, double alpha, - bool bypass_tips, unsigned int threads, const char* format, - unsigned int pcoa_dims, const char *mmap_dir) -{ - bool fp64; - compute_status rc = is_fp64(unifrac_method, format, fp64); - - if (rc==okay) { - if (fp64) { - mat_full_fp64_t* result; - rc = one_off_matrix(biom_filename, tree_filename, - unifrac_method, variance_adjust, alpha, - bypass_tips, threads, mmap_dir, - &result); - - if (rc==okay) { - // we have no alternative to hdf5 right now - IOStatus iostatus = write_mat_from_matrix_hdf5(out_filename, result, pcoa_dims); - destroy_mat_full_fp64(&result); - - if (iostatus!=write_okay) rc=output_error; - } - } else { - mat_full_fp32_t* result; - rc = one_off_matrix_fp32(biom_filename, tree_filename, - unifrac_method, variance_adjust, alpha, - bypass_tips, threads, mmap_dir, - &result); - - if (rc==okay) { - // we have no alternative to hdf5 right now - IOStatus iostatus = write_mat_from_matrix_hdf5_fp32(out_filename, result, pcoa_dims); - destroy_mat_full_fp32(&result); - - if (iostatus!=write_okay) rc=output_error; - } - } - } - - return rc; -} - -IOStatus write_mat(const char* output_filename, mat_t* result) { - std::ofstream output; - output.open(output_filename); - - uint64_t comb_N = su::comb_2(result->n_samples); - uint64_t comb_N_minus = 0; - double v; - - for(unsigned int i = 0; i < result->n_samples; i++) - output << "\t" << result->sample_ids[i]; - output << std::endl; - - for(unsigned int i = 0; i < result->n_samples; i++) { - output << result->sample_ids[i]; - for(unsigned int j = 0; j < result->n_samples; j++) { - if(i < j) { // upper triangle - comb_N_minus = su::comb_2(result->n_samples - i); - v = result->condensed_form[comb_N - comb_N_minus + (j - i - 1)]; - } else if (i > j) { // lower triangle - comb_N_minus = su::comb_2(result->n_samples - j); - v = result->condensed_form[comb_N - comb_N_minus + (i - j - 1)]; - } else { - v = 0.0; - } - output << std::setprecision(16) << "\t" << v; - } - output << std::endl; - } - output.close(); - - return write_okay; -} - -IOStatus write_mat_from_matrix(const char* output_filename, mat_full_fp64_t* result) { - const double *buf2d = result->matrix; - - std::ofstream output; - output.open(output_filename); - - double v; - const uint64_t n_samples_64 = result->n_samples; // 64-bit to avoid overflow - - for(unsigned int i = 0; i < result->n_samples; i++) - output << "\t" << result->sample_ids[i]; - output << std::endl; - - for(unsigned int i = 0; i < result->n_samples; i++) { - output << result->sample_ids[i]; - for(unsigned int j = 0; j < result->n_samples; j++) { - v = buf2d[i*n_samples_64+j]; - output << std::setprecision(16) << "\t" << v; - } - output << std::endl; - } - output.close(); - - return write_okay; -} - -herr_t write_hdf5_string(hid_t output_file_id,const char *dname, const char *str) -{ - // this is the convoluted way to store a string - // Will use the FORTRAN forma, so we do not depend on null termination - hid_t filetype_id = H5Tcopy (H5T_FORTRAN_S1); - H5Tset_size(filetype_id, strlen(str)); - hid_t memtype_id = H5Tcopy (H5T_C_S1); - H5Tset_size(memtype_id, strlen(str)+1); - - hsize_t dims[1] = {1}; - hid_t dataspace_id = H5Screate_simple (1, dims, NULL); - - hid_t dataset_id = H5Dcreate(output_file_id,dname, filetype_id, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, - H5P_DEFAULT); - herr_t status = H5Dwrite(dataset_id, memtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, str); - - H5Dclose(dataset_id); - H5Sclose(dataspace_id); - H5Tclose(memtype_id); - H5Tclose(filetype_id); - - return status; -} - -// Internal: Make sure TReal and real_id match -template -IOStatus write_mat_from_matrix_hdf5_T(const char* output_filename, TMat * result, hid_t real_id, unsigned int pcoa_dims) { - /* Create a new file using default properties. */ - hid_t output_file_id = H5Fcreate(output_filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); - if (output_file_id<0) return write_error; - - // simple header - if (write_hdf5_string(output_file_id,"format","BDSM")<0) { - H5Fclose (output_file_id); - return write_error; - } - if (write_hdf5_string(output_file_id,"version","2020.12")<0) { - H5Fclose (output_file_id); - return write_error; - } - - // save the ids - { - hsize_t dims[1]; - dims[0] = result->n_samples; - hid_t dataspace_id = H5Screate_simple(1, dims, NULL); - - // this is the convoluted way to store an array of strings - hid_t datatype_id = H5Tcopy(H5T_C_S1); - H5Tset_size(datatype_id,H5T_VARIABLE); - - hid_t dcpl_id = H5Pcreate (H5P_DATASET_CREATE); - - hid_t dataset_id = H5Dcreate1(output_file_id, "order", datatype_id, dataspace_id, dcpl_id); - - herr_t status = H5Dwrite(dataset_id, datatype_id, H5S_ALL, H5S_ALL, - H5P_DEFAULT, result->sample_ids); - - H5Dclose(dataset_id); - H5Tclose(datatype_id); - H5Sclose(dataspace_id); - H5Pclose(dcpl_id); - - // check status after cleanup, for simplicity - if (status<0) { - H5Fclose (output_file_id); - return write_error; - } - } - - // save the matrix - { - hsize_t dims[2]; - dims[0] = result->n_samples; - dims[1] = result->n_samples; - hid_t dataspace_id = H5Screate_simple(2, dims, NULL); - - hid_t dcpl_id = H5Pcreate (H5P_DATASET_CREATE); - - hid_t dataset_id = H5Dcreate2(output_file_id, "matrix",real_id, dataspace_id, - H5P_DEFAULT, dcpl_id, H5P_DEFAULT); - herr_t status = H5Dwrite(dataset_id, real_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, - result->matrix); - - - H5Pclose(dcpl_id); - H5Dclose(dataset_id); - H5Sclose(dataspace_id); - - // check status after cleanup, for simplicity - if (status<0) { - H5Fclose (output_file_id); - return write_error; - } - } - - if (pcoa_dims>0) { - // compute pcoa and save it in the file - // use inplace variant to keep memory use in check; we don't need matrix anymore - TReal * eigenvalues; - TReal * samples; - TReal * proportion_explained; - - su::pcoa_inplace(result->matrix, result->n_samples, pcoa_dims, eigenvalues, samples, proportion_explained); - - - if (write_hdf5_string(output_file_id,"pcoa_method","FSVD")<0) { - H5Fclose (output_file_id); - return write_error; - } - - // save the eigenvalues - { - hsize_t dims[1]; - dims[0] = pcoa_dims; - hid_t dataspace_id = H5Screate_simple(1, dims, NULL); - - hid_t dcpl_id = H5Pcreate (H5P_DATASET_CREATE); - - hid_t dataset_id = H5Dcreate2(output_file_id, "pcoa_eigvals",real_id, dataspace_id, - H5P_DEFAULT, dcpl_id, H5P_DEFAULT); - herr_t status = H5Dwrite(dataset_id, real_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, - eigenvalues); - - - H5Pclose(dcpl_id); - H5Dclose(dataset_id); - H5Sclose(dataspace_id); - - // check status after cleanup, for simplicity - if (status<0) { - H5Fclose (output_file_id); - free(samples); - free(proportion_explained); - free(eigenvalues); - return write_error; - } - } - - // save the proportion_explained - { - hsize_t dims[1]; - dims[0] = pcoa_dims; - hid_t dataspace_id = H5Screate_simple(1, dims, NULL); - - hid_t dcpl_id = H5Pcreate (H5P_DATASET_CREATE); - - hid_t dataset_id = H5Dcreate2(output_file_id, "pcoa_proportion_explained",real_id, dataspace_id, - H5P_DEFAULT, dcpl_id, H5P_DEFAULT); - herr_t status = H5Dwrite(dataset_id, real_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, - proportion_explained); - - - H5Pclose(dcpl_id); - H5Dclose(dataset_id); - H5Sclose(dataspace_id); - - // check status after cleanup, for simplicity - if (status<0) { - H5Fclose (output_file_id); - free(samples); - free(proportion_explained); - free(eigenvalues); - return write_error; - } - } - - // save the samples - { - hsize_t dims[2]; - dims[0] = result->n_samples; - dims[1] = pcoa_dims; - hid_t dataspace_id = H5Screate_simple(2, dims, NULL); - - hid_t dcpl_id = H5Pcreate (H5P_DATASET_CREATE); - - hid_t dataset_id = H5Dcreate2(output_file_id, "pcoa_samples",real_id, dataspace_id, - H5P_DEFAULT, dcpl_id, H5P_DEFAULT); - herr_t status = H5Dwrite(dataset_id, real_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, - samples); - - - H5Pclose(dcpl_id); - H5Dclose(dataset_id); - H5Sclose(dataspace_id); - - // check status after cleanup, for simplicity - if (status<0) { - H5Fclose (output_file_id); - free(samples); - free(proportion_explained); - free(eigenvalues); - return write_error; - } - } - - free(samples); - free(proportion_explained); - free(eigenvalues); - - } - - H5Fclose (output_file_id); - return write_okay; -} - -// Internal: Make sure TReal and real_id match -template -IOStatus write_mat_hdf5_T(const char* output_filename, mat_t* result,hid_t real_id, unsigned int pcoa_dims) { - // compute the matrix - TMat mat_full; - mat_full.n_samples = result->n_samples; - - const uint64_t n_samples = result->n_samples; - mat_full.flags = 0; - mat_full.matrix = (TReal*) malloc(n_samples*n_samples*sizeof(TReal)); - if (mat_full.matrix==NULL) { - return open_error; // we don't have a better error code - } - - mat_full.sample_ids = result->sample_ids; // just link - - condensed_form_to_matrix_T(result->condensed_form, n_samples, mat_full.matrix); - IOStatus err = write_mat_from_matrix_hdf5_T(output_filename, &mat_full, real_id, pcoa_dims); - - free(mat_full.matrix); - return err; -} - -IOStatus write_mat_hdf5(const char* output_filename, mat_t* result, unsigned int pcoa_dims) { - return write_mat_hdf5_T(output_filename,result,H5T_IEEE_F64LE,pcoa_dims); -} - -IOStatus write_mat_hdf5_fp32(const char* output_filename, mat_t* result, unsigned int pcoa_dims) { - return write_mat_hdf5_T(output_filename,result,H5T_IEEE_F32LE,pcoa_dims); -} - -IOStatus write_mat_from_matrix_hdf5(const char* output_filename, mat_full_fp64_t* result, unsigned int pcoa_dims) { - return write_mat_from_matrix_hdf5_T(output_filename,result,H5T_IEEE_F64LE,pcoa_dims); -} - -IOStatus write_mat_from_matrix_hdf5_fp32(const char* output_filename, mat_full_fp32_t* result, unsigned int pcoa_dims) { - return write_mat_from_matrix_hdf5_T(output_filename,result,H5T_IEEE_F32LE,pcoa_dims); -} - -IOStatus write_vec(const char* output_filename, r_vec* result) { - std::ofstream output; - output.open(output_filename); - - // write sample ids in first column of file and faith's pd in second column - output << "#SampleID\tfaith_pd" << std::endl; - for(unsigned int i = 0; i < result->n_samples; i++) { - output << result->sample_ids[i]; - output << std::setprecision(16) << "\t" << result->values[i]; - output << std::endl; - } - output.close(); - - return write_okay; -} - -IOStatus write_partial(const char* output_filename, const partial_mat_t* result) { - int fd = open(output_filename, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR ); - if (fd==-1) return write_error; - - int cnt = -1; - - uint32_t n_stripes = result->stripe_stop - result->stripe_start; - - uint32_t sample_id_length = 0; - for(unsigned int i = 0; i < result->n_samples; i++) { - sample_id_length += strlen(result->sample_ids[i])+1; - } - - { - char * const samples_buf = (char *)malloc(sample_id_length); - - char *samples_ptr = samples_buf; - - /* sample IDs */ - for(unsigned int i = 0; i < result->n_samples; i++) { - uint32_t length = strlen(result->sample_ids[i])+1; - memcpy(samples_ptr,result->sample_ids[i],length); - samples_ptr+= length; - } - - int max_compressed = LZ4_compressBound(sample_id_length); - char * const cmp_buf = (char *)malloc(max_compressed); - - int sample_id_length_compressed = LZ4_compress_default(samples_buf,cmp_buf,sample_id_length,max_compressed); - if (sample_id_length_compressed<1) {close(fd); return open_error;} - - uint32_t header[8]; - header[0] = PARTIAL_MAGIC_V2; - header[1] = result->n_samples; - header[2] = n_stripes; - header[3] = result->stripe_start; - header[4] = result->stripe_total; - header[5] = result->is_upper_triangle; - header[6] = sample_id_length; - header[7] = sample_id_length_compressed; - - cnt=write(fd,header, 8 * sizeof(uint32_t)); - if (cnt<1) {close(fd); return write_error;} - - cnt=write(fd,cmp_buf, sample_id_length_compressed); - if (cnt<1) {close(fd); return write_error;} - - free(cmp_buf); - free(samples_buf); - } - - { - int max_compressed = LZ4_compressBound(sizeof(double) * result->n_samples); - char * const cmp_buf_raw = (char *)malloc(max_compressed+sizeof(uint32_t)); - char * const cmp_buf = cmp_buf_raw + sizeof(uint32_t); - - /* stripe information */ - for(unsigned int i = 0; i < n_stripes; i++) { - int cmp_size = LZ4_compress_default((const char *) result->stripes[i],cmp_buf,sizeof(double) * result->n_samples,max_compressed); - if (cmp_size<1) {close(fd); return open_error;} - - uint32_t *cmp_buf_size_p = (uint32_t *)cmp_buf_raw; - *cmp_buf_size_p = cmp_size; - - cnt=write(fd, cmp_buf_raw, cmp_size+sizeof(uint32_t)); - if (cnt<1) {return write_error;} - } - - free(cmp_buf_raw); - } - - /* footer */ - { - uint32_t header[1]; - header[0] = PARTIAL_MAGIC_V2; - - cnt=write(fd,header, 1 * sizeof(uint32_t)); - if (cnt<1) {close(fd); return open_error;} - } - - close(fd); - - return write_okay; -} - -IOStatus _is_partial_file(const char* input_filename) { - int fd = open(input_filename, O_RDONLY ); - if (fd==-1) return open_error; - - uint32_t header[1]; - int cnt = read(fd,header,sizeof(uint32_t)); - close(fd); - - if (cnt!=sizeof(uint32_t)) return magic_incompatible; - if ( header[0] != PARTIAL_MAGIC_V2) return magic_incompatible; - - return read_okay; -} - -template -inline IOStatus read_partial_header_fd(int fd, TPMat &result) { - int cnt=-1; - - uint32_t header[8]; - cnt = read(fd,header,8*sizeof(uint32_t)); - if (cnt != (8*sizeof(uint32_t))) {return magic_incompatible;} - - if ( header[0] != PARTIAL_MAGIC_V2) {return magic_incompatible;} - - const uint32_t n_samples = header[1]; - const uint32_t n_stripes = header[2]; - const uint32_t stripe_start = header[3]; - const uint32_t stripe_total = header[4]; - const bool is_upper_triangle = header[5]; - - /* sanity check header */ - if(n_samples <= 0 || n_stripes <= 0 || stripe_total <= 0 || is_upper_triangle < 0) - {return bad_header;} - if(stripe_total >= n_samples || n_stripes > stripe_total || stripe_start >= stripe_total || stripe_start + n_stripes > stripe_total) - {return bad_header;} - - /* initialize the partial result structure */ - result.n_samples = n_samples; - result.sample_ids = (char**)malloc(sizeof(char*) * n_samples); - result.stripes = (double**)malloc(sizeof(double*) * (n_stripes)); - result.stripe_start = stripe_start; - result.stripe_stop = stripe_start + n_stripes; - result.is_upper_triangle = is_upper_triangle; - result.stripe_total = stripe_total; - - /* load samples */ - { - const uint32_t sample_id_length = header[6]; - const uint32_t sample_id_length_compressed = header[7]; - - /* sanity check header */ - if (sample_id_length<=0 || sample_id_length_compressed <=0) - { return bad_header;} - - char * const cmp_buf = (char *)malloc(sample_id_length_compressed); - if (cmp_buf==NULL) { return bad_header;} // no better error code - cnt = read(fd,cmp_buf,sample_id_length_compressed); - if (cnt != sample_id_length_compressed) {free(cmp_buf); return magic_incompatible;} - - char *samples_buf = (char *)malloc(sample_id_length); - if (samples_buf==NULL) { free(cmp_buf); return bad_header;} // no better error code - - cnt = LZ4_decompress_safe(cmp_buf,samples_buf,sample_id_length_compressed,sample_id_length); - if (cnt!=sample_id_length) {free(samples_buf); free(cmp_buf); return magic_incompatible;} - - const char *samples_ptr = samples_buf; - - for(int i = 0; i < n_samples; i++) { - uint32_t sample_length = strlen(samples_ptr); - if ((samples_ptr+sample_length+1)>(samples_buf+sample_id_length)) {free(samples_buf); free(cmp_buf); return magic_incompatible;} - - result.sample_ids[i] = (char*)malloc(sample_length + 1); - memcpy(result.sample_ids[i],samples_ptr,sample_length + 1); - samples_ptr += sample_length + 1; - } - free(samples_buf); - free(cmp_buf); - } - - return read_okay; -} - -template -inline IOStatus read_partial_data_fd(int fd, TPMat &result) { - int cnt=-1; - - const uint32_t n_samples = result.n_samples; - const uint32_t n_stripes = result.stripe_stop-result.stripe_start; - - /* load stripes */ - { - int max_compressed = LZ4_compressBound(sizeof(double) * n_samples); - char * const cmp_buf = (char *)malloc(max_compressed+sizeof(uint32_t)); - if (cmp_buf==NULL) { return bad_header;} // no better error code - - uint32_t *cmp_buf_size_p = (uint32_t *)cmp_buf; - - cnt = read(fd,cmp_buf_size_p , sizeof(uint32_t) ); - if (cnt != sizeof(uint32_t) ) {free(cmp_buf); return magic_incompatible;} - - for(int i = 0; i < n_stripes; i++) { - uint32_t cmp_size = *cmp_buf_size_p; - - uint32_t read_size = cmp_size; - if ( (i+1) -inline IOStatus read_partial_one_stripe_fd(int fd, TPMat &result, uint32_t stripe_idx) { - int cnt=-1; - - const uint32_t n_samples = result.n_samples; - - /* load stripes */ - { - int max_compressed = LZ4_compressBound(sizeof(double) * n_samples); - char * const cmp_buf = (char *)malloc(max_compressed+sizeof(uint32_t)); - if (cmp_buf==NULL) { return bad_header;} // no better error code - - uint32_t *cmp_buf_size_p = (uint32_t *)cmp_buf; - - uint32_t curr_idx = stripe_idx; - while (result.offsets[curr_idx]==0) --curr_idx; // must start reading from the first known offset - - for (;curr_idx(fd, *result); - if (sts==read_okay) - sts = read_partial_data_fd(fd, *result); - - if (sts==read_okay) { - IOStatus sts = read_okay; - /* sanity check the footer */ - uint32_t header[1]; - header[0] = 0; - int cnt = read(fd,header,sizeof(uint32_t)); - if (cnt != (sizeof(uint32_t))) {sts= magic_incompatible;} - - if (sts==read_okay) { - if ( header[0] != PARTIAL_MAGIC_V2) {sts= magic_incompatible;} - } - } - - close(fd); - - if (sts==read_okay) { - (*result_out) = result; - } else { - free(result); - (*result_out) = NULL; - } - return sts; -} - -IOStatus read_partial_header(const char* input_filename, partial_dyn_mat_t** result_out) { - int fd = open(input_filename, O_RDONLY ); - if (fd==-1) return open_error; - - /* initialize the partial result structure */ - partial_dyn_mat_t* result = (partial_dyn_mat_t*)malloc(sizeof(partial_dyn_mat)); - { - IOStatus sts = read_partial_header_fd(fd, *result); - if (sts!=read_okay) {free(result); close(fd); return sts;} - } - - // save the offset of the first stripe - const uint32_t n_stripes = result->stripe_stop-result->stripe_start; - result->stripes = (double**) calloc(n_stripes,sizeof(double*)); - result->offsets = (uint64_t*) calloc(n_stripes,sizeof(uint64_t)); - result->offsets[0] = lseek(fd,0,SEEK_CUR); - - close(fd); - - result->filename= strdup(input_filename); - - (*result_out) = result; - return read_okay; -} - -IOStatus read_partial_one_stripe(partial_dyn_mat_t* result, uint32_t stripe_idx) { - if (result->stripes[stripe_idx]!=0) return read_okay; // will not re-read - - int fd = open(result->filename, O_RDONLY ); - if (fd==-1) return open_error; - - IOStatus sts = read_partial_one_stripe_fd(fd, *result, stripe_idx); - - close(fd); - return sts; -} - - -template -MergeStatus check_partial(const TPMat* const * partial_mats, int n_partials, bool verbose) { - if(n_partials <= 0) { - fprintf(stderr, "Zero or less partials.\n"); - exit(EXIT_FAILURE); - } - - // sanity check - int n_samples = partial_mats[0]->n_samples; - bool *stripe_map = (bool*)calloc(sizeof(bool), partial_mats[0]->stripe_total); - int stripe_count = 0; - - for(int i = 0; i < n_partials; i++) { - if(partial_mats[i]->n_samples != n_samples) { - free(stripe_map); - if (verbose) { - fprintf(stderr, "Wrong number of samples in %i, %i!=%i\n", - i,int(partial_mats[i]->n_samples),int(n_samples)); - } - return partials_mismatch; - } - - if(partial_mats[0]->stripe_total != partial_mats[i]->stripe_total) { - free(stripe_map); - if (verbose) { - fprintf(stderr, "Wrong number of stripes in %i, %i!=%i\n", - i,int(partial_mats[0]->stripe_total), int(partial_mats[i]->stripe_total)); - } - return partials_mismatch; - } - if(partial_mats[0]->is_upper_triangle != partial_mats[i]->is_upper_triangle) { - free(stripe_map); - if (verbose) { - fprintf(stderr, "Wrong number of is_upper_triangle in %i, %i!=%i\n", - i,int(partial_mats[0]->is_upper_triangle),int(partial_mats[i]->is_upper_triangle)); - } - return square_mismatch; - } - for(int j = 0; j < n_samples; j++) { - if(strcmp(partial_mats[0]->sample_ids[j], partial_mats[i]->sample_ids[j]) != 0) { - free(stripe_map); - if (verbose) { - fprintf(stderr, "Wrong number of sample id %i in %i, %s!=%s\n", - j,i,partial_mats[0]->sample_ids[j], partial_mats[i]->sample_ids[j]); - } - return sample_id_consistency; - } - } - for(int j = partial_mats[i]->stripe_start; j < partial_mats[i]->stripe_stop; j++) { - if(stripe_map[j]) { - if (verbose) { - fprintf(stderr, "Overlap in %i vs %i\n", - i,j); - } - free(stripe_map); - return stripes_overlap; - } - stripe_map[j] = true; - stripe_count += 1; - } - } - free(stripe_map); - - if(stripe_count != partial_mats[0]->stripe_total) { - if (verbose) { - fprintf(stderr, "Insufficient number of stripes found, %i!=%i\n", - int(stripe_count), int(partial_mats[0]->stripe_total)); - } - return incomplete_stripe_set; - } - - return merge_okay; -} - -MergeStatus validate_partial(const partial_dyn_mat_t* const * partial_mats, int n_partials) { - return check_partial(partial_mats, n_partials, true); -} - - -MergeStatus merge_partial(partial_mat_t** partial_mats, int n_partials, unsigned int nthreads, mat_t** result) { - MergeStatus err = check_partial(partial_mats, n_partials, false); - if (err!=merge_okay) return err; - - int n_samples = partial_mats[0]->n_samples; - std::vector stripes(partial_mats[0]->stripe_total); - std::vector stripes_totals(partial_mats[0]->stripe_total); // not actually used but destroy_stripes needs this to "exist" - for(int i = 0; i < n_partials; i++) { - int n_stripes = partial_mats[i]->stripe_stop - partial_mats[i]->stripe_start; - for(int j = 0; j < n_stripes; j++) { - // as this is potentially a large amount of memory, don't copy, just adopt - *&(stripes[j + partial_mats[i]->stripe_start]) = partial_mats[i]->stripes[j]; - } - } - - initialize_mat_no_biom(*result, partial_mats[0]->sample_ids, n_samples, partial_mats[0]->is_upper_triangle); - if ((*result)==NULL) return incomplete_stripe_set; - if ((*result)->condensed_form==NULL) return incomplete_stripe_set; - if ((*result)->sample_ids==NULL) return incomplete_stripe_set; - - su::stripes_to_condensed_form(stripes, n_samples, (*result)->condensed_form, 0, partial_mats[0]->stripe_total); - - destroy_stripes(stripes, stripes_totals, n_samples, 0, n_partials); - - return merge_okay; -} - -// Will keep only the strictly necessary stripes in memory... reading just in time -class PartialStripes : public su::ManagedStripes { - private: - const uint32_t n_partials; - mutable partial_dyn_mat_t* * partial_mats; // link only, not owned - - static bool in_range(const partial_dyn_mat_t &partial_mat, uint32_t stripe) { - return (stripe>=partial_mat.stripe_start) && (stripestripe_start; - - if (partial_mat->stripes[sidx]==NULL) { - read_partial_one_stripe(partial_mat,sidx); - // ignore any errors, not clear what to do - // will just return NULL - } - - return partial_mat->stripes[sidx]; - } - virtual void release_stripe(uint32_t stripe) const { - uint32_t pidx = find_partial_idx(stripe); - partial_dyn_mat_t * const partial_mat = partial_mats[pidx]; - uint32_t sidx = stripe-partial_mat->stripe_start; - - if (partial_mat->stripes[sidx]!=NULL) { - free(partial_mat->stripes[sidx]); - partial_mat->stripes[sidx]=NULL; - } - } -}; - -template -MergeStatus merge_partial_to_matrix_T(partial_dyn_mat_t* * partial_mats, int n_partials, - const char *mmap_dir, /* if NULL or "", use malloc */ - TMat** result /* out */ ) { - if (mmap_dir!=NULL) { - if (mmap_dir[0]==0) mmap_dir = NULL; // easier to have a simple test going on - } - - MergeStatus err = check_partial(partial_mats, n_partials, false); - if (err!=merge_okay) return err; - - initialize_mat_full_no_biom_T(*result, partial_mats[0]->sample_ids, partial_mats[0]->n_samples,mmap_dir); - - if ((*result)==NULL) return incomplete_stripe_set; - if ((*result)->matrix==NULL) return incomplete_stripe_set; - if ((*result)->sample_ids==NULL) return incomplete_stripe_set; - - PartialStripes ps(n_partials,partial_mats); - const uint32_t tile_size = (mmap_dir==NULL) ? \ - (128/sizeof(TReal)) : /* keep it small for memory access, to fit in chip cache */ \ - (4096/sizeof(TReal)); /* make it larger for mmap, as the limiting factor is swapping */ - su::stripes_to_matrix_T(ps, partial_mats[0]->n_samples, partial_mats[0]->stripe_total, (*result)->matrix, tile_size); - - return merge_okay; -} - -MergeStatus merge_partial_to_matrix(partial_dyn_mat_t* * partial_mats, int n_partials, mat_full_fp64_t** result) { - return merge_partial_to_matrix_T(partial_mats, n_partials, NULL, result); -} - -MergeStatus merge_partial_to_matrix_fp32(partial_dyn_mat_t* * partial_mats, int n_partials, mat_full_fp32_t** result) { - return merge_partial_to_matrix_T(partial_mats, n_partials, NULL, result); -} - -MergeStatus merge_partial_to_mmap_matrix(partial_dyn_mat_t* * partial_mats, int n_partials, const char *mmap_dir, mat_full_fp64_t** result) { - return merge_partial_to_matrix_T(partial_mats, n_partials, mmap_dir, result); -} - -MergeStatus merge_partial_to_mmap_matrix_fp32(partial_dyn_mat_t* * partial_mats, int n_partials, const char *mmap_dir, mat_full_fp32_t** result) { - return merge_partial_to_matrix_T(partial_mats, n_partials, mmap_dir, result); -} - - -// skbio_alt pass-thoughs - - -// Find eigen values and vectors -// Based on N. Halko, P.G. Martinsson, Y. Shkolnisky, and M. Tygert. -// Original Paper: https://arxiv.org/abs/1007.5510 -// centered == n x n, must be symmetric, Note: will be used in-place as temp buffer - -void find_eigens_fast(const uint32_t n_samples, const uint32_t n_dims, double * centered, double **eigenvalues, double **eigenvectors) { - su::find_eigens_fast(n_samples, n_dims, centered, *eigenvalues, *eigenvectors); -} - -void find_eigens_fast_fp32(const uint32_t n_samples, const uint32_t n_dims, float * centered, float **eigenvalues, float **eigenvectors) { - su::find_eigens_fast(n_samples, n_dims, centered, *eigenvalues, *eigenvectors); -} - -/* - Perform Principal Coordinate Analysis. - - Principal Coordinate Analysis (PCoA) is a method similar - to Principal Components Analysis (PCA) with the difference that PCoA - operates on distance matrices, typically with non-euclidian and thus - ecologically meaningful distances like UniFrac in microbiome research. - - In ecology, the euclidean distance preserved by Principal - Component Analysis (PCA) is often not a good choice because it - deals poorly with double zeros (Species have unimodal - distributions along environmental gradients, so if a species is - absent from two sites at the same site, it can't be known if an - environmental variable is too high in one of them and too low in - the other, or too low in both, etc. On the other hand, if an - species is present in two sites, that means that the sites are - similar.). - - Note that the returned eigenvectors are not normalized to unit length. -*/ - -// mat - in, result of unifrac compute -// n_samples - in, size of the matrix (n x n) -// n_dims - in, Dimensions to reduce the distance matrix to. This number determines how many eigenvectors and eigenvalues will be returned. -// eigenvalues - out, alocated buffer of size n_dims -// samples - out, alocated buffer of size n_dims x n_samples -// proportion_explained - out, allocated buffer of size n_dims - -void pcoa(const double * mat, const uint32_t n_samples, const uint32_t n_dims, double * *eigenvalues, double * *samples, double * *proportion_explained) { - su::pcoa(mat, n_samples, n_dims, *eigenvalues, *samples, *proportion_explained); -} - -void pcoa_fp32(const float * mat, const uint32_t n_samples, const uint32_t n_dims, float * *eigenvalues, float * *samples, float * *proportion_explained) { - su::pcoa(mat, n_samples, n_dims, *eigenvalues, *samples, *proportion_explained); -} - -void pcoa_mixed(const double * mat, const uint32_t n_samples, const uint32_t n_dims, float * *eigenvalues, float * *samples, float * *proportion_explained) { - su::pcoa(mat, n_samples, n_dims, *eigenvalues, *samples, *proportion_explained); -} - diff --git a/sucpp/api.hpp b/sucpp/api.hpp deleted file mode 100644 index be3bd63f..00000000 --- a/sucpp/api.hpp +++ /dev/null @@ -1,581 +0,0 @@ -#include "task_parameters.hpp" - -#ifdef __cplusplus -#include -#define EXTERN extern "C" - - -#else -#include -#define EXTERN -#endif - -#define PARTIAL_MAGIC "SSU-PARTIAL-01" -#define PARTIAL_MAGIC_V2 0x088ABA02 - - -typedef enum compute_status {okay=0, tree_missing, table_missing, table_empty, unknown_method, table_and_tree_do_not_overlap, output_error} ComputeStatus; -typedef enum io_status {read_okay=0, write_okay, open_error, read_error, magic_incompatible, bad_header, unexpected_end, write_error} IOStatus; -typedef enum merge_status {merge_okay=0, incomplete_stripe_set, sample_id_consistency, square_mismatch, partials_mismatch, stripes_overlap} MergeStatus; - -/* a result matrix - * - * n_samples the number of samples. - * cf_size the size of the condensed form. - * is_upper_triangle if true, indicates condensed_form represents a square - * matrix, and only the upper triangle is contained. if false, - * condensed_form represents the lower triangle of a matrix. - * condensed_form the matrix values of length cf_size. - * sample_ids the sample IDs of length n_samples. - */ -typedef struct mat { - unsigned int n_samples; - unsigned int cf_size; - bool is_upper_triangle; - double* condensed_form; - char** sample_ids; -} mat_t; - -/* a result matrix, full, fp64 - * - * n_samples the number of samples. - * matrix the matrix values, n_sample**2 size - * sample_ids the sample IDs of length n_samples. - */ -typedef struct mat_full_fp64 { - uint32_t n_samples; - uint32_t flags; //opaque, 0 for default behavior - double* matrix; - char** sample_ids; -} mat_full_fp64_t; - -/* a result matrix, full, fp32 - * - * n_samples the number of samples. - * matrix the matrix values, n_sample**2 size - * sample_ids the sample IDs of length n_samples. - */ -typedef struct mat_full_fp32 { - uint32_t n_samples; - uint32_t flags; //opaque, 0 for default behavior - float* matrix; - char** sample_ids; -} mat_full_fp32_t; - - - -/* a result vector - * - * n_samples the number of samples. - * values the score values of length n_samples. - * sample_ids the sample IDs of length n_samples. - */ -typedef struct results_vec{ - unsigned int n_samples; - double* values; - char** sample_ids; -} r_vec; - -/* a partial result containing stripe data - * - * n_samples the number of samples. - * sample_ids the sample IDs of length n_samples. - * stripes the stripe data of dimension (stripe_stop - stripe_start, n_samples) - * stripe_start the logical starting stripe in the final matrix. - * stripe_stop the logical stopping stripe in the final matrix. - * stripe_total the total number of stripes present in the final matrix. - * is_upper_triangle whether the stripes correspond to the upper triangle of the resulting matrix. - * This is useful for asymmetric unifrac metrics. - */ -typedef struct partial_mat { - uint32_t n_samples; - char** sample_ids; - double** stripes; - uint32_t stripe_start; - uint32_t stripe_stop; - uint32_t stripe_total; - bool is_upper_triangle; -} partial_mat_t; - -/* a partial resuly, can be populated dynamically - * - * n_samples the number of samples. - * sample_ids the sample IDs of length n_samples. - * offsets offsets to the stripes in the file; 0 means unknown - * stripes the stripe data of dimension (stripe_stop - stripe_start, n_samples) - * stripe_start the logical starting stripe in the final matrix. - * stripe_stop the logical stopping stripe in the final matrix. - * stripe_total the total number of stripes present in the final matrix. - * is_upper_triangle whether the stripes correspond to the upper triangle of the resulting matrix. - * This is useful for asymmetric unifrac metrics. - * filename Name of the file from which to read - */ -typedef struct partial_dyn_mat { - uint32_t n_samples; - char** sample_ids; - uint64_t* offsets; - double** stripes; - uint32_t stripe_start; - uint32_t stripe_stop; - uint32_t stripe_total; - bool is_upper_triangle; - char* filename; -} partial_dyn_mat_t; - - - -void destroy_mat(mat_t** result); -void destroy_mat_full_fp64(mat_full_fp64_t** result); -void destroy_mat_full_fp32(mat_full_fp32_t** result); -void destroy_partial_mat(partial_mat_t** result); -void destroy_partial_dyn_mat(partial_dyn_mat_t** result); -void destroy_results_vec(r_vec** result); - -/* Compute UniFrac - condensed form - * - * biom_filename the filename to the biom table. - * tree_filename the filename to the correspodning tree. - * unifrac_method the requested unifrac method. - * variance_adjust whether to apply variance adjustment. - * alpha GUniFrac alpha, only relevant if method == generalized. - * bypass_tips disregard tips, reduces compute by about 50% - * threads the number of threads to use. - * result the resulting distance matrix in condensed form, this is initialized within the method so using ** - * - * one_off returns the following error codes: - * - * okay : no problems encountered - * table_missing : the filename for the table does not exist - * tree_missing : the filename for the tree does not exist - * unknown_method : the requested method is unknown. - * table_empty : the table does not have any entries - */ -EXTERN ComputeStatus one_off(const char* biom_filename, const char* tree_filename, - const char* unifrac_method, bool variance_adjust, double alpha, - bool bypass_tips, unsigned int threads, mat_t** result); - -/* Compute UniFrac - matrix form - * - * biom_filename the filename to the biom table. - * tree_filename the filename to the correspodning tree. - * unifrac_method the requested unifrac method. - * variance_adjust whether to apply variance adjustment. - * alpha GUniFrac alpha, only relevant if method == generalized. - * bypass_tips disregard tips, reduces compute by about 50% - * threads the number of threads/blocks to use. - * mmap_dir If not NULL, area to use for temp memory storage - * result the resulting distance matrix in matrix form, this is initialized within the method so using ** - * - * one_off_matrix returns the following error codes: - * - * okay : no problems encountered - * table_missing : the filename for the table does not exist - * tree_missing : the filename for the tree does not exist - * unknown_method : the requested method is unknown. - * table_empty : the table does not have any entries - */ -EXTERN ComputeStatus one_off_matrix(const char* biom_filename, const char* tree_filename, - const char* unifrac_method, bool variance_adjust, double alpha, - bool bypass_tips, unsigned int nthreads, - const char *mmap_dir, - mat_full_fp64_t** result); - -/* Compute UniFrac - matrix form, fp32 variant - * - * biom_filename the filename to the biom table. - * tree_filename the filename to the correspodning tree. - * unifrac_method the requested unifrac method. - * variance_adjust whether to apply variance adjustment. - * alpha GUniFrac alpha, only relevant if method == generalized. - * bypass_tips disregard tips, reduces compute by about 50% - * threads the number of threads/blocks to use. - * mmap_dir If not NULL, area to use for temp memory storage - * result the resulting distance matrix in matrix form, this is initialized within the method so using ** - * - * one_off_matrix_fp32 returns the following error codes: - * - * okay : no problems encountered - * table_missing : the filename for the table does not exist - * tree_missing : the filename for the tree does not exist - * unknown_method : the requested method is unknown. - * table_empty : the table does not have any entries - */ -EXTERN ComputeStatus one_off_matrix_fp32(const char* biom_filename, const char* tree_filename, - const char* unifrac_method, bool variance_adjust, double alpha, - bool bypass_tips, unsigned int nthreads, - const char *mmap_dir, - mat_full_fp32_t** result); - - -/* compute Faith PD - * biom_filename the filename to the biom table. - * tree_filename the filename to the correspodning tree. - * result the resulting vector of computed Faith PD values - * - * faith_pd_one_off returns the following error codes: - * - * okay : no problems encountered - * table_missing : the filename for the table does not exist - * tree_missing : the filename for the tree does not exist - * table_empty : the table does not have any entries - */ -EXTERN ComputeStatus faith_pd_one_off(const char* biom_filename, const char* tree_filename, - r_vec** result); - -/* Compute UniFrac and save to file - * - * biom_filename the filename to the biom table. - * tree_filename the filename to the correspodning tree. - * out_filename the filename of the output file. - * unifrac_method the requested unifrac method. - * variance_adjust whether to apply variance adjustment. - * alpha GUniFrac alpha, only relevant if method == generalized. - * bypass_tips disregard tips, reduces compute by about 50% - * threads the number of threads to use. - * format output format to use. - * pcoa_dims if not 0, number of dimensions to use or PCoA - * mmap_dir if not empty, temp dir to use for disk-based memory - * - * unifrac_to_file returns the following error codes: - * - * okay : no problems encountered - * table_missing : the filename for the table does not exist - * tree_missing : the filename for the tree does not exist - * unknown_method : the requested method is unknown. - * table_empty : the table does not have any entries - * output_error : failed to properly write the output file - */ -EXTERN ComputeStatus unifrac_to_file(const char* biom_filename, const char* tree_filename, const char* out_filename, - const char* unifrac_method, bool variance_adjust, double alpha, - bool bypass_tips, unsigned int threads, const char* format, - unsigned int pcoa_dims, const char *mmap_dir); - -/* Write a matrix object - * - * filename the file to write into - * result the results object - * - * The following error codes are returned: - * - * write_okay : no problems - */ -EXTERN IOStatus write_mat(const char* filename, mat_t* result); - -/* Write a matrix object using hdf5 format - * - * filename the file to write into - * result the results object - * pcoa_dims PCoAdimensions to compute, if >0 - * - * The following error codes are returned: - * - * write_okay : no problems - */ -EXTERN IOStatus write_mat_hdf5(const char* filename, mat_t* result, unsigned int pcoa_dims); - -/* Write a matrix object using hdf5 format, using fp32 precision - * - * filename the file to write into - * result the results object - * pcoa_dims PCoAdimensions to compute, if >0 - * - * The following error codes are returned: - * - * write_okay : no problems - */ -EXTERN IOStatus write_mat_hdf5_fp32(const char* filename, mat_t* result, unsigned int pcoa_dims); - -/* Write a matrix object - * - * filename the file to write into - * result the results object - * - * The following error codes are returned: - * - * write_okay : no problems - */ -EXTERN IOStatus write_mat_from_matrix(const char* filename, mat_full_fp64_t* result); - - -/* Write a matrix object from buffer using hdf5 format - * - * filename the file to write into - * result the results object - * pcoa_dims PCoAdimensions to compute, if >0 - * - * The following error codes are returned: - * - * write_okay : no problems - */ -EXTERN IOStatus write_mat_from_matrix_hdf5(const char* filename, mat_full_fp64_t* result, unsigned int pcoa_dims); - -/* Write a matrix object from buffer using hdf5 format, using fp32 precision - * - * filename the file to write into - * result the results object - * pcoa_dims PCoAdimensions to compute, if >0 - * - * The following error codes are returned: - * - * write_okay : no problems - */ -EXTERN IOStatus write_mat_from_matrix_hdf5_fp32(const char* filename, mat_full_fp32_t* result, unsigned int pcoa_dims); - -/* Write a series - * - * filename the file to write into - * result the results object - * - * The following error codes are returned: - * - * write_okay : no problems - */ -EXTERN IOStatus write_vec(const char* filename, r_vec* result); - -/* Read a matrix object - * - * filename the file to write into - * result the results object - * - * The following error codes are returned: - * - * read_okay : no problems - * open_error : could not open the file - * magic_incompatible : format magic not found or incompatible - * unexpected_end : format end not found in expected location - */ -//EXTERN IOStatus read_mat(const char* filename, mat_t** result); - -/* Compute a subset of a UniFrac distance matrix - * - * biom_filename the filename to the biom table. - * tree_filename the filename to the correspodning tree. - * unifrac_method the requested unifrac method. - * variance_adjust whether to apply variance adjustment. - * alpha GUniFrac alpha, only relevant if method == generalized. - * bypass_tips disregard tips, reduces compute by about 50% - * threads the number of threads to use. - * stripe_start the starting stripe to compute - * stripe_stop the last stripe to compute - * dm_stripes the unique branch length stripes. This is expected to be - * uninitialized, and is an output parameter. - * dm_stripes_total the total branch length stripes. This is expected to be - * uninitialized, and is an output parameter. - * result the resulting distance matrix in condensed form, this is initialized within the method so using ** - * - * partial returns the following error codes: - * - * okay : no problems encountered - * table_missing : the filename for the table does not exist - * tree_missing : the filename for the tree does not exist - * unknown_method : the requested method is unknown. - */ - -EXTERN ComputeStatus partial(const char* biom_filename, const char* tree_filename, - const char* unifrac_method, bool variance_adjust, double alpha, - bool bypass_tips, unsigned int threads, unsigned int stripe_start, - unsigned int stripe_stop, partial_mat_t** result); - -/* Write a partial matrix object - * - * filename the file to write into - * result the partial results object - * - * The following error codes are returned: - * - * write_okay : no problems - * open_error : could not open the file - * - * The structure of the binary output file is as follows. Newlines added for clarity, but are not stored. - * The file has logical blocks, but are not explicitly denoted in the format. These logical blocks are - * just used to improve readability here, and are denoted by ### marks. - * - * ### HEADER ### - * : uint16_t, the length of the magic - * : char, e.g., SSU-PARTIAL-01 - * : uint32_t, the number of samples - * : uint32_t, the number of stripes represented in this file - * : uint32_t, the starting stripe number - * : uint32_t, the total number of stripes in the full matrix - * : uint8_t, zero is false, nonzero is true - * - * ### SAMPLE IDS ### - * : uint16_t, the length of the next sample ID - * : LEN bytes, char - * ... : ... repeated - * : uint16_t, the length of the next sample ID - * : LEN bytes, char - * - * ### STRIPE VALUES; SS -> STRIPE_START, NS -> N_STRIPES - * : double, the first value in the 0th stripe - * ... : ... repeated for N_SAMPLES values - * : double, the last value in the 0th stripe - * : double, the first value in the Kth stripe - * ... : ... repeated for N_SAMPLES values - * : double, the last value in the Kth stripe - * - * ### FOOTER ### - * : char, e.g., SSU-PARTIAL-01, same as starting magic - */ -EXTERN IOStatus write_partial(const char* filename, const partial_mat_t* result); - -/* Read a partial matrix object - * - * filename the file to write into - * result the partial results object, output parameter - * - * The following error codes are returned: - * - * read_okay : no problems - * open_error : could not open the file - * magic_incompatible : format magic not found or incompatible - * bad_header : header seems malformed - * unexpected_end : format end not found in expected location - */ -EXTERN IOStatus read_partial(const char* filename, partial_mat_t** result); - -/* Read a partial matrix object header - * - * filename the file to write into - * result the partial results object, output parameter - * - * The following error codes are returned: - * - * read_okay : no problems - * open_error : could not open the file - * magic_incompatible : format magic not found or incompatible - * bad_header : header seems malformed - * unexpected_end : format end not found in expected location - */ -EXTERN IOStatus read_partial_header(const char* input_filename, partial_dyn_mat_t** result_out); - -/* Read a stripe of a partial matrix - * - * filename the file to write into - * result the partial results object - * stripe_idx relative stripe number - * - * The following error codes are returned: - * - * read_okay : no problems - * open_error : could not open the file - * magic_incompatible : format magic not found or incompatible - * bad_header : header seems malformed - * unexpected_end : format end not found in expected location - */ -EXTERN IOStatus read_partial_one_stripe(partial_dyn_mat_t* result, uint32_t stripe_idx); - -/* - * Description TBD - */ -EXTERN MergeStatus validate_partial(const partial_dyn_mat_t* const * partial_mats, int n_partials); - -/* Merge partial results - * - * results an array of partial_mat_t*, the buffers will be destroyed in the process - * n_partials number of partial mats - * merged the full matrix, output parameters, this is initialized in the method so using ** - * - * The following error codes are returned: - * - * merge_okay : no problems - * incomplete_stripe_set : not all stripes needed to create a full matrix were foun - * sample_id_consistency : samples described by stripes are inconsistent - * square_mismatch : inconsistency on denotation of square matrix - */ -EXTERN MergeStatus merge_partial(partial_mat_t** partial_mats, int n_partials, unsigned int nthreads, mat_t** result); - -/* Merge partial results - * - * partial_mats an array of partial_dyn_mat_t* - * n_partials number of partial mats - * result the full matrix, output parameters, this is initialized in the method so using ** - * - * The following error codes are returned: - * - * merge_okay : no problems - * incomplete_stripe_set : not all stripes needed to create a full matrix were foun - * sample_id_consistency : samples described by stripes are inconsistent - * square_mismatch : inconsistency on denotation of square matrix - */ -MergeStatus merge_partial_to_matrix(partial_dyn_mat_t* * partial_mats, int n_partials, mat_full_fp64_t** result); - -/* Merge partial results - * - * partial_mats an array of partial_dyn_mat_t* - * n_partials number of partial mats - * result the full matrix, output parameters, this is initialized in the method so using ** - * - * The following error codes are returned: - * - * merge_okay : no problems - * incomplete_stripe_set : not all stripes needed to create a full matrix were foun - * sample_id_consistency : samples described by stripes are inconsistent - * square_mismatch : inconsistency on denotation of square matrix - */ -MergeStatus merge_partial_to_matrix_fp32(partial_dyn_mat_t* * partial_mats, int n_partials, mat_full_fp32_t** result); - - -/* Merge partial results - * - * partial_mats an array of partial_dyn_mat_t* - * n_partials number of partial mats - * mmap_dir Where to host the mmap file - * result the full matrix, output parameters, this is initialized in the method so using ** - * - * The following error codes are returned: - * - * merge_okay : no problems - * incomplete_stripe_set : not all stripes needed to create a full matrix were foun - * sample_id_consistency : samples described by stripes are inconsistent - * square_mismatch : inconsistency on denotation of square matrix - */ -MergeStatus merge_partial_to_mmap_matrix(partial_dyn_mat_t* * partial_mats, int n_partials, const char *mmap_dir, mat_full_fp64_t** result); - -/* Merge partial results - * - * partial_mats an array of partial_dyn_mat_t* - * n_partials number of partial mats - * mmap_dir Where to host the mmap file - * result the full matrix, output parameters, this is initialized in the method so using ** - * - * The following error codes are returned: - * - * merge_okay : no problems - * incomplete_stripe_set : not all stripes needed to create a full matrix were foun - * sample_id_consistency : samples described by stripes are inconsistent - * square_mismatch : inconsistency on denotation of square matrix - */ -MergeStatus merge_partial_to_mmap_matrix_fp32(partial_dyn_mat_t* * partial_mats, int n_partials, const char *mmap_dir, mat_full_fp32_t** result); - - -// Find eigen values and vectors -// Based on N. Halko, P.G. Martinsson, Y. Shkolnisky, and M. Tygert. -// Original Paper: https://arxiv.org/abs/1007.5510 -// centered == n x n, must be symmetric, Note: will be used in-place as temp buffer -void find_eigens_fast(const uint32_t n_samples, const uint32_t n_dims, double * centered, double **eigenvalues, double **eigenvectors); -void find_eigens_fast_p32(const uint32_t n_samples, const uint32_t n_dims,float * centered, float **eigenvalues, float **eigenvectors); - -// Perform Principal Coordinate Analysis -// mat - in, result of unifrac compute -// n_samples - in, size of the matrix (n x n) -// n_dims - in, Dimensions to reduce the distance matrix to. This number determines how many eigenvectors and eigenvalues will be returned. -// eigenvalues - out, alocated buffer of size n_dims -// samples - out, alocated buffer of size n_dims x n_samples -// proportion_explained - out, allocated buffer of size n_dims -void pcoa(const double * mat, const uint32_t n_samples, const uint32_t n_dims, double **eigenvalues, double **samples, double **proportion_explained); -void pcoa_fp32(const float * mat, const uint32_t n_samples, const uint32_t n_dims, float * *eigenvalues, float * *samples, float * *proportion_explained); -void pcoa_mixed(const double * mat, const uint32_t n_samples, const uint32_t n_dims, float * *eigenvalues, float * *samples, float * *proportion_explained); - - -#ifdef __cplusplus -// TODO: only needed for testing, should be encased in a macro -void set_tasks(std::vector &tasks, - double alpha, - unsigned int n_samples, - unsigned int stripe_start, - unsigned int stripe_stop, - bool bypass_tips, - unsigned int nthreads); - -#endif diff --git a/sucpp/benchtest.sh b/sucpp/benchtest.sh deleted file mode 100644 index ab9280b1..00000000 --- a/sucpp/benchtest.sh +++ /dev/null @@ -1,17 +0,0 @@ -set -e -set -x - -basedir=bench_tables_trees -resdir=$basedir/results -mkdir -p $resdir -for f in $basedir/*.biom -do - bench=${basedir}/$(basename $f .biom) - res=${resdir}/$(basename $f .biom) - for method in {unweighted,weighted_normalized,weighted_unnormalized} - do - /usr/bin/time -l ./su ${bench}.tre ${bench}.biom $method > ${res}.${method}.su.dm 2> ${res}.${method}.su.stats - /usr/bin/time -l ./sk ${bench}.tre ${bench}.biom $method > ${res}.${method}.sk.dm 2> ${res}.${method}.sk.stats - python compare_dms.py ${res}.${method}.sk.dm ${res}.${method}.su.dm - done -done diff --git a/sucpp/biom.cpp b/sucpp/biom.cpp deleted file mode 100644 index a62a9e27..00000000 --- a/sucpp/biom.cpp +++ /dev/null @@ -1,324 +0,0 @@ -/* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ - -#include -#include -#include -#include "biom.hpp" - -using namespace H5; -using namespace su; - -/* datasets defined by the BIOM 2.x spec */ -const std::string OBS_INDPTR = std::string("/observation/matrix/indptr"); -const std::string OBS_INDICES = std::string("/observation/matrix/indices"); -const std::string OBS_DATA = std::string("/observation/matrix/data"); -const std::string OBS_IDS = std::string("/observation/ids"); - -const std::string SAMPLE_INDPTR = std::string("/sample/matrix/indptr"); -const std::string SAMPLE_INDICES = std::string("/sample/matrix/indices"); -const std::string SAMPLE_DATA = std::string("/sample/matrix/data"); -const std::string SAMPLE_IDS = std::string("/sample/ids"); - -biom::biom(std::string filename) { - file = H5File(filename.c_str(), H5F_ACC_RDONLY); - - /* establish the datasets */ - obs_indices = file.openDataSet(OBS_INDICES.c_str()); - obs_data = file.openDataSet(OBS_DATA.c_str()); - sample_indices = file.openDataSet(SAMPLE_INDICES.c_str()); - sample_data = file.openDataSet(SAMPLE_DATA.c_str()); - - /* cache IDs and indptr */ - sample_ids = std::vector(); - obs_ids = std::vector(); - sample_indptr = std::vector(); - obs_indptr = std::vector(); - - load_ids(OBS_IDS.c_str(), obs_ids); - load_ids(SAMPLE_IDS.c_str(), sample_ids); - load_indptr(OBS_INDPTR.c_str(), obs_indptr); - load_indptr(SAMPLE_INDPTR.c_str(), sample_indptr); - - /* cache shape and nnz info */ - n_samples = sample_ids.size(); - n_obs = obs_ids.size(); - set_nnz(); - - /* define a mapping between an ID and its corresponding offset */ - obs_id_index = std::unordered_map(); - sample_id_index = std::unordered_map(); - - create_id_index(obs_ids, obs_id_index); - create_id_index(sample_ids, sample_id_index); - - /* load obs sparse data */ - obs_indices_resident = (uint32_t**)malloc(sizeof(uint32_t**) * n_obs); - if(obs_indices_resident == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(uint32_t**) * n_obs, __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - obs_data_resident = (double**)malloc(sizeof(double**) * n_obs); - if(obs_data_resident == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(double**) * n_obs, __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - obs_counts_resident = (unsigned int*)malloc(sizeof(unsigned int) * n_obs); - if(obs_counts_resident == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(unsigned int) * n_obs, __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - - uint32_t *current_indices = NULL; - double *current_data = NULL; - for(unsigned int i = 0; i < obs_ids.size(); i++) { - std::string id_ = obs_ids[i]; - unsigned int n = get_obs_data_direct(id_, current_indices, current_data); - obs_counts_resident[i] = n; - obs_indices_resident[i] = current_indices; - obs_data_resident[i] = current_data; - } - sample_counts = get_sample_counts(); -} - -biom::~biom() { - for(unsigned int i = 0; i < n_obs; i++) { - free(obs_indices_resident[i]); - free(obs_data_resident[i]); - } - free(obs_indices_resident); - free(obs_data_resident); - free(obs_counts_resident); -} - -void biom::set_nnz() { - // should these be cached? - DataType dtype = obs_data.getDataType(); - DataSpace dataspace = obs_data.getSpace(); - - hsize_t dims[1]; - dataspace.getSimpleExtentDims(dims, NULL); - nnz = dims[0]; -} - -void biom::load_ids(const char *path, std::vector &ids) { - DataSet ds_ids = file.openDataSet(path); - DataType dtype = ds_ids.getDataType(); - DataSpace dataspace = ds_ids.getSpace(); - - hsize_t dims[1]; - dataspace.getSimpleExtentDims(dims, NULL); - - /* the IDs are a dataset of variable length strings */ - char **dataout = (char**)malloc(sizeof(char*) * dims[0]); - if(dataout == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(char*) * dims[0], __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - ds_ids.read((void*)dataout, dtype); - - ids.reserve(dims[0]); - for(unsigned int i = 0; i < dims[0]; i++) { - ids.push_back(dataout[i]); - } - - for(unsigned int i = 0; i < dims[0]; i++) - free(dataout[i]); - free(dataout); -} - -void biom::load_indptr(const char *path, std::vector &indptr) { - DataSet ds = file.openDataSet(path); - DataType dtype = ds.getDataType(); - DataSpace dataspace = ds.getSpace(); - - hsize_t dims[1]; - dataspace.getSimpleExtentDims(dims, NULL); - - uint32_t *dataout = (uint32_t*)malloc(sizeof(uint32_t) * dims[0]); - if(dataout == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(uint32_t) * dims[0], __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - ds.read((void*)dataout, dtype); - - indptr.reserve(dims[0]); - for(unsigned int i = 0; i < dims[0]; i++) - indptr.push_back(dataout[i]); - free(dataout); -} - -void biom::create_id_index(std::vector &ids, - std::unordered_map &map) { - uint32_t count = 0; - map.reserve(ids.size()); - for(auto i = ids.begin(); i != ids.end(); i++, count++) { - map[*i] = count; - } -} - -unsigned int biom::get_obs_data_direct(const std::string &id, uint32_t *& current_indices_out, double *& current_data_out) { - uint32_t idx = obs_id_index.at(id); - uint32_t start = obs_indptr[idx]; - uint32_t end = obs_indptr[idx + 1]; - - hsize_t count[1] = {end - start}; - hsize_t offset[1] = {start}; - - DataType indices_dtype = obs_indices.getDataType(); - DataType data_dtype = obs_data.getDataType(); - - DataSpace indices_dataspace = obs_indices.getSpace(); - DataSpace data_dataspace = obs_data.getSpace(); - - DataSpace indices_memspace(1, count, NULL); - DataSpace data_memspace(1, count, NULL); - - indices_dataspace.selectHyperslab(H5S_SELECT_SET, count, offset); - data_dataspace.selectHyperslab(H5S_SELECT_SET, count, offset); - - current_indices_out = (uint32_t*)malloc(sizeof(uint32_t) * count[0]); - if(current_indices_out == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(uint32_t) * count[0], __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - current_data_out = (double*)malloc(sizeof(double) * count[0]); - if(current_data_out == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(double) * count[0], __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - - obs_indices.read((void*)current_indices_out, indices_dtype, indices_memspace, indices_dataspace); - obs_data.read((void*)current_data_out, data_dtype, data_memspace, data_dataspace); - - return count[0]; -} - -template -void biom::get_obs_data_TT(const std::string &id, TFloat* out) const { - uint32_t idx = obs_id_index.at(id); - unsigned int count = obs_counts_resident[idx]; - const uint32_t * const indices = obs_indices_resident[idx]; - const double * const data = obs_data_resident[idx]; - - // reset our output buffer - for(unsigned int i = 0; i < n_samples; i++) - out[i] = 0.0; - - for(unsigned int i = 0; i < count; i++) { - out[indices[i]] = data[i]; - } -} - -void biom::get_obs_data(const std::string &id, double* out) const { - biom::get_obs_data_TT(id,out); -} - -void biom::get_obs_data(const std::string &id, float* out) const { - biom::get_obs_data_TT(id,out); -} - - -// note: out is supposed to be fully filled, i.e. out[start:end] -template -void biom::get_obs_data_range_TT(const std::string &id, unsigned int start, unsigned int end, bool normalize, TFloat* out) const { - uint32_t idx = obs_id_index.at(id); - unsigned int count = obs_counts_resident[idx]; - const uint32_t * const indices = obs_indices_resident[idx]; - const double * const data = obs_data_resident[idx]; - - // reset our output buffer - for(unsigned int i = start; i < end; i++) - out[i-start] = 0.0; - - if (normalize) { - for(unsigned int i = 0; i < count; i++) { - const int32_t j = indices[i]; - if ((j>=start)&&(j=start)&&(j -#include -#include -#include - -#include "biom_interface.hpp" - -namespace su { - class biom : public biom_interface { - public: - /* default constructor - * - * @param filename The path to the BIOM table to read - */ - biom(std::string filename); - - /* default destructor - * - * Temporary arrays are freed - */ - virtual ~biom(); - - /* get a dense vector of observation data - * - * @param id The observation ID to fetch - * @param out An allocated array of at least size n_samples. - * Values of an index position [0, n_samples) which do not - * have data will be zero'd. - */ - void get_obs_data(const std::string &id, double* out) const; - void get_obs_data(const std::string &id, float* out) const; - - /* get a dense vector of a range of observation data - * - * @param id The observation ID to fetc - * @param start Initial index - * @param end First index past the end - * @param normalize If set, divide by sample_counts - * @param out An allocated array of at least size (end-start). First element will corrrectpoint to index start. - * Values of an index position [0, (end-start)) which do not - * have data will be zero'd. - */ - void get_obs_data_range(const std::string &id, unsigned int start, unsigned int end, bool normalize, double* out) const; - void get_obs_data_range(const std::string &id, unsigned int start, unsigned int end, bool normalize, float* out) const; - - private: - /* retain DataSet handles within the HDF5 file */ - H5::DataSet obs_indices; - H5::DataSet sample_indices; - H5::DataSet obs_data; - H5::DataSet sample_data; - H5::H5File file; - uint32_t **obs_indices_resident; - double **obs_data_resident; - unsigned int *obs_counts_resident; - - unsigned int get_obs_data_direct(const std::string &id, uint32_t *& current_indices_out, double *& current_data_out); - unsigned int get_sample_data_direct(const std::string &id, uint32_t *& current_indices_out, double *& current_data_out); - double* get_sample_counts(); - - /* At construction, lookups mapping IDs -> index position within an - * axis are defined - */ - std::unordered_map obs_id_index; - std::unordered_map sample_id_index; - - /* load ids from an axis - * - * @param path The dataset path to the ID dataset to load - * @param ids The variable representing the IDs to load into - */ - void load_ids(const char *path, std::vector &ids); - - /* load the index pointer for an axis - * - * @param path The dataset path to the index pointer to load - * @param indptr The vector to load the data into - */ - void load_indptr(const char *path, std::vector &indptr); - - /* count the number of nonzero values and set nnz */ - void set_nnz(); - - /* create an index mapping an ID to its corresponding index - * position. - * - * @param ids A vector of IDs to index - * @param map A hash table to populate - */ - void create_id_index(std::vector &ids, - std::unordered_map &map); - - - // templatized version - template void get_obs_data_TT(const std::string &id, TFloat* out) const; - template void get_obs_data_range_TT(const std::string &id, unsigned int start, unsigned int end, bool normalize, TFloat* out) const; - }; -} - -#endif /* _UNIFRAC_BIOM_H */ - diff --git a/sucpp/biom_interface.hpp b/sucpp/biom_interface.hpp deleted file mode 100644 index bbfc80e4..00000000 --- a/sucpp/biom_interface.hpp +++ /dev/null @@ -1,72 +0,0 @@ -/* - * BSD 3-Clause License - * - * Copyright (c) 2021-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ - - -#ifndef _UNIFRAC_BIOM_INTERFACE_H -#define _UNIFRAC_BIOM_INTERFACE_H - -#include -#include - -namespace su { - class biom_interface { - public: - // cache the IDs contained within the table - std::vector sample_ids; - std::vector obs_ids; - - // cache both index pointers into both CSC and CSR representations - std::vector sample_indptr; - std::vector obs_indptr; - - uint32_t n_samples; // the number of samples - uint32_t n_obs; // the number of observations - uint32_t nnz; // the total number of nonzero entries - double *sample_counts; - - /* default constructor - * - * Automatically create the needed objects. - * All other initialization happens in children constructors. - */ - biom_interface() {} - - /* default destructor - * - * Automatically destroy the objects. - * All other cleanup must have been performed by the children constructors. - */ - virtual ~biom_interface() {} - - /* get a dense vector of observation data - * - * @param id The observation ID to fetch - * @param out An allocated array of at least size n_samples. - * Values of an index position [0, n_samples) which do not - * have data will be zero'd. - */ - virtual void get_obs_data(const std::string &id, double* out) const = 0; - virtual void get_obs_data(const std::string &id, float* out) const = 0; - - /* get a dense vector of a range of observation data - * - * @param id The observation ID to fetc - * @param start Initial index - * @param end First index past the end - * @param normalize If set, divide by sample_counts - * @param out An allocated array of at least size (end-start). First element will corrrectpoint to index start. - * Values of an index position [0, (end-start)) which do not - * have data will be zero'd. - */ - virtual void get_obs_data_range(const std::string &id, unsigned int start, unsigned int end, bool normalize, double* out) const = 0; - virtual void get_obs_data_range(const std::string &id, unsigned int start, unsigned int end, bool normalize, float* out) const = 0; - }; -} - -#endif /* _UNIFRAC_BIOOM_INTERFACE_H */ diff --git a/sucpp/capi_test.c b/sucpp/capi_test.c deleted file mode 100644 index c1981f04..00000000 --- a/sucpp/capi_test.c +++ /dev/null @@ -1,71 +0,0 @@ -#include -#include -#include -#include -#include "api.hpp" - -#ifndef bool -#define bool char -#define true 1 -#define false 0 -#endif - -void err(bool condition, const char* msg) { - if(condition) { - fprintf(stderr, "%s\n", msg); - exit(1); - } -} - -void test_su(int num_cores){ - mat_t* result = NULL; - const char* table = "test.biom"; - const char* tree = "test.tre"; - const char* method = "unweighted"; - double exp[] = {0.2, 0.57142857, 0.6, 0.5, 0.2, 0.42857143, 0.66666667, 0.6, 0.33333333, 0.71428571, 0.85714286, 0.42857143, 0.33333333, 0.4, 0.6}; - - ComputeStatus status; - status = one_off(table, tree, method, - false, 1.0, false, num_cores, &result); - - err(status != okay, "Compute failed"); - err(result == NULL, "Empty result"); - err(result->n_samples != 6, "Wrong number of samples"); - err(result->cf_size != 15, "Wrong condensed form size"); - err(!result->is_upper_triangle, "Result is not squaure"); - - for(unsigned int i = 0; i < result->cf_size; i++) - err(fabs(exp[i] - result->condensed_form[i]) > 0.00001, "Result is wrong"); - -} - -void test_faith_pd(){ - r_vec* result = NULL; - const char* table = "test.biom"; - const char* tree = "test.tre"; - double exp[] = {4, 5, 6, 3, 2, 5}; - - ComputeStatus status; - status = faith_pd_one_off(table, tree, &result); - - err(status != okay, "Compute failed"); - err(result == NULL, "Empty result"); - err(result->n_samples != 6, "Wrong number of samples"); - - for(unsigned int i = 0; i < result->n_samples; i++) - err(fabs(exp[i] - result->values[i]) > 0.00001, "Result is wrong"); - -} - -int main(int argc, char** argv) { - int num_cores = strtol(argv[1], NULL, 10); - - printf("Testing Striped UniFrac...\n"); - test_su(num_cores); - printf("Tests passed.\n"); - printf("Testing Faith's PD...\n"); - test_faith_pd(); - printf("Tests passed.\n"); - return 0; -} - diff --git a/sucpp/cmd.cpp b/sucpp/cmd.cpp deleted file mode 100644 index 4cfd3108..00000000 --- a/sucpp/cmd.cpp +++ /dev/null @@ -1 +0,0 @@ -#include "cmd.hpp" diff --git a/sucpp/cmd.hpp b/sucpp/cmd.hpp deleted file mode 100644 index 408a169e..00000000 --- a/sucpp/cmd.hpp +++ /dev/null @@ -1,32 +0,0 @@ -#include -#include -#include -#include - -class InputParser{ - /* this object was shamelessly adapted from - http://stackoverflow.com/a/868894 - */ - public: - InputParser (int &argc, char **argv){ - for (int i=1; i < argc; ++i) - this->tokens.push_back(std::string(argv[i])); - } - /// @author iain - const std::string& getCmdOption(const std::string &option) const{ - std::vector::const_iterator itr; - itr = std::find(this->tokens.begin(), this->tokens.end(), option); - if (itr != this->tokens.end() && ++itr != this->tokens.end()){ - return *itr; - } - return empty; - } - /// @author iain - bool cmdOptionExists(const std::string &option) const{ - return std::find(this->tokens.begin(), this->tokens.end(), option) - != this->tokens.end(); - } - private: - std::vector tokens; - const std::string empty; -}; diff --git a/sucpp/faithpd.cpp b/sucpp/faithpd.cpp deleted file mode 100644 index 20674965..00000000 --- a/sucpp/faithpd.cpp +++ /dev/null @@ -1,84 +0,0 @@ -#include -#include -#include -#include -#include "api.hpp" -#include "cmd.hpp" -#include "tree.hpp" -#include "biom.hpp" -#include "unifrac.hpp" - - -void usage() { - std::cout << "usage: faithpd -i -t -o " << std::endl; - std::cout << std::endl; - std::cout << " -i\t\tThe input BIOM table." << std::endl; - std::cout << " -t\t\tThe input phylogeny in newick." << std::endl; - std::cout << " -o\t\tThe output series." << std::endl; - std::cout << std::endl; - std::cout << "Citations: " << std::endl; - std::cout << " For Faith's PD, please see:" << std::endl; - std::cout << " Faith Biological Conservation 1992; DOI: 10.1016/0006-3207(92)91201-3" << std::endl; - std::cout << std::endl; - -} - -const char* compute_status_messages[7] = {"No error.", - "The tree file cannot be found.", - "The table file cannot be found.", - "The table file contains an empty table.", - "An unknown method was requested.", - "Table observation IDs are not a subset of the tree tips. This error can also be triggered if a node name contains a single quote (this is unlikely).", - "Error creating the output."}; - -void err(std::string msg) { - std::cerr << "ERROR: " << msg << std::endl << std::endl; - usage(); -} - -int faith_cli_one_off(std::string table_filename, std::string tree_filename, - std::string output_filename) { - if(output_filename.empty()) { - err("output filename missing"); - return EXIT_FAILURE; - } - - if(table_filename.empty()) { - err("table filename missing"); - return EXIT_FAILURE; - } - - if(tree_filename.empty()) { - err("tree filename missing"); - return EXIT_FAILURE; - } - - r_vec *result = NULL; - compute_status status; - status = faith_pd_one_off(table_filename.c_str(), tree_filename.c_str(), &result); - if(status != okay || result == NULL) { - fprintf(stderr, "Compute failed in faith_pd_one_off: %s\n", compute_status_messages[status]); - exit(EXIT_FAILURE); - } - - write_vec(output_filename.c_str(), result); - destroy_results_vec(&result); - - return EXIT_SUCCESS; -} - -int main(int argc, char **argv){ - InputParser input(argc, argv); - if(input.cmdOptionExists("-h") || input.cmdOptionExists("--help") || argc == 1) { - usage(); - return EXIT_SUCCESS; - } - - const std::string &table_filename = input.getCmdOption("-i"); - const std::string &tree_filename = input.getCmdOption("-t"); - const std::string &output_filename = input.getCmdOption("-o"); - - faith_cli_one_off(table_filename, tree_filename, output_filename); - - return EXIT_SUCCESS; -} diff --git a/sucpp/skbio_alt.cpp b/sucpp/skbio_alt.cpp deleted file mode 100644 index 464f6f78..00000000 --- a/sucpp/skbio_alt.cpp +++ /dev/null @@ -1,617 +0,0 @@ -/* - * Classes, methods and unction that provide skbio-like unctionality - */ - -#include "skbio_alt.hpp" -#include - -#include - -// Not using anything mkl specific, but this is what we get from Conda -#include -#include - -// Compute the E_matrix with means -// centered must be pre-allocated and same size as mat (n_samples*n_samples)...will work even if centered==mat -// row_means must be pre-allocated and n_samples in size -template -inline void E_matrix_means(const TRealIn * mat, const uint32_t n_samples, // IN - TReal * centered, TReal * row_means, TReal &global_mean) { // OUT - /* - Compute E matrix from a distance matrix and store in temp centered matrix. - - Squares and divides by -2 the input elementwise. Eq. 9.20 in - Legendre & Legendre 1998. - - Compute sum of the rows at the same time. - */ - - TReal global_sum = 0.0; - -#pragma omp parallel for shared(mat,centered,row_means) reduction(+: global_sum) - for (uint32_t row=0; row -inline void F_matrix_inplace(const TReal * __restrict__ row_means, const TReal global_mean, TReal * __restrict__ centered, const uint32_t n_samples) { - /* - Compute F matrix from E matrix. - - Centring step: for each element, the mean of the corresponding - row and column are substracted, and the mean of the whole - matrix is added. Eq. 9.21 in Legendre & Legendre 1998. - Pseudo-code: - row_means = E_matrix.mean(axis=1, keepdims=True) - col_means = Transpose(row_means) - matrix_mean = E_matrix.mean() - return E_matrix - row_means - col_means + matrix_mean - */ - - // use a tiled pattern to maximize locality of row_means -#pragma omp parallel for shared(centered,row_means) - for (uint32_t trow=0; trow -inline void mat_to_centered_T(const TRealIn * mat, const uint32_t n_samples, TReal * centered) { - - TReal global_mean; - TReal *row_means = (TReal *) malloc(uint64_t(n_samples)*sizeof(TReal)); - E_matrix_means(mat, n_samples, centered, row_means, global_mean); - F_matrix_inplace(row_means, global_mean, centered, n_samples); - free(row_means); -} - -void su::mat_to_centered(const double * mat, const uint32_t n_samples, double * centered) { - mat_to_centered_T(mat,n_samples,centered); -} - -void su::mat_to_centered(const float * mat, const uint32_t n_samples, float * centered) { - mat_to_centered_T(mat,n_samples,centered); -} - -void su::mat_to_centered(const double * mat, const uint32_t n_samples, float * centered) { - mat_to_centered_T(mat,n_samples,centered); -} - -// Matrix dot multiplication -// Expects FORTRAN-style ColOrder -// mat must be cols x rows -// other must be cols x rows (ColOrder... rows elements together) -template -inline void mat_dot_T(const TReal *mat, const TReal *other, const uint32_t rows, const uint32_t cols, TReal *out); - -template<> -inline void mat_dot_T(const double *mat, const double *other, const uint32_t rows, const uint32_t cols, double *out) -{ - cblas_dgemm(CblasColMajor,CblasNoTrans,CblasNoTrans, rows , cols, rows, 1.0, mat, rows, other, rows, 0.0, out, rows); -} - -template<> -inline void mat_dot_T(const float *mat, const float *other, const uint32_t rows, const uint32_t cols, float *out) -{ - cblas_sgemm(CblasColMajor,CblasNoTrans,CblasNoTrans, rows , cols, rows, 1.0, mat, rows, other, rows, 0.0, out, rows); -} - -// Expects FORTRAN-style ColOrder -// Based on N. Halko, P.G. Martinsson, Y. Shkolnisky, and M. Tygert. -// Original Paper: https://arxiv.org/abs/1007.5510 -// Step 1 -// centered == n x n -// randomized = k*2 x n (ColOrder... n elements together) -template -inline void centered_randomize_T(const TReal * centered, const uint32_t n_samples, const uint32_t k, TReal * randomized) { - uint64_t matrix_els = uint64_t(n_samples)*uint64_t(k); - TReal * tmp = (TReal *) malloc(matrix_els*sizeof(TReal)); - - // Form a real n x k matrix whose entries are independent, identically - // distributed Gaussian random variables of zero mean and unit variance - TReal *G = tmp; - { - std::default_random_engine generator; - std::normal_distribution distribution; - for (uint64_t i=0; i(centered,G,n_samples,k,randomized); - - // power method... single iteration.. store in 2nd part of output - // Reusing tmp buffer for intermediate storage - mat_dot_T(centered,randomized,n_samples,k,tmp); - mat_dot_T(centered,tmp,n_samples,k,randomized+matrix_els); - - free(tmp); -} - -// templated LAPACKE wrapper - -// Compute QR -// H is in,overwritten by Q on out -// H is (r x c), Q is (r x qc), with rc<=c -template -inline int qr_i_T(const uint32_t rows, const uint32_t cols, TReal *H, uint32_t &qcols); - -template<> -inline int qr_i_T(const uint32_t rows, const uint32_t cols, double *H, uint32_t &qcols) { - qcols= std::min(rows,cols); - double *tau= new double[qcols]; - int rc = LAPACKE_dgeqrf(LAPACK_COL_MAJOR, rows, cols, H, rows, tau); - if (rc==0) { - qcols= std::min(rows,cols); - rc = LAPACKE_dorgqr(LAPACK_COL_MAJOR, rows, qcols, qcols, H, rows, tau); - } - delete[] tau; - return rc; -} - -template<> -inline int qr_i_T(const uint32_t rows, const uint32_t cols, float *H, uint32_t &qcols) { - qcols= std::min(rows,cols); - float *tau= new float[qcols]; - int rc = LAPACKE_sgeqrf(LAPACK_COL_MAJOR, rows, cols, H, rows, tau); - if (rc==0) { - qcols= std::min(rows,cols); - rc = LAPACKE_sorgqr(LAPACK_COL_MAJOR, rows, qcols, qcols, H, rows, tau); - } - delete[] tau; - return rc; -} - -namespace su { - -// helper class, since QR ops are multi function -template -class QR { - public: - uint32_t rows; - uint32_t cols; - - TReal *Q; - - // will take ownership of _H - QR(const uint32_t _rows, const uint32_t _cols, TReal *_H) - : rows(_rows) - , Q(_H) - { - int rc = qr_i_T(_rows, _cols, Q, cols); - if (rc!=0) { - fprintf(stderr, "qr_i_T(_rows,_cols, H, cols) failed with %i\n", rc); - exit(1); // should never fail - } - } - - ~QR() { - free(Q); - } - - // res = mat * Q - // mat must be rows x rows - // res will be rows * cols - void qdot_r_sq(const TReal *mat, TReal *res); - - // res = Q * mat - // mat must be cols * cols - // res will be rows * cols - void qdot_l_sq(const TReal *mat, TReal *res); - -}; - -} - -template<> -inline void su::QR::qdot_r_sq(const double *mat, double *res) { - cblas_dgemm(CblasColMajor,CblasNoTrans,CblasNoTrans, rows , cols, rows, 1.0, mat, rows, Q, rows, 0.0, res, rows); -} - -template<> -inline void su::QR::qdot_r_sq(const float *mat, float *res) { - cblas_sgemm(CblasColMajor,CblasNoTrans,CblasNoTrans, rows , cols, rows, 1.0, mat, rows, Q, rows, 0.0, res, rows); -} - -template<> -inline void su::QR::qdot_l_sq(const double *mat, double *res) { - cblas_dgemm(CblasColMajor,CblasNoTrans,CblasNoTrans, rows , cols, cols, 1.0, Q, rows, mat, cols, 0.0, res, rows); -} - -template<> -inline void su::QR::qdot_l_sq(const float *mat, float *res) { - cblas_sgemm(CblasColMajor,CblasNoTrans,CblasNoTrans, rows , cols, cols, 1.0, Q, rows, mat, cols, 0.0, res, rows); -} - -// compute svd, and return S and V -// T = input -// S output -// T is Vt on output -template -inline int svd_it_T(const uint32_t rows, const uint32_t cols, TReal *T, TReal *S); - -template<> -inline int svd_it_T(const uint32_t rows, const uint32_t cols, double *T, double *S) { - double *superb = (double *) malloc(sizeof(double)*rows); - int res =LAPACKE_dgesvd(LAPACK_COL_MAJOR, 'N', 'O', rows, cols, T, rows, S, NULL, rows, NULL, cols, superb); - free(superb); - - return res; -} - -template<> -inline int svd_it_T(const uint32_t rows, const uint32_t cols, float *T, float *S) { - float *superb = (float *) malloc(sizeof(float)*rows); - int res =LAPACKE_sgesvd(LAPACK_COL_MAJOR, 'N', 'O', rows, cols, T, rows, S, NULL, rows, NULL, cols, superb); - free(superb); - - return res; -} - -// square matrix transpose, with org not alingned -template -inline void transpose_sq_st_T(const uint64_t n, const uint64_t stride, const TReal *in, TReal *out) { - // n expected to be small, so simple single-thread perfect - // org_n>=n guaranteed - for (uint64_t i=0; i -inline void transpose_T(const uint64_t rows, const uint64_t cols, const TReal *in, TReal *out) { - // To be optimizedc - for (uint64_t i=0; i -inline void find_eigens_fast_T(const uint32_t n_samples, const uint32_t n_dims, TReal * centered, TReal * &eigenvalues, TReal * &eigenvectors) { - const uint32_t k = n_dims+2; - - int rc; - - TReal *S = (TReal *) malloc(uint64_t(n_samples)*sizeof(TReal)); // take worst case size as a start - TReal *Ut = NULL; - - { - TReal *H = (TReal *) malloc(sizeof(TReal)*uint64_t(n_samples)*uint64_t(k)*2); - - // step 1 - centered_randomize_T(centered, n_samples, k, H); - - // step 2 - // QR decomposition of H - - su::QR qr_obj(n_samples, k*2, H); // H is now owned by qr_obj, as Q - - // step 3 - // T = centered * Q (since centered^T == centered, due to being symmetric) - // centered = n x n - // T = n x ref - - TReal *T = (TReal *) malloc(sizeof(TReal)*uint64_t(qr_obj.rows)*uint64_t(qr_obj.cols)); - qr_obj.qdot_r_sq(centered,T); - - // step 4 - // compute svd - // update T in-place, Wt on output (Vt according to the LAPACK nomenclature) - rc=svd_it_T(qr_obj.rows,qr_obj.cols, T, S); - if (rc!=0) { - fprintf(stderr, "svd_it_T(n_samples, T, S) failed with %i\n",rc); - exit(1); // should never fail - } - - // step 5 - // Compute U = Q*Wt^t - { - // transpose Wt -> W, Wt uses n_samples strides - TReal * W = (TReal *) malloc(sizeof(TReal)*uint64_t(qr_obj.cols)*uint64_t(qr_obj.cols)); - transpose_sq_st_T(qr_obj.cols, qr_obj.rows, T, W); // Wt == T on input - - Ut = T; // Ut takes ownership of the T buffer - qr_obj.qdot_l_sq(W, Ut); - - free(W); - } - - } // we don't need qr_obj anymore, release memory - - // step 6 - // get the interesting subset, and return - - // simply truncate the values, since it is a vector - eigenvalues = (TReal *) realloc(S, sizeof(TReal)*n_dims); - - // *eigenvectors = U = Vt - // use only the truncated part of W, then transpose - TReal *U = (TReal *) malloc(uint64_t(n_samples)*uint64_t(n_dims)*sizeof(TReal)); - - transpose_T(n_samples, n_dims, Ut, U); - eigenvectors = U; - - free(Ut); -} - -void su::find_eigens_fast(const uint32_t n_samples, const uint32_t n_dims, double * centered, double * &eigenvalues, double * &eigenvectors) { - find_eigens_fast_T(n_samples, n_dims, centered, eigenvalues, eigenvectors); -} - -void su::find_eigens_fast(const uint32_t n_samples, const uint32_t n_dims, float * centered, float * &eigenvalues, float * &eigenvectors) { - find_eigens_fast_T(n_samples, n_dims, centered, eigenvalues, eigenvectors); -} - -// helper class - -namespace su { - -template -class NewCentered { -private: - const uint32_t n_samples; - const uint32_t n_dims; - TReal * centered_buf; -public: - NewCentered(const uint32_t _n_samples, const uint32_t _n_dims) - : n_samples(_n_samples) - , n_dims(_n_dims) - , centered_buf(NULL) - {} - - TReal * get_buf() { - if (centered_buf==NULL) centered_buf = (TReal *) malloc(sizeof(TReal)*uint64_t(n_samples)*uint64_t(n_samples)); - return centered_buf; - } - - void release_buf() { - if (centered_buf!=NULL) free(centered_buf); - centered_buf=NULL; - } - - ~NewCentered() { - if (centered_buf!=NULL) release_buf(); - } - -private: - NewCentered(const NewCentered &other) = delete; - NewCentered& operator=(const NewCentered &other) = delete; -}; - -template -class InPlaceCentered { -private: - TReal * mat; -public: - InPlaceCentered(TReal * _mat) - : mat(_mat) - {} - - TReal * get_buf() { return mat; } - - void release_buf() {} - - ~InPlaceCentered() {} -}; - -} - -/* - Perform Principal Coordinate Analysis. - - Principal Coordinate Analysis (PCoA) is a method similar - to Principal Components Analysis (PCA) with the difference that PCoA - operates on distance matrices, typically with non-euclidian and thus - ecologically meaningful distances like UniFrac in microbiome research. - - In ecology, the euclidean distance preserved by Principal - Component Analysis (PCA) is often not a good choice because it - deals poorly with double zeros (Species have unimodal - distributions along environmental gradients, so if a species is - absent from two sites at the same site, it can't be known if an - environmental variable is too high in one of them and too low in - the other, or too low in both, etc. On the other hand, if an - species is present in two sites, that means that the sites are - similar.). - - Note that the returned eigenvectors are not normalized to unit length. -*/ - -// mat - in, result of unifrac compute -// inplace - in, if true, use mat as a work buffer -// n_samples - in, size of the matrix (n x n) -// n_dims - in, Dimensions to reduce the distance matrix to. This number determines how many eigenvectors and eigenvalues will be returned. -// eigenvalues - out, alocated buffer of size n_dims -// samples - out, alocated buffer of size n_dims x n_samples -// proportion_explained - out, allocated buffer of size n_dims - -template -inline void pcoa_T(TRealIn * mat, TCenter ¢er_obj, const uint32_t n_samples, const uint32_t n_dims, TReal * &eigenvalues, TReal * &samples,TReal * &proportion_explained) { - proportion_explained = (TReal *) malloc(sizeof(TReal)*n_dims); - - TReal diag_sum = 0.0; - TReal *eigenvectors = NULL; - - { - TReal *centered = center_obj.get_buf(); - - // First must center the matrix - mat_to_centered_T(mat,n_samples,centered); - - // get the sum of the diagonal, needed later - // and centered will be updated in-place in find_eigen - for (uint32_t i=0; i(n_samples,n_dims,centered,eigenvalues,eigenvectors); - - center_obj.release_buf(); - } - - // expects eigenvalues to be ordered and non-negative - // The above unction guarantees that - - - // Scale eigenvalues to have length = sqrt(eigenvalue). This - // works because np.linalg.eigh returns normalized - // eigenvectors. Each row contains the coordinates of the - // objects in the space of principal coordinates. Note that at - // least one eigenvalue is zero because only n-1 axes are - // needed to represent n points in a euclidean space. - // samples = eigvecs * np.sqrt(eigvals) - // we will just update in place and pass out - samples = eigenvectors; - - // use proportion_explained as tmp buffer here - { - TReal *sqvals = proportion_explained; - for (uint32_t i=0; i cobj(n_samples, n_dims); - pcoa_T(mat, cobj , n_samples, n_dims, eigenvalues, samples, proportion_explained); -} - -void su::pcoa(const float * mat, const uint32_t n_samples, const uint32_t n_dims, float * &eigenvalues, float * &samples, float * &proportion_explained) { - su::NewCentered cobj(n_samples, n_dims); - pcoa_T(mat, cobj, n_samples, n_dims, eigenvalues, samples, proportion_explained); -} - -void su::pcoa(const double * mat, const uint32_t n_samples, const uint32_t n_dims, float * &eigenvalues, float * &samples, float * &proportion_explained) { - su::NewCentered cobj(n_samples, n_dims); - pcoa_T(mat, cobj, n_samples, n_dims, eigenvalues, samples, proportion_explained); -} - -void su::pcoa_inplace(double * mat, const uint32_t n_samples, const uint32_t n_dims, double * &eigenvalues, double * &samples, double * &proportion_explained) { - su::InPlaceCentered cobj(mat); - pcoa_T(mat, cobj, n_samples, n_dims, eigenvalues, samples, proportion_explained); -} - -void su::pcoa_inplace(float * mat, const uint32_t n_samples, const uint32_t n_dims, float * &eigenvalues, float * &samples, float * &proportion_explained) { - su::InPlaceCentered cobj(mat); - pcoa_T(mat, cobj, n_samples, n_dims, eigenvalues, samples, proportion_explained); -} - diff --git a/sucpp/skbio_alt.hpp b/sucpp/skbio_alt.hpp deleted file mode 100644 index 284f0ea6..00000000 --- a/sucpp/skbio_alt.hpp +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Classes, methods and unction that provide skbio-like unctionality - */ - -#ifndef UNIFRAC_SKBIO_ALT_H -#define UNIFRAC_SKBIO_ALT_H - -#include - -namespace su { - -// Center the matrix -// mat and center must be nxn and symmetric -// centered must be pre-allocated and same size as mat...will work even if centered==mat -void mat_to_centered(const double * mat, const uint32_t n_samples, double * centered); -void mat_to_centered(const float * mat, const uint32_t n_samples, float * centered); -void mat_to_centered(const double * mat, const uint32_t n_samples, float * centered); - -// Find eigen values and vectors -// Based on N. Halko, P.G. Martinsson, Y. Shkolnisky, and M. Tygert. -// Original Paper: https://arxiv.org/abs/1007.5510 -// centered == n x n, must be symmetric, Note: will be used in-place as temp buffer -void find_eigens_fast(const uint32_t n_samples, const uint32_t n_dims, double * centered, double * &eigenvalues, double * &eigenvectors); -void find_eigens_fast(const uint32_t n_samples, const uint32_t n_dims, float * centered, float * &eigenvalues, float * &eigenvectors); - -// Perform Principal Coordinate Analysis -// mat - in, result of unifrac compute -// n_samples - in, size of the matrix (n x n) -// n_dims - in, Dimensions to reduce the distance matrix to. This number determines how many eigenvectors and eigenvalues will be returned. -// eigenvalues - out, alocated buffer of size n_dims -// samples - out, alocated buffer of size n_dims x n_samples -// proportion_explained - out, allocated buffer of size n_dims -void pcoa(const double * mat, const uint32_t n_samples, const uint32_t n_dims, double * &eigenvalues, double * &samples, double * &proportion_explained); -void pcoa(const float * mat, const uint32_t n_samples, const uint32_t n_dims, float * &eigenvalues, float * &samples, float * &proportion_explained); -void pcoa(const double * mat, const uint32_t n_samples, const uint32_t n_dims, float * &eigenvalues, float * &samples, float * &proportion_explained); - -// in-place version, will use mat as temp buffer internally -void pcoa_inplace(double * mat, const uint32_t n_samples, const uint32_t n_dims, double * &eigenvalues, double * &samples, double * &proportion_explained); -void pcoa_inplace(float * mat, const uint32_t n_samples, const uint32_t n_dims, float * &eigenvalues, float * &samples, float * &proportion_explained); - - -} - -#endif diff --git a/sucpp/su.cpp b/sucpp/su.cpp deleted file mode 100644 index 5640846c..00000000 --- a/sucpp/su.cpp +++ /dev/null @@ -1,538 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include "api.hpp" -#include "cmd.hpp" -#include "tree.hpp" -#include "biom.hpp" -#include "unifrac.hpp" - -enum Format {format_invalid,format_ascii, format_hdf5_fp32, format_hdf5_fp64}; - -void usage() { - std::cout << "usage: ssu -i -o -m [METHOD] -t [-n threads] [-a alpha] [-f] [--vaw]" << std::endl; - std::cout << " [--mode [MODE]] [--start starting-stripe] [--stop stopping-stripe] [--partial-pattern ]" << std::endl; - std::cout << " [--n-partials number_of_partitions] [--report-bare] [--format|-r out-mode]" << std::endl; - std::cout << std::endl; - std::cout << " -i\t\tThe input BIOM table." << std::endl; - std::cout << " -t\t\tThe input phylogeny in newick." << std::endl; - std::cout << " -m\t\tThe method, [unweighted | weighted_normalized | weighted_unnormalized | generalized | unweighted_fp32 | weighted_normalized_fp32 | weighted_unnormalized_fp32 | generalized_fp32]." << std::endl; - std::cout << " -o\t\tThe output distance matrix." << std::endl; - std::cout << " -n\t\t[OPTIONAL] The number of threads, default is 1." << std::endl; - std::cout << " -a\t\t[OPTIONAL] Generalized UniFrac alpha, default is 1." << std::endl; - std::cout << " -f\t\t[OPTIONAL] Bypass tips, reduces compute by about 50%." << std::endl; - std::cout << " --vaw\t[OPTIONAL] Variance adjusted, default is to not adjust for variance." << std::endl; - std::cout << " --mode\t[OPTIONAL] Mode of operation:" << std::endl; - std::cout << " \t\t one-off : [DEFAULT] compute UniFrac." << std::endl; - std::cout << " \t\t partial : Compute UniFrac over a subset of stripes." << std::endl; - std::cout << " \t\t partial-report : Start and stop suggestions for partial compute." << std::endl; - std::cout << " \t\t merge-partial : Merge partial UniFrac results." << std::endl; - std::cout << " \t\t check-partial : Check partial UniFrac results." << std::endl; - std::cout << " --start\t[OPTIONAL] If mode==partial, the starting stripe." << std::endl; - std::cout << " --stop\t[OPTIONAL] If mode==partial, the stopping stripe." << std::endl; - std::cout << " --partial-pattern\t[OPTIONAL] If mode==merge-partial or check-partial, a glob pattern for partial outputs to merge." << std::endl; - std::cout << " --n-partials\t[OPTIONAL] If mode==partial-report, the number of partitions to compute." << std::endl; - std::cout << " --report-bare\t[OPTIONAL] If mode==partial-report, produce barebones output." << std::endl; - std::cout << " --format|-r\t[OPTIONAL] Output format:" << std::endl; - std::cout << " \t\t ascii : [DEFAULT] Original ASCII format." << std::endl; - std::cout << " \t\t hfd5 : HFD5 format. May be fp32 or fp64, depending on method." << std::endl; - std::cout << " \t\t hdf5_fp32 : HFD5 format, using fp32 precision." << std::endl; - std::cout << " \t\t hdf5_fp64 : HFD5 format, using fp64 precision." << std::endl; - std::cout << " --pcoa\t[OPTIONAL] Number of PCoA dimensions to compute (default: 10, do not compute if 0)" << std::endl; - std::cout << " --diskbuf\t[OPTIONAL] Use a disk buffer to reduce memory footprint. Provide path to a fast partition (ideally NVMe)." << std::endl; - std::cout << std::endl; - std::cout << "Citations: " << std::endl; - std::cout << " For UniFrac, please see:" << std::endl; - std::cout << " McDonald et al. Nature Methods 2018; DOI: 10.1038/s41592-018-0187-8" << std::endl; - std::cout << " Lozupone and Knight Appl Environ Microbiol 2005; DOI: 10.1128/AEM.71.12.8228-8235.2005" << std::endl; - std::cout << " Lozupone et al. Appl Environ Microbiol 2007; DOI: 10.1128/AEM.01996-06" << std::endl; - std::cout << " Hamady et al. ISME 2010; DOI: 10.1038/ismej.2009.97" << std::endl; - std::cout << " Lozupone et al. ISME 2011; DOI: 10.1038/ismej.2010.133" << std::endl; - std::cout << " For Generalized UniFrac, please see: " << std::endl; - std::cout << " Chen et al. Bioinformatics 2012; DOI: 10.1093/bioinformatics/bts342" << std::endl; - std::cout << " For Variance Adjusted UniFrac, please see: " << std::endl; - std::cout << " Chang et al. BMC Bioinformatics 2011; DOI: 10.1186/1471-2105-12-118" << std::endl; - std::cout << std::endl; - std::cout << "Runtime progress can be obtained by issuing a SIGUSR1 signal. If running with " << std::endl; - std::cout << "multiple threads, this signal will only be honored if issued to the master PID. " << std::endl; - std::cout << "The report will yield the following information: " << std::endl; - std::cout << std::endl; - std::cout << "tid: start: stop: k: total:" << std::endl; - std::cout << std::endl; - std::cout << "The proportion of the tree that has been evaluated can be determined from (k / total)." << std::endl; - std::cout << std::endl; -} - -const char* compute_status_messages[7] = {"No error.", - "The tree file cannot be found.", - "The table file cannot be found.", - "The table file contains an empty table.", - "An unknown method was requested.", - "Table observation IDs are not a subset of the tree tips. This error can also be triggered if a node name contains a single quote (this is unlikely).", - "Error creating the output."}; - - -// https://stackoverflow.com/questions/8401777/simple-glob-in-c-on-unix-system -inline std::vector glob(const std::string& pat){ - using namespace std; - glob_t glob_result; - glob(pat.c_str(),GLOB_TILDE,NULL,&glob_result); - vector ret; - for(unsigned int i=0;i partials = glob(partial_pattern); - partial_dyn_mat_t** partial_mats = (partial_dyn_mat_t**)malloc(sizeof(partial_dyn_mat_t*) * partials.size()); - for(size_t i = 0; i < partials.size(); i++) { - IOStatus io_err = read_partial_header(partials[i].c_str(), &partial_mats[i]); - if(io_err != read_okay) { - std::ostringstream msg; - msg << "Unable to parse file (" << partials[i] << "); err " << io_err; - err(msg.str()); - return EXIT_FAILURE; - } - } - - const char * mmap_dir_c = mmap_dir.empty() ? NULL : mmap_dir.c_str(); - - int status; - if (format_val==format_hdf5_fp64) { - status = mode_merge_partial_fp64(output_filename.c_str(), format_val, pcoa_dims, partials.size(), partial_mats, mmap_dir_c); - } else if (format_val==format_hdf5_fp32) { - status = mode_merge_partial_fp32(output_filename.c_str(), format_val, pcoa_dims, partials.size(), partial_mats, mmap_dir_c); - } else { - status = mode_merge_partial_fp64(output_filename.c_str(), format_val, pcoa_dims, partials.size(), partial_mats, mmap_dir_c); - } - - for(size_t i = 0; i < partials.size(); i++) { - destroy_partial_dyn_mat(&partial_mats[i]); - } - - return status; -} - -int mode_check_partial(const std::string &partial_pattern) { - if(partial_pattern.empty()) { - std::string msg("Partial file pattern missing. For instance, if your partial results\n" \ - "are named 'ssu.unweighted.start0.partial', 'ssu.unweighted.start10.partial', \n" \ - "etc, then a pattern of 'ssu.unweighted.start*.partial' would make sense"); - err(msg); - return EXIT_FAILURE; - } - - std::vector partials = glob(partial_pattern); - std::cout << "Processing " << partials.size() << " partial files." << std::endl; - - - partial_dyn_mat_t** partial_mats = (partial_dyn_mat_t**)malloc(sizeof(partial_dyn_mat_t*) * partials.size()); - for(size_t i = 0; i < partials.size(); i++) { - IOStatus io_err = read_partial_header(partials[i].c_str(), &partial_mats[i]); - if(io_err != read_okay) { - std::ostringstream msg; - msg << "Unable to parse file (" << partials[i] << "); err " << io_err; - err(msg.str()); - return EXIT_FAILURE; - } - } - - int status = validate_partial(partial_mats,partials.size()); - - if (status==merge_okay) { - std::cout << "Partials are ready to be merged" << std::endl; - uint64_t n_samples_64 = partial_mats[0]->n_samples; - uint64_t msize32 = sizeof(float) * n_samples_64 * n_samples_64; - uint64_t msize64 = sizeof(double) * n_samples_64 * n_samples_64; - - std::cout << "n_samples = " << n_samples_64 << std::endl; - std::cout << "Matrix RAM needs: fp32 = " << msize32/(1024.0*1024*1024) << "GB, fp64 = " << msize64/(1024.0*1024*1024) << "GB"<< std::endl; - } - - for(size_t i = 0; i < partials.size(); i++) { - destroy_partial_dyn_mat(&partial_mats[i]); - } - - return status; -} - -int mode_partial(std::string table_filename, std::string tree_filename, - std::string output_filename, std::string method_string, - bool vaw, double g_unifrac_alpha, bool bypass_tips, - unsigned int nthreads, int start_stripe, int stop_stripe) { - if(output_filename.empty()) { - err("output filename missing"); - return EXIT_FAILURE; - } - - if(table_filename.empty()) { - err("table filename missing"); - return EXIT_FAILURE; - } - - if(tree_filename.empty()) { - err("tree filename missing"); - return EXIT_FAILURE; - } - - if(method_string.empty()) { - err("method missing"); - return EXIT_FAILURE; - } - - if(start_stripe < 0) { - err("Starting stripe must be >= 0"); - return EXIT_FAILURE; - } - if(stop_stripe <= start_stripe) { - err("In '--mode partial', the stop and start stripes must be specified, and the stop stripe must be > start stripe"); - return EXIT_FAILURE; - } - - partial_mat_t *result = NULL; - compute_status status; - status = partial(table_filename.c_str(), tree_filename.c_str(), method_string.c_str(), - vaw, g_unifrac_alpha, bypass_tips, nthreads, start_stripe, stop_stripe, &result); - if(status != okay || result == NULL) { - fprintf(stderr, "Compute failed in partial: %s\n", compute_status_messages[status]); - exit(EXIT_FAILURE); - } - - io_status err = write_partial(output_filename.c_str(), result); - destroy_partial_mat(&result); - - if(err != write_okay){ - fprintf(stderr, "Write failed: %s\n", err == open_error ? "could not open output" : "unknown error"); - return EXIT_FAILURE; - } - - return EXIT_SUCCESS; -} - -int mode_one_off(const std::string &table_filename, const std::string &tree_filename, - const std::string &output_filename, const std::string &format_str, Format format_val, - const std::string &method_string, unsigned int pcoa_dims, - bool vaw, double g_unifrac_alpha, bool bypass_tips, - unsigned int nthreads, const std::string &mmap_dir) { - if(output_filename.empty()) { - err("output filename missing"); - return EXIT_FAILURE; - } - - if(table_filename.empty()) { - err("table filename missing"); - return EXIT_FAILURE; - } - - if(tree_filename.empty()) { - err("tree filename missing"); - return EXIT_FAILURE; - } - - if(method_string.empty()) { - err("method missing"); - return EXIT_FAILURE; - } - - compute_status status = okay; - if (format_val==format_ascii) { - mat_t *result = NULL; - - status = one_off(table_filename.c_str(), tree_filename.c_str(), method_string.c_str(), - vaw, g_unifrac_alpha, bypass_tips, nthreads, &result); - if(status != okay || result == NULL) { - fprintf(stderr, "Compute failed in one_off: %s\n", compute_status_messages[status]); - exit(EXIT_FAILURE); - } - - IOStatus iostatus = write_mat(output_filename.c_str(), result); - destroy_mat(&result); - - if(iostatus!=write_okay) { - err("Failed to write output file."); - status = output_error; - } - - } else { - const char * mmap_dir_c = mmap_dir.empty() ? NULL : mmap_dir.c_str(); - - status = unifrac_to_file(table_filename.c_str(), tree_filename.c_str(), output_filename.c_str(), - method_string.c_str(), vaw, g_unifrac_alpha, bypass_tips, nthreads, format_str.c_str(), - pcoa_dims, mmap_dir_c); - - if (status != okay) { - fprintf(stderr, "Compute failed in one_off: %s\n", compute_status_messages[status]); - } - } - - return (status==okay) ? EXIT_SUCCESS : EXIT_FAILURE; -} - -void ssu_sig_handler(int signo) { - if (signo == SIGUSR1) { - printf("Status cannot be reported.\n"); - } -} - -Format get_format(const std::string &format_string, const std::string &method_string) { - Format format_val = format_invalid; - if (format_string.empty()) { - format_val = format_ascii; - } else if (format_string == "ascii") { - format_val = format_ascii; - } else if (format_string == "hdf5_fp32") { - format_val = format_hdf5_fp32; - } else if (format_string == "hdf5_fp64") { - format_val = format_hdf5_fp64; - } else if (format_string == "hdf5") { - if ((method_string=="unweighted_fp32") || (method_string=="weighted_normalized_fp32") || (method_string=="weighted_unnormalized_fp32") || (method_string=="generalized_fp32")) - format_val = format_hdf5_fp32; - else - format_val = format_hdf5_fp64; - } - - return format_val; -} - -int main(int argc, char **argv){ - signal(SIGUSR1, ssu_sig_handler); - InputParser input(argc, argv); - if(input.cmdOptionExists("-h") || input.cmdOptionExists("--help") || argc == 1) { - usage(); - return EXIT_SUCCESS; - } - - unsigned int nthreads; - std::string table_filename = input.getCmdOption("-i"); - std::string tree_filename = input.getCmdOption("-t"); - std::string output_filename = input.getCmdOption("-o"); - std::string method_string = input.getCmdOption("-m"); - std::string nthreads_arg = input.getCmdOption("-n"); - std::string gunifrac_arg = input.getCmdOption("-a"); - std::string mode_arg = input.getCmdOption("--mode"); - std::string start_arg = input.getCmdOption("--start"); - std::string stop_arg = input.getCmdOption("--stop"); - std::string partial_pattern = input.getCmdOption("--partial-pattern"); - std::string npartials = input.getCmdOption("--n-partials"); - std::string report_bare = input.getCmdOption("--report-bare"); - std::string format_arg = input.getCmdOption("--format"); - std::string sformat_arg = input.getCmdOption("-r"); - std::string pcoa_arg = input.getCmdOption("--pcoa"); - std::string diskbuf_arg = input.getCmdOption("--diskbuf"); - - if(nthreads_arg.empty()) { - nthreads = 1; - } else { - nthreads = atoi(nthreads_arg.c_str()); - } - - bool vaw = input.cmdOptionExists("--vaw"); - bool bare = input.cmdOptionExists("--report-bare"); - bool bypass_tips = input.cmdOptionExists("-f"); - double g_unifrac_alpha; - - if(gunifrac_arg.empty()) { - g_unifrac_alpha = 1.0; - } else { - g_unifrac_alpha = atof(gunifrac_arg.c_str()); - } - - int start_stripe; - if(start_arg.empty()) - start_stripe = 0; - else - start_stripe = atoi(start_arg.c_str()); - - int stop_stripe; - if(stop_arg.empty()) - stop_stripe = 0; - else - stop_stripe = atoi(stop_arg.c_str()); - - int n_partials; - if(npartials.empty()) - n_partials = 1; - else - n_partials = atoi(npartials.c_str()); - - if(n_partials<1) { - err("--n-partials cannot be < 1"); - return EXIT_FAILURE; - } - if(n_partials>1000000000) { - err("--n-partials cannot be > 1G"); - return EXIT_FAILURE; - } - - Format format_val = format_invalid; - if(!format_arg.empty()) { - format_val = get_format(format_arg,method_string); - } else { - format_val = get_format(sformat_arg,method_string); - format_arg=sformat_arg; // easier to use a single variable - } - if(format_val==format_invalid) { - err("Invalid format, must be one of ascii|hdf5|hdf5_fp32|hdf5_fp64"); - return EXIT_FAILURE; - } - - unsigned int pcoa_dims; - if(pcoa_arg.empty()) - pcoa_dims = 10; - else - pcoa_dims = atoi(pcoa_arg.c_str()); - - - if(mode_arg.empty() || mode_arg == "one-off") - return mode_one_off(table_filename, tree_filename, output_filename, format_arg, format_val, method_string, pcoa_dims, vaw, g_unifrac_alpha, bypass_tips, nthreads, diskbuf_arg); - else if(mode_arg == "partial") - return mode_partial(table_filename, tree_filename, output_filename, method_string, vaw, g_unifrac_alpha, bypass_tips, nthreads, start_stripe, stop_stripe); - else if(mode_arg == "merge-partial") - return mode_merge_partial(output_filename, format_val, pcoa_dims, partial_pattern, diskbuf_arg); - else if(mode_arg == "check-partial") - return mode_check_partial(partial_pattern); - else if(mode_arg == "partial-report") - return mode_partial_report(table_filename, uint32_t(n_partials), bare); - else - err("Unknown mode. Valid options are: one-off, partial, merge-partial, check-partial, partial-report"); - - return EXIT_SUCCESS; -} - diff --git a/sucpp/su_R.cpp b/sucpp/su_R.cpp deleted file mode 100644 index 0e2b595e..00000000 --- a/sucpp/su_R.cpp +++ /dev/null @@ -1,44 +0,0 @@ -#include -#include -#include -#include "api.hpp" - -using namespace std; -using namespace Rcpp; - - -// [[Rcpp::export]] -Rcpp::List unifrac(const char* table, const char* tree, int nthreads){ - mat_t* result = NULL; - const char* method = "unweighted"; - ComputeStatus status; - status = one_off(table, tree, method, false, 1.0, false, nthreads, &result); - vector cf; - //push result->condensed_form into a vector becuase R doesn't like double* - for(int i=0; icf_size; i++){ - cf.push_back(result->condensed_form[i]); - } - - return Rcpp::List::create(Rcpp::Named("n_samples") = result->n_samples, - Rcpp::Named("is_upper_triangle") = result->is_upper_triangle, - Rcpp::Named("cf_size") = result->cf_size, - Rcpp::Named("c_form") = cf); - -} - -// [[Rcpp::export]] -Rcpp::List faith_pd(const char* table, const char* tree){ - r_vec* result = NULL; - ComputeStatus status; - status = faith_pd_one_off(table, tree, &result); - vector values; - for(int i = 0; i < result->n_samples; i++){ - values.push_back(result->values[i]); - } - - return Rcpp::List::create(Rcpp::Named("n_samples") = result->n_samples, - Rcpp::Named("faith_pd") = values); - -} - - diff --git a/sucpp/task_parameters.hpp b/sucpp/task_parameters.hpp deleted file mode 100644 index 8f3d5327..00000000 --- a/sucpp/task_parameters.hpp +++ /dev/null @@ -1,35 +0,0 @@ -#include -#include - -#ifndef __su_task_parameters - #ifdef __cplusplus - namespace su { - #endif - - /* task specific compute parameters - * - * n_samples the number of samples being processed - * start the first stride to process - * stop the last stride to process - * tid the thread identifier - * bypass_tips ignore tips on compute, reduces compute by ~50% - * g_unifrac_alpha an alpha value for generalized unifrac - */ - struct task_parameters { - uint32_t n_samples; // number of samples - unsigned int start; // starting stripe - unsigned int stop; // stopping stripe - unsigned int tid; // thread ID - bool bypass_tips; // avoid compute at tips - - // task specific arguments below - double g_unifrac_alpha; // generalized unifrac alpha - }; - - #ifdef __cplusplus - } - #endif - -#define __su_task_parameters -#endif - diff --git a/sucpp/test.biom b/sucpp/test.biom deleted file mode 100644 index b3c019bf..00000000 Binary files a/sucpp/test.biom and /dev/null differ diff --git a/sucpp/test.tre b/sucpp/test.tre deleted file mode 100644 index 1ba14a5b..00000000 --- a/sucpp/test.tre +++ /dev/null @@ -1 +0,0 @@ -(GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1):1,(GG_OTU_5:1,GG_OTU_4:1):1); diff --git a/sucpp/test_api.cpp b/sucpp/test_api.cpp deleted file mode 100644 index ac87716c..00000000 --- a/sucpp/test_api.cpp +++ /dev/null @@ -1,743 +0,0 @@ -#include -#include "api.hpp" -#include -#include -#include -#include -#include -#include -#include -#include - -/* - * test harness adapted from - * https://github.com/noporpoise/BitArray/blob/master/dev/bit_array_test.c - */ -const char *suite_name; -char suite_pass; -int suites_run = 0, suites_failed = 0, suites_empty = 0; -int tests_in_suite = 0, tests_run = 0, tests_failed = 0; - -#define QUOTE(str) #str -#define ASSERT(x) {tests_run++; tests_in_suite++; if(!(x)) \ - { fprintf(stderr, "failed assert [%s:%i] %s\n", __FILE__, __LINE__, QUOTE(x)); \ - suite_pass = 0; tests_failed++; }} - -void SUITE_START(const char *name) { - suite_pass = 1; - suite_name = name; - suites_run++; - tests_in_suite = 0; -} - -void SUITE_END() { - printf("Testing %s ", suite_name); - size_t suite_i; - for(suite_i = strlen(suite_name); suite_i < 80-8-5; suite_i++) printf("."); - printf("%s\n", suite_pass ? " pass" : " fail"); - if(!suite_pass) suites_failed++; - if(!tests_in_suite) suites_empty++; -} -/* - * End adapted code - */ - - -//void test_write_mat() { -// SUITE_START("test write mat_t"); -// SUITE_END(); -//} -// -//void test_read_mat() { -// SUITE_START("test read mat_t"); -// SUITE_END(); -//} -// - -template -void fill_test_pm(TMat* pm, int case_id) { - pm->n_samples = 6; - pm->sample_ids = (char**)malloc(sizeof(char*) * 6); - pm->sample_ids[0] = (char*)malloc(sizeof(char) * 2); - pm->sample_ids[0][0] = 'A'; pm->sample_ids[0][1] = '\0'; - pm->sample_ids[1] = (char*)malloc(sizeof(char) * 2); - pm->sample_ids[1][0] = 'B'; pm->sample_ids[1][1] = '\0'; - pm->sample_ids[2] = (char*)malloc(sizeof(char) * 3); - pm->sample_ids[2][0] = 'C'; pm->sample_ids[2][1] = 'x'; pm->sample_ids[2][2] = '\0'; - pm->sample_ids[3] = (char*)malloc(sizeof(char) * 2); - pm->sample_ids[3][0] = 'D'; pm->sample_ids[3][1] = '\0'; - pm->sample_ids[4] = (char*)malloc(sizeof(char) * 2); - pm->sample_ids[4][0] = 'E'; pm->sample_ids[4][1] = '\0'; - pm->sample_ids[5] = (char*)malloc(sizeof(char) * 2); - pm->sample_ids[5][0] = 'F'; pm->sample_ids[5][1] = '\0'; - - if (case_id==0) { - pm->stripe_start = 0; - pm->stripe_stop = 3; - pm->stripe_total = 3; - pm->stripes = (TReal**)malloc(sizeof(TReal*) * 3); - pm->stripes[0] = (TReal*)malloc(sizeof(TReal) * 6); - pm->stripes[0][0] = 1; pm->stripes[0][1] = 2; pm->stripes[0][2] = 3; pm->stripes[0][3] = 4; pm->stripes[0][4] = 5; pm->stripes[0][5] = 6; - pm->stripes[1] = (TReal*)malloc(sizeof(TReal) * 6); - pm->stripes[1][0] = 7; pm->stripes[1][1] = 8; pm->stripes[1][2] = 9; pm->stripes[1][3] = 10; pm->stripes[1][4] = 11; pm->stripes[1][5] = 12; - pm->stripes[2] = (TReal*)malloc(sizeof(TReal) * 6); - pm->stripes[2][0] = 13; pm->stripes[2][1] = 14; pm->stripes[2][2] = 15; pm->stripes[2][3] = 16; pm->stripes[2][4] = 17; pm->stripes[2][5] = 18; - } else if (case_id==1) { - pm->stripe_start = 0; - pm->stripe_stop = 2; - pm->stripe_total = 3; - pm->stripes = (TReal**)malloc(sizeof(TReal*) * 2); - pm->stripes[0] = (TReal*)malloc(sizeof(TReal) * 6); - pm->stripes[0][0] = 1; pm->stripes[0][1] = 2; pm->stripes[0][2] = 3; pm->stripes[0][3] = 4; pm->stripes[0][4] = 5; pm->stripes[0][5] = 6; - pm->stripes[1] = (TReal*)malloc(sizeof(TReal) * 6); - pm->stripes[1][0] = 7; pm->stripes[1][1] = 8; pm->stripes[1][2] = 9; pm->stripes[1][3] = 10; pm->stripes[1][4] = 11; pm->stripes[1][5] = 12; - } else { // assume 2 - pm->stripe_start = 2; - pm->stripe_stop = 3; - pm->stripe_total = 3; - pm->stripes = (TReal**)malloc(sizeof(TReal*) * 1); - pm->stripes[0] = (TReal*)malloc(sizeof(TReal) * 6); - pm->stripes[0][0] = 16; pm->stripes[0][1] = 17; pm->stripes[0][2] = 18; pm->stripes[0][3] = 16; pm->stripes[0][4] = 17; pm->stripes[0][5] = 18; - } - pm->is_upper_triangle = true; -} - -partial_mat_t* make_test_pm(int case_id) { - partial_mat_t* pm = (partial_mat_t*)malloc(sizeof(partial_mat_t)); - - fill_test_pm(pm,case_id); - return pm; -} - -partial_dyn_mat_t* make_test_pdm(int case_id) { - partial_dyn_mat_t* pm = (partial_dyn_mat_t*)malloc(sizeof(partial_dyn_mat_t)); - fill_test_pm(pm,case_id); - pm->offsets = (uint64_t*)calloc(pm->stripe_stop-pm->stripe_start,sizeof(uint64_t)); - pm->filename = strdup("dummy"); - - return pm; -} - -mat_t* mat_three_rep() { - mat_t* res = (mat_t*)malloc(sizeof(mat_t)); - res->n_samples = 6; - res->cf_size = 15; - res->is_upper_triangle = true; - res->condensed_form = (double*)malloc(sizeof(double) * 15); - // using second half of third stripe. the last stripe when operating on even numbers of samples is normally redundant with the first half, - // but that was more annoying in to write up in the tests. - res->condensed_form[0] = 1; res->condensed_form[1] = 7; res->condensed_form[2] = 16; res->condensed_form[3] = 11; res->condensed_form[4] = 6; - res->condensed_form[5] = 2; res->condensed_form[6] = 8; res->condensed_form[7] = 17; res->condensed_form[8] = 12; - res->condensed_form[9] = 3; res->condensed_form[10] = 9; res->condensed_form[11] = 18; - res->condensed_form[12] = 4; res->condensed_form[13] = 10; - res->condensed_form[14] = 5; - res->sample_ids = (char**)malloc(sizeof(char*) * 6); - res->sample_ids[0] = (char*)malloc(sizeof(char) * 2); - res->sample_ids[0][0] = 'A'; res->sample_ids[0][1] = '\0'; - res->sample_ids[1] = (char*)malloc(sizeof(char) * 2); - res->sample_ids[1][0] = 'B'; res->sample_ids[1][1] = '\0'; - res->sample_ids[2] = (char*)malloc(sizeof(char) * 3); - res->sample_ids[2][0] = 'C'; res->sample_ids[2][1] = 'x'; res->sample_ids[2][2] = '\0'; - res->sample_ids[3] = (char*)malloc(sizeof(char) * 2); - res->sample_ids[3][0] = 'D'; res->sample_ids[3][1] = '\0'; - res->sample_ids[4] = (char*)malloc(sizeof(char) * 2); - res->sample_ids[4][0] = 'E'; res->sample_ids[4][1] = '\0'; - res->sample_ids[5] = (char*)malloc(sizeof(char) * 2); - res->sample_ids[5][0] = 'F'; res->sample_ids[5][1] = '\0'; - - return res; -} - -template -TMat* mat_full_three_rep() { - TMat* res = (TMat*)malloc(sizeof(TMat)); - res->n_samples = 6; - res->flags=0; - res->matrix = (TReal*)malloc(sizeof(TReal) * 36); - TReal * m=res->matrix ; - m[ 0] = 0; m[ 1] = 1; m[ 2] = 7; m[ 3] = 16; m[ 4] = 11; m[ 5] = 6; - m[ 6] = 1; m[ 7] = 0; m[ 8] = 2; m[ 9] = 8; m[10] = 17; m[11] = 12; - m[12] = 7; m[13] = 2; m[14] = 0; m[15] = 3; m[16] = 9; m[17] = 18; - m[18] = 16; m[19] = 8; m[20] = 3; m[21] = 0; m[22] = 4; m[23] = 10; - m[24] = 11; m[25] = 17; m[26] = 9; m[27] = 4; m[28] = 0; m[29] = 5; - m[30] = 6; m[31] = 12; m[32] = 18; m[33] = 10; m[34] = 5; m[35] = 0; - - res->sample_ids = (char**)malloc(sizeof(char*) * 6); - res->sample_ids[0] = (char*)malloc(sizeof(char) * 2); - res->sample_ids[0][0] = 'A'; res->sample_ids[0][1] = '\0'; - res->sample_ids[1] = (char*)malloc(sizeof(char) * 2); - res->sample_ids[1][0] = 'B'; res->sample_ids[1][1] = '\0'; - res->sample_ids[2] = (char*)malloc(sizeof(char) * 3); - res->sample_ids[2][0] = 'C'; res->sample_ids[2][1] = 'x'; res->sample_ids[2][2] = '\0'; - res->sample_ids[3] = (char*)malloc(sizeof(char) * 2); - res->sample_ids[3][0] = 'D'; res->sample_ids[3][1] = '\0'; - res->sample_ids[4] = (char*)malloc(sizeof(char) * 2); - res->sample_ids[4][0] = 'E'; res->sample_ids[4][1] = '\0'; - res->sample_ids[5] = (char*)malloc(sizeof(char) * 2); - res->sample_ids[5][0] = 'F'; res->sample_ids[5][1] = '\0'; - - return res; -} - -void test_read_write_partial_mat() { - SUITE_START("test read/write partial_mat_t"); - - partial_mat_t* pm = make_test_pm(0); - - io_status err = write_partial("/tmp/ssu_io.dat", pm); - ASSERT(err == write_okay); - - { - partial_mat_t *obs = NULL; - err = read_partial("/tmp/ssu_io.dat", &obs); - - ASSERT(err == read_okay); - ASSERT(obs->n_samples == 6); - ASSERT(obs->stripe_start == 0); - ASSERT(obs->stripe_stop == 3); - ASSERT(obs->stripe_total == 3); - ASSERT(strcmp(obs->sample_ids[0], "A") == 0); - ASSERT(strcmp(obs->sample_ids[1], "B") == 0); - ASSERT(strcmp(obs->sample_ids[2], "Cx") == 0); - ASSERT(strcmp(obs->sample_ids[3], "D") == 0); - ASSERT(strcmp(obs->sample_ids[4], "E") == 0); - ASSERT(strcmp(obs->sample_ids[5], "F") == 0); - - for(int i = 0; i < 3; i++) { - for(int j = 0; j < 6; j++) { - ASSERT(obs->stripes[i][j] == ((i * 6) + j + 1)); - } - } - - destroy_partial_mat(&obs); - } - - { - partial_dyn_mat_t *obs = NULL; - err = read_partial_header("/tmp/ssu_io.dat", &obs); - - ASSERT(err == read_okay); - ASSERT(obs->n_samples == 6); - ASSERT(obs->stripe_start == 0); - ASSERT(obs->stripe_stop == 3); - ASSERT(obs->stripe_total == 3); - ASSERT(strcmp(obs->sample_ids[0], "A") == 0); - ASSERT(strcmp(obs->sample_ids[1], "B") == 0); - ASSERT(strcmp(obs->sample_ids[2], "Cx") == 0); - ASSERT(strcmp(obs->sample_ids[3], "D") == 0); - ASSERT(strcmp(obs->sample_ids[4], "E") == 0); - ASSERT(strcmp(obs->sample_ids[5], "F") == 0); - - for(int i = 0; i < 3; i++) { - ASSERT(obs->stripes[i]==NULL); - } - - err = read_partial_one_stripe(obs,1); - ASSERT(err == read_okay); - - ASSERT(obs->stripes[0]==NULL); - ASSERT(obs->stripes[1]!=NULL); - ASSERT(obs->stripes[2]==NULL); - - { - const int i = 1; - for(int j = 0; j < 6; j++) { - ASSERT(obs->stripes[i][j] == ((i * 6) + j + 1)); - } - } - - err = read_partial_one_stripe(obs,0); - ASSERT(err == read_okay); - - ASSERT(obs->stripes[0]!=NULL); - ASSERT(obs->stripes[1]!=NULL); - ASSERT(obs->stripes[2]==NULL); - - { - const int i = 0; - for(int j = 0; j < 6; j++) { - ASSERT(obs->stripes[i][j] == ((i * 6) + j + 1)); - } - } - - err = read_partial_one_stripe(obs,2); - ASSERT(err == read_okay); - - - for(int i = 0; i < 3; i++) { - ASSERT(obs->stripes[i]!=NULL); - for(int j = 0; j < 6; j++) { - ASSERT(obs->stripes[i][j] == ((i * 6) + j + 1)); - } - } - - destroy_partial_dyn_mat(&obs); - } - - unlink("/tmp/ssu_io.dat"); - - SUITE_END(); -} - -void test_merge_partial_mat() { - SUITE_START("test merge partial_mat_t"); - - // the easy test - partial_mat_t* pm1 = make_test_pm(1); - partial_mat_t* pm2 = make_test_pm(2); - - mat_t* exp = mat_three_rep(); - - partial_mat_t* pms[2]; - pms[0] = pm1; - pms[1] = pm2; - - mat_t* obs = NULL; - merge_status err = merge_partial(pms, 2, 1, &obs); - ASSERT(err == merge_okay); - ASSERT(obs->cf_size == exp->cf_size); - ASSERT(obs->n_samples == exp->n_samples); - ASSERT(obs->is_upper_triangle == exp->is_upper_triangle); - for(unsigned int i = 0; i < obs->cf_size; i++) { - ASSERT(obs->condensed_form[i] == exp->condensed_form[i]); - } - for(unsigned int i = 0; i < obs->n_samples; i++) - ASSERT(strcmp(obs->sample_ids[i], exp->sample_ids[i]) == 0); - - // out of order test - - pms[0] = pm2; - pms[1] = pm1; - - obs = NULL; - err = merge_partial(pms, 2, 1, &obs); - ASSERT(err == merge_okay); - ASSERT(obs->cf_size == exp->cf_size); - ASSERT(obs->n_samples == exp->n_samples); - ASSERT(obs->is_upper_triangle == exp->is_upper_triangle); - for(unsigned int i = 0; i < obs->cf_size; i++) { - ASSERT(obs->condensed_form[i] == exp->condensed_form[i]); - } - for(unsigned int i = 0; i < obs->n_samples; i++) - ASSERT(strcmp(obs->sample_ids[i], exp->sample_ids[i]) == 0); - - // error checking - pm1->stripe_start = 0; - pm1->stripe_stop = 3; - pm1->stripe_total = 9; - pm1->is_upper_triangle = true; - - pm2->stripe_start = 3; - pm2->stripe_stop = 5; - pm2->stripe_total = 9; - pm2->is_upper_triangle = true; - - partial_mat_t* pm3 = make_test_pm(0); - pm3->stripe_start = 6; - pm3->stripe_stop = 9; - pm3->stripe_total = 9; - - partial_mat_t* pms_err[3]; - - pms_err[2] = pm1; - pms_err[0] = pm2; - pms_err[1] = pm3; - - err = merge_partial(pms_err, 3, 1, &obs); - ASSERT(err == incomplete_stripe_set); - - pm2->stripe_start = 2; - pm2->stripe_stop = 6; - err = merge_partial(pms_err, 3, 1, &obs); - ASSERT(err == stripes_overlap); - - pm2->stripe_start = 3; - pm2->sample_ids[2][0] = 'X'; - err = merge_partial(pms_err, 3, 1, &obs); - ASSERT(err == sample_id_consistency); - - pm2->sample_ids[2][0] = 'C'; - pm3->n_samples = 2; - err = merge_partial(pms_err, 3, 1, &obs); - ASSERT(err == partials_mismatch); - - pm3->n_samples = 6; - pm3->stripe_total = 12; - err = merge_partial(pms_err, 3, 1, &obs); - ASSERT(err == partials_mismatch); - - pm3->is_upper_triangle = false; - pm3->stripe_total = 9; - err = merge_partial(pms_err, 3, 1, &obs); - ASSERT(err == square_mismatch); - - SUITE_END(); -} - -void test_merge_partial_dyn_mat() { - SUITE_START("test merge partial_dyn_mat_t"); - - // the easy test - partial_dyn_mat_t* pm1 = make_test_pdm(1); - partial_dyn_mat_t* pm2 = make_test_pdm(2); - - mat_full_fp64_t* exp = mat_full_three_rep(); - - partial_dyn_mat_t* pms[2]; - pms[0] = pm1; - pms[1] = pm2; - - mat_full_fp64_t* obs = NULL; - merge_status err = merge_partial_to_matrix(pms, 2, &obs); - ASSERT(err == merge_okay); - ASSERT(obs->n_samples == exp->n_samples); - for(unsigned int i = 0; i < (obs->n_samples*obs->n_samples); i++) { - ASSERT(obs->matrix[i] == exp->matrix[i]); - } - for(unsigned int i = 0; i < obs->n_samples; i++) { - ASSERT(strcmp(obs->sample_ids[i], exp->sample_ids[i]) == 0); - } - // out of order test - - // recreate deallocated stripes - ASSERT(pm1->stripes[0]==NULL); - ASSERT(pm1->stripes[1]==NULL); - ASSERT(pm2->stripes[0]==NULL); - destroy_partial_dyn_mat(&pm1); - destroy_partial_dyn_mat(&pm2); - - - pm1 = make_test_pdm(1); - pm2 = make_test_pdm(2); - - pms[0] = pm2; - pms[1] = pm1; - - mat_full_fp32_t* exp2 = mat_full_three_rep(); - mat_full_fp32_t *obs2 = NULL; - err = merge_partial_to_matrix_fp32(pms, 2, &obs2); - ASSERT(err == merge_okay); - ASSERT(obs2->n_samples == exp2->n_samples); - for(unsigned int i = 0; i < (obs2->n_samples*obs2->n_samples); i++) { - ASSERT(obs2->matrix[i] == exp2->matrix[i]); - } - for(unsigned int i = 0; i < obs2->n_samples; i++) - ASSERT(strcmp(obs2->sample_ids[i], exp2->sample_ids[i]) == 0); - - - ASSERT(pm2->stripes[0]==NULL); - ASSERT(pm1->stripes[0]==NULL); - ASSERT(pm1->stripes[1]==NULL); - destroy_partial_dyn_mat(&pm1); - destroy_partial_dyn_mat(&pm2); - - - pm1 = make_test_pdm(1); - pm2 = make_test_pdm(2); - - - // error checking - pm1->stripe_start = 0; - pm1->stripe_stop = 3; - pm1->stripe_total = 9; - pm1->is_upper_triangle = true; - - pm2->stripe_start = 3; - pm2->stripe_stop = 5; - pm2->stripe_total = 9; - pm2->is_upper_triangle = true; - - partial_dyn_mat_t* pm3 = make_test_pdm(0); - pm3->stripe_start = 6; - pm3->stripe_stop = 9; - pm3->stripe_total = 9; - - partial_dyn_mat_t* pms_err[3]; - - pms_err[2] = pm1; - pms_err[0] = pm2; - pms_err[1] = pm3; - - err = merge_partial_to_matrix(pms_err, 3, &obs); - ASSERT(err == incomplete_stripe_set); - - pm2->stripe_start = 2; - pm2->stripe_stop = 6; - err = merge_partial_to_matrix(pms_err, 3, &obs); - ASSERT(err == stripes_overlap); - - pm2->stripe_start = 3; - pm2->sample_ids[2][0] = 'X'; - err = merge_partial_to_matrix(pms_err, 3, &obs); - ASSERT(err == sample_id_consistency); - - pm2->sample_ids[2][0] = 'C'; - pm3->n_samples = 2; - err = merge_partial_to_matrix(pms_err, 3, &obs); - ASSERT(err == partials_mismatch); - - pm3->n_samples = 6; - pm3->stripe_total = 12; - err = merge_partial_to_matrix(pms_err, 3, &obs); - ASSERT(err == partials_mismatch); - - /* - * Disable for now... not dealing properly with is_upper_triangle == false - - pm3->is_upper_triangle = false; - pm3->stripe_total = 9; - err = merge_partial_to_matrix(pms_err, 3, &obs); - ASSERT(err == square_mismatch); - */ - - destroy_mat_full_fp64(&obs); - // note, we cannot cleanly destroy the partial_dyn_mat_t structures that have been hacked by hand - - SUITE_END(); -} - -void test_merge_partial_io() { - SUITE_START("test merge partial_io"); - - // the easy test - partial_mat_t* s1 = make_test_pm(1); - partial_mat_t* s2 = make_test_pm(2); - - io_status ierr; - - ierr = write_partial("/tmp/ssu_io_1.dat", s1); - ASSERT(ierr == write_okay); - - ierr = write_partial("/tmp/ssu_io_2.dat", s2); - ASSERT(ierr == write_okay); - - partial_dyn_mat_t* pm1 = NULL; - partial_dyn_mat_t* pm2 = NULL; - - ierr = read_partial_header("/tmp/ssu_io_1.dat", &pm1); - ASSERT(ierr == read_okay); - - ierr = read_partial_header("/tmp/ssu_io_2.dat", &pm2); - ASSERT(ierr == read_okay); - - mat_full_fp64_t* exp = mat_full_three_rep(); - - partial_dyn_mat_t* pms[2]; - pms[0] = pm1; - pms[1] = pm2; - - mat_full_fp64_t* obs = NULL; - merge_status err = merge_partial_to_matrix(pms, 2, &obs); - ASSERT(err == merge_okay); - ASSERT(obs->n_samples == exp->n_samples); - for(unsigned int i = 0; i < (obs->n_samples*obs->n_samples); i++) { - ASSERT(obs->matrix[i] == exp->matrix[i]); - } - for(unsigned int i = 0; i < obs->n_samples; i++) { - ASSERT(strcmp(obs->sample_ids[i], exp->sample_ids[i]) == 0); - } - ASSERT(pm1->stripes[0]==NULL); - ASSERT(pm1->stripes[1]==NULL); - ASSERT(pm2->stripes[0]==NULL); - - destroy_mat_full_fp64(&obs); - destroy_partial_dyn_mat(&pm1); - destroy_partial_dyn_mat(&pm2); - - // out of order test - partial_dyn_mat_t* pm1b = NULL; - partial_dyn_mat_t* pm2b = NULL; - - ierr = read_partial_header("/tmp/ssu_io_1.dat", &pm1b); - ASSERT(ierr == read_okay); - - ierr = read_partial_header("/tmp/ssu_io_2.dat", &pm2b); - ASSERT(ierr == read_okay); - - pms[0] = pm2b; - pms[1] = pm1b; - - mat_full_fp32_t* exp2 = mat_full_three_rep(); - mat_full_fp32_t *obs2 = NULL; - err = merge_partial_to_matrix_fp32(pms, 2, &obs2); - ASSERT(err == merge_okay); - ASSERT(obs2->n_samples == exp2->n_samples); - for(unsigned int i = 0; i < (obs2->n_samples*obs2->n_samples); i++) { - ASSERT(obs2->matrix[i] == exp2->matrix[i]); - } - for(unsigned int i = 0; i < obs2->n_samples; i++) - ASSERT(strcmp(obs2->sample_ids[i], exp2->sample_ids[i]) == 0); - - - ASSERT(pm2b->stripes[0]==NULL); - ASSERT(pm1b->stripes[0]==NULL); - ASSERT(pm1b->stripes[1]==NULL); - - destroy_mat_full_fp32(&obs2); - destroy_partial_dyn_mat(&pm1b); - destroy_partial_dyn_mat(&pm2b); - - unlink("/tmp/ssu_io_1.dat"); - unlink("/tmp/ssu_io_2.dat"); - - SUITE_END(); -} - -void test_merge_partial_mmap() { - SUITE_START("test merge partial_mmap"); - - // the easy test - partial_mat_t* s1 = make_test_pm(1); - partial_mat_t* s2 = make_test_pm(2); - - io_status ierr; - - ierr = write_partial("/tmp/ssu_io_1.dat", s1); - ASSERT(ierr == write_okay); - - ierr = write_partial("/tmp/ssu_io_2.dat", s2); - ASSERT(ierr == write_okay); - - partial_dyn_mat_t* pm1 = NULL; - partial_dyn_mat_t* pm2 = NULL; - - ierr = read_partial_header("/tmp/ssu_io_1.dat", &pm1); - ASSERT(ierr == read_okay); - - ierr = read_partial_header("/tmp/ssu_io_2.dat", &pm2); - ASSERT(ierr == read_okay); - - mat_full_fp64_t* exp = mat_full_three_rep(); - - partial_dyn_mat_t* pms[2]; - pms[0] = pm1; - pms[1] = pm2; - - mat_full_fp32_t* obs = NULL; - merge_status err = merge_partial_to_mmap_matrix_fp32(pms, 2, "/tmp", &obs); - ASSERT(err == merge_okay); - ASSERT(obs->n_samples == exp->n_samples); - ASSERT(obs->flags != 0); - for(unsigned int i = 0; i < (obs->n_samples*obs->n_samples); i++) { - ASSERT(obs->matrix[i] == exp->matrix[i]); - } - for(unsigned int i = 0; i < obs->n_samples; i++) { - ASSERT(strcmp(obs->sample_ids[i], exp->sample_ids[i]) == 0); - } - ASSERT(pm1->stripes[0]==NULL); - ASSERT(pm1->stripes[1]==NULL); - ASSERT(pm2->stripes[0]==NULL); - - destroy_mat_full_fp32(&obs); - destroy_partial_dyn_mat(&pm1); - destroy_partial_dyn_mat(&pm2); - - // test failure due to FS problems - - ierr = read_partial_header("/tmp/ssu_io_1.dat", &pm1); - ASSERT(ierr == read_okay); - - ierr = read_partial_header("/tmp/ssu_io_2.dat", &pm2); - ASSERT(ierr == read_okay); - - pms[0] = pm1; - pms[1] = pm2; - - - err = merge_partial_to_mmap_matrix_fp32(pms, 2, "/santa/goes/skiing", &obs); - ASSERT(err != merge_okay); - destroy_partial_dyn_mat(&pm1); - destroy_partial_dyn_mat(&pm2); - - destroy_partial_mat(&s1); - destroy_partial_mat(&s2); - - unlink("/tmp/ssu_io_1.dat"); - unlink("/tmp/ssu_io_2.dat"); - - - SUITE_END(); -} - -void test_to_file_one(const char *method) { - - static const char h5name[]="/tmp/ssu_t1.h5"; - struct stat sbuf; - - ComputeStatus urc; - int frc; - - // ensure file does not already exist - frc=stat(h5name,&sbuf); - if (frc == 0) { - unlink(h5name); - frc=stat(h5name,&sbuf); - } - ASSERT(frc != 0); - - urc=unifrac_to_file("test.biom","test.tre",h5name,method,false,1.0,false,1,"hdf5",0,NULL); - ASSERT(urc == okay); - - // first, we check it does exist - frc=stat(h5name,&sbuf); - ASSERT(frc == 0); - - { - try { - H5::H5File file(h5name, H5F_ACC_RDONLY); - H5::DataSet mds(file.openDataSet("matrix")); - H5::DataSpace dataspace(mds.getSpace()); - - ASSERT(dataspace.isSimple() == true); - ASSERT(dataspace.getSimpleExtentNdims() == 2); - - hsize_t dims[2]; - dataspace.getSimpleExtentDims(dims, NULL); - ASSERT(dims[0] == 6); - ASSERT(dims[1] == 6); - } catch(...) { - int rc=1; - ASSERT(rc == 0); // if we get here is always an error, just to get a nice message - } - } - - unlink(h5name); - -} - -void test_to_file() { - SUITE_START("test unifrac_to_file"); - - test_to_file_one("unweighted"); - test_to_file_one("unweighted_fp32"); - test_to_file_one("weighted_normalized"); - test_to_file_one("weighted_normalized_fp32"); - test_to_file_one("weighted_unnormalized"); - test_to_file_one("weighted_unnormalized_fp32"); - test_to_file_one("generalized"); - test_to_file_one("generalized_fp32"); - - SUITE_END(); -} - -int main(int argc, char** argv) { - /* one_off and partial are executed as integration tests */ - - //test_write_mat(); - //test_read_mat(); - test_read_write_partial_mat(); - test_merge_partial_mat(); - test_merge_partial_dyn_mat(); - test_merge_partial_io(); - test_merge_partial_mmap(); - test_to_file(); - - printf("\n"); - printf(" %i / %i suites failed\n", suites_failed, suites_run); - printf(" %i / %i suites empty\n", suites_empty, suites_run); - printf(" %i / %i tests failed\n", tests_failed, tests_run); - - printf("\n THE END.\n"); - - return tests_failed ? EXIT_FAILURE : EXIT_SUCCESS; -} diff --git a/sucpp/test_ska.cpp b/sucpp/test_ska.cpp deleted file mode 100644 index 949c9d66..00000000 --- a/sucpp/test_ska.cpp +++ /dev/null @@ -1,516 +0,0 @@ -#include -#include "skbio_alt.hpp" -#include -#include -#include - -#include "api.hpp" - -/* - * test harness adapted from - * https://github.com/noporpoise/BitArray/blob/master/dev/bit_array_test.c - */ -const char *suite_name; -char suite_pass; -int suites_run = 0, suites_failed = 0, suites_empty = 0; -int tests_in_suite = 0, tests_run = 0, tests_failed = 0; - -#define QUOTE(str) #str -#define ASSERT(x) {tests_run++; tests_in_suite++; if(!(x)) \ - { fprintf(stderr, "failed assert [%s:%i] %s\n", __FILE__, __LINE__, QUOTE(x)); \ - suite_pass = 0; tests_failed++; }} - -void SUITE_START(const char *name) { - suite_pass = 1; - suite_name = name; - suites_run++; - tests_in_suite = 0; -} - -void SUITE_END() { - printf("Testing %s ", suite_name); - size_t suite_i; - for(suite_i = strlen(suite_name); suite_i < 80-8-5; suite_i++) printf("."); - printf("%s\n", suite_pass ? " pass" : " fail"); - if(!suite_pass) suites_failed++; - if(!tests_in_suite) suites_empty++; -} -/* - * End adapted code - */ - - -void test_center_mat() { - SUITE_START("test center mat"); - - // unweighted unifrac of test.biom - double matrix[] = { - 0.0000000000, 0.2000000000, 0.5714285714, 0.6000000000, 0.5000000000, 0.2000000000, - 0.2000000000, 0.0000000000, 0.4285714286 ,0.6666666667, 0.6000000000, 0.3333333333, - 0.5714285714, 0.4285714286, 0.0000000000, 0.7142857143, 0.8571428571, 0.4285714286, - 0.6000000000, 0.6666666667, 0.7142857143, 0.0000000000, 0.3333333333, 0.4000000000, - 0.5000000000, 0.6000000000, 0.8571428571, 0.3333333333, 0.0000000000, 0.6000000000, - 0.2000000000, 0.3333333333, 0.4285714286, 0.4000000000, 0.6000000000, 0.0000000000}; - - const uint32_t n_samples = 6; - - double exp[] = { 0.05343726, 0.04366213, -0.0329743 , -0.07912698, -0.00495654, 0.01995843, - 0.04366213, 0.073887 , 0.04867914, -0.11112434, -0.04973167,-0.00537226, - -0.0329743 , 0.04867914, 0.20714475, -0.07737528, -0.17044974, 0.02497543, - -0.07912698, -0.11112434, -0.07737528, 0.14830877, 0.11192366, 0.00739418, - -0.00495654, -0.04973167, -0.17044974, 0.11192366, 0.18664966,-0.07343537, - 0.01995843, -0.00537226, 0.02497543, 0.00739418, -0.07343537, 0.02647959 }; - - { - double *centered = (double *) malloc(6*6*sizeof(double)); - - su::mat_to_centered(matrix, n_samples, centered); - - for(int i = 0; i < (6*6); i++) { - //printf("%i %f %f\n",i,float(centered[i]),float(exp[i])); - ASSERT(fabs(centered[i] - exp[i]) < 0.000001); - } - - free(centered); - } - - float *matrix_fp32 = (float *) malloc(6*6*sizeof(float)); - for(int i = 0; i < (6*6); i++) matrix_fp32[i] = matrix[i]; - - - { - float *centered_fp32 = (float *) malloc(6*6*sizeof(float)); - - su::mat_to_centered(matrix_fp32, n_samples, centered_fp32); - - for(int i = 0; i < (6*6); i++) { - //printf("%i %f %f\n",i,float(centered_fp32[i]),float(exp[i])); - ASSERT(fabs(centered_fp32[i] - exp[i]) < 0.000001); - } - - free(centered_fp32); - } - - free(matrix_fp32); - - - SUITE_END(); -} - -void test_pcoa() { - SUITE_START("test pcoa"); - - // unweighted unifrac of crawford.biom - double matrix[] = { - 0. , 0.71836067 , 0.71317361 , 0.69746044 , 0.62587207 , 0.72826674 - , 0.72065895 , 0.72640581 , 0.73606053, - 0.71836067 , 0. , 0.70302967 , 0.73407301 , 0.6548042 , 0.71547381 - , 0.78397813 , 0.72318399 , 0.76138933, - 0.71317361 , 0.70302967 , 0. , 0.61041275 , 0.62331299 , 0.71848305 - , 0.70416337 , 0.75258475 , 0.79249029, - 0.69746044 , 0.73407301 , 0.61041275 , 0. , 0.6439278 , 0.70052733 - , 0.69832716 , 0.77818938 , 0.72959894, - 0.62587207 , 0.6548042 , 0.62331299 , 0.6439278 , 0. , 0.75782689 - , 0.71005144 , 0.75065046 , 0.78944369, - 0.72826674 , 0.71547381 , 0.71848305 , 0.70052733 , 0.75782689 , 0. - , 0.63593642 , 0.71283615 , 0.58314638, - 0.72065895 , 0.78397813 , 0.70416337 , 0.69832716 , 0.71005144 , 0.63593642 - , 0. , 0.69200762 , 0.68972056, - 0.72640581 , 0.72318399 , 0.75258475 , 0.77818938 , 0.75065046 , 0.71283615 - , 0.69200762 , 0. , 0.71514083, - 0.73606053 , 0.76138933 , 0.79249029 , 0.72959894 , 0.78944369 , 0.58314638 - , 0.68972056 , 0.71514083 , 0. }; - - - const uint32_t n_samples = 9; - - // Test centering - - double exp2[] = { - 0.22225336 , -0.025481 , -0.03491711 , -0.02614685 , 0.01899659 , -0.05103589 - , -0.03971812 , -0.02699392 , -0.03695706, - -0.025481 , 0.24282669 , -0.01744751 , -0.04206624 , 0.0107569 , -0.03151438 - , -0.07706765 , -0.0143721 , -0.0456347, - -0.03491711 , -0.01744751 , 0.21652901 , 0.02791465 , 0.0177328 , -0.04682078 - , -0.03082866 , -0.04921529 , -0.08294711, - -0.02614685 , -0.04206624 , 0.02791465 , 0.21190401 , 0.00235833 , -0.03639361 - , -0.02904855 , -0.07112525 , -0.03739649, - 0.01899659 , 0.0107569 , 0.0177328 , 0.00235833 , 0.20745566 , -0.08039931 - , -0.03952883 , -0.05229812 , -0.08507403, - -0.05103589 , -0.03151438 , -0.04682078 , -0.03639361 , -0.08039931 , 0.20604732 - , 0.00964595 , -0.02533192 , 0.05580262, - -0.03971812 , -0.07706765 , -0.03082866 , -0.02904855 , -0.03952883 , 0.00964595 - , 0.21765972 , -0.00489531 , -0.00621856, - -0.02699392 , -0.0143721 , -0.04921529 , -0.07112525 , -0.05229812 , -0.02533192 - , -0.00489531 , 0.2514242 , -0.00719229, - -0.03695706 , -0.0456347 , -0.08294711 , -0.03739649 , -0.08507403 , 0.05580262 - , -0.00621856 , -0.00719229 , 0.24561762 }; - - double *centered = (double *) malloc(9*9*sizeof(double)); - - su::mat_to_centered(matrix, n_samples, centered); - - for(int i = 0; i < (9*9); i++) { - //printf("%i %f %f\n",i,float(centered[i]),float(exp[i])); - ASSERT(fabs(centered[i] - exp2[i]) < 0.000001); - } - - // Test eigens - - double exp3a[] = {0.45752162, 0.3260088 , 0.2791141 , 0.26296948, 0.20924533}; - double exp3b[] = { - -0.17316152, 0.17579996, 0.23301609, -0.74519625, -0.05194624, - -0.19959264, 0.53235665, -0.53370018, 0.2173474 , 0.26736004, - -0.35794942, -0.27956624, 0.01114096, 0.40488848, -0.13121464, - -0.2296467 , -0.47494333, -0.12571292, 0.02313551, -0.46916459, - -0.44501584, 0.05597451, 0.07717711, -0.15881922, 0.24594442, - 0.40335552, -0.16290597, -0.30327343, 0.03778646, 0.21664806, - 0.23769142, -0.29034629, 0.46813757, 0.13858945, 0.58624834, - 0.2407584 , 0.51300752, 0.48211607, 0.34422672, -0.42416046, - 0.52356078, -0.0693768 , -0.30890127, -0.26195855, -0.23971493}; - - { - double *eigenvalues; - double *eigenvectors; - su::find_eigens_fast(n_samples, 5, centered, eigenvalues, eigenvectors); - - for(int i = 0; i < 5; i++) { - //printf("%i %f %f\n",i,float(eigenvalues[i]),float(exp3a[i])); - ASSERT(fabs(eigenvalues[i] - exp3a[i]) < 0.000001); - } - - // signs may flip, that's normal - for(int i = 0; i < (5*9); i++) { - //printf("%i %f %f %f\n",i,float(eigenvectors[i]),float(exp3b[i]),float(fabs(eigenvectors[i]) - fabs(exp3b[i]))); - ASSERT( fabs(fabs(eigenvectors[i]) - fabs(exp3b[i])) < 0.000001); - } - - free(eigenvectors); - free(eigenvalues); - } - free(centered); - - - // Test PCoA (incudes the above calls - - double *exp4a = exp3a; // same eigenvals; - double exp4b[] = { - -0.11712705, 0.10037682, -0.12310531, -0.38214073, -0.02376195, - -0.13500515, 0.30396064, 0.28196047, 0.11145694, 0.12229942, - -0.24211822, -0.15962444, -0.00588591, 0.20762904, -0.06002196, - -0.15533382, -0.27117925, 0.06641571, 0.01186402, -0.21461156, - -0.30101024, 0.03195987, -0.04077363, -0.08144337, 0.1125032 , - 0.27283106, -0.09301471, 0.16022314, 0.0193771 , 0.09910206, - 0.16077529, -0.16577955, -0.24732293, 0.07106943, 0.26816958, - 0.16284981, 0.29291283, -0.25470794, 0.17652136, -0.19402517, - 0.35413832, -0.0396122 , 0.1631964 , -0.13433378, -0.10965362}; - double exp4c[] = {0.22630343, 0.16125338, 0.13805791, 0.13007231, 0.10349879}; - - { - double *eigenvalues; - double *samples; - double *proportion_explained; - - su::pcoa(matrix, n_samples, 5, eigenvalues, samples, proportion_explained); - - for(int i = 0; i < 5; i++) { - //printf("%i %f %f\n",i,float(eigenvalues[i]),float(exp4a[i])); - ASSERT(fabs(eigenvalues[i] - exp4a[i]) < 0.000001); - } - - // signs may flip, that's normal - for(int i = 0; i < (5*9); i++) { - //printf("%i %f %f %f\n",i,float(samples[i]),float(exp4b[i]),float(fabs(samples[i]) - fabs(exp4b[i]))); - ASSERT( fabs(fabs(samples[i]) - fabs(exp4b[i])) < 0.000001); - } - - for(int i = 0; i < 5; i++) { - //printf("%i %f %f\n",i,float(proportion_explained[i]),float(exp4c[i])); - ASSERT(fabs(proportion_explained[i] - exp4c[i]) < 0.000001); - } - - free(eigenvalues); - free(samples); - free(proportion_explained); - } - - // Test PCoA mixed mode - { - float *eigenvalues_fp32; - float *samples_fp32; - float *proportion_explained_fp32; - - su::pcoa(matrix, n_samples, 5, eigenvalues_fp32, samples_fp32, proportion_explained_fp32); - - for(int i = 0; i < 5; i++) { - //printf("%i %f %f\n",i,float(eigenvalues_fp32[i]),float(exp4a[i])); - ASSERT(fabs(eigenvalues_fp32[i] - exp4a[i]) < 0.000001); - } - - // signs may flip, that's normal - for(int i = 0; i < (5*9); i++) { - //printf("%i %f %f %f\n",i,float(samples_fp32[i]),float(exp4b[i]),float(fabs(samples_fp32[i]) - fabs(exp4b[i]))); - ASSERT( fabs(fabs(samples_fp32[i]) - fabs(exp4b[i])) < 0.000001); - } - - for(int i = 0; i < 5; i++) { - //printf("%i %f %f\n",i,float(proportion_explained_fp32[i]),float(exp4c[i])); - ASSERT(fabs(proportion_explained_fp32[i] - exp4c[i]) < 0.000001); - } - - free(eigenvalues_fp32); - free(samples_fp32); - free(proportion_explained_fp32); - } - - // test in-place - { - double *eigenvalues; - double *samples; - double *proportion_explained; - - su::pcoa_inplace(matrix, n_samples, 5, eigenvalues, samples, proportion_explained); - // Note: matrix content has been destroyed - - for(int i = 0; i < 5; i++) { - //printf("%i %f %f\n",i,float(eigenvalues[i]),float(exp4a[i])); - ASSERT(fabs(eigenvalues[i] - exp4a[i]) < 0.000001); - } - - // signs may flip, that's normal - for(int i = 0; i < (5*9); i++) { - //printf("%i %f %f %f\n",i,float(samples[i]),float(exp4b[i]),float(fabs(samples[i]) - fabs(exp4b[i]))); - ASSERT( fabs(fabs(samples[i]) - fabs(exp4b[i])) < 0.000001); - } - - for(int i = 0; i < 5; i++) { - //printf("%i %f %f\n",i,float(proportion_explained[i]),float(exp4c[i])); - ASSERT(fabs(proportion_explained[i] - exp4c[i]) < 0.000001); - } - - free(eigenvalues); - free(samples); - free(proportion_explained); - } - - SUITE_END(); -} - -void test_pcoa_big() { - SUITE_START("test pcoa big"); - - //too big to inline, use support file - FILE *fptr = fopen("test_ska_pcoa_big.dat", "r"); - ASSERT(fptr != NULL) - - unsigned int n_samples = 0; - fscanf(fptr,"# unifrac %u\n",&n_samples); - ASSERT(n_samples == 57); - - - // first 57 rows/cols of unweighted unifrac of EMP - double matrix[57*57]; - for (unsigned int i=0; i<(57*57); i++) - fscanf(fptr,"%lf\n",&(matrix[i])); - - unsigned int n_dims = 0; - fscanf(fptr,"# pcoa %u\n",&n_dims); - ASSERT(n_dims == 7); - - double exp1[7]; - for (unsigned int i=0; i<(7); i++) - fscanf(fptr,"%lf\n",&(exp1[i])); - - double exp2[7*57]; - for (unsigned int i=0; i<(7*57); i++) - fscanf(fptr,"%lf\n",&(exp2[i])); - - double exp3[7]; - for (unsigned int i=0; i<(7); i++) - fscanf(fptr,"%lf\n",&(exp3[i])); - - fclose(fptr); - - { - double *eigenvalues; - double *samples; - double *proportion_explained; - - su::pcoa(matrix, n_samples, n_dims, eigenvalues, samples, proportion_explained); - - // last three eignes are very close to each other and could come back in reverse order - - for(unsigned int i = 0; i < n_dims ; i++) { - //printf("%i %f %f %f\n",i,float(eigenvalues[i]),float(exp1[i]),float(fabs(eigenvalues[i] - exp1[i]))); - const double max_err = (i<4) ? 0.01 : 0.1; // the values are approximate, based on a random number in the algo - ASSERT(fabs(eigenvalues[i] - exp1[i]) < max_err); - } - - // signs may flip, that's normal - for(unsigned int i = 0; i < (n_samples*n_dims); i++) { - if ((i%n_dims)<4) { - //printf("%i %f %f %f\n",i,float(samples[i]),float(exp2[i]),float(fabs(samples[i]) - fabs(exp2[i]))); - ASSERT( fabs(fabs(samples[i]) - fabs(exp2[i])) < 0.05) - } else { - // any of the 3 will do - unsigned int ibase = (i/n_dims)*n_dims; - //printf("%i %f %f %f %f\n",i,float(samples[i]),float(exp2[ibase+4]),float(exp2[ibase+5]),float(exp2[ibase+6])); - ASSERT( (fabs(fabs(samples[i]) - fabs(exp2[ibase+4])) < 0.15) || (fabs(fabs(samples[i]) - fabs(exp2[ibase+5])) < 0.15) || (fabs(fabs(samples[i]) - fabs(exp2[ibase+6])) < 0.15) ); - } - } - - for(unsigned int i = 0; i < n_dims; i++) { - //printf("%i %f %f\n",i,float(proportion_explained[i]),float(exp3[i]),float(fabs(proportion_explained[i] - exp3[i]))); - const double max_err = (i<4) ? 0.001 : 0.01; - ASSERT(fabs(proportion_explained[i] - exp3[i]) < max_err); - } - - free(proportion_explained); - free(samples); - free(eigenvalues); - } - - { - float *eigenvalues; - float *samples; - float *proportion_explained; - - su::pcoa(matrix, n_samples, n_dims, eigenvalues, samples, proportion_explained); - - // last three eignes are very close to each other and could come back in reverse order - - for(unsigned int i = 0; i < n_dims ; i++) { - //printf("%i %f %f %f\n",i,float(eigenvalues[i]),float(exp1[i]),float(fabs(eigenvalues[i] - exp1[i]))); - const float max_err = (i<4) ? 0.01 : 0.1; // the values are approximate, based on a random number in the algo - ASSERT(fabs(eigenvalues[i] - exp1[i]) < max_err); - } - - // signs may flip, that's normal - for(unsigned int i = 0; i < (n_samples*n_dims); i++) { - if ((i%n_dims)<4) { - //printf("%i %f %f %f\n",i,float(samples[i]),float(exp2[i]),float(fabs(samples[i]) - fabs(exp2[i]))); - ASSERT( fabs(fabs(samples[i]) - fabs(exp2[i])) < 0.1) - } else { - // any of the 3 will do - unsigned int ibase = (i/n_dims)*n_dims; - //printf("%i %f %f %f %f\n",i,float(samples[i]),float(exp2[ibase+4]),float(exp2[ibase+5]),float(exp2[ibase+6])); - ASSERT( (fabs(fabs(samples[i]) - fabs(exp2[ibase+4])) < 0.15) || (fabs(fabs(samples[i]) - fabs(exp2[ibase+5])) < 0.15) || (fabs(fabs(samples[i]) - fabs(exp2[ibase+6])) < 0.15) ); - } - } - - for(unsigned int i = 0; i < n_dims; i++) { - //printf("%i %f %f\n",i,float(proportion_explained[i]),float(exp3[i]),float(fabs(proportion_explained[i] - exp3[i]))); - const float max_err = (i<4) ? 0.001 : 0.01; - ASSERT(fabs(proportion_explained[i] - exp3[i]) < max_err); - } - - free(proportion_explained); - free(samples); - free(eigenvalues); - } - - { - float matrix_fp32[57*57]; - for (unsigned int i=0; i<(57*57); i++) - matrix_fp32[i] = matrix[i]; - - { - float *eigenvalues; - float *samples; - float *proportion_explained; - - su::pcoa(matrix_fp32, n_samples, n_dims, eigenvalues, samples, proportion_explained); - - // last three eignes are very close to each other and could come back in reverse order - - for(unsigned int i = 0; i < n_dims ; i++) { - //printf("%i %f %f %f\n",i,float(eigenvalues[i]),float(exp1[i]),float(fabs(eigenvalues[i] - exp1[i]))); - const float max_err = (i<4) ? 0.01 : 0.1; // the values are approximate, based on a random number in the algo - ASSERT(fabs(eigenvalues[i] - exp1[i]) < max_err); - } - - // signs may flip, that's normal - for(unsigned int i = 0; i < (n_samples*n_dims); i++) { - if ((i%n_dims)<4) { - //printf("%i %f %f %f\n",i,float(samples[i]),float(exp2[i]),float(fabs(samples[i]) - fabs(exp2[i]))); - ASSERT( fabs(fabs(samples[i]) - fabs(exp2[i])) < 0.1) - } else { - // any of the 3 will do - unsigned int ibase = (i/n_dims)*n_dims; - //printf("%i %f %f %f %f\n",i,float(samples[i]),float(exp2[ibase+4]),float(exp2[ibase+5]),float(exp2[ibase+6])); - ASSERT( (fabs(fabs(samples[i]) - fabs(exp2[ibase+4])) < 0.15) || (fabs(fabs(samples[i]) - fabs(exp2[ibase+5])) < 0.15) || (fabs(fabs(samples[i]) - fabs(exp2[ibase+6])) < 0.15) ); - } - } - - for(unsigned int i = 0; i < n_dims; i++) { - //printf("%i %f %f\n",i,float(proportion_explained[i]),float(exp3[i]),float(fabs(proportion_explained[i] - exp3[i]))); - const float max_err = (i<4) ? 0.001 : 0.01; - ASSERT(fabs(proportion_explained[i] - exp3[i]) < max_err); - } - - free(proportion_explained); - free(samples); - free(eigenvalues); - } - - { - float *eigenvalues; - float *samples; - float *proportion_explained; - - su::pcoa_inplace(matrix_fp32, n_samples, n_dims, eigenvalues, samples, proportion_explained); - // Note: content of matrix_fp32 has been destroyed - - // last three eignes are very close to each other and could come back in reverse order - - for(unsigned int i = 0; i < n_dims ; i++) { - //printf("%i %f %f %f\n",i,float(eigenvalues[i]),float(exp1[i]),float(fabs(eigenvalues[i] - exp1[i]))); - const float max_err = (i<4) ? 0.01 : 0.1; // the values are approximate, based on a random number in the algo - ASSERT(fabs(eigenvalues[i] - exp1[i]) < max_err); - } - - // signs may flip, that's normal - for(unsigned int i = 0; i < (n_samples*n_dims); i++) { - if ((i%n_dims)<4) { - //printf("%i %f %f %f\n",i,float(samples[i]),float(exp2[i]),float(fabs(samples[i]) - fabs(exp2[i]))); - ASSERT( fabs(fabs(samples[i]) - fabs(exp2[i])) < 0.1) - } else { - // any of the 3 will do - unsigned int ibase = (i/n_dims)*n_dims; - //printf("%i %f %f %f %f\n",i,float(samples[i]),float(exp2[ibase+4]),float(exp2[ibase+5]),float(exp2[ibase+6])); - ASSERT( (fabs(fabs(samples[i]) - fabs(exp2[ibase+4])) < 0.15) || (fabs(fabs(samples[i]) - fabs(exp2[ibase+5])) < 0.15) || (fabs(fabs(samples[i]) - fabs(exp2[ibase+6])) < 0.15) ); - } - } - - for(unsigned int i = 0; i < n_dims; i++) { - //printf("%i %f %f\n",i,float(proportion_explained[i]),float(exp3[i]),float(fabs(proportion_explained[i] - exp3[i]))); - const float max_err = (i<4) ? 0.001 : 0.01; - ASSERT(fabs(proportion_explained[i] - exp3[i]) < max_err); - } - - free(proportion_explained); - free(samples); - free(eigenvalues); - } - - } - - - - SUITE_END(); -} - -int main(int argc, char** argv) { - test_center_mat(); - test_pcoa(); - test_pcoa_big(); - - printf("\n"); - printf(" %i / %i suites failed\n", suites_failed, suites_run); - printf(" %i / %i suites empty\n", suites_empty, suites_run); - printf(" %i / %i tests failed\n", tests_failed, tests_run); - - printf("\n THE END.\n"); - - return tests_failed ? EXIT_FAILURE : EXIT_SUCCESS; -} diff --git a/sucpp/test_ska_pcoa_big.dat b/sucpp/test_ska_pcoa_big.dat deleted file mode 100644 index 6e10db75..00000000 --- a/sucpp/test_ska_pcoa_big.dat +++ /dev/null @@ -1,3664 +0,0 @@ -# unifrac 57 -0.000000 -0.950883 -0.925467 -0.942685 -0.946867 -0.900857 -0.918038 -0.936913 -0.922809 -0.955770 -0.923860 -0.942869 -0.945097 -0.934522 -0.951878 -0.942467 -0.836106 -0.931903 -0.888912 -0.962021 -0.919973 -0.945621 -0.953109 -0.936558 -0.952173 -0.942877 -0.937624 -0.947284 -0.948317 -0.947515 -0.936059 -0.950720 -0.929309 -0.949459 -0.803216 -0.926292 -0.947140 -0.953137 -0.954502 -0.770338 -0.946883 -0.935992 -0.932617 -0.952604 -0.947546 -0.933947 -0.949483 -0.959825 -0.949005 -0.911958 -0.927311 -0.930302 -0.752238 -0.945606 -0.937430 -0.707954 -0.949925 -0.950883 -0.000000 -0.875905 -0.868694 -0.762639 -0.855361 -0.877736 -0.845327 -0.843290 -0.916377 -0.893390 -0.802567 -0.869932 -0.854706 -0.877391 -0.865799 -0.951840 -0.845430 -0.954491 -0.961021 -0.865376 -0.923502 -0.717260 -0.754222 -0.775748 -0.821892 -0.928893 -0.900920 -0.792339 -0.728443 -0.784663 -0.711012 -0.940213 -0.899511 -0.952925 -0.916895 -0.897356 -0.714006 -0.929754 -0.932686 -0.636528 -0.897664 -0.822672 -0.736650 -0.875698 -0.854048 -0.940416 -0.927072 -0.708374 -0.894228 -0.867627 -0.880325 -0.921660 -0.789248 -0.808261 -0.954224 -0.920468 -0.925467 -0.875905 -0.000000 -0.658556 -0.857434 -0.861288 -0.819924 -0.753185 -0.816612 -0.947886 -0.827102 -0.837861 -0.597003 -0.806763 -0.883921 -0.870304 -0.893920 -0.733847 -0.907924 -0.927714 -0.613704 -0.864190 -0.885613 -0.856325 -0.835157 -0.798392 -0.850569 -0.839578 -0.813228 -0.865080 -0.860362 -0.865913 -0.884919 -0.834627 -0.905982 -0.904453 -0.892967 -0.877850 -0.845049 -0.896810 -0.864871 -0.849646 -0.742261 -0.875701 -0.618568 -0.684099 -0.901057 -0.824650 -0.861649 -0.834687 -0.870877 -0.542812 -0.880030 -0.845482 -0.831448 -0.921790 -0.877044 -0.942685 -0.868694 -0.658556 -0.000000 -0.846807 -0.873695 -0.812930 -0.761420 -0.852546 -0.949922 -0.845547 -0.838229 -0.604431 -0.796921 -0.882208 -0.869152 -0.932455 -0.646338 -0.932789 -0.923012 -0.637784 -0.843161 -0.873020 -0.853422 -0.831626 -0.776793 -0.861113 -0.863207 -0.797057 -0.867799 -0.850210 -0.855860 -0.898218 -0.850414 -0.938152 -0.904979 -0.886563 -0.871417 -0.833733 -0.909678 -0.858432 -0.862019 -0.753962 -0.874623 -0.522917 -0.647334 -0.917839 -0.813452 -0.853833 -0.838460 -0.883248 -0.602860 -0.895554 -0.845276 -0.819952 -0.943930 -0.872523 -0.946867 -0.762639 -0.857434 -0.846807 -0.000000 -0.870644 -0.873040 -0.813825 -0.839988 -0.921838 -0.889059 -0.810507 -0.838611 -0.849573 -0.880202 -0.861283 -0.945135 -0.825211 -0.939177 -0.951187 -0.846258 -0.899207 -0.773214 -0.781258 -0.750694 -0.808270 -0.900223 -0.896811 -0.779932 -0.790001 -0.825784 -0.764520 -0.927103 -0.899211 -0.945712 -0.904577 -0.876027 -0.767545 -0.913633 -0.929382 -0.777761 -0.891562 -0.816742 -0.767742 -0.851561 -0.839805 -0.936841 -0.907887 -0.748085 -0.885075 -0.866901 -0.856020 -0.925318 -0.786194 -0.723351 -0.947850 -0.888425 -0.900857 -0.855361 -0.861288 -0.873695 -0.870644 -0.000000 -0.872952 -0.825385 -0.809442 -0.938280 -0.727602 -0.780119 -0.866841 -0.788788 -0.905208 -0.882055 -0.827841 -0.853270 -0.829922 -0.944036 -0.845505 -0.908617 -0.862270 -0.816307 -0.867666 -0.811319 -0.892198 -0.846356 -0.864241 -0.855938 -0.791353 -0.862115 -0.878718 -0.816999 -0.853319 -0.895402 -0.888764 -0.866750 -0.894509 -0.845506 -0.829934 -0.688612 -0.856817 -0.873884 -0.871570 -0.851693 -0.881896 -0.872935 -0.860692 -0.827825 -0.876230 -0.862114 -0.828973 -0.828217 -0.847962 -0.883075 -0.910120 -0.918038 -0.877736 -0.819924 -0.812930 -0.873040 -0.872952 -0.000000 -0.828825 -0.833276 -0.932334 -0.827191 -0.864539 -0.823283 -0.831230 -0.902836 -0.892363 -0.886899 -0.824955 -0.902814 -0.915614 -0.802998 -0.887478 -0.889484 -0.884003 -0.877410 -0.841325 -0.877335 -0.860102 -0.863821 -0.902459 -0.889057 -0.887147 -0.871284 -0.854576 -0.904143 -0.902250 -0.893084 -0.888426 -0.869574 -0.889108 -0.853758 -0.868578 -0.812848 -0.885668 -0.832409 -0.805251 -0.902814 -0.867349 -0.881852 -0.851747 -0.858642 -0.783179 -0.884548 -0.865299 -0.861431 -0.906366 -0.908390 -0.936913 -0.845327 -0.753185 -0.761420 -0.813825 -0.825385 -0.828825 -0.000000 -0.825094 -0.941972 -0.784170 -0.787034 -0.738858 -0.739644 -0.880329 -0.848105 -0.917500 -0.791516 -0.913328 -0.924667 -0.738329 -0.846000 -0.847644 -0.829801 -0.794750 -0.751581 -0.847152 -0.783164 -0.727386 -0.849298 -0.827218 -0.833450 -0.874353 -0.789677 -0.921891 -0.903657 -0.874444 -0.835680 -0.838468 -0.898200 -0.843558 -0.787206 -0.784409 -0.853389 -0.770840 -0.795344 -0.880527 -0.825790 -0.827601 -0.817439 -0.861289 -0.751222 -0.871600 -0.798566 -0.809562 -0.937328 -0.855658 -0.922809 -0.843290 -0.816612 -0.852546 -0.839988 -0.809442 -0.833276 -0.825094 -0.000000 -0.944595 -0.728395 -0.664039 -0.848241 -0.801649 -0.932820 -0.919769 -0.821895 -0.836012 -0.835205 -0.865372 -0.827969 -0.862659 -0.861032 -0.787222 -0.872557 -0.805243 -0.830876 -0.773147 -0.869150 -0.840588 -0.830998 -0.858487 -0.802052 -0.760895 -0.852578 -0.920793 -0.901146 -0.859814 -0.852220 -0.869178 -0.822584 -0.726266 -0.866938 -0.881020 -0.855606 -0.840409 -0.841951 -0.864574 -0.861642 -0.828571 -0.908068 -0.833921 -0.872648 -0.684464 -0.789884 -0.908162 -0.896148 -0.955770 -0.916377 -0.947886 -0.949922 -0.921838 -0.938280 -0.932334 -0.941972 -0.944595 -0.000000 -0.956371 -0.944121 -0.951666 -0.949948 -0.931704 -0.923679 -0.963884 -0.925379 -0.971254 -0.978283 -0.938921 -0.962231 -0.922165 -0.921767 -0.924406 -0.943199 -0.959528 -0.963827 -0.928214 -0.922299 -0.928957 -0.926129 -0.971932 -0.966729 -0.964529 -0.906220 -0.904738 -0.925656 -0.974445 -0.961677 -0.928553 -0.963465 -0.921866 -0.913509 -0.948605 -0.931662 -0.969119 -0.973554 -0.920689 -0.943497 -0.910883 -0.947843 -0.959480 -0.940082 -0.936510 -0.960491 -0.957545 -0.923860 -0.893390 -0.827102 -0.845547 -0.889059 -0.727602 -0.827191 -0.784170 -0.728395 -0.956371 -0.000000 -0.708313 -0.830500 -0.748020 -0.935275 -0.909958 -0.829307 -0.858332 -0.806232 -0.874527 -0.806928 -0.860428 -0.898361 -0.833302 -0.898159 -0.789890 -0.817735 -0.736903 -0.890843 -0.895605 -0.853789 -0.901642 -0.797962 -0.663999 -0.851102 -0.910949 -0.898879 -0.908923 -0.782570 -0.823080 -0.859871 -0.661393 -0.871310 -0.908754 -0.862771 -0.841580 -0.760037 -0.795391 -0.896856 -0.787621 -0.916908 -0.808847 -0.808474 -0.818109 -0.824802 -0.906598 -0.887389 -0.942869 -0.802567 -0.837861 -0.838229 -0.810507 -0.780119 -0.864539 -0.787034 -0.664039 -0.944121 -0.708313 -0.000000 -0.822147 -0.742381 -0.923600 -0.898260 -0.895721 -0.832202 -0.894713 -0.909344 -0.837216 -0.870698 -0.815486 -0.752309 -0.838107 -0.775544 -0.840725 -0.774224 -0.825276 -0.791348 -0.781558 -0.813188 -0.865207 -0.743496 -0.906662 -0.919087 -0.891428 -0.820383 -0.840883 -0.882307 -0.785919 -0.760875 -0.846168 -0.845756 -0.828628 -0.825924 -0.879512 -0.855618 -0.815462 -0.832354 -0.903268 -0.835632 -0.860977 -0.641763 -0.794373 -0.929273 -0.887581 -0.945097 -0.869932 -0.597003 -0.604431 -0.838611 -0.866841 -0.823283 -0.738858 -0.848241 -0.951666 -0.830500 -0.822147 -0.000000 -0.779653 -0.880308 -0.866160 -0.930244 -0.732232 -0.931979 -0.924401 -0.609553 -0.850718 -0.874764 -0.853619 -0.824201 -0.782077 -0.851878 -0.856889 -0.807442 -0.873230 -0.857711 -0.852395 -0.900272 -0.842107 -0.937206 -0.914639 -0.887293 -0.870079 -0.843047 -0.912681 -0.864160 -0.865284 -0.676093 -0.869403 -0.557087 -0.680547 -0.915238 -0.815637 -0.858187 -0.846706 -0.877012 -0.555591 -0.875825 -0.838191 -0.831433 -0.941377 -0.864181 -0.934522 -0.854706 -0.806763 -0.796921 -0.849573 -0.788788 -0.831230 -0.739644 -0.801649 -0.949948 -0.748020 -0.742381 -0.779653 -0.000000 -0.891363 -0.873945 -0.909146 -0.817432 -0.907505 -0.912798 -0.780554 -0.857758 -0.858283 -0.828901 -0.844395 -0.732841 -0.848633 -0.727164 -0.822670 -0.857814 -0.809884 -0.842681 -0.861622 -0.648252 -0.917201 -0.913087 -0.885236 -0.853515 -0.834871 -0.869767 -0.821753 -0.783962 -0.812697 -0.865936 -0.803072 -0.806727 -0.892124 -0.849324 -0.855732 -0.837628 -0.879363 -0.789095 -0.798033 -0.794027 -0.816773 -0.926091 -0.874297 -0.951878 -0.877391 -0.883921 -0.882208 -0.880202 -0.905208 -0.902836 -0.880329 -0.932820 -0.931704 -0.935275 -0.923600 -0.880308 -0.891363 -0.000000 -0.749514 -0.960920 -0.864243 -0.967715 -0.981110 -0.868196 -0.943592 -0.887409 -0.904329 -0.843285 -0.897206 -0.954303 -0.943078 -0.853192 -0.883723 -0.891212 -0.869927 -0.966141 -0.941037 -0.962509 -0.916917 -0.906709 -0.864075 -0.956684 -0.948882 -0.889704 -0.934235 -0.817651 -0.863783 -0.901197 -0.867519 -0.966131 -0.945948 -0.858308 -0.927258 -0.744567 -0.882602 -0.940086 -0.924701 -0.917692 -0.954688 -0.928355 -0.942467 -0.865799 -0.870304 -0.869152 -0.861283 -0.882055 -0.892363 -0.848105 -0.919769 -0.923679 -0.909958 -0.898260 -0.866160 -0.873945 -0.749514 -0.000000 -0.952727 -0.855890 -0.956342 -0.977546 -0.857017 -0.934475 -0.873316 -0.881214 -0.806514 -0.873499 -0.942442 -0.924663 -0.833253 -0.862126 -0.867990 -0.852253 -0.958611 -0.928291 -0.950624 -0.900132 -0.896308 -0.844800 -0.949426 -0.936616 -0.874121 -0.915228 -0.801362 -0.831545 -0.890550 -0.850525 -0.953656 -0.933293 -0.832836 -0.912394 -0.736560 -0.857737 -0.920944 -0.908606 -0.890825 -0.938848 -0.920676 -0.836106 -0.951840 -0.893920 -0.932455 -0.945135 -0.827841 -0.886899 -0.917500 -0.821895 -0.963884 -0.829307 -0.895721 -0.930244 -0.909146 -0.960920 -0.952727 -0.000000 -0.919761 -0.716719 -0.897720 -0.913009 -0.916036 -0.951941 -0.927812 -0.957944 -0.933629 -0.876379 -0.890158 -0.954408 -0.947436 -0.930663 -0.955896 -0.850370 -0.882120 -0.542144 -0.932575 -0.944352 -0.957306 -0.904083 -0.780567 -0.932759 -0.827212 -0.939655 -0.955149 -0.929799 -0.924389 -0.871519 -0.916662 -0.955649 -0.859642 -0.944841 -0.895509 -0.811097 -0.907153 -0.916950 -0.816379 -0.934616 -0.931903 -0.845430 -0.733847 -0.646338 -0.825211 -0.853270 -0.824955 -0.791516 -0.836012 -0.925379 -0.858332 -0.832202 -0.732232 -0.817432 -0.864243 -0.855890 -0.919761 -0.000000 -0.931987 -0.942071 -0.723892 -0.874810 -0.857536 -0.834621 -0.808548 -0.798743 -0.882593 -0.873912 -0.790672 -0.841393 -0.829001 -0.834144 -0.910931 -0.871231 -0.929060 -0.883695 -0.857249 -0.848843 -0.874677 -0.909399 -0.832504 -0.871769 -0.774739 -0.838087 -0.717265 -0.660444 -0.924109 -0.867395 -0.834165 -0.834184 -0.854987 -0.707400 -0.887298 -0.836763 -0.816007 -0.934203 -0.883222 -0.888912 -0.954491 -0.907924 -0.932789 -0.939177 -0.829922 -0.902814 -0.913328 -0.835205 -0.971254 -0.806232 -0.894713 -0.931979 -0.907505 -0.967715 -0.956342 -0.716719 -0.931987 -0.000000 -0.875950 -0.917642 -0.917229 -0.951869 -0.910324 -0.958785 -0.934473 -0.882419 -0.870228 -0.957783 -0.952526 -0.923094 -0.958702 -0.802795 -0.857247 -0.699790 -0.943958 -0.944872 -0.960012 -0.876585 -0.849576 -0.932192 -0.785758 -0.945018 -0.959706 -0.932009 -0.931867 -0.833975 -0.900267 -0.957490 -0.874993 -0.953218 -0.903465 -0.874400 -0.907508 -0.919066 -0.862560 -0.934037 -0.962021 -0.961021 -0.927714 -0.923012 -0.951187 -0.944036 -0.915614 -0.924667 -0.865372 -0.978283 -0.874527 -0.909344 -0.924401 -0.912798 -0.981110 -0.977546 -0.897720 -0.942071 -0.875950 -0.000000 -0.922815 -0.888662 -0.959884 -0.949138 -0.965696 -0.927568 -0.826457 -0.858766 -0.958165 -0.959130 -0.960104 -0.963644 -0.801723 -0.839943 -0.892223 -0.968230 -0.958453 -0.968865 -0.781823 -0.930421 -0.955341 -0.867128 -0.959928 -0.972917 -0.920111 -0.938839 -0.778247 -0.866601 -0.963352 -0.891169 -0.977175 -0.921329 -0.933289 -0.897590 -0.901006 -0.955194 -0.935049 -0.919973 -0.865376 -0.613704 -0.637784 -0.846258 -0.845505 -0.802998 -0.738329 -0.827969 -0.938921 -0.806928 -0.837216 -0.609553 -0.780554 -0.868196 -0.857017 -0.913009 -0.723892 -0.917642 -0.922815 -0.000000 -0.845463 -0.871787 -0.850431 -0.834139 -0.777714 -0.853826 -0.836158 -0.808749 -0.867151 -0.855487 -0.851500 -0.884926 -0.817082 -0.919464 -0.894557 -0.873376 -0.869461 -0.843130 -0.893616 -0.853156 -0.836409 -0.735911 -0.876239 -0.634180 -0.708674 -0.898311 -0.826903 -0.850139 -0.811091 -0.847908 -0.612305 -0.871796 -0.848518 -0.816455 -0.927580 -0.867447 -0.945621 -0.923502 -0.864190 -0.843161 -0.899207 -0.908617 -0.887478 -0.846000 -0.862659 -0.962231 -0.860428 -0.870698 -0.850718 -0.857758 -0.943592 -0.934475 -0.916036 -0.874810 -0.917229 -0.888662 -0.845463 -0.000000 -0.914953 -0.906093 -0.907259 -0.862791 -0.718051 -0.866324 -0.893466 -0.914863 -0.912569 -0.912017 -0.878760 -0.869738 -0.924687 -0.906586 -0.897381 -0.916192 -0.826615 -0.916632 -0.919078 -0.873053 -0.903896 -0.924657 -0.862125 -0.883323 -0.898415 -0.837839 -0.913961 -0.816016 -0.936246 -0.855601 -0.914592 -0.884170 -0.869485 -0.943072 -0.744470 -0.953109 -0.717260 -0.885613 -0.873020 -0.773214 -0.862270 -0.889484 -0.847644 -0.861032 -0.922165 -0.898361 -0.815486 -0.874764 -0.858283 -0.887409 -0.873316 -0.951941 -0.857536 -0.951869 -0.959884 -0.871787 -0.914953 -0.000000 -0.760564 -0.786031 -0.830820 -0.926962 -0.905507 -0.796966 -0.692234 -0.803213 -0.714538 -0.939118 -0.894099 -0.954729 -0.911829 -0.889738 -0.722764 -0.932857 -0.931972 -0.726367 -0.900258 -0.825420 -0.761180 -0.873218 -0.861618 -0.942424 -0.925933 -0.728019 -0.893324 -0.879332 -0.892313 -0.921435 -0.801181 -0.814978 -0.951026 -0.925351 -0.936558 -0.754222 -0.856325 -0.853422 -0.781258 -0.816307 -0.884003 -0.829801 -0.787222 -0.921767 -0.833302 -0.752309 -0.853619 -0.828901 -0.904329 -0.881214 -0.927812 -0.834621 -0.910324 -0.949138 -0.850431 -0.906093 -0.760564 -0.000000 -0.802637 -0.808634 -0.901023 -0.864355 -0.802803 -0.737050 -0.769249 -0.764051 -0.918822 -0.864335 -0.924319 -0.903756 -0.882520 -0.767074 -0.913712 -0.909577 -0.737789 -0.850038 -0.844195 -0.792691 -0.849186 -0.847649 -0.920961 -0.905015 -0.770907 -0.881864 -0.879245 -0.856793 -0.901609 -0.737237 -0.785313 -0.933553 -0.906292 -0.952173 -0.775748 -0.835157 -0.831626 -0.750694 -0.867666 -0.877410 -0.794750 -0.872557 -0.924406 -0.898159 -0.838107 -0.824201 -0.844395 -0.843285 -0.806514 -0.957944 -0.808548 -0.958785 -0.965696 -0.834139 -0.907259 -0.786031 -0.802637 -0.000000 -0.819161 -0.923242 -0.908694 -0.687380 -0.763825 -0.803676 -0.749803 -0.944622 -0.908830 -0.960087 -0.900328 -0.883823 -0.738799 -0.923401 -0.942839 -0.793717 -0.906077 -0.787596 -0.766479 -0.837486 -0.825720 -0.942659 -0.918513 -0.744747 -0.898959 -0.825632 -0.847438 -0.922993 -0.837325 -0.823325 -0.955825 -0.890439 -0.942877 -0.821892 -0.798392 -0.776793 -0.808270 -0.811319 -0.841325 -0.751581 -0.805243 -0.943199 -0.789890 -0.775544 -0.782077 -0.732841 -0.897206 -0.873499 -0.933629 -0.798743 -0.934473 -0.927568 -0.777714 -0.862791 -0.830820 -0.808634 -0.819161 -0.000000 -0.870424 -0.739865 -0.787772 -0.836186 -0.800454 -0.831337 -0.900986 -0.796902 -0.941981 -0.913404 -0.884096 -0.842792 -0.856129 -0.905075 -0.813429 -0.812679 -0.800721 -0.846014 -0.788722 -0.801647 -0.918970 -0.832165 -0.829254 -0.846914 -0.874460 -0.784047 -0.853731 -0.785806 -0.742033 -0.944013 -0.881855 -0.937624 -0.928893 -0.850569 -0.861113 -0.900223 -0.892198 -0.877335 -0.847152 -0.830876 -0.959528 -0.817735 -0.840725 -0.851878 -0.848633 -0.954303 -0.942442 -0.876379 -0.882593 -0.882419 -0.826457 -0.853826 -0.718051 -0.926962 -0.901023 -0.923242 -0.870424 -0.000000 -0.850305 -0.917159 -0.921319 -0.921190 -0.928276 -0.824692 -0.820211 -0.899825 -0.902174 -0.898520 -0.929705 -0.770882 -0.906576 -0.916638 -0.863008 -0.905446 -0.937377 -0.860483 -0.873073 -0.859844 -0.781712 -0.925182 -0.778392 -0.944693 -0.841203 -0.910795 -0.857501 -0.855692 -0.930540 -0.783420 -0.947284 -0.900920 -0.839578 -0.863207 -0.896811 -0.846356 -0.860102 -0.783164 -0.773147 -0.963827 -0.736903 -0.774224 -0.856889 -0.727164 -0.943078 -0.924663 -0.890158 -0.873912 -0.870228 -0.858766 -0.836158 -0.866324 -0.905507 -0.864355 -0.908694 -0.739865 -0.850305 -0.000000 -0.881150 -0.898662 -0.860559 -0.890558 -0.803062 -0.636981 -0.887731 -0.936790 -0.920664 -0.907248 -0.774277 -0.867285 -0.878913 -0.718362 -0.887245 -0.912964 -0.863577 -0.863759 -0.826435 -0.819138 -0.899484 -0.829570 -0.926428 -0.841469 -0.820646 -0.799146 -0.833500 -0.926227 -0.904176 -0.948317 -0.792339 -0.813228 -0.797057 -0.779932 -0.864241 -0.863821 -0.727386 -0.869150 -0.928214 -0.890843 -0.825276 -0.807442 -0.822670 -0.853192 -0.833253 -0.954408 -0.790672 -0.957783 -0.958165 -0.808749 -0.893466 -0.796966 -0.802803 -0.687380 -0.787772 -0.917159 -0.881150 -0.000000 -0.780308 -0.796663 -0.755878 -0.940108 -0.895824 -0.954318 -0.912689 -0.892855 -0.750739 -0.911208 -0.939494 -0.800118 -0.900279 -0.777003 -0.793876 -0.817278 -0.823293 -0.955950 -0.903424 -0.764888 -0.892821 -0.826622 -0.834270 -0.910857 -0.827449 -0.821033 -0.957131 -0.899640 -0.947515 -0.728443 -0.865080 -0.867799 -0.790001 -0.855938 -0.902459 -0.849298 -0.840588 -0.922299 -0.895605 -0.791348 -0.873230 -0.857814 -0.883723 -0.862126 -0.947436 -0.841393 -0.952526 -0.959130 -0.867151 -0.914863 -0.692234 -0.737050 -0.763825 -0.836186 -0.921319 -0.898662 -0.780308 -0.000000 -0.751886 -0.719425 -0.936347 -0.892971 -0.952022 -0.903712 -0.882232 -0.714749 -0.926346 -0.934537 -0.737428 -0.905128 -0.832854 -0.752983 -0.858813 -0.858596 -0.949268 -0.925634 -0.723345 -0.890657 -0.874871 -0.883653 -0.921293 -0.786472 -0.806069 -0.947624 -0.913597 -0.936059 -0.784663 -0.860362 -0.850210 -0.825784 -0.791353 -0.889057 -0.827218 -0.830998 -0.928957 -0.853789 -0.781558 -0.857711 -0.809884 -0.891212 -0.867990 -0.930663 -0.829001 -0.923094 -0.960104 -0.855487 -0.912569 -0.803213 -0.769249 -0.803676 -0.800454 -0.921190 -0.860559 -0.796663 -0.751886 -0.000000 -0.782800 -0.933176 -0.865918 -0.928229 -0.908566 -0.893143 -0.782897 -0.922950 -0.915242 -0.781485 -0.859733 -0.830658 -0.809034 -0.858182 -0.856418 -0.929825 -0.914351 -0.790455 -0.884912 -0.875503 -0.867131 -0.893267 -0.797123 -0.839236 -0.935346 -0.917793 -0.950720 -0.711012 -0.865913 -0.855860 -0.764520 -0.862115 -0.887147 -0.833450 -0.858487 -0.926129 -0.901642 -0.813188 -0.852395 -0.842681 -0.869927 -0.852253 -0.955896 -0.834144 -0.958702 -0.963644 -0.851500 -0.912017 -0.714538 -0.764051 -0.749803 -0.831337 -0.928276 -0.890558 -0.755878 -0.719425 -0.782800 -0.000000 -0.943899 -0.897472 -0.956239 -0.914761 -0.890615 -0.639162 -0.931756 -0.937362 -0.712509 -0.900716 -0.807277 -0.729098 -0.862889 -0.853044 -0.953931 -0.923615 -0.641989 -0.899960 -0.857703 -0.872383 -0.916511 -0.794821 -0.811914 -0.952683 -0.914494 -0.929309 -0.940213 -0.884919 -0.898218 -0.927103 -0.878718 -0.871284 -0.874353 -0.802052 -0.971932 -0.797962 -0.865207 -0.900272 -0.861622 -0.966141 -0.958611 -0.850370 -0.910931 -0.802795 -0.801723 -0.884926 -0.878760 -0.939118 -0.918822 -0.944622 -0.900986 -0.824692 -0.803062 -0.940108 -0.936347 -0.933176 -0.943899 -0.000000 -0.762895 -0.832676 -0.950029 -0.945642 -0.945849 -0.788316 -0.853949 -0.918682 -0.802488 -0.930815 -0.947058 -0.912301 -0.904872 -0.790943 -0.843174 -0.944547 -0.852711 -0.958361 -0.879649 -0.867632 -0.875682 -0.883767 -0.905340 -0.917888 -0.949459 -0.899511 -0.834627 -0.850414 -0.899211 -0.816999 -0.854576 -0.789677 -0.760895 -0.966729 -0.663999 -0.743496 -0.842107 -0.648252 -0.941037 -0.928291 -0.882120 -0.871231 -0.857247 -0.839943 -0.817082 -0.869738 -0.894099 -0.864335 -0.908830 -0.796902 -0.820211 -0.636981 -0.895824 -0.892971 -0.865918 -0.897472 -0.762895 -0.000000 -0.881163 -0.933882 -0.912060 -0.904571 -0.729431 -0.855515 -0.872241 -0.684781 -0.884953 -0.919154 -0.862154 -0.850844 -0.793510 -0.779120 -0.906414 -0.821792 -0.929125 -0.823093 -0.796242 -0.783315 -0.824896 -0.921563 -0.901819 -0.803216 -0.952925 -0.905982 -0.938152 -0.945712 -0.853319 -0.904143 -0.921891 -0.852578 -0.964529 -0.851102 -0.906662 -0.937206 -0.917201 -0.962509 -0.950624 -0.542144 -0.929060 -0.699790 -0.892223 -0.919464 -0.924687 -0.954729 -0.924319 -0.960087 -0.941981 -0.899825 -0.887731 -0.954318 -0.952022 -0.928229 -0.956239 -0.832676 -0.881163 -0.000000 -0.935666 -0.952245 -0.958035 -0.895682 -0.762524 -0.933683 -0.836377 -0.937877 -0.958717 -0.939207 -0.937281 -0.863981 -0.912676 -0.957816 -0.876890 -0.941163 -0.904716 -0.774927 -0.914524 -0.925710 -0.781257 -0.940375 -0.926292 -0.916895 -0.904453 -0.904979 -0.904577 -0.895402 -0.902250 -0.903657 -0.920793 -0.906220 -0.910949 -0.919087 -0.914639 -0.913087 -0.916917 -0.900132 -0.932575 -0.883695 -0.943958 -0.968230 -0.894557 -0.906586 -0.911829 -0.903756 -0.900328 -0.913404 -0.902174 -0.936790 -0.912689 -0.903712 -0.908566 -0.914761 -0.950029 -0.933882 -0.935666 -0.000000 -0.831623 -0.913250 -0.937409 -0.925973 -0.907864 -0.930160 -0.898136 -0.902527 -0.910510 -0.887977 -0.943023 -0.923687 -0.897929 -0.861837 -0.897041 -0.902561 -0.928325 -0.922908 -0.912161 -0.931455 -0.900095 -0.947140 -0.897356 -0.892967 -0.886563 -0.876027 -0.888764 -0.893084 -0.874444 -0.901146 -0.904738 -0.898879 -0.891428 -0.887293 -0.885236 -0.906709 -0.896308 -0.944352 -0.857249 -0.944872 -0.958453 -0.873376 -0.897381 -0.889738 -0.882520 -0.883823 -0.884096 -0.898520 -0.920664 -0.892855 -0.882232 -0.893143 -0.890615 -0.945642 -0.912060 -0.952245 -0.831623 -0.000000 -0.889066 -0.919315 -0.939234 -0.894408 -0.917852 -0.877111 -0.880268 -0.890413 -0.869984 -0.935777 -0.897052 -0.885462 -0.843482 -0.899083 -0.885854 -0.930093 -0.897703 -0.880936 -0.946664 -0.878372 -0.953137 -0.714006 -0.877850 -0.871417 -0.767545 -0.866750 -0.888426 -0.835680 -0.859814 -0.925656 -0.908923 -0.820383 -0.870079 -0.853515 -0.864075 -0.844800 -0.957306 -0.848843 -0.960012 -0.968865 -0.869461 -0.916192 -0.722764 -0.767074 -0.738799 -0.842792 -0.929705 -0.907248 -0.750739 -0.714749 -0.782897 -0.639162 -0.945849 -0.904571 -0.958035 -0.913250 -0.889066 -0.000000 -0.937645 -0.938813 -0.735424 -0.906944 -0.808339 -0.720128 -0.879512 -0.856406 -0.950513 -0.927583 -0.639156 -0.910368 -0.849307 -0.884420 -0.926082 -0.805649 -0.821317 -0.953189 -0.916071 -0.954502 -0.929754 -0.845049 -0.833733 -0.913633 -0.894509 -0.869574 -0.838468 -0.852220 -0.974445 -0.782570 -0.840883 -0.843047 -0.834871 -0.956684 -0.949426 -0.904083 -0.874677 -0.876585 -0.781823 -0.843130 -0.826615 -0.932857 -0.913712 -0.923401 -0.856129 -0.770882 -0.774277 -0.911208 -0.926346 -0.922950 -0.931756 -0.788316 -0.729431 -0.895682 -0.937409 -0.919315 -0.937645 -0.000000 -0.897487 -0.924747 -0.800132 -0.913225 -0.945567 -0.844709 -0.858780 -0.834043 -0.730372 -0.929070 -0.803822 -0.949493 -0.832769 -0.884390 -0.872897 -0.865719 -0.943816 -0.881451 -0.770338 -0.932686 -0.896810 -0.909678 -0.929382 -0.845506 -0.889108 -0.898200 -0.869178 -0.961677 -0.823080 -0.882307 -0.912681 -0.869767 -0.948882 -0.936616 -0.780567 -0.909399 -0.849576 -0.930421 -0.893616 -0.916632 -0.931972 -0.909577 -0.942839 -0.905075 -0.906576 -0.867285 -0.939494 -0.934537 -0.915242 -0.937362 -0.853949 -0.855515 -0.762524 -0.925973 -0.939234 -0.938813 -0.897487 -0.000000 -0.908346 -0.839569 -0.917403 -0.941624 -0.921612 -0.901517 -0.860516 -0.903196 -0.937531 -0.859051 -0.928029 -0.885310 -0.720668 -0.897929 -0.898561 -0.719750 -0.933423 -0.946883 -0.636528 -0.864871 -0.858432 -0.777761 -0.829934 -0.853758 -0.843558 -0.822584 -0.928553 -0.859871 -0.785919 -0.864160 -0.821753 -0.889704 -0.874121 -0.932759 -0.832504 -0.932192 -0.955341 -0.853156 -0.919078 -0.726367 -0.737789 -0.793717 -0.813429 -0.916638 -0.878913 -0.800118 -0.737428 -0.781485 -0.712509 -0.918682 -0.872241 -0.933683 -0.907864 -0.894408 -0.735424 -0.924747 -0.908346 -0.000000 -0.867647 -0.819892 -0.776396 -0.864574 -0.852297 -0.921454 -0.914833 -0.740568 -0.884682 -0.871818 -0.870633 -0.907203 -0.771314 -0.796839 -0.944352 -0.918706 -0.935992 -0.897664 -0.849646 -0.862019 -0.891562 -0.688612 -0.868578 -0.787206 -0.726266 -0.963465 -0.661393 -0.760875 -0.865284 -0.783962 -0.934235 -0.915228 -0.827212 -0.871769 -0.785758 -0.867128 -0.836409 -0.873053 -0.900258 -0.850038 -0.906077 -0.812679 -0.863008 -0.718362 -0.900279 -0.905128 -0.859733 -0.900716 -0.802488 -0.684781 -0.836377 -0.930160 -0.917852 -0.906944 -0.800132 -0.839569 -0.867647 -0.000000 -0.890338 -0.917167 -0.867577 -0.865466 -0.736989 -0.805205 -0.901127 -0.801914 -0.919693 -0.835646 -0.835644 -0.820083 -0.837722 -0.907819 -0.907351 -0.932617 -0.822672 -0.742261 -0.753962 -0.816742 -0.856817 -0.812848 -0.784409 -0.866938 -0.921866 -0.871310 -0.846168 -0.676093 -0.812697 -0.817651 -0.801362 -0.939655 -0.774739 -0.945018 -0.959928 -0.735911 -0.903896 -0.825420 -0.844195 -0.787596 -0.800721 -0.905446 -0.887245 -0.777003 -0.832854 -0.830658 -0.807277 -0.930815 -0.884953 -0.937877 -0.898136 -0.877111 -0.808339 -0.913225 -0.917403 -0.819892 -0.890338 -0.000000 -0.813397 -0.729006 -0.751626 -0.936662 -0.888097 -0.805401 -0.885772 -0.792721 -0.722159 -0.886030 -0.846184 -0.855958 -0.933318 -0.898957 -0.952604 -0.736650 -0.875701 -0.874623 -0.767742 -0.873884 -0.885668 -0.853389 -0.881020 -0.913509 -0.908754 -0.845756 -0.869403 -0.865936 -0.863783 -0.831545 -0.955149 -0.838087 -0.959706 -0.972917 -0.876239 -0.924657 -0.761180 -0.792691 -0.766479 -0.846014 -0.937377 -0.912964 -0.793876 -0.752983 -0.809034 -0.729098 -0.947058 -0.919154 -0.958717 -0.902527 -0.880268 -0.720128 -0.945567 -0.941624 -0.776396 -0.917167 -0.813397 -0.000000 -0.880910 -0.855394 -0.957285 -0.932567 -0.717257 -0.907124 -0.850880 -0.876019 -0.929508 -0.840390 -0.831807 -0.953786 -0.919621 -0.947546 -0.875698 -0.618568 -0.522917 -0.851561 -0.871570 -0.832409 -0.770840 -0.855606 -0.948605 -0.862771 -0.828628 -0.557087 -0.803072 -0.901197 -0.890550 -0.929799 -0.717265 -0.932009 -0.920111 -0.634180 -0.862125 -0.873218 -0.849186 -0.837486 -0.788722 -0.860483 -0.863577 -0.817278 -0.858813 -0.858182 -0.862889 -0.912301 -0.862154 -0.939207 -0.910510 -0.890413 -0.879512 -0.844709 -0.921612 -0.864574 -0.867577 -0.729006 -0.880910 -0.000000 -0.670354 -0.928384 -0.824604 -0.869033 -0.851857 -0.895514 -0.582926 -0.893055 -0.842267 -0.836818 -0.945575 -0.881091 -0.933947 -0.854048 -0.684099 -0.647334 -0.839805 -0.851693 -0.805251 -0.795344 -0.840409 -0.931662 -0.841580 -0.825924 -0.680547 -0.806727 -0.867519 -0.850525 -0.924389 -0.660444 -0.931867 -0.938839 -0.708674 -0.883323 -0.861618 -0.847649 -0.825720 -0.801647 -0.873073 -0.863759 -0.823293 -0.858596 -0.856418 -0.853044 -0.904872 -0.850844 -0.937281 -0.887977 -0.869984 -0.856406 -0.858780 -0.901517 -0.852297 -0.865466 -0.751626 -0.855394 -0.670354 -0.000000 -0.915137 -0.837632 -0.846402 -0.842468 -0.849197 -0.644026 -0.894072 -0.838863 -0.822049 -0.929155 -0.888046 -0.949483 -0.940416 -0.901057 -0.917839 -0.936841 -0.881896 -0.902814 -0.880527 -0.841951 -0.969119 -0.760037 -0.879512 -0.915238 -0.892124 -0.966131 -0.953656 -0.871519 -0.924109 -0.833975 -0.778247 -0.898311 -0.898415 -0.942424 -0.920961 -0.942659 -0.918970 -0.859844 -0.826435 -0.955950 -0.949268 -0.929825 -0.953931 -0.790943 -0.793510 -0.863981 -0.943023 -0.935777 -0.950513 -0.834043 -0.860516 -0.921454 -0.736989 -0.936662 -0.957285 -0.928384 -0.915137 -0.000000 -0.850593 -0.952523 -0.825051 -0.957029 -0.906156 -0.903132 -0.877912 -0.880524 -0.932198 -0.923248 -0.959825 -0.927072 -0.824650 -0.813452 -0.907887 -0.872935 -0.867349 -0.825790 -0.864574 -0.973554 -0.795391 -0.855618 -0.815637 -0.849324 -0.945948 -0.933293 -0.916662 -0.867395 -0.900267 -0.866601 -0.826903 -0.837839 -0.925933 -0.905015 -0.918513 -0.832165 -0.781712 -0.819138 -0.903424 -0.925634 -0.914351 -0.923615 -0.843174 -0.779120 -0.912676 -0.923687 -0.897052 -0.927583 -0.730372 -0.903196 -0.914833 -0.805205 -0.888097 -0.932567 -0.824604 -0.837632 -0.850593 -0.000000 -0.921609 -0.809971 -0.935589 -0.815000 -0.888111 -0.883408 -0.871643 -0.946302 -0.854362 -0.949005 -0.708374 -0.861649 -0.853833 -0.748085 -0.860692 -0.881852 -0.827601 -0.861642 -0.920689 -0.896856 -0.815462 -0.858187 -0.855732 -0.858308 -0.832836 -0.955649 -0.834165 -0.957490 -0.963352 -0.850139 -0.913961 -0.728019 -0.770907 -0.744747 -0.829254 -0.925182 -0.899484 -0.764888 -0.723345 -0.790455 -0.641989 -0.944547 -0.906414 -0.957816 -0.897929 -0.885462 -0.639156 -0.929070 -0.937531 -0.740568 -0.901127 -0.805401 -0.717257 -0.869033 -0.846402 -0.952523 -0.921609 -0.000000 -0.891645 -0.846430 -0.867635 -0.924010 -0.813015 -0.819523 -0.953641 -0.911920 -0.911958 -0.894228 -0.834687 -0.838460 -0.885075 -0.827825 -0.851747 -0.817439 -0.828571 -0.943497 -0.787621 -0.832354 -0.846706 -0.837628 -0.927258 -0.912394 -0.859642 -0.834184 -0.874993 -0.891169 -0.811091 -0.816016 -0.893324 -0.881864 -0.898959 -0.846914 -0.778392 -0.829570 -0.892821 -0.890657 -0.884912 -0.899960 -0.852711 -0.821792 -0.876890 -0.861837 -0.843482 -0.910368 -0.803822 -0.859051 -0.884682 -0.801914 -0.885772 -0.907124 -0.851857 -0.842468 -0.825051 -0.809971 -0.891645 -0.000000 -0.912017 -0.833501 -0.869701 -0.862490 -0.833461 -0.914433 -0.852035 -0.927311 -0.867627 -0.870877 -0.883248 -0.866901 -0.876230 -0.858642 -0.861289 -0.908068 -0.910883 -0.916908 -0.903268 -0.877012 -0.879363 -0.744567 -0.736560 -0.944841 -0.854987 -0.953218 -0.977175 -0.847908 -0.936246 -0.879332 -0.879245 -0.825632 -0.874460 -0.944693 -0.926428 -0.826622 -0.874871 -0.875503 -0.857703 -0.958361 -0.929125 -0.941163 -0.897041 -0.899083 -0.849307 -0.949493 -0.928029 -0.871818 -0.919693 -0.792721 -0.850880 -0.895514 -0.849197 -0.957029 -0.935589 -0.846430 -0.912017 -0.000000 -0.869408 -0.914691 -0.908689 -0.906407 -0.927567 -0.925254 -0.930302 -0.880325 -0.542812 -0.602860 -0.856020 -0.862114 -0.783179 -0.751222 -0.833921 -0.947843 -0.808847 -0.835632 -0.555591 -0.789095 -0.882602 -0.857737 -0.895509 -0.707400 -0.903465 -0.921329 -0.612305 -0.855601 -0.892313 -0.856793 -0.847438 -0.784047 -0.841203 -0.841469 -0.834270 -0.883653 -0.867131 -0.872383 -0.879649 -0.823093 -0.904716 -0.902561 -0.885854 -0.884420 -0.832769 -0.885310 -0.870633 -0.835646 -0.722159 -0.876019 -0.582926 -0.644026 -0.906156 -0.815000 -0.867635 -0.833501 -0.869408 -0.000000 -0.861319 -0.857344 -0.829615 -0.917790 -0.863200 -0.752238 -0.921660 -0.880030 -0.895554 -0.925318 -0.828973 -0.884548 -0.871600 -0.872648 -0.959480 -0.808474 -0.860977 -0.875825 -0.798033 -0.940086 -0.920944 -0.811097 -0.887298 -0.874400 -0.933289 -0.871796 -0.914592 -0.921435 -0.901609 -0.922993 -0.853731 -0.910795 -0.820646 -0.910857 -0.921293 -0.893267 -0.916511 -0.867632 -0.796242 -0.774927 -0.928325 -0.930093 -0.926082 -0.884390 -0.720668 -0.907203 -0.835644 -0.886030 -0.929508 -0.893055 -0.894072 -0.903132 -0.888111 -0.924010 -0.869701 -0.914691 -0.861319 -0.000000 -0.888850 -0.884985 -0.763524 -0.932314 -0.945606 -0.789248 -0.845482 -0.845276 -0.786194 -0.828217 -0.865299 -0.798566 -0.684464 -0.940082 -0.818109 -0.641763 -0.838191 -0.794027 -0.924701 -0.908606 -0.907153 -0.836763 -0.907508 -0.897590 -0.848518 -0.884170 -0.801181 -0.737237 -0.837325 -0.785806 -0.857501 -0.799146 -0.827449 -0.786472 -0.797123 -0.794821 -0.875682 -0.783315 -0.914524 -0.922908 -0.897703 -0.805649 -0.872897 -0.897929 -0.771314 -0.820083 -0.846184 -0.840390 -0.842267 -0.838863 -0.877912 -0.883408 -0.813015 -0.862490 -0.908689 -0.857344 -0.888850 -0.000000 -0.717762 -0.940333 -0.900145 -0.937430 -0.808261 -0.831448 -0.819952 -0.723351 -0.847962 -0.861431 -0.809562 -0.789884 -0.936510 -0.824802 -0.794373 -0.831433 -0.816773 -0.917692 -0.890825 -0.916950 -0.816007 -0.919066 -0.901006 -0.816455 -0.869485 -0.814978 -0.785313 -0.823325 -0.742033 -0.855692 -0.833500 -0.821033 -0.806069 -0.839236 -0.811914 -0.883767 -0.824896 -0.925710 -0.912161 -0.880936 -0.821317 -0.865719 -0.898561 -0.796839 -0.837722 -0.855958 -0.831807 -0.836818 -0.822049 -0.880524 -0.871643 -0.819523 -0.833461 -0.906407 -0.829615 -0.884985 -0.717762 -0.000000 -0.937239 -0.888949 -0.707954 -0.954224 -0.921790 -0.943930 -0.947850 -0.883075 -0.906366 -0.937328 -0.908162 -0.960491 -0.906598 -0.929273 -0.941377 -0.926091 -0.954688 -0.938848 -0.816379 -0.934203 -0.862560 -0.955194 -0.927580 -0.943072 -0.951026 -0.933553 -0.955825 -0.944013 -0.930540 -0.926227 -0.957131 -0.947624 -0.935346 -0.952683 -0.905340 -0.921563 -0.781257 -0.931455 -0.946664 -0.953189 -0.943816 -0.719750 -0.944352 -0.907819 -0.933318 -0.953786 -0.945575 -0.929155 -0.932198 -0.946302 -0.953641 -0.914433 -0.927567 -0.917790 -0.763524 -0.940333 -0.937239 -0.000000 -0.949230 -0.949925 -0.920468 -0.877044 -0.872523 -0.888425 -0.910120 -0.908390 -0.855658 -0.896148 -0.957545 -0.887389 -0.887581 -0.864181 -0.874297 -0.928355 -0.920676 -0.934616 -0.883222 -0.934037 -0.935049 -0.867447 -0.744470 -0.925351 -0.906292 -0.890439 -0.881855 -0.783420 -0.904176 -0.899640 -0.913597 -0.917793 -0.914494 -0.917888 -0.901819 -0.940375 -0.900095 -0.878372 -0.916071 -0.881451 -0.933423 -0.918706 -0.907351 -0.898957 -0.919621 -0.881091 -0.888046 -0.923248 -0.854362 -0.911920 -0.852035 -0.925254 -0.863200 -0.932314 -0.900145 -0.888949 -0.949230 -0.000000 -# pcoa 7 -2.168005 -1.574025 -1.227855 -0.900611 -0.704779 -0.581530 -0.559244 --0.192075 --0.207311 --0.348201 --0.097258 -0.088603 -0.231521 -0.059255 -0.280048 --0.185872 -0.074479 --0.037334 -0.079270 --0.001951 -0.033606 -0.027739 -0.341673 --0.095510 --0.107295 -0.068901 --0.077782 -0.014564 -0.077884 -0.364900 --0.084066 --0.072390 -0.064100 -0.001953 --0.018466 -0.219950 --0.085335 -0.051682 -0.037181 -0.093314 --0.069101 -0.067707 --0.077937 --0.100183 -0.048756 --0.154636 --0.120306 --0.059691 --0.274384 --0.038334 -0.079469 --0.071806 -0.003354 -0.021635 --0.083695 -0.108326 -0.042096 -0.144665 -0.074504 --0.023725 --0.073548 -0.023292 -0.024106 --0.104899 --0.059740 -0.203512 --0.106816 -0.008558 --0.107489 --0.039773 -0.042856 --0.144491 --0.163523 -0.173162 --0.124125 -0.127131 --0.239202 --0.229443 -0.012475 -0.194221 --0.097726 --0.129551 --0.028685 --0.087311 -0.006787 --0.054166 -0.249236 --0.121856 --0.009655 --0.010173 --0.048338 -0.075680 -0.374098 --0.082520 --0.091528 -0.033436 -0.029490 --0.013894 --0.031920 -0.086668 -0.149950 --0.105206 --0.123219 -0.200629 --0.091900 -0.145995 --0.031906 --0.215689 -0.140329 --0.318254 --0.062663 -0.123972 -0.157521 --0.038314 --0.193321 -0.113082 --0.316252 --0.032439 -0.069508 --0.328734 --0.187233 --0.189316 --0.108424 -0.105680 --0.265798 --0.004367 -0.115909 -0.208368 --0.089977 --0.052814 -0.020370 --0.012117 --0.098755 --0.312927 --0.158982 --0.064739 --0.055882 -0.005194 --0.246936 --0.069985 --0.252646 --0.034188 -0.108059 -0.207065 --0.025757 --0.009759 -0.273261 -0.037024 -0.316102 --0.081534 --0.070383 -0.004637 --0.054179 -0.070929 --0.116689 -0.080958 -0.035224 -0.308609 -0.178686 -0.076792 --0.056186 -0.262588 --0.175840 -0.065301 --0.003661 -0.095006 -0.009553 -0.020488 -0.170877 --0.137210 -0.117488 --0.078427 -0.071994 --0.048350 --0.081237 -0.271248 --0.045139 --0.022187 -0.039911 --0.025813 --0.050702 -0.076074 -0.062794 -0.095236 -0.130382 --0.072658 --0.023898 -0.071467 -0.002213 --0.196860 -0.078374 -0.089664 -0.294806 -0.183966 -0.054528 --0.001771 --0.186684 -0.030587 -0.232855 --0.054548 --0.140039 -0.145553 -0.050403 -0.249164 -0.004253 --0.012192 --0.000330 --0.033895 -0.029764 -0.060379 -0.269771 --0.176470 -0.070173 --0.018950 -0.074526 -0.015746 --0.033918 -0.162830 --0.117626 -0.065009 --0.095115 --0.046510 -0.065788 --0.155267 -0.308090 --0.155099 -0.051736 --0.015288 -0.099203 -0.042273 -0.037244 --0.286694 --0.043648 -0.103329 -0.062286 --0.027735 --0.000761 -0.108388 --0.226487 -0.047757 -0.267113 --0.062763 --0.130721 -0.136719 -0.012490 --0.339884 --0.218370 --0.225095 --0.116981 -0.066345 --0.204247 -0.052749 --0.002003 --0.062462 --0.163352 -0.228329 --0.016391 --0.034144 --0.218505 -0.033690 --0.024752 --0.088974 -0.229494 --0.001732 --0.065266 --0.252932 -0.309604 --0.180729 -0.036605 -0.025882 -0.060809 --0.049508 -0.127139 --0.231294 -0.114033 -0.152245 -0.172568 -0.018677 -0.086941 -0.128806 --0.265657 --0.156657 --0.195836 --0.150766 -0.082936 -0.124123 -0.028828 -0.225437 --0.156579 -0.094597 --0.072473 -0.106094 --0.053832 --0.010189 --0.237767 --0.022535 -0.204800 --0.108568 --0.156155 --0.080263 --0.103420 -0.181521 -0.147557 --0.141015 --0.049381 --0.089806 --0.007473 -0.043550 -0.280903 --0.150726 --0.009160 -0.030012 -0.016795 -0.044567 --0.026837 -0.074686 -0.373055 --0.088752 --0.095494 -0.103198 --0.015283 -0.021577 -0.085172 -0.274134 --0.097577 --0.067013 -0.007028 --0.060041 -0.016655 --0.278475 --0.062192 -0.129874 -0.087910 --0.124719 --0.123203 -0.106648 --0.176028 -0.151982 -0.100615 -0.164741 -0.050861 --0.006560 -0.075675 -0.307862 --0.153358 -0.029753 -0.009904 -0.054364 --0.021066 -0.071642 --0.153611 -0.043243 -0.035722 -0.147258 -0.058168 --0.070435 --0.100033 -0.142762 --0.050531 --0.218945 -0.092206 --0.328138 -0.009011 -0.042064 -0.015417 -0.382789 --0.106001 --0.104048 -0.046575 --0.042862 --0.047175 --0.223217 --0.102351 --0.151940 --0.191822 -0.018894 -0.281433 --0.011423 -0.065335 --0.098850 -0.236787 --0.107015 -0.020345 -0.032019 -0.056171 -0.069956 --0.021944 -0.143036 --0.002126 -0.126027 --0.045578 -0.101623 --0.232695 --0.202490 --0.321746 --0.119927 -0.120516 -0.172395 -0.071798 --0.056234 -0.050902 --0.023732 -0.322540 -0.131511 -0.089340 --0.172564 -0.102652 -0.074528 -0.058137 -0.042643 -0.033370 -0.027535 -0.026479 diff --git a/sucpp/test_su.cpp b/sucpp/test_su.cpp deleted file mode 100644 index 9ef3d334..00000000 --- a/sucpp/test_su.cpp +++ /dev/null @@ -1,1872 +0,0 @@ -#include -#include "api.hpp" -#include "tree.hpp" -#include "biom.hpp" -#include "unifrac.hpp" -#include "unifrac_internal.hpp" -#include -#include -#include - -/* - * test harness adapted from - * https://github.com/noporpoise/BitArray/blob/master/dev/bit_array_test.c - */ -const char *suite_name; -char suite_pass; -int suites_run = 0, suites_failed = 0, suites_empty = 0; -int tests_in_suite = 0, tests_run = 0, tests_failed = 0; - -#define QUOTE(str) #str -#define ASSERT(x) {tests_run++; tests_in_suite++; if(!(x)) \ - { fprintf(stderr, "failed assert [%s:%i] %s\n", __FILE__, __LINE__, QUOTE(x)); \ - suite_pass = 0; tests_failed++; }} - -void SUITE_START(const char *name) { - suite_pass = 1; - suite_name = name; - suites_run++; - tests_in_suite = 0; -} - -void SUITE_END() { - printf("Testing %s ", suite_name); - size_t suite_i; - for(suite_i = strlen(suite_name); suite_i < 80-8-5; suite_i++) printf("."); - printf("%s\n", suite_pass ? " pass" : " fail"); - if(!suite_pass) suites_failed++; - if(!tests_in_suite) suites_empty++; -} -/* - * End adapted code - */ - -std::vector _bool_array_to_vector(bool *arr, unsigned int n) { - std::vector vec; - - for(unsigned int i = 0; i < n; i++) - vec.push_back(arr[i]); - - return vec; -} - -std::vector _uint32_array_to_vector(uint32_t *arr, unsigned int n) { - std::vector vec; - - for(unsigned int i = 0; i < n; i++) - vec.push_back(arr[i]); - - return vec; -} - -std::vector _double_array_to_vector(double *arr, unsigned int n) { - std::vector vec; - - for(unsigned int i = 0; i < n; i++) - vec.push_back(arr[i]); - - return vec; -} - -std::vector _string_array_to_vector(std::string *arr, unsigned int n) { - std::vector vec; - - for(unsigned int i = 0; i < n; i++) - vec.push_back(arr[i]); - - return vec; -} - -bool vec_almost_equal(std::vector a, std::vector b) { - if(a.size() != b.size()) { - return false; - } - for(unsigned int i = 0; i < a.size(); i++) { - if(!(fabs(a[i] - b[i]) < 0.000001)) { // sufficient given the tests - return false; - } - } - return true; -} - - -void test_bptree_constructor_simple() { - SUITE_START("bptree constructor simple"); - //01234567 - //11101000 - su::BPTree tree = su::BPTree("(('123:foo; bar':1,b:2)c);"); - - unsigned int exp_nparens = 8; - - std::vector exp_structure; - exp_structure.push_back(true); - exp_structure.push_back(true); - exp_structure.push_back(true); - exp_structure.push_back(false); - exp_structure.push_back(true); - exp_structure.push_back(false); - exp_structure.push_back(false); - exp_structure.push_back(false); - - std::vector exp_openclose; - exp_openclose.push_back(7); - exp_openclose.push_back(6); - exp_openclose.push_back(3); - exp_openclose.push_back(2); - exp_openclose.push_back(5); - exp_openclose.push_back(4); - exp_openclose.push_back(1); - exp_openclose.push_back(0); - - std::vector exp_names; - exp_names.push_back(std::string()); - exp_names.push_back(std::string("c")); - exp_names.push_back(std::string("123:foo; bar")); - exp_names.push_back(std::string()); - exp_names.push_back(std::string("b")); - exp_names.push_back(std::string()); - exp_names.push_back(std::string()); - exp_names.push_back(std::string()); - - std::vector exp_lengths; - exp_lengths.push_back(0.0); - exp_lengths.push_back(0.0); - exp_lengths.push_back(1.0); - exp_lengths.push_back(0.0); - exp_lengths.push_back(2.0); - exp_lengths.push_back(0.0); - exp_lengths.push_back(0.0); - exp_lengths.push_back(0.0); - - ASSERT(tree.nparens == exp_nparens); - ASSERT(tree.get_structure() == exp_structure); - ASSERT(tree.get_openclose() == exp_openclose); - ASSERT(tree.lengths == exp_lengths); - ASSERT(tree.names == exp_names); - - SUITE_END(); -} - -void test_bptree_constructor_from_existing() { - SUITE_START("bptree constructor from_existing"); - //01234567 - //11101000 - su::BPTree existing = su::BPTree("(('123:foo; bar':1,b:2)c);"); - su::BPTree tree = su::BPTree(existing.get_structure(), existing.lengths, existing.names); - - unsigned int exp_nparens = 8; - - std::vector exp_structure; - exp_structure.push_back(true); - exp_structure.push_back(true); - exp_structure.push_back(true); - exp_structure.push_back(false); - exp_structure.push_back(true); - exp_structure.push_back(false); - exp_structure.push_back(false); - exp_structure.push_back(false); - - std::vector exp_openclose; - exp_openclose.push_back(7); - exp_openclose.push_back(6); - exp_openclose.push_back(3); - exp_openclose.push_back(2); - exp_openclose.push_back(5); - exp_openclose.push_back(4); - exp_openclose.push_back(1); - exp_openclose.push_back(0); - - std::vector exp_names; - exp_names.push_back(std::string()); - exp_names.push_back(std::string("c")); - exp_names.push_back(std::string("123:foo; bar")); - exp_names.push_back(std::string()); - exp_names.push_back(std::string("b")); - exp_names.push_back(std::string()); - exp_names.push_back(std::string()); - exp_names.push_back(std::string()); - - std::vector exp_lengths; - exp_lengths.push_back(0.0); - exp_lengths.push_back(0.0); - exp_lengths.push_back(1.0); - exp_lengths.push_back(0.0); - exp_lengths.push_back(2.0); - exp_lengths.push_back(0.0); - exp_lengths.push_back(0.0); - exp_lengths.push_back(0.0); - - ASSERT(tree.nparens == exp_nparens); - ASSERT(tree.get_structure() == exp_structure); - ASSERT(tree.get_openclose() == exp_openclose); - ASSERT(tree.lengths == exp_lengths); - ASSERT(tree.names == exp_names); - - SUITE_END(); -} - -void test_bptree_mask() { - SUITE_START("bptree mask"); - //01234567 - //11101000 - //111000 - std::vector mask = {true, true, true, true, false, false, true, true}; - su::BPTree base = su::BPTree("(('123:foo; bar':1,b:2)c);"); - su::BPTree tree = base.mask(mask, base.lengths); - unsigned int exp_nparens = 6; - - std::vector exp_structure; - exp_structure.push_back(true); - exp_structure.push_back(true); - exp_structure.push_back(true); - exp_structure.push_back(false); - exp_structure.push_back(false); - exp_structure.push_back(false); - - std::vector exp_openclose; - exp_openclose.push_back(5); - exp_openclose.push_back(4); - exp_openclose.push_back(3); - exp_openclose.push_back(2); - exp_openclose.push_back(1); - exp_openclose.push_back(0); - - std::vector exp_names; - exp_names.push_back(std::string()); - exp_names.push_back(std::string("c")); - exp_names.push_back(std::string("123:foo; bar")); - exp_names.push_back(std::string()); - exp_names.push_back(std::string()); - exp_names.push_back(std::string()); - - std::vector exp_lengths; - exp_lengths.push_back(0.0); - exp_lengths.push_back(0.0); - exp_lengths.push_back(1.0); - exp_lengths.push_back(0.0); - exp_lengths.push_back(0.0); - exp_lengths.push_back(0.0); - - ASSERT(tree.nparens == exp_nparens); - ASSERT(tree.get_structure() == exp_structure); - ASSERT(tree.get_openclose() == exp_openclose); - ASSERT(tree.lengths == exp_lengths); - ASSERT(tree.names == exp_names); - - SUITE_END(); -} - -void test_bptree_constructor_single_descendent() { - SUITE_START("bptree constructor single descendent"); - - su::BPTree tree = su::BPTree("(((a)b)c,((d)e)f,g)r;"); - - unsigned int exp_nparens = 16; - - bool structure_arr[] = {1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0}; - std::vector exp_structure = _bool_array_to_vector(structure_arr, exp_nparens); - - double length_arr[] = {0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}; - std::vector exp_lengths = _double_array_to_vector(length_arr, exp_nparens); - - std::string names_arr[] = {"r", "c", "b", "a", "", "", "", "f", "e", "d", "", "", "", "g", "", ""}; - std::vector exp_names = _string_array_to_vector(names_arr, exp_nparens); - - ASSERT(tree.nparens == exp_nparens); - ASSERT(tree.get_structure() == exp_structure); - ASSERT(vec_almost_equal(tree.lengths, exp_lengths)); - ASSERT(tree.names == exp_names); - - SUITE_END(); -} - -void test_bptree_constructor_complex() { - SUITE_START("bp tree constructor complex"); - su::BPTree tree = su::BPTree("(((a:1,b:2.5)c:6,d:8,(e),(f,g,(h:1,i:2)j:1)k:1.2)l,m:2)r;"); - - unsigned int exp_nparens = 30; - - bool structure_arr[] = {1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0}; - std::vector exp_structure = _bool_array_to_vector(structure_arr, exp_nparens); - - double length_arr[] = {0, 0, 6, 1, 0, 2.5, 0, 0, 8, 0, 0, 0, 0, 0, 1.2, 0, 0, 0, 0, 1, 1, 0, 2, 0, 0, 0, 0, 2, 0, 0}; - std::vector exp_lengths = _double_array_to_vector(length_arr, exp_nparens); - - std::string names_arr[] = {"r", "l", "c", "a", "", "b", "", "", "d", "", "", "e", "", "", "k", "f", "", "g", "", "j", "h", "", "i", "", "", "", "", "m", "", ""}; - std::vector exp_names = _string_array_to_vector(names_arr, exp_nparens); - - ASSERT(tree.nparens == exp_nparens); - ASSERT(tree.get_structure() == exp_structure); - ASSERT(vec_almost_equal(tree.lengths, exp_lengths)); - ASSERT(tree.names == exp_names); - SUITE_END(); -} - -void test_bptree_constructor_semicolon() { - SUITE_START("bp tree constructor semicolon"); - su::BPTree tree = su::BPTree("((a,(b,c):5)'d','e; foo':10,((f))g)r;"); - - unsigned int exp_nparens = 20; - - bool structure_arr[] = {1, 1, 1, 0, 1, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0}; - std::vector exp_structure = _bool_array_to_vector(structure_arr, exp_nparens); - - double length_arr[] = {0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0}; - std::vector exp_lengths = _double_array_to_vector(length_arr, exp_nparens); - - std::string names_arr[] = {"r", "d", "a", "", "", "b", "", "c", "", "", "", "e; foo", "", "g", "", "f", "", "", "", ""}; - std::vector exp_names = _string_array_to_vector(names_arr, exp_nparens); - - ASSERT(tree.nparens == exp_nparens); - ASSERT(tree.get_structure() == exp_structure); - ASSERT(vec_almost_equal(tree.lengths, exp_lengths)); - ASSERT(tree.names == exp_names); - SUITE_END(); -} - -void test_bptree_constructor_edgecases() { - SUITE_START("bp tree constructor edgecases"); - - su::BPTree tree1 = su::BPTree("((a,b));"); - bool structure_arr1[] = {1, 1, 1, 0, 1, 0, 0, 0}; - std::vector exp_structure1 = _bool_array_to_vector(structure_arr1, 8); - - su::BPTree tree2 = su::BPTree("(a);"); - bool structure_arr2[] = {1, 1, 0, 0}; - std::vector exp_structure2 = _bool_array_to_vector(structure_arr2, 4); - - su::BPTree tree3 = su::BPTree("();"); - bool structure_arr3[] = {1, 1, 0, 0}; - std::vector exp_structure3 = _bool_array_to_vector(structure_arr3, 4); - - su::BPTree tree4 = su::BPTree("((a,b),c);"); - bool structure_arr4[] = {1, 1, 1, 0, 1, 0, 0, 1, 0, 0}; - std::vector exp_structure4 = _bool_array_to_vector(structure_arr4, 10); - - su::BPTree tree5 = su::BPTree("(a,(b,c));"); - bool structure_arr5[] = {1, 1, 0, 1, 1, 0, 1, 0, 0, 0}; - std::vector exp_structure5 = _bool_array_to_vector(structure_arr5, 10); - - ASSERT(tree1.get_structure() == exp_structure1); - ASSERT(tree2.get_structure() == exp_structure2); - ASSERT(tree3.get_structure() == exp_structure3); - ASSERT(tree4.get_structure() == exp_structure4); - ASSERT(tree5.get_structure() == exp_structure5); - - SUITE_END(); -} - -void test_bptree_constructor_quoted_comma() { - SUITE_START("quoted comma bug"); - su::BPTree tree = su::BPTree("((3,'foo,bar')x,c)r;"); - std::vector exp_names = {"r", "x", "3", "", "foo,bar", "", "", "c", "", ""}; - ASSERT(exp_names.size() == tree.names.size()); - - for(unsigned int i = 0; i < tree.names.size(); i++) { - ASSERT(exp_names[i] == tree.names[i]); - } - SUITE_END(); -} - -void test_bptree_constructor_quoted_parens() { - SUITE_START("quoted parens"); - su::BPTree tree = su::BPTree("((3,'foo(b)ar')x,c)r;"); - std::vector exp_names = {"r", "x", "3", "", "foo(b)ar", "", "", "c", "", ""}; - ASSERT(exp_names.size() == tree.names.size()); - - for(unsigned int i = 0; i < tree.names.size(); i++) { - ASSERT(exp_names[i] == tree.names[i]); - } - SUITE_END(); -} -void test_bptree_postorder() { - SUITE_START("postorderselect"); - - // fig1 from https://www.dcc.uchile.cl/~gnavarro/ps/tcs16.2.pdf - su::BPTree tree = su::BPTree("((3,4,(6)5)2,7,((10,100)9)8)1;"); - uint32_t exp[] = {2, 4, 7, 6, 1, 11, 15, 17, 14, 13, 0}; - uint32_t obs[tree.nparens / 2]; - - for(unsigned int i = 0; i < (tree.nparens / 2); i++) - obs[i] = tree.postorderselect(i); - - std::vector exp_v = _uint32_array_to_vector(exp, tree.nparens / 2); - std::vector obs_v = _uint32_array_to_vector(obs, tree.nparens / 2); - - ASSERT(obs_v == exp_v); - SUITE_END(); -} - -void test_bptree_preorder() { - SUITE_START("preorderselect"); - - // fig1 from https://www.dcc.uchile.cl/~gnavarro/ps/tcs16.2.pdf - su::BPTree tree = su::BPTree("((3,4,(6)5)2,7,((10,100)9)8)1;"); - uint32_t exp[] = {0, 1, 2, 4, 6, 7, 11, 13, 14, 15, 17}; - uint32_t obs[tree.nparens / 2]; - - for(unsigned int i = 0; i < (tree.nparens / 2); i++) - obs[i] = tree.preorderselect(i); - - std::vector exp_v = _uint32_array_to_vector(exp, tree.nparens / 2); - std::vector obs_v = _uint32_array_to_vector(obs, tree.nparens / 2); - - ASSERT(obs_v == exp_v); - SUITE_END(); -} - -void test_bptree_parent() { - SUITE_START("parent"); - - // fig1 from https://www.dcc.uchile.cl/~gnavarro/ps/tcs16.2.pdf - su::BPTree tree = su::BPTree("((3,4,(6)5)2,7,((10,100)9)8)1;"); - uint32_t exp[] = {0, 1, 1, 1, 1, 1, 6, 6, 1, 0, 0, 0, 0, 13, 14, 14, 14, 14, 13, 0}; - - // all the -2 and +1 garbage is to avoid testing the root. - uint32_t obs[tree.nparens - 2]; - - for(int i = 0; i < (int(tree.nparens) - 2); i++) - obs[i] = tree.parent(i+1); - - std::vector exp_v = _uint32_array_to_vector(exp, tree.nparens - 2); - std::vector obs_v = _uint32_array_to_vector(obs, tree.nparens - 2); - - ASSERT(obs_v == exp_v); - SUITE_END(); -} - -void test_biom_constructor() { - SUITE_START("biom constructor"); - - su::biom table = su::biom("test.biom"); - uint32_t exp_n_samples = 6; - uint32_t exp_n_obs = 5; - - std::string sids[] = {"Sample1", "Sample2", "Sample3", "Sample4", "Sample5", "Sample6"}; - std::vector exp_sids = _string_array_to_vector(sids, exp_n_samples); - - std::string oids[] = {"GG_OTU_1", "GG_OTU_2","GG_OTU_3", "GG_OTU_4", "GG_OTU_5"}; - std::vector exp_oids = _string_array_to_vector(oids, exp_n_obs); - - uint32_t s_indptr[] = {0, 2, 5, 9, 11, 12, 15}; - std::vector exp_s_indptr = _uint32_array_to_vector(s_indptr, exp_n_samples + 1); - - uint32_t o_indptr[] = {0, 1, 6, 9, 13, 15}; - std::vector exp_o_indptr = _uint32_array_to_vector(o_indptr, exp_n_obs + 1); - - uint32_t exp_nnz = 15; - - ASSERT(table.n_samples == exp_n_samples); - ASSERT(table.n_obs == exp_n_obs); - ASSERT(table.nnz == exp_nnz); - ASSERT(table.sample_ids == exp_sids); - ASSERT(table.obs_ids == exp_oids); - ASSERT(table.sample_indptr == exp_s_indptr); - ASSERT(table.obs_indptr == exp_o_indptr); - - SUITE_END(); -} - -void test_biom_get_obs_data() { - SUITE_START("biom get obs data"); - - su::biom table = su::biom("test.biom"); - double exp0[] = {0.0, 0.0, 1.0, 0.0, 0.0, 0.0}; - std::vector exp0_vec = _double_array_to_vector(exp0, 6); - double exp1[] = {5.0, 1.0, 0.0, 2.0, 3.0, 1.0}; - std::vector exp1_vec = _double_array_to_vector(exp1, 6); - double exp2[] = {0.0, 0.0, 1.0, 4.0, 0.0, 2.0}; - std::vector exp2_vec = _double_array_to_vector(exp2, 6); - double exp3[] = {2.0, 1.0, 1.0, 0.0, 0.0, 1.0}; - std::vector exp3_vec = _double_array_to_vector(exp3, 6); - double exp4[] = {0.0, 1.0, 1.0, 0.0, 0.0, 0.0}; - std::vector exp4_vec = _double_array_to_vector(exp4, 6); - - double *out = (double*)malloc(sizeof(double) * 6); - std::vector obs_vec; - - table.get_obs_data(std::string("GG_OTU_1").c_str(), out); - obs_vec = _double_array_to_vector(out, 6); - ASSERT(vec_almost_equal(obs_vec, exp0_vec)); - - table.get_obs_data(std::string("GG_OTU_2").c_str(), out); - obs_vec = _double_array_to_vector(out, 6); - ASSERT(vec_almost_equal(obs_vec, exp1_vec)); - - table.get_obs_data(std::string("GG_OTU_3").c_str(), out); - obs_vec = _double_array_to_vector(out, 6); - ASSERT(vec_almost_equal(obs_vec, exp2_vec)); - - table.get_obs_data(std::string("GG_OTU_4").c_str(), out); - obs_vec = _double_array_to_vector(out, 6); - ASSERT(vec_almost_equal(obs_vec, exp3_vec)); - - table.get_obs_data(std::string("GG_OTU_5").c_str(), out); - obs_vec = _double_array_to_vector(out, 6); - ASSERT(vec_almost_equal(obs_vec, exp4_vec)); - - free(out); - SUITE_END(); -} - -void test_bptree_leftchild() { - SUITE_START("test bptree left child"); - su::BPTree tree = su::BPTree("((3,4,(6)5)2,7,((10,100)9)8)1;"); - - uint32_t exp[] = {1, 2, 0, 0, 7, 0, 0, 14, 15, 0, 0}; - std::vector structure = tree.get_structure(); - - uint32_t exp_pos = 0; - for(unsigned int i = 0; i < tree.nparens; i++) { - if(structure[i]) - ASSERT(tree.leftchild(i) == exp[exp_pos++]); - } - SUITE_END(); -} - -void test_bptree_rightchild() { - SUITE_START("test bptree right child"); - su::BPTree tree = su::BPTree("((3,4,(6)5)2,7,((10,100)9)8)1;"); - - uint32_t exp[] = {13, 6, 0, 0, 7, 0, 0, 14, 17, 0, 0}; - std::vector structure = tree.get_structure(); - - uint32_t exp_pos = 0; - for(unsigned int i = 0; i < tree.nparens; i++) { - if(structure[i]) - ASSERT(tree.rightchild(i) == exp[exp_pos++]); - } - SUITE_END(); -} - -void test_bptree_rightsibling() { - SUITE_START("test bptree rightsibling"); - su::BPTree tree = su::BPTree("((3,4,(6)5)2,7,((10,100)9)8)1;"); - - uint32_t exp[] = {0, 11, 4, 6, 0, 0, 13, 0, 0, 17, 0}; - std::vector structure = tree.get_structure(); - - uint32_t exp_pos = 0; - for(unsigned int i = 0; i < tree.nparens; i++) { - if(structure[i]) - ASSERT(tree.rightsibling(i) == exp[exp_pos++]); - } - SUITE_END(); -} - -void test_propstack_constructor() { - SUITE_START("test propstack constructor"); - su::PropStack ps(10); - // nothing to test directly... - SUITE_END(); -} - -void test_propstack_push_and_pop() { - SUITE_START("test propstack push and pop"); - su::PropStack ps(10); - - double *vec1 = ps.pop(1); - double *vec2 = ps.pop(2); - double *vec3 = ps.pop(3); - double *vec1_obs; - double *vec2_obs; - double *vec3_obs; - - ps.push(1); - ps.push(2); - ps.push(3); - - vec3_obs = ps.pop(4); - vec2_obs = ps.pop(5); - vec1_obs = ps.pop(6); - - ASSERT(vec1 == vec1_obs); - ASSERT(vec2 == vec2_obs); - ASSERT(vec3 == vec3_obs); - SUITE_END(); -} - -void test_propstack_get() { - SUITE_START("test propstack get"); - su::PropStack ps(10); - - double *vec1 = ps.pop(1); - double *vec2 = ps.pop(2); - double *vec3 = ps.pop(3); - - double *vec1_obs = ps.get(1); - double *vec2_obs = ps.get(2); - double *vec3_obs = ps.get(3); - - ASSERT(vec1 == vec1_obs); - ASSERT(vec2 == vec2_obs); - ASSERT(vec3 == vec3_obs); - SUITE_END(); -} - -void test_unifrac_set_proportions() { - SUITE_START("test unifrac set proportions"); - // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 - // ( ( ) ( ( ) ( ) ) ( ( ) ( ) ) ) - su::BPTree tree = su::BPTree("(GG_OTU_1,(GG_OTU_2,GG_OTU_3),(GG_OTU_5,GG_OTU_4));"); - su::biom table = su::biom("test.biom"); - su::PropStack ps(table.n_samples); - - double *obs = ps.pop(4); // GG_OTU_2 - double exp4[] = {0.714285714286, 0.333333333333, 0.0, 0.333333333333, 1.0, 0.25}; - set_proportions(obs, tree, 4, table, ps); - for(unsigned int i = 0; i < table.n_samples; i++) - ASSERT(fabs(obs[i] - exp4[i]) < 0.000001); - - obs = ps.pop(6); // GG_OTU_3 - double exp6[] = {0.0, 0.0, 0.25, 0.666666666667, 0.0, 0.5}; - set_proportions(obs, tree, 6, table, ps); - for(unsigned int i = 0; i < table.n_samples; i++) - ASSERT(fabs(obs[i] - exp6[i]) < 0.000001); - - obs = ps.pop(3); // node containing GG_OTU_2 and GG_OTU_3 - double exp3[] = {0.71428571, 0.33333333, 0.25, 1.0, 1.0, 0.75}; - set_proportions(obs, tree, 3, table, ps); - for(unsigned int i = 0; i < table.n_samples; i++) - ASSERT(fabs(obs[i] - exp3[i]) < 0.000001); - SUITE_END(); -} - -void test_unifrac_set_proportions_range() { - SUITE_START("test unifrac set proportions range"); - // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 - // ( ( ) ( ( ) ( ) ) ( ( ) ( ) ) ) - su::BPTree tree = su::BPTree("(GG_OTU_1,(GG_OTU_2,GG_OTU_3),(GG_OTU_5,GG_OTU_4));"); - su::biom table = su::biom("test.biom"); - - const double exp4[] = {0.714285714286, 0.333333333333, 0.0, 0.333333333333, 1.0, 0.25}; - const double exp6[] = {0.0, 0.0, 0.25, 0.666666666667, 0.0, 0.5}; - const double exp3[] = {0.71428571, 0.33333333, 0.25, 1.0, 1.0, 0.75}; - - - // first the whole table - { - su::PropStack ps(table.n_samples); - - double *obs = ps.pop(4); // GG_OTU_2 - set_proportions_range(obs, tree, 4, table, 0, table.n_samples, ps); - for(unsigned int i = 0; i < table.n_samples; i++) - ASSERT(fabs(obs[i] - exp4[i]) < 0.000001); - - obs = ps.pop(6); // GG_OTU_3 - set_proportions_range(obs, tree, 6, table, 0, table.n_samples, ps); - for(unsigned int i = 0; i < table.n_samples; i++) - ASSERT(fabs(obs[i] - exp6[i]) < 0.000001); - - obs = ps.pop(3); // node containing GG_OTU_2 and GG_OTU_3 - set_proportions_range(obs, tree, 3, table, 0, table.n_samples, ps); - for(unsigned int i = 0; i < table.n_samples; i++) - ASSERT(fabs(obs[i] - exp3[i]) < 0.000001); - } - - // beginning - { - su::PropStack ps(3); - - double *obs = ps.pop(4); // GG_OTU_2 - set_proportions_range(obs, tree, 4, table, 0, 3, ps); - for(unsigned int i = 0; i < 3; i++) - ASSERT(fabs(obs[i] - exp4[i]) < 0.000001); - - obs = ps.pop(6); // GG_OTU_3 - set_proportions_range(obs, tree, 6, table, 0, 3, ps); - for(unsigned int i = 0; i < 3; i++) - ASSERT(fabs(obs[i] - exp6[i]) < 0.000001); - - obs = ps.pop(3); // node containing GG_OTU_2 and GG_OTU_3 - set_proportions_range(obs, tree, 3, table, 0, 3, ps); - for(unsigned int i = 0; i < 3; i++) - ASSERT(fabs(obs[i] - exp3[i]) < 0.000001); - } - - - // end - { - su::PropStack ps(4); - - double *obs = ps.pop(4); // GG_OTU_2 - set_proportions_range(obs, tree, 4, table, 2, table.n_samples, ps); - for(unsigned int i = 2; i < table.n_samples; i++) - ASSERT(fabs(obs[i-2] - exp4[i]) < 0.000001); - - obs = ps.pop(6); // GG_OTU_3 - set_proportions_range(obs, tree, 6, table, 2, table.n_samples, ps); - for(unsigned int i = 2; i < table.n_samples; i++) - ASSERT(fabs(obs[i-2] - exp6[i]) < 0.000001); - - obs = ps.pop(3); // node containing GG_OTU_2 and GG_OTU_3 - set_proportions_range(obs, tree, 3, table, 2, table.n_samples, ps); - for(unsigned int i = 2; i < table.n_samples; i++) - ASSERT(fabs(obs[i-2] - exp3[i]) < 0.000001); - } - - - // middle - { - const unsigned int start = 1; - const unsigned int end = 4; - su::PropStack ps(end-start); - - double *obs = ps.pop(4); // GG_OTU_2 - set_proportions_range(obs, tree, 4, table, start, end, ps); - for(unsigned int i =start; i < end; i++) - ASSERT(fabs(obs[i-start] - exp4[i]) < 0.000001); - - obs = ps.pop(6); // GG_OTU_3 - set_proportions_range(obs, tree, 6, table, start, end, ps); - for(unsigned int i = start; i < end; i++) - ASSERT(fabs(obs[i-start] - exp6[i]) < 0.000001); - - obs = ps.pop(3); // node containing GG_OTU_2 and GG_OTU_3 - set_proportions_range(obs, tree, 3, table, start, end, ps); - for(unsigned int i = start; i < end; i++) - ASSERT(fabs(obs[i-start] - exp3[i]) < 0.000001); - } - - SUITE_END(); -} - -void test_unifrac_set_proportions_range_float() { - SUITE_START("test unifrac set proportions range float"); - // 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 - // ( ( ) ( ( ) ( ) ) ( ( ) ( ) ) ) - su::BPTree tree = su::BPTree("(GG_OTU_1,(GG_OTU_2,GG_OTU_3),(GG_OTU_5,GG_OTU_4));"); - su::biom table = su::biom("test.biom"); - - const float exp4[] = {0.714285714286, 0.333333333333, 0.0, 0.333333333333, 1.0, 0.25}; - const float exp6[] = {0.0, 0.0, 0.25, 0.666666666667, 0.0, 0.5}; - const float exp3[] = {0.71428571, 0.33333333, 0.25, 1.0, 1.0, 0.75}; - - // just midle - { - const unsigned int start = 1; - const unsigned int end = 4; - su::PropStack ps(end-start); - - float *obs = ps.pop(4); // GG_OTU_2 - set_proportions_range(obs, tree, 4, table, start, end, ps); - for(unsigned int i =start; i < end; i++) - ASSERT(fabs(obs[i-start] - exp4[i]) < 0.000001); - - obs = ps.pop(6); // GG_OTU_3 - set_proportions_range(obs, tree, 6, table, start, end, ps); - for(unsigned int i = start; i < end; i++) - ASSERT(fabs(obs[i-start] - exp6[i]) < 0.000001); - - obs = ps.pop(3); // node containing GG_OTU_2 and GG_OTU_3 - set_proportions_range(obs, tree, 3, table, start, end, ps); - for(unsigned int i = start; i < end; i++) - ASSERT(fabs(obs[i-start] - exp3[i]) < 0.000001); - } - - SUITE_END(); -} - - - -void test_unifrac_deconvolute_stripes() { - SUITE_START("test deconvolute stripes"); - std::vector stripes; - double s1[] = {1, 1, 1, 1, 1, 1}; - double s2[] = {2, 2, 2, 2, 2, 2}; - double s3[] = {3, 3, 3, 3, 3, 3}; - stripes.push_back(s1); - stripes.push_back(s2); - stripes.push_back(s3); - - double exp[6][6] = { {0, 1, 2, 3, 2, 1}, - {1, 0, 1, 2, 3, 2}, - {2, 1, 0, 1, 2, 3}, - {3, 2, 1, 0, 1, 2}, - {2, 3, 2, 1, 0, 1}, - {1, 2, 3, 2, 1, 0} }; - double **obs = su::deconvolute_stripes(stripes, 6); - for(unsigned int i = 0; i < 6; i++) { - for(unsigned int j = 0; j < 6; j++) { - ASSERT(exp[i][j] == obs[i][j]); - } - } - free(obs); - SUITE_END(); -} - -void test_unifrac_stripes_to_condensed_form_even() { - SUITE_START("test stripes_to_condensed_form even samples"); - std::vector stripes; - double s1[] = {0, 9, 17, 24, 30, 35, 39, 42, 44, 8}; - double s2[] = {1, 10, 18, 25, 31, 36, 40, 43, 7, 16}; - double s3[] = {2, 11, 19, 26, 32, 37, 41, 6, 15, 23}; - double s4[] = {3, 12, 20, 27, 33, 38, 5, 14, 22, 29}; - double s5[] = {4, 13, 21, 28, 34, 4, 13, 21, 28, 34}; - stripes.push_back(s1); - stripes.push_back(s2); - stripes.push_back(s3); - stripes.push_back(s4); - stripes.push_back(s5); - - double exp[45] = {/* 0, */ 0, 1, 2, 3, 4, 5, 6, 7, 8, - /* *, 0, */ 9, 10, 11, 12, 13, 14, 15, 16, - /* *, *, 0, */ 17, 18, 19, 20, 21, 22, 23, - /* *, *, *, 0, */ 24, 25, 26, 27, 28, 29, - /* *, *, *, *, 0, */ 30, 31, 32, 33, 34, - /* *, *, *, *, *, 0, */ 35, 36, 37, 38, - /* *, *, *, *, *, *, 0, */ 39, 40, 41, - /* *, *, *, *, *, *, *, 0, */ 42, 43, - /* *, *, *, *, *, *, *, *, 0, */ 44}; - /* *, *, *, *, *, *, *, *, *, *, 0 */ - - double *obs = (double*)malloc(sizeof(double) * 45); - su::stripes_to_condensed_form(stripes, 10, obs, 0, 5); - for(unsigned int i = 0; i < 45; i++) { - ASSERT(exp[i] == obs[i]); - } - free(obs); - SUITE_END(); -} - -void test_unifrac_stripes_to_condensed_form_odd() { - SUITE_START("test stripes_to_condensed_form odd samples"); - std::vector stripes; - double s1[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0}; - double s2[] = {20, 19, 18, 17, 16, 15, 14 ,13, 12, 11, 1}; - double s3[] = {21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 2}; - double s4[] = {40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 3}; - double s5[] = {41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 4}; - stripes.push_back(s1); - stripes.push_back(s2); - stripes.push_back(s3); - stripes.push_back(s4); - stripes.push_back(s5); - - double exp[55] = {/* 0, */ 1, 20, 21, 40, 41, 47, 33, 29, 11, 0, - /* 1, 0, */ 2, 19, 22, 39, 42, 48, 32, 30, 1, - /*20, 2, 0, */ 3, 18, 23, 38, 43, 49, 31, 2, - /*21, 19, 3, 0, */ 4, 17, 24, 37, 44, 50, 3, - /*40, 22, 18, 4, 0, */ 5, 16, 25 ,36, 45 , 4, - /*41, 39, 23, 17, 5, 0, */ 6, 15, 26, 35, 46, - /*47, 42, 38, 24, 16, 6, 0, */ 7, 14, 27, 34, - /*33, 48, 43, 37, 25, 15, 7, 0,*/ 8, 13, 28, - /*29, 32, 49, 44, 36, 26, 14, 8, 0, */ 9, 12, - /*11, 30, 31, 50, 45, 35, 27, 13, 9, 0,*/ 10}; - /* 0, 1, 2, 3, 4, 46, 34, 28, 12, 10, 0}; */ - double *obs = (double*)malloc(sizeof(double) * 55); - su::stripes_to_condensed_form(stripes, 11, obs, 0, 5); - for(unsigned int i = 0; i < 55; i++) { - ASSERT(exp[i] == obs[i]); - } - free(obs); - SUITE_END(); -} - -void test_unifrac_stripes_to_condensed_form_odd2() { - SUITE_START("test stripes_to_condensed_form odd(2) samples"); - std::vector stripes; - double s1[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9}; - double s2[] = {18, 17, 16, 15, 14, 13, 12 ,11, 10}; - double s3[] = {19, 20, 21, 22, 23, 24, 25, 26, 27}; - double s4[] = {36, 35, 34, 33, 32, 31, 30, 29, 28}; - double s5[] = {31, 30, 29, 28, 36, 35, 34, 33, 32}; - stripes.push_back(s1); - stripes.push_back(s2); - stripes.push_back(s3); - stripes.push_back(s4); - stripes.push_back(s5); - - double exp[36] = {/* 0, */ 1, 18, 19, 36, 31, 25, 11, 9, - /* 1, 0, */ 2, 17, 20, 35, 30, 26, 10, - /*20, 2, 0, */ 3, 16, 21, 34, 29, 27, - /*21, 19, 3, 0, */ 4, 15, 22, 33, 28, - /*40, 22, 18, 4, 0, */ 5, 14, 23 ,32, - /*41, 39, 23, 17, 5, 0, */ 6, 13, 24, - /*47, 42, 38, 24, 16, 6, 0, */ 7, 12, - /*47, 42, 38, 24, 16, 6, 7, 0, */ 8}; - /* 0, 1, 2, 3, 4, 46, 34, 8, 8, 0}; */ - double *obs = (double*)malloc(sizeof(double) * 36); - su::stripes_to_condensed_form(stripes, 9, obs, 0, 5); - for(unsigned int i = 0; i < 36; i++) { - ASSERT(exp[i] == obs[i]); - } - free(obs); - SUITE_END(); -} - -class ValidatedMemoryStripes : public su::MemoryStripes { - private: - const uint32_t n_stripes; - mutable std::vector stripe_status; // 0 new, 1 allocated, 2 deallocated, 3 reallocated, 6 deallocate after rellocation - public: - ValidatedMemoryStripes(uint32_t _n_stripes, std::vector &_stripes) - : su::MemoryStripes(_stripes) - , n_stripes(_n_stripes) - , stripe_status(n_stripes) - { - for (uint32_t i=0; i2); - return out; - } - - -}; - - -void test_unifrac_stripes_to_matrix_even() { - SUITE_START("test stripes_to_matrix even samples"); - std::vector stripes; - double s1[] = {0, 9, 17, 24, 30, 35, 39, 42, 44, 8}; - double s2[] = {1, 10, 18, 25, 31, 36, 40, 43, 7, 16}; - double s3[] = {2, 11, 19, 26, 32, 37, 41, 6, 15, 23}; - double s4[] = {3, 12, 20, 27, 33, 38, 5, 14, 22, 29}; - double s5[] = {4, 13, 21, 28, 34, 4, 13, 21, 28, 34}; - stripes.push_back(s1); - stripes.push_back(s2); - stripes.push_back(s3); - stripes.push_back(s4); - stripes.push_back(s5); - - // test also double to float conversion - float exp[100] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, - 0, 0, 9, 10, 11, 12, 13, 14, 15, 16, - 1, 9, 0, 17, 18, 19, 20, 21, 22, 23, - 2, 10, 17, 0, 24, 25, 26, 27, 28, 29, - 3, 11, 18, 24, 0, 30, 31, 32, 33, 34, - 4, 12, 19, 25, 30, 0, 35, 36, 37, 38, - 5, 13, 20, 26, 31, 35, 0, 39, 40, 41, - 6, 14, 21, 27, 32, 36, 39, 0, 42, 43, - 7, 15, 22, 28, 33, 37, 40, 42, 0, 44, - 8, 16, 23, 29, 34, 38, 41, 43, 44, 0}; - { - float *obs = (float*)malloc(sizeof(float) * 100); - ValidatedMemoryStripes vs(5,stripes); - su::stripes_to_matrix_fp32(vs, 10, 5, obs); - for(unsigned int i = 0; i < 100; i++) { - ASSERT(exp[i] == obs[i]); - } - - ASSERT(vs.allInitialized() == true); - ASSERT(vs.allDealocated() == true); - ASSERT(vs.anyRealocated() == false); - - free(obs); - } - - { // small tiles - float *obs = (float*)malloc(sizeof(float) * 100); - ValidatedMemoryStripes vs(5,stripes); - su::stripes_to_matrix_fp32(vs, 10, 5, obs, 4); - for(unsigned int i = 0; i < 100; i++) { - ASSERT(exp[i] == obs[i]); - } - - ASSERT(vs.allInitialized() == true); - ASSERT(vs.allDealocated() == true); - ASSERT(vs.anyRealocated() == false); - - free(obs); - } - - { // large tiles - float *obs = (float*)malloc(sizeof(float) * 100); - ValidatedMemoryStripes vs(5,stripes); - su::stripes_to_matrix_fp32(vs, 10, 5, obs, 128); - for(unsigned int i = 0; i < 100; i++) { - ASSERT(exp[i] == obs[i]); - } - - ASSERT(vs.allInitialized() == true); - ASSERT(vs.allDealocated() == true); - ASSERT(vs.anyRealocated() == false); - - free(obs); - } - - - // test also intermediate, 2-step procedure - double *obsC = (double*)malloc(sizeof(double) * 45); - su::stripes_to_condensed_form(stripes, 10, obsC, 0, 5); - - float *obs2 = (float*)malloc(sizeof(float) * 100); - su::condensed_form_to_matrix_fp32(obsC, 10, obs2); - - for(unsigned int i = 0; i < 100; i++) { - ASSERT(exp[i] == obs2[i]); - } - - free(obs2); - free(obsC); - SUITE_END(); -} - -void test_unifrac_stripes_to_matrix_odd() { - SUITE_START("test stripes_to_matrix odd samples"); - std::vector stripes; - double s1[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0}; - double s2[] = {20, 19, 18, 17, 16, 15, 14 ,13, 12, 11, 1}; - double s3[] = {21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 2}; - double s4[] = {40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 3}; - double s5[] = {41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 4}; - stripes.push_back(s1); - stripes.push_back(s2); - stripes.push_back(s3); - stripes.push_back(s4); - stripes.push_back(s5); - - double exp[121] = { 0, 1, 20, 21, 40, 41, 47, 33, 29, 11, 0, - 1, 0, 2, 19, 22, 39, 42, 48, 32, 30, 1, - 20, 2, 0, 3, 18, 23, 38, 43, 49, 31, 2, - 21, 19, 3, 0, 4, 17, 24, 37, 44, 50, 3, - 40, 22, 18, 4, 0, 5, 16, 25 ,36, 45 , 4, - 41, 39, 23, 17, 5, 0, 6, 15, 26, 35, 46, - 47, 42, 38, 24, 16, 6, 0, 7, 14, 27, 34, - 33, 48, 43, 37, 25, 15, 7, 0, 8, 13, 28, - 29, 32, 49, 44, 36, 26, 14, 8, 0, 9, 12, - 11, 30, 31, 50, 45, 35, 27, 13, 9, 0, 10, - 0, 1, 2, 3, 4, 46, 34, 28, 12, 10, 0}; - - { - double *obs = (double*)malloc(sizeof(double) * 121); - ValidatedMemoryStripes vs(5,stripes); - su::stripes_to_matrix(vs, 11, 5, obs); - for(unsigned int i = 0; i < 121; i++) { - ASSERT(exp[i] == obs[i]); - } - ASSERT(vs.allInitialized() == true); - ASSERT(vs.allDealocated() == true); - ASSERT(vs.anyRealocated() == false); - free(obs); - } - - { // small tiling - double *obs = (double*)malloc(sizeof(double) * 121); - ValidatedMemoryStripes vs(5,stripes); - su::stripes_to_matrix(vs, 11, 5, obs,4); - for(unsigned int i = 0; i < 121; i++) { - ASSERT(exp[i] == obs[i]); - } - ASSERT(vs.allInitialized() == true); - ASSERT(vs.allDealocated() == true); - ASSERT(vs.anyRealocated() == false); - free(obs); - } - - { // large tiling - double *obs = (double*)malloc(sizeof(double) * 121); - ValidatedMemoryStripes vs(5,stripes); - su::stripes_to_matrix(vs, 11, 5, obs,128); - for(unsigned int i = 0; i < 121; i++) { - ASSERT(exp[i] == obs[i]); - } - ASSERT(vs.allInitialized() == true); - ASSERT(vs.allDealocated() == true); - ASSERT(vs.anyRealocated() == false); - free(obs); - } - - - // test also intermediate, 2-step procedure - double *obsC = (double*)malloc(sizeof(double) * 55); - su::stripes_to_condensed_form(stripes, 11, obsC, 0, 5); - - double *obs2 = (double*)malloc(sizeof(double) * 121); - su::condensed_form_to_matrix(obsC, 11, obs2); - - for(unsigned int i = 0; i < 121; i++) { - ASSERT(exp[i] == obs2[i]); - } - - free(obs2); - free(obsC); - SUITE_END(); -} - -void test_unifrac_stripes_to_matrix_odd2() { - SUITE_START("test stripes_to_matrix odd(2) samples"); - std::vector stripes; - double s1[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9}; - double s2[] = {18, 17, 16, 15, 14, 13, 12 ,11, 10}; - double s3[] = {19, 20, 21, 22, 23, 24, 25, 26, 27}; - double s4[] = {36, 35, 34, 33, 32, 31, 30, 29, 28}; - double s5[] = {31, 30, 29, 28, 36, 35, 34, 33, 32}; - stripes.push_back(s1); - stripes.push_back(s2); - stripes.push_back(s3); - stripes.push_back(s4); - stripes.push_back(s5); - - double exp[81] = { 0, 1, 18, 19, 36, 31, 25, 11, 9, - 1, 0, 2, 17, 20, 35, 30, 26, 10, - 18, 2, 0, 3, 16, 21, 34, 29, 27, - 19, 17, 3, 0, 4, 15, 22, 33, 28, - 36, 20, 16, 4, 0, 5, 14, 23 ,32, - 31, 35, 21, 15, 5, 0, 6, 13, 24, - 25, 30, 34, 22, 14, 6, 0, 7, 12, - 11, 26, 29, 33, 23, 13, 7, 0, 8, - 9, 10, 27, 28, 32, 24, 12, 8, 0}; - - { - double *obs = (double*)malloc(sizeof(double) * 81); - ValidatedMemoryStripes vs(5,stripes); - su::stripes_to_matrix(vs, 9, 5, obs); - for(unsigned int i = 0; i < 81; i++) { - ASSERT(exp[i] == obs[i]); - } - ASSERT(vs.allInitialized() == true); - ASSERT(vs.allDealocated() == true); - ASSERT(vs.anyRealocated() == false); - free(obs); - } - - { // small tile - double *obs = (double*)malloc(sizeof(double) * 81); - ValidatedMemoryStripes vs(5,stripes); - su::stripes_to_matrix(vs, 9, 5, obs,4); - for(unsigned int i = 0; i < 81; i++) { - ASSERT(exp[i] == obs[i]); - } - ASSERT(vs.allInitialized() == true); - ASSERT(vs.allDealocated() == true); - ASSERT(vs.anyRealocated() == false); - free(obs); - } - - { // large tile - double *obs = (double*)malloc(sizeof(double) * 81); - ValidatedMemoryStripes vs(5,stripes); - su::stripes_to_matrix(vs, 9, 5, obs,128); - for(unsigned int i = 0; i < 81; i++) { - ASSERT(exp[i] == obs[i]); - } - ASSERT(vs.allInitialized() == true); - ASSERT(vs.allDealocated() == true); - ASSERT(vs.anyRealocated() == false); - free(obs); - } - - // test also intermediate, 2-step procedure - double *obsC = (double*)malloc(sizeof(double) * 36); - su::stripes_to_condensed_form(stripes, 9, obsC, 0, 5); - - double *obs2 = (double*)malloc(sizeof(double) * 81); - su::condensed_form_to_matrix(obsC, 9, obs2); - - for(unsigned int i = 0; i < 81; i++) { - ASSERT(exp[i] == obs2[i]); - } - - free(obs2); - free(obsC); - SUITE_END(); -} - - -void test_unnormalized_weighted_unifrac() { - SUITE_START("test unnormalized weighted unifrac"); - - std::vector threads(1); - su::BPTree tree = su::BPTree("(GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1):1,(GG_OTU_5:1,GG_OTU_4:1):1);"); - su::biom table = su::biom("test.biom"); - - std::vector exp; - double stride1[] = {1.52380952, 1.25, 2.75, 1.33333333, 2., 1.07142857}; - double stride2[] = {2.17857143, 2.66666667, 3.25, 1.0, 1.14285714, 1.83333333}; - double stride3[] = {1.9047619, 2.66666667, 1.75, 1.9047619, 2.66666667, 1.75}; - exp.push_back(stride1); - exp.push_back(stride2); - exp.push_back(stride3); - std::vector strides = su::make_strides(6); - std::vector strides_total = su::make_strides(6); - - su::task_parameters task_p; - task_p.start = 0; task_p.stop = 3; task_p.tid = 0; task_p.n_samples = 6; task_p.bypass_tips = false; - - std::vector tasks; - tasks.push_back(task_p); - su::process_stripes(std::ref(table), - std::ref(tree), - su::weighted_unnormalized, - false, - std::ref(strides), - std::ref(strides_total), - std::ref(threads), - std::ref(tasks)); - - for(unsigned int i = 0; i < 3; i++) { - for(unsigned int j = 0; j < 6; j++) { - ASSERT(fabs(strides[i][j] - exp[i][j]) < 0.000001); - } - free(strides[i]); - } - SUITE_END(); -} - -void test_generalized_unifrac() { - SUITE_START("test generalized unifrac"); - - std::vector threads(1); - su::BPTree tree = su::BPTree("(GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1):1,(GG_OTU_5:1,GG_OTU_4:1):1);"); - su::biom table = su::biom("test.biom"); - - // weighted normalized unifrac as computed above - std::vector w_exp; - double w_stride1[] = {0.38095238, 0.33333333, 0.73333333, 0.33333333, 0.5, 0.26785714}; - double w_stride2[] = {0.58095238, 0.66666667, 0.86666667, 0.25, 0.28571429, 0.45833333}; - double w_stride3[] = {0.47619048, 0.66666667, 0.46666667, 0.47619048, 0.66666667, 0.46666667}; - w_exp.push_back(w_stride1); - w_exp.push_back(w_stride2); - w_exp.push_back(w_stride3); - std::vector w_strides = su::make_strides(6); - std::vector w_strides_total = su::make_strides(6); - su::task_parameters w_task_p; - w_task_p.start = 0; w_task_p.stop = 3; w_task_p.tid = 0; w_task_p.n_samples = 6; w_task_p.bypass_tips = false; - w_task_p.g_unifrac_alpha = 1.0; - - std::vector tasks; - tasks.push_back(w_task_p); - su::process_stripes(std::ref(table), - std::ref(tree), - su::generalized, - false, - std::ref(w_strides), - std::ref(w_strides_total), - std::ref(threads), - std::ref(tasks)); - - // as computed by GUniFrac v1.0 - // Sample1 Sample2 Sample3 Sample4 Sample5 Sample6 - //Sample1 0.0000000 0.4408392 0.6886965 0.7060606 0.5833333 0.3278410 - //Sample2 0.4408392 0.0000000 0.5102041 0.7500000 0.8000000 0.5208125 - //Sample3 0.6886965 0.5102041 0.0000000 0.8649351 0.9428571 0.5952381 - //Sample4 0.7060606 0.7500000 0.8649351 0.0000000 0.5000000 0.4857143 - //Sample5 0.5833333 0.8000000 0.9428571 0.5000000 0.0000000 0.7485714 - //Sample6 0.3278410 0.5208125 0.5952381 0.4857143 0.7485714 0.0000000 - std::vector d0_exp; - double d0_stride1[] = {0.4408392, 0.5102041, 0.8649351, 0.5000000, 0.7485714, 0.3278410}; - double d0_stride2[] = {0.6886965, 0.7500000, 0.9428571, 0.4857143, 0.5833333, 0.5208125}; - double d0_stride3[] = {0.7060606, 0.8000000, 0.5952381, 0.7060606, 0.8000000, 0.5952381}; - d0_exp.push_back(d0_stride1); - d0_exp.push_back(d0_stride2); - d0_exp.push_back(d0_stride3); - std::vector d0_strides = su::make_strides(6); - std::vector d0_strides_total = su::make_strides(6); - su::task_parameters d0_task_p; - d0_task_p.start = 0; d0_task_p.stop = 3; d0_task_p.tid = 0; d0_task_p.n_samples = 6; d0_task_p.bypass_tips = false; - d0_task_p.g_unifrac_alpha = 0.0; - - tasks.clear(); - tasks.push_back(d0_task_p); - su::process_stripes(std::ref(table), - std::ref(tree), - su::generalized, - false, - std::ref(d0_strides), - std::ref(d0_strides_total), - std::ref(threads), - std::ref(tasks)); - - // as computed by GUniFrac v1.0 - // Sample1 Sample2 Sample3 Sample4 Sample5 Sample6 - //Sample1 0.0000000 0.4040518 0.6285560 0.5869439 0.4082483 0.2995673 - //Sample2 0.4040518 0.0000000 0.4160597 0.7071068 0.7302479 0.4860856 - //Sample3 0.6285560 0.4160597 0.0000000 0.8005220 0.9073159 0.5218198 - //Sample4 0.5869439 0.7071068 0.8005220 0.0000000 0.4117216 0.3485667 - //Sample5 0.4082483 0.7302479 0.9073159 0.4117216 0.0000000 0.6188282 - //Sample6 0.2995673 0.4860856 0.5218198 0.3485667 0.6188282 0.0000000 - std::vector d05_exp; - double d05_stride1[] = {0.4040518, 0.4160597, 0.8005220, 0.4117216, 0.6188282, 0.2995673}; - double d05_stride2[] = {0.6285560, 0.7071068, 0.9073159, 0.3485667, 0.4082483, 0.4860856}; - double d05_stride3[] = {0.5869439, 0.7302479, 0.5218198, 0.5869439, 0.7302479, 0.5218198}; - d05_exp.push_back(d05_stride1); - d05_exp.push_back(d05_stride2); - d05_exp.push_back(d05_stride3); - std::vector d05_strides = su::make_strides(6); - std::vector d05_strides_total = su::make_strides(6); - su::task_parameters d05_task_p; - d05_task_p.start = 0; d05_task_p.stop = 3; d05_task_p.tid = 0; d05_task_p.n_samples = 6; d05_task_p.bypass_tips = false; - d05_task_p.g_unifrac_alpha = 0.5; - - tasks.clear(); - tasks.push_back(d05_task_p); - su::process_stripes(std::ref(table), - std::ref(tree), - su::generalized, - false, - std::ref(d05_strides), - std::ref(d05_strides_total), - std::ref(threads), - std::ref(tasks)); - - for(unsigned int i = 0; i < 3; i++) { - for(unsigned int j = 0; j < 6; j++) { - ASSERT(fabs(w_strides[i][j] - w_exp[i][j]) < 0.000001); - ASSERT(fabs(d0_strides[i][j] - d0_exp[i][j]) < 0.000001); - ASSERT(fabs(d05_strides[i][j] - d05_exp[i][j]) < 0.000001); - } - free(w_strides[i]); - free(d0_strides[i]); - free(d05_strides[i]); - } - SUITE_END(); -} - -void test_vaw_unifrac_weighted_normalized() { - SUITE_START("test vaw weighted normalized unifrac"); - - std::vector threads(1); - su::BPTree tree = su::BPTree("(GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1):1,(GG_OTU_5:1,GG_OTU_4:1):1);"); - su::biom table = su::biom("test.biom"); - - // as computed by GUniFrac, the original implementation of VAW-UniFrac - // could not be found. - // Sample1 Sample2 Sample3 Sample4 Sample5 Sample6 - //Sample1 0.0000000 0.4086040 0.6240185 0.4639481 0.2857143 0.2766318 - //Sample2 0.4086040 0.0000000 0.3798594 0.6884992 0.6807616 0.4735781 - //Sample3 0.6240185 0.3798594 0.0000000 0.7713254 0.8812897 0.5047114 - //Sample4 0.4639481 0.6884992 0.7713254 0.0000000 0.6666667 0.2709298 - //Sample5 0.2857143 0.6807616 0.8812897 0.6666667 0.0000000 0.4735991 - //Sample6 0.2766318 0.4735781 0.5047114 0.2709298 0.4735991 0.0000000 - // weighted normalized unifrac as computed above - - std::vector w_exp; - double w_stride1[] = {0.4086040, 0.3798594, 0.7713254, 0.6666667, 0.4735991, 0.2766318}; - double w_stride2[] = {0.6240185, 0.6884992, 0.8812897, 0.2709298, 0.2857143, 0.4735781}; - double w_stride3[] = {0.4639481, 0.6807616, 0.5047114, 0.4639481, 0.6807616, 0.5047114}; - w_exp.push_back(w_stride1); - w_exp.push_back(w_stride2); - w_exp.push_back(w_stride3); - std::vector w_strides = su::make_strides(6); - std::vector w_strides_total = su::make_strides(6); - su::task_parameters w_task_p; - w_task_p.start = 0; w_task_p.stop = 3; w_task_p.tid = 0; w_task_p.n_samples = 6; w_task_p.bypass_tips = false; - w_task_p.g_unifrac_alpha = 1.0; - - std::vector tasks; - tasks.push_back(w_task_p); - su::process_stripes(std::ref(table), - std::ref(tree), - su::weighted_normalized, - true, - std::ref(w_strides), - std::ref(w_strides_total), - std::ref(threads), - std::ref(tasks)); - - for(unsigned int i = 0; i < 3; i++) { - for(unsigned int j = 0; j < 6; j++) { - ASSERT(fabs(w_strides[i][j] - w_exp[i][j]) < 0.000001); - } - free(w_strides[i]); - } - SUITE_END(); -} - - -void test_make_strides() { - SUITE_START("test make stripes"); - std::vector exp; - double stride[] = {0., 0., 0.}; - exp.push_back(stride); - exp.push_back(stride); - exp.push_back(stride); - - std::vector obs = su::make_strides(3); - for(unsigned int i = 0; i < 3; i++) { - for(unsigned int j = 0; j < 6; j++) { - ASSERT(fabs(obs[i][j] - exp[i][j]) < 0.000001); - } - free(obs[i]); - } -} - -void test_faith_pd() { - SUITE_START("test faith PD"); - - // Note this tree is binary (opposed to example below) - su::BPTree tree = su::BPTree("((GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1):1):2,(GG_OTU_5:1,GG_OTU_4:1):1);"); - su::biom table = su::biom("test.biom"); - - // make vector of expectations from faith PD - double exp[6] = {6., 7., 8., 5., 4., 7.}; - - // run faith PD to get obs - double obs[6] = {0, 0, 0, 0, 0, 0}; - - su::faith_pd(table, tree, obs); - - // ASSERT that results = expectation - for (unsigned int i = 0; i < 6; i++){ - ASSERT(fabs(exp[i]-obs[i]) < 0.000001) - } - SUITE_END(); -} - -void test_faith_pd_shear(){ - SUITE_START("test faith PD extra OTUs in tree"); - - su::BPTree tree = su::BPTree("((GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1,GG_OTU_ex:9):1):2,(GG_OTU_5:1,GG_OTU_4:1,GG_OTU_ex2:12):1);"); - su::biom table = su::biom("test.biom"); - - // make vector of expectations from faith PD - double exp[6] = {6., 7., 8., 5., 4., 7.}; - - // run faith PD to get obs - double obs[6] = {0, 0, 0, 0, 0, 0}; - - std::unordered_set to_keep(table.obs_ids.begin(), \ - table.obs_ids.end()); \ - su::BPTree tree_sheared = tree.shear(to_keep).collapse(); - su::faith_pd(table, tree_sheared, obs); - - // ASSERT that results = expectation - for (unsigned int i = 0; i < 6; i++){ - ASSERT(fabs(exp[i]-obs[i]) < 0.000001) - } - SUITE_END(); -} - -void test_unweighted_unifrac() { - SUITE_START("test unweighted unifrac"); - std::vector threads(1); - su::BPTree tree = su::BPTree("(GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1):1,(GG_OTU_5:1,GG_OTU_4:1):1);"); - su::biom table = su::biom("test.biom"); - - std::vector exp; - double stride1[] = {0.2, 0.42857143, 0.71428571, 0.33333333, 0.6, 0.2}; - double stride2[] = {0.57142857, 0.66666667, 0.85714286, 0.4, 0.5, 0.33333333}; - double stride3[] = {0.6, 0.6, 0.42857143, 0.6, 0.6, 0.42857143}; - exp.push_back(stride1); - exp.push_back(stride2); - exp.push_back(stride3); - std::vector strides = su::make_strides(6); - std::vector strides_total = su::make_strides(6); - - su::task_parameters task_p; - task_p.start = 0; task_p.stop = 3; task_p.tid = 0; task_p.n_samples = 6; task_p.bypass_tips = false; - - std::vector tasks; - tasks.push_back(task_p); - su::process_stripes(std::ref(table), - std::ref(tree), - su::unweighted, - false, - std::ref(strides), - std::ref(strides_total), - std::ref(threads), - std::ref(tasks)); - - for(unsigned int i = 0; i < 3; i++) { - for(unsigned int j = 0; j < 6; j++) { - ASSERT(fabs(strides[i][j] - exp[i][j]) < 0.000001); - } - free(strides[i]); - } - SUITE_END(); -} - -void test_unweighted_unifrac_fast() { - SUITE_START("test unweighted unifrac no tips"); - std::vector threads(1); - su::BPTree tree = su::BPTree("(GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1):1,(GG_OTU_5:1,GG_OTU_4:1):1);"); - su::biom table = su::biom("test.biom"); - - std::vector exp; - double stride1[] = {0., 0., 0.5, 0., 0.5, 0.}; - double stride2[] = {0., 0.5, 0.5, 0.5, 0.5, 0.}; - double stride3[] = {0.5, 0.5, 0., 0.5, 0.5, 0.}; - exp.push_back(stride1); - exp.push_back(stride2); - exp.push_back(stride3); - std::vector strides = su::make_strides(6); - std::vector strides_total = su::make_strides(6); - - su::task_parameters task_p; - task_p.start = 0; task_p.stop = 3; task_p.tid = 0; task_p.n_samples = 6; task_p.bypass_tips = true; - - std::vector tasks; - tasks.push_back(task_p); - su::process_stripes(std::ref(table), - std::ref(tree), - su::unweighted, - false, - std::ref(strides), - std::ref(strides_total), - std::ref(threads), - std::ref(tasks)); - - for(unsigned int i = 0; i < 3; i++) { - for(unsigned int j = 0; j < 6; j++) { - ASSERT(fabs(strides[i][j] - exp[i][j]) < 0.000001); - } - free(strides[i]); - } - SUITE_END(); -} - -void test_normalized_weighted_unifrac() { - SUITE_START("test normalized weighted unifrac"); - std::vector threads(1); - su::BPTree tree = su::BPTree("(GG_OTU_1:1,(GG_OTU_2:1,GG_OTU_3:1):1,(GG_OTU_5:1,GG_OTU_4:1):1);"); - su::biom table = su::biom("test.biom"); - - std::vector exp; - double stride1[] = {0.38095238, 0.33333333, 0.73333333, 0.33333333, 0.5, 0.26785714}; - double stride2[] = {0.58095238, 0.66666667, 0.86666667, 0.25, 0.28571429, 0.45833333}; - double stride3[] = {0.47619048, 0.66666667, 0.46666667, 0.47619048, 0.66666667, 0.46666667}; - exp.push_back(stride1); - exp.push_back(stride2); - exp.push_back(stride3); - std::vector strides = su::make_strides(6); - std::vector strides_total = su::make_strides(6); - - su::task_parameters task_p; - task_p.start = 0; task_p.stop = 3; task_p.tid = 0; task_p.n_samples = 6; task_p.bypass_tips = false; - - - std::vector tasks; - tasks.push_back(task_p); - su::process_stripes(std::ref(table), - std::ref(tree), - su::weighted_normalized, - false, - std::ref(strides), - std::ref(strides_total), - std::ref(threads), - std::ref(tasks)); - - for(unsigned int i = 0; i < 3; i++) { - for(unsigned int j = 0; j < 6; j++) { - ASSERT(fabs(strides[i][j] - exp[i][j]) < 0.000001); - } - free(strides[i]); - } - SUITE_END(); -} - -void test_bptree_shear_simple() { - SUITE_START("test bptree shear simple"); - su::BPTree tree = su::BPTree("((3:2,4:3,(6:5)5:4)2:1,7:6,((10:9,11:10)9:8)8:7)r"); - - // simple - std::unordered_set to_keep = {"4", "6", "7", "10", "11"}; - - uint32_t exp_nparens = 20; - std::vector exp_structure = {true, true, true, false, true, true, false, false, false, true, - false, true, true, true, false, true, false, false, false, false}; - std::vector exp_names = {"r", "2", "4", "", "5", "6", "", "", "", "7", "", "8", "9", "10", "", - "11", "", "", "", ""}; - std::vector exp_lengths = {0, 1, 3, 0, 4, 5, 0, 0, 0, 6, 0, 7, 8, 9, 0, 10, 0, 0, 0, 0}; - - su::BPTree obs = tree.shear(to_keep); - ASSERT(obs.get_structure() == exp_structure); - ASSERT(exp_nparens == obs.nparens); - ASSERT(vec_almost_equal(exp_lengths, obs.lengths)); - ASSERT(obs.names == exp_names); - SUITE_END(); -} - -void test_bptree_shear_deep() { - SUITE_START("test bptree shear deep"); - su::BPTree tree = su::BPTree("((3:2,4:3,(6:5)5:4)2:1,7:6,((10:9,11:10)9:8)8:7)r"); - - // deep - std::unordered_set to_keep = {"10", "11"}; - - uint32_t exp_nparens = 10; - std::vector exp_structure = {true, true, true, true, false, true, false, false, false, false}; - std::vector exp_names = {"r", "8", "9", "10", "", "11", "", "", "", ""}; - std::vector exp_lengths = {0, 7, 8, 9, 0, 10, 0, 0, 0, 0}; - - su::BPTree obs = tree.shear(to_keep); - ASSERT(exp_nparens == obs.nparens); - ASSERT(obs.get_structure() == exp_structure); - ASSERT(vec_almost_equal(exp_lengths, obs.lengths)); - ASSERT(obs.names == exp_names); - SUITE_END(); -} - -void test_test_table_ids_are_subset_of_tree() { - SUITE_START("test test_table_ids_are_subset_of_tree"); - - su::BPTree tree = su::BPTree("(a:1,b:2)r;"); - su::biom table = su::biom("test.biom"); - std::string expected = "GG_OTU_1"; - std::string observed = su::test_table_ids_are_subset_of_tree(table, tree); - ASSERT(observed == expected); - - su::BPTree tree2 = su::BPTree("(GG_OTU_1,GG_OTU_5,GG_OTU_6,GG_OTU_2,GG_OTU_3,GG_OTU_4);"); - su::biom table2 = su::biom("test.biom"); - expected = ""; - observed = su::test_table_ids_are_subset_of_tree(table2, tree2); - ASSERT(observed == expected); - SUITE_END(); -} - - -void test_bptree_get_tip_names() { - SUITE_START("test bptree get_tip_names"); - su::BPTree tree = su::BPTree("((a:2,b:3,(c:5)d:4)e:1,f:6,((g:9,h:10)i:8)j:7)r"); - - std::unordered_set expected = {"a", "b", "c", "f", "g", "h"}; - std::unordered_set observed = tree.get_tip_names(); - ASSERT(observed == expected); - SUITE_END(); -} - -void test_bptree_collapse_simple() { - SUITE_START("test bptree collapse simple"); - su::BPTree tree = su::BPTree("((3:2,4:3,(6:5)5:4)2:1,7:6,((10:9,11:10)9:8)8:7)r"); - - uint32_t exp_nparens = 18; - std::vector exp_structure = {true, true, true, false, true, false, true, false, false, - true, false, true, true, false, true, false, false, false}; - std::vector exp_names = {"r", "2", "3", "", "4", "", "6", "", "", "7", "", "9", "10", "", "11", "", "", ""}; - std::vector exp_lengths = {0, 1, 2, 0, 3, 0, 9, 0, 0, 6, 0, 15, 9, 0, 10, 0, 0, 0}; - - su::BPTree obs = tree.collapse(); - - ASSERT(obs.get_structure() == exp_structure); - ASSERT(exp_nparens == obs.nparens); - ASSERT(vec_almost_equal(exp_lengths, obs.lengths)); - ASSERT(obs.names == exp_names); - SUITE_END(); -} - -void test_bptree_collapse_edge() { - SUITE_START("test bptree collapse edge case against root"); - - su::BPTree tree = su::BPTree("((a),b)r;"); - su::BPTree exp = su::BPTree("(a,b)r;"); - su::BPTree obs = tree.collapse(); - ASSERT(obs.get_structure() == exp.get_structure()); - ASSERT(obs.names == exp.names); - ASSERT(vec_almost_equal(obs.lengths, exp.lengths)); - - SUITE_END(); -} - -void test_unifrac_sample_counts() { - SUITE_START("test unifrac sample counts"); - su::biom table = su::biom("test.biom"); - double* obs = table.sample_counts; - double exp[] = {7, 3, 4, 6, 3, 4}; - for(unsigned int i = 0; i < 6; i++) - ASSERT(obs[i] == exp[i]); - SUITE_END(); -} - -void test_set_tasks() { - SUITE_START("test set tasks"); - std::vector obs(1); - std::vector exp(1); - - exp[0].g_unifrac_alpha = 1.0; - exp[0].n_samples = 100; - exp[0].bypass_tips = false; - exp[0].start = 0; - exp[0].stop = 100; - exp[0].tid = 0; - - set_tasks(obs, 1.0, 100, 0, 100, false, 1); - ASSERT(obs[0].g_unifrac_alpha == exp[0].g_unifrac_alpha); - ASSERT(obs[0].n_samples == exp[0].n_samples); - ASSERT(obs[0].start == exp[0].start); - ASSERT(obs[0].stop == exp[0].stop); - ASSERT(obs[0].tid == exp[0].tid); - - std::vector obs2(2); - std::vector exp2(2); - - exp2[0].g_unifrac_alpha = 1.0; - exp2[0].n_samples = 100; - exp2[0].bypass_tips = false; - exp2[0].start = 0; - exp2[0].stop = 50; - exp2[0].tid = 0; - exp2[1].g_unifrac_alpha = 1.0; - exp2[1].n_samples = 100; - exp2[1].bypass_tips = false; - exp2[1].start = 50; - exp2[1].stop = 100; - exp2[1].tid = 1; - - set_tasks(obs2, 1.0, 100, 0, 100, false, 2); - for(unsigned int i=0; i < 2; i++) { - ASSERT(obs2[i].g_unifrac_alpha == exp2[i].g_unifrac_alpha); - ASSERT(obs2[i].n_samples == exp2[i].n_samples); - ASSERT(obs2[i].start == exp2[i].start); - ASSERT(obs2[i].stop == exp2[i].stop); - ASSERT(obs2[i].tid == exp2[i].tid); - } - - std::vector obs3(3); - std::vector exp3(3); - - exp3[0].g_unifrac_alpha = 1.0; - exp3[0].n_samples = 100; - exp3[0].bypass_tips = false; - exp3[0].start = 25; - exp3[0].stop = 50; - exp3[0].tid = 0; - exp3[1].g_unifrac_alpha = 1.0; - exp3[1].n_samples = 100; - exp3[1].bypass_tips = false; - exp3[1].start = 50; - exp3[1].stop = 75; - exp3[1].tid = 1; - exp3[2].g_unifrac_alpha = 1.0; - exp3[2].n_samples = 100; - exp3[2].bypass_tips = false; - exp3[2].start = 75; - exp3[2].stop = 100; - exp3[2].tid = 2; - - set_tasks(obs3, 1.0, 100, 25, 100, false, 3); - for(unsigned int i=0; i < 3; i++) { - ASSERT(obs3[i].g_unifrac_alpha == exp3[i].g_unifrac_alpha); - ASSERT(obs3[i].n_samples == exp3[i].n_samples); - ASSERT(obs3[i].start == exp3[i].start); - ASSERT(obs3[i].stop == exp3[i].stop); - ASSERT(obs3[i].tid == exp3[i].tid); - } - - std::vector obs4(3); - std::vector exp4(3); - - exp4[0].g_unifrac_alpha = 1.0; - exp4[0].n_samples = 100; - exp4[0].bypass_tips = false; - exp4[0].start = 26; - exp4[0].stop = 51; - exp4[0].tid = 0; - exp4[1].g_unifrac_alpha = 1.0; - exp4[1].n_samples = 100; - exp4[1].bypass_tips = false; - exp4[1].start = 51; - exp4[1].stop = 76; - exp4[1].tid = 1; - exp4[2].g_unifrac_alpha = 1.0; - exp4[2].n_samples = 100; - exp4[2].bypass_tips = false; - exp4[2].start = 76; - exp4[2].stop = 100; - exp4[2].tid = 2; - - set_tasks(obs4, 1.0, 100, 26, 100, false, 3); - for(unsigned int i=0; i < 3; i++) { - ASSERT(obs4[i].g_unifrac_alpha == exp4[i].g_unifrac_alpha); - ASSERT(obs4[i].n_samples == exp4[i].n_samples); - ASSERT(obs4[i].start == exp4[i].start); - ASSERT(obs4[i].stop == exp4[i].stop); - ASSERT(obs4[i].tid == exp4[i].tid); - } - - // set_tasks boundary bug - std::vector obs16(16); - std::vector exp16(16); - set_tasks(obs16, 1.0, 9511, 0, 0, false, 16); - exp16[15].start = 4459; - exp16[15].stop = 4756; - ASSERT(obs16[15].start == exp16[15].start); - ASSERT(obs16[15].stop == exp16[15].stop); - SUITE_END(); -} - -void test_bptree_constructor_newline_bug() { - SUITE_START("test bptree constructor newline bug"); - su::BPTree tree = su::BPTree("((362be41f31fd26be95ae43a8769b91c0:0.116350803,(a16679d5a10caa9753f171977552d920:0.105836235,((a7acc2abb505c3ee177a12e514d3b994:0.008268754,(4e22aa3508b98813f52e1a12ffdb74ad:0.03144211,8139c4ac825dae48454fb4800fb87896:0.043622957)0.923:0.046588301)0.997:0.120902074,((2d3df7387323e2edcbbfcb6e56a02710:0.031543994,3f6752aabcc291b67a063fb6492fd107:0.091571442)0.759:0.016335166,((d599ebe277afb0dfd4ad3c2176afc50e:5e-09,84d0affc7243c7d6261f3a7d680b873f:0.010245188)0.883:0.048993011,51121722488d0c3da1388d1b117cd239:0.119447926)0.763:0.035660204)0.921:0.058191474)0.776:0.02854575)0.657:0.052060833)0.658:0.032547569,(99647b51f775c8ddde8ed36a7d60dbcd:0.173334268,(f18a9c8112372e2916a66a9778f3741b:0.194813398,(5833416522de0cca717a1abf720079ac:5e-09,(2bf1067d2cd4f09671e3ebe5500205ca:0.031692682,(b32621bcd86cb99e846d8f6fee7c9ab8:0.031330707,1016319c25196d73bdb3096d86a9df2f:5e-09)0.058:0.01028612)0.849:0.010284866)0.791:0.041353384)0.922:0.109470534):0.022169824000000005)root;\n\n"); - SUITE_END(); -} - -int main(int argc, char** argv) { - test_bptree_constructor_simple(); - test_bptree_constructor_newline_bug(); - test_bptree_constructor_from_existing(); - test_bptree_constructor_single_descendent(); - test_bptree_constructor_complex(); - test_bptree_constructor_semicolon(); - test_bptree_constructor_edgecases(); - test_bptree_constructor_quoted_comma(); - test_bptree_constructor_quoted_parens(); - test_bptree_postorder(); - test_bptree_preorder(); - test_bptree_parent(); - test_bptree_leftchild(); - test_bptree_rightchild(); - test_bptree_rightsibling(); - test_bptree_get_tip_names(); - test_bptree_mask(); - test_bptree_shear_simple(); - test_bptree_shear_deep(); - test_bptree_collapse_simple(); - test_bptree_collapse_edge(); - - test_biom_constructor(); - test_biom_get_obs_data(); - - test_propstack_constructor(); - test_propstack_push_and_pop(); - test_propstack_get(); - - test_unifrac_set_proportions(); - test_unifrac_set_proportions_range(); - test_unifrac_set_proportions_range_float(); - test_unifrac_deconvolute_stripes(); - test_unifrac_stripes_to_condensed_form_even(); - test_unifrac_stripes_to_condensed_form_odd(); - test_unifrac_stripes_to_condensed_form_odd2(); - test_unifrac_stripes_to_matrix_even(); - test_unifrac_stripes_to_matrix_odd(); - test_unifrac_stripes_to_matrix_odd2(); - test_unweighted_unifrac(); - test_unweighted_unifrac_fast(); - test_unnormalized_weighted_unifrac(); - test_normalized_weighted_unifrac(); - test_generalized_unifrac(); - test_vaw_unifrac_weighted_normalized(); - test_unifrac_sample_counts(); - test_set_tasks(); - test_test_table_ids_are_subset_of_tree(); - - test_faith_pd(); - test_faith_pd_shear(); - - printf("\n"); - printf(" %i / %i suites failed\n", suites_failed, suites_run); - printf(" %i / %i suites empty\n", suites_empty, suites_run); - printf(" %i / %i tests failed\n", tests_failed, tests_run); - - printf("\n THE END.\n"); - - return tests_failed ? EXIT_FAILURE : EXIT_SUCCESS; -} diff --git a/sucpp/tree.cpp b/sucpp/tree.cpp deleted file mode 100644 index 3dc6f98e..00000000 --- a/sucpp/tree.cpp +++ /dev/null @@ -1,462 +0,0 @@ -#include "tree.hpp" -#include -#include - -using namespace su; - -BPTree::BPTree(std::string newick) { - openclose = std::vector(); - lengths = std::vector(); - names = std::vector(); - excess = std::vector(); - - select_0_index = std::vector(); - select_1_index = std::vector(); - structure = std::vector(); - structure.reserve(500000); // a fair sized tree... avoid reallocs, and its not _that_ much waste if this is wrong - - // three pass for parse. not ideal, but easier to map from IOW code - newick_to_bp(newick); - - // resize is correct here as we are not performing a push_back - openclose.resize(nparens); - lengths.resize(nparens); - names.resize(nparens); - select_0_index.resize(nparens / 2); - select_1_index.resize(nparens / 2); - excess.resize(nparens); - - structure_to_openclose(); - newick_to_metadata(newick); - index_and_cache(); -} - -BPTree::BPTree(std::vector input_structure, std::vector input_lengths, std::vector input_names) { - structure = input_structure; - lengths = input_lengths; - names = input_names; - - nparens = structure.size(); - - openclose = std::vector(); - select_0_index = std::vector(); - select_1_index = std::vector(); - openclose.resize(nparens); - select_0_index.resize(nparens / 2); - select_1_index.resize(nparens / 2); - excess.resize(nparens); - - structure_to_openclose(); - index_and_cache(); -} - -BPTree BPTree::mask(std::vector topology_mask, std::vector in_lengths) { - - std::vector new_structure = std::vector(); - std::vector new_lengths = std::vector(); - std::vector new_names = std::vector(); - - uint32_t count = 0; - for(auto i = topology_mask.begin(); i != topology_mask.end(); i++) { - if(*i) - count++; - } - - new_structure.resize(count); - new_lengths.resize(count); - new_names.resize(count); - - auto mask_it = topology_mask.begin(); - auto base_it = this->structure.begin(); - uint32_t new_idx = 0; - uint32_t old_idx = 0; - for(; mask_it != topology_mask.end(); mask_it++, base_it++, old_idx++) { - if(*mask_it) { - new_structure[new_idx] = this->structure[old_idx]; - new_lengths[new_idx] = in_lengths[old_idx]; - new_names[new_idx] = this->names[old_idx]; - new_idx++; - } - } - - return BPTree(new_structure, new_lengths, new_names); -} - -std::unordered_set BPTree::get_tip_names() { - std::unordered_set observed; - - for(unsigned int i = 0; i < this->nparens; i++) { - if(this->isleaf(i)) { - observed.insert(this->names[i]); - } - } - - return observed; -} - -BPTree BPTree::shear(std::unordered_set to_keep) { - std::vector shearmask = std::vector(this->nparens); - int32_t p; - - for(unsigned int i = 0; i < this->nparens; i++) { - if(this->isleaf(i) && to_keep.count(this->names[i]) > 0) { - shearmask[i] = true; - shearmask[i+1] = true; - - p = this->parent(i); - while(p != -1 && !shearmask[p]) { - shearmask[p] = true; - shearmask[this->close(p)] = true; - p = this->parent(p); - } - } - } - return this->mask(shearmask, this->lengths); -} - -BPTree BPTree::collapse() { - std::vector collapsemask = std::vector(this->nparens); - std::vector new_lengths = std::vector(this->lengths); - - uint32_t current, first, last; - - for(uint32_t i = 0; i < this->nparens / 2; i++) { - current = this->preorderselect(i); - - if(this->isleaf(current) or (current == 0)) { // 0 == root - collapsemask[current] = true; - collapsemask[this->close(current)] = true; - } else { - first = this->leftchild(current); - last = this->rightchild(current); - - if(first == last) { - new_lengths[first] = new_lengths[first] + new_lengths[current]; - } else { - collapsemask[current] = true; - collapsemask[this->close(current)] = true; - } - } - } - - return this->mask(collapsemask, new_lengths); -} - /* - mask = bit_array_create(self.B.size) - bit_array_set_bit(mask, self.root()) - bit_array_set_bit(mask, self.close(self.root())) - - new_lengths = self._lengths.copy() - new_lengths_ptr = new_lengths.data - - with nogil: - for i in range(n): - current = self.preorderselect(i) - - if self.isleaf(current): - bit_array_set_bit(mask, current) - bit_array_set_bit(mask, self.close(current)) - else: - first = self.fchild(current) - last = self.lchild(current) - - if first == last: - new_lengths_ptr[first] = new_lengths_ptr[first] + \ - new_lengths_ptr[current] - else: - bit_array_set_bit(mask, current) - bit_array_set_bit(mask, self.close(current)) - - new_bp = self._mask_from_self(mask, new_lengths) - bit_array_free(mask) - return new_bp -*/ - - -BPTree::~BPTree() { -} - -void BPTree::index_and_cache() { - // should probably do the open/close in here too - unsigned int idx = 0; - auto i = structure.begin(); - auto k0 = select_0_index.begin(); - auto k1 = select_1_index.begin(); - auto e_it = excess.begin(); - unsigned int e = 0; - - for(; i != structure.end(); i++, idx++ ) { - if(*i) { - *(k1++) = idx; - *(e_it++) = ++e; - } - else { - *(k0++) = idx; - *(e_it++) = --e; - } - } -} - -uint32_t BPTree::postorderselect(uint32_t k) const { - return open(select_0_index[k]); -} - -uint32_t BPTree::preorderselect(uint32_t k) const { - return select_1_index[k]; -} - -inline uint32_t BPTree::open(uint32_t i) const { - return structure[i] ? i : openclose[i]; -} - -inline uint32_t BPTree::close(uint32_t i) const { - return structure[i] ? openclose[i] : i; -} - -bool BPTree::isleaf(unsigned int idx) const { - return (structure[idx] && !structure[idx + 1]); -} - -uint32_t BPTree::leftchild(uint32_t i) const { - // aka fchild - if(isleaf(i)) - return 0; // this is awkward, using 0 which is root, but a root cannot be a child. edge case - else - return i + 1; -} - -uint32_t BPTree::rightchild(uint32_t i) const { - // aka lchild - if(isleaf(i)) - return 0; // this is awkward, using 0 which is root, but a root cannot be a child. edge case - else - return open(close(i) - 1); -} - -uint32_t BPTree::rightsibling(uint32_t i) const { - // aka nsibling - uint32_t position = close(i) + 1; - if(position >= nparens) - return 0; // will return 0 if no sibling as root cannot have a sibling - else if(structure[position]) - return position; - else - return 0; -} - -int32_t BPTree::parent(uint32_t i) const { - return enclose(i); -} - -int32_t BPTree::enclose(uint32_t i) const { - if(structure[i]) - return bwd(i, -2) + 1; - else - return bwd(i - 1, -2) + 1; -} - -int32_t BPTree::bwd(uint32_t i, int d) const { - uint32_t target_excess = excess[i] + d; - for(int current_idx = i - 1; current_idx >= 0; current_idx--) { - if(excess[current_idx] == target_excess) - return current_idx; - } - return -1; -} - -void BPTree::newick_to_bp(std::string newick) { - char last_structure; - bool potential_single_descendent = false; - int count = 0; - bool in_quote = false; - for(auto c = newick.begin(); c != newick.end(); c++) { - if(*c == '\'') - in_quote = !in_quote; - - if(in_quote) - continue; - - switch(*c) { - case '(': - // opening of a node - count++; - structure.push_back(true); - last_structure = *c; - potential_single_descendent = true; - break; - case ')': - // closing of a node - if(potential_single_descendent || (last_structure == ',')) { - // we have a single descendent or a last child (i.e. ",)" scenario) - count += 3; - structure.push_back(true); - structure.push_back(false); - structure.push_back(false); - potential_single_descendent = false; - } else { - // it is possible still to have a single descendent in the case of - // multiple single descendents (e.g., (...()...) ) - count += 1; - structure.push_back(false); - } - last_structure = *c; - break; - case ',': - if(last_structure != ')') { - // we have a new tip - count += 2; - structure.push_back(true); - structure.push_back(false); - } - potential_single_descendent = false; - last_structure = *c; - break; - default: - break; - } - } - nparens = structure.size(); -} - - -void BPTree::structure_to_openclose() { - std::stack oc; - unsigned int open_idx; - unsigned int i = 0; - - for(auto it = structure.begin(); it != structure.end(); it++, i++) { - if(*it) { - oc.push(i); - } else { - open_idx = oc.top(); - oc.pop(); - openclose[i] = open_idx; - openclose[open_idx] = i; - } - } -} -// trim from end -// from http://stackoverflow.com/a/217605 -static inline std::string &rtrim(std::string &s) { - s.erase(std::find_if(s.rbegin(), s.rend(), - std::not1(std::ptr_fun(std::isspace))).base(), s.end()); - return s; -} - - -//// WEIRDNESS. THIS SOLVES IT WITH THE RTRIM. ISOLATE, MOVE TO CONSTRUCTOR. -void BPTree::newick_to_metadata(std::string newick) { - newick = rtrim(newick); - - std::string::iterator start = newick.begin(); - std::string::iterator end = newick.end(); - std::string token; - char last_structure = '\0'; - - unsigned int structure_idx = 0; - unsigned int lag = 0; - unsigned int open_idx; - - while(start != end) { - token = tokenize(start, end); - // this sucks. - if(token.length() == 1 && is_structure_character(token[0])) { - switch(token[0]) { - case '(': - structure_idx++; - break; - case ')': - case ',': - structure_idx++; - if(last_structure == ')') - lag++; - break; - } - } else { - // puts us on the corresponding closing parenthesis - structure_idx += lag; - lag = 0; - - open_idx = open(structure_idx); - set_node_metadata(open_idx, token); - // std::cout << structure_idx << " <-> " << open_idx << " " << token << std::endl; - // make sure to advance an extra position if we are a leaf as the - // as a leaf is by definition a 10, and doing a single advancement - // would put the structure to token mapping out of sync - if(isleaf(open_idx)) - structure_idx += 2; - else - structure_idx += 1; - - } - last_structure = token[0]; - } -} - -void BPTree::set_node_metadata(unsigned int open_idx, std::string &token) { - double length = 0.0; - std::string name = std::string(); - unsigned int colon_idx = token.find_last_of(':'); - - if(colon_idx == 0) - length = std::stof(token.substr(1)); - else if(colon_idx < token.length()) { - name = token.substr(0, colon_idx); - length = std::stof(token.substr(colon_idx + 1)); - } else - name = token; - - names[open_idx] = name; - lengths[open_idx] = length; -} - -inline bool BPTree::is_structure_character(char c) const { - return (c == '(' || c == ')' || c == ',' || c == ';'); -} - -std::string BPTree::tokenize(std::string::iterator &start, const std::string::iterator &end) { - bool inquote = false; - bool isquote = false; - char c; - std::string token; - - do { - c = *start; - start++; - - if(c == '\n') { - continue; - } - - isquote = c == '\''; - - if(inquote && isquote) { - inquote = false; - continue; - } else if(!inquote && isquote) { - inquote = true; - continue; - } - - if(is_structure_character(c) && !inquote) { - if(token.length() == 0) - token.push_back(c); - break; - } - - token.push_back(c); - - - } while(start != end); - - return token; -} - -std::vector BPTree::get_structure() { - return structure; -} - -std::vector BPTree::get_openclose() { - return openclose; -} - diff --git a/sucpp/tree.hpp b/sucpp/tree.hpp deleted file mode 100644 index 99f61dfe..00000000 --- a/sucpp/tree.hpp +++ /dev/null @@ -1,138 +0,0 @@ -/* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ - -#ifndef __UNIFRAC_TREE_H -#define __UNIFRAC_TREE_H 1 - -#include -#include -#include -#include -#include - -namespace su { - class BPTree { - public: - /* tracked attributes */ - std::vector lengths; - std::vector names; - - /* total number of parentheses */ - uint32_t nparens; - - /* default constructor - * - * @param newick A newick string - */ - BPTree(std::string newick); - - /* constructor from a defined topology - * - * @param input_structure A boolean vector defining the topology - * @param input_lengths A vector of double of the branch lengths - * @param input_names A vector of str of the vertex names - */ - BPTree(std::vector input_structure, std::vector input_lengths, std::vector input_names); - ~BPTree(); - - /* postorder tree traversal - * - * Get the index position of the ith node in a postorder tree - * traversal. - * - * @param i The ith node in a postorder traversal - */ - uint32_t postorderselect(uint32_t i)const ; - - /* preorder tree traversal - * - * Get the index position of the ith node in a preorder tree - * traversal. - * - * @param i The ith node in a preorder traversal - */ - uint32_t preorderselect(uint32_t i) const; - - /* Test if the node at an index position is a leaf - * - * @param i The node to evaluate - */ - bool isleaf(uint32_t i) const; - - /* Get the left child of a node - * - * @param i The node to obtain the left child from - */ - uint32_t leftchild(uint32_t i) const ; - - /* Get the right child of a node - * - * @param i The node to obtain the right child from - */ - uint32_t rightchild(uint32_t i) const; - - /* Get the right sibling of a node - * - * @param i The node to obtain the right sibling from - */ - uint32_t rightsibling(uint32_t i) const; - - /* Get the parent of a node - * - * @param i The node to obtain the parent of - */ - int32_t parent(uint32_t i) const; - - /* get the names at the tips of the tree */ - std::unordered_set get_tip_names(); - - /* public getters */ - std::vector get_structure(); - std::vector get_openclose(); - - /* serialize the structure as a sequence of 1s and 0s */ - void print() { - for(auto c = structure.begin(); c != structure.end(); c++) { - if(*c) - std::cout << "1"; - else - std::cout << "0"; - } - std::cout << std::endl; - } - BPTree mask(std::vector topology_mask, std::vector in_lengths); // mask self - - BPTree shear(std::unordered_set to_keep); - - BPTree collapse(); - - private: - std::vector structure; // the topology - std::vector openclose; // cache'd mapping between parentheses - std::vector select_0_index; // cache of select 0 - std::vector select_1_index; // cache of select 1 - std::vector excess; - - void index_and_cache(); // construct the select caches - void newick_to_bp(std::string newick); // convert a newick string to parentheses - void newick_to_metadata(std::string newick); // convert newick to attributes - void structure_to_openclose(); // set the cache mapping between parentheses pairs - void set_node_metadata(unsigned int open_idx, std::string &token); // set attributes for a node - bool is_structure_character(char c) const; // test if a character is a newick structure - inline uint32_t open(uint32_t i) const; // obtain the index of the opening for a given parenthesis - inline uint32_t close(uint32_t i) const; // obtain the index of the closing for a given parenthesis - std::string tokenize(std::string::iterator &start, const std::string::iterator &end); // newick -> tokens - - int32_t bwd(uint32_t i, int32_t d) const; - int32_t enclose(uint32_t i) const; - }; -} - -#endif /* UNIFRAC_TREE_H */ - diff --git a/sucpp/unifrac.cpp b/sucpp/unifrac.cpp deleted file mode 100644 index 809196ad..00000000 --- a/sucpp/unifrac.cpp +++ /dev/null @@ -1,494 +0,0 @@ -/* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ - -#include "tree.hpp" -#include "biom_interface.hpp" -#include "unifrac.hpp" -#include "affinity.hpp" -#include -#include -#include -#include -#include -#include -#include -#include - -#include "unifrac_internal.hpp" - -// We will always have the CPU version -#define SUCMP_NM su_cpu -#include "unifrac_cmp.hpp" -#undef SUCMP_NM - -#ifdef UNIFRAC_ENABLE_ACC -#define SUCMP_NM su_acc -#include "unifrac_cmp.hpp" -#undef SUCMP_NM -#endif - -using namespace su; - -std::string su::test_table_ids_are_subset_of_tree(su::biom_interface &table, su::BPTree &tree) { - std::unordered_set tip_names = tree.get_tip_names(); - std::unordered_set::const_iterator hit; - std::string a_missing_name = ""; - - for(auto i : table.obs_ids) { - hit = tip_names.find(i); - if(hit == tip_names.end()) { - a_missing_name = i; - break; - } - } - - return a_missing_name; -} - -double** su::deconvolute_stripes(std::vector &stripes, uint32_t n) { - // would be better to just do striped_to_condensed_form - double **dm; - dm = (double**)malloc(sizeof(double*) * n); - if(dm == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(double*) * n, __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - for(unsigned int i = 0; i < n; i++) { - dm[i] = (double*)malloc(sizeof(double) * n); - if(dm[i] == NULL) { - fprintf(stderr, "Failed to allocate %zd bytes; [%s]:%d\n", - sizeof(double) * n, __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - dm[i][i] = 0; - } - - for(unsigned int i = 0; i < stripes.size(); i++) { - double *vec = stripes[i]; - unsigned int k = 0; - for(unsigned int row = 0, col = i + 1; row < n; row++, col++) { - if(col < n) { - dm[row][col] = vec[k]; - dm[col][row] = vec[k]; - } else { - dm[col % n][row] = vec[k]; - dm[row][col % n] = vec[k]; - } - k++; - } - } - return dm; -} - - -void su::stripes_to_condensed_form(std::vector &stripes, uint32_t n, double* cf, unsigned int start, unsigned int stop) { - // n must be >= 2, but that should be enforced upstream as that would imply - // computing unifrac on a single sample. - - uint64_t comb_N = comb_2(n); - for(unsigned int stripe = start; stripe < stop; stripe++) { - // compute the (i, j) position of each element in each stripe - uint64_t i = 0; - uint64_t j = stripe + 1; - for(uint64_t k = 0; k < n; k++, i++, j++) { - if(j == n) { - i = 0; - j = n - (stripe + 1); - } - // determine the position in the condensed form vector for a given (i, j) - // based off of - // https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.distance.squareform.html - uint64_t comb_N_minus_i = comb_2(n - i); - cf[comb_N - comb_N_minus_i + (j - i - 1)] = stripes[stripe][k]; - } - } -} - - -// write in a 2D matrix -// also suitable for writing to disk -template -void su::condensed_form_to_matrix_T(const double* __restrict__ cf, const uint32_t n, TReal* __restrict__ buf2d) { - const uint64_t comb_N = su::comb_2(n); - for(uint64_t i = 0; i < n; i++) { - for(uint64_t j = 0; j < n; j++) { - TReal v; - if(i < j) { // upper triangle - const uint64_t comb_N_minus = su::comb_2(n - i); - v = cf[comb_N - comb_N_minus + (j - i - 1)]; - } else if (i > j) { // lower triangle - const uint64_t comb_N_minus = su::comb_2(n - j); - v = cf[comb_N - comb_N_minus + (i - j - 1)]; - } else { - v = 0.0; - } - buf2d[i*n+j] = v; - } - } -} - - -// make sure it is instantiated -template void su::condensed_form_to_matrix_T(const double* __restrict__ cf, const uint32_t n, double* __restrict__ buf2d); -template void su::condensed_form_to_matrix_T(const double* __restrict__ cf, const uint32_t n, float* __restrict__ buf2d); - -void su::condensed_form_to_matrix(const double* __restrict__ cf, const uint32_t n, double* __restrict__ buf2d) { - su::condensed_form_to_matrix_T(cf,n,buf2d); -} - -void su::condensed_form_to_matrix_fp32(const double* __restrict__ cf, const uint32_t n, float* __restrict__ buf2d) { - su::condensed_form_to_matrix_T(cf,n,buf2d); -} - -/* - * The stripes end up computing the following positions in the distance - * matrix. - * - * x A B C x x - * x x A B C x - * x x x A B C - * C x x x A B - * B C x x x A - * A B C x x x - * - * However, we store those stripes as vectors, ie - * [ A A A A A A ] - */ - - -// Helper class -// Will cache pointers and automatically release stripes when all elements are used -class OnceManagedStripes { - private: - const uint32_t n_samples; - const uint32_t n_stripes; - const ManagedStripes &stripes; - std::vector stripe_ptr; - std::vector stripe_accessed; - - const double *get_stripe(const uint32_t stripe) { - if (stripe_ptr[stripe]==0) stripe_ptr[stripe]=stripes.get_stripe(stripe); - return stripe_ptr[stripe]; - } - - void release_stripe(const uint32_t stripe) { - stripes.release_stripe(stripe); - stripe_ptr[stripe]=0; - } - - public: - OnceManagedStripes(const ManagedStripes &_stripes, const uint32_t _n_samples, const uint32_t _n_stripes) - : n_samples(_n_samples), n_stripes(_n_stripes) - , stripes(_stripes) - , stripe_ptr(n_stripes) - , stripe_accessed(n_stripes) - {} - - ~OnceManagedStripes() - { - for(uint32_t i = 0; i < n_stripes; i++) { - if (stripe_ptr[i]!=0) { - release_stripe(i); - } - } - } - - double get_val(const uint32_t stripe, const uint32_t el) - { - if (stripe_ptr[stripe]==0) stripe_ptr[stripe]=stripes.get_stripe(stripe); - const double *mystripe = stripe_ptr[stripe]; - double val = mystripe[el]; - - stripe_accessed[stripe]++; - if (stripe_accessed[stripe]==n_samples) release_stripe(stripe); // we will not use this stripe anymore - - return val; - } - - -}; - -// write in a 2D matrix -// also suitable for writing to disk -template -void su::stripes_to_matrix_T(const ManagedStripes &_stripes, const uint32_t n_samples, const uint32_t n_stripes, TReal* __restrict__ buf2d, uint32_t tile_size) { - // n_samples must be >= 2, but that should be enforced upstream as that would imply - // computing unifrac on a single sample. - - // tile for for better memory access pattern - const uint32_t TILE = (tile_size>0) ? tile_size : (128/sizeof(TReal)); - const uint32_t n_samples_tup = (n_samples+(TILE-1))/TILE; // round up - - OnceManagedStripes stripes(_stripes, n_samples, n_stripes); - - - for(uint32_t oi = 0; oi < n_samples_tup; oi++) { // off diagonal - // alternate between inner and outer off-diagonal, due to wrap around in stripes - const uint32_t o = ((oi%2)==0) ? \ - (oi/2)*TILE : /* close to diagonal */ \ - (n_samples_tup-(oi/2)-1)*TILE; /* far from diagonal */ - - for(uint32_t d = 0; d < (n_samples-o); d+=TILE) { // diagonal - - uint32_t iOut = d; - uint32_t jOut = d+o; - - uint32_t iMax = std::min(iOut+TILE,n_samples); - uint32_t jMax = std::min(jOut+TILE,n_samples); - - - if (iOut==jOut) { - // on diagonal - for(uint64_t i = iOut; i < iMax; i++) { - buf2d[i*n_samples+i] = 0.0; - - int64_t stripe=0; - - uint64_t j = i+1; - for(; (stripen_stripes) { - // ops, we overshoot... roll back - j-=(stripe-n_stripes); - stripe=n_stripes; - } - for(; (stripe(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, double* __restrict__ buf2d, uint32_t tile_size); -template void su::stripes_to_matrix_T(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, float* __restrict__ buf2d, uint32_t tile_size); - -void su::stripes_to_matrix(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, double* __restrict__ buf2d, uint32_t tile_size) { - return su::stripes_to_matrix_T(stripes, n_samples, n_stripes, buf2d, tile_size); -} - -void su::stripes_to_matrix_fp32(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, float* __restrict__ buf2d, uint32_t tile_size) { - return su::stripes_to_matrix_T(stripes, n_samples, n_stripes, buf2d, tile_size); -} - - -void progressbar(float progress) { - // from http://stackoverflow.com/a/14539953 - // - // could encapsulate into a classs for displaying time elapsed etc - int barWidth = 70; - std::cout << "["; - int pos = barWidth * progress; - for (int i = 0; i < barWidth; ++i) { - if (i < pos) std::cout << "="; - else if (i == pos) std::cout << ">"; - else std::cout << " "; - } - std::cout << "] " << int(progress * 100.0) << " %\r"; - std::cout.flush(); -} - -// Computes Faith's PD for the samples in `table` over the phylogenetic -// tree given by `tree`. -// Assure that tree does not contain ids that are not in table -void su::faith_pd(biom_interface &table, - BPTree &tree, - double* result) { - PropStack propstack(table.n_samples); - - uint32_t node; - double *node_proportions; - double length; - - // for node in postorderselect - for(unsigned int k = 0; k < (tree.nparens / 2) - 1; k++) { - node = tree.postorderselect(k); - // get branch length - length = tree.lengths[node]; - - // get node proportions and set intermediate scores - node_proportions = propstack.pop(node); - set_proportions(node_proportions, tree, node, table, propstack); - - for (unsigned int sample = 0; sample < table.n_samples; sample++){ - // calculate contribution of node to score - result[sample] += (node_proportions[sample] > 0) * length; - } - } -} - - -#ifdef UNIFRAC_ENABLE_ACC - -// test only once, then use persistent value -static int proc_use_acc = -1; - -inline bool use_acc() { - if (proc_use_acc!=-1) return (proc_use_acc!=0); - int has_nvidia_gpu_rc = access("/proc/driver/nvidia/gpus", F_OK); - - bool print_info = false; - - if (const char* env_p = std::getenv("UNIFRAC_GPU_INFO")) { - print_info = true; - std::string env_s(env_p); - if ((env_s=="NO") || (env_s=="N") || (env_s=="no") || (env_s=="n") || - (env_s=="NEVER") || (env_s=="never")) { - print_info = false; - } - } - - - if (has_nvidia_gpu_rc != 0) { - if (print_info) printf("INFO (unifrac): GPU not found, using CPU\n"); - proc_use_acc=0; - return false; - } - - if (const char* env_p = std::getenv("UNIFRAC_USE_GPU")) { - std::string env_s(env_p); - if ((env_s=="NO") || (env_s=="N") || (env_s=="no") || (env_s=="n") || - (env_s=="NEVER") || (env_s=="never")) { - if (print_info) printf("INFO (unifrac): Use of GPU explicitly disabled, using CPU\n"); - proc_use_acc=0; - return false; - } - } - - if (print_info) printf("INFO (unifrac): Using GPU\n"); - proc_use_acc=1; - return true; -} -#endif - -void su::unifrac(biom_interface &table, - BPTree &tree, - Method unifrac_method, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - const su::task_parameters* task_p) { -#ifdef UNIFRAC_ENABLE_ACC - if (use_acc()) { - su_acc::unifrac(table, tree, unifrac_method, dm_stripes, dm_stripes_total, task_p); - } else { -#else - if (true) { -#endif - su_cpu::unifrac(table, tree, unifrac_method, dm_stripes, dm_stripes_total, task_p); - } -} - - -void su::unifrac_vaw(biom_interface &table, - BPTree &tree, - Method unifrac_method, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - const su::task_parameters* task_p) { -#ifdef UNIFRAC_ENABLE_ACC - if (use_acc()) { - su_acc::unifrac_vaw(table, tree, unifrac_method, dm_stripes, dm_stripes_total, task_p); - } else { -#else - if (true) { -#endif - su_cpu::unifrac_vaw(table, tree, unifrac_method, dm_stripes, dm_stripes_total, task_p); - } -} - - -void su::process_stripes(biom_interface &table, - BPTree &tree_sheared, - Method method, - bool variance_adjust, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - std::vector &threads, - std::vector &tasks) { - - // register a signal handler so we can ask the master thread for its - // progress - register_report_status(); - - // cannot use threading with openacc or openmp - for(unsigned int tid = 0; tid < threads.size(); tid++) { - if(variance_adjust) - su::unifrac_vaw( - std::ref(table), - std::ref(tree_sheared), - method, - std::ref(dm_stripes), - std::ref(dm_stripes_total), - &tasks[tid]); - else - su::unifrac( - std::ref(table), - std::ref(tree_sheared), - method, - std::ref(dm_stripes), - std::ref(dm_stripes_total), - &tasks[tid]); - } - - remove_report_status(); -} diff --git a/sucpp/unifrac.hpp b/sucpp/unifrac.hpp deleted file mode 100644 index 842c4aeb..00000000 --- a/sucpp/unifrac.hpp +++ /dev/null @@ -1,112 +0,0 @@ -/* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ - -#include -#include -#include -#include -#include - -#ifndef __UNIFRAC - -#include "task_parameters.hpp" -#include "biom_interface.hpp" - - namespace su { - enum Method {unweighted, weighted_normalized, weighted_unnormalized, generalized, unweighted_fp32, weighted_normalized_fp32, weighted_unnormalized_fp32, generalized_fp32}; - - void faith_pd(biom_interface &table, BPTree &tree, double* result); - - std::string test_table_ids_are_subset_of_tree(biom_interface &table, BPTree &tree); - void unifrac(biom_interface &table, - BPTree &tree, - Method unifrac_method, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - const task_parameters* task_p); - - void unifrac_vaw(biom_interface &table, - BPTree &tree, - Method unifrac_method, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - const task_parameters* task_p); - - double** deconvolute_stripes(std::vector &stripes, uint32_t n); - - class ManagedStripes { - public: - virtual ~ManagedStripes() {} - virtual const double *get_stripe(uint32_t stripe) const = 0; - virtual void release_stripe(uint32_t stripe) const = 0; - }; - - class MemoryStripes : public ManagedStripes { - private: - const double * const * stripes; // just a pointer, not owned - public: - MemoryStripes(const double * const * _stripes) : stripes(_stripes) {} - MemoryStripes(std::vector &_stripes) : stripes(_stripes.data()) {} - MemoryStripes(const std::vector &_stripes) : stripes(_stripes.data()) {} - MemoryStripes(const std::vector &_stripes) : stripes(_stripes.data()) {} - MemoryStripes(std::vector &_stripes) : stripes(_stripes.data()) {} - - virtual const double *get_stripe(uint32_t stripe) const {return stripes[stripe];} - virtual void release_stripe(uint32_t stripe) const {}; - }; - - - void stripes_to_condensed_form(std::vector &stripes, uint32_t n, double* cf, unsigned int start, unsigned int stop); - - // tile_size==0 means memory optimized - template void stripes_to_matrix_T(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, TReal* __restrict__ buf2d, uint32_t tile_size=0); - void stripes_to_matrix(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, double* __restrict__ buf2d, uint32_t tile_size=0); - void stripes_to_matrix_fp32(const ManagedStripes &stripes, const uint32_t n_samples, const uint32_t n_stripes, float* __restrict__ buf2d, uint32_t tile_size=0); - - - template void condensed_form_to_matrix_T(const double* __restrict__ cf, const uint32_t n, TReal* __restrict__ buf2d); - void condensed_form_to_matrix(const double* __restrict__ cf, const uint32_t n, double* __restrict__ buf2d); - void condensed_form_to_matrix_fp32(const double* __restrict__ cf, const uint32_t n, float* __restrict__ buf2d); - - inline uint64_t comb_2(uint64_t N) { - // based off of _comb_int_long - // https://github.com/scipy/scipy/blob/v0.19.1/scipy/special/_comb.pyx - - // Compute binom(N, k) for integers. - // - // we're disregarding overflow as that practically should not - // happen unless the number of samples processed is in excess - // of 4 billion - uint64_t val, j, M, nterms; - uint64_t k = 2; - - M = N + 1; - nterms = k < (N - k) ? k : N - k; - - val = 1; - - for(j = 1; j < nterms + 1; j++) { - val *= M - j; - val /= j; - } - return val; - } - - // process the stripes described by tasks - void process_stripes(biom_interface &table, - BPTree &tree_sheared, - Method method, - bool variance_adjust, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - std::vector &threads, - std::vector &tasks); - } -#define __UNIFRAC 1 -#endif diff --git a/sucpp/unifrac_cmp.cpp b/sucpp/unifrac_cmp.cpp deleted file mode 100644 index a2c22c91..00000000 --- a/sucpp/unifrac_cmp.cpp +++ /dev/null @@ -1,395 +0,0 @@ -/* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ - -#include "tree.hpp" -#include "biom_interface.hpp" -#include -#include -#include -#include - -#include "unifrac_internal.hpp" - -#include "unifrac_task.hpp" -// Note: unifrac_task.hpp defines SUCMP_NM, needed by unifrac_cmp.hpp -#include "unifrac_cmp.hpp" - -// embed in this file, to properly instantiate the templatized functions -#include "unifrac_task.cpp" - -using namespace SUCMP_NM; - -template -inline void initialize_sample_counts(TFloat*& _counts, const su::task_parameters* task_p, const su::biom_interface &table) { - const unsigned int n_samples = task_p->n_samples; - const uint64_t n_samples_r = ((n_samples + UNIFRAC_BLOCK-1)/UNIFRAC_BLOCK)*UNIFRAC_BLOCK; // round up - TFloat * counts = NULL; - int err = 0; - err = posix_memalign((void **)&counts, 4096, sizeof(TFloat) * n_samples_r); - if(counts == NULL || err != 0) { - fprintf(stderr, "Failed to allocate %zd bytes, err %d; [%s]:%d\n", - sizeof(TFloat) * n_samples_r, err, __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - for(unsigned int i = 0; i < n_samples; i++) { - counts[i] = table.sample_counts[i]; - } - // avoid NaNs - for(unsigned int i = n_samples; i < n_samples_r; i++) { - counts[i] = 0.0; - } - - _counts=counts; -} - -template -inline void unifracTT(const su::biom_interface &table, - const su::BPTree &tree, - const bool want_total, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - const su::task_parameters* task_p) { - int err; - // no processor affinity whenusing openacc or openmp - - if(table.n_samples != task_p->n_samples) { - fprintf(stderr, "Task and table n_samples not equal\n"); - exit(EXIT_FAILURE); - } - const unsigned int n_samples = task_p->n_samples; - const uint64_t n_samples_r = ((n_samples + UNIFRAC_BLOCK-1)/UNIFRAC_BLOCK)*UNIFRAC_BLOCK; // round up - - - su::PropStackMulti propstack_multi(table.n_samples); - - const unsigned int max_emb = TaskT::RECOMMENDED_MAX_EMBS; - - su::initialize_stripes(std::ref(dm_stripes), std::ref(dm_stripes_total), want_total, task_p); - - TaskT taskObj(std::ref(dm_stripes), std::ref(dm_stripes_total),max_emb,task_p); - - TFloat *lengths = NULL; - err = posix_memalign((void **)&lengths, 4096, sizeof(TFloat) * max_emb); - if(err != 0) { - fprintf(stderr, "posix_memalign(%d) failed: %d\n", sizeof(TFloat) * max_emb, err); - exit(EXIT_FAILURE); - } -#pragma acc enter data create(lengths[:max_emb]) - - /* - * The values in the example vectors correspond to index positions of an - * element in the resulting distance matrix. So, in the example below, - * the following can be interpreted: - * - * [0 1 2] - * [1 2 3] - * - * As comparing the sample for row 0 against the sample for col 1, the - * sample for row 1 against the sample for col 2, the sample for row 2 - * against the sample for col 3. - * - * In other words, we're computing stripes of a distance matrix. In the - * following example, we're computing over 6 samples requiring 3 - * stripes. - * - * A; stripe == 0 - * [0 1 2 3 4 5] - * [1 2 3 4 5 0] - * - * B; stripe == 1 - * [0 1 2 3 4 5] - * [2 3 4 5 0 1] - * - * C; stripe == 2 - * [0 1 2 3 4 5] - * [3 4 5 0 1 2] - * - * The stripes end up computing the following positions in the distance - * matrix. - * - * x A B C x x - * x x A B C x - * x x x A B C - * C x x x A B - * B C x x x A - * A B C x x x - * - * However, we store those stripes as vectors, ie - * [ A A A A A A ] - * - * We end up performing N / 2 redundant calculations on the last stripe - * (see C) but that is small over large N. - */ - - unsigned int k = 0; // index in tree - const unsigned int max_k = (tree.nparens / 2) - 1; - - const unsigned int num_prop_chunks = propstack_multi.get_num_stacks(); - while (k &propstack = propstack_multi.get_prop_stack(ck); - const unsigned int tstart = propstack_multi.get_start(ck); - const unsigned int tend = propstack_multi.get_end(ck); - unsigned int my_filled_emb = 0; - unsigned int my_k=k_start; - - while ((my_filled_embbypass_tips && tree.isleaf(node)) - continue; - - if (ck==0) { // they all do the same thing, so enough for the first to update the global state - lengths[filled_emb] = tree.lengths[node]; - filled_emb++; - } - taskObj.embed_proportions_range(node_proportions, tstart, tend, my_filled_emb); - my_filled_emb++; - } - if (ck==0) { // they all do the same thing, so enough for the first to update the global state - k=my_k; - } - } - - taskObj.sync_embedded_proportions(filled_emb); -#ifdef _OPENACC - // lengths may be still in use in async mode, wait -#pragma acc wait -#pragma acc update device(lengths[:filled_emb]) -#endif - taskObj._run(filled_emb,lengths); - filled_emb=0; - - su::try_report(task_p, k, max_k); - } - -#pragma acc wait - - if(want_total) { - const uint64_t start_idx = task_p->start; - const uint64_t stop_idx = task_p->stop; - - TFloat * const dm_stripes_buf = taskObj.dm_stripes.buf; - const TFloat * const dm_stripes_total_buf = taskObj.dm_stripes_total.buf; - -#pragma acc parallel loop collapse(2) present(dm_stripes_buf,dm_stripes_total_buf) - for(uint64_t i = start_idx; i < stop_idx; i++) - for(uint64_t j = 0; j < n_samples; j++) { - uint64_t idx = (i-start_idx)*n_samples_r+j; - dm_stripes_buf[idx]=dm_stripes_buf[idx]/dm_stripes_total_buf[idx]; - // taskObj.dm_stripes[i][j] = taskObj.dm_stripes[i][j] / taskObj.dm_stripes_total[i][j]; - } - - } - -#pragma acc exit data delete(lengths[:max_emb]) - free(lengths); -} - -void SUCMP_NM::unifrac(const su::biom_interface &table, - const su::BPTree &tree, - su::Method unifrac_method, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - const su::task_parameters* task_p) { - switch(unifrac_method) { - case su::unweighted: - unifracTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::weighted_normalized: - unifracTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::weighted_unnormalized: - unifracTT,double>( table, tree, false, dm_stripes,dm_stripes_total,task_p); - break; - case su::generalized: - unifracTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::unweighted_fp32: - unifracTT,float>( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::weighted_normalized_fp32: - unifracTT,float>( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::weighted_unnormalized_fp32: - unifracTT,float>( table, tree, false, dm_stripes,dm_stripes_total,task_p); - break; - case su::generalized_fp32: - unifracTT,float>( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - default: - fprintf(stderr, "Unknown unifrac task\n"); - exit(1); - break; - } -} - - -template -inline void unifrac_vawTT(const su::biom_interface &table, - const su::BPTree &tree, - const bool want_total, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - const su::task_parameters* task_p) { - int err; - // no processor affinity whenusing openacc or openmp - - if(table.n_samples != task_p->n_samples) { - fprintf(stderr, "Task and table n_samples not equal\n"); - exit(EXIT_FAILURE); - } - const unsigned int n_samples = task_p->n_samples; - const uint64_t n_samples_r = ((n_samples + UNIFRAC_BLOCK-1)/UNIFRAC_BLOCK)*UNIFRAC_BLOCK; // round up - - su::PropStackMulti propstack_multi(table.n_samples); - su::PropStackMulti countstack_multi(table.n_samples); - - const unsigned int max_emb = TaskT::RECOMMENDED_MAX_EMBS; - - TFloat *sample_total_counts; - - initialize_sample_counts(sample_total_counts, task_p, table); -#pragma acc enter data copyin(sample_total_counts[:n_samples_r]) - su::initialize_stripes(std::ref(dm_stripes), std::ref(dm_stripes_total), want_total, task_p); - - TaskT taskObj(std::ref(dm_stripes), std::ref(dm_stripes_total), sample_total_counts, max_emb, task_p); - - TFloat *lengths = NULL; - err = posix_memalign((void **)&lengths, 4096, sizeof(TFloat) * max_emb); - if(err != 0) { - fprintf(stderr, "posix_memalign(%d) failed: %d\n", sizeof(TFloat) * max_emb, err); - exit(EXIT_FAILURE); - } -#pragma acc enter data create(lengths[:max_emb]) - - unsigned int k = 0; // index in tree - const unsigned int max_k = (tree.nparens / 2) - 1; - - const unsigned int num_prop_chunks = propstack_multi.get_num_stacks(); - while (k &propstack = propstack_multi.get_prop_stack(ck); - su::PropStack &countstack = countstack_multi.get_prop_stack(ck); - const unsigned int tstart = propstack_multi.get_start(ck); - const unsigned int tend = propstack_multi.get_end(ck); - unsigned int my_filled_emb = 0; - unsigned int my_k=k_start; - - while ((my_filled_embbypass_tips && tree.isleaf(node)) - continue; - - if (ck==0) { // they all do the same thing, so enough for the first to update the global state - lengths[filled_emb] = tree.lengths[node]; - filled_emb++; - } - taskObj.embed_range(node_proportions, node_counts, tstart, tend, my_filled_emb); - my_filled_emb++; - } - if (ck==0) { // they all do the same thing, so enough for the first to update the global state - k=my_k; - } - } - -#pragma acc wait -#pragma acc update device(lengths[:filled_emb]) - taskObj.sync_embedded(filled_emb); - taskObj._run(filled_emb,lengths); - filled_emb = 0; - - su::try_report(task_p, k, max_k); - } - -#pragma acc wait - if(want_total) { - const uint64_t start_idx = task_p->start; - const uint64_t stop_idx = task_p->stop; - - TFloat * const dm_stripes_buf = taskObj.dm_stripes.buf; - const TFloat * const dm_stripes_total_buf = taskObj.dm_stripes_total.buf; - -#pragma acc parallel loop collapse(2) present(dm_stripes_buf,dm_stripes_total_buf) - for(uint64_t i = start_idx; i < stop_idx; i++) - for(uint64_t j = 0; j < n_samples; j++) { - uint64_t idx = (i-start_idx)*n_samples_r+j; - dm_stripes_buf[idx]=dm_stripes_buf[idx]/dm_stripes_total_buf[idx]; - // taskObj.dm_stripes[i][j] = taskObj.dm_stripes[i][j] / taskObj.dm_stripes_total[i][j]; - } - - } - - -#pragma acc exit data delete(lengths[:max_emb]) -#pragma acc exit data delete(sample_total_counts[:n_samples_r]) - free(lengths); - free(sample_total_counts); -} - -void SUCMP_NM::unifrac_vaw(const su::biom_interface &table, - const su::BPTree &tree, - su::Method unifrac_method, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - const su::task_parameters* task_p) { - switch(unifrac_method) { - case su::unweighted: - unifrac_vawTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::weighted_normalized: - unifrac_vawTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::weighted_unnormalized: - unifrac_vawTT,double>( table, tree, false, dm_stripes,dm_stripes_total,task_p); - break; - case su::generalized: - unifrac_vawTT,double>( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::unweighted_fp32: - unifrac_vawTT,float >( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::weighted_normalized_fp32: - unifrac_vawTT,float >( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - case su::weighted_unnormalized_fp32: - unifrac_vawTT,float >( table, tree, false, dm_stripes,dm_stripes_total,task_p); - break; - case su::generalized_fp32: - unifrac_vawTT,float >( table, tree, true, dm_stripes,dm_stripes_total,task_p); - break; - default: - fprintf(stderr, "Unknown unifrac task\n"); - exit(1); - break; - } -} - diff --git a/sucpp/unifrac_cmp.hpp b/sucpp/unifrac_cmp.hpp deleted file mode 100644 index 9d5a5787..00000000 --- a/sucpp/unifrac_cmp.hpp +++ /dev/null @@ -1,38 +0,0 @@ -/* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ - -#ifdef SUCMP_NM - -/* Note: Allow multiple definitions of this header, using different SUCMP_NM */ - -#include "task_parameters.hpp" -#include "tree.hpp" -#include "biom_interface.hpp" - -#include "unifrac_internal.hpp" - -namespace SUCMP_NM { - - void unifrac(const su::biom_interface &table, - const su::BPTree &tree, - su::Method unifrac_method, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - const su::task_parameters* task_p); - - void unifrac_vaw(const su::biom_interface &table, - const su::BPTree &tree, - su::Method unifrac_method, - std::vector &dm_stripes, - std::vector &dm_stripes_total, - const su::task_parameters* task_p); - -} - -#endif /* SUCMP_NM */ diff --git a/sucpp/unifrac_internal.cpp b/sucpp/unifrac_internal.cpp deleted file mode 100644 index 1fff6e14..00000000 --- a/sucpp/unifrac_internal.cpp +++ /dev/null @@ -1,290 +0,0 @@ -/* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ - -#include "tree.hpp" -#include "biom_interface.hpp" -#include "affinity.hpp" -#include -#include -#include -#include -#include -#include -#include - -#include "unifrac_internal.hpp" - -static pthread_mutex_t printf_mutex; -static bool* report_status; - -static int sync_printf(const char *format, ...) { - // https://stackoverflow.com/a/23587285/19741 - va_list args; - va_start(args, format); - - pthread_mutex_lock(&printf_mutex); - vprintf(format, args); - pthread_mutex_unlock(&printf_mutex); - - va_end(args); -} - -static void sig_handler(int signo) { - // http://www.thegeekstuff.com/2012/03/catch-signals-sample-c-code - if (signo == SIGUSR1) { - if(report_status == NULL) - fprintf(stderr, "Cannot report status.\n"); - else { - for(int i = 0; i < CPU_SETSIZE; i++) { - report_status[i] = true; - } - } - } -} - -using namespace su; - -void su::try_report(const su::task_parameters* task_p, unsigned int k, unsigned int max_k) { - if(__builtin_expect(report_status[task_p->tid], false)) { - sync_printf("tid:%u\tstart:%u\tstop:%u\tk:%u\ttotal:%u\n", task_p->tid, task_p->start, task_p->stop, k, max_k); - report_status[task_p->tid] = false; - } -} - -void su::register_report_status() { - // register a signal handler so we can ask the master thread for its - // progress - if (signal(SIGUSR1, sig_handler) == SIG_ERR) - fprintf(stderr, "Can't catch SIGUSR1\n"); - - report_status = (bool*)calloc(sizeof(bool), CPU_SETSIZE); - pthread_mutex_init(&printf_mutex, NULL); -} - -void su::remove_report_status() { - if(report_status != NULL) { - pthread_mutex_destroy(&printf_mutex); - free(report_status); - report_status = NULL; - } -} - -template -PropStack::PropStack(uint32_t vecsize) -: prop_stack() -, prop_map() -, defaultsize(vecsize) -{ - prop_map.reserve(1000); -} - -template -PropStack::~PropStack() { - // drain stack - for(unsigned int i = 0; i < prop_stack.size(); i++) { - TFloat *vec = prop_stack.top(); - prop_stack.pop(); - free(vec); - } - - // drain the map - for(auto it = prop_map.begin(); it != prop_map.end(); it++) { - TFloat *vec = it->second; - free(vec); - } - prop_map.clear(); -} - -template -TFloat* PropStack::get(uint32_t i) { - return prop_map[i]; -} - -template -void PropStack::push(uint32_t node) { - TFloat* vec = prop_map[node]; - prop_map.erase(node); - prop_stack.push(vec); -} - -template -TFloat* PropStack::pop(uint32_t node) { - /* - * if we don't have any available vectors, create one - * add it to our record of known vectors so we can track our mallocs - */ - TFloat *vec; - int err = 0; - if(prop_stack.empty()) { - err = posix_memalign((void **)&vec, 32, sizeof(TFloat) * defaultsize); - if(vec == NULL || err != 0) { - fprintf(stderr, "Failed to allocate %zd bytes, err %d; [%s]:%d\n", - sizeof(TFloat) * defaultsize, err, __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - } - else { - vec = prop_stack.top(); - prop_stack.pop(); - } - - prop_map[node] = vec; - return vec; -} - -// make sure they get instantiated -template class su::PropStack; -template class su::PropStack; - - -void su::initialize_stripes(std::vector &dm_stripes, - std::vector &dm_stripes_total, - bool want_total, - const su::task_parameters* task_p) { - int err = 0; - for(unsigned int i = task_p->start; i < task_p->stop; i++){ - err = posix_memalign((void **)&dm_stripes[i], 4096, sizeof(double) * task_p->n_samples); - if(dm_stripes[i] == NULL || err != 0) { - fprintf(stderr, "Failed to allocate %zd bytes, err %d; [%s]:%d\n", - sizeof(double) * task_p->n_samples, err, __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - for(unsigned int j = 0; j < task_p->n_samples; j++) - dm_stripes[i][j] = 0.; - - if(want_total) { - err = posix_memalign((void **)&dm_stripes_total[i], 4096, sizeof(double) * task_p->n_samples); - if(dm_stripes_total[i] == NULL || err != 0) { - fprintf(stderr, "Failed to allocate %zd bytes err %d; [%s]:%d\n", - sizeof(double) * task_p->n_samples, err, __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - for(unsigned int j = 0; j < task_p->n_samples; j++) - dm_stripes_total[i][j] = 0.; - } - } -} - -template -void su::set_proportions(TFloat* __restrict__ props, - const BPTree &tree, - uint32_t node, - const biom_interface &table, - PropStack &ps, - bool normalize) { - if(tree.isleaf(node)) { - table.get_obs_data(tree.names[node], props); - if (normalize) { -#pragma omp parallel for schedule(static) - for(unsigned int i = 0; i < table.n_samples; i++) { - props[i] /= table.sample_counts[i]; - } - } - - } else { - unsigned int current = tree.leftchild(node); - unsigned int right = tree.rightchild(node); - -#pragma omp parallel for schedule(static) - for(unsigned int i = 0; i < table.n_samples; i++) - props[i] = 0; - - while(current <= right && current != 0) { - TFloat * __restrict__ vec = ps.get(current); // pull from prop map - ps.push(current); // remove from prop map, place back on stack - -#pragma omp parallel for schedule(static) - for(unsigned int i = 0; i < table.n_samples; i++) - props[i] = props[i] + vec[i]; - - current = tree.rightsibling(current); - } - } -} - -// make sure they get instantiated -template void su::set_proportions(float* __restrict__ props, - const BPTree &tree, - uint32_t node, - const biom_interface &table, - PropStack &ps, - bool normalize); -template void su::set_proportions(double* __restrict__ props, - const BPTree &tree, - uint32_t node, - const biom_interface &table, - PropStack &ps, - bool normalize); - -template -void su::set_proportions_range(TFloat* __restrict__ props, - const BPTree &tree, - uint32_t node, - const biom_interface &table, - unsigned int start, unsigned int end, - PropStack &ps, - bool normalize) { - const unsigned int els = end-start; - if(tree.isleaf(node)) { - table.get_obs_data_range(tree.names[node], start, end, normalize, props); - } else { - const unsigned int right = tree.rightchild(node); - unsigned int current = tree.leftchild(node); - - for(unsigned int i = 0; i < els; i++) - props[i] = 0; - - while(current <= right && current != 0) { - const TFloat * __restrict__ vec = ps.get(current); // pull from prop map - ps.push(current); // remove from prop map, place back on stack - - for(unsigned int i = 0; i < els; i++) - props[i] += vec[i]; - - current = tree.rightsibling(current); - } - } -} - -// make sure they get instantiated -template void su::set_proportions_range(float* __restrict__ props, - const BPTree &tree, - uint32_t node, - const biom_interface &table, - unsigned int start, unsigned int end, - PropStack &ps, - bool normalize); -template void su::set_proportions_range(double* __restrict__ props, - const BPTree &tree, - uint32_t node, - const biom_interface &table, - unsigned int start, unsigned int end, - PropStack &ps, - bool normalize); - -std::vector su::make_strides(unsigned int n_samples) { - uint32_t n_rotations = (n_samples + 1) / 2; - std::vector dm_stripes(n_rotations); - - int err = 0; - for(unsigned int i = 0; i < n_rotations; i++) { - double* tmp; - err = posix_memalign((void **)&tmp, 32, sizeof(double) * n_samples); - if(tmp == NULL || err != 0) { - fprintf(stderr, "Failed to allocate %zd bytes, err %d; [%s]:%d\n", - sizeof(double) * n_samples, err, __FILE__, __LINE__); - exit(EXIT_FAILURE); - } - for(unsigned int j = 0; j < n_samples; j++) - tmp[j] = 0.0; - dm_stripes[i] = tmp; - } - return dm_stripes; -} - diff --git a/sucpp/unifrac_internal.hpp b/sucpp/unifrac_internal.hpp deleted file mode 100644 index b53b0b5a..00000000 --- a/sucpp/unifrac_internal.hpp +++ /dev/null @@ -1,96 +0,0 @@ -/* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ - -#ifndef __UNIFRAC_INTERNAL -#define __UNIFRAC_INTERNAL 1 - -#include -#include -#include -#include "biom_interface.hpp" -#include "task_parameters.hpp" -#include "unifrac.hpp" - -namespace su { - // helper reporting functions - void register_report_status(); - void remove_report_status(); - void try_report(const su::task_parameters* task_p, unsigned int k, unsigned int max_k); - - template - class PropStack { - private: - std::stack prop_stack; - std::unordered_map prop_map; - uint32_t defaultsize; - public: - PropStack(uint32_t vecsize); - virtual ~PropStack(); - TFloat* pop(uint32_t i); - void push(uint32_t i); - TFloat* get(uint32_t i); - }; - - // Helper class with default constructor - // The default is small enough to fit in L1 cache - template - class PropStackFixed : public PropStack { - public: - static const uint32_t DEF_VEC_SIZE = 1024*sizeof(double)/sizeof(TFloat); - - PropStackFixed() : PropStack(DEF_VEC_SIZE) {} - }; - - // Helper class that splits a large vec_size into several smaller chunks of def_size - template - class PropStackMulti { - protected: - const uint32_t vecsize; - std::vector > multi; - - public: - PropStackMulti(uint32_t _vecsize) - : vecsize(_vecsize) - , multi((vecsize + (PropStackFixed::DEF_VEC_SIZE-1))/PropStackFixed::DEF_VEC_SIZE) // round up - {} - ~PropStackMulti() {} - - uint32_t get_num_stacks() const {return (vecsize + (PropStackFixed::DEF_VEC_SIZE-1))/PropStackFixed::DEF_VEC_SIZE;} - - uint32_t get_start(uint32_t idx) const {return idx*PropStackFixed::DEF_VEC_SIZE;} - uint32_t get_end(uint32_t idx) const {return std::min((idx+1)*PropStackFixed::DEF_VEC_SIZE, vecsize);} - - PropStackFixed &get_prop_stack(uint32_t idx) {return multi[idx];} - }; - - template - void set_proportions(TFloat* __restrict__ props, - const BPTree &tree, uint32_t node, - const biom_interface &table, - PropStack &ps, - bool normalize = true); - - template - void set_proportions_range(TFloat* __restrict__ props, - const BPTree &tree, uint32_t node, - const biom_interface &table,unsigned int start, unsigned int end, - PropStack &ps, - bool normalize = true); - - - void initialize_stripes(std::vector &dm_stripes, - std::vector &dm_stripes_total, - bool want_total, - const su::task_parameters* task_p); - - std::vector make_strides(unsigned int n_samples); - -} - -#endif diff --git a/sucpp/unifrac_task.cpp b/sucpp/unifrac_task.cpp deleted file mode 100644 index 0a40a731..00000000 --- a/sucpp/unifrac_task.cpp +++ /dev/null @@ -1,785 +0,0 @@ -#include -#include "unifrac_task.hpp" -#include - - - - -template -void SUCMP_NM::UnifracUnnormalizedWeightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { - const uint64_t start_idx = this->task_p->start; - const uint64_t stop_idx = this->task_p->stop; - const uint64_t n_samples = this->task_p->n_samples; - const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - - // openacc only works well with local variables - const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; - TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; - - bool * const __restrict__ zcheck = this->zcheck; - TFloat * const __restrict__ sums = this->sums; - - const uint64_t step_size = SUCMP_NM::UnifracUnnormalizedWeightedTask::step_size; - const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up - - // check for zero values and pre-compute single column sums -#ifdef _OPENACC -#pragma acc parallel loop present(embedded_proportions,lengths,zcheck,sums) -#else -#pragma omp parallel for default(shared) -#endif - for(uint64_t k=0; k::acc_vector_size; -#pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,dm_stripes_buf,lengths,zcheck,sums) async -#else - // use dynamic scheduling due to non-homogeneity in the loop -#pragma omp parallel for default(shared) schedule(dynamic,1) -#endif - for(uint64_t sk = 0; sk < sample_steps ; sk++) { - for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { - for(uint64_t ik = 0; ik < step_size ; ik++) { - const uint64_t k = sk*step_size + ik; - - if (k>=n_samples) continue; // past the limit - - const bool zcheck_k = zcheck[sk]; // due to loop collapse in ACC, must load in here - - const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound - - const bool allzero_k = zcheck[k]; - const bool allzero_l1 = zcheck[l1]; - - if (allzero_k && allzero_l1) { - // nothing to do, would have to add 0 - } else { - TFloat my_stripe; - - if (allzero_k || allzero_l1) { - // one side has all zeros - // we can use the distributed property, and use the pre-computed values - - const uint64_t ridx = (allzero_k) ? l1 : // if (nonzero_l1) ridx=l1 // fabs(k-l1), with k==0 - k; // if (nonzero_k) ridx=k // fabs(k-l1), with l1==0 - - // keep reads in the same place to maximize GPU warp performance - my_stripe = sums[ridx]; - - } else { - // both sides non zero, use the explicit but slow approach - my_stripe = 0.0; - -#pragma acc loop seq - for (uint64_t emb=0; embembedded_proportions,this->embedded_proportions_alt); -#endif -} - -template -void SUCMP_NM::UnifracVawUnnormalizedWeightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { - const uint64_t start_idx = this->task_p->start; - const uint64_t stop_idx = this->task_p->stop; - const uint64_t n_samples = this->task_p->n_samples; - const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - - // openacc only works well with local variables - const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; - const TFloat * const __restrict__ embedded_counts = this->embedded_counts; - const TFloat * const __restrict__ sample_total_counts = this->sample_total_counts; - TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; - - const uint64_t step_size = SUCMP_NM::UnifracVawUnnormalizedWeightedTask::step_size; - const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up - - // point of thread -#ifdef _OPENACC - const unsigned int acc_vector_size = SUCMP_NM::UnifracVawUnnormalizedWeightedTask::acc_vector_size; -#pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,embedded_counts,sample_total_counts,dm_stripes_buf,lengths) async -#else -#pragma omp parallel for default(shared) schedule(dynamic,1) -#endif - for(uint64_t sk = 0; sk < sample_steps ; sk++) { - for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { - for(uint64_t ik = 0; ik < step_size ; ik++) { - const uint64_t k = sk*step_size + ik; - const uint64_t idx = (stripe-start_idx) * n_samples_r; - TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; - //TFloat *dm_stripe = dm_stripes[stripe]; - - if (k>=n_samples) continue; // past the limit - - const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound - - const TFloat m = sample_total_counts[k] + sample_total_counts[l1]; - - TFloat my_stripe = dm_stripe[k]; - -#pragma acc loop seq - for (uint64_t emb=0; emb 0) { - TFloat u1 = embedded_proportions[offset + k]; - TFloat v1 = embedded_proportions[offset + l1]; - TFloat diff1 = fabs(u1 - v1); - TFloat length = lengths[emb]; - - my_stripe += (diff1 * length) / vaw; - } - } - - dm_stripe[k] = my_stripe; - } - - } - } - -#ifdef _OPENACC - // next iteration will use the alternative space - std::swap(this->embedded_proportions,this->embedded_proportions_alt); -#endif -} - -template -void SUCMP_NM::UnifracNormalizedWeightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { - const uint64_t start_idx = this->task_p->start; - const uint64_t stop_idx = this->task_p->stop; - const uint64_t n_samples = this->task_p->n_samples; - const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - - // openacc only works well with local variables - const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; - TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; - TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; - - bool * const __restrict__ zcheck = this->zcheck; - TFloat * const __restrict__ sums = this->sums; - - const uint64_t step_size = SUCMP_NM::UnifracNormalizedWeightedTask::step_size; - const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up - - // check for zero values and pre-compute single column sums -#ifdef _OPENACC -#pragma acc parallel loop present(embedded_proportions,lengths,zcheck,sums) -#else -#pragma omp parallel for default(shared) -#endif - for(uint64_t k=0; k::acc_vector_size; -#pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,dm_stripes_buf,dm_stripes_total_buf,lengths,zcheck,sums) async -#else - // use dynamic scheduling due to non-homogeneity in the loop -#pragma omp parallel for schedule(dynamic,1) default(shared) -#endif - for(uint64_t sk = 0; sk < sample_steps ; sk++) { - for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { - for(uint64_t ik = 0; ik < step_size ; ik++) { - const uint64_t k = sk*step_size + ik; - - if (k>=n_samples) continue; // past the limit - - const bool zcheck_k = zcheck[sk]; // due to loop collapse in ACC, must load in here - - const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound - - const bool allzero_k = zcheck[k]; - const bool allzero_l1 = zcheck[l1]; - - if (allzero_k && allzero_l1) { - // nothing to do, would have to add 0 - } else { - const uint64_t idx = (stripe-start_idx) * n_samples_r; - TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; - TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; - //TFloat *dm_stripe = dm_stripes[stripe]; - //TFloat *dm_stripe_total = dm_stripes_total[stripe]; - - // the totals can always use the distributed property - dm_stripe_total[k] += sums[k] + sums[l1]; - - TFloat my_stripe; - - if (allzero_k || allzero_l1) { - // one side has all zeros - // we can use the distributed property, and use the pre-computed values - - const uint64_t ridx = (allzero_k) ? l1 : // if (nonzero_l1) ridx=l1 // fabs(k-l1), with k==0 - k; // if (nonzero_k) ridx=k // fabs(k-l1), with l1==0 - - // keep reads in the same place to maximize GPU warp performance - my_stripe = sums[ridx]; - - } else { - // both sides non zero, use the explicit but slow approach - - my_stripe = 0.0; - -#pragma acc loop seq - for (uint64_t emb=0; embembedded_proportions,this->embedded_proportions_alt); -#endif -} - -template -void SUCMP_NM::UnifracVawNormalizedWeightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { - const uint64_t start_idx = this->task_p->start; - const uint64_t stop_idx = this->task_p->stop; - const uint64_t n_samples = this->task_p->n_samples; - const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - - // openacc only works well with local variables - const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; - const TFloat * const __restrict__ embedded_counts = this->embedded_counts; - const TFloat * const __restrict__ sample_total_counts = this->sample_total_counts; - TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; - TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; - - const uint64_t step_size = SUCMP_NM::UnifracVawNormalizedWeightedTask::step_size; - const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up - - // point of thread -#ifdef _OPENACC - const unsigned int acc_vector_size = SUCMP_NM::UnifracVawNormalizedWeightedTask::acc_vector_size; -#pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,embedded_counts,sample_total_counts,dm_stripes_buf,dm_stripes_total_buf,lengths) async -#else -#pragma omp parallel for schedule(dynamic,1) default(shared) -#endif - for(uint64_t sk = 0; sk < sample_steps ; sk++) { - for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { - for(uint64_t ik = 0; ik < step_size ; ik++) { - const uint64_t k = sk*step_size + ik; - const uint64_t idx = (stripe-start_idx) * n_samples_r; - TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; - TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; - //TFloat *dm_stripe = dm_stripes[stripe]; - //TFloat *dm_stripe_total = dm_stripes_total[stripe]; - - if (k>=n_samples) continue; // past the limit - - const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound - - const TFloat m = sample_total_counts[k] + sample_total_counts[l1]; - - TFloat my_stripe = dm_stripe[k]; - TFloat my_stripe_total = dm_stripe_total[k]; - -#pragma acc loop seq - for (uint64_t emb=0; emb 0) { - TFloat u1 = embedded_proportions[offset + k]; - TFloat v1 = embedded_proportions[offset + l1]; - TFloat diff1 = fabs(u1 - v1); - TFloat length = lengths[emb]; - - my_stripe += (diff1 * length) / vaw; - my_stripe_total += ((u1 + v1) * length) / vaw; - } - } - - dm_stripe[k] = my_stripe; - dm_stripe_total[k] = my_stripe_total; - - } - - } - } - -#ifdef _OPENACC - // next iteration will use the alternative space - std::swap(this->embedded_proportions,this->embedded_proportions_alt); -#endif -} - -template -void SUCMP_NM::UnifracGeneralizedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { - const uint64_t start_idx = this->task_p->start; - const uint64_t stop_idx = this->task_p->stop; - const uint64_t n_samples = this->task_p->n_samples; - const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - - // openacc only works well with local variables - const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; - TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; - TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; - - const TFloat g_unifrac_alpha = this->task_p->g_unifrac_alpha; - - const uint64_t step_size = SUCMP_NM::UnifracGeneralizedTask::step_size; - const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up - - // point of thread -#ifdef _OPENACC - const unsigned int acc_vector_size = SUCMP_NM::UnifracGeneralizedTask::acc_vector_size; -#pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,dm_stripes_buf,dm_stripes_total_buf,lengths) async -#else -#pragma omp parallel for schedule(dynamic,1) default(shared) -#endif - for(uint64_t sk = 0; sk < sample_steps ; sk++) { - for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { - for(uint64_t ik = 0; ik < step_size ; ik++) { - const uint64_t k = sk*step_size + ik; - const uint64_t idx = (stripe-start_idx) * n_samples_r; - TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; - TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; - //TFloat *dm_stripe = dm_stripes[stripe]; - //TFloat *dm_stripe_total = dm_stripes_total[stripe]; - - if (k>=n_samples) continue; // past the limit - - const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound - - TFloat my_stripe = dm_stripe[k]; - TFloat my_stripe_total = dm_stripe_total[k]; - -#pragma acc loop seq - for (uint64_t emb=0; embembedded_proportions,this->embedded_proportions_alt); -#endif -} - -template -void SUCMP_NM::UnifracVawGeneralizedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { - const uint64_t start_idx = this->task_p->start; - const uint64_t stop_idx = this->task_p->stop; - const uint64_t n_samples = this->task_p->n_samples; - const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - - const TFloat g_unifrac_alpha = this->task_p->g_unifrac_alpha; - - // openacc only works well with local variables - const TFloat * const __restrict__ embedded_proportions = this->embedded_proportions; - const TFloat * const __restrict__ embedded_counts = this->embedded_counts; - const TFloat * const __restrict__ sample_total_counts = this->sample_total_counts; - TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; - TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; - - const uint64_t step_size = SUCMP_NM::UnifracVawGeneralizedTask::step_size; - const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up - // quick hack, to be finished - - // point of thread -#ifdef _OPENACC - const unsigned int acc_vector_size = SUCMP_NM::UnifracVawGeneralizedTask::acc_vector_size; -#pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,embedded_counts,sample_total_counts,dm_stripes_buf,dm_stripes_total_buf,lengths) async -#else -#pragma omp parallel for schedule(dynamic,1) default(shared) -#endif - for(uint64_t sk = 0; sk < sample_steps ; sk++) { - for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { - for(uint64_t ik = 0; ik < step_size ; ik++) { - const uint64_t k = sk*step_size + ik; - const uint64_t idx = (stripe-start_idx) * n_samples_r; - TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; - TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; - //TFloat *dm_stripe = dm_stripes[stripe]; - //TFloat *dm_stripe_total = dm_stripes_total[stripe]; - - if (k>=n_samples) continue; // past the limit - - const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound - - const TFloat m = sample_total_counts[k] + sample_total_counts[l1]; - - TFloat my_stripe = dm_stripe[k]; - TFloat my_stripe_total = dm_stripe_total[k]; - -#pragma acc loop seq - for (uint64_t emb=0; emb 0) { - TFloat u1 = embedded_proportions[offset + k]; - TFloat v1 = embedded_proportions[offset + l1]; - TFloat length = lengths[emb]; - - TFloat sum1 = (u1 + v1) / vaw; - TFloat sub1 = fabs(u1 - v1) / vaw; - TFloat sum_pow1 = pow(sum1, g_unifrac_alpha) * length; - - my_stripe += sum_pow1 * (sub1 / sum1); - my_stripe_total += sum_pow1; - } - } - - dm_stripe[k] = my_stripe; - dm_stripe_total[k] = my_stripe_total; - - } - } - } - -#ifdef _OPENACC - // next iteration will use the alternative space - std::swap(this->embedded_proportions,this->embedded_proportions_alt); -#endif -} - -template -void SUCMP_NM::UnifracUnweightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { - const uint64_t start_idx = this->task_p->start; - const uint64_t stop_idx = this->task_p->stop; - const uint64_t n_samples = this->task_p->n_samples; - const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - - // openacc only works well with local variables - const uint64_t * const __restrict__ embedded_proportions = this->embedded_proportions; - TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; - TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; - - TFloat * const __restrict__ sums = this->sums; - - const uint64_t step_size = SUCMP_NM::UnifracUnweightedTask::step_size; - const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up - - const uint64_t filled_embs_els = filled_embs/64; - const uint64_t filled_embs_rem = filled_embs%64; - - const uint64_t filled_embs_els_round = (filled_embs+63)/64; - - - // pre-compute sums of length elements, since they are likely to be accessed many times - // We will use a 8-bit map, to keep it small enough to keep in L1 cache -#ifdef _OPENACC -#pragma acc parallel loop collapse(2) gang present(lengths,sums) async -#else -#pragma omp parallel for default(shared) -#endif - for (uint64_t emb_el=0; emb_el> 0) & 1) * pl[0]) + (((b8_i >> 1) & 1) * pl[1]) + - (((b8_i >> 2) & 1) * pl[2]) + (((b8_i >> 3) & 1) * pl[3]) + - (((b8_i >> 4) & 1) * pl[4]) + (((b8_i >> 5) & 1) * pl[5]) + - (((b8_i >> 6) & 1) * pl[6]) + (((b8_i >> 7) & 1) * pl[7]); - } - } - } - if (filled_embs_rem>0) { // add also the overflow elements - const uint64_t emb_el=filled_embs_els; -#ifdef _OPENACC -#pragma acc parallel loop gang present(lengths,sums) async -#else - // no advantage of OMP, too small -#endif - for (uint64_t sub8=0; sub8<8; sub8++) { - // we are summing we have enough buffer in sums - const uint64_t emb8 = emb_el*8+sub8; - TFloat * __restrict__ psum = &(sums[emb8<<8]); - -#pragma acc loop vector - // compute all the combinations for this block, set to 0 any past the limit - // as above - for (uint64_t b8_i=0; b8_i<0x100; b8_i++) { - TFloat val= 0; - for (uint64_t li=(emb8*8); li> (li-(emb8*8))) & 1) * lengths[li]; - } - psum[b8_i] = val; - } - } - } - - // point of thread -#ifdef _OPENACC -#pragma acc wait - const unsigned int acc_vector_size = SUCMP_NM::UnifracUnweightedTask::acc_vector_size; -#pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,dm_stripes_buf,dm_stripes_total_buf,sums) async -#else - // use dynamic scheduling due to non-homogeneity in the loop -#pragma omp parallel for schedule(dynamic,1) default(shared) -#endif - for(uint64_t sk = 0; sk < sample_steps ; sk++) { - for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { - for(uint64_t ik = 0; ik < step_size ; ik++) { - const uint64_t k = sk*step_size + ik; - const uint64_t idx = (stripe-start_idx) * n_samples_r; - TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; - TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; - //TFloat *dm_stripe = dm_stripes[stripe]; - //TFloat *dm_stripe_total = dm_stripes_total[stripe]; - - if (k>=n_samples) continue; // past the limit - - const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound - - bool did_update = false; - TFloat my_stripe = 0.0; - TFloat my_stripe_total = 0.0; - -#pragma acc loop seq - for (uint64_t emb_el=0; emb_el> 8) & 0xff)] + - psum[0x200+((x1 >> 16) & 0xff)] + - psum[0x300+((x1 >> 24) & 0xff)] + - psum[0x400+((x1 >> 32) & 0xff)] + - psum[0x500+((x1 >> 40) & 0xff)] + - psum[0x600+((x1 >> 48) & 0xff)] + - psum[0x700+((x1 >> 56) )]; - my_stripe_total += psum[ (o1 & 0xff)] + - psum[0x100+((o1 >> 8) & 0xff)] + - psum[0x200+((o1 >> 16) & 0xff)] + - psum[0x300+((o1 >> 24) & 0xff)] + - psum[0x400+((o1 >> 32) & 0xff)] + - psum[0x500+((o1 >> 40) & 0xff)] + - psum[0x600+((o1 >> 48) & 0xff)] + - psum[0x700+((o1 >> 56) )]; - } - } - - if (did_update) { - dm_stripe[k] += my_stripe; - dm_stripe_total[k] += my_stripe_total; - } - - } - - } - } - -#ifdef _OPENACC - // next iteration will use the alternative space - std::swap(this->embedded_proportions,this->embedded_proportions_alt); -#endif -} - -template -void SUCMP_NM::UnifracVawUnweightedTask::_run(unsigned int filled_embs, const TFloat * __restrict__ lengths) { - const uint64_t start_idx = this->task_p->start; - const uint64_t stop_idx = this->task_p->stop; - const uint64_t n_samples = this->task_p->n_samples; - const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - - // openacc only works well with local variables - const uint32_t * const __restrict__ embedded_proportions = this->embedded_proportions; - const TFloat * const __restrict__ embedded_counts = this->embedded_counts; - const TFloat * const __restrict__ sample_total_counts = this->sample_total_counts; - TFloat * const __restrict__ dm_stripes_buf = this->dm_stripes.buf; - TFloat * const __restrict__ dm_stripes_total_buf = this->dm_stripes_total.buf; - - const uint64_t step_size = SUCMP_NM::UnifracVawUnweightedTask::step_size; - const uint64_t sample_steps = (n_samples+(step_size-1))/step_size; // round up - - const uint64_t filled_embs_els = (filled_embs+31)/32; // round up - - // point of thread -#ifdef _OPENACC - const unsigned int acc_vector_size = SUCMP_NM::UnifracVawUnweightedTask::acc_vector_size; -#pragma acc parallel loop collapse(3) vector_length(acc_vector_size) present(embedded_proportions,embedded_counts,sample_total_counts,dm_stripes_buf,dm_stripes_total_buf,lengths) async -#else -#pragma omp parallel for schedule(dynamic,1) default(shared) -#endif - for(uint64_t sk = 0; sk < sample_steps ; sk++) { - for(uint64_t stripe = start_idx; stripe < stop_idx; stripe++) { - for(uint64_t ik = 0; ik < step_size ; ik++) { - const uint64_t k = sk*step_size + ik; - const uint64_t idx = (stripe-start_idx) * n_samples_r; - TFloat * const __restrict__ dm_stripe = dm_stripes_buf+idx; - TFloat * const __restrict__ dm_stripe_total = dm_stripes_total_buf+idx; - //TFloat *dm_stripe = dm_stripes[stripe]; - //TFloat *dm_stripe_total = dm_stripes_total[stripe]; - - if (k>=n_samples) continue; // past the limit - - const uint64_t l1 = (k + stripe + 1)%n_samples; // wraparound - - TFloat my_stripe = dm_stripe[k]; - TFloat my_stripe_total = dm_stripe_total[k]; - - const TFloat m = sample_total_counts[k] + sample_total_counts[l1]; - -#pragma acc loop seq - for (uint64_t emb_el=0; emb_el 0) { - TFloat length = lengths[emb]; - TFloat lv1 = length / vaw; - - my_stripe += ((x1 >> ei) & 1)*lv1; - my_stripe_total += ((o1 >> ei) & 1)*lv1; - } - } - } - } - - dm_stripe[k] = my_stripe; - dm_stripe_total[k] = my_stripe_total; - - } - - } - } - -#ifdef _OPENACC - // next iteration will use the alternative space - std::swap(this->embedded_proportions,this->embedded_proportions_alt); -#endif -} - diff --git a/sucpp/unifrac_task.hpp b/sucpp/unifrac_task.hpp deleted file mode 100644 index 7424c8e0..00000000 --- a/sucpp/unifrac_task.hpp +++ /dev/null @@ -1,577 +0,0 @@ -/* - * BSD 3-Clause License - * - * Copyright (c) 2016-2021, UniFrac development team. - * All rights reserved. - * - * See LICENSE file for more details - */ - -#include "task_parameters.hpp" -#include -#include -#include -#include -#include -#include - - -#ifndef __UNIFRAC_TASKS -#define __UNIFRAC_TASKS 1 - -#ifdef _OPENACC - -#define SUCMP_NM su_acc - - #ifndef SMALLGPU - // defaultt on larger alignment, which improves performance on GPUs like V100 -#define UNIFRAC_BLOCK 64 - #else - // smaller GPUs prefer smaller allignment -#define UNIFRAC_BLOCK 32 - #endif - -#else - -#define SUCMP_NM su_cpu - - -// CPUs don't need such a big alignment -#define UNIFRAC_BLOCK 16 -#endif - -namespace SUCMP_NM { - - // Note: This adds a copy, which is suboptimal - // But was the easiest way to get a contiguous buffer - // And it does allow for fp32 compute, when desired - template - class UnifracTaskVector { - private: - std::vector &dm_stripes; - const su::task_parameters* const task_p; - - public: - const unsigned int start_idx; - const unsigned int n_samples; - const uint64_t n_samples_r; - TFloat* const buf; - - UnifracTaskVector(std::vector &_dm_stripes, const su::task_parameters* _task_p) - : dm_stripes(_dm_stripes), task_p(_task_p) - , start_idx(task_p->start), n_samples(task_p->n_samples) - , n_samples_r(((n_samples + UNIFRAC_BLOCK-1)/UNIFRAC_BLOCK)*UNIFRAC_BLOCK) // round up - , buf((dm_stripes[start_idx]==NULL) ? NULL : new TFloat[n_samples_r*(task_p->stop-start_idx)]) // dm_stripes could be null, in which case keep it null - { - TFloat* const ibuf = buf; - if (ibuf != NULL) { -#ifdef _OPENACC - const uint64_t bufels = n_samples_r * (task_p->stop-start_idx); -#endif - for(unsigned int stripe=start_idx; stripe < task_p->stop; stripe++) { - double * dm_stripe = dm_stripes[stripe]; - TFloat * buf_stripe = this->operator[](stripe); - for(unsigned int j=0; jstop-start_idx); -#pragma acc exit data copyout(ibuf[:bufels]) -#endif - for(unsigned int stripe=start_idx; stripe < task_p->stop; stripe++) { - double * dm_stripe = dm_stripes[stripe]; - TFloat * buf_stripe = this->operator[](stripe); - for(unsigned int j=0; j - class UnifracTaskBase { - public: - UnifracTaskVector dm_stripes; - UnifracTaskVector dm_stripes_total; - - const su::task_parameters* task_p; - - const unsigned int max_embs; - TEmb * embedded_proportions; -#ifdef _OPENACC - protected: - // alternate buffer only needed in async environments, like openacc - TEmb * embedded_proportions_alt; // used as temp - public: -#endif - - UnifracTaskBase(std::vector &_dm_stripes, std::vector &_dm_stripes_total, unsigned int _max_embs, const su::task_parameters* _task_p) - : dm_stripes(_dm_stripes,_task_p), dm_stripes_total(_dm_stripes_total,_task_p), task_p(_task_p) - , max_embs(_max_embs) - , embedded_proportions(initialize_embedded(dm_stripes.n_samples_r,_max_embs)) -#ifdef _OPENACC - , embedded_proportions_alt(initialize_embedded(dm_stripes.n_samples_r,_max_embs)) -#endif - {} - - /* remove - // Note: not const, since they share a mutable state - UnifracTaskBase(UnifracTaskBase &baseObj) - : dm_stripes(baseObj.dm_stripes), dm_stripes_total(baseObj.dm_stripes_total), task_p(baseObj.task_p) {} - */ - - virtual ~UnifracTaskBase() - { -#ifdef _OPENACC - const uint64_t n_samples_r = dm_stripes.n_samples_r; - const uint64_t bsize = n_samples_r * get_emb_els(max_embs); -#pragma acc exit data delete(embedded_proportions_alt[:bsize]) -#pragma acc exit data delete(embedded_proportions[:bsize]) - free(embedded_proportions_alt); -#endif - free(embedded_proportions); - } - - void sync_embedded_proportions(unsigned int filled_embs) - { -#ifdef _OPENACC - const uint64_t n_samples_r = dm_stripes.n_samples_r; - const uint64_t bsize = n_samples_r * get_emb_els(filled_embs); -#pragma acc update device(embedded_proportions[:bsize]) -#endif - } - - static unsigned int get_emb_els(unsigned int max_embs); - - static TEmb *initialize_embedded(const uint64_t n_samples_r, unsigned int max_embs) { - uint64_t bsize = n_samples_r * get_emb_els(max_embs); - - TEmb* buf = NULL; - int err = posix_memalign((void **)&buf, 4096, sizeof(TEmb) * bsize); - if(buf == NULL || err != 0) { - fprintf(stderr, "Failed to allocate %zd bytes, err %d; [%s]:%d\n", - sizeof(TEmb) * bsize, err, __FILE__, __LINE__); - exit(EXIT_FAILURE); - } -#pragma acc enter data create(buf[:bsize]) - return buf; - } - - void embed_proportions_range(const TFloat* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb); - void embed_proportions(const TFloat* __restrict__ in, unsigned int emb) {embed_proportions_range(in,0,dm_stripes.n_samples,emb);} - - - - // - // ===== Internal, do not use directly ======= - // - - - // Just copy from one buffer to another - // May convert between fp formats in the process (if TOut!=double) - template void embed_proportions_range_straight(TOut* __restrict__ out, const TFloat* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) const - { - const unsigned int n_samples = dm_stripes.n_samples; - const uint64_t n_samples_r = dm_stripes.n_samples_r; - const uint64_t offset = emb * n_samples_r; - - for(unsigned int i = start; i < end; i++) { - out[offset + i] = in[i-start]; - } - - if (end==n_samples) { - // avoid NaNs - for(unsigned int i = n_samples; i < n_samples_r; i++) { - out[offset + i] = 0.0; - } - } - } - - // packed bool - // Compute (in[:]>0) on each element, and store only the boolean bit. - // The output values are stored in a multi-byte format, one bit per emb index, - // so it will likely take multiple passes to store all the values - // - // Note: assumes we are processing emb in increasing order, starting from 0 - template void embed_proportions_range_bool(TOut* __restrict__ out, const TFloat* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) const - { - const unsigned int n_packed = sizeof(TOut)*8;// e.g. 32 for unit32_t - const unsigned int n_samples = dm_stripes.n_samples; - const uint64_t n_samples_r = dm_stripes.n_samples_r; - // The output values are stored in a multi-byte format, one bit per emb index - // Compute the element to store the bit into, as well as whichbit in that element - unsigned int emb_block = emb/n_packed; // beginning of the element block - unsigned int emb_bit = emb%n_packed; // bit inside the elements - const uint64_t offset = emb_block * n_samples_r; - - if (emb_bit==0) { - // assign for emb_bit==0, so it clears the other bits - // assumes we processing emb in increasing order, starting from 0 - for(unsigned int i = start; i < end; i++) { - out[offset + i] = (in[i-start] > 0); - } - - if (end==n_samples) { - // avoid NaNs - for(unsigned int i = n_samples; i < n_samples_r; i++) { - out[offset + i] = 0; - } - } - } else { - // just update my bit - for(unsigned int i = start; i < end; i++) { - out[offset + i] |= (TOut(in[i-start] > 0) << emb_bit); - } - - // the rest of the els are already OK - } - } - }; - - // straight embeded_proportions - template<> inline void UnifracTaskBase::embed_proportions_range(const double* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) {embed_proportions_range_straight(embedded_proportions,in,start,end,emb);} - template<> inline void UnifracTaskBase::embed_proportions_range(const double* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) {embed_proportions_range_straight(embedded_proportions,in,start,end,emb);} - template<> inline void UnifracTaskBase::embed_proportions_range(const float* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) {embed_proportions_range_straight(embedded_proportions,in,start,end,emb);} - template<> inline unsigned int UnifracTaskBase::get_emb_els(unsigned int max_embs) {return max_embs;} - template<> inline unsigned int UnifracTaskBase::get_emb_els(unsigned int max_embs) {return max_embs;} - template<> inline unsigned int UnifracTaskBase::get_emb_els(unsigned int max_embs) {return max_embs;} - - //packed bool embeded_proportions - template<> inline void UnifracTaskBase::embed_proportions_range(const double* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) {embed_proportions_range_bool(embedded_proportions,in,start,end,emb);} - template<> inline void UnifracTaskBase::embed_proportions_range(const float* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) {embed_proportions_range_bool(embedded_proportions,in,start,end,emb);} - template<> inline unsigned int UnifracTaskBase::get_emb_els(unsigned int max_embs) {return (max_embs+31)/32;} - template<> inline unsigned int UnifracTaskBase::get_emb_els(unsigned int max_embs) {return (max_embs+31)/32;} - - template<> inline void UnifracTaskBase::embed_proportions_range(const double* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) {embed_proportions_range_bool(embedded_proportions,in,start,end,emb);} - template<> inline void UnifracTaskBase::embed_proportions_range(const float* __restrict__ in, unsigned int start, unsigned int end, unsigned int emb) {embed_proportions_range_bool(embedded_proportions,in,start,end,emb);} - template<> inline unsigned int UnifracTaskBase::get_emb_els(unsigned int max_embs) {return (max_embs+63)/64;} - template<> inline unsigned int UnifracTaskBase::get_emb_els(unsigned int max_embs) {return (max_embs+63)/64;} - - /* void unifrac tasks - * - * all methods utilize the same function signature. that signature is as follows: - * - * dm_stripes vector the stripes of the distance matrix being accumulated - * into for unique branch length - * dm_stripes vector the stripes of the distance matrix being accumulated - * into for total branch length (e.g., to normalize unweighted unifrac) - * embedded_proportions the proportions vector for a sample, or rather - * the counts vector normalized to 1. this vector is embedded as it is - * duplicated: if A, B and C are proportions for features A, B, and C, the - * vector will look like [A B C A B C]. - * length the branch length of the current node to its parent. - * task_p task specific parameters. - */ - - template - class UnifracTask : public UnifracTaskBase { - protected: - // Use one cache line on CPU - // On GPU, shaing a cache line is actually a good thing - static const unsigned int step_size = 16*4/sizeof(TFloat); - -#ifdef _OPENACC - // Use as big vector size as we can, to maximize cache line reuse - static const unsigned int acc_vector_size = 2048; -#endif - - public: - - UnifracTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, unsigned int _max_embs, const su::task_parameters* _task_p) - : UnifracTaskBase(_dm_stripes, _dm_stripes_total, _max_embs, _task_p) {} - - /* delete - UnifracTask(UnifracTaskBase &baseObj, const TEmb * _embedded_proportions, unsigned int _max_embs) - : UnifracTaskBase(baseObj) - , embedded_proportions(_embedded_proportions), max_embs(_max_embs) {} - */ - - - virtual ~UnifracTask() {} - - virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) = 0; - - protected: - static const unsigned int RECOMMENDED_MAX_EMBS_STRAIGHT = 128-16; // a little less to leave a bit of space of maxed-out L1 - // packed uses 32x less memory,so this should be 32x larger than straight... but there are additional structures, so use half of that - static const unsigned int RECOMMENDED_MAX_EMBS_BOOL = 64*32; - - }; - - - template - class UnifracUnnormalizedWeightedTask : public UnifracTask { - public: - static const unsigned int RECOMMENDED_MAX_EMBS = UnifracTask::RECOMMENDED_MAX_EMBS_STRAIGHT; - - UnifracUnnormalizedWeightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, unsigned int _max_embs, const su::task_parameters* _task_p) - : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) - { - const unsigned int n_samples = this->task_p->n_samples; - - zcheck = NULL; - sums = NULL; - posix_memalign((void **)&zcheck, 4096, sizeof(bool) * n_samples); - posix_memalign((void **)&sums , 4096, sizeof(TFloat) * n_samples); -#pragma acc enter data create(zcheck[:n_samples],sums[:n_samples]) - } - - virtual ~UnifracUnnormalizedWeightedTask() - { -#ifdef _OPENACC - const unsigned int n_samples = this->task_p->n_samples; -#pragma acc exit data delete(sums[:n_samples],zcheck[:n_samples]) -#endif - free(sums); - free(zcheck); - } - - virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} - - void _run(unsigned int filled_embs, const TFloat * __restrict__ length); - protected: - // temp buffers - bool *zcheck; - TFloat *sums; - }; - template - class UnifracNormalizedWeightedTask : public UnifracTask { - public: - static const unsigned int RECOMMENDED_MAX_EMBS = UnifracTask::RECOMMENDED_MAX_EMBS_STRAIGHT; - - UnifracNormalizedWeightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, unsigned int _max_embs, const su::task_parameters* _task_p) - : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) - { - const unsigned int n_samples = this->task_p->n_samples; - - zcheck = NULL; - sums = NULL; - posix_memalign((void **)&zcheck, 4096, sizeof(bool) * n_samples); - posix_memalign((void **)&sums , 4096, sizeof(TFloat) * n_samples); -#pragma acc enter data create(zcheck[:n_samples],sums[:n_samples]) - } - - virtual ~UnifracNormalizedWeightedTask() - { -#ifdef _OPENACC - const unsigned int n_samples = this->task_p->n_samples; -#pragma acc exit data delete(sums[:n_samples],zcheck[:n_samples]) -#endif - free(sums); - free(zcheck); - } - - virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} - - void _run(unsigned int filled_embs, const TFloat * __restrict__ length); - protected: - // temp buffers - bool *zcheck; - TFloat *sums; - }; - template - class UnifracUnweightedTask : public UnifracTask { - public: - static const unsigned int RECOMMENDED_MAX_EMBS = UnifracTask::RECOMMENDED_MAX_EMBS_BOOL; - - // Note: _max_emb MUST be multiple of 64 - UnifracUnweightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, unsigned int _max_embs, const su::task_parameters* _task_p) - : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) - { - const unsigned int bsize = _max_embs*(0x400/32); - sums = NULL; - posix_memalign((void **)&sums, 4096, sizeof(TFloat) * bsize); -#pragma acc enter data create(sums[:bsize]) - } - - virtual ~UnifracUnweightedTask() - { -#ifdef _OPENACC - const unsigned int bsize = this->max_embs*(0x400/32); -#pragma acc exit data delete(sums[:bsize]) -#endif - free(sums); - } - - virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} - - void _run(unsigned int filled_embs, const TFloat * __restrict__ length); - private: - TFloat *sums; // temp buffer - }; - template - class UnifracGeneralizedTask : public UnifracTask { - public: - static const unsigned int RECOMMENDED_MAX_EMBS = UnifracTask::RECOMMENDED_MAX_EMBS_STRAIGHT; - - UnifracGeneralizedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, unsigned int _max_embs, const su::task_parameters* _task_p) - : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) {} - - virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} - - void _run(unsigned int filled_embs, const TFloat * __restrict__ length); - }; - - /* void unifrac_vaw tasks - * - * all methods utilize the same function signature. that signature is as follows: - * - * dm_stripes vector the stripes of the distance matrix being accumulated - * into for unique branch length - * dm_stripes vector the stripes of the distance matrix being accumulated - * into for total branch length (e.g., to normalize unweighted unifrac) - * embedded_proportions the proportions vector for a sample, or rather - * the counts vector normalized to 1. this vector is embedded as it is - * duplicated: if A, B and C are proportions for features A, B, and C, the - * vector will look like [A B C A B C]. - * embedded_counts the counts vector embedded in the same way and order as - * embedded_proportions. the values of this array are unnormalized feature - * counts for the subtree. - * sample_total_counts the total unnormalized feature counts for all samples - * embedded in the same way and order as embedded_proportions. - * length the branch length of the current node to its parent. - * task_p task specific parameters. - */ - template - class UnifracVawTask : public UnifracTaskBase { - protected: -#ifdef _OPENACC - // The parallel nature of GPUs needs a largish step - #ifndef SMALLGPU - // default to larger step, which makes a big difference for bigger GPUs like V100 - static const unsigned int step_size = 32; - // keep the vector size just big enough to keep the used emb array inside the 32k buffer - static const unsigned int acc_vector_size = 32*32*8/sizeof(TFloat); - #else - // smaller GPUs prefer a slightly smaller step - static const unsigned int step_size = 16; - // keep the vector size just big enough to keep the used emb array inside the 32k buffer - static const unsigned int acc_vector_size = 32*32*8/sizeof(TFloat); - #endif -#else - // The serial nature of CPU cores prefers a small step - static const unsigned int step_size = 4; -#endif - - public: - TFloat * const embedded_counts; - const TFloat * const sample_total_counts; - - static const unsigned int RECOMMENDED_MAX_EMBS = 128; - - UnifracVawTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, - const TFloat * _sample_total_counts, - unsigned int _max_embs, const su::task_parameters* _task_p) - : UnifracTaskBase(_dm_stripes, _dm_stripes_total, _max_embs, _task_p) - , embedded_counts(UnifracTaskBase::initialize_embedded(this->dm_stripes.n_samples_r,_max_embs)), sample_total_counts(_sample_total_counts) {} - - - /* delete - UnifracVawTask(UnifracTaskBase &baseObj, - const TEmb * _embedded_proportions, const TFloat * _sample_total_counts, unsigned int _max_embs) - : UnifracTaskBase(baseObj) - , embedded_proportions(_embedded_proportions), embedded_counts(initialize_embedded()), sample_total_counts(_sample_total_counts), max_embs(_max_embs) {} - */ - - - virtual ~UnifracVawTask() {} - - void sync_embedded_counts(unsigned int filled_embs) - { -#ifdef _OPENACC - const uint64_t n_samples_r = this->dm_stripes.n_samples_r; - const uint64_t bsize = n_samples_r * filled_embs; -#pragma acc update device(embedded_counts[:bsize]) -#endif - } - - void sync_embedded(unsigned int filled_embs) { this->sync_embedded_proportions(filled_embs); this->sync_embedded_counts(filled_embs);} - - void embed_range(const TFloat* __restrict__ in_proportions, const TFloat* __restrict__ in_counts, unsigned int start, unsigned int end, unsigned int emb) { - this->embed_proportions_range(in_proportions,start,end,emb); - this->embed_proportions_range_straight(this->embedded_counts,in_counts,start,end,emb); - } - void embed(const TFloat* __restrict__ in_proportions, const double* __restrict__ in_counts, unsigned int emb) { embed_range(in_proportions,in_counts,0,this->dm_stripes.n_samples,emb);} - - virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) = 0; - }; - - template - class UnifracVawUnnormalizedWeightedTask : public UnifracVawTask { - public: - UnifracVawUnnormalizedWeightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, - const TFloat * _sample_total_counts, - unsigned int _max_embs, const su::task_parameters* _task_p) - : UnifracVawTask(_dm_stripes,_dm_stripes_total,_sample_total_counts,_max_embs,_task_p) {} - - virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} - - void _run(unsigned int filled_embs, const TFloat * __restrict__ length); - }; - template - class UnifracVawNormalizedWeightedTask : public UnifracVawTask { - public: - UnifracVawNormalizedWeightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, - const TFloat * _sample_total_counts, - unsigned int _max_embs, const su::task_parameters* _task_p) - : UnifracVawTask(_dm_stripes,_dm_stripes_total,_sample_total_counts,_max_embs,_task_p) {} - - virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} - - void _run(unsigned int filled_embs, const TFloat * __restrict__ length); - }; - template - class UnifracVawUnweightedTask : public UnifracVawTask { - public: - UnifracVawUnweightedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, - const TFloat * _sample_total_counts, - unsigned int _max_embs, const su::task_parameters* _task_p) - : UnifracVawTask(_dm_stripes,_dm_stripes_total,_sample_total_counts,_max_embs,_task_p) {} - - virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} - - void _run(unsigned int filled_embs, const TFloat * __restrict__ length); - }; - template - class UnifracVawGeneralizedTask : public UnifracVawTask { - public: - UnifracVawGeneralizedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, - const TFloat * _sample_total_counts, - unsigned int _max_embs, const su::task_parameters* _task_p) - : UnifracVawTask(_dm_stripes,_dm_stripes_total,_sample_total_counts,_max_embs,_task_p) {} - - virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} - - void _run(unsigned int filled_embs, const TFloat * __restrict__ length); - }; - -} - -#endif diff --git a/unifrac/_api.pxd b/unifrac/_api.pxd index 74102f2d..f4032b94 100644 --- a/unifrac/_api.pxd +++ b/unifrac/_api.pxd @@ -1,7 +1,7 @@ #distutils: language = c++ from libcpp cimport bool -cdef extern from "../sucpp/api.hpp": +cdef extern from "api.hpp": struct mat: double* condensed_form unsigned int n_samples