Remove dependency on MPI compilers (#350)

* Remove our dependency on MPI compilers.  This includes:

 - Require only mpi4py (all MPI operations done in python).  This
   allows us to build our compiled extensions with any compiler
   rather than requiring consistency with the compilers used to
   build mpi4py.

 - Port the high-level C++ atmosphere simulation code to python
   and use compiled code only on a single process.  Change the
   on-disk format of cached atmosphere realizations to use hdf5.

 - Update build system to not look for MPI compilers.

 - Add a new log level, "VERBOSE" which is even lower than "DEBUG".
   The intention for this is to contain messages that might come
   from every single process.

 - The MPIShared and MPILock classes are found in the upstream
   pshmem package.  Bundle a tagged release of that package for
   now until it is widely available on PyPI / conda-forge.

 - Clean up the generation of version.cpp so that it only happens
   if the version actually changes.

* Change logic to use only a single "use_cache" option.

* When caching atmosphere sims, write to a temporary file and move it into place at the end.

* Comment out pshmem test import, which imports MPI

* Support both new and old package structure within numba.

* Only attempt to set the numba backend if the utils.set_numba_threading()
function is actually called.  TOAST does not use numba since it conflicts
with our existing explicit parallelism.  The set_numba_threading() function
is designed to be used before calling external packages that use numba,
in order to ensure consistency between toast and numba.

* Add global timers for the individual pieces of atm_sim_compute_slice

* Actually use verbose level messages.

* Fix typo in debug printing of the kolmogorov spectrum.  Also fix a bug that the kolmo_x spectrum was not being reduced across processes.

* No need to reduce kolmo_x since it is computed on every process.

* Fix a missing cast from float to int

* Fix typo bug introduced during code reorganization.

* Use explicit bitwidth types (like int64_t) rather than long.

.atom/config.cson

@@ -9,7 +9,7 @@
     python: {}
   editor:
     autoIndentOnPaste: false
-    fontSize: 16
+    fontSize: 18
     preferredLineLength: 88
     scrollPastEnd: true
     showInvisibles: true

.github/workflows/test.yml

@@ -9,22 +9,6 @@ on:
     branches: [ master ]
 
 jobs:
-  py35:
-    name: Python 3.5
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Pull Dependency Image
-        run: docker pull hpc4cmb/toast-deps-py35:latest
-      - name: Compile
-        run: docker run -v "$(pwd)":/home/toast --name="test_py35" hpc4cmb/toast-deps-py35:latest /home/toast/platforms/install_test_runner.sh && docker commit -m "test runner" test_py35 test_runner:py35
-      - name: Test Documentation Build
-        run: docker run -v "$(pwd)":/home/toast test_runner:py35 /home/toast/docs/build_docs.sh
-      - name: Run Serial Tests
-        run: docker run test_runner:py35 python -c 'import toast.tests; toast.tests.run()'
-      - name: Run MPI Tests
-        run: docker run test_runner:py35 mpirun -np 2 python -c 'import toast.tests; toast.tests.run()'
   py36:
     name: Python 3.6
     runs-on: ubuntu-latest

CMakeLists.txt

@@ -74,7 +74,8 @@ find_package(AATM)
 
 find_package(SuiteSparse)
 
-find_package(MPI)
+# Re-enable if we ever add MPI compiled extensions.
+# find_package(MPI)
 
 find_package(PythonInterp REQUIRED)
 

platforms/cori-gcc.sh

@@ -5,8 +5,6 @@ opts="$@"
 cmake \
     -DCMAKE_C_COMPILER="${CRAYPE_DIR}/bin/cc" \
     -DCMAKE_CXX_COMPILER="${CRAYPE_DIR}/bin/CC" \
-    -DMPI_C_COMPILER="${CRAYPE_DIR}/bin/cc" \
-    -DMPI_CXX_COMPILER="${CRAYPE_DIR}/bin/CC" \
     -DCMAKE_C_FLAGS="-O3 -g -fPIC -pthread" \
     -DCMAKE_CXX_FLAGS="-O3 -g -fPIC -pthread -std=c++11" \
     -DPYTHON_EXECUTABLE:FILEPATH=$(which python3) \

platforms/cori-intel.sh

@@ -5,8 +5,6 @@ opts="$@"
 cmake \
     -DCMAKE_C_COMPILER="${CRAYPE_DIR}/bin/cc" \
     -DCMAKE_CXX_COMPILER="${CRAYPE_DIR}/bin/CC" \
-    -DMPI_C_COMPILER="${CRAYPE_DIR}/bin/cc" \
-    -DMPI_CXX_COMPILER="${CRAYPE_DIR}/bin/CC" \
     -DCMAKE_C_FLAGS="-O3 -g -fPIC -xcore-avx2 -axmic-avx512 -pthread" \
     -DCMAKE_CXX_FLAGS="-O3 -g -fPIC -xcore-avx2 -axmic-avx512 -pthread -std=c++11" \
     -DPYTHON_EXECUTABLE:FILEPATH=$(which python3) \
@@ -18,5 +16,3 @@ cmake \
     -DSUITESPARSE_LIBRARY_DIR_HINTS="${CMBENV_AUX_ROOT}/lib" \
     ${opts} \
     ..
-
-

platforms/install_test_runner.sh

@@ -6,18 +6,16 @@
 set -e
 
 # Get the absolute path to the source tree
-pushd $(dirname $(dirname $0)) > /dev/null
+pushd $(dirname $(dirname $0)) >/dev/null 2>&1
 toastdir=$(pwd -P)
-popd > /dev/null
+popd >/dev/null 2>&1
 
 mkdir build
-pushd build
+pushd build >/dev/null 2>&1
 
 cmake \
     -DCMAKE_C_COMPILER="gcc" \
     -DCMAKE_CXX_COMPILER="g++" \
-    -DMPI_C_COMPILER="mpicc" \
-    -DMPI_CXX_COMPILER="mpicxx" \
     -DCMAKE_C_FLAGS="-O3 -g -fPIC -pthread" \
     -DCMAKE_CXX_FLAGS="-O3 -g -fPIC -pthread -std=c++11" \
     -DPYTHON_EXECUTABLE:FILEPATH=$(which python3) \
@@ -27,4 +25,4 @@ cmake \
 
 make -j 2 install
 
-popd > /dev/null
+popd >/dev/null 2>&1

platforms/linux-gcc.sh

@@ -8,8 +8,6 @@ opts="$@"
 cmake \
     -DCMAKE_C_COMPILER="gcc" \
     -DCMAKE_CXX_COMPILER="g++" \
-    -DMPI_C_COMPILER="mpicc" \
-    -DMPI_CXX_COMPILER="mpicxx" \
     -DCMAKE_C_FLAGS="-O3 -g -fPIC -pthread" \
     -DCMAKE_CXX_FLAGS="-O3 -g -fPIC -pthread -std=c++11" \
     -DPYTHON_EXECUTABLE:FILEPATH=$(which python3) \

platforms/linux-intel.sh

@@ -8,8 +8,6 @@ opts="$@"
 cmake \
     -DCMAKE_C_COMPILER="icc" \
     -DCMAKE_CXX_COMPILER="icpc" \
-    -DMPI_C_COMPILER="mpiicc" \
-    -DMPI_CXX_COMPILER="mpiicpc" \
     -DCMAKE_C_FLAGS="-O3 -g -fPIC -pthread" \
     -DCMAKE_CXX_FLAGS="-O3 -g -fPIC -pthread -std=c++11" \
     -DBLAS_LIBRARIES=${MKLROOT}/lib/intel64/libmkl_rt.so \

platforms/osx_homebrew.sh

@@ -13,8 +13,6 @@ opts="$@"
 cmake \
     -DCMAKE_C_COMPILER="clang" \
     -DCMAKE_CXX_COMPILER="clang++" \
-    -DMPI_C_COMPILER="mpicc" \
-    -DMPI_CXX_COMPILER="mpicxx" \
     -DCMAKE_C_FLAGS="-O3 -g -fPIC" \
     -DCMAKE_CXX_FLAGS="-O3 -g -fPIC -std=c++11" \
     -DPYTHON_EXECUTABLE:FILEPATH=$(which python3) \

setup.py

@@ -241,6 +241,7 @@ def readme():
 conf["provides"] = "toast"
 conf["python_requires"] = ">=3.6.0"
 conf["install_requires"] = ["cmake", "numpy", "scipy", "healpy", "matplotlib", "ephem"]
+conf["extras_require"] = {"mpi": ["mpi4py>=3.0"]}
 conf["packages"] = find_packages("src")
 conf["package_dir"] = {"": "src"}
 conf["ext_modules"] = ext_modules

src/libtoast/CMakeLists.txt

@@ -6,21 +6,20 @@ add_subdirectory(gtest EXCLUDE_FROM_ALL)
 set(gtest_force_shared_crt TRUE)
 include_directories("${gtest_SOURCE_DIR}/include")
 
-# Add a custom command that produces version.cpp, plus
-# a dummy output that's not actually produced, in order
-# to force version.cmake to always be re-run before the build
-add_custom_command(
-    OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/_version.cpp
-        ${CMAKE_CURRENT_BINARY_DIR}/src/version.cpp
-    COMMAND ${CMAKE_COMMAND} -P
-        ${CMAKE_CURRENT_SOURCE_DIR}/version.cmake
+# Add a custom command that produces version.cpp
+
+set(versioncpp ${CMAKE_CURRENT_BINARY_DIR}/version.cpp)
+
+add_custom_command(OUTPUT ${versioncpp}
+    COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_SOURCE_DIR}/version.cmake
+    COMMENT "Updating version.cpp if needed ..."
+    WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
 )
 
 # Define the sources
 
 set(toast_SOURCES
     toast.cpp
-    src/version.cpp
     src/toast_sys_environment.cpp
     src/toast_sys_utils.cpp
     src/toast_math_lapack.cpp
@@ -36,6 +35,8 @@ set(toast_SOURCES
     src/toast_tod_simnoise.cpp
     src/toast_atm_utils.cpp
     src/toast_atm.cpp
+    src/toast_atm_sim.cpp
+    src/toast_atm_observe.cpp
     tests/toast_test_runner.cpp
     tests/toast_test_env.cpp
     tests/toast_test_utils.cpp
@@ -48,7 +49,7 @@ set(toast_SOURCES
     tests/toast_test_polyfilter.cpp
 )
 
-add_library(toast ${toast_SOURCES})
+add_library(toast ${versioncpp} ${toast_SOURCES})
 
 target_include_directories(toast BEFORE PRIVATE
     "${CMAKE_CURRENT_SOURCE_DIR}"
@@ -68,6 +69,10 @@ if(MPI_FOUND)
     target_compile_definitions(toast PRIVATE HAVE_MPI=1)
 endif(MPI_FOUND)
 
+if(MPI4PY_FOUND)
+    target_compile_definitions(toast PRIVATE HAVE_MPI4PY=1)
+endif(MPI4PY_FOUND)
+
 # Dependencies
 
 target_link_libraries(toast gtest)