Remove our dependency on MPI compilers. This includes:

 - Require only mpi4py (all MPI operations done in python).  This
   allows us to build our compiled extensions with any compiler
   rather than requiring consistency with the compilers used to
   build mpi4py.

 - Port the high-level C++ atmosphere simulation code to python
   and use compiled code only on a single process.  Change the
   on-disk format of cached atmosphere realizations to use hdf5.

 - Update build system to not look for MPI compilers.

 - Add a new log level, "VERBOSE" which is even lower than "DEBUG".
   The intention for this is to contain messages that might come
   from every single process.

 - The MPIShared and MPILock classes are found in the upstream
   pshmem package.  Bundle a tagged release of that package for
   now until it is widely available on PyPI / conda-forge.

 - Clean up the generation of version.cpp so that it only happens
   if the version actually changes.

.atom/config.cson

@@ -9,7 +9,7 @@
     python: {}
   editor:
     autoIndentOnPaste: false
-    fontSize: 16
+    fontSize: 18
     preferredLineLength: 88
     scrollPastEnd: true
     showInvisibles: true

.github/workflows/test.yml

@@ -9,22 +9,6 @@ on:
     branches: [ master ]
 
 jobs:
-  py35:
-    name: Python 3.5
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Pull Dependency Image
-        run: docker pull hpc4cmb/toast-deps-py35:latest
-      - name: Compile
-        run: docker run -v "$(pwd)":/home/toast --name="test_py35" hpc4cmb/toast-deps-py35:latest /home/toast/platforms/install_test_runner.sh && docker commit -m "test runner" test_py35 test_runner:py35
-      - name: Test Documentation Build
-        run: docker run -v "$(pwd)":/home/toast test_runner:py35 /home/toast/docs/build_docs.sh
-      - name: Run Serial Tests
-        run: docker run test_runner:py35 python -c 'import toast.tests; toast.tests.run()'
-      - name: Run MPI Tests
-        run: docker run test_runner:py35 mpirun -np 2 python -c 'import toast.tests; toast.tests.run()'
   py36:
     name: Python 3.6
     runs-on: ubuntu-latest

CMakeLists.txt

@@ -74,7 +74,8 @@ find_package(AATM)
 
 find_package(SuiteSparse)
 
-find_package(MPI)
+# Re-enable if we ever add MPI compiled extensions.
+# find_package(MPI)
 
 find_package(PythonInterp REQUIRED)
 

platforms/cori-gcc.sh

@@ -5,8 +5,6 @@ opts="$@"
 cmake \
     -DCMAKE_C_COMPILER="${CRAYPE_DIR}/bin/cc" \
     -DCMAKE_CXX_COMPILER="${CRAYPE_DIR}/bin/CC" \
-    -DMPI_C_COMPILER="${CRAYPE_DIR}/bin/cc" \
-    -DMPI_CXX_COMPILER="${CRAYPE_DIR}/bin/CC" \
     -DCMAKE_C_FLAGS="-O3 -g -fPIC -pthread" \
     -DCMAKE_CXX_FLAGS="-O3 -g -fPIC -pthread -std=c++11" \
     -DPYTHON_EXECUTABLE:FILEPATH=$(which python3) \

platforms/cori-intel.sh

@@ -5,8 +5,6 @@ opts="$@"
 cmake \
     -DCMAKE_C_COMPILER="${CRAYPE_DIR}/bin/cc" \
     -DCMAKE_CXX_COMPILER="${CRAYPE_DIR}/bin/CC" \
-    -DMPI_C_COMPILER="${CRAYPE_DIR}/bin/cc" \
-    -DMPI_CXX_COMPILER="${CRAYPE_DIR}/bin/CC" \
     -DCMAKE_C_FLAGS="-O3 -g -fPIC -xcore-avx2 -axmic-avx512 -pthread" \
     -DCMAKE_CXX_FLAGS="-O3 -g -fPIC -xcore-avx2 -axmic-avx512 -pthread -std=c++11" \
     -DPYTHON_EXECUTABLE:FILEPATH=$(which python3) \
@@ -18,5 +16,3 @@ cmake \
     -DSUITESPARSE_LIBRARY_DIR_HINTS="${CMBENV_AUX_ROOT}/lib" \
     ${opts} \
     ..
-
-

platforms/install_test_runner.sh

@@ -6,18 +6,16 @@
 set -e
 
 # Get the absolute path to the source tree
-pushd $(dirname $(dirname $0)) > /dev/null
+pushd $(dirname $(dirname $0)) >/dev/null 2>&1
 toastdir=$(pwd -P)
-popd > /dev/null
+popd >/dev/null 2>&1
 
 mkdir build
-pushd build
+pushd build >/dev/null 2>&1
 
 cmake \
     -DCMAKE_C_COMPILER="gcc" \
     -DCMAKE_CXX_COMPILER="g++" \
-    -DMPI_C_COMPILER="mpicc" \
-    -DMPI_CXX_COMPILER="mpicxx" \
     -DCMAKE_C_FLAGS="-O3 -g -fPIC -pthread" \
     -DCMAKE_CXX_FLAGS="-O3 -g -fPIC -pthread -std=c++11" \
     -DPYTHON_EXECUTABLE:FILEPATH=$(which python3) \
@@ -27,4 +25,4 @@ cmake \
 
 make -j 2 install
 
-popd > /dev/null
+popd >/dev/null 2>&1

platforms/linux-gcc.sh

@@ -8,8 +8,6 @@ opts="$@"
 cmake \
     -DCMAKE_C_COMPILER="gcc" \
     -DCMAKE_CXX_COMPILER="g++" \
-    -DMPI_C_COMPILER="mpicc" \
-    -DMPI_CXX_COMPILER="mpicxx" \
     -DCMAKE_C_FLAGS="-O3 -g -fPIC -pthread" \
     -DCMAKE_CXX_FLAGS="-O3 -g -fPIC -pthread -std=c++11" \
     -DPYTHON_EXECUTABLE:FILEPATH=$(which python3) \

platforms/linux-intel.sh

@@ -8,8 +8,6 @@ opts="$@"
 cmake \
     -DCMAKE_C_COMPILER="icc" \
     -DCMAKE_CXX_COMPILER="icpc" \
-    -DMPI_C_COMPILER="mpiicc" \
-    -DMPI_CXX_COMPILER="mpiicpc" \
     -DCMAKE_C_FLAGS="-O3 -g -fPIC -pthread" \
     -DCMAKE_CXX_FLAGS="-O3 -g -fPIC -pthread -std=c++11" \
     -DBLAS_LIBRARIES=${MKLROOT}/lib/intel64/libmkl_rt.so \

platforms/osx_homebrew.sh

@@ -13,8 +13,6 @@ opts="$@"
 cmake \
     -DCMAKE_C_COMPILER="clang" \
     -DCMAKE_CXX_COMPILER="clang++" \
-    -DMPI_C_COMPILER="mpicc" \
-    -DMPI_CXX_COMPILER="mpicxx" \
     -DCMAKE_C_FLAGS="-O3 -g -fPIC" \
     -DCMAKE_CXX_FLAGS="-O3 -g -fPIC -std=c++11" \
     -DPYTHON_EXECUTABLE:FILEPATH=$(which python3) \

setup.py

@@ -241,6 +241,7 @@ def readme():
 conf["provides"] = "toast"
 conf["python_requires"] = ">=3.6.0"
 conf["install_requires"] = ["cmake", "numpy", "scipy", "healpy", "matplotlib", "ephem"]
+conf["extras_require"] = {"mpi": ["mpi4py>=3.0"]}
 conf["packages"] = find_packages("src")
 conf["package_dir"] = {"": "src"}
 conf["ext_modules"] = ext_modules

src/libtoast/CMakeLists.txt

@@ -6,21 +6,20 @@ add_subdirectory(gtest EXCLUDE_FROM_ALL)
 set(gtest_force_shared_crt TRUE)
 include_directories("${gtest_SOURCE_DIR}/include")
 
-# Add a custom command that produces version.cpp, plus
-# a dummy output that's not actually produced, in order
-# to force version.cmake to always be re-run before the build
-add_custom_command(
-    OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/_version.cpp
-        ${CMAKE_CURRENT_BINARY_DIR}/src/version.cpp
-    COMMAND ${CMAKE_COMMAND} -P
-        ${CMAKE_CURRENT_SOURCE_DIR}/version.cmake
+# Add a custom command that produces version.cpp
+
+set(versioncpp ${CMAKE_CURRENT_BINARY_DIR}/version.cpp)
+
+add_custom_command(OUTPUT ${versioncpp}
+    COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_SOURCE_DIR}/version.cmake
+    COMMENT "Updating version.cpp if needed ..."
+    WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
 )
 
 # Define the sources
 
 set(toast_SOURCES
     toast.cpp
-    src/version.cpp
     src/toast_sys_environment.cpp
     src/toast_sys_utils.cpp
     src/toast_math_lapack.cpp
@@ -33,9 +32,16 @@ set(toast_SOURCES
     src/toast_fod_psd.cpp
     src/toast_tod_filter.cpp
     src/toast_tod_pointing.cpp
-    src/toast_tod_simnoise.cpp
+    src/toast_tod_simnoise.cpp#
+# add_custom_target(genversion COMMAND
+#     ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_SOURCE_DIR}/version.cmake
+#     COMMENT "Updating version.cpp if needed ..."
+#     WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+# )
     src/toast_atm_utils.cpp
     src/toast_atm.cpp
+    src/toast_atm_sim.cpp
+    src/toast_atm_observe.cpp
     tests/toast_test_runner.cpp
     tests/toast_test_env.cpp
     tests/toast_test_utils.cpp
@@ -48,7 +54,7 @@ set(toast_SOURCES
     tests/toast_test_polyfilter.cpp
 )
 
-add_library(toast ${toast_SOURCES})
+add_library(toast ${versioncpp} ${toast_SOURCES})
 
 target_include_directories(toast BEFORE PRIVATE
     "${CMAKE_CURRENT_SOURCE_DIR}"
@@ -68,6 +74,10 @@ if(MPI_FOUND)
     target_compile_definitions(toast PRIVATE HAVE_MPI=1)
 endif(MPI_FOUND)
 
+if(MPI4PY_FOUND)
+    target_compile_definitions(toast PRIVATE HAVE_MPI4PY=1)
+endif(MPI4PY_FOUND)
+
 # Dependencies
 
 target_link_libraries(toast gtest)