Test and deploy binary pip wheels:

- Add scripts to be run inside a cibuildwheel docker image on both linux and macos which installs static versions of OpenBLAS (linux only), fftw, and suitesparse. Link our internal _libtoast extension to these static packages, which ensures consistency regardless of other shared libraries loaded by other packages. - Add a github workflow which tests pip wheel building daily, and preserves the wheels as artifacts which can be downloaded and tested as needed. This also runs the unit tests on the installed wheel packages (linux and macos, python 3.6-3.8). - Add a deployment workflow executed on git tags which uploads to pypi. this will likely need some testing during 2.3.7 tag preparation. - Cleanup package definitions in setup.py, version generation, and add a MANIFEST.in file to ensure that extra package files are bundled.
hpc4cmb · May 28, 2020 · 0ce7d3b · 0ce7d3b
1 parent 3471da9
commit 0ce7d3b
Show file tree

Hide file tree

Showing 17 changed files with 597 additions and 46 deletions.
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -0,0 +1,110 @@
+# Deploy tagged releases.
+
+name:  Deploy Release
+
+on:
+  push:
+    tags:
+      - '*'
+
+env:
+  TWINE_USERNAME: __token__
+  TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
+
+jobs:
+  py36:
+    name: Python 3.6 wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    env:
+      CIBW_BUILD: cp36-macosx_x86_64 cp36-manylinux_x86_64
+      CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+      CIBW_MANYLINUX_I686_IMAGE: manylinux2014
+      CIBW_BUILD_VERBOSITY: 3
+      CIBW_ENVIRONMENT_LINUX: "TOAST_STATIC_DEPS=1 TOAST_BUILD_BLAS_LIBRARIES='-lopenblas -fopenmp -lm -lgfortran' TOAST_BUILD_LAPACK_LIBRARIES='-lopenblas -fopenmp -lm -lgfortran'"
+      CIBW_ENVIRONMENT_MACOS: "TOAST_STATIC_DEPS=1"
+      CIBW_BEFORE_BUILD_LINUX: ./wheels/install_deps_linux.sh
+      CIBW_BEFORE_BUILD_MACOS: ./wheels/install_deps_osx.sh
+      CIBW_BEFORE_TEST: echo "" >/dev/null
+      CIBW_TEST_COMMAND: export OMP_NUM_THREADS=2; python -c 'import toast.tests; toast.tests.run()'
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+      name: Install Python
+      with:
+        python-version: '3.7'
+    - name: Install cibuildwheel
+      run: |
+        python -m pip install twine cibuildwheel==1.4.2
+    - name: Build wheel
+      run: |
+        python -m cibuildwheel --output-dir wheelhouse
+    - name: Upload to PyPI
+      run: |
+        python -m twine upload wheelhouse/toast*.whl
+  py37:
+    name: Python 3.7 wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    env:
+      CIBW_BUILD: cp37-macosx_x86_64 cp37-manylinux_x86_64
+      CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+      CIBW_MANYLINUX_I686_IMAGE: manylinux2014
+      CIBW_BUILD_VERBOSITY: 3
+      CIBW_ENVIRONMENT_LINUX: "TOAST_STATIC_DEPS=1 TOAST_BUILD_BLAS_LIBRARIES='-lopenblas -fopenmp -lm -lgfortran' TOAST_BUILD_LAPACK_LIBRARIES='-lopenblas -fopenmp -lm -lgfortran'"
+      CIBW_ENVIRONMENT_MACOS: "TOAST_STATIC_DEPS=1"
+      CIBW_BEFORE_BUILD_LINUX: ./wheels/install_deps_linux.sh
+      CIBW_BEFORE_BUILD_MACOS: ./wheels/install_deps_osx.sh
+      CIBW_BEFORE_TEST: echo "" >/dev/null
+      CIBW_TEST_COMMAND: export OMP_NUM_THREADS=2; python -c 'import toast.tests; toast.tests.run()'
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+      name: Install Python
+      with:
+        python-version: '3.7'
+    - name: Install cibuildwheel
+      run: |
+        python -m pip install twine cibuildwheel==1.4.2
+    - name: Build wheel
+      run: |
+        python -m cibuildwheel --output-dir wheelhouse
+    - name: Upload to PyPI
+      run: |
+        python -m twine upload wheelhouse/toast*.whl
+  py38:
+    name: Python 3.8 wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    env:
+      CIBW_BUILD: cp38-macosx_x86_64 cp38-manylinux_x86_64
+      CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+      CIBW_MANYLINUX_I686_IMAGE: manylinux2014
+      CIBW_BUILD_VERBOSITY: 3
+      CIBW_ENVIRONMENT_LINUX: "TOAST_STATIC_DEPS=1 TOAST_BUILD_BLAS_LIBRARIES='-lopenblas -fopenmp -lm -lgfortran' TOAST_BUILD_LAPACK_LIBRARIES='-lopenblas -fopenmp -lm -lgfortran'"
+      CIBW_ENVIRONMENT_MACOS: "TOAST_STATIC_DEPS=1"
+      CIBW_BEFORE_BUILD_LINUX: ./wheels/install_deps_linux.sh
+      CIBW_BEFORE_BUILD_MACOS: ./wheels/install_deps_osx.sh
+      CIBW_BEFORE_TEST: echo "" >/dev/null
+      CIBW_TEST_COMMAND: export OMP_NUM_THREADS=2; python -c 'import toast.tests; toast.tests.run()'
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+      name: Install Python
+      with:
+        python-version: '3.7'
+    - name: Install cibuildwheel
+      run: |
+        python -m pip install twine cibuildwheel==1.4.2
+    - name: Build wheel
+      run: |
+        python -m cibuildwheel --output-dir wheelhouse
+    - name: Upload to PyPI
+      run: |
+        python -m twine upload wheelhouse/toast*.whl
diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
@@ -0,0 +1,115 @@
+
+name:  Test Binary Wheels
+
+# Uncomment here for testing and comment out schedule below.
+# on:
+#   push:
+#     branches: [ master ]
+#   pull_request:
+#     branches: [ master ]
+
+# Uncomment here for production and comment out push / PR above.
+on:
+  schedule:
+    - cron: '0 4 * * *'
+
+jobs:
+  py36:
+    name: Python 3.6 wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    env:
+      CIBW_BUILD: cp36-macosx_x86_64 cp36-manylinux_x86_64
+      CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+      CIBW_MANYLINUX_I686_IMAGE: manylinux2014
+      CIBW_BUILD_VERBOSITY: 3
+      CIBW_ENVIRONMENT_LINUX: "TOAST_STATIC_DEPS=1 TOAST_BUILD_BLAS_LIBRARIES='-lopenblas -fopenmp -lm -lgfortran' TOAST_BUILD_LAPACK_LIBRARIES='-lopenblas -fopenmp -lm -lgfortran'"
+      CIBW_ENVIRONMENT_MACOS: "TOAST_STATIC_DEPS=1"
+      CIBW_BEFORE_BUILD_LINUX: ./wheels/install_deps_linux.sh
+      CIBW_BEFORE_BUILD_MACOS: ./wheels/install_deps_osx.sh
+      CIBW_BEFORE_TEST: echo "" >/dev/null
+      CIBW_TEST_COMMAND: export OMP_NUM_THREADS=2; python -c 'import toast.tests; toast.tests.run()'
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+      name: Install Python
+      with:
+        python-version: '3.7'
+    - name: Install cibuildwheel
+      run: |
+        python -m pip install cibuildwheel==1.4.2
+    - name: Build wheel
+      run: |
+        python -m cibuildwheel --output-dir wheelhouse
+    - uses: actions/upload-artifact@v2
+      with:
+        name: wheels
+        path: ./wheelhouse
+  py37:
+    name: Python 3.7 wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    env:
+      CIBW_BUILD: cp37-macosx_x86_64 cp37-manylinux_x86_64
+      CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+      CIBW_MANYLINUX_I686_IMAGE: manylinux2014
+      CIBW_BUILD_VERBOSITY: 3
+      CIBW_ENVIRONMENT_LINUX: "TOAST_STATIC_DEPS=1 TOAST_BUILD_BLAS_LIBRARIES='-lopenblas -fopenmp -lm -lgfortran' TOAST_BUILD_LAPACK_LIBRARIES='-lopenblas -fopenmp -lm -lgfortran'"
+      CIBW_ENVIRONMENT_MACOS: "TOAST_STATIC_DEPS=1"
+      CIBW_BEFORE_BUILD_LINUX: ./wheels/install_deps_linux.sh
+      CIBW_BEFORE_BUILD_MACOS: ./wheels/install_deps_osx.sh
+      CIBW_BEFORE_TEST: echo "" >/dev/null
+      CIBW_TEST_COMMAND: export OMP_NUM_THREADS=2; python -c 'import toast.tests; toast.tests.run()'
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+      name: Install Python
+      with:
+        python-version: '3.7'
+    - name: Install cibuildwheel
+      run: |
+        python -m pip install cibuildwheel==1.4.2
+    - name: Build wheel
+      run: |
+        python -m cibuildwheel --output-dir wheelhouse
+    - uses: actions/upload-artifact@v2
+      with:
+        name: wheels
+        path: ./wheelhouse
+  py38:
+    name: Python 3.8 wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    env:
+      CIBW_BUILD: cp38-macosx_x86_64 cp38-manylinux_x86_64
+      CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+      CIBW_MANYLINUX_I686_IMAGE: manylinux2014
+      CIBW_BUILD_VERBOSITY: 3
+      CIBW_ENVIRONMENT_LINUX: "TOAST_STATIC_DEPS=1 TOAST_BUILD_BLAS_LIBRARIES='-lopenblas -fopenmp -lm -lgfortran' TOAST_BUILD_LAPACK_LIBRARIES='-lopenblas -fopenmp -lm -lgfortran'"
+      CIBW_ENVIRONMENT_MACOS: "TOAST_STATIC_DEPS=1"
+      CIBW_BEFORE_BUILD_LINUX: ./wheels/install_deps_linux.sh
+      CIBW_BEFORE_BUILD_MACOS: ./wheels/install_deps_osx.sh
+      CIBW_BEFORE_TEST: echo "" >/dev/null
+      CIBW_TEST_COMMAND: export OMP_NUM_THREADS=2; python -c 'import toast.tests; toast.tests.run()'
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+      name: Install Python
+      with:
+        python-version: '3.7'
+    - name: Install cibuildwheel
+      run: |
+        python -m pip install cibuildwheel==1.4.2
+    - name: Build wheel
+      run: |
+        python -m cibuildwheel --output-dir wheelhouse
+    - uses: actions/upload-artifact@v2
+      with:
+        name: wheels
+        path: ./wheelhouse
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -24,13 +24,6 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 # Auxiliary files
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake")
 
-# Copy the RELEASE file into the build directory
-configure_file(
-    "${CMAKE_CURRENT_SOURCE_DIR}/RELEASE"
-    "${CMAKE_CURRENT_BINARY_DIR}/RELEASE"
-    COPYONLY
-)
-
 # Use GNUInstallDirs to install libraries into correct locations on all
 # platforms.
 include(GNUInstallDirs)
@@ -69,8 +62,10 @@ if(BLAS_FOUND)
     if(LAPACK_FOUND)
         find_package(LAPACKnames)
     else(LAPACK_FOUND)
-        message(SEND_ERROR "Could not find a working LAPACK installation")
+        message(FATAL_ERROR "Could not find a working LAPACK installation")
     endif(LAPACK_FOUND)
+else(BLAS_FOUND)
+    message(FATAL_ERROR "Could not find a working BLAS installation")
 endif(BLAS_FOUND)
 
 find_package(FFTW)

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -0,0 +1,10 @@
+include README.md
+include AUTHORS
+include LICENSE
+include src
+include CMakeLists.txt
+include tutorial
+include example
+include platforms
+include cmake
+include docs
diff --git a/RELEASE b/RELEASE
diff --git a/cmake/FindAATM.cmake b/cmake/FindAATM.cmake
@@ -26,19 +26,18 @@ if(PKG_CONFIG_FOUND AND NOT AATM_ROOT)
 endif()
 
 # Check whether to search static or dynamic libs
-set(CMAKE_FIND_LIBRARY_SUFFIXES_SAV ${CMAKE_FIND_LIBRARY_SUFFIXES})
+
+set(AATM_LIB_NAME "aatm")
 
 if(${AATM_USE_STATIC_LIBS})
-  set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_STATIC_LIBRARY_SUFFIX})
-else()
-  set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_SAV})
+  set(AATM_LIB_NAME "aatm_static")
 endif()
 
 if(AATM_ROOT)
     # find libs
     find_library(
         AATM_LIB
-        NAMES "aatm"
+        NAMES ${AATM_LIB_NAME}
         PATHS ${AATM_ROOT}
         PATH_SUFFIXES "lib" "lib64"
         NO_DEFAULT_PATH
@@ -54,7 +53,7 @@ if(AATM_ROOT)
 else(AATM_ROOT)
     find_library(
         AATM_LIB
-        NAMES "aatm"
+        NAMES ${AATM_LIB_NAME}
         PATHS "${PKG_AATM_PREFIX}"
         PATH_SUFFIXES "lib" "lib64"
     )
@@ -70,9 +69,6 @@ if(AATM_LIB)
     set(AATM_LIBRARIES ${AATM_LIB})
 endif(AATM_LIB)
 
-# Restore variable
-set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_SAV})
-
 include(FindPackageHandleStandardArgs)
 
 find_package_handle_standard_args(AATM