Wheels: ADIOS2 -> v2.9.0, blosc2

.github/workflows/build.yml

@@ -40,10 +40,10 @@ jobs:
               CMAKE_GENERATOR: "Visual Studio 16 2019"
               CMAKE_GENERATOR_PLATFORM: "Win32"
 
-          - os: macos-10.15
+          - os: macos-11
             arch: "x86_64"
             env:
-              MACOSX_DEPLOYMENT_TARGET: 10.15
+              MACOSX_DEPLOYMENT_TARGET: 11.0
 
           # Apple Silicon M1/arm64/aarch64 builds:
           #   https://cibuildwheel.readthedocs.io/en/stable/faq/#apple-silicon
@@ -86,29 +86,19 @@ jobs:
         python -m pip install --upgrade pip setuptools wheel
         python -m pip install cibuildwheel==2.12.1
 
-#    # 0.15.0: CMake: Fix Python Install Directory
-#    # https://github.com/openPMD/openPMD-api/pull/1393
-#    - name: Download Patch 1/2
-#      uses: suisei-cn/actions-download-file@v1
-#      id: patch1
-#      with:
-#        url: "https://github.com/ax3l/openPMD-api/commit/b622cc5ea770f866c1e373185a9e389c04bdb54c.patch"
-#        target: src/.patch/
-#
-#    # 0.15.0.post2 bump
-#    - name: Download Patch 2/2
-#      uses: suisei-cn/actions-download-file@v1
-#      id: setupversion
-#      with:
-#        url: "https://gist.githubusercontent.com/ax3l/5e83edefe4b05cf6aa2a971649285fe0/raw/023cbd69e06715e5191ece741bb1de56560a9f96/0001-Bump-setup.py.patch"
-#        target: src/.patch/
-
-#    - name: Apply Patches
-#      run: |
-#        python -m pip install "patch==1.*"
-#        cd src
-#        python -m patch .patch/b622cc5ea770f866c1e373185a9e389c04bdb54c.patch
-#        python -m patch .patch/0001-Bump-setup.py.patch
+    # Hack: cmake --trace-expand
+    #- name: Download Patch 2/2
+    #  uses: suisei-cn/actions-download-file@v1
+    #  id: setupversion
+    #  with:
+    #    url: "https://gist.githubusercontent.com/ax3l/9b15c5bc73d7f3c1c66784187a016474/raw/e31dd54498c9c8fc465a7cbb6f900f768b9d0c0b/0001-Hack-setup.py-CMake-trace-expand.patch"
+    #    target: src/.patch/
+
+    #- name: Apply Patches
+    #  run: |
+    #    python -m pip install "patch==1.*"
+    #    cd src
+    #    python -m patch .patch/0001-Hack-setup.py-CMake-trace-expand.patch
 
     - name: Build wheel
       env:
@@ -124,10 +114,10 @@ jobs:
         # for the openPMD-api build, CMake shall search for
         # static dependencies of HDF5 and ADIOS1 (see setup.py)
         CIBW_ENVIRONMENT: HDF5_USE_STATIC_LIBRARIES='ON' ZLIB_USE_STATIC_LIBS='ON' ADIOS_USE_STATIC_LIBS='ON' openPMD_CMAKE_openPMD_USE_HDF5='ON' openPMD_CMAKE_openPMD_USE_ADIOS2='ON'
-        CIBW_ENVIRONMENT_WINDOWS: HDF5_USE_STATIC_LIBRARIES='ON' ZLIB_USE_STATIC_LIBS='ON' openPMD_CMAKE_openPMD_USE_HDF5='ON' openPMD_CMAKE_openPMD_USE_ADIOS2='ON' CMAKE_PREFIX_PATH='C:/Program Files (x86)/ADIOS2;C:/Program Files (x86)/blosc;C:/Program Files (x86)/HDF5;C:/Program Files (x86)/ZFP;C:/Program Files (x86)/zlib'
+        CIBW_ENVIRONMENT_WINDOWS: HDF5_USE_STATIC_LIBRARIES='ON' ZLIB_USE_STATIC_LIBS='ON' openPMD_CMAKE_openPMD_USE_HDF5='ON' openPMD_CMAKE_openPMD_USE_ADIOS2='ON' CMAKE_PREFIX_PATH='C:/Program Files (x86)/ADIOS2;C:/Program Files (x86)/blosc;C:/Program Files (x86)/blosc2;C:/Program Files (x86)/HDF5;C:/Program Files (x86)/ZFP;C:/Program Files (x86)/zlib'
         # C++17 support in macOS 10.13+ (partial) and 10.14+ (std::visit) and 10.15+ (std::filesystem::path)
         #   https://cibuildwheel.readthedocs.io/en/stable/cpp_standards/#macos-and-deployment-target-versions
-        # arm64 Python interpreters are built with 11.0
+        # arm64/aarch64 (M1/M2) requires 11.0+
         MACOSX_DEPLOYMENT_TARGET: "${{ matrix.env.MACOSX_DEPLOYMENT_TARGET }}"
         CMAKE_OSX_ARCHITECTURES: "${{ matrix.env.CMAKE_OSX_ARCHITECTURES }}"
         # Windows cross-compilation

.travis.yml

@@ -155,7 +155,7 @@ install:
 
 script:
   - cd src
-  - python3 -m cibuildwheel --output-dir ../wheelhouse
+  - travis_wait 45 python3 -m cibuildwheel --output-dir ../wheelhouse
   - cd ..
 
 deploy:

library_builders.bat

@@ -16,25 +16,26 @@ exit /b 0
 
 :build_adios2
   if exist adios2-stamp exit /b 0
-  curl -sLo adios2-2.7.1.zip ^
-    https://github.com/ornladios/ADIOS2/archive/v2.7.1.zip
-  powershell Expand-Archive adios2-2.7.1.zip -DestinationPath dep-adios2
+  curl -sLo adios2-2.9.0.zip ^
+    https://github.com/ornladios/ADIOS2/archive/v2.9.0.zip
+  powershell Expand-Archive adios2-2.9.0.zip -DestinationPath dep-adios2
 
-  :: Patch Blosc Compression w/ ADIOS 2.7.1
-  curl -sLo adios2-blosc.patch ^
-    https://patch-diff.githubusercontent.com/raw/ornladios/ADIOS2/pull/2746.patch
-  python -m patch -p 1 -d dep-adios2/ADIOS2-2.7.1 adios2-blosc.patch
+  :: https://github.com/ornladios/ADIOS2/issues/3680#issuecomment-1615308336
+  curl -sLo adios2-cblosc-stdmin.patch ^
+    https://patch-diff.githubusercontent.com/raw/ornladios/ADIOS2/pull/3681.patch
+  python -m patch -p 1 -d dep-adios2/ADIOS2-2.9.0 adios2-cblosc-stdmin.patch
 
-  cmake -S dep-adios2/ADIOS2-2.7.1 -B build-adios2 ^
+  cmake -S dep-adios2/ADIOS2-2.9.0 -B build-adios2 ^
     -DCMAKE_BUILD_TYPE=Release  ^
     -DBUILD_SHARED_LIBS=OFF     ^
     -DBUILD_TESTING=OFF         ^
     -DADIOS2_USE_MPI=OFF        ^
     -DADIOS2_BUILD_EXAMPLES=OFF ^
-    -DADIOS2_USE_Blosc=ON       ^
+    -DADIOS2_USE_Blosc2=ON      ^
     -DADIOS2_USE_BZip2=OFF      ^
     -DADIOS2_USE_Fortran=OFF    ^
     -DADIOS2_USE_HDF5=OFF       ^
+    -DADIOS2_USE_MHS=OFF        ^
     -DADIOS2_USE_PNG=OFF        ^
     -DADIOS2_USE_Profiling=OFF  ^
     -DADIOS2_USE_Python=OFF     ^
@@ -88,14 +89,62 @@ exit /b 0
   if errorlevel 1 exit 1
 exit /b 0
 
+:build_blosc2
+  if exist blosc2-stamp exit /b 0
+
+  curl -sLo blosc2-2.9.3.zip ^
+    https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.9.3.zip
+  powershell Expand-Archive blosc2-2.9.3.zip -DestinationPath dep-blosc2
+
+  :: https://github.com/Blosc/c-blosc2/pull/525
+  curl -sLo c-blosc2-cmake.patch ^
+    https://patch-diff.githubusercontent.com/raw/Blosc/c-blosc2/pull/525.patch
+  python -m patch -p 1 -d dep-blosc2/c-blosc2-2.9.3 c-blosc2-cmake.patch
+
+  :: https://github.com/Blosc/c-blosc2/issues/526
+  curl -sLo c-blosc2-cxx20.patch ^
+    https://github.com/Blosc/c-blosc2/pull/527.patch
+  python -m patch -p 1 -d dep-blosc2/c-blosc2-2.9.3 c-blosc2-cxx20.patch
+
+  :: https://github.com/Blosc/c-blosc2/pull/529
+  curl -sLo c-blosc2-external-zlib.patch ^
+    https://github.com/Blosc/c-blosc2/pull/529.patch
+  python -m patch -p 1 -d dep-blosc2/c-blosc2-2.9.3 c-blosc2-external-zlib.patch
+
+  cmake -S dep-blosc2/c-blosc2-2.9.3 -B build-blosc2 ^
+    -DCMAKE_BUILD_TYPE=Release  ^
+    -DBUILD_SHARED=OFF          ^
+    -DBUILD_STATIC=ON           ^
+    -DBUILD_BENCHMARKS=OFF      ^
+    -DBUILD_EXAMPLES=OFF        ^
+    -DBUILD_FUZZERS=OFF         ^
+    -DBUILD_PLUGINS=OFF         ^
+    -DBUILD_TESTS=OFF
+::    -DPREFER_EXTERNAL_ZLIB=ON
+::    -DZLIB_USE_STATIC_LIBS=ON
+  if errorlevel 1 exit 1
+
+  cmake --build build-blosc2 --parallel %CPU_COUNT%
+  if errorlevel 1 exit 1
+
+  cmake --build build-blosc2 --target install --config Release
+  if errorlevel 1 exit 1
+
+  rmdir /s /q build-blosc2
+  if errorlevel 1 exit 1
+
+  break > blosc2-stamp
+  if errorlevel 1 exit 1
+exit /b 0
+
 :build_hdf5
   if exist hdf5-stamp exit /b 0
 
-  curl -sLo hdf5-1.12.2.zip ^
-    https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.12/hdf5-1.12.2/src/hdf5-1.12.2.zip
-  powershell Expand-Archive hdf5-1.12.2.zip -DestinationPath dep-hdf5
+  curl -sLo hdf5-1.14.1-2.zip ^
+    https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.14/hdf5-1.14.1/src/hdf5-1.14.1-2.zip
+  powershell Expand-Archive hdf5-1.14.1-2.zip -DestinationPath dep-hdf5
 
-  cmake -S dep-hdf5/hdf5-1.12.2 -B build-hdf5 ^
+  cmake -S dep-hdf5/hdf5-1.14.1-2 -B build-hdf5 ^
     -DCMAKE_BUILD_TYPE=Release  ^
     -DCMAKE_VERBOSE_MAKEFILE=ON ^
     -DBUILD_SHARED_LIBS=OFF     ^
@@ -193,7 +242,8 @@ call :install_buildessentials
 call :build_zlib
 :: build_bzip2
 :: build_szip
-call :build_blosc
 call :build_zfp
+call :build_blosc
+call :build_blosc2
 call :build_hdf5
 call :build_adios2

library_builders.sh

@@ -4,7 +4,14 @@
 set -eu -o pipefail
 
 BUILD_PREFIX="${BUILD_PREFIX:-/usr/local}"
-CPU_COUNT="${CPU_COUNT:-2}"
+
+# https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
+if [ "$(uname -s)" = "Darwin" ]
+then
+    CPU_COUNT="${CPU_COUNT:-3}"
+else
+    CPU_COUNT="${CPU_COUNT:-2}"
+fi
 
 function install_buildessentials {
     if [ -e buildessentials-stamp ]; then return; fi
@@ -96,57 +103,39 @@ function build_adios1 {
 function build_adios2 {
     if [ -e adios2-stamp ]; then return; fi
 
-    curl -sLo adios2-2.7.1.tar.gz \
-        https://github.com/ornladios/ADIOS2/archive/v2.7.1.tar.gz
+    #curl -sLo adios2-2.9.0.tar.gz \
+    #    https://github.com/ornladios/ADIOS2/archive/v2.9.0.tar.gz
+    curl -sLo adios2-fix-blosc2-findpackage.tar.gz \
+        https://github.com/ax3l/ADIOS2/archive/refs/heads/fix-blosc2-findpackage.tar.gz
     file adios2*.tar.gz
     tar -xzf adios2*.tar.gz
     rm adios2*.tar.gz
 
-    # Patch PThread Propagation
-    curl -sLo adios-pthread.patch \
-        https://patch-diff.githubusercontent.com/raw/ornladios/ADIOS2/pull/2768.patch
-    python3 -m patch -p 1 -d ADIOS2-2.7.1 adios-pthread.patch
-
-    # DILL macOS arm64 or universal2 binary
-    #   https://github.com/ornladios/ADIOS2/issues/3116
-    #   needs rebase (or use ADIOS2-2.8.0)
-    #curl -sLo dill-universal.patch \
-    #    https://patch-diff.githubusercontent.com/raw/ornladios/ADIOS2/pull/3118.patch
-    #python3 -m patch -p 1 -d ADIOS2-2.7.1 dill-universal.patch
-    ADIOS2_USE_SST=ON
-    if [[ "${CMAKE_OSX_ARCHITECTURES-}" == "arm64" ]]; then
-        ADIOS2_USE_SST=OFF
-    fi
-
+    # build
     mkdir build-adios2
     cd build-adios2
     PY_BIN=$(which python3)
     CMAKE_BIN="$(${PY_BIN} -m pip show cmake 2>/dev/null | grep Location | cut -d' ' -f2)/cmake/data/bin/"
-    if [ "$(uname -s)" = "Linux" ]
-    then
-        EVPATH_ZPL="ON"
-    else
-        # ZPL in EVPATH disabled because it does not build with older macOS
-        #       https://github.com/GTkorvo/evpath/issues/47
-        EVPATH_ZPL="OFF"
-    fi
     PATH=${CMAKE_BIN}:${PATH} cmake               \
         -DBUILD_SHARED_LIBS=OFF                   \
         -DBUILD_TESTING=OFF                       \
         -DCMAKE_POSITION_INDEPENDENT_CODE=ON      \
         -DADIOS2_BUILD_EXAMPLES=OFF               \
         -DADIOS2_USE_BZip2=OFF                    \
+        -DADIOS2_USE_Blosc2=ON                    \
         -DADIOS2_USE_Fortran=OFF                  \
+        -DADIOS2_USE_HDF5=OFF                     \
+        -DADIOS2_USE_MHS=OFF                      \
         -DADIOS2_USE_MPI=OFF                      \
         -DADIOS2_USE_PNG=OFF                      \
-        -DADIOS2_USE_SST=${ADIOS2_USE_SST}        \
+        -DADIOS2_USE_SST=ON                       \
         -DADIOS2_USE_ZFP=ON                       \
         -DADIOS2_RUN_INSTALL_TEST=OFF             \
-        -DEVPATH_USE_ZPL_ENET=${EVPATH_ZPL}       \
         -DHDF5_USE_STATIC_LIBRARIES:BOOL=ON       \
         -DCMAKE_VERBOSE_MAKEFILE=ON               \
         -DCMAKE_DISABLE_FIND_PACKAGE_LibFFI=TRUE  \
         -DCMAKE_DISABLE_FIND_PACKAGE_BISON=TRUE   \
+        -DADIOS2_INSTALL_GENERATE_CONFIG=OFF      \
         -DCMAKE_INSTALL_PREFIX=${BUILD_PREFIX} ../ADIOS2-*
 
     make -j${CPU_COUNT}
@@ -203,6 +192,43 @@ function build_blosc {
     touch blosc-stamp
 }
 
+function build_blosc2 {
+    if [ -e blosc-stamp2 ]; then return; fi
+
+    #curl -sLo blosc2-v2.10.0.tar.gz \
+    curl -sLo blosc2-topic-cmake-install-targets.tar.gz \
+        https://github.com/Blosc/c-blosc2/archive/d510951c492db9f09ab9171675c3dddbf8ffd4fa.tar.gz
+    file blosc2*.tar.gz
+    tar -xzf blosc2*.tar.gz
+    rm blosc2*.tar.gz
+
+    mkdir build-blosc2
+    cd build-blosc2
+    PY_BIN=$(which python3)
+    CMAKE_BIN="$(${PY_BIN} -m pip show cmake 2>/dev/null | grep Location | cut -d' ' -f2)/cmake/data/bin/"
+    PATH=${CMAKE_BIN}:${PATH} cmake          \
+      -DCMAKE_POSITION_INDEPENDENT_CODE=ON   \
+      -DBUILD_STATIC=ON                      \
+      -DBUILD_SHARED=OFF                     \
+      -DBUILD_BENCHMARKS=OFF                 \
+      -DBUILD_EXAMPLES=OFF                   \
+      -DBUILD_FUZZERS=OFF                    \
+      -DBUILD_PLUGINS=OFF                    \
+      -DBUILD_TESTS=OFF                      \
+      -DCMAKE_VERBOSE_MAKEFILE=ON            \
+      -DCMAKE_INSTALL_PREFIX=${BUILD_PREFIX} \
+      -DPREFER_EXTERNAL_ZLIB=ON              \
+      -DZLIB_USE_STATIC_LIBS=ON              \
+      ../c-blosc2-*
+    make -j${CPU_COUNT}
+    make install
+    cd -
+
+    rm -rf build-blosc2
+
+    touch blosc-stamp2
+}
+
 function build_zfp {
     if [ -e zfp-stamp ]; then return; fi
 
@@ -271,6 +297,7 @@ function build_hdf5 {
 
     # macOS cross-compile
     HOST_ARG=""
+    #   heavily based on conda-forge hdf5-feedstock and h5py's cibuildwheel instructions
     #   https://github.com/conda-forge/hdf5-feedstock/blob/cbbd57d58f7f5350ca679eaad49354c11dd32b95/recipe/build.sh#L53-L80
     if [[ "${CMAKE_OSX_ARCHITECTURES-}" == "arm64" ]]; then
         # https://github.com/h5py/h5py/blob/fcaca1d1b81d25c0d83b11d5bdf497469b5980e9/ci/configure_hdf5_mac.sh
@@ -306,13 +333,18 @@ function build_hdf5 {
         --prefix=${BUILD_PREFIX}
 
     if [[ "${CMAKE_OSX_ARCHITECTURES-}" == "arm64" ]]; then
+        (
         # https://github.com/h5py/h5py/blob/fcaca1d1b81d25c0d83b11d5bdf497469b5980e9/ci/configure_hdf5_mac.sh - build_h5detect
         mkdir -p native-build/bin
         pushd native-build/bin
-        CFLAGS= $CC ../../src/H5detect.c -I ../../src/ -o H5detect
-        CFLAGS= $CC ../../src/H5make_libsettings.c -I ../../src/ -o H5make_libsettings
-        popd
 
+        # MACOSX_DEPLOYMENT_TARGET is for the target_platform and not for build_platform
+        unset MACOSX_DEPLOYMENT_TARGET
+
+        CFLAGS="" $CC ../../src/H5detect.c -I ../../src/ -o H5detect
+        CFLAGS="" $CC ../../src/H5make_libsettings.c -I ../../src/ -o H5make_libsettings
+        popd
+        )
         export PATH="$(pwd)/native-build/bin:$PATH"
     fi
 
@@ -339,11 +371,16 @@ fi
 
 install_buildessentials
 build_zlib
-build_blosc
 build_zfp
+if [[ "$(uname -m)" != "ppc64le" ]]; then
+    # builds too long for Travis-CI
+    build_blosc
+fi
+build_blosc2
 build_hdf5
-# skip ADIOS1 build for M1
-if [[ "${CMAKE_OSX_ARCHITECTURES-}" != "arm64" ]]; then
+if [[ "${CMAKE_OSX_ARCHITECTURES-}" != "arm64" && "$(uname -m)" != "ppc64le" ]]; then
+    # macOS: skip ADIOS1 build for M1
+    # Linux: with ADIOS2 also enabled, this builds too long for Travis-CI
     build_adios1
 fi
 build_adios2