From 7c6ac1f7718ecfac8a898ce003281806a81ce0fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 8 Mar 2023 14:28:45 +0100 Subject: [PATCH 01/54] Update blosc -> blosc2, ADIOS2 -> v2.9.0-rc1 ADIOS2 should be replace by v2.9.0 finally --- library_builders.sh | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/library_builders.sh b/library_builders.sh index 6f7dba2ed1..d7067f0b75 100755 --- a/library_builders.sh +++ b/library_builders.sh @@ -96,8 +96,8 @@ function build_adios1 { function build_adios2 { if [ -e adios2-stamp ]; then return; fi - curl -sLo adios2-2.7.1.tar.gz \ - https://github.com/ornladios/ADIOS2/archive/v2.7.1.tar.gz + curl -sLo adios2-2.9.0-rc1.tar.gz \ + https://github.com/ornladios/ADIOS2/archive/v2.9.0-rc1.tar.gz file adios2*.tar.gz tar -xzf adios2*.tar.gz rm adios2*.tar.gz @@ -105,7 +105,7 @@ function build_adios2 { # Patch PThread Propagation curl -sLo adios-pthread.patch \ https://patch-diff.githubusercontent.com/raw/ornladios/ADIOS2/pull/2768.patch - python3 -m patch -p 1 -d ADIOS2-2.7.1 adios-pthread.patch + python3 -m patch -p 1 -d ADIOS2-2.9.0-rc1 adios-pthread.patch # DILL macOS arm64 or universal2 binary # https://github.com/ornladios/ADIOS2/issues/3116 @@ -136,6 +136,7 @@ function build_adios2 { -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ -DADIOS2_BUILD_EXAMPLES=OFF \ -DADIOS2_USE_BZip2=OFF \ + -DADIOS2_USE_Blosc=ON \ -DADIOS2_USE_Fortran=OFF \ -DADIOS2_USE_MPI=OFF \ -DADIOS2_USE_PNG=OFF \ @@ -161,23 +162,23 @@ function build_adios2 { function build_blosc { if [ -e blosc-stamp ]; then return; fi - curl -sLo c-blosc-1.21.0.tar.gz \ - https://github.com/Blosc/c-blosc/archive/v1.21.0.tar.gz - file c-blosc*.tar.gz - tar -xzf c-blosc*.tar.gz - rm c-blosc*.tar.gz + curl -sLo c-blosc2-v2.7.1.tar.gz \ + https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.7.1.tar.gz + file c-blosc2*.tar.gz + tar -xzf c-blosc2*.tar.gz + rm c-blosc2*.tar.gz # Patch PThread Propagation - curl -sLo blosc-pthread.patch \ - https://patch-diff.githubusercontent.com/raw/Blosc/c-blosc/pull/318.patch - python3 -m patch -p 1 -d c-blosc-1.21.0 blosc-pthread.patch + # curl -sLo blosc-pthread.patch \ + # https://patch-diff.githubusercontent.com/raw/Blosc/c-blosc/pull/318.patch + # python3 -m patch -p 1 -d c-blosc-1.21.0 blosc-pthread.patch # SSE2 support # https://github.com/Blosc/c-blosc/issues/334 - DEACTIVATE_SSE2=OFF + WITH_SSE2=ON if [[ "${CMAKE_OSX_ARCHITECTURES-}" == *"arm64"* ]]; then # error: SSE2 is not supported by the target architecture/platform and/or this compiler. - DEACTIVATE_SSE2=ON + WITH_SSE2=OFF fi mkdir build-blosc @@ -186,14 +187,14 @@ function build_blosc { CMAKE_BIN="$(${PY_BIN} -m pip show cmake 2>/dev/null | grep Location | cut -d' ' -f2)/cmake/data/bin/" PATH=${CMAKE_BIN}:${PATH} cmake \ -DDEACTIVATE_SNAPPY=ON \ - -DDEACTIVATE_SSE2=${DEACTIVATE_SSE2} \ + -DWITH_SSE2=${WITH_SSE2} \ -DBUILD_SHARED=OFF \ -DBUILD_TESTS=OFF \ -DBUILD_BENCHMARKS=OFF \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DCMAKE_INSTALL_PREFIX=${BUILD_PREFIX} \ -DZLIB_USE_STATIC_LIBS=ON \ - ../c-blosc-* + ../c-blosc2-* make -j${CPU_COUNT} make install cd - From f0b3f5285535ad9624c3b81936065a7b3403213b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Wed, 8 Mar 2023 20:41:16 +0100 Subject: [PATCH 02/54] Fix Blosc2 CMake params --- library_builders.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) mode change 100755 => 100644 library_builders.sh diff --git a/library_builders.sh b/library_builders.sh old mode 100755 new mode 100644 index d7067f0b75..c608d5f0fa --- a/library_builders.sh +++ b/library_builders.sh @@ -185,11 +185,11 @@ function build_blosc { cd build-blosc PY_BIN=$(which python3) CMAKE_BIN="$(${PY_BIN} -m pip show cmake 2>/dev/null | grep Location | cut -d' ' -f2)/cmake/data/bin/" + # Blosc2 runs into a linking error without testing enabled??? PATH=${CMAKE_BIN}:${PATH} cmake \ - -DDEACTIVATE_SNAPPY=ON \ - -DWITH_SSE2=${WITH_SSE2} \ + -DWITH_SSE2=${WITH_SSE2} \ -DBUILD_SHARED=OFF \ - -DBUILD_TESTS=OFF \ + -DBUILD_TESTS=ON \ -DBUILD_BENCHMARKS=OFF \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DCMAKE_INSTALL_PREFIX=${BUILD_PREFIX} \ From b6ea33499bdf857b22c9077934a7f7a74f4686c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 9 Mar 2023 16:29:40 +0100 Subject: [PATCH 03/54] Guard against CMAKE_OSX_ARCHITECTURES --- library_builders.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/library_builders.sh b/library_builders.sh index c608d5f0fa..e531fc319f 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -184,6 +184,12 @@ function build_blosc { mkdir build-blosc cd build-blosc PY_BIN=$(which python3) + if [ -n "${CMAKE_OSX_ARCHITECTURES+x}" -a -z "${CMAKE_OSX_ARCHITECTURES}" ]; then + # This if branch gets active when the env variable is defined, but empty + # Our builders somehow define CMAKE_OSC_ARCHITECTURES="" and the CMake + # script of blosc2 runs into trouble then + unset CMAKE_OSX_ARCHITECTURES + fi CMAKE_BIN="$(${PY_BIN} -m pip show cmake 2>/dev/null | grep Location | cut -d' ' -f2)/cmake/data/bin/" # Blosc2 runs into a linking error without testing enabled??? PATH=${CMAKE_BIN}:${PATH} cmake \ From c601cb3634e50a29971412f571a2b11489f242ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 9 Mar 2023 16:47:47 +0100 Subject: [PATCH 04/54] For now, deactivate the pthread patch in ADIOS2 --- library_builders.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/library_builders.sh b/library_builders.sh index e531fc319f..ec17c69003 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -103,9 +103,9 @@ function build_adios2 { rm adios2*.tar.gz # Patch PThread Propagation - curl -sLo adios-pthread.patch \ - https://patch-diff.githubusercontent.com/raw/ornladios/ADIOS2/pull/2768.patch - python3 -m patch -p 1 -d ADIOS2-2.9.0-rc1 adios-pthread.patch + # curl -sLo adios-pthread.patch \ + # https://patch-diff.githubusercontent.com/raw/ornladios/ADIOS2/pull/2768.patch + # python3 -m patch -p 1 -d ADIOS2-2.9.0-rc1 adios-pthread.patch # DILL macOS arm64 or universal2 binary # https://github.com/ornladios/ADIOS2/issues/3116 From 43024bf66bbd803caab9386a829b664c7deb46fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 10 Mar 2023 11:46:31 +0100 Subject: [PATCH 05/54] Test small patch for blosc2 CMake --- library_builders.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/library_builders.sh b/library_builders.sh index ec17c69003..28f9ecd17b 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -181,15 +181,12 @@ function build_blosc { WITH_SSE2=OFF fi + # @todo: proper patch for this + sed -Ei 's|if\(\$ENV\{CMAKE_OSX_ARCHITECTURES\} STREQUAL "arm64"\)|if("$ENV{CMAKE_OSX_ARCHITECTURES}" STREQUAL "arm64")|' CMakeLists.txt + mkdir build-blosc cd build-blosc PY_BIN=$(which python3) - if [ -n "${CMAKE_OSX_ARCHITECTURES+x}" -a -z "${CMAKE_OSX_ARCHITECTURES}" ]; then - # This if branch gets active when the env variable is defined, but empty - # Our builders somehow define CMAKE_OSC_ARCHITECTURES="" and the CMake - # script of blosc2 runs into trouble then - unset CMAKE_OSX_ARCHITECTURES - fi CMAKE_BIN="$(${PY_BIN} -m pip show cmake 2>/dev/null | grep Location | cut -d' ' -f2)/cmake/data/bin/" # Blosc2 runs into a linking error without testing enabled??? PATH=${CMAKE_BIN}:${PATH} cmake \ From 6c983745c4ee154e60b3b598a6f6ea63ccd9213f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Fri, 10 Mar 2023 14:39:23 +0100 Subject: [PATCH 06/54] Add -fPIC flag --- library_builders.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/library_builders.sh b/library_builders.sh index 28f9ecd17b..15647fae87 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -191,6 +191,7 @@ function build_blosc { # Blosc2 runs into a linking error without testing enabled??? PATH=${CMAKE_BIN}:${PATH} cmake \ -DWITH_SSE2=${WITH_SSE2} \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ -DBUILD_SHARED=OFF \ -DBUILD_TESTS=ON \ -DBUILD_BENCHMARKS=OFF \ From 64b3efba9b1a190a473b79acfadb4522a57e431b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 13 Mar 2023 10:48:37 +0100 Subject: [PATCH 07/54] MacOS compatibility --- library_builders.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/library_builders.sh b/library_builders.sh index 15647fae87..dca5956f26 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -182,7 +182,8 @@ function build_blosc { fi # @todo: proper patch for this - sed -Ei 's|if\(\$ENV\{CMAKE_OSX_ARCHITECTURES\} STREQUAL "arm64"\)|if("$ENV{CMAKE_OSX_ARCHITECTURES}" STREQUAL "arm64")|' CMakeLists.txt + sed -E -i.bak 's|if\(\$ENV\{CMAKE_OSX_ARCHITECTURES\} STREQUAL "arm64"\)|if("$ENV{CMAKE_OSX_ARCHITECTURES}" STREQUAL "arm64")|' c-blosc2-*/CMakeLists.txt + cat c-blosc2-*/CMakeLists.txt mkdir build-blosc cd build-blosc From 24ddef27bc7e7542beb663d2089e97066a14a55a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 13 Mar 2023 13:22:12 +0100 Subject: [PATCH 08/54] -DADIOS2_USE_Blosc2=ON --- library_builders.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library_builders.sh b/library_builders.sh index dca5956f26..9d871679fc 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -136,7 +136,7 @@ function build_adios2 { -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ -DADIOS2_BUILD_EXAMPLES=OFF \ -DADIOS2_USE_BZip2=OFF \ - -DADIOS2_USE_Blosc=ON \ + -DADIOS2_USE_Blosc2=ON \ -DADIOS2_USE_Fortran=OFF \ -DADIOS2_USE_MPI=OFF \ -DADIOS2_USE_PNG=OFF \ From c41110167fb0cabf8c45215afe77c30bf01329da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 16 Mar 2023 10:53:23 +0100 Subject: [PATCH 09/54] Build atl and ffs manually to avoid CMake import error --- library_builders.sh | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/library_builders.sh b/library_builders.sh index 9d871679fc..b5a01c051b 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -102,6 +102,41 @@ function build_adios2 { tar -xzf adios2*.tar.gz rm adios2*.tar.gz + CMAKE_BIN="$(${PY_BIN} -m pip show cmake 2>/dev/null | grep Location | cut -d' ' -f2)/cmake/data/bin/" + + mkdir build-atl + cd build-atl + PATH=${CMAKE_BIN}:${PATH} cmake \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_TESTING=OFF \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DCMAKE_INSTALL_PREFIX=${BUILD_PREFIX} ../ADIOS2-*/thirdparty/atl/atl + make -j${CPU_COUNT} + make install + cd - + + mkdir build-dill + cd build-dill + PATH=${CMAKE_BIN}:${PATH} cmake \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_TESTING=OFF \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DCMAKE_INSTALL_PREFIX=${BUILD_PREFIX} ../ADIOS2-*/thirdparty/dill/dill + make -j${CPU_COUNT} + make install + cd - + + mkdir build-ffs + cd build-ffs + PATH=${CMAKE_BIN}:${PATH} cmake \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_TESTING=OFF \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DCMAKE_INSTALL_PREFIX=${BUILD_PREFIX} ../ADIOS2-*/thirdparty/ffs/ffs + make -j${CPU_COUNT} + make install + cd - + # Patch PThread Propagation # curl -sLo adios-pthread.patch \ # https://patch-diff.githubusercontent.com/raw/ornladios/ADIOS2/pull/2768.patch @@ -121,7 +156,6 @@ function build_adios2 { mkdir build-adios2 cd build-adios2 PY_BIN=$(which python3) - CMAKE_BIN="$(${PY_BIN} -m pip show cmake 2>/dev/null | grep Location | cut -d' ' -f2)/cmake/data/bin/" if [ "$(uname -s)" = "Linux" ] then EVPATH_ZPL="ON" @@ -135,6 +169,8 @@ function build_adios2 { -DBUILD_TESTING=OFF \ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ -DADIOS2_BUILD_EXAMPLES=OFF \ + -DADIOS2_USE_EXTERNAL_ATL=ON \ + -DADIOS2_USE_EXTERNAL_FFS=ON \ -DADIOS2_USE_BZip2=OFF \ -DADIOS2_USE_Blosc2=ON \ -DADIOS2_USE_Fortran=OFF \ From 275330b7501fbd585a5b4b71344e30f5d7232cb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 16 Mar 2023 16:23:55 +0100 Subject: [PATCH 10/54] Revert "Build atl and ffs manually to avoid CMake import error" This reverts commit c5469a3fa79715743ff7e177d111a6ec8e1d389a. --- library_builders.sh | 38 +------------------------------------- 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/library_builders.sh b/library_builders.sh index b5a01c051b..9d871679fc 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -102,41 +102,6 @@ function build_adios2 { tar -xzf adios2*.tar.gz rm adios2*.tar.gz - CMAKE_BIN="$(${PY_BIN} -m pip show cmake 2>/dev/null | grep Location | cut -d' ' -f2)/cmake/data/bin/" - - mkdir build-atl - cd build-atl - PATH=${CMAKE_BIN}:${PATH} cmake \ - -DBUILD_SHARED_LIBS=OFF \ - -DBUILD_TESTING=OFF \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -DCMAKE_INSTALL_PREFIX=${BUILD_PREFIX} ../ADIOS2-*/thirdparty/atl/atl - make -j${CPU_COUNT} - make install - cd - - - mkdir build-dill - cd build-dill - PATH=${CMAKE_BIN}:${PATH} cmake \ - -DBUILD_SHARED_LIBS=OFF \ - -DBUILD_TESTING=OFF \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -DCMAKE_INSTALL_PREFIX=${BUILD_PREFIX} ../ADIOS2-*/thirdparty/dill/dill - make -j${CPU_COUNT} - make install - cd - - - mkdir build-ffs - cd build-ffs - PATH=${CMAKE_BIN}:${PATH} cmake \ - -DBUILD_SHARED_LIBS=OFF \ - -DBUILD_TESTING=OFF \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -DCMAKE_INSTALL_PREFIX=${BUILD_PREFIX} ../ADIOS2-*/thirdparty/ffs/ffs - make -j${CPU_COUNT} - make install - cd - - # Patch PThread Propagation # curl -sLo adios-pthread.patch \ # https://patch-diff.githubusercontent.com/raw/ornladios/ADIOS2/pull/2768.patch @@ -156,6 +121,7 @@ function build_adios2 { mkdir build-adios2 cd build-adios2 PY_BIN=$(which python3) + CMAKE_BIN="$(${PY_BIN} -m pip show cmake 2>/dev/null | grep Location | cut -d' ' -f2)/cmake/data/bin/" if [ "$(uname -s)" = "Linux" ] then EVPATH_ZPL="ON" @@ -169,8 +135,6 @@ function build_adios2 { -DBUILD_TESTING=OFF \ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ -DADIOS2_BUILD_EXAMPLES=OFF \ - -DADIOS2_USE_EXTERNAL_ATL=ON \ - -DADIOS2_USE_EXTERNAL_FFS=ON \ -DADIOS2_USE_BZip2=OFF \ -DADIOS2_USE_Blosc2=ON \ -DADIOS2_USE_Fortran=OFF \ From 90eeab9104367de98a74780b0731c59f1e094b59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 11 May 2023 18:13:00 +0200 Subject: [PATCH 11/54] RC1 -> full release --- library_builders.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/library_builders.sh b/library_builders.sh index 9d871679fc..d60e0ce579 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -96,8 +96,8 @@ function build_adios1 { function build_adios2 { if [ -e adios2-stamp ]; then return; fi - curl -sLo adios2-2.9.0-rc1.tar.gz \ - https://github.com/ornladios/ADIOS2/archive/v2.9.0-rc1.tar.gz + curl -sLo adios2-2.9.0.tar.gz \ + https://github.com/ornladios/ADIOS2/archive/v2.9.0.tar.gz file adios2*.tar.gz tar -xzf adios2*.tar.gz rm adios2*.tar.gz @@ -105,7 +105,7 @@ function build_adios2 { # Patch PThread Propagation # curl -sLo adios-pthread.patch \ # https://patch-diff.githubusercontent.com/raw/ornladios/ADIOS2/pull/2768.patch - # python3 -m patch -p 1 -d ADIOS2-2.9.0-rc1 adios-pthread.patch + # python3 -m patch -p 1 -d ADIOS2-2.9.0 adios-pthread.patch # DILL macOS arm64 or universal2 binary # https://github.com/ornladios/ADIOS2/issues/3116 From 1716fb88482a62c85549b976eca0eb77da634282 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 6 Jun 2023 16:31:20 +0200 Subject: [PATCH 12/54] Don't build ADIOS1 --- library_builders.sh | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/library_builders.sh b/library_builders.sh index d60e0ce579..0261f74e2d 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -65,34 +65,6 @@ function install_buildessentials { touch buildessentials-stamp } -function build_adios1 { - if [ -e adios1-stamp ]; then return; fi - - curl -sLo adios-1.13.1.tar.gz \ - http://users.nccs.gov/~pnorbert/adios-1.13.1.tar.gz - file adios*.tar.gz - tar -xzf adios*.tar.gz - rm adios*.tar.gz - cd adios-* - - # Cross-Compile hints for autotools based builds - HOST_ARG="" - if [[ "${CMAKE_OSX_ARCHITECTURES-}" == "arm64" ]]; then - HOST_ARG="--host=aarch64-apple-darwin" - fi - - ./configure --enable-static --disable-shared --disable-fortran --without-mpi ${HOST_ARG} --prefix=${BUILD_PREFIX} --with-blosc=/usr - make -j${CPU_COUNT} - make install - cd - - - # note: for universal binaries on macOS - # https://developer.apple.com/documentation/apple-silicon/building-a-universal-macos-binary - #lipo -create -output universal_app x86_app arm_app - - touch adios1-stamp -} - function build_adios2 { if [ -e adios2-stamp ]; then return; fi @@ -348,8 +320,4 @@ build_zlib build_blosc build_zfp build_hdf5 -# skip ADIOS1 build for M1 -if [[ "${CMAKE_OSX_ARCHITECTURES-}" != "arm64" ]]; then - build_adios1 -fi build_adios2 From 143e6aa440fde5465bf8a5ea5d86ea086ec7dbf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Tue, 6 Jun 2023 16:38:52 +0200 Subject: [PATCH 13/54] -DADIOS2_USE_MHS=OFF --- library_builders.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/library_builders.sh b/library_builders.sh index 0261f74e2d..952f9a77fa 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -110,6 +110,7 @@ function build_adios2 { -DADIOS2_USE_BZip2=OFF \ -DADIOS2_USE_Blosc2=ON \ -DADIOS2_USE_Fortran=OFF \ + -DADIOS2_USE_MHS=OFF \ -DADIOS2_USE_MPI=OFF \ -DADIOS2_USE_PNG=OFF \ -DADIOS2_USE_SST=${ADIOS2_USE_SST} \ From a82c4a4bd72c495157747082a338ec1dab4f5110 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Thu, 15 Jun 2023 11:08:36 +0200 Subject: [PATCH 14/54] Use -DADIOS2_INSTALL_GENERATE_CONFIG=OFF workaround See https://github.com/ornladios/ADIOS2/issues/3348#issuecomment-1517084004 --- library_builders.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/library_builders.sh b/library_builders.sh index 952f9a77fa..65697db955 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -121,6 +121,7 @@ function build_adios2 { -DCMAKE_VERBOSE_MAKEFILE=ON \ -DCMAKE_DISABLE_FIND_PACKAGE_LibFFI=TRUE \ -DCMAKE_DISABLE_FIND_PACKAGE_BISON=TRUE \ + -DADIOS2_INSTALL_GENERATE_CONFIG=OFF \ -DCMAKE_INSTALL_PREFIX=${BUILD_PREFIX} ../ADIOS2-* make -j${CPU_COUNT} From 7065b8914aeda3aa2ebf40efa882dca49402506b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 19 Jun 2023 14:53:26 +0200 Subject: [PATCH 15/54] Do not disable find calls --- library_builders.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/library_builders.sh b/library_builders.sh index 65697db955..000542a0e1 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -119,8 +119,6 @@ function build_adios2 { -DEVPATH_USE_ZPL_ENET=${EVPATH_ZPL} \ -DHDF5_USE_STATIC_LIBRARIES:BOOL=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DCMAKE_DISABLE_FIND_PACKAGE_LibFFI=TRUE \ - -DCMAKE_DISABLE_FIND_PACKAGE_BISON=TRUE \ -DADIOS2_INSTALL_GENERATE_CONFIG=OFF \ -DCMAKE_INSTALL_PREFIX=${BUILD_PREFIX} ../ADIOS2-* From f96d6ad4efd7e5bf7a0d0bbf5a18e4941099bee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 19 Jun 2023 16:49:56 +0200 Subject: [PATCH 16/54] Revert "Do not disable find calls" This reverts commit 7065b8914aeda3aa2ebf40efa882dca49402506b. --- library_builders.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/library_builders.sh b/library_builders.sh index 000542a0e1..65697db955 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -119,6 +119,8 @@ function build_adios2 { -DEVPATH_USE_ZPL_ENET=${EVPATH_ZPL} \ -DHDF5_USE_STATIC_LIBRARIES:BOOL=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DCMAKE_DISABLE_FIND_PACKAGE_LibFFI=TRUE \ + -DCMAKE_DISABLE_FIND_PACKAGE_BISON=TRUE \ -DADIOS2_INSTALL_GENERATE_CONFIG=OFF \ -DCMAKE_INSTALL_PREFIX=${BUILD_PREFIX} ../ADIOS2-* From 928219d47aa36bc87353f167de96bd4f16f437f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 19 Jun 2023 17:00:02 +0200 Subject: [PATCH 17/54] Try setting ADIOS2_PATH --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index deacff688e..d70e54e8aa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -123,7 +123,7 @@ jobs: CIBW_BEFORE_BUILD_WINDOWS: 'cmd /E:ON /V:ON /C .github\library_builders.bat' # for the openPMD-api build, CMake shall search for # static dependencies of HDF5 and ADIOS1 (see setup.py) - CIBW_ENVIRONMENT: HDF5_USE_STATIC_LIBRARIES='ON' ZLIB_USE_STATIC_LIBS='ON' ADIOS_USE_STATIC_LIBS='ON' openPMD_CMAKE_openPMD_USE_HDF5='ON' openPMD_CMAKE_openPMD_USE_ADIOS2='ON' + CIBW_ENVIRONMENT: HDF5_USE_STATIC_LIBRARIES='ON' ZLIB_USE_STATIC_LIBS='ON' ADIOS_USE_STATIC_LIBS='ON' openPMD_CMAKE_openPMD_USE_HDF5='ON' openPMD_CMAKE_openPMD_USE_ADIOS2='ON' ADIOS2_DIR=/usr/local CIBW_ENVIRONMENT_WINDOWS: HDF5_USE_STATIC_LIBRARIES='ON' ZLIB_USE_STATIC_LIBS='ON' openPMD_CMAKE_openPMD_USE_HDF5='ON' openPMD_CMAKE_openPMD_USE_ADIOS2='ON' CMAKE_PREFIX_PATH='C:/Program Files (x86)/ADIOS2;C:/Program Files (x86)/blosc;C:/Program Files (x86)/HDF5;C:/Program Files (x86)/ZFP;C:/Program Files (x86)/zlib' # C++17 support in macOS 10.13+ (partial) and 10.14+ (std::visit) and 10.15+ (std::filesystem::path) # https://cibuildwheel.readthedocs.io/en/stable/cpp_standards/#macos-and-deployment-target-versions @@ -136,6 +136,7 @@ jobs: # Show a bit more output (pip -v) CIBW_BUILD_VERBOSITY: 1 run: | + env cd src python -m pip install --upgrade pip setuptools wheel python -m pip install --upgrade cmake From 20ae61e97a1e496f9c79ccb2608bc90f1fd96082 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 27 Jun 2023 11:52:59 +0200 Subject: [PATCH 18/54] Keep ADIOS1 + c-blosc1 --- library_builders.sh | 93 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 79 insertions(+), 14 deletions(-) diff --git a/library_builders.sh b/library_builders.sh index 65697db955..19ad4b61dd 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -65,6 +65,34 @@ function install_buildessentials { touch buildessentials-stamp } +function build_adios1 { + if [ -e adios1-stamp ]; then return; fi + + curl -sLo adios-1.13.1.tar.gz \ + http://users.nccs.gov/~pnorbert/adios-1.13.1.tar.gz + file adios*.tar.gz + tar -xzf adios*.tar.gz + rm adios*.tar.gz + cd adios-* + + # Cross-Compile hints for autotools based builds + HOST_ARG="" + if [[ "${CMAKE_OSX_ARCHITECTURES-}" == "arm64" ]]; then + HOST_ARG="--host=aarch64-apple-darwin" + fi + + ./configure --enable-static --disable-shared --disable-fortran --without-mpi ${HOST_ARG} --prefix=${BUILD_PREFIX} --with-blosc=/usr + make -j${CPU_COUNT} + make install + cd - + + # note: for universal binaries on macOS + # https://developer.apple.com/documentation/apple-silicon/building-a-universal-macos-binary + #lipo -create -output universal_app x86_app arm_app + + touch adios1-stamp +} + function build_adios2 { if [ -e adios2-stamp ]; then return; fi @@ -136,31 +164,63 @@ function build_adios2 { function build_blosc { if [ -e blosc-stamp ]; then return; fi - curl -sLo c-blosc2-v2.7.1.tar.gz \ - https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.7.1.tar.gz - file c-blosc2*.tar.gz - tar -xzf c-blosc2*.tar.gz - rm c-blosc2*.tar.gz + curl -sLo c-blosc-1.21.0.tar.gz \ + https://github.com/Blosc/c-blosc/archive/v1.21.0.tar.gz + file c-blosc*.tar.gz + tar -xzf c-blosc*.tar.gz + rm c-blosc*.tar.gz # Patch PThread Propagation - # curl -sLo blosc-pthread.patch \ - # https://patch-diff.githubusercontent.com/raw/Blosc/c-blosc/pull/318.patch - # python3 -m patch -p 1 -d c-blosc-1.21.0 blosc-pthread.patch + curl -sLo blosc-pthread.patch \ + https://patch-diff.githubusercontent.com/raw/Blosc/c-blosc/pull/318.patch + python3 -m patch -p 1 -d c-blosc-1.21.0 blosc-pthread.patch # SSE2 support # https://github.com/Blosc/c-blosc/issues/334 - WITH_SSE2=ON + DEACTIVATE_SSE2=OFF if [[ "${CMAKE_OSX_ARCHITECTURES-}" == *"arm64"* ]]; then # error: SSE2 is not supported by the target architecture/platform and/or this compiler. - WITH_SSE2=OFF + DEACTIVATE_SSE2=ON fi + mkdir build-blosc + cd build-blosc + PY_BIN=$(which python3) + CMAKE_BIN="$(${PY_BIN} -m pip show cmake 2>/dev/null | grep Location | cut -d' ' -f2)/cmake/data/bin/" + PATH=${CMAKE_BIN}:${PATH} cmake \ + -DDEACTIVATE_SNAPPY=ON \ + -DDEACTIVATE_SSE2=${DEACTIVATE_SSE2} \ + -DBUILD_SHARED=OFF \ + -DBUILD_TESTS=OFF \ + -DBUILD_BENCHMARKS=OFF \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DCMAKE_INSTALL_PREFIX=${BUILD_PREFIX} \ + -DZLIB_USE_STATIC_LIBS=ON \ + ../c-blosc-* + make -j${CPU_COUNT} + make install + cd - + + rm -rf build-blosc + + touch blosc-stamp +} + +function build_blosc2 { + if [ -e blosc-stamp ]; then return; fi + + curl -sLo c-blosc2-v2.7.1.tar.gz \ + https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.7.1.tar.gz + file c-blosc2*.tar.gz + tar -xzf c-blosc2*.tar.gz + rm c-blosc2*.tar.gz + # @todo: proper patch for this sed -E -i.bak 's|if\(\$ENV\{CMAKE_OSX_ARCHITECTURES\} STREQUAL "arm64"\)|if("$ENV{CMAKE_OSX_ARCHITECTURES}" STREQUAL "arm64")|' c-blosc2-*/CMakeLists.txt cat c-blosc2-*/CMakeLists.txt - mkdir build-blosc - cd build-blosc + mkdir build-blosc2 + cd build-blosc2 PY_BIN=$(which python3) CMAKE_BIN="$(${PY_BIN} -m pip show cmake 2>/dev/null | grep Location | cut -d' ' -f2)/cmake/data/bin/" # Blosc2 runs into a linking error without testing enabled??? @@ -178,9 +238,9 @@ function build_blosc { make install cd - - rm -rf build-blosc + rm -rf build-blosc2 - touch blosc-stamp + touch blosc2-stamp } function build_zfp { @@ -320,6 +380,11 @@ fi install_buildessentials build_zlib build_blosc +build_blosc2 build_zfp build_hdf5 +# skip ADIOS1 build for M1 +if [[ "${CMAKE_OSX_ARCHITECTURES-}" != "arm64" ]]; then + build_adios1 +fi build_adios2 From c9cf3eb567adc06878ac1b52693f68629bca259f Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 27 Jun 2023 11:54:14 +0200 Subject: [PATCH 19/54] Undo `build.yml` Changes --- .github/workflows/build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d70e54e8aa..deacff688e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -123,7 +123,7 @@ jobs: CIBW_BEFORE_BUILD_WINDOWS: 'cmd /E:ON /V:ON /C .github\library_builders.bat' # for the openPMD-api build, CMake shall search for # static dependencies of HDF5 and ADIOS1 (see setup.py) - CIBW_ENVIRONMENT: HDF5_USE_STATIC_LIBRARIES='ON' ZLIB_USE_STATIC_LIBS='ON' ADIOS_USE_STATIC_LIBS='ON' openPMD_CMAKE_openPMD_USE_HDF5='ON' openPMD_CMAKE_openPMD_USE_ADIOS2='ON' ADIOS2_DIR=/usr/local + CIBW_ENVIRONMENT: HDF5_USE_STATIC_LIBRARIES='ON' ZLIB_USE_STATIC_LIBS='ON' ADIOS_USE_STATIC_LIBS='ON' openPMD_CMAKE_openPMD_USE_HDF5='ON' openPMD_CMAKE_openPMD_USE_ADIOS2='ON' CIBW_ENVIRONMENT_WINDOWS: HDF5_USE_STATIC_LIBRARIES='ON' ZLIB_USE_STATIC_LIBS='ON' openPMD_CMAKE_openPMD_USE_HDF5='ON' openPMD_CMAKE_openPMD_USE_ADIOS2='ON' CMAKE_PREFIX_PATH='C:/Program Files (x86)/ADIOS2;C:/Program Files (x86)/blosc;C:/Program Files (x86)/HDF5;C:/Program Files (x86)/ZFP;C:/Program Files (x86)/zlib' # C++17 support in macOS 10.13+ (partial) and 10.14+ (std::visit) and 10.15+ (std::filesystem::path) # https://cibuildwheel.readthedocs.io/en/stable/cpp_standards/#macos-and-deployment-target-versions @@ -136,7 +136,6 @@ jobs: # Show a bit more output (pip -v) CIBW_BUILD_VERBOSITY: 1 run: | - env cd src python -m pip install --upgrade pip setuptools wheel python -m pip install --upgrade cmake From 176ff4eb283c273f46e7931699ddc967fd375b2a Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 27 Jun 2023 11:55:36 +0200 Subject: [PATCH 20/54] ADIOS2: no c-blosc1 pthread issue --- library_builders.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/library_builders.sh b/library_builders.sh index 19ad4b61dd..d70a361710 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -102,11 +102,6 @@ function build_adios2 { tar -xzf adios2*.tar.gz rm adios2*.tar.gz - # Patch PThread Propagation - # curl -sLo adios-pthread.patch \ - # https://patch-diff.githubusercontent.com/raw/ornladios/ADIOS2/pull/2768.patch - # python3 -m patch -p 1 -d ADIOS2-2.9.0 adios-pthread.patch - # DILL macOS arm64 or universal2 binary # https://github.com/ornladios/ADIOS2/issues/3116 # needs rebase (or use ADIOS2-2.8.0) From 2bb602e38df90b2c7e86117e8560c4694440934b Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 27 Jun 2023 11:59:40 +0200 Subject: [PATCH 21/54] HDF5: 1.14.1-2 --- library_builders.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/library_builders.sh b/library_builders.sh index d70a361710..e0f0d03fd6 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -113,10 +113,7 @@ function build_adios2 { ADIOS2_USE_SST=OFF fi - mkdir build-adios2 - cd build-adios2 - PY_BIN=$(which python3) - CMAKE_BIN="$(${PY_BIN} -m pip show cmake 2>/dev/null | grep Location | cut -d' ' -f2)/cmake/data/bin/" + # ZPL issues on non-Linux platforms if [ "$(uname -s)" = "Linux" ] then EVPATH_ZPL="ON" @@ -125,6 +122,12 @@ function build_adios2 { # https://github.com/GTkorvo/evpath/issues/47 EVPATH_ZPL="OFF" fi + + # build + mkdir build-adios2 + cd build-adios2 + PY_BIN=$(which python3) + CMAKE_BIN="$(${PY_BIN} -m pip show cmake 2>/dev/null | grep Location | cut -d' ' -f2)/cmake/data/bin/" PATH=${CMAKE_BIN}:${PATH} cmake \ -DBUILD_SHARED_LIBS=OFF \ -DBUILD_TESTING=OFF \ @@ -297,8 +300,8 @@ function build_zlib { function build_hdf5 { if [ -e hdf5-stamp ]; then return; fi - curl -sLo hdf5-1.12.2.tar.gz \ - https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.12/hdf5-1.12.2/src/hdf5-1.12.2.tar.gz + curl -sLo hdf5-1.14.1-2.tar.gz \ + https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.14/hdf5-1.14.1/src/hdf5-1.14.1-2.tar.gz file hdf5*.tar.gz tar -xzf hdf5*.tar.gz rm hdf5*.tar.gz From becf8cdd003c73d59c474d1821a6f568b61ca760 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 27 Jun 2023 12:02:51 +0200 Subject: [PATCH 22/54] macOS CI: Now has 3 cores --- library_builders.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/library_builders.sh b/library_builders.sh index e0f0d03fd6..eeb95933e1 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -4,7 +4,14 @@ set -eu -o pipefail BUILD_PREFIX="${BUILD_PREFIX:-/usr/local}" -CPU_COUNT="${CPU_COUNT:-2}" + +# https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources +if [ "$(uname -s)" = "Darwin" ] +then + CPU_COUNT="${CPU_COUNT:-3}" +else + CPU_COUNT="${CPU_COUNT:-2}" +fi function install_buildessentials { if [ -e buildessentials-stamp ]; then return; fi From 6f209889bcf227fd9244e7933994fb7dfc1cb203 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 30 Jun 2023 14:17:22 -0700 Subject: [PATCH 23/54] C-Blosc2 Updates - CMake options - Windows - build stamp --- .github/workflows/build.yml | 2 +- library_builders.bat | 48 +++++++++++++++++++++++++++++-------- library_builders.sh | 10 ++++---- 3 files changed, 45 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index deacff688e..39b8c44be7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -124,7 +124,7 @@ jobs: # for the openPMD-api build, CMake shall search for # static dependencies of HDF5 and ADIOS1 (see setup.py) CIBW_ENVIRONMENT: HDF5_USE_STATIC_LIBRARIES='ON' ZLIB_USE_STATIC_LIBS='ON' ADIOS_USE_STATIC_LIBS='ON' openPMD_CMAKE_openPMD_USE_HDF5='ON' openPMD_CMAKE_openPMD_USE_ADIOS2='ON' - CIBW_ENVIRONMENT_WINDOWS: HDF5_USE_STATIC_LIBRARIES='ON' ZLIB_USE_STATIC_LIBS='ON' openPMD_CMAKE_openPMD_USE_HDF5='ON' openPMD_CMAKE_openPMD_USE_ADIOS2='ON' CMAKE_PREFIX_PATH='C:/Program Files (x86)/ADIOS2;C:/Program Files (x86)/blosc;C:/Program Files (x86)/HDF5;C:/Program Files (x86)/ZFP;C:/Program Files (x86)/zlib' + CIBW_ENVIRONMENT_WINDOWS: HDF5_USE_STATIC_LIBRARIES='ON' ZLIB_USE_STATIC_LIBS='ON' openPMD_CMAKE_openPMD_USE_HDF5='ON' openPMD_CMAKE_openPMD_USE_ADIOS2='ON' CMAKE_PREFIX_PATH='C:/Program Files (x86)/ADIOS2;C:/Program Files (x86)/blosc;C:/Program Files (x86)/blosc2;C:/Program Files (x86)/HDF5;C:/Program Files (x86)/ZFP;C:/Program Files (x86)/zlib' # C++17 support in macOS 10.13+ (partial) and 10.14+ (std::visit) and 10.15+ (std::filesystem::path) # https://cibuildwheel.readthedocs.io/en/stable/cpp_standards/#macos-and-deployment-target-versions # arm64 Python interpreters are built with 11.0 diff --git a/library_builders.bat b/library_builders.bat index 383e2d13d1..a3fc08fc81 100644 --- a/library_builders.bat +++ b/library_builders.bat @@ -16,25 +16,21 @@ exit /b 0 :build_adios2 if exist adios2-stamp exit /b 0 - curl -sLo adios2-2.7.1.zip ^ - https://github.com/ornladios/ADIOS2/archive/v2.7.1.zip - powershell Expand-Archive adios2-2.7.1.zip -DestinationPath dep-adios2 + curl -sLo adios2-2.9.0.zip ^ + https://github.com/ornladios/ADIOS2/archive/v2.9.0.zip + powershell Expand-Archive adios2-2.9.0.zip -DestinationPath dep-adios2 - :: Patch Blosc Compression w/ ADIOS 2.7.1 - curl -sLo adios2-blosc.patch ^ - https://patch-diff.githubusercontent.com/raw/ornladios/ADIOS2/pull/2746.patch - python -m patch -p 1 -d dep-adios2/ADIOS2-2.7.1 adios2-blosc.patch - - cmake -S dep-adios2/ADIOS2-2.7.1 -B build-adios2 ^ + cmake -S dep-adios2/ADIOS2-2.9.0 -B build-adios2 ^ -DCMAKE_BUILD_TYPE=Release ^ -DBUILD_SHARED_LIBS=OFF ^ -DBUILD_TESTING=OFF ^ -DADIOS2_USE_MPI=OFF ^ -DADIOS2_BUILD_EXAMPLES=OFF ^ - -DADIOS2_USE_Blosc=ON ^ + -DADIOS2_USE_Blosc2=ON ^ -DADIOS2_USE_BZip2=OFF ^ -DADIOS2_USE_Fortran=OFF ^ -DADIOS2_USE_HDF5=OFF ^ + -DADIOS2_USE_MHS=OFF ^ -DADIOS2_USE_PNG=OFF ^ -DADIOS2_USE_Profiling=OFF ^ -DADIOS2_USE_Python=OFF ^ @@ -88,6 +84,37 @@ exit /b 0 if errorlevel 1 exit 1 exit /b 0 +:build_blosc2 + if exist blosc2-stamp exit /b 0 + + curl -sLo blosc2-2.7.1.zip ^ + https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.7.1.zip + powershell Expand-Archive blosc2-2.7.1.zip -DestinationPath dep-blosc2 + + cmake -S dep-blosc2/c-blosc2-2.7.1 -B build-blosc2 ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DBUILD_SHARED=OFF ^ + -DBUILD_STATIC=ON ^ + -DBUILD_BENCHMARKS=OFF ^ + -DBUILD_EXAMPLES=OFF ^ + -DBUILD_FUZZERS=OFF ^ + -DBUILD_TESTS=OFF ^ + -DZLIB_USE_STATIC_LIBS=ON + if errorlevel 1 exit 1 + + cmake --build build-blosc2 --parallel %CPU_COUNT% + if errorlevel 1 exit 1 + + cmake --build build-blosc2 --target install --config Release + if errorlevel 1 exit 1 + + rmdir /s /q build-blosc2 + if errorlevel 1 exit 1 + + break > blosc2-stamp + if errorlevel 1 exit 1 +exit /b 0 + :build_hdf5 if exist hdf5-stamp exit /b 0 @@ -194,6 +221,7 @@ call :build_zlib :: build_bzip2 :: build_szip call :build_blosc +call :build_blosc2 call :build_zfp call :build_hdf5 call :build_adios2 diff --git a/library_builders.sh b/library_builders.sh index eeb95933e1..72f8581993 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -212,7 +212,7 @@ function build_blosc { } function build_blosc2 { - if [ -e blosc-stamp ]; then return; fi + if [ -e blosc-stamp2 ]; then return; fi curl -sLo c-blosc2-v2.7.1.tar.gz \ https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.7.1.tar.gz @@ -230,11 +230,13 @@ function build_blosc2 { CMAKE_BIN="$(${PY_BIN} -m pip show cmake 2>/dev/null | grep Location | cut -d' ' -f2)/cmake/data/bin/" # Blosc2 runs into a linking error without testing enabled??? PATH=${CMAKE_BIN}:${PATH} cmake \ - -DWITH_SSE2=${WITH_SSE2} \ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DBUILD_STATIC=ON \ -DBUILD_SHARED=OFF \ - -DBUILD_TESTS=ON \ -DBUILD_BENCHMARKS=OFF \ + -DBUILD_EXAMPLES=OFF \ + -DBUILD_FUZZERS=OFF \ + -DBUILD_TESTS=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DCMAKE_INSTALL_PREFIX=${BUILD_PREFIX} \ -DZLIB_USE_STATIC_LIBS=ON \ @@ -245,7 +247,7 @@ function build_blosc2 { rm -rf build-blosc2 - touch blosc2-stamp + touch blosc-stamp2 } function build_zfp { From dede51c0e70bfe3862a8db344be7263fe7104e7e Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 30 Jun 2023 14:42:39 -0700 Subject: [PATCH 24/54] Windows Updates --- library_builders.bat | 8 ++++---- library_builders.sh | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/library_builders.bat b/library_builders.bat index a3fc08fc81..05e2487301 100644 --- a/library_builders.bat +++ b/library_builders.bat @@ -118,11 +118,11 @@ exit /b 0 :build_hdf5 if exist hdf5-stamp exit /b 0 - curl -sLo hdf5-1.12.2.zip ^ - https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.12/hdf5-1.12.2/src/hdf5-1.12.2.zip - powershell Expand-Archive hdf5-1.12.2.zip -DestinationPath dep-hdf5 + curl -sLo hdf5-1.14.1-2.zip ^ + https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.14/hdf5-1.14.1/src/hdf5-1.14.1-2.zip + powershell Expand-Archive hdf5-1.14.1-2.zip -DestinationPath dep-hdf5 - cmake -S dep-hdf5/hdf5-1.12.2 -B build-hdf5 ^ + cmake -S dep-hdf5/hdf5-1.14.1-2 -B build-hdf5 ^ -DCMAKE_BUILD_TYPE=Release ^ -DCMAKE_VERBOSE_MAKEFILE=ON ^ -DBUILD_SHARED_LIBS=OFF ^ diff --git a/library_builders.sh b/library_builders.sh index 72f8581993..e458010791 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -334,11 +334,11 @@ function build_hdf5 { HOST_ARG="--host=aarch64-apple-darwin" curl -sLo osx_cross_configure.patch \ - https://raw.githubusercontent.com/h5py/h5py/fcaca1d1b81d25c0d83b11d5bdf497469b5980e9/ci/osx_cross_configure.patch + https://raw.githubusercontent.com/conda-forge/hdf5-feedstock/ca7b3b40d58cb949638ead75c9527b3fd1112a3f/recipe/patches/osx_cross_configure.patch python3 -m patch -p 0 -d . osx_cross_configure.patch curl -sLo osx_cross_src_makefile.patch \ - https://raw.githubusercontent.com/h5py/h5py/fcaca1d1b81d25c0d83b11d5bdf497469b5980e9/ci/osx_cross_src_makefile.patch + https://raw.githubusercontent.com/conda-forge/hdf5-feedstock/ca7b3b40d58cb949638ead75c9527b3fd1112a3f/recipe/patches/osx_cross_makefile.patch #python3 -m patch -p 0 -d . osx_cross_src_makefile.patch patch -p 0 < osx_cross_src_makefile.patch fi From 64164bee779975921ba4cc2ae59b8d9cb7c66b32 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 30 Jun 2023 14:55:01 -0700 Subject: [PATCH 25/54] HDF5 Patches --- library_builders.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/library_builders.sh b/library_builders.sh index e458010791..6047364f15 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -339,8 +339,7 @@ function build_hdf5 { curl -sLo osx_cross_src_makefile.patch \ https://raw.githubusercontent.com/conda-forge/hdf5-feedstock/ca7b3b40d58cb949638ead75c9527b3fd1112a3f/recipe/patches/osx_cross_makefile.patch - #python3 -m patch -p 0 -d . osx_cross_src_makefile.patch - patch -p 0 < osx_cross_src_makefile.patch + python3 -m patch -p 0 -d . osx_cross_src_makefile.patch fi ./configure \ From c8b5be21a7040ea0ec75167653c3e4f515ffdc1e Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 30 Jun 2023 15:02:14 -0700 Subject: [PATCH 26/54] C-Blosc 2.9.3 + CMake Patch --- library_builders.bat | 14 ++++++++++---- library_builders.sh | 15 ++++++++------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/library_builders.bat b/library_builders.bat index 05e2487301..b2f04eab0f 100644 --- a/library_builders.bat +++ b/library_builders.bat @@ -87,11 +87,16 @@ exit /b 0 :build_blosc2 if exist blosc2-stamp exit /b 0 - curl -sLo blosc2-2.7.1.zip ^ - https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.7.1.zip - powershell Expand-Archive blosc2-2.7.1.zip -DestinationPath dep-blosc2 + curl -sLo blosc2-2.9.3.zip ^ + https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.9.3.zip + powershell Expand-Archive blosc2-2.9.3.zip -DestinationPath dep-blosc2 - cmake -S dep-blosc2/c-blosc2-2.7.1 -B build-blosc2 ^ + :: https://github.com/Blosc/c-blosc2/pull/525 + curl -sLo c-blosc2-cmake.patch ^ + https://patch-diff.githubusercontent.com/raw/Blosc/c-blosc2/pull/525.patch + python -m patch -p 1 -d dep-blosc2/c-blosc2-2.9.3 c-blosc2-cmake.patch + + cmake -S dep-blosc2/c-blosc2-2.9.3 -B build-blosc2 ^ -DCMAKE_BUILD_TYPE=Release ^ -DBUILD_SHARED=OFF ^ -DBUILD_STATIC=ON ^ @@ -99,6 +104,7 @@ exit /b 0 -DBUILD_EXAMPLES=OFF ^ -DBUILD_FUZZERS=OFF ^ -DBUILD_TESTS=OFF ^ + -DPREFER_EXTERNAL_ZLIB=ON ^ -DZLIB_USE_STATIC_LIBS=ON if errorlevel 1 exit 1 diff --git a/library_builders.sh b/library_builders.sh index 6047364f15..e12879b0f5 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -214,21 +214,21 @@ function build_blosc { function build_blosc2 { if [ -e blosc-stamp2 ]; then return; fi - curl -sLo c-blosc2-v2.7.1.tar.gz \ - https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.7.1.tar.gz + curl -sLo c-blosc2-v2.9.3.tar.gz \ + https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.9.3.tar.gz file c-blosc2*.tar.gz tar -xzf c-blosc2*.tar.gz rm c-blosc2*.tar.gz - # @todo: proper patch for this - sed -E -i.bak 's|if\(\$ENV\{CMAKE_OSX_ARCHITECTURES\} STREQUAL "arm64"\)|if("$ENV{CMAKE_OSX_ARCHITECTURES}" STREQUAL "arm64")|' c-blosc2-*/CMakeLists.txt - cat c-blosc2-*/CMakeLists.txt + # https://github.com/Blosc/c-blosc2/pull/525 + curl -sLo c-blosc2-cmake.patch \ + https://patch-diff.githubusercontent.com/raw/Blosc/c-blosc2/pull/525.patch + python3 -m patch -p 1 -d c-blosc2-2.9.3 c-blosc2-cmake.patch mkdir build-blosc2 cd build-blosc2 PY_BIN=$(which python3) CMAKE_BIN="$(${PY_BIN} -m pip show cmake 2>/dev/null | grep Location | cut -d' ' -f2)/cmake/data/bin/" - # Blosc2 runs into a linking error without testing enabled??? PATH=${CMAKE_BIN}:${PATH} cmake \ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ -DBUILD_STATIC=ON \ @@ -236,9 +236,10 @@ function build_blosc2 { -DBUILD_BENCHMARKS=OFF \ -DBUILD_EXAMPLES=OFF \ -DBUILD_FUZZERS=OFF \ - -DBUILD_TESTS=ON \ + -DBUILD_TESTS=OFF \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DCMAKE_INSTALL_PREFIX=${BUILD_PREFIX} \ + -DPREFER_EXTERNAL_ZLIB=ON \ -DZLIB_USE_STATIC_LIBS=ON \ ../c-blosc2-* make -j${CPU_COUNT} From e63a576784bb9e9d971b4cde5f45f21cbf3278f2 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 30 Jun 2023 15:32:29 -0700 Subject: [PATCH 27/54] ARM: H5Detect --- library_builders.sh | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/library_builders.sh b/library_builders.sh index e12879b0f5..7406894006 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -352,17 +352,6 @@ function build_hdf5 { ${HOST_ARG} \ --prefix=${BUILD_PREFIX} - if [[ "${CMAKE_OSX_ARCHITECTURES-}" == "arm64" ]]; then - # https://github.com/h5py/h5py/blob/fcaca1d1b81d25c0d83b11d5bdf497469b5980e9/ci/configure_hdf5_mac.sh - build_h5detect - mkdir -p native-build/bin - pushd native-build/bin - CFLAGS= $CC ../../src/H5detect.c -I ../../src/ -o H5detect - CFLAGS= $CC ../../src/H5make_libsettings.c -I ../../src/ -o H5make_libsettings - popd - - export PATH="$(pwd)/native-build/bin:$PATH" - fi - make -j${CPU_COUNT} make install cd .. From ee1baf6cf11f5d9961dd2d510781977f0305a9e8 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 30 Jun 2023 16:33:02 -0700 Subject: [PATCH 28/54] Windows: Avoid CXX20 in C-Blosc2 --- library_builders.bat | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/library_builders.bat b/library_builders.bat index b2f04eab0f..6d9d78e324 100644 --- a/library_builders.bat +++ b/library_builders.bat @@ -96,6 +96,11 @@ exit /b 0 https://patch-diff.githubusercontent.com/raw/Blosc/c-blosc2/pull/525.patch python -m patch -p 1 -d dep-blosc2/c-blosc2-2.9.3 c-blosc2-cmake.patch + :: https://github.com/Blosc/c-blosc2/issues/526 + curl -sLo c-blosc2-cxx20.patch ^ + https://github.com/Blosc/c-blosc2/pull/527.patch + python -m patch -p 1 -d dep-blosc2/c-blosc2-2.9.3 c-blosc2-cxx20.patch + cmake -S dep-blosc2/c-blosc2-2.9.3 -B build-blosc2 ^ -DCMAKE_BUILD_TYPE=Release ^ -DBUILD_SHARED=OFF ^ From df0f071470c29c0694b56dd52b21eb5843b61cc3 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 30 Jun 2023 16:40:32 -0700 Subject: [PATCH 29/54] macOS arm64 HDF5: H5detect cross-compile --- library_builders.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/library_builders.sh b/library_builders.sh index 7406894006..8927791cb3 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -319,6 +319,7 @@ function build_hdf5 { # macOS cross-compile HOST_ARG="" + # heavily based on conda-forge hdf5-feedstock and h5py's cibuildwheel instructions # https://github.com/conda-forge/hdf5-feedstock/blob/cbbd57d58f7f5350ca679eaad49354c11dd32b95/recipe/build.sh#L53-L80 if [[ "${CMAKE_OSX_ARCHITECTURES-}" == "arm64" ]]; then # https://github.com/h5py/h5py/blob/fcaca1d1b81d25c0d83b11d5bdf497469b5980e9/ci/configure_hdf5_mac.sh @@ -352,6 +353,22 @@ function build_hdf5 { ${HOST_ARG} \ --prefix=${BUILD_PREFIX} + if [[ "${CMAKE_OSX_ARCHITECTURES-}" == "arm64" ]]; then + ( + # https://github.com/h5py/h5py/blob/fcaca1d1b81d25c0d83b11d5bdf497469b5980e9/ci/configure_hdf5_mac.sh - build_h5detect + mkdir -p native-build/bin + pushd native-build/bin + + # MACOSX_DEPLOYMENT_TARGET is for the target_platform and not for build_platform + unset MACOSX_DEPLOYMENT_TARGET + + $CC ../../src/H5detect.c -I ../../src/ -o H5detect + $CC ../../src/H5make_libsettings.c -I ../../src/ -o H5make_libsettings + popd + ) + export PATH="$(pwd)/native-build/bin:$PATH" + fi + make -j${CPU_COUNT} make install cd .. From 377f6501e53705d42c2e1df1dc22cb00f54948fe Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 30 Jun 2023 17:00:56 -0700 Subject: [PATCH 30/54] HDF5 arm64: cross-compiling H5detect --- library_builders.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/library_builders.sh b/library_builders.sh index 8927791cb3..874a22f798 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -362,8 +362,8 @@ function build_hdf5 { # MACOSX_DEPLOYMENT_TARGET is for the target_platform and not for build_platform unset MACOSX_DEPLOYMENT_TARGET - $CC ../../src/H5detect.c -I ../../src/ -o H5detect - $CC ../../src/H5make_libsettings.c -I ../../src/ -o H5make_libsettings + CFLAGS="" $CC ../../src/H5detect.c -I ../../src/ -o H5detect + CFLAGS="" $CC ../../src/H5make_libsettings.c -I ../../src/ -o H5make_libsettings popd ) export PATH="$(pwd)/native-build/bin:$PATH" From c76edd1fabed1f7791454119dfffd9cacb4d266f Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 30 Jun 2023 17:41:10 -0700 Subject: [PATCH 31/54] [Patch] ADIOS2 c-blosc2 windows.h std::min --- library_builders.bat | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/library_builders.bat b/library_builders.bat index 6d9d78e324..bb1dfb0625 100644 --- a/library_builders.bat +++ b/library_builders.bat @@ -20,6 +20,11 @@ exit /b 0 https://github.com/ornladios/ADIOS2/archive/v2.9.0.zip powershell Expand-Archive adios2-2.9.0.zip -DestinationPath dep-adios2 + :: https://github.com/ornladios/ADIOS2/issues/3680#issuecomment-1615308336 + curl -sLo adios2-cblosc-stdmin.patch ^ + https://patch-diff.githubusercontent.com/raw/ornladios/ADIOS2/pull/3681.patch + python -m patch -p 1 -d dep-adios2/ADIOS2-2.9.0 adios2-cblosc-stdmin.patch + cmake -S dep-adios2/ADIOS2-2.9.0 -B build-adios2 ^ -DCMAKE_BUILD_TYPE=Release ^ -DBUILD_SHARED_LIBS=OFF ^ From bab56193dedaff35e3041b1171af519336932c70 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 30 Jun 2023 22:31:01 -0700 Subject: [PATCH 32/54] [Patch] C-Blosc2: External Zlib --- library_builders.bat | 8 +++++++- library_builders.sh | 10 ++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/library_builders.bat b/library_builders.bat index bb1dfb0625..8b65d02236 100644 --- a/library_builders.bat +++ b/library_builders.bat @@ -106,6 +106,11 @@ exit /b 0 https://github.com/Blosc/c-blosc2/pull/527.patch python -m patch -p 1 -d dep-blosc2/c-blosc2-2.9.3 c-blosc2-cxx20.patch + :: https://github.com/Blosc/c-blosc2/pull/529 + curl -sLo c-blosc2-external-zlib.patch ^ + https://github.com/Blosc/c-blosc2/pull/529.patch + python -m patch -p 1 -d dep-blosc2/c-blosc2-2.9.3 c-blosc2-external-zlib.patch + cmake -S dep-blosc2/c-blosc2-2.9.3 -B build-blosc2 ^ -DCMAKE_BUILD_TYPE=Release ^ -DBUILD_SHARED=OFF ^ @@ -113,6 +118,7 @@ exit /b 0 -DBUILD_BENCHMARKS=OFF ^ -DBUILD_EXAMPLES=OFF ^ -DBUILD_FUZZERS=OFF ^ + -DBUILD_PLUGINS=OFF ^ -DBUILD_TESTS=OFF ^ -DPREFER_EXTERNAL_ZLIB=ON ^ -DZLIB_USE_STATIC_LIBS=ON @@ -236,8 +242,8 @@ call :install_buildessentials call :build_zlib :: build_bzip2 :: build_szip +call :build_zfp call :build_blosc call :build_blosc2 -call :build_zfp call :build_hdf5 call :build_adios2 diff --git a/library_builders.sh b/library_builders.sh index 874a22f798..32a871f48b 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -225,6 +225,11 @@ function build_blosc2 { https://patch-diff.githubusercontent.com/raw/Blosc/c-blosc2/pull/525.patch python3 -m patch -p 1 -d c-blosc2-2.9.3 c-blosc2-cmake.patch + # https://github.com/Blosc/c-blosc2/pull/529 + curl -sLo c-blosc2-external-zlib.patch \ + https://patch-diff.githubusercontent.com/raw/Blosc/c-blosc2/pull/529.patch + python3 -m patch -p 1 -d c-blosc2-2.9.3 c-blosc2-external-zlib.patch + mkdir build-blosc2 cd build-blosc2 PY_BIN=$(which python3) @@ -236,12 +241,13 @@ function build_blosc2 { -DBUILD_BENCHMARKS=OFF \ -DBUILD_EXAMPLES=OFF \ -DBUILD_FUZZERS=OFF \ + -DBUILD_PLUGINS=OFF \ -DBUILD_TESTS=OFF \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DCMAKE_INSTALL_PREFIX=${BUILD_PREFIX} \ -DPREFER_EXTERNAL_ZLIB=ON \ -DZLIB_USE_STATIC_LIBS=ON \ - ../c-blosc2-* + ../c-blosc2-2.9.3 make -j${CPU_COUNT} make install cd - @@ -392,9 +398,9 @@ fi install_buildessentials build_zlib +build_zfp build_blosc build_blosc2 -build_zfp build_hdf5 # skip ADIOS1 build for M1 if [[ "${CMAKE_OSX_ARCHITECTURES-}" != "arm64" ]]; then From 1de81daf3d57e87262528e55e3296fa503e42a80 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 30 Jun 2023 23:28:24 -0700 Subject: [PATCH 33/54] [Hack] Windows: Ignore Ext. ZLIB for c-blosc2 Just build again and close eyes. --- library_builders.bat | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/library_builders.bat b/library_builders.bat index 8b65d02236..5d013142df 100644 --- a/library_builders.bat +++ b/library_builders.bat @@ -119,9 +119,9 @@ exit /b 0 -DBUILD_EXAMPLES=OFF ^ -DBUILD_FUZZERS=OFF ^ -DBUILD_PLUGINS=OFF ^ - -DBUILD_TESTS=OFF ^ - -DPREFER_EXTERNAL_ZLIB=ON ^ - -DZLIB_USE_STATIC_LIBS=ON + -DBUILD_TESTS=OFF +:: -DPREFER_EXTERNAL_ZLIB=ON +:: -DZLIB_USE_STATIC_LIBS=ON if errorlevel 1 exit 1 cmake --build build-blosc2 --parallel %CPU_COUNT% From 4ed8604f65c465833664ff947220af726f391028 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 25 Jul 2023 10:24:04 -0700 Subject: [PATCH 34/54] ppc64le: Skip C-Blosc & ADIOS1 Due to time limits on Travis-CI. --- library_builders.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/library_builders.sh b/library_builders.sh index 32a871f48b..790b9b7794 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -399,11 +399,15 @@ fi install_buildessentials build_zlib build_zfp -build_blosc +if [[ "$(uname -m)" != "ppc64le" ]]; then + # builds too long for Travis-CI + build_blosc +fi build_blosc2 build_hdf5 -# skip ADIOS1 build for M1 -if [[ "${CMAKE_OSX_ARCHITECTURES-}" != "arm64" ]]; then +if [[ "${CMAKE_OSX_ARCHITECTURES-}" != "arm64" && "$(uname -m)" != "ppc64le" ]]; then + # macOS: skip ADIOS1 build for M1 + # Linux: with ADIOS2 also enabled, this builds too long for Travis-CI build_adios1 fi build_adios2 From a93c13797ec82af5b44d511963c8cc303ad69c09 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 25 Jul 2023 11:04:58 -0700 Subject: [PATCH 35/54] Unix: Keep HDF5 1.12.2 for Now - issues with cross-compile cannot quickly be solved for macOS arm64/aarch64 - disable HDF5 support in ADIOS2 for now --- library_builders.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/library_builders.sh b/library_builders.sh index 790b9b7794..efbaa4ea7d 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -143,6 +143,7 @@ function build_adios2 { -DADIOS2_USE_BZip2=OFF \ -DADIOS2_USE_Blosc2=ON \ -DADIOS2_USE_Fortran=OFF \ + -DADIOS2_USE_HDF5=OFF \ -DADIOS2_USE_MHS=OFF \ -DADIOS2_USE_MPI=OFF \ -DADIOS2_USE_PNG=OFF \ @@ -316,8 +317,8 @@ function build_zlib { function build_hdf5 { if [ -e hdf5-stamp ]; then return; fi - curl -sLo hdf5-1.14.1-2.tar.gz \ - https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.14/hdf5-1.14.1/src/hdf5-1.14.1-2.tar.gz + curl -sLo hdf5-1.12.2.tar.gz \ + https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.12/hdf5-1.12.2/src/hdf5-1.12.2.tar.gz file hdf5*.tar.gz tar -xzf hdf5*.tar.gz rm hdf5*.tar.gz @@ -342,12 +343,13 @@ function build_hdf5 { HOST_ARG="--host=aarch64-apple-darwin" curl -sLo osx_cross_configure.patch \ - https://raw.githubusercontent.com/conda-forge/hdf5-feedstock/ca7b3b40d58cb949638ead75c9527b3fd1112a3f/recipe/patches/osx_cross_configure.patch + https://raw.githubusercontent.com/h5py/h5py/fcaca1d1b81d25c0d83b11d5bdf497469b5980e9/ci/osx_cross_configure.patch python3 -m patch -p 0 -d . osx_cross_configure.patch curl -sLo osx_cross_src_makefile.patch \ - https://raw.githubusercontent.com/conda-forge/hdf5-feedstock/ca7b3b40d58cb949638ead75c9527b3fd1112a3f/recipe/patches/osx_cross_makefile.patch - python3 -m patch -p 0 -d . osx_cross_src_makefile.patch + https://raw.githubusercontent.com/h5py/h5py/fcaca1d1b81d25c0d83b11d5bdf497469b5980e9/ci/osx_cross_src_makefile.patch + #python3 -m patch -p 0 -d . osx_cross_src_makefile.patch + patch -p 0 < osx_cross_src_makefile.patch fi ./configure \ From 612bd31184e22acca9ade8055c9a423b84780f18 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 25 Jul 2023 14:19:49 -0700 Subject: [PATCH 36/54] [Patch] C-Blosc2 PUBLIC Linkage Zlib --- library_builders.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/library_builders.sh b/library_builders.sh index efbaa4ea7d..6a2c66e64a 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -231,6 +231,13 @@ function build_blosc2 { https://patch-diff.githubusercontent.com/raw/Blosc/c-blosc2/pull/529.patch python3 -m patch -p 1 -d c-blosc2-2.9.3 c-blosc2-external-zlib.patch + # https://github.com/Blosc/c-blosc2/pull/536 + curl -sLo c-blosc2-external-zlib-public.patch \ + https://patch-diff.githubusercontent.com/raw/Blosc/c-blosc2/pull/536.patch + python3 -m patch -p 1 -d c-blosc2-2.9.3 c-blosc2-external-zlib-public.patch + + rm -rf *.patch + mkdir build-blosc2 cd build-blosc2 PY_BIN=$(which python3) From 2d187089fb2146dcf5ba733fd64f29bc3db7df66 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 25 Jul 2023 17:39:08 -0700 Subject: [PATCH 37/54] Use Blosc2Config.cmake Package --- library_builders.sh | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/library_builders.sh b/library_builders.sh index 6a2c66e64a..896495d847 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -103,8 +103,10 @@ function build_adios1 { function build_adios2 { if [ -e adios2-stamp ]; then return; fi - curl -sLo adios2-2.9.0.tar.gz \ - https://github.com/ornladios/ADIOS2/archive/v2.9.0.tar.gz + #curl -sLo adios2-2.9.0.tar.gz \ + # https://github.com/ornladios/ADIOS2/archive/v2.9.0.tar.gz + curl -sLo adios2-fix-blosc2-findpackage.tar.gz \ + https://github.com/ax3l/ADIOS2/archive/refs/heads/fix-blosc2-findpackage.tar.gz file adios2*.tar.gz tar -xzf adios2*.tar.gz rm adios2*.tar.gz @@ -215,28 +217,14 @@ function build_blosc { function build_blosc2 { if [ -e blosc-stamp2 ]; then return; fi - curl -sLo c-blosc2-v2.9.3.tar.gz \ - https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.9.3.tar.gz + #curl -sLo c-blosc2-v2.10.0.tar.gz \ + curl -sLo c-blosc2-topic-cmake-install-targets.tar.gz \ + https://github.com/ax3l/c-blosc2/archive/refs/heads/topic-cmake-install-targets.tar.gz file c-blosc2*.tar.gz tar -xzf c-blosc2*.tar.gz rm c-blosc2*.tar.gz - # https://github.com/Blosc/c-blosc2/pull/525 - curl -sLo c-blosc2-cmake.patch \ - https://patch-diff.githubusercontent.com/raw/Blosc/c-blosc2/pull/525.patch - python3 -m patch -p 1 -d c-blosc2-2.9.3 c-blosc2-cmake.patch - - # https://github.com/Blosc/c-blosc2/pull/529 - curl -sLo c-blosc2-external-zlib.patch \ - https://patch-diff.githubusercontent.com/raw/Blosc/c-blosc2/pull/529.patch - python3 -m patch -p 1 -d c-blosc2-2.9.3 c-blosc2-external-zlib.patch - - # https://github.com/Blosc/c-blosc2/pull/536 - curl -sLo c-blosc2-external-zlib-public.patch \ - https://patch-diff.githubusercontent.com/raw/Blosc/c-blosc2/pull/536.patch - python3 -m patch -p 1 -d c-blosc2-2.9.3 c-blosc2-external-zlib-public.patch - - rm -rf *.patch + ls mkdir build-blosc2 cd build-blosc2 @@ -255,7 +243,7 @@ function build_blosc2 { -DCMAKE_INSTALL_PREFIX=${BUILD_PREFIX} \ -DPREFER_EXTERNAL_ZLIB=ON \ -DZLIB_USE_STATIC_LIBS=ON \ - ../c-blosc2-2.9.3 + ../c-blosc2-topic-cmake-install-targets make -j${CPU_COUNT} make install cd - From 2bf81e4a707eed3bdac21bccac9535c57b673ee0 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 26 Jul 2023 10:33:22 -0700 Subject: [PATCH 38/54] Cleanup and `cat` ADIOS2 Config File --- library_builders.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/library_builders.sh b/library_builders.sh index 896495d847..1aa462c93e 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -166,6 +166,8 @@ function build_adios2 { rm -rf build-adios2 + cat /usr/local/lib/cmake/adios2/adios2-config.cmake + touch adios2-stamp } @@ -217,14 +219,12 @@ function build_blosc { function build_blosc2 { if [ -e blosc-stamp2 ]; then return; fi - #curl -sLo c-blosc2-v2.10.0.tar.gz \ - curl -sLo c-blosc2-topic-cmake-install-targets.tar.gz \ + #curl -sLo blosc2-v2.10.0.tar.gz \ + curl -sLo blosc2-topic-cmake-install-targets.tar.gz \ https://github.com/ax3l/c-blosc2/archive/refs/heads/topic-cmake-install-targets.tar.gz - file c-blosc2*.tar.gz - tar -xzf c-blosc2*.tar.gz - rm c-blosc2*.tar.gz - - ls + file blosc2*.tar.gz + tar -xzf blosc2*.tar.gz + rm blosc2*.tar.gz mkdir build-blosc2 cd build-blosc2 @@ -243,7 +243,7 @@ function build_blosc2 { -DCMAKE_INSTALL_PREFIX=${BUILD_PREFIX} \ -DPREFER_EXTERNAL_ZLIB=ON \ -DZLIB_USE_STATIC_LIBS=ON \ - ../c-blosc2-topic-cmake-install-targets + ../c-blosc2-* make -j${CPU_COUNT} make install cd - From 502834d71521b307f0d575f21aa7d67cfec088af Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 26 Jul 2023 17:53:32 -0700 Subject: [PATCH 39/54] [Patch] [Hack] CMake --trace-expand --- .github/workflows/build.yml | 36 +++++++++++++----------------------- 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 39b8c44be7..2c86629ceb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -86,29 +86,19 @@ jobs: python -m pip install --upgrade pip setuptools wheel python -m pip install cibuildwheel==2.12.1 -# # 0.15.0: CMake: Fix Python Install Directory -# # https://github.com/openPMD/openPMD-api/pull/1393 -# - name: Download Patch 1/2 -# uses: suisei-cn/actions-download-file@v1 -# id: patch1 -# with: -# url: "https://github.com/ax3l/openPMD-api/commit/b622cc5ea770f866c1e373185a9e389c04bdb54c.patch" -# target: src/.patch/ -# -# # 0.15.0.post2 bump -# - name: Download Patch 2/2 -# uses: suisei-cn/actions-download-file@v1 -# id: setupversion -# with: -# url: "https://gist.githubusercontent.com/ax3l/5e83edefe4b05cf6aa2a971649285fe0/raw/023cbd69e06715e5191ece741bb1de56560a9f96/0001-Bump-setup.py.patch" -# target: src/.patch/ - -# - name: Apply Patches -# run: | -# python -m pip install "patch==1.*" -# cd src -# python -m patch .patch/b622cc5ea770f866c1e373185a9e389c04bdb54c.patch -# python -m patch .patch/0001-Bump-setup.py.patch + # Hack: cmake --trace-expand + - name: Download Patch 2/2 + uses: suisei-cn/actions-download-file@v1 + id: setupversion + with: + url: "https://gist.githubusercontent.com/ax3l/9b15c5bc73d7f3c1c66784187a016474/raw/e31dd54498c9c8fc465a7cbb6f900f768b9d0c0b/0001-Hack-setup.py-CMake-trace-expand.patch" + target: src/.patch/ + + - name: Apply Patches + run: | + python -m pip install "patch==1.*" + cd src + python -m patch .patch/0001-Hack-setup.py-CMake-trace-expand.patch - name: Build wheel env: From 69565d7bc60d9b8bab67816c0c969e90610206c2 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 26 Jul 2023 23:45:06 -0700 Subject: [PATCH 40/54] Cleanup: cat adios2-config.cmake --- library_builders.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/library_builders.sh b/library_builders.sh index 1aa462c93e..59e1a809c2 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -166,8 +166,6 @@ function build_adios2 { rm -rf build-adios2 - cat /usr/local/lib/cmake/adios2/adios2-config.cmake - touch adios2-stamp } From 09a93e4c146b69ea54ee39cf1bc47bc986d3a682 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 27 Jul 2023 11:58:41 -0700 Subject: [PATCH 41/54] [Travis] PPC64le: `travis_wait 30` Takes longer now w/o output to the terminal than the default wait time of 10min. https://docs.travis-ci.com/user/common-build-problems/#build-times-out-because-no-output-was-received --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 448256570a..e823c97de0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -155,7 +155,7 @@ install: script: - cd src - - python3 -m cibuildwheel --output-dir ../wheelhouse + - travis_wait 30 python3 -m cibuildwheel --output-dir ../wheelhouse - cd .. deploy: From 1936444d97b21fedbe2029f895e7e3adc70bbeef Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 27 Jul 2023 16:57:07 -0700 Subject: [PATCH 42/54] [ADIOS2] CMake --trace-expand --- library_builders.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library_builders.sh b/library_builders.sh index 59e1a809c2..3b002a4f68 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -137,7 +137,7 @@ function build_adios2 { cd build-adios2 PY_BIN=$(which python3) CMAKE_BIN="$(${PY_BIN} -m pip show cmake 2>/dev/null | grep Location | cut -d' ' -f2)/cmake/data/bin/" - PATH=${CMAKE_BIN}:${PATH} cmake \ + PATH=${CMAKE_BIN}:${PATH} cmake --trace-expand \ -DBUILD_SHARED_LIBS=OFF \ -DBUILD_TESTING=OFF \ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ From 03bcbbdc87edcd1460d6f4196f3b23bd29434e0d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 27 Jul 2023 23:41:20 -0700 Subject: [PATCH 43/54] Undo Trace Expand Hacks --- .github/workflows/build.yml | 24 ++++++++++++------------ library_builders.sh | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2c86629ceb..cfb6840288 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -87,18 +87,18 @@ jobs: python -m pip install cibuildwheel==2.12.1 # Hack: cmake --trace-expand - - name: Download Patch 2/2 - uses: suisei-cn/actions-download-file@v1 - id: setupversion - with: - url: "https://gist.githubusercontent.com/ax3l/9b15c5bc73d7f3c1c66784187a016474/raw/e31dd54498c9c8fc465a7cbb6f900f768b9d0c0b/0001-Hack-setup.py-CMake-trace-expand.patch" - target: src/.patch/ - - - name: Apply Patches - run: | - python -m pip install "patch==1.*" - cd src - python -m patch .patch/0001-Hack-setup.py-CMake-trace-expand.patch + #- name: Download Patch 2/2 + # uses: suisei-cn/actions-download-file@v1 + # id: setupversion + # with: + # url: "https://gist.githubusercontent.com/ax3l/9b15c5bc73d7f3c1c66784187a016474/raw/e31dd54498c9c8fc465a7cbb6f900f768b9d0c0b/0001-Hack-setup.py-CMake-trace-expand.patch" + # target: src/.patch/ + + #- name: Apply Patches + # run: | + # python -m pip install "patch==1.*" + # cd src + # python -m patch .patch/0001-Hack-setup.py-CMake-trace-expand.patch - name: Build wheel env: diff --git a/library_builders.sh b/library_builders.sh index 3b002a4f68..59e1a809c2 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -137,7 +137,7 @@ function build_adios2 { cd build-adios2 PY_BIN=$(which python3) CMAKE_BIN="$(${PY_BIN} -m pip show cmake 2>/dev/null | grep Location | cut -d' ' -f2)/cmake/data/bin/" - PATH=${CMAKE_BIN}:${PATH} cmake --trace-expand \ + PATH=${CMAKE_BIN}:${PATH} cmake \ -DBUILD_SHARED_LIBS=OFF \ -DBUILD_TESTING=OFF \ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ From 31c9ae3ba63e93ff7ca1d2cefd3408e4effdd151 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 28 Jul 2023 11:53:34 -0700 Subject: [PATCH 44/54] [Travis] PPC64le: `travis_wait 45` --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e823c97de0..90438a8484 100644 --- a/.travis.yml +++ b/.travis.yml @@ -155,7 +155,7 @@ install: script: - cd src - - travis_wait 30 python3 -m cibuildwheel --output-dir ../wheelhouse + - travis_wait 45 python3 -m cibuildwheel --output-dir ../wheelhouse - cd .. deploy: From a474a6101e0b38cceee8a9c8f3651653f8bd1451 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 28 Jul 2023 14:22:10 -0700 Subject: [PATCH 45/54] ADIOS2: w/ SST --- library_builders.sh | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/library_builders.sh b/library_builders.sh index 59e1a809c2..1462f781aa 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -111,17 +111,6 @@ function build_adios2 { tar -xzf adios2*.tar.gz rm adios2*.tar.gz - # DILL macOS arm64 or universal2 binary - # https://github.com/ornladios/ADIOS2/issues/3116 - # needs rebase (or use ADIOS2-2.8.0) - #curl -sLo dill-universal.patch \ - # https://patch-diff.githubusercontent.com/raw/ornladios/ADIOS2/pull/3118.patch - #python3 -m patch -p 1 -d ADIOS2-2.7.1 dill-universal.patch - ADIOS2_USE_SST=ON - if [[ "${CMAKE_OSX_ARCHITECTURES-}" == "arm64" ]]; then - ADIOS2_USE_SST=OFF - fi - # ZPL issues on non-Linux platforms if [ "$(uname -s)" = "Linux" ] then @@ -149,7 +138,7 @@ function build_adios2 { -DADIOS2_USE_MHS=OFF \ -DADIOS2_USE_MPI=OFF \ -DADIOS2_USE_PNG=OFF \ - -DADIOS2_USE_SST=${ADIOS2_USE_SST} \ + -DADIOS2_USE_SST=ON \ -DADIOS2_USE_ZFP=ON \ -DADIOS2_RUN_INSTALL_TEST=OFF \ -DEVPATH_USE_ZPL_ENET=${EVPATH_ZPL} \ From 8fd3271219070278402f13c56b0f5f775738e7af Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 28 Jul 2023 14:33:41 -0700 Subject: [PATCH 46/54] Blosc2: CMake Config PR Merged --- library_builders.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library_builders.sh b/library_builders.sh index 1462f781aa..d8430ba6dd 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -208,7 +208,7 @@ function build_blosc2 { #curl -sLo blosc2-v2.10.0.tar.gz \ curl -sLo blosc2-topic-cmake-install-targets.tar.gz \ - https://github.com/ax3l/c-blosc2/archive/refs/heads/topic-cmake-install-targets.tar.gz + https://github.com/Blosc/c-blosc2/archive/d510951c492db9f09ab9171675c3dddbf8ffd4fa.tar.gz file blosc2*.tar.gz tar -xzf blosc2*.tar.gz rm blosc2*.tar.gz From 8e1b1cac0bdf80e9f739f0458c562e5a66422372 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sun, 30 Jul 2023 20:44:14 -0700 Subject: [PATCH 47/54] macOS (x86): 10.15 -> 11.0 The macOS 10.15 build images on GH Actions are gone, so we remove this EOL platform for good now. --- .github/workflows/build.yml | 6 +++--- library_builders.sh | 11 ----------- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cfb6840288..e28902bfc9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,10 +40,10 @@ jobs: CMAKE_GENERATOR: "Visual Studio 16 2019" CMAKE_GENERATOR_PLATFORM: "Win32" - - os: macos-10.15 + - os: macos-11 arch: "x86_64" env: - MACOSX_DEPLOYMENT_TARGET: 10.15 + MACOSX_DEPLOYMENT_TARGET: 11.0 # Apple Silicon M1/arm64/aarch64 builds: # https://cibuildwheel.readthedocs.io/en/stable/faq/#apple-silicon @@ -117,7 +117,7 @@ jobs: CIBW_ENVIRONMENT_WINDOWS: HDF5_USE_STATIC_LIBRARIES='ON' ZLIB_USE_STATIC_LIBS='ON' openPMD_CMAKE_openPMD_USE_HDF5='ON' openPMD_CMAKE_openPMD_USE_ADIOS2='ON' CMAKE_PREFIX_PATH='C:/Program Files (x86)/ADIOS2;C:/Program Files (x86)/blosc;C:/Program Files (x86)/blosc2;C:/Program Files (x86)/HDF5;C:/Program Files (x86)/ZFP;C:/Program Files (x86)/zlib' # C++17 support in macOS 10.13+ (partial) and 10.14+ (std::visit) and 10.15+ (std::filesystem::path) # https://cibuildwheel.readthedocs.io/en/stable/cpp_standards/#macos-and-deployment-target-versions - # arm64 Python interpreters are built with 11.0 + # arm64/aarch64 (M1/M2) requires 11.0+ MACOSX_DEPLOYMENT_TARGET: "${{ matrix.env.MACOSX_DEPLOYMENT_TARGET }}" CMAKE_OSX_ARCHITECTURES: "${{ matrix.env.CMAKE_OSX_ARCHITECTURES }}" # Windows cross-compilation diff --git a/library_builders.sh b/library_builders.sh index d8430ba6dd..f29f0c50a3 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -111,16 +111,6 @@ function build_adios2 { tar -xzf adios2*.tar.gz rm adios2*.tar.gz - # ZPL issues on non-Linux platforms - if [ "$(uname -s)" = "Linux" ] - then - EVPATH_ZPL="ON" - else - # ZPL in EVPATH disabled because it does not build with older macOS - # https://github.com/GTkorvo/evpath/issues/47 - EVPATH_ZPL="OFF" - fi - # build mkdir build-adios2 cd build-adios2 @@ -141,7 +131,6 @@ function build_adios2 { -DADIOS2_USE_SST=ON \ -DADIOS2_USE_ZFP=ON \ -DADIOS2_RUN_INSTALL_TEST=OFF \ - -DEVPATH_USE_ZPL_ENET=${EVPATH_ZPL} \ -DHDF5_USE_STATIC_LIBRARIES:BOOL=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DCMAKE_DISABLE_FIND_PACKAGE_LibFFI=TRUE \ From bde03770e9187f16bfdb110687b0d8d095f22d3d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 8 Aug 2023 10:38:41 -0700 Subject: [PATCH 48/54] Windows: Skip Building Blosc1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Franz Pöschel --- library_builders.bat | 1 - 1 file changed, 1 deletion(-) diff --git a/library_builders.bat b/library_builders.bat index 5d013142df..90a5a01bc1 100644 --- a/library_builders.bat +++ b/library_builders.bat @@ -243,7 +243,6 @@ call :build_zlib :: build_bzip2 :: build_szip call :build_zfp -call :build_blosc call :build_blosc2 call :build_hdf5 call :build_adios2 From 0c7dc37925b83789460a98506e2c9a6af5ee5c96 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 9 Aug 2023 15:49:40 -0700 Subject: [PATCH 49/54] Python: 3.8+ Skip older Python releases, 3.7 is EOL since last month. --- .github/workflows/build.yml | 4 ++-- .travis.yml | 27 ++++++--------------------- 2 files changed, 8 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e28902bfc9..2f5f020be4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -79,7 +79,7 @@ jobs: - uses: actions/setup-python@v1 name: Install Python with: - python-version: '3.7' + python-version: '3.8' - name: Install cibuildwheel run: | @@ -106,7 +106,7 @@ jobs: # https://github.com/pypa/manylinux/issues/899 CIBW_SKIP: "pp*-manylinux*" CIBW_ARCHS: "${{ matrix.arch }}" - CIBW_PROJECT_REQUIRES_PYTHON: ">=3.7" + CIBW_PROJECT_REQUIRES_PYTHON: ">=3.8" # Install dependencies CIBW_BEFORE_BUILD_LINUX: bash -x .github/library_builders.sh CIBW_BEFORE_BUILD_MACOS: bash -x .github/library_builders.sh diff --git a/.travis.yml b/.travis.yml index 90438a8484..a70ab7d609 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ env: global: - OPENPMD_GIT_REF="0.15.1" - - CIBW_PROJECT_REQUIRES_PYTHON=">=3.7" + - CIBW_PROJECT_REQUIRES_PYTHON=">=3.8" # Install dependencies on Linux and OSX - CIBW_BEFORE_BUILD="bash -x .github/library_builders.sh" # for the openPMD-api build, CMake shall search for @@ -28,17 +28,17 @@ jobs: arch: arm64 dist: focal env: - - CIBW_BUILD="cp37-manylinux_aarch64 cp38-manylinux_aarch64" + - CIBW_BUILD="cp38-manylinux_aarch64 cp39-manylinux_aarch64" - services: docker arch: arm64 dist: focal env: - - CIBW_BUILD="cp39-manylinux_aarch64 cp310-manylinux_aarch64" + - CIBW_BUILD="cp310-manylinux_aarch64 cp311-manylinux_aarch64" - services: docker arch: arm64 dist: focal env: - - CIBW_BUILD="cp311-manylinux_aarch64 pp37-manylinux_aarch64" + - CIBW_BUILD="pp310-manylinux_aarch64" - services: docker arch: arm64 dist: focal @@ -49,24 +49,14 @@ jobs: arch: arm64 dist: focal env: - - CIBW_BUILD="cp37-musllinux_aarch64 cp38-musllinux_aarch64" + - CIBW_BUILD="cp38-musllinux_aarch64 cp39-musllinux_aarch64" - services: docker arch: arm64 dist: focal env: - - CIBW_BUILD="cp39-musllinux_aarch64 cp310-musllinux_aarch64" - - services: docker - arch: arm64 - dist: focal - env: - - CIBW_BUILD="cp311-musllinux_aarch64" + - CIBW_BUILD="cp310-musllinux_aarch64 cp311-musllinux_aarch64" # perform a linux PPC64LE build - - services: docker - arch: ppc64le - dist: focal - env: - - CIBW_BUILD="cp37-manylinux_ppc64le" - services: docker arch: ppc64le dist: focal @@ -87,11 +77,6 @@ jobs: dist: focal env: - CIBW_BUILD="cp311-manylinux_ppc64le" - - services: docker - arch: ppc64le - dist: focal - env: - - CIBW_BUILD="cp37-musllinux_ppc64le" - services: docker arch: ppc64le dist: focal From f7c01018a35f9fd038199f2dc0533740524ed16f Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 16 Aug 2023 16:41:16 -0400 Subject: [PATCH 50/54] [Patch] ADIOS2: Remove FindBlosc2.cmake --- library_builders.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/library_builders.sh b/library_builders.sh index f29f0c50a3..ff219844ef 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -140,6 +140,10 @@ function build_adios2 { make -j${CPU_COUNT} make install + + # CMake Config package of C-Blosc 2.10.1+ only + rm -rf ${BUILD_PREFIX}/lib*/cmake/adios2/FindBlosc2.cmake + cd - rm -rf build-adios2 From 7caefd6e60f83c2abd7e72528627c6f49477e044 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 16 Aug 2023 16:43:51 -0400 Subject: [PATCH 51/54] C-Blosc 2.10.1 --- library_builders.bat | 25 +++++-------------------- library_builders.sh | 5 ++--- 2 files changed, 7 insertions(+), 23 deletions(-) diff --git a/library_builders.bat b/library_builders.bat index 90a5a01bc1..8bd96cb71f 100644 --- a/library_builders.bat +++ b/library_builders.bat @@ -92,26 +92,11 @@ exit /b 0 :build_blosc2 if exist blosc2-stamp exit /b 0 - curl -sLo blosc2-2.9.3.zip ^ - https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.9.3.zip - powershell Expand-Archive blosc2-2.9.3.zip -DestinationPath dep-blosc2 - - :: https://github.com/Blosc/c-blosc2/pull/525 - curl -sLo c-blosc2-cmake.patch ^ - https://patch-diff.githubusercontent.com/raw/Blosc/c-blosc2/pull/525.patch - python -m patch -p 1 -d dep-blosc2/c-blosc2-2.9.3 c-blosc2-cmake.patch - - :: https://github.com/Blosc/c-blosc2/issues/526 - curl -sLo c-blosc2-cxx20.patch ^ - https://github.com/Blosc/c-blosc2/pull/527.patch - python -m patch -p 1 -d dep-blosc2/c-blosc2-2.9.3 c-blosc2-cxx20.patch - - :: https://github.com/Blosc/c-blosc2/pull/529 - curl -sLo c-blosc2-external-zlib.patch ^ - https://github.com/Blosc/c-blosc2/pull/529.patch - python -m patch -p 1 -d dep-blosc2/c-blosc2-2.9.3 c-blosc2-external-zlib.patch - - cmake -S dep-blosc2/c-blosc2-2.9.3 -B build-blosc2 ^ + curl -sLo blosc2-2.10.1.zip ^ + https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.10.1.zip + powershell Expand-Archive blosc2-2.10.1.zip -DestinationPath dep-blosc2 + + cmake -S dep-blosc2/c-blosc2-2.10.1 -B build-blosc2 ^ -DCMAKE_BUILD_TYPE=Release ^ -DBUILD_SHARED=OFF ^ -DBUILD_STATIC=ON ^ diff --git a/library_builders.sh b/library_builders.sh index ff219844ef..590acfc881 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -199,9 +199,8 @@ function build_blosc { function build_blosc2 { if [ -e blosc-stamp2 ]; then return; fi - #curl -sLo blosc2-v2.10.0.tar.gz \ - curl -sLo blosc2-topic-cmake-install-targets.tar.gz \ - https://github.com/Blosc/c-blosc2/archive/d510951c492db9f09ab9171675c3dddbf8ffd4fa.tar.gz + curl -sLo blosc2-v2.10.1.tar.gz \ + https://github.com/Blosc/c-blosc2/archive/refs/tags/v2.10.1.tar.gz file blosc2*.tar.gz tar -xzf blosc2*.tar.gz rm blosc2*.tar.gz From 23b66c0b51869b823abf25053840a5b37f2c1b94 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 16 Aug 2023 16:47:11 -0400 Subject: [PATCH 52/54] Win: ADIOS 2.9.1 --- library_builders.bat | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/library_builders.bat b/library_builders.bat index 8bd96cb71f..2eebba39ea 100644 --- a/library_builders.bat +++ b/library_builders.bat @@ -16,16 +16,11 @@ exit /b 0 :build_adios2 if exist adios2-stamp exit /b 0 - curl -sLo adios2-2.9.0.zip ^ - https://github.com/ornladios/ADIOS2/archive/v2.9.0.zip - powershell Expand-Archive adios2-2.9.0.zip -DestinationPath dep-adios2 + curl -sLo adios2-2.9.1.zip ^ + https://github.com/ornladios/ADIOS2/archive/v2.9.1.zip + powershell Expand-Archive adios2-2.9.1.zip -DestinationPath dep-adios2 - :: https://github.com/ornladios/ADIOS2/issues/3680#issuecomment-1615308336 - curl -sLo adios2-cblosc-stdmin.patch ^ - https://patch-diff.githubusercontent.com/raw/ornladios/ADIOS2/pull/3681.patch - python -m patch -p 1 -d dep-adios2/ADIOS2-2.9.0 adios2-cblosc-stdmin.patch - - cmake -S dep-adios2/ADIOS2-2.9.0 -B build-adios2 ^ + cmake -S dep-adios2/ADIOS2-2.9.1 -B build-adios2 ^ -DCMAKE_BUILD_TYPE=Release ^ -DBUILD_SHARED_LIBS=OFF ^ -DBUILD_TESTING=OFF ^ From 2c97be6531c688161c4da6e89c94e0c417122fb0 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 16 Aug 2023 16:50:33 -0400 Subject: [PATCH 53/54] ADIOS2_Blosc2_PREFER_SHARED=OFF https://github.com/ornladios/ADIOS2/pull/3715#discussion_r1296413167 --- library_builders.bat | 1 + library_builders.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/library_builders.bat b/library_builders.bat index 2eebba39ea..653c69a3ee 100644 --- a/library_builders.bat +++ b/library_builders.bat @@ -26,6 +26,7 @@ exit /b 0 -DBUILD_TESTING=OFF ^ -DADIOS2_USE_MPI=OFF ^ -DADIOS2_BUILD_EXAMPLES=OFF ^ + -DADIOS2_Blosc2_PREFER_SHARED=OFF ^ -DADIOS2_USE_Blosc2=ON ^ -DADIOS2_USE_BZip2=OFF ^ -DADIOS2_USE_Fortran=OFF ^ diff --git a/library_builders.sh b/library_builders.sh index 590acfc881..ae036d87a3 100644 --- a/library_builders.sh +++ b/library_builders.sh @@ -121,6 +121,7 @@ function build_adios2 { -DBUILD_TESTING=OFF \ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ -DADIOS2_BUILD_EXAMPLES=OFF \ + -DADIOS2_Blosc2_PREFER_SHARED=OFF \ -DADIOS2_USE_BZip2=OFF \ -DADIOS2_USE_Blosc2=ON \ -DADIOS2_USE_Fortran=OFF \ From 9352359ca8878135fc9294a20c518f1601556d12 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 16 Aug 2023 20:22:51 -0400 Subject: [PATCH 54/54] cibuildwheel 2.12.1 -> 2.14.1 For `pp310-manylinux_aarch64` --- .github/workflows/build.yml | 2 +- .travis.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2f5f020be4..cc9ad207a9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -84,7 +84,7 @@ jobs: - name: Install cibuildwheel run: | python -m pip install --upgrade pip setuptools wheel - python -m pip install cibuildwheel==2.12.1 + python -m pip install cibuildwheel==2.14.1 # Hack: cmake --trace-expand #- name: Download Patch 2/2 diff --git a/.travis.yml b/.travis.yml index a70ab7d609..74bb660581 100644 --- a/.travis.yml +++ b/.travis.yml @@ -116,7 +116,7 @@ install: - git clone --branch ${OPENPMD_GIT_REF} --depth 1 https://github.com/openPMD/openPMD-api.git src - cp library_builders.sh src/.github/ - python -m pip install --upgrade pip setuptools wheel - - python -m pip install cibuildwheel==2.12.1 + - python -m pip install cibuildwheel==2.14.1 # twine & cryptography: see # https://github.com/scikit-build/cmake-python-distributions/blob/4730aeee240917303f293dffc89a8d8d5a4787c4/requirements-deploy.txt # https://github.com/pyca/cryptography/issues/6086