From 6e5baa3290dbc57e99f390700a9ec44f7f60166b Mon Sep 17 00:00:00 2001 From: h-vetinari Date: Thu, 28 Sep 2023 22:18:07 +1100 Subject: [PATCH] GH-37621: [Packaging][Conda] Sync conda recipes with feedstocks (#37624) Syncing after the release of 13.0.0 + a couple of migrations (state as of https://github.com/conda-forge/arrow-cpp-feedstock/pull/1168 & https://github.com/conda-forge/r-arrow-feedstock/pull/68) Relevant updates: * we're not building twice for different protobuf versions anymore * new abseil version (fixes #36908) * we've finally upgraded the aws-sdk to 1.11 * the default R versions (on unix) are now 4.2 & 4.3. Also some further hardening of the activation scripts & clean-ups for dependencies & test skips. * Closes: #37621 Lead-authored-by: H. Vetinari Co-authored-by: h-vetinari Signed-off-by: Joris Van den Bossche --- .../linux_64_cuda_compiler_version11.2.yaml | 20 +++--- .../linux_64_cuda_compiler_versionNone.yaml | 20 +++--- ...nux_aarch64_cuda_compiler_version11.2.yaml | 20 +++--- ...nux_aarch64_cuda_compiler_versionNone.yaml | 20 +++--- ...nux_ppc64le_cuda_compiler_version11.2.yaml | 20 +++--- ...nux_ppc64le_cuda_compiler_versionNone.yaml | 20 +++--- .../conda-recipes/.ci_support/osx_64_.yaml | 22 +++---- .../conda-recipes/.ci_support/osx_arm64_.yaml | 20 +++--- ...r_base4.1.yaml => linux_64_r_base4.3.yaml} | 2 +- ...e4.1.yaml => linux_aarch64_r_base4.3.yaml} | 2 +- ...4_r_base4.1.yaml => osx_64_r_base4.3.yaml} | 2 +- ..._base4.1.yaml => osx_arm64_r_base4.3.yaml} | 2 +- .../win_64_cuda_compiler_version11.2.yaml | 22 +++---- .../win_64_cuda_compiler_versionNone.yaml | 22 +++---- dev/tasks/conda-recipes/arrow-cpp/activate.sh | 17 +++-- .../conda-recipes/arrow-cpp/build-arrow.sh | 4 +- .../conda-recipes/arrow-cpp/build-pyarrow.sh | 4 ++ dev/tasks/conda-recipes/arrow-cpp/meta.yaml | 65 ++++++++----------- dev/tasks/tasks.yml | 24 +++---- 19 files changed, 151 insertions(+), 177 deletions(-) rename dev/tasks/conda-recipes/.ci_support/r/{linux_64_r_base4.1.yaml => linux_64_r_base4.3.yaml} (98%) rename dev/tasks/conda-recipes/.ci_support/r/{linux_aarch64_r_base4.1.yaml => linux_aarch64_r_base4.3.yaml} (98%) rename dev/tasks/conda-recipes/.ci_support/r/{osx_64_r_base4.1.yaml => osx_64_r_base4.3.yaml} (98%) rename dev/tasks/conda-recipes/.ci_support/r/{osx_arm64_r_base4.1.yaml => osx_arm64_r_base4.3.yaml} (98%) diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version11.2.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version11.2.yaml index 1cdcec199e7ba..042e2364d1c49 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version11.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version11.2.yaml @@ -1,7 +1,7 @@ aws_crt_cpp: -- 0.20.3 +- 0.23.1 aws_sdk_cpp: -- 1.10.57 +- 1.11.156 bzip2: - '1' c_compiler: @@ -33,20 +33,18 @@ glog: google_cloud_cpp: - '2.12' libabseil: -- '20230125' +- '20230802' libgrpc: -- '1.54' -- '1.56' +- '1.57' libprotobuf: -- '3.21' -- 4.23.3 +- 4.23.4 lz4_c: - 1.9.3 numpy: -- '1.21' +- '1.22' - '1.23' -- '1.21' -- '1.21' +- '1.22' +- '1.22' openssl: - '3' orc: @@ -67,7 +65,7 @@ snappy: target_platform: - linux-64 thrift_cpp: -- 0.18.1 +- 0.19.0 ucx: - 1.14.0 zip_keys: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml index 5be5b58a73932..9885e6db38cd7 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNone.yaml @@ -1,7 +1,7 @@ aws_crt_cpp: -- 0.20.3 +- 0.23.1 aws_sdk_cpp: -- 1.10.57 +- 1.11.156 bzip2: - '1' c_compiler: @@ -33,20 +33,18 @@ glog: google_cloud_cpp: - '2.12' libabseil: -- '20230125' +- '20230802' libgrpc: -- '1.54' -- '1.56' +- '1.57' libprotobuf: -- '3.21' -- 4.23.3 +- 4.23.4 lz4_c: - 1.9.3 numpy: -- '1.21' +- '1.22' - '1.23' -- '1.21' -- '1.21' +- '1.22' +- '1.22' openssl: - '3' orc: @@ -67,7 +65,7 @@ snappy: target_platform: - linux-64 thrift_cpp: -- 0.18.1 +- 0.19.0 ucx: - 1.14.0 zip_keys: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_version11.2.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_version11.2.yaml index 1677b03564c08..788b584504ec4 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_version11.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_version11.2.yaml @@ -1,9 +1,9 @@ BUILD: - aarch64-conda_cos7-linux-gnu aws_crt_cpp: -- 0.20.3 +- 0.23.1 aws_sdk_cpp: -- 1.10.57 +- 1.11.156 bzip2: - '1' c_compiler: @@ -37,20 +37,18 @@ glog: google_cloud_cpp: - '2.12' libabseil: -- '20230125' +- '20230802' libgrpc: -- '1.54' -- '1.56' +- '1.57' libprotobuf: -- '3.21' -- 4.23.3 +- 4.23.4 lz4_c: - 1.9.3 numpy: -- '1.21' +- '1.22' - '1.23' -- '1.21' -- '1.21' +- '1.22' +- '1.22' openssl: - '3' orc: @@ -71,7 +69,7 @@ snappy: target_platform: - linux-aarch64 thrift_cpp: -- 0.18.1 +- 0.19.0 ucx: - 1.14.0 zip_keys: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml index 88fdf1254e661..a1e4b8571abaf 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_aarch64_cuda_compiler_versionNone.yaml @@ -1,9 +1,9 @@ BUILD: - aarch64-conda_cos7-linux-gnu aws_crt_cpp: -- 0.20.3 +- 0.23.1 aws_sdk_cpp: -- 1.10.57 +- 1.11.156 bzip2: - '1' c_compiler: @@ -37,20 +37,18 @@ glog: google_cloud_cpp: - '2.12' libabseil: -- '20230125' +- '20230802' libgrpc: -- '1.54' -- '1.56' +- '1.57' libprotobuf: -- '3.21' -- 4.23.3 +- 4.23.4 lz4_c: - 1.9.3 numpy: -- '1.21' +- '1.22' - '1.23' -- '1.21' -- '1.21' +- '1.22' +- '1.22' openssl: - '3' orc: @@ -71,7 +69,7 @@ snappy: target_platform: - linux-aarch64 thrift_cpp: -- 0.18.1 +- 0.19.0 ucx: - 1.14.0 zip_keys: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_version11.2.yaml b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_version11.2.yaml index 3585db7b99baa..e21c4cbe853f8 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_version11.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_version11.2.yaml @@ -1,7 +1,7 @@ aws_crt_cpp: -- 0.20.3 +- 0.23.1 aws_sdk_cpp: -- 1.10.57 +- 1.11.156 bzip2: - '1' c_compiler: @@ -33,20 +33,18 @@ glog: google_cloud_cpp: - '2.12' libabseil: -- '20230125' +- '20230802' libgrpc: -- '1.54' -- '1.56' +- '1.57' libprotobuf: -- '3.21' -- 4.23.3 +- 4.23.4 lz4_c: - 1.9.3 numpy: -- '1.21' +- '1.22' - '1.23' -- '1.21' -- '1.21' +- '1.22' +- '1.22' openssl: - '3' orc: @@ -67,7 +65,7 @@ snappy: target_platform: - linux-ppc64le thrift_cpp: -- 0.18.1 +- 0.19.0 ucx: - 1.14.0 zip_keys: diff --git a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml index c13a522254286..89f1049ebdd84 100644 --- a/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml +++ b/dev/tasks/conda-recipes/.ci_support/linux_ppc64le_cuda_compiler_versionNone.yaml @@ -1,7 +1,7 @@ aws_crt_cpp: -- 0.20.3 +- 0.23.1 aws_sdk_cpp: -- 1.10.57 +- 1.11.156 bzip2: - '1' c_compiler: @@ -33,20 +33,18 @@ glog: google_cloud_cpp: - '2.12' libabseil: -- '20230125' +- '20230802' libgrpc: -- '1.54' -- '1.56' +- '1.57' libprotobuf: -- '3.21' -- 4.23.3 +- 4.23.4 lz4_c: - 1.9.3 numpy: -- '1.21' +- '1.22' - '1.23' -- '1.21' -- '1.21' +- '1.22' +- '1.22' openssl: - '3' orc: @@ -67,7 +65,7 @@ snappy: target_platform: - linux-ppc64le thrift_cpp: -- 0.18.1 +- 0.19.0 ucx: - 1.14.0 zip_keys: diff --git a/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml b/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml index dd4a230760ef2..2a5f8c5b36bd3 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_64_.yaml @@ -1,9 +1,9 @@ MACOSX_DEPLOYMENT_TARGET: -- '10.9' +- '10.13' aws_crt_cpp: -- 0.20.3 +- 0.23.1 aws_sdk_cpp: -- 1.10.57 +- 1.11.156 bzip2: - '1' c_compiler: @@ -27,22 +27,20 @@ glog: google_cloud_cpp: - '2.12' libabseil: -- '20230125' +- '20230802' libgrpc: -- '1.54' -- '1.56' +- '1.57' libprotobuf: -- '3.21' -- 4.23.3 +- 4.23.4 lz4_c: - 1.9.3 macos_machine: - x86_64-apple-darwin13.4.0 numpy: -- '1.21' +- '1.22' - '1.23' -- '1.21' -- '1.21' +- '1.22' +- '1.22' openssl: - '3' orc: @@ -63,7 +61,7 @@ snappy: target_platform: - osx-64 thrift_cpp: -- 0.18.1 +- 0.19.0 zip_keys: - - c_compiler_version - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml b/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml index 6a6713a54fe86..211b71226cae8 100644 --- a/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml +++ b/dev/tasks/conda-recipes/.ci_support/osx_arm64_.yaml @@ -1,9 +1,9 @@ MACOSX_DEPLOYMENT_TARGET: - '11.0' aws_crt_cpp: -- 0.20.3 +- 0.23.1 aws_sdk_cpp: -- 1.10.57 +- 1.11.156 bzip2: - '1' c_compiler: @@ -27,22 +27,20 @@ glog: google_cloud_cpp: - '2.12' libabseil: -- '20230125' +- '20230802' libgrpc: -- '1.54' -- '1.56' +- '1.57' libprotobuf: -- '3.21' -- 4.23.3 +- 4.23.4 lz4_c: - 1.9.3 macos_machine: - arm64-apple-darwin20.0.0 numpy: -- '1.21' +- '1.22' - '1.23' -- '1.21' -- '1.21' +- '1.22' +- '1.22' openssl: - '3' orc: @@ -63,7 +61,7 @@ snappy: target_platform: - osx-arm64 thrift_cpp: -- 0.18.1 +- 0.19.0 zip_keys: - - c_compiler_version - cxx_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.3.yaml similarity index 98% rename from dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml rename to dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.3.yaml index e63767cbe9771..a4d06c9f20cdd 100644 --- a/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml +++ b/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.3.yaml @@ -19,7 +19,7 @@ pin_run_as_build: min_pin: x.x max_pin: x.x r_base: -- '4.1' +- '4.3' target_platform: - linux-64 zip_keys: diff --git a/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.3.yaml similarity index 98% rename from dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml rename to dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.3.yaml index 2b80b020fdc0b..028b190bb1ef5 100644 --- a/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.1.yaml +++ b/dev/tasks/conda-recipes/.ci_support/r/linux_aarch64_r_base4.3.yaml @@ -23,7 +23,7 @@ pin_run_as_build: min_pin: x.x max_pin: x.x r_base: -- '4.1' +- '4.3' target_platform: - linux-aarch64 zip_keys: diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.3.yaml similarity index 98% rename from dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml rename to dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.3.yaml index 6be6c2f5462c5..7b8b62d8e00bb 100644 --- a/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml +++ b/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.3.yaml @@ -19,7 +19,7 @@ pin_run_as_build: min_pin: x.x max_pin: x.x r_base: -- '4.1' +- '4.3' target_platform: - osx-64 zip_keys: diff --git a/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.1.yaml b/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.3.yaml similarity index 98% rename from dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.1.yaml rename to dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.3.yaml index 0ce856fcccf5c..a8e8aab83d598 100644 --- a/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.1.yaml +++ b/dev/tasks/conda-recipes/.ci_support/r/osx_arm64_r_base4.3.yaml @@ -19,7 +19,7 @@ pin_run_as_build: min_pin: x.x max_pin: x.x r_base: -- '4.1' +- '4.3' target_platform: - osx-arm64 zip_keys: diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version11.2.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version11.2.yaml index f75d92e276d9e..32da33c072019 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version11.2.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_version11.2.yaml @@ -1,11 +1,9 @@ aws_crt_cpp: -- 0.20.3 +- 0.23.1 aws_sdk_cpp: -- 1.10.57 +- 1.11.156 bzip2: - '1' -c_ares: -- '1' c_compiler: - vs2019 channel_sources: @@ -27,24 +25,22 @@ glog: google_cloud_cpp: - '2.12' libabseil: -- '20230125' +- '20230802' libcrc32c: - '1.1' libcurl: - '8' libgrpc: -- '1.54' -- '1.56' +- '1.57' libprotobuf: -- '3.21' -- 4.23.3 +- 4.23.4 lz4_c: - 1.9.3 numpy: -- '1.21' +- '1.22' - '1.23' -- '1.21' -- '1.21' +- '1.22' +- '1.22' openssl: - '3' orc: @@ -65,7 +61,7 @@ snappy: target_platform: - win-64 thrift_cpp: -- 0.18.1 +- 0.19.0 zip_keys: - - cuda_compiler - cuda_compiler_version diff --git a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml index 6d8fb15b15a2a..6a33b86b9d65e 100644 --- a/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml +++ b/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNone.yaml @@ -1,11 +1,9 @@ aws_crt_cpp: -- 0.20.3 +- 0.23.1 aws_sdk_cpp: -- 1.10.57 +- 1.11.156 bzip2: - '1' -c_ares: -- '1' c_compiler: - vs2019 channel_sources: @@ -27,24 +25,22 @@ glog: google_cloud_cpp: - '2.12' libabseil: -- '20230125' +- '20230802' libcrc32c: - '1.1' libcurl: - '8' libgrpc: -- '1.54' -- '1.56' +- '1.57' libprotobuf: -- '3.21' -- 4.23.3 +- 4.23.4 lz4_c: - 1.9.3 numpy: -- '1.21' +- '1.22' - '1.23' -- '1.21' -- '1.21' +- '1.22' +- '1.22' openssl: - '3' orc: @@ -65,7 +61,7 @@ snappy: target_platform: - win-64 thrift_cpp: -- 0.18.1 +- 0.19.0 zip_keys: - - cuda_compiler - cuda_compiler_version diff --git a/dev/tasks/conda-recipes/arrow-cpp/activate.sh b/dev/tasks/conda-recipes/arrow-cpp/activate.sh index 8757612781bbe..19d037ff4127a 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/activate.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/activate.sh @@ -23,6 +23,13 @@ _la_log "Beginning libarrow activation." # where the GDB wrappers get installed _la_gdb_prefix="$CONDA_PREFIX/share/gdb/auto-load" +# If the directory is not writable, nothing can be done +if [ ! -w "$_la_gdb_prefix" ]; then + _la_log 'No rights to modify $_la_gdb_prefix, cannot create symlink!' + _la_log 'Unless you plan to use the GDB debugger with libarrow, this warning can be safely ignored.' + return +fi + # this needs to be in sync with ARROW_GDB_INSTALL_DIR in build.sh _la_placeholder="replace_this_section_with_absolute_slashed_path_to_CONDA_PREFIX" # the paths here are intentionally stacked, see #935, resp. @@ -44,7 +51,7 @@ for _la_target in "$_la_orig_install_dir/"*.py; do # If the file doesn't exist, skip this iteration of the loop. # (This happens when no files are found, in which case the # loop runs with target equal to the pattern itself.) - _la_log 'Folder $_la_orig_install_dir seems to not contain .py files, skipping' + _la_log 'Folder $_la_orig_install_dir seems to not contain .py files, skipping.' continue fi _la_symlink="$_la_symlink_dir/$(basename "$_la_target")" @@ -54,13 +61,13 @@ for _la_target in "$_la_orig_install_dir/"*.py; do _la_log 'symlink $_la_symlink already exists and points to $_la_target, skipping.' continue fi - _la_log 'Creating symlink $_la_symlink pointing to $_la_target' + _la_log 'Creating symlink $_la_symlink pointing to $_la_target.' mkdir -p "$_la_symlink_dir" || true # this check also creates the symlink; if it fails, we enter the if-branch. if ! ln -sf "$_la_target" "$_la_symlink"; then - echo -n "${BASH_SOURCE[0]} ERROR: Failed to create symlink from " - echo -n "'$_la_target' to '$_la_symlink'" - echo + echo -n "${BASH_SOURCE[0]} WARNING: Failed to create symlink from " + echo "'$_la_target' to '$_la_symlink'!" + echo "Unless you plan to use the GDB debugger with libarrow, this warning can be safely ignored." continue fi done diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh index dc588f9473870..ef0b038812a01 100755 --- a/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh @@ -30,7 +30,7 @@ fi # Enable CUDA support if [[ ! -z "${cuda_compiler_version+x}" && "${cuda_compiler_version}" != "None" ]] then - EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=ON -DCUDA_TOOLKIT_ROOT_DIR=${CUDA_HOME} -DCMAKE_LIBRARY_PATH=${CONDA_BUILD_SYSROOT}/lib" + EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=ON -DCUDAToolkit_ROOT=${CUDA_HOME} -DCMAKE_LIBRARY_PATH=${CONDA_BUILD_SYSROOT}/lib" else EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=OFF" fi @@ -43,8 +43,8 @@ if [[ "${build_platform}" != "${target_platform}" ]]; then fi EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DCLANG_EXECUTABLE=${BUILD_PREFIX}/bin/${CONDA_TOOLCHAIN_HOST}-clang" EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DLLVM_LINK_EXECUTABLE=${BUILD_PREFIX}/bin/llvm-link" + EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DARROW_JEMALLOC_LG_PAGE=16" sed -ie "s;protoc-gen-grpc.*$;protoc-gen-grpc=${BUILD_PREFIX}/bin/grpc_cpp_plugin\";g" ../src/arrow/flight/CMakeLists.txt - sed -ie 's;"--with-jemalloc-prefix\=je_arrow_";"--with-jemalloc-prefix\=je_arrow_" "--with-lg-page\=16";g' ../cmake_modules/ThirdpartyToolchain.cmake fi # disable -fno-plt, which causes problems with GCC on PPC diff --git a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh index 9c12321a1c115..f39e06874ca0e 100755 --- a/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh +++ b/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh @@ -24,6 +24,10 @@ BUILD_EXT_FLAGS="" # Enable CUDA support if [[ ! -z "${cuda_compiler_version+x}" && "${cuda_compiler_version}" != "None" ]]; then export PYARROW_WITH_CUDA=1 + if [[ "${build_platform}" != "${target_platform}" ]]; then + export CUDAToolkit_ROOT=${CUDA_HOME} + export CMAKE_LIBRARY_PATH=${CONDA_BUILD_SYSROOT}/lib + fi else export PYARROW_WITH_CUDA=0 fi diff --git a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml index fbe40af3dae01..371b62245bb72 100644 --- a/dev/tasks/conda-recipes/arrow-cpp/meta.yaml +++ b/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -61,7 +61,7 @@ outputs: build: string: h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} run_exports: - - {{ pin_subpackage("libarrow", max_pin="x.x.x") }} + - {{ pin_subpackage("libarrow", max_pin="x") }} ignore_run_exports_from: - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] # arrow only uses headers, apparently @@ -114,6 +114,8 @@ outputs: - libgrpc - libprotobuf - libutf8proc + # gandiva requires shared libllvm + - llvm # [unix] - lz4-c - nlohmann_json # gandiva depends on openssl @@ -133,8 +135,6 @@ outputs: # its host deps (which aren't yet covered above) leak into the build here - libcrc32c # [win] - libcurl # [win] - # same for libgrpc (before 1.55.0, which is coupled with libprotobuf 4.23.x) - - c-ares # [win and libprotobuf == "3.21"] run_constrained: - apache-arrow-proc =*={{ build_ext }} # make sure we don't co-install with old version of old package name @@ -198,8 +198,6 @@ outputs: requirements: host: - {{ pin_subpackage('libarrow', exact=True) }} - # avoid wrappers for different builds colliding due to identical hashes - - libprotobuf run: - {{ pin_subpackage('libarrow', exact=True) }} test: @@ -235,9 +233,7 @@ outputs: - cmake - ninja host: - # we're building for two protobuf versions, cannot pin exactly - # - {{ pin_subpackage('libarrow', exact=True) }} - - libarrow ={{ version }}=*_{{ PKG_BUILDNUM }}_{{ build_ext }} + - {{ pin_subpackage('libarrow', exact=True) }} - clangdev {{ llvm_version }} - llvmdev {{ llvm_version }} - cython <3 @@ -246,8 +242,7 @@ outputs: - setuptools - setuptools_scm <8.0.0 run: - # - {{ pin_subpackage('libarrow', exact=True) }} - - libarrow ={{ version }}=*_{{ PKG_BUILDNUM }}_{{ build_ext }} + - {{ pin_subpackage('libarrow', exact=True) }} - {{ pin_compatible('numpy') }} - python run_constrained: @@ -336,28 +331,28 @@ outputs: # crossbow CI: reduce to one python version, except on (unemulated) linux, where it's fast enough {% if linux64 or py == 311 %} - # {% if not (aarch64 or ppc64le) or py in (310, 311) %} - # only run the full test suite for one python version when in emulation (each run takes ~45min); - # there's essentially zero divergence in behaviour across python versions anyway, and otherwise - # CUDA builds for aarch/ppc consistently run out of disk space on azure for some reason + # {% if not (aarch64 or ppc64le) or py == 311 %} + # only run the full test suite for one python version when in emulation + # (each run can take up to ~45min); there's essentially zero divergence + # in behaviour across python versions anyway test: requires: - # vary protobuf version in test suite (historically, test failures only have a very - # weak dependency on python version, so we don't lose coverage by doing half & half) - - libprotobuf <4 # [py % 2 == 0] # test_cpp_extension_in_python requires a compiler - {{ compiler("cxx") }} # [linux] - # temporary pin due to missing fixture - - pytest <7.4.0 + - pytest - pytest-lazy-fixture - backports.zoneinfo # [py<39] + - boto3 - cffi - cloudpickle - cython <3 - fastparquet - fsspec - hypothesis + # currently disabled due to GH-37692 + # - minio-server - pandas + - s3fs >=2023 - scipy # these are generally (far) behind on migrating abseil/grpc/protobuf, # and using them as test dependencies blocks the migrator unnecessarily @@ -372,8 +367,8 @@ outputs: source_files: - testing/data commands: - - cd ${SP_DIR}/pyarrow/tests # [unix] - - cd %SP_DIR%\pyarrow\tests # [win] + - cd ${SP_DIR} # [unix] + - cd %SP_DIR% # [win] - export ARROW_TEST_DATA="${SRC_DIR}/testing/data" # [unix] - set "ARROW_TEST_DATA=%SRC_DIR%\testing\data" # [win] @@ -382,34 +377,26 @@ outputs: {% set tests_to_skip = tests_to_skip + " or test_cuda" %} # skip tests that raise SIGINT and crash the test suite {% set tests_to_skip = tests_to_skip + " or (test_csv and test_cancellation)" %} # [linux] - {% set tests_to_skip = tests_to_skip + " or (test_flight and test_interrupt)" %} # [linux] - # tests that may crash the agent due to out-of-bound memory writes or other risky stuff - {% set tests_to_skip = tests_to_skip + " or test_debug_memory_pool" %} # [aarch64 or ppc64le] - # cannot pass -D_LIBCPP_DISABLE_AVAILABILITY to test suite for our older macos sdk - {% set tests_to_skip = tests_to_skip + " or test_cpp_extension_in_python" %} # [osx] + # skip test that intentionally writes out of bounds and then expects no error message + {% set tests_to_skip = tests_to_skip + " or test_debug_memory_pool_disabled[system_memory_pool]" %} # [osx] # skip tests that make invalid(-for-conda) assumptions about the compilers setup {% set tests_to_skip = tests_to_skip + " or test_cython_api" %} # [unix] {% set tests_to_skip = tests_to_skip + " or test_visit_strings" %} # [unix] # skip tests that cannot succeed in emulation {% set tests_to_skip = tests_to_skip + " or test_debug_memory_pool_disabled" %} # [aarch64 or ppc64le] {% set tests_to_skip = tests_to_skip + " or test_env_var_io_thread_count" %} # [aarch64 or ppc64le] + # XMinioInvalidObjectName on osx/win: "Object name contains unsupported characters" + {% set tests_to_skip = tests_to_skip + " or test_write_to_dataset_with_partitions_s3fs" %} # [osx or win] # vvvvvvv TESTS THAT SHOULDN'T HAVE TO BE SKIPPED vvvvvvv - {% set tests_to_skip = tests_to_skip + " or test_extension_to_pandas_storage_type" %} - # segfaults on OSX: to investigate ASAP - {% set tests_to_skip = tests_to_skip + " or test_flight" %} # [osx] + # currently broken + {% set tests_to_skip = tests_to_skip + " or test_fastparquet_cross_compatibility" %} # gandiva tests are segfaulting on ppc - {% set tests_to_skip = tests_to_skip + " or test_gandiva" %} # [ppc64le] - # test failures on ppc + {% set tests_to_skip = tests_to_skip + " or test_gandiva" %} # [ppc64le] + # test failures on ppc (both failing with: Float value was truncated converting to int32) {% set tests_to_skip = tests_to_skip + " or test_safe_cast_from_float_with_nans_to_int" %} # [ppc64le] - # gandiva tests are segfaulting on ppc - {% set tests_to_skip = tests_to_skip + " or test_float_with_null_as_integer" %} # [ppc64le] - # test is broken; header is in $PREFIX, not $SP_DIR - {% set tests_to_skip = tests_to_skip + " or (test_misc and test_get_include)" %} # [unix] - # flaky tests that fail occasionally - {% set tests_to_skip = tests_to_skip + " or test_total_bytes_allocated " %} # [linux] - {% set tests_to_skip = tests_to_skip + " or test_feather_format " %} # [linux] + {% set tests_to_skip = tests_to_skip + " or test_float_with_null_as_integer" %} # [ppc64le] # ^^^^^^^ TESTS THAT SHOULDN'T HAVE TO BE SKIPPED ^^^^^^^ - - pytest -rfEs -k "not ({{ tests_to_skip }})" + - pytest pyarrow/ -rfEs -k "not ({{ tests_to_skip }})" {% endif %} about: diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 29e038a922412..859ff8ddb5b44 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -246,15 +246,15 @@ tasks: # generated and to be synced regularly from the feedstock. We have no way # yet to generate them inside the arrow repository automatically. - conda-linux-x64-cpu-r41: + conda-linux-x64-cpu-r43: ci: azure template: conda-recipes/azure.linux.yml params: config: linux_64_cuda_compiler_versionNone - r_config: linux_64_r_base4.1 + r_config: linux_64_r_base4.3 artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda + - r-arrow-{no_rc_version}-r43(h[a-z0-9]+)_0.conda conda-linux-x64-cpu-r42: ci: azure @@ -292,15 +292,15 @@ tasks: ########################### Conda Linux (aarch64) ########################### - conda-linux-aarch64-cpu-r41: + conda-linux-aarch64-cpu-r43: ci: azure template: conda-recipes/azure.linux.yml params: config: linux_aarch64_cuda_compiler_versionNone - r_config: linux_aarch64_r_base4.1 + r_config: linux_aarch64_r_base4.3 artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda + - r-arrow-{no_rc_version}-r43(h[a-z0-9]+)_0.conda conda-linux-aarch64-cpu-r42: ci: azure @@ -364,15 +364,15 @@ tasks: ############################## Conda OSX (x64) ############################## - conda-osx-x64-cpu-r41: + conda-osx-x64-cpu-r43: ci: azure template: conda-recipes/azure.osx.yml params: config: osx_64_ - r_config: osx_64_r_base4.1 + r_config: osx_64_r_base4.3 artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda + - r-arrow-{no_rc_version}-r43(h[a-z0-9]+)_0.conda conda-osx-x64-cpu-r42: ci: azure @@ -398,15 +398,15 @@ tasks: ############################# Conda OSX (arm64) ############################# - conda-osx-arm64-cpu-r41: + conda-osx-arm64-cpu-r43: ci: azure template: conda-recipes/azure.osx.yml params: config: osx_arm64_ - r_config: osx_arm64_r_base4.1 + r_config: osx_arm64_r_base4.3 artifacts: - libarrow-{no_rc_version}-(h[a-z0-9]+)_0_cpu.conda - - r-arrow-{no_rc_version}-r41(h[a-z0-9]+)_0.conda + - r-arrow-{no_rc_version}-r43(h[a-z0-9]+)_0.conda conda-osx-arm64-cpu-r42: ci: azure