Merge remote-tracking branch 'upstream/main' into apachegh-41978-pand…

…as-datetime64-unit

.github/CODEOWNERS

@@ -37,7 +37,7 @@
 /go/ @zeroshade
 /java/ @lidavidm
 /js/ @domoritz @trxcllnt
-/matlab/ @kevingurney @kou
+/matlab/ @kevingurney @kou @sgilmore10
 /python/pyarrow/_flight.pyx @lidavidm
 /python/pyarrow/**/*gandiva* @wjones127
 /r/ @paleolimbot @thisisnic

.github/workflows/cpp.yml

@@ -21,6 +21,7 @@ on:
   push:
     paths:
       - '.github/workflows/cpp.yml'
+      - 'ci/conda_env_*'
       - 'ci/docker/**'
       - 'ci/scripts/cpp_*'
       - 'ci/scripts/install_azurite.sh'
@@ -35,6 +36,7 @@ on:
   pull_request:
     paths:
       - '.github/workflows/cpp.yml'
+      - 'ci/conda_env_*'
       - 'ci/docker/**'
       - 'ci/scripts/cpp_*'
       - 'ci/scripts/install_azurite.sh'

.github/workflows/issue_bot.yml

@@ -21,7 +21,6 @@ on:
   issues:
     types:
       - opened
-      - edited
 
 permissions:
   contents: read

.github/workflows/ruby.yml

@@ -197,9 +197,7 @@ jobs:
         mingw-n-bits:
           - 64
         ruby-version:
-          # TODO: Use the latest Ruby again when we fix GH-39130.
-          # - ruby
-          - "3.1"
+          - ruby
     env:
       ARROW_BUILD_STATIC: OFF
       ARROW_BUILD_TESTS: OFF

.gitignore

@@ -102,8 +102,8 @@ __debug_bin
 .envrc
 
 # Develocity
-.mvn/.gradle-enterprise/
-.mvn/.develocity/
+java/.mvn/.gradle-enterprise/
+java/.mvn/.develocity/
 
 # rat
 filtered_rat.txt

ci/appveyor-cpp-setup.bat

@@ -66,6 +66,9 @@ set CONDA_PACKAGES=%CONDA_PACKAGES% --file=ci\conda_env_cpp.txt
 @rem Force conda to use conda-forge
 conda config --add channels conda-forge
 conda config --remove channels defaults
+@rem Ensure using the latest information. If there are invalid caches,
+@rem mamba may use invalid download URL.
+mamba clean --all -y
 @rem Arrow conda environment
 mamba create -n arrow -y -c conda-forge ^
   --file=ci\conda_env_python.txt ^

ci/conda_env_cpp.txt

@@ -21,7 +21,7 @@ azure-identity-cpp>=1.6.0
 azure-storage-blobs-cpp>=12.10.0
 azure-storage-common-cpp>=12.5.0
 azure-storage-files-datalake-cpp>=12.9.0
-benchmark>=1.6.0
+benchmark>=1.6.0,!=1.8.4
 boost-cpp>=1.68.0
 brotli
 bzip2

ci/scripts/java_full_build.sh

@@ -49,21 +49,13 @@ fi
 # build the entire project
 mvn clean \
     install \
-    assembly:single \
-    source:jar \
-    javadoc:jar \
     -Papache-release \
     -Parrow-c-data \
     -Parrow-jni \
     -Darrow.cpp.build.dir=$dist_dir \
-    -Darrow.c.jni.dist.dir=$dist_dir \
-    -DdescriptorId=source-release
+    -Darrow.c.jni.dist.dir=$dist_dir
 
 # copy all jar, zip and pom files to the distribution folder
-find . \
-     "(" -name "*-javadoc.jar" -o -name "*-sources.jar" ")" \
-     -exec echo {} ";" \
-     -exec cp {} $dist_dir ";"
 find ~/.m2/repository/org/apache/arrow \
      "(" \
      -name "*.jar" -o \

ci/scripts/python_wheel_manylinux_build.sh

@@ -160,6 +160,26 @@ export CMAKE_PREFIX_PATH=/tmp/arrow-dist
 pushd /arrow/python
 python setup.py bdist_wheel
 
+echo "=== Strip symbols from wheel ==="
+mkdir -p dist/temp-fix-wheel
+mv dist/pyarrow-*.whl dist/temp-fix-wheel
+
+pushd dist/temp-fix-wheel
+wheel_name=$(ls pyarrow-*.whl)
+# Unzip and remove old wheel
+unzip $wheel_name
+rm $wheel_name
+for filename in $(ls pyarrow/*.so pyarrow/*.so.*); do
+    echo "Stripping debug symbols from: $filename";
+    strip --strip-debug $filename
+done
+# Zip wheel again after stripping symbols
+zip -r $wheel_name .
+mv $wheel_name ..
+popd
+
+rm -rf dist/temp-fix-wheel
+
 echo "=== (${PYTHON_VERSION}) Tag the wheel with manylinux${MANYLINUX_VERSION} ==="
 auditwheel repair -L . dist/pyarrow-*.whl -w repaired_wheels
 popd

cpp/CMakePresets.json

@@ -220,6 +220,14 @@
         "PARQUET_REQUIRE_ENCRYPTION": "ON"
       }
     },
+    {
+      "name": "features-valgrind",
+      "hidden": true,
+      "cacheVariables": {
+        "ARROW_RUNTIME_SIMD_LEVEL": "AVX2",
+        "ARROW_TEST_MEMCHECK": "ON"
+      }
+    },
     {
       "name": "ninja-debug-minimal",
       "inherits": [
@@ -331,6 +339,46 @@
       "displayName": "Debug build with everything enabled (except benchmarks)",
       "cacheVariables": {}
     },
+    {
+      "name": "ninja-debug-valgrind-basic",
+      "inherits": [
+        "base-debug",
+        "features-basic",
+        "features-valgrind"
+      ],
+      "displayName": "Debug build for Valgrind with reduced dependencies",
+      "cacheVariables": {}
+    },
+    {
+      "name": "ninja-debug-valgrind",
+      "inherits": [
+        "base-debug",
+        "features-main",
+        "features-valgrind"
+      ],
+      "displayName": "Debug build for Valgrind with more optional components",
+      "cacheVariables": {}
+    },
+    {
+      "name": "ninja-debug-valgrind-minimal",
+      "inherits": [
+        "base-debug",
+        "features-minimal",
+        "features-valgrind"
+      ],
+      "displayName": "Debug build for Valgrind without anything enabled",
+      "cacheVariables": {}
+    },
+    {
+      "name": "ninja-debug-valgrind-maximal",
+      "inherits": [
+        "base-debug",
+        "features-maximal",
+        "features-valgrind"
+      ],
+      "displayName": "Debug build for Valgrind with everything enabled",
+      "cacheVariables": {}
+    },
     {
       "name": "ninja-release-minimal",
       "inherits": [
@@ -491,4 +539,4 @@
       }
     }
   ]
-}
+}

cpp/cmake_modules/Findutf8proc.cmake

@@ -19,7 +19,7 @@ if(utf8proc_FOUND)
   return()
 endif()
 
-if(ARROW_PACKAGE_KIND STREQUAL "vcpkg")
+if(ARROW_PACKAGE_KIND STREQUAL "vcpkg" OR VCPKG_TOOLCHAIN)
   set(find_package_args "")
   if(utf8proc_FIND_VERSION)
     list(APPEND find_package_args ${utf8proc_FIND_VERSION})

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -2819,11 +2819,13 @@ macro(build_utf8proc)
 endmacro()
 
 if(ARROW_WITH_UTF8PROC)
-  resolve_dependency(utf8proc
-                     PC_PACKAGE_NAMES
-                     libutf8proc
-                     REQUIRED_VERSION
-                     "2.2.0")
+  set(utf8proc_resolve_dependency_args utf8proc PC_PACKAGE_NAMES libutf8proc)
+  if(NOT VCPKG_TOOLCHAIN)
+    # utf8proc in vcpkg doesn't provide version information:
+    # https://github.com/microsoft/vcpkg/issues/39176
+    list(APPEND utf8proc_resolve_dependency_args REQUIRED_VERSION "2.2.0")
+  endif()
+  resolve_dependency(${utf8proc_resolve_dependency_args})
 endif()
 
 macro(build_cares)
@@ -4611,8 +4613,11 @@ macro(build_opentelemetry)
   set(_OPENTELEMETRY_LIBS
       common
       http_client_curl
+      logs
+      ostream_log_record_exporter
       ostream_span_exporter
       otlp_http_client
+      otlp_http_log_record_exporter
       otlp_http_exporter
       otlp_recordable
       proto
@@ -4645,6 +4650,14 @@ macro(build_opentelemetry)
       set(_OPENTELEMETRY_STATIC_LIBRARY
           "${OPENTELEMETRY_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}opentelemetry_exporter_otlp_http${CMAKE_STATIC_LIBRARY_SUFFIX}"
       )
+    elseif(_OPENTELEMETRY_LIB STREQUAL "otlp_http_log_record_exporter")
+      set(_OPENTELEMETRY_STATIC_LIBRARY
+          "${OPENTELEMETRY_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}opentelemetry_exporter_otlp_http_log${CMAKE_STATIC_LIBRARY_SUFFIX}"
+      )
+    elseif(_OPENTELEMETRY_LIB STREQUAL "ostream_log_record_exporter")
+      set(_OPENTELEMETRY_STATIC_LIBRARY
+          "${OPENTELEMETRY_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}opentelemetry_exporter_ostream_logs${CMAKE_STATIC_LIBRARY_SUFFIX}"
+      )
     else()
       set(_OPENTELEMETRY_STATIC_LIBRARY
           "${OPENTELEMETRY_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}opentelemetry_${_OPENTELEMETRY_LIB}${CMAKE_STATIC_LIBRARY_SUFFIX}"
@@ -4679,9 +4692,16 @@ macro(build_opentelemetry)
                       IMPORTED_LOCATION)
   list(APPEND
        OPENTELEMETRY_CMAKE_ARGS
-       -DWITH_OTLP=ON
        -DWITH_OTLP_HTTP=ON
        -DWITH_OTLP_GRPC=OFF
+       # Disabled because it seemed to cause linking errors. May be worth a closer look.
+       -DWITH_FUNC_TESTS=OFF
+       # These options are slated for removal in v1.14 and their features are deemed stable
+       # as of v1.13. However, setting their corresponding ENABLE_* macros in headers seems
+       # finicky - resulting in build failures or ABI-related runtime errors during HTTP
+       # client initialization. There may still be a solution, but we disable them for now.
+       -DWITH_OTLP_HTTP_SSL_PREVIEW=OFF
+       -DWITH_OTLP_HTTP_SSL_TLS_PREVIEW=OFF
        "-DProtobuf_INCLUDE_DIR=${OPENTELEMETRY_PROTOBUF_INCLUDE_DIR}"
        "-DProtobuf_LIBRARY=${OPENTELEMETRY_PROTOBUF_INCLUDE_DIR}"
        "-DProtobuf_PROTOC_EXECUTABLE=${OPENTELEMETRY_PROTOC_EXECUTABLE}")
@@ -4755,19 +4775,25 @@ macro(build_opentelemetry)
   target_link_libraries(opentelemetry-cpp::resources INTERFACE opentelemetry-cpp::common)
   target_link_libraries(opentelemetry-cpp::trace INTERFACE opentelemetry-cpp::common
                                                            opentelemetry-cpp::resources)
+  target_link_libraries(opentelemetry-cpp::logs INTERFACE opentelemetry-cpp::common
+                                                          opentelemetry-cpp::resources)
   target_link_libraries(opentelemetry-cpp::http_client_curl
-                        INTERFACE opentelemetry-cpp::ext CURL::libcurl)
+                        INTERFACE opentelemetry-cpp::common opentelemetry-cpp::ext
+                                  CURL::libcurl)
   target_link_libraries(opentelemetry-cpp::proto INTERFACE ${ARROW_PROTOBUF_LIBPROTOBUF})
   target_link_libraries(opentelemetry-cpp::otlp_recordable
-                        INTERFACE opentelemetry-cpp::trace opentelemetry-cpp::resources
-                                  opentelemetry-cpp::proto)
+                        INTERFACE opentelemetry-cpp::logs opentelemetry-cpp::trace
+                                  opentelemetry-cpp::resources opentelemetry-cpp::proto)
   target_link_libraries(opentelemetry-cpp::otlp_http_client
-                        INTERFACE opentelemetry-cpp::sdk opentelemetry-cpp::proto
+                        INTERFACE opentelemetry-cpp::common opentelemetry-cpp::proto
                                   opentelemetry-cpp::http_client_curl
                                   nlohmann_json::nlohmann_json)
   target_link_libraries(opentelemetry-cpp::otlp_http_exporter
                         INTERFACE opentelemetry-cpp::otlp_recordable
                                   opentelemetry-cpp::otlp_http_client)
+  target_link_libraries(opentelemetry-cpp::otlp_http_log_record_exporter
+                        INTERFACE opentelemetry-cpp::otlp_recordable
+                                  opentelemetry-cpp::otlp_http_client)
 
   foreach(_OPENTELEMETRY_LIB ${_OPENTELEMETRY_LIBS})
     add_dependencies(opentelemetry-cpp::${_OPENTELEMETRY_LIB} opentelemetry_ep)
@@ -4789,7 +4815,11 @@ if(ARROW_WITH_OPENTELEMETRY)
   set(opentelemetry-cpp_SOURCE "AUTO")
   resolve_dependency(opentelemetry-cpp)
   set(ARROW_OPENTELEMETRY_LIBS
-      opentelemetry-cpp::trace opentelemetry-cpp::ostream_span_exporter
+      opentelemetry-cpp::trace
+      opentelemetry-cpp::logs
+      opentelemetry-cpp::otlp_http_log_record_exporter
+      opentelemetry-cpp::ostream_log_record_exporter
+      opentelemetry-cpp::ostream_span_exporter
       opentelemetry-cpp::otlp_http_exporter)
   get_target_property(OPENTELEMETRY_INCLUDE_DIR opentelemetry-cpp::api
                       INTERFACE_INCLUDE_DIRECTORIES)

cpp/cmake_modules/Usevcpkg.cmake

@@ -237,9 +237,6 @@ set(LZ4_ROOT
     CACHE STRING "")
 
 if(CMAKE_HOST_WIN32)
-  set(utf8proc_MSVC_STATIC_LIB_SUFFIX
-      ""
-      CACHE STRING "")
   set(LZ4_MSVC_LIB_PREFIX
       ""
       CACHE STRING "")