Merge remote-tracking branch 'upstream/main' into 38325-capsule-devic…

…e-spec

.github/workflows/ruby.yml

@@ -323,6 +323,7 @@ jobs:
       ARROW_FLIGHT: ON
       ARROW_FLIGHT_SQL: ON
       ARROW_GANDIVA: OFF
+      ARROW_GLIB_VAPI: "false"
       ARROW_HDFS: OFF
       ARROW_HOME: "${{ github.workspace }}/dist"
       ARROW_JEMALLOC: OFF
@@ -345,6 +346,7 @@ jobs:
       CMAKE_UNITY_BUILD: ON
       VCPKG_BINARY_SOURCES: 'clear;nuget,GitHub,readwrite'
       VCPKG_ROOT: "${{ github.workspace }}/vcpkg"
+      VCPKG_TRIPLET: x64-windows
     permissions:
       packages: write
     steps:
@@ -405,7 +407,7 @@ jobs:
             -source "https://nuget.pkg.github.com/$GITHUB_REPOSITORY_OWNER/index.json"
       - name: Build C++ vcpkg dependencies
         run: |
-          vcpkg\vcpkg.exe install --triplet x64-windows --x-manifest-root cpp --x-install-root build\cpp\vcpkg_installed
+          vcpkg\vcpkg.exe install --triplet $env:VCPKG_TRIPLET --x-manifest-root cpp --x-install-root build\cpp\vcpkg_installed
       - name: Build C++
         shell: cmd
         run: |

c_glib/arrow-dataset-glib/enums.h.template

@@ -22,6 +22,8 @@
 
 #include <glib-object.h>
 
+#include <arrow-dataset-glib/version.h>
+
 G_BEGIN_DECLS
 /*** END file-header ***/
 
@@ -31,6 +33,7 @@ G_BEGIN_DECLS
 /*** END file-production ***/
 
 /*** BEGIN value-header ***/
+GADATASET_AVAILABLE_IN_ALL
 GType @enum_name@_get_type(void) G_GNUC_CONST;
 #define @ENUMPREFIX@_TYPE_@ENUMSHORT@ (@enum_name@_get_type())
 /*** END value-header ***/

c_glib/arrow-dataset-glib/meson.build

@@ -82,6 +82,7 @@ libarrow_dataset_glib = library('arrow-dataset-glib',
                                 implicit_include_directories: false,
                                 include_directories: base_include_directories,
                                 cpp_args: ['-DGADATASET_COMPILATION'],
+                                c_args: ['-DGADATASET_COMPILATION'],
                                 soversion: so_version,
                                 version: library_version)
 arrow_dataset_glib = declare_dependency(link_with: libarrow_dataset_glib,

c_glib/arrow-glib/enums.h.template

@@ -22,6 +22,8 @@
 
 #include <glib-object.h>
 
+#include <arrow-glib/version.h>
+
 G_BEGIN_DECLS
 /*** END file-header ***/
 
@@ -31,6 +33,7 @@ G_BEGIN_DECLS
 /*** END file-production ***/
 
 /*** BEGIN value-header ***/
+GARROW_AVAILABLE_IN_ALL
 GType @enum_name@_get_type(void) G_GNUC_CONST;
 #define @ENUMPREFIX@_TYPE_@ENUMSHORT@ (@enum_name@_get_type())
 /*** END value-header ***/

c_glib/arrow-glib/meson.build

@@ -250,6 +250,7 @@ libarrow_glib = library('arrow-glib',
                         implicit_include_directories: false,
                         include_directories: base_include_directories,
                         cpp_args: ['-DGARROW_COMPILATION'],
+                        c_args: ['-DGARROW_COMPILATION'],
                         soversion: so_version,
                         version: library_version)
 arrow_glib = declare_dependency(link_with: libarrow_glib,

c_glib/gandiva-glib/enums.h.template

@@ -22,6 +22,8 @@
 
 #include <glib-object.h>
 
+#include <gandiva-glib/version.h>
+
 G_BEGIN_DECLS
 /*** END file-header ***/
 
@@ -31,6 +33,7 @@ G_BEGIN_DECLS
 /*** END file-production ***/
 
 /*** BEGIN value-header ***/
+GGANDIVA_AVAILABLE_IN_ALL
 GType @enum_name@_get_type(void) G_GNUC_CONST;
 #define @ENUMPREFIX@_TYPE_@ENUMSHORT@ (@enum_name@_get_type())
 /*** END value-header ***/

c_glib/gandiva-glib/meson.build

@@ -86,6 +86,7 @@ libgandiva_glib = library('gandiva-glib',
                           implicit_include_directories: false,
                           include_directories: base_include_directories,
                           cpp_args: ['-DGGANDIVA_COMPILATION'],
+                          c_args: ['-DGGANDIVA_COMPILATION'],
                           soversion: so_version,
                           version: library_version)
 gandiva_glib = declare_dependency(link_with: libgandiva_glib,

c_glib/tool/generate-version-header.py

@@ -84,7 +84,7 @@ def write_header(
 
 
 def generate_visibility_macros(library: str) -> str:
-    return f"""#if (defined(_WIN32) || defined(__CYGWIN__)) && defined(_MSVC_LANG) && \
+    return f"""#if (defined(_WIN32) || defined(__CYGWIN__)) && defined(_MSC_VER) && \
   !defined({library}_STATIC_COMPILATION)
 #  define {library}_EXPORT __declspec(dllexport)
 #  define {library}_IMPORT __declspec(dllimport)

c_glib/vcpkg.json

@@ -3,6 +3,7 @@
   "version-string": "17.0.0-SNAPSHOT",
   "dependencies": [
     "glib",
+    "gobject-introspection",
     "pkgconf"
   ]
 }

ci/scripts/c_glib_build.sh

@@ -34,15 +34,22 @@ if [ -n "${MSYSTEM:-}" ]; then
   export ARROW_HOME="$(cygpath --unix "${ARROW_HOME}")"
 fi
 
+export PATH="${ARROW_HOME}/bin:${PATH}"
+
 meson_pkg_config_path="${ARROW_HOME}/lib/pkgconfig"
 
 mkdir -p ${build_dir}
 
-if [ -n "${VCPKG_ROOT:-}" ]; then
+if [ -n "${VCPKG_ROOT:-}" -a -n "${VCPKG_TRIPLET:-}" ]; then
   vcpkg_install_root="${build_root}/vcpkg_installed"
   $VCPKG_ROOT/vcpkg install --x-manifest-root=${source_dir} --x-install-root=${vcpkg_install_root}
-  export PKG_CONFIG="${vcpkg_install_root}/x64-windows/tools/pkgconf/pkgconf.exe"
-  meson_pkg_config_path="${vcpkg_install_root}/x64-windows/lib/pkgconfig:${meson_pkg_config_path}"
+  export PKG_CONFIG="${vcpkg_install_root}/${VCPKG_TRIPLET}/tools/pkgconf/pkgconf.exe"
+  meson_pkg_config_path="${vcpkg_install_root}/${VCPKG_TRIPLET}/lib/pkgconfig:${meson_pkg_config_path}"
+  # Configure PATH for libraries required by the gobject-introspection generated binary
+  cpp_vcpkg_install_root="${build_root}/cpp/vcpkg_installed"
+  PATH="${cpp_vcpkg_install_root}/${VCPKG_TRIPLET}/debug/bin:${PATH}"
+  PATH="${cpp_vcpkg_install_root}/${VCPKG_TRIPLET}/bin:${PATH}"
+  export PATH="${vcpkg_install_root}/${VCPKG_TRIPLET}/bin:${PATH}"
 fi
 
 if [ -n "${VCToolsInstallDir:-}" -a -n "${MSYSTEM:-}" ]; then

ci/scripts/java_full_build.sh

@@ -53,7 +53,8 @@ mvn clean \
     -Parrow-c-data \
     -Parrow-jni \
     -Darrow.cpp.build.dir=$dist_dir \
-    -Darrow.c.jni.dist.dir=$dist_dir
+    -Darrow.c.jni.dist.dir=$dist_dir \
+    --no-transfer-progress
 
 # copy all jar, zip and pom files to the distribution folder
 find ~/.m2/repository/org/apache/arrow \

ci/scripts/swift_test.sh

@@ -34,10 +34,14 @@ popd
 
 source_dir=${1}/swift/Arrow
 pushd ${source_dir}
+sed 's/\/\/ build://g' Package.swift > Package.swift.build
+mv Package.swift.build Package.swift
 swift test
 popd
 
 source_dir=${1}/swift/ArrowFlight
 pushd ${source_dir}
+sed 's/\/\/ build://g' Package.swift > Package.swift.build
+mv Package.swift.build Package.swift
 swift test
 popd

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -634,8 +634,10 @@ endif()
 if(DEFINED ENV{ARROW_CARES_URL})
   set(CARES_SOURCE_URL "$ENV{ARROW_CARES_URL}")
 else()
+  string(REPLACE "." "_" ARROW_CARES_BUILD_VERSION_UNDERSCORES
+                 ${ARROW_CARES_BUILD_VERSION})
   set_urls(CARES_SOURCE_URL
-           "https://c-ares.haxx.se/download/c-ares-${ARROW_CARES_BUILD_VERSION}.tar.gz"
+           "https://github.com/c-ares/c-ares/releases/download/cares-${ARROW_CARES_BUILD_VERSION_UNDERSCORES}/c-ares-${ARROW_CARES_BUILD_VERSION}.tar.gz"
            "${THIRDPARTY_MIRROR_URL}/cares-${ARROW_CARES_BUILD_VERSION}.tar.gz")
 endif()
 

cpp/src/arrow/c/bridge.cc

@@ -528,7 +528,7 @@ namespace {
 struct ExportedArrayPrivateData : PoolAllocationMixin<ExportedArrayPrivateData> {
   // The buffers are owned by the ArrayData member
   SmallVector<const void*, 3> buffers_;
-  struct ArrowArray dictionary_;
+  struct ArrowArray dictionary_ {};
   SmallVector<struct ArrowArray, 1> children_;
   SmallVector<struct ArrowArray*, 4> child_pointers_;
 

cpp/src/arrow/compute/kernels/scalar_cast_extension.cc

@@ -56,8 +56,9 @@ Status CastToExtension(KernelContext* ctx, const ExecSpan& batch, ExecResult* ou
 std::shared_ptr<CastFunction> GetCastToExtension(std::string name) {
   auto func = std::make_shared<CastFunction>(std::move(name), Type::EXTENSION);
   for (Type::type in_ty : AllTypeIds()) {
-    DCHECK_OK(
-        func->AddKernel(in_ty, {InputType(in_ty)}, kOutputTargetType, CastToExtension));
+    DCHECK_OK(func->AddKernel(in_ty, {InputType(in_ty)}, kOutputTargetType,
+                              CastToExtension, NullHandling::COMPUTED_NO_PREALLOCATE,
+                              MemAllocation::NO_PREALLOCATE));
   }
   return func;
 }

cpp/src/arrow/flight/sql/client.cc

@@ -24,7 +24,6 @@
 #include <google/protobuf/any.pb.h>
 
 #include "arrow/buffer.h"
-#include "arrow/flight/otel_logging_internal.h"
 #include "arrow/flight/sql/protocol_internal.h"
 #include "arrow/flight/types.h"
 #include "arrow/io/memory.h"
@@ -64,7 +63,6 @@ arrow::Result<FlightDescriptor> GetFlightDescriptorForCommand(
 arrow::Result<std::unique_ptr<FlightInfo>> GetFlightInfoForCommand(
     FlightSqlClient* client, const FlightCallOptions& options,
     const google::protobuf::Message& command) {
-  ARROW_FLIGHT_OTELLOG_SQL_CLIENT(INFO, "[Example message] func=", __func__);
   ARROW_ASSIGN_OR_RAISE(FlightDescriptor descriptor,
                         GetFlightDescriptorForCommand(command));
   return client->GetFlightInfo(options, descriptor);

cpp/src/arrow/flight/sql/server.cc

@@ -26,13 +26,11 @@
 
 #include "arrow/buffer.h"
 #include "arrow/builder.h"
-#include "arrow/flight/otel_logging_internal.h"
 #include "arrow/flight/serialization_internal.h"
 #include "arrow/flight/sql/protocol_internal.h"
 #include "arrow/flight/sql/sql_info_internal.h"
 #include "arrow/type.h"
 #include "arrow/util/checked_cast.h"
-#include "arrow/util/logger.h"
 
 #define PROPERTY_TO_OPTIONAL(COMMAND, PROPERTY) \
   COMMAND.has_##PROPERTY() ? std::make_optional(COMMAND.PROPERTY()) : std::nullopt
@@ -578,7 +576,6 @@ arrow::Result<std::string> CreateStatementQueryTicket(
 Status FlightSqlServerBase::GetFlightInfo(const ServerCallContext& context,
                                           const FlightDescriptor& request,
                                           std::unique_ptr<FlightInfo>* info) {
-  ARROW_FLIGHT_OTELLOG_SQL_SERVER(INFO, "[Example message] func=", __func__);
   google::protobuf::Any any;
   if (!any.ParseFromArray(request.cmd.data(), static_cast<int>(request.cmd.size()))) {
     return Status::Invalid("Unable to parse command");

cpp/src/arrow/flight/transport/grpc/grpc_client.cc

@@ -50,7 +50,6 @@
 #include "arrow/flight/client_middleware.h"
 #include "arrow/flight/cookie_internal.h"
 #include "arrow/flight/middleware.h"
-#include "arrow/flight/otel_logging_internal.h"
 #include "arrow/flight/serialization_internal.h"
 #include "arrow/flight/transport.h"
 #include "arrow/flight/transport/grpc/serialization_internal.h"
@@ -929,7 +928,6 @@ class GrpcClientImpl : public internal::ClientTransport {
   Status GetFlightInfo(const FlightCallOptions& options,
                        const FlightDescriptor& descriptor,
                        std::unique_ptr<FlightInfo>* info) override {
-    ARROW_FLIGHT_OTELLOG_CLIENT(INFO, "[Example message] func=", __func__);
     pb::FlightDescriptor pb_descriptor;
     pb::FlightInfo pb_response;
 

cpp/src/arrow/flight/transport/grpc/grpc_server.cc

@@ -28,7 +28,6 @@
 #include <grpcpp/grpcpp.h>
 
 #include "arrow/buffer.h"
-#include "arrow/flight/otel_logging_internal.h"
 #include "arrow/flight/serialization_internal.h"
 #include "arrow/flight/server.h"
 #include "arrow/flight/server_middleware.h"
@@ -37,7 +36,6 @@
 #include "arrow/flight/transport/grpc/util_internal.h"
 #include "arrow/flight/transport_server.h"
 #include "arrow/flight/types.h"
-#include "arrow/util/logger.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/uri.h"
 
@@ -404,7 +402,6 @@ class GrpcServiceHandler final : public FlightService::Service {
   ::grpc::Status GetFlightInfo(ServerContext* context,
                                const pb::FlightDescriptor* request,
                                pb::FlightInfo* response) {
-    ARROW_FLIGHT_OTELLOG_SERVER(INFO, "[Example message] func=", __func__);
     GrpcServerCallContext flight_context(context);
     GRPC_RETURN_NOT_GRPC_OK(
         CheckAuth(FlightMethod::GetFlightInfo, context, flight_context));

cpp/src/arrow/gpu/cuda_benchmark.cc

@@ -36,7 +36,7 @@ constexpr int64_t kGpuNumber = 0;
 static void CudaBufferWriterBenchmark(benchmark::State& state, const int64_t total_bytes,
                                       const int64_t chunksize,
                                       const int64_t buffer_size) {
-  CudaDeviceManager* manager;
+  CudaDeviceManager* manager = nullptr;
   ABORT_NOT_OK(CudaDeviceManager::Instance().Value(&manager));
   std::shared_ptr<CudaContext> context;
   ABORT_NOT_OK(manager->GetContext(kGpuNumber).Value(&context));

cpp/src/arrow/telemetry/telemetry_test.cc

@@ -22,6 +22,7 @@
 #include "arrow/telemetry/util_internal.h"
 #include "arrow/testing/gtest_util.h"
 #include "arrow/testing/util.h"
+#include "arrow/util/io_util.h"
 #include "arrow/util/logging.h"
 #include "arrow/util/tracing_internal.h"
 
@@ -35,11 +36,20 @@
 namespace arrow {
 namespace telemetry {
 
+static constexpr char kLoggingEnvVar[] = "ARROW_LOGGING_BACKEND";
+
 class OtelEnvironment : public ::testing::Environment {
  public:
   static constexpr std::string_view kLoggerName = "arrow-telemetry-test";
 
   void SetUp() override {
+    auto maybe_env_var = arrow::internal::GetEnvVar(kLoggingEnvVar);
+    // The env variable is required to be set to obtain a valid logger, so if it hasn't
+    // been set in the environment then we do that here.
+    if (maybe_env_var.status().IsKeyError()) {
+      ASSERT_OK(arrow::internal::SetEnvVar(kLoggingEnvVar, "arrow_otlp_stderr"));
+    }
+
     // Implicitly sets up span processors + tracer provider
     auto tracer = arrow::internal::tracing::GetTracer();
     ARROW_UNUSED(tracer);

cpp/thirdparty/versions.txt

@@ -144,7 +144,7 @@ DEPENDENCIES=(
   "ARROW_BOOST_URL boost-${ARROW_BOOST_BUILD_VERSION}.tar.gz https://apache.jfrog.io/artifactory/arrow/thirdparty/7.0.0/boost_${ARROW_BOOST_BUILD_VERSION//./_}.tar.gz"
   "ARROW_BROTLI_URL brotli-${ARROW_BROTLI_BUILD_VERSION}.tar.gz https://github.com/google/brotli/archive/${ARROW_BROTLI_BUILD_VERSION}.tar.gz"
   "ARROW_BZIP2_URL bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz https://sourceware.org/pub/bzip2/bzip2-${ARROW_BZIP2_BUILD_VERSION}.tar.gz"
-  "ARROW_CARES_URL cares-${ARROW_CARES_BUILD_VERSION}.tar.gz https://c-ares.haxx.se/download/c-ares-${ARROW_CARES_BUILD_VERSION}.tar.gz"
+  "ARROW_CARES_URL cares-${ARROW_CARES_BUILD_VERSION}.tar.gz https://github.com/c-ares/c-ares/releases/download/cares-${ARROW_CARES_BUILD_VERSION//./_}/c-ares-${ARROW_CARES_BUILD_VERSION}.tar.gz"
   "ARROW_CRC32C_URL crc32c-${ARROW_CRC32C_BUILD_VERSION}.tar.gz https://github.com/google/crc32c/archive/refs/tags/${ARROW_CRC32C_BUILD_VERSION}.tar.gz"
   "ARROW_GBENCHMARK_URL gbenchmark-${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz https://github.com/google/benchmark/archive/${ARROW_GBENCHMARK_BUILD_VERSION}.tar.gz"
   "ARROW_GFLAGS_URL gflags-${ARROW_GFLAGS_BUILD_VERSION}.tar.gz https://github.com/gflags/gflags/archive/${ARROW_GFLAGS_BUILD_VERSION}.tar.gz"

dev/archery/archery/integration/datagen.py

@@ -1932,7 +1932,6 @@ def _temp_path():
         .skip_tester('Rust'),
 
         generate_binary_view_case()
-        .skip_tester('Java')
         .skip_tester('JS')
         .skip_tester('nanoarrow')
         .skip_tester('Rust'),

dev/tasks/java-jars/github.yml

@@ -242,6 +242,9 @@ jobs:
           test -f arrow/java-dist/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll
           test -f arrow/java-dist/arrow_orc_jni/x86_64/arrow_orc_jni.dll
       - name: Build bundled jar
+        env:
+          MAVEN_ARGS: >-
+            --no-transfer-progress
         run: |
           set -e
           pushd arrow/java

docker-compose.yml

@@ -1565,6 +1565,7 @@ services:
       # so some build might pass without this setting, but we want to ensure that we stay to AVX2 regardless of runner.
       EXTRA_CMAKE_FLAGS: "-DARROW_RUNTIME_SIMD_LEVEL=AVX2"
       ARROW_SOURCE_HOME: "/arrow"
+      LIBARROW_DOWNLOAD: "false"
     volumes: *ubuntu-volumes
     command: >
       /bin/bash -c "

go/arrow/array/data.go

@@ -201,7 +201,9 @@ func (d *Data) SizeInBytes() uint64 {
 	}
 
 	for _, b := range d.Buffers() {
-		size += uint64(b.Len())
+		if b != nil {
+			size += uint64(b.Len())
+		}
 	}
 	for _, c := range d.Children() {
 		size += c.SizeInBytes()

go/arrow/array/data_test.go

@@ -17,6 +17,7 @@
 package array
 
 import (
+	"slices"
 	"testing"
 
 	"github.com/apache/arrow/go/v17/arrow"
@@ -60,6 +61,17 @@ func TestSizeInBytes(t *testing.T) {
 	var arrayData arrow.ArrayData = data
 	dataWithChild := NewData(&arrow.StringType{}, 10, buffers1, []arrow.ArrayData{arrayData}, 0, 0)
 
+	buffers2 := slices.Clone(buffers1)
+	buffers2[0] = nil
+	dataWithNilBuffer := NewData(&arrow.StringType{}, 10, buffers2, nil, 0, 0)
+
+	t.Run("nil buffers", func(t *testing.T) {
+		expectedSize := uint64(30)
+		if actualSize := dataWithNilBuffer.SizeInBytes(); actualSize != expectedSize {
+			t.Errorf("expected size %d, got %d", expectedSize, actualSize)
+		}
+	})
+
 	t.Run("buffers only", func(t *testing.T) {
 		expectedSize := uint64(45)
 		if actualSize := data.SizeInBytes(); actualSize != expectedSize {

go/arrow/example_test.go

@@ -27,7 +27,6 @@ import (
 )
 
 // This example demonstrates how to build an array of int64 values using a builder and Append.
-// Whilst convenient for small arrays,
 func Example_minimal() {
 	// Create an allocator.
 	pool := memory.NewGoAllocator()