cc

.github/workflows/macos.yml

@@ -63,9 +63,7 @@ jobs:
         env:
           HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: "TRUE"
         run: |
-          source scripts/setup-macos.sh
-          install_build_prerequisites
-          install_velox_deps_from_brew
+          scripts/setup-macos.sh
 
           echo "NJOBS=`sysctl -n hw.ncpu`" >> $GITHUB_ENV
           brew unlink protobuf || echo "protobuf not installed"

scripts/setup-macos.sh

@@ -28,6 +28,36 @@
 set -e # Exit on error.
 set -x # Print commands that are executed.
 
+
+# Build dependencies versions.
+FB_OS_VERSION="v2024.05.20.00"
+FMT_VERSION="10.1.1"
+BOOST_VERSION="boost-1.84.0"
+ARROW_VERSION="15.0.0"
+DUCKDB_VERSION="v0.8.1"
+PROTOBUF_VERSION="21.8"
+MINIO_VERSION="2022-05-26T05-48-41Z"
+MINIO_BINARY_NAME="minio-2022-05-26"
+AWS_SDK_VERSION="1.11.321"
+XSIMD_VERSION="10.0.0"
+SIMDJSON_VERSION="3.9.3"
+CPR_VERSION="1.10.5"
+DOUBLE_CONVERSION_VERSION="v3.1.5"
+RANGE_V3_VERSION="0.12.0"
+RE2_VERSION="2022-02-01"
+GLOG_VERSION="v0.6.0"
+LZO_VERSION="2.10"
+SNAPPY_VERSION="1.1.8"
+
+# Adapter related versions.
+ABSEIL_VERSION="20240116.2"
+GRPC_VERSION="v1.48.1"
+CRC32_VERSION="1.1.2"
+NLOHMAN_JSON_VERSION="v3.11.3"
+GOOGLE_CLOUD_CPP_VERSION="v2.22.0"
+HADOOP_VERSION="2.10.1"
+AZURE_SDK_VERSION="12.8.0"
+
 SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}")
 source $SCRIPTDIR/setup-helper-functions.sh
 PYTHON_VENV=${PYHTON_VENV:-"${SCRIPTDIR}/../.venv"}
@@ -37,14 +67,16 @@ NPROC=$(getconf _NPROCESSORS_ONLN)
 DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)}
 MACOS_VELOX_DEPS="bison boost double-conversion flex gflags glog googletest icu4c libevent libsodium lz4 lzo openssl protobuf@21 simdjson snappy thrift xz xsimd zstd"
 MACOS_BUILD_DEPS="ninja cmake"
-FB_OS_VERSION="v2024.05.20.00"
-FMT_VERSION="10.1.1"
+
+CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}"
+BUILD_DUCKDB="${BUILD_DUCKDB:-true}"
+USE_CLANG="${USE_CLANG:-false}"
 
 function update_brew {
   DEFAULT_BREW_PATH=/usr/local/bin/brew
-  if [ `arch` == "arm64" ] ;
+  if [ "$(arch)" == "arm64" ];
     then
-      DEFAULT_BREW_PATH=$(which brew) ;
+      DEFAULT_BREW_PATH=$(which brew)
   fi
   BREW_PATH=${BREW_PATH:-$DEFAULT_BREW_PATH}
   $BREW_PATH update --auto-update --verbose
@@ -61,9 +93,9 @@ function install_from_brew {
     tap="velox/local-${pkg}"
     brew tap-new "${tap}"
     brew extract "--version=${ver}" "${pkg}" "${tap}"
-    brew install "${tap}/${pkg}@${ver}" || ( echo "Failed to install ${tap}/${pkg}@${ver}" ; exit 1 )
+    brew install "${tap}/${pkg}@${ver}" || { echo "Failed to install ${tap}/${pkg}@${ver}"; exit 1; }
   else
-    ( brew install --formula "${pkg}" && echo "Installation of ${pkg} is successful" || brew upgrade --formula "$pkg" ) || ( echo "Failed to install ${pkg}" ; exit 1 )
+    (brew install --formula "${pkg}" && echo "Installation of ${pkg} is successful") || brew upgrade --formula "${pkg}" || { echo "Failed to install ${pkg}"; exit 1; }
   fi
 }
 
@@ -83,7 +115,7 @@ function install_build_prerequisites {
 }
 
 function install_velox_deps_from_brew {
-  for pkg in ${MACOS_VELOX_DEPS}
+  for pkg in ${MACOS_VELOX_DEPS};
   do
     install_from_brew ${pkg}
   done
@@ -134,6 +166,133 @@ function install_re2 {
   cmake_install re2 -DRE2_BUILD_TESTING=OFF
 }
 
+function install_xsimd {
+  wget_and_untar https://github.com/xtensor-stack/xsimd/archive/refs/tags/${XSIMD_VERSION}.tar.gz xsimd
+  cmake_install xsimd
+}
+
+function install_arrow {
+   set -e  # Exit on first error
+   echo "Downloading and extracting Apache Arrow version ${ARROW_VERSION}..."
+   wget_and_untar https://archive.apache.org/dist/arrow/arrow-${ARROW_VERSION}/apache-arrow-${ARROW_VERSION}.tar.gz arrow
+
+   echo "Starting the build process..."
+   (
+     cd arrow/cpp
+     cmake_install \
+       -DARROW_PARQUET=OFF \
+       -DARROW_WITH_THRIFT=ON \
+       -DARROW_WITH_LZ4=ON \
+       -DARROW_WITH_SNAPPY=ON \
+       -DARROW_WITH_ZLIB=ON \
+       -DARROW_WITH_ZSTD=ON \
+       -DARROW_JEMALLOC=OFF \
+       -DARROW_SIMD_LEVEL=NONE \
+       -DARROW_RUNTIME_SIMD_LEVEL=NONE \
+       -DARROW_WITH_UTF8PROC=OFF \
+       -DARROW_TESTING=ON \
+       -DCMAKE_INSTALL_PREFIX=/usr/local \
+       -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
+       -DARROW_BUILD_STATIC=ON \
+       -DThrift_SOURCE=BUNDLED
+
+     echo "Installing Thrift..."
+     cd _build/thrift_ep-prefix/src/thrift_ep-build
+     ${SUDO} cmake --install ./ --prefix /usr/local/
+   )
+   echo "Apache Arrow installation complete."
+}
+
+function install_duckdb {
+  if $BUILD_DUCKDB ; then
+    echo 'Building DuckDB'
+    wget_and_untar https://github.com/duckdb/duckdb/archive/refs/tags/${DUCKDB_VERSION}.tar.gz duckdb
+    cmake_install duckdb -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+  fi
+}
+
+function install_boost {
+  wget_and_untar https://github.com/boostorg/boost/releases/download/${BOOST_VERSION}/${BOOST_VERSION}.tar.gz boost
+  (
+    cd boost
+    if [[ ${USE_CLANG} != "false" ]]; then
+      ./bootstrap.sh --prefix=/usr/local --with-toolset="clang-15"
+      # Switch the compiler from the clang-15 toolset which doesn't exist (clang-15.jam) to
+      # clang of version 15 when toolset clang-15 is used.
+      # This reconciles the project-config.jam generation with what the b2 build system allows for customization.
+      sed -i 's/using clang-15/using clang : 15/g' project-config.jam
+      ${SUDO} ./b2 "-j${NPROC}" -d0 install threading=multi toolset=clang-15 --without-python
+    else
+      ./bootstrap.sh --prefix=/usr/local
+      ${SUDO} ./b2 "-j${NPROC}" -d0 install threading=multi --without-python
+    fi
+  )
+}
+
+function install_proxygen {
+  wget_and_untar https://github.com/facebook/proxygen/archive/refs/tags/${FB_OS_VERSION}.tar.gz proxygen
+  cmake_install proxygen -DBUILD_TESTS=OFF
+}
+
+function install_protobuf {
+  wget_and_untar https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-all-${PROTOBUF_VERSION}.tar.gz protobuf
+  cmake_install protobuf -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_ABSL_PROVIDER=package
+}
+
+function install_simdjson {
+  wget_and_untar https://github.com/simdjson/simdjson/archive/refs/tags/v${SIMDJSON_VERSION}.tar.gz simdjson
+  cmake_install simdjson
+}
+
+function install_gcs {
+  # Install gcs dependencies
+  # https://github.com/googleapis/google-cloud-cpp/blob/main/doc/packaging.md#required-libraries
+
+  # abseil-cpp
+  github_checkout abseil/abseil-cpp ${ABSEIL_VERSION} --depth 1
+  cmake_install \
+    -DABSL_BUILD_TESTING=OFF \
+    -DCMAKE_CXX_STANDARD=17 \
+    -DABSL_PROPAGATE_CXX_STD=ON \
+    -DABSL_ENABLE_INSTALL=ON
+
+  # protobuf
+  github_checkout protocolbuffers/protobuf v${PROTOBUF_VERSION} --depth 1
+  cmake_install \
+    -Dprotobuf_BUILD_TESTS=OFF \
+    -Dprotobuf_ABSL_PROVIDER=package
+
+  # grpc
+  github_checkout grpc/grpc ${GRPC_VERSION} --depth 1
+  cmake_install \
+    -DgRPC_BUILD_TESTS=OFF \
+    -DgRPC_ABSL_PROVIDER=package \
+    -DgRPC_ZLIB_PROVIDER=package \
+    -DgRPC_CARES_PROVIDER=package \
+    -DgRPC_RE2_PROVIDER=package \
+    -DgRPC_SSL_PROVIDER=package \
+    -DgRPC_PROTOBUF_PROVIDER=package \
+    -DgRPC_INSTALL=ON
+
+  # crc32
+  github_checkout google/crc32c ${CRC32_VERSION} --depth 1
+  cmake_install \
+    -DCRC32C_BUILD_TESTS=OFF \
+    -DCRC32C_BUILD_BENCHMARKS=OFF \
+    -DCRC32C_USE_GLOG=OFF
+
+  # nlohmann json
+  github_checkout nlohmann/json ${NLOHMAN_JSON_VERSION} --depth 1
+  cmake_install \
+    -DJSON_BuildTests=OFF
+
+  # google-cloud-cpp
+  github_checkout googleapis/google-cloud-cpp ${GOOGLE_CLOUD_CPP_VERSION} --depth 1
+  cmake_install \
+    -DGOOGLE_CLOUD_CPP_ENABLE_EXAMPLES=OFF \
+    -DGOOGLE_CLOUD_CPP_ENABLE=storage
+}
+
 function install_velox_deps {
   run_and_time install_velox_deps_from_brew
   run_and_time install_ranges_v3
@@ -145,6 +304,13 @@ function install_velox_deps {
   run_and_time install_wangle
   run_and_time install_mvfst
   run_and_time install_fbthrift
+  run_and_time install_xsimd
+  run_and_time install_proxygen
+  run_and_time install_arrow
+  run_and_time install_duckdb
+  run_and_time install_protobuf
+  run_and_time install_simdjson
+  run_and_time install_simdjson
 }
 
 (return 2> /dev/null) && return # If script was sourced, don't run commands.