diff --git a/.github/workflows/linux-build.yml b/.github/workflows/linux-build.yml index f2db425f37e1..2f358e8522b1 100644 --- a/.github/workflows/linux-build.yml +++ b/.github/workflows/linux-build.yml @@ -25,6 +25,8 @@ on: - "CMake/**" - "third_party/**" - "scripts/setup-ubuntu.sh" + - "scripts/setup-common.sh" + - "scripts/setup-versions.sh" - "scripts/setup-helper-functions.sh" - ".github/workflows/linux-build.yml" @@ -36,6 +38,8 @@ on: - "CMake/**" - "third_party/**" - "scripts/setup-ubuntu.sh" + - "scripts/setup-common.sh" + - "scripts/setup-versions.sh" - "scripts/setup-helper-functions.sh" - ".github/workflows/linux-build.yml" @@ -171,7 +175,7 @@ jobs: MAKEFLAGS: "NUM_THREADS=8 MAX_HIGH_MEM_JOBS=4 MAX_LINK_JOBS=3" EXTRA_CMAKE_FLAGS: "-DVELOX_ENABLE_ARROW=ON -DVELOX_ENABLE_PARQUET=ON" run: | - make debug + make debug - name: CCache after run: | diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index bb02a13dca87..6a4583ab54d1 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -61,8 +61,7 @@ jobs: env: HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: "TRUE" run: | - source scripts/setup-macos.sh - brew install $MACOS_BUILD_DEPS $MACOS_VELOX_DEPS + scripts/setup-macos.sh echo "NJOBS=`sysctl -n hw.ncpu`" >> $GITHUB_ENV brew unlink protobuf || echo "protobuf not installed" diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 4b1370fb0036..6f4176ea5fd9 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -23,6 +23,8 @@ on: - "CMake/**" - "third_party/**" - "scripts/setup-ubuntu.sh" + - "scripts/setup-common.sh" + - "scripts/setup-versions.sh" - "scripts/setup-helper-functions.sh" - ".github/workflows/scheduled.yml" @@ -36,6 +38,8 @@ on: - "CMake/**" - "third_party/**" - "scripts/setup-ubuntu.sh" + - "scripts/setup-common.sh" + - "scripts/setup-versions.sh" - "scripts/setup-helper-functions.sh" - ".github/workflows/scheduled.yml" diff --git a/README.md b/README.md index 9bc7a46e0fe7..521c0395c5ef 100644 --- a/README.md +++ b/README.md @@ -132,11 +132,13 @@ libraries. Once you have checked out Velox, you can setup and build like so: ```shell $ ./scripts/setup-centos9.sh -$ ./scripts/setup-adapters.sh +$ ./scripts/setup-centos9.sh install_adapters $ make ``` -Note that `setup-adapters.sh` supports MacOS and Ubuntu 20.04 or later. +Note that the `install_adapters` command is available for the supported MacOS and +Ubuntu (20.04 or later) scripts. Individual adapters can be installed by specifying +the individual install command, e.g. `setup-centos9.sh install_aws`. ### Using Clang on Linux diff --git a/scripts/adapters.dockerfile b/scripts/adapters.dockerfile index 7dac26b94992..a47883e3f7e2 100644 --- a/scripts/adapters.dockerfile +++ b/scripts/adapters.dockerfile @@ -15,15 +15,18 @@ ARG image=ghcr.io/facebookincubator/velox-dev:centos9 FROM $image -COPY scripts/setup-adapters.sh / +COPY scripts/setup-centos9.sh / +COPY scripts/setup-common.sh / +COPY scripts/setup-versions.sh / +COPY scripts/setup-helper-functions.sh / RUN mkdir build && ( cd build && source /opt/rh/gcc-toolset-12/enable && \ - bash /setup-adapters.sh ) && rm -rf build && dnf remove -y conda && dnf clean all + bash /setup-centos9.sh install_adapters ) && rm -rf build && dnf remove -y conda && dnf clean all # install miniforge RUN curl -L -o /tmp/miniforge.sh https://github.com/conda-forge/miniforge/releases/download/23.11.0-0/Mambaforge-23.11.0-0-Linux-x86_64.sh && \ bash /tmp/miniforge.sh -b -p /opt/miniforge && \ rm /tmp/miniforge.sh -ENV PATH=/opt/miniforge/condabin:${PATH} +ENV PATH=/opt/miniforge/condabin:${PATH} # install test dependencies RUN mamba create -y --name adapters python=3.8 diff --git a/scripts/centos.dockerfile b/scripts/centos.dockerfile index 50a18e9a7243..12f9ff4e1a50 100644 --- a/scripts/centos.dockerfile +++ b/scripts/centos.dockerfile @@ -16,6 +16,8 @@ ARG image=quay.io/centos/centos:stream9 FROM $image COPY scripts/setup-helper-functions.sh / +COPY scripts/setup-versions.sh / +COPY scripts/setup-common.sh / COPY scripts/setup-centos9.sh / # The removal of the build dir has to happen in the same layer as the build # to minimize the image size. gh & jq are required for CI diff --git a/scripts/setup-adapters.sh b/scripts/setup-adapters.sh deleted file mode 100755 index a8c832ea54c5..000000000000 --- a/scripts/setup-adapters.sh +++ /dev/null @@ -1,231 +0,0 @@ -#!/bin/bash -# Copyright (c) Facebook, Inc. and its affiliates. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -SCRIPTDIR=$(dirname "$0") -source $SCRIPTDIR/setup-helper-functions.sh - -# Propagate errors and improve debugging. -set -eufx -o pipefail - -SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") -source $SCRIPTDIR/setup-helper-functions.sh -DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)} -CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" -MACHINE=$(uname -m) - -function install_aws_deps { - local AWS_REPO_NAME="aws/aws-sdk-cpp" - local AWS_SDK_VERSION="1.11.321" - - github_checkout $AWS_REPO_NAME $AWS_SDK_VERSION --depth 1 --recurse-submodules - cmake_install -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS:BOOL=OFF -DMINIMIZE_SIZE:BOOL=ON -DENABLE_TESTING:BOOL=OFF -DBUILD_ONLY:STRING="s3;identity-management" - # Dependencies for S3 testing - # We need this specific version of Minio for testing. - local MINIO_ARCH=$MACHINE - if [[ $MACHINE == aarch64 ]]; then - MINIO_ARCH="arm64" - elif [[ $MACHINE == x86_64 ]]; then - MINIO_ARCH="amd64" - fi - local MINIO_BINARY="minio-2022-05-26" - local MINIO_OS="linux" - if [[ "$OSTYPE" == darwin* ]]; then - # minio will have to approved under the Privacy & Security on MacOS on first use. - MINIO_OS="darwin" - fi - wget https://dl.min.io/server/minio/release/${MINIO_OS}-${MINIO_ARCH}/archive/minio.RELEASE.2022-05-26T05-48-41Z -O ${MINIO_BINARY} - chmod +x ./${MINIO_BINARY} - mv ./${MINIO_BINARY} /usr/local/bin/ -} - -function install_gcs-sdk-cpp { - # Install gcs dependencies - # https://github.com/googleapis/google-cloud-cpp/blob/main/doc/packaging.md#required-libraries - - # abseil-cpp - github_checkout abseil/abseil-cpp 20240116.2 --depth 1 - cmake_install \ - -DABSL_BUILD_TESTING=OFF \ - -DCMAKE_CXX_STANDARD=17 \ - -DABSL_PROPAGATE_CXX_STD=ON \ - -DABSL_ENABLE_INSTALL=ON - - # protobuf - github_checkout protocolbuffers/protobuf v21.8 --depth 1 - cmake_install \ - -Dprotobuf_BUILD_TESTS=OFF \ - -Dprotobuf_ABSL_PROVIDER=package - - # grpc - github_checkout grpc/grpc v1.48.1 --depth 1 - cmake_install \ - -DgRPC_BUILD_TESTS=OFF \ - -DgRPC_ABSL_PROVIDER=package \ - -DgRPC_ZLIB_PROVIDER=package \ - -DgRPC_CARES_PROVIDER=package \ - -DgRPC_RE2_PROVIDER=package \ - -DgRPC_SSL_PROVIDER=package \ - -DgRPC_PROTOBUF_PROVIDER=package \ - -DgRPC_INSTALL=ON - - # crc32 - github_checkout google/crc32c 1.1.2 --depth 1 - cmake_install \ - -DCRC32C_BUILD_TESTS=OFF \ - -DCRC32C_BUILD_BENCHMARKS=OFF \ - -DCRC32C_USE_GLOG=OFF - - # nlohmann json - github_checkout nlohmann/json v3.11.3 --depth 1 - cmake_install \ - -DJSON_BuildTests=OFF - - # google-cloud-cpp - github_checkout googleapis/google-cloud-cpp v2.22.0 --depth 1 - cmake_install \ - -DGOOGLE_CLOUD_CPP_ENABLE_EXAMPLES=OFF \ - -DGOOGLE_CLOUD_CPP_ENABLE=storage -} - -function install_azure-storage-sdk-cpp { - # Disable VCPKG to install additional static dependencies under the VCPKG installed path - # instead of using system pre-installed dependencies. - export AZURE_SDK_DISABLE_AUTO_VCPKG=ON - vcpkg_commit_id=7a6f366cefd27210f6a8309aed10c31104436509 - github_checkout azure/azure-sdk-for-cpp azure-storage-files-datalake_12.8.0 - sed -i "s/set(VCPKG_COMMIT_STRING .*)/set(VCPKG_COMMIT_STRING $vcpkg_commit_id)/" cmake-modules/AzureVcpkg.cmake - - azure_core_dir="sdk/core/azure-core" - if ! grep -q "baseline" $azure_core_dir/vcpkg.json; then - # build and install azure-core with the version compatible with system pre-installed openssl - openssl_version=$(openssl version -v | awk '{print $2}') - if [[ "$openssl_version" == 1.1.1* ]]; then - openssl_version="1.1.1n" - fi - sed -i "s/\"version-string\"/\"builtin-baseline\": \"$vcpkg_commit_id\",\"version-string\"/" $azure_core_dir/vcpkg.json - sed -i "s/\"version-string\"/\"overrides\": [{ \"name\": \"openssl\", \"version-string\": \"$openssl_version\" }],\"version-string\"/" $azure_core_dir/vcpkg.json - fi - cmake_install $azure_core_dir -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF - - # install azure-storage-common - cmake_install sdk/storage/azure-storage-common -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF - - # install azure-storage-blobs - cmake_install sdk/storage/azure-storage-blobs -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF - - # install azure-storage-files-datalake - cmake_install sdk/storage/azure-storage-files-datalake -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF -} - -function install_hdfs_deps { - github_checkout apache/hawq master - libhdfs3_dir=$DEPENDENCY_DIR/hawq/depends/libhdfs3 - if [[ "$OSTYPE" == darwin* ]]; then - sed -i '' -e "/FIND_PACKAGE(GoogleTest REQUIRED)/d" $libhdfs3_dir/CMakeLists.txt - sed -i '' -e "s/dumpversion/dumpfullversion/" $libhdfs3_dir/CMakeLists.txt - fi - - if [[ "$OSTYPE" == linux-gnu* ]]; then - sed -i "/FIND_PACKAGE(GoogleTest REQUIRED)/d" $libhdfs3_dir/CMakeLists.txt - sed -i "s/dumpversion/dumpfullversion/" $libhdfs3_dir/CMake/Platform.cmake - # Dependencies for Hadoop testing - wget_and_untar https://archive.apache.org/dist/hadoop/common/hadoop-2.10.1/hadoop-2.10.1.tar.gz hadoop - cp -a hadoop /usr/local/ - fi - cmake_install $libhdfs3_dir -} - -cd "${DEPENDENCY_DIR}" || exit -# aws-sdk-cpp missing dependencies - -if [[ "$OSTYPE" == "linux-gnu"* ]]; then - # /etc/os-release is a standard way to query various distribution - # information and is available everywhere - LINUX_DISTRIBUTION=$(. /etc/os-release && echo ${ID}) - if [[ "$LINUX_DISTRIBUTION" == "ubuntu" || "$LINUX_DISTRIBUTION" == "debian" ]]; then - apt install -y --no-install-recommends libxml2-dev libgsasl7-dev uuid-dev - # Dependencies of GCS, probably a workaround until the docker image is rebuilt - apt install -y --no-install-recommends libc-ares-dev libcurl4-openssl-dev - # Dependencies of Azure Storage Blob cpp - apt install -y openssl - else # Assume Fedora/CentOS - dnf -y install libxml2-devel libgsasl-devel libuuid-devel krb5-devel - # Dependencies of GCS, probably a workaround until the docker image is rebuilt - dnf -y install npm curl-devel c-ares-devel - # Dependencies of Azure Storage Blob Cpp - dnf -y install perl-IPC-Cmd - dnf -y install openssl - fi -fi - -if [[ "$OSTYPE" == darwin* ]]; then - brew install libxml2 gsasl -fi - -install_aws=0 -install_gcs=0 -install_hdfs=0 -install_abfs=0 - -if [ "$#" -eq 0 ]; then - # Install all adapters by default - install_aws=1 - install_gcs=1 - install_hdfs=1 - install_abfs=1 -fi - -while [[ $# -gt 0 ]]; do - case $1 in - gcs) - install_gcs=1 - shift # past argument - ;; - aws) - install_aws=1 - shift # past argument - ;; - hdfs) - install_hdfs=1 - shift # past argument - ;; - abfs) - install_abfs=1 - shift # past argument - ;; - *) - echo "ERROR: Unknown option $1! will be ignored!" - shift - ;; - esac -done - -if [ $install_gcs -eq 1 ]; then - install_gcs-sdk-cpp -fi -if [ $install_aws -eq 1 ]; then - install_aws_deps -fi -if [ $install_hdfs -eq 1 ]; then - install_hdfs_deps -fi -if [ $install_abfs -eq 1 ]; then - install_azure-storage-sdk-cpp -fi - -_ret=$? -if [ $_ret -eq 0 ] ; then - echo "All deps for Velox adapters installed!" -fi diff --git a/scripts/setup-centos9.sh b/scripts/setup-centos9.sh index 825d2d29e32a..0ab5527c3aaf 100755 --- a/scripts/setup-centos9.sh +++ b/scripts/setup-centos9.sh @@ -29,18 +29,16 @@ set -efx -o pipefail # Some of the packages must be build with the same compiler flags # so that some low level types are the same size. Also, disable warnings. SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") -source $SCRIPTDIR/setup-helper-functions.sh -NPROC=$(getconf _NPROCESSORS_ONLN) -export CXXFLAGS=$(get_cxx_flags) # Used by boost. +source $SCRIPTDIR/setup-common.sh + +# Folly must be built with the same compiler flags so that some low level types +# are the same size. +export CXXFLAGS=$(get_cxx_flags) # Used by boost. export CFLAGS=${CXXFLAGS//"-std=c++17"/} # Used by LZO. -CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" -BUILD_DUCKDB="${BUILD_DUCKDB:-true}" -USE_CLANG="${USE_CLANG:-false}" +export COMPILER_FLAGS=${CXXFLAGS} -FB_OS_VERSION="v2024.05.20.00" -FMT_VERSION="10.1.1" -BOOST_VERSION="boost-1.84.0" -ARROW_VERSION="15.0.0" +SUDO="${SUDO:-""}" +USE_CLANG="${USE_CLANG:-false}" function dnf_install { dnf install -y -q --setopt=install_weak_deps=False "$@" @@ -71,141 +69,57 @@ function install_velox_deps_from_dnf { dnf_install libevent-devel \ openssl-devel re2-devel libzstd-devel lz4-devel double-conversion-devel \ libdwarf-devel elfutils-libelf-devel curl-devel libicu-devel bison flex \ - libsodium-devel zlib-devel - - # install sphinx for doc gen - pip install sphinx sphinx-tabs breathe sphinx_rtd_theme + libsodium-devel zlib-devel gtest-devel gmock-devel } function install_conda { dnf_install conda } -function install_gflags { - # Remove an older version if present. - dnf remove -y gflags - wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags - cmake_install gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 -} - -function install_glog { - wget_and_untar https://github.com/google/glog/archive/v0.6.0.tar.gz glog - cmake_install glog -DBUILD_SHARED_LIBS=ON -} - -function install_lzo { - wget_and_untar http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz lzo - ( - cd lzo - ./configure --prefix=/usr --enable-shared --disable-static --docdir=/usr/share/doc/lzo-2.10 - make "-j$(nproc)" - make install - ) -} - -function install_boost { - wget_and_untar https://github.com/boostorg/boost/releases/download/${BOOST_VERSION}/${BOOST_VERSION}.tar.gz boost - ( - cd boost - if [[ ${USE_CLANG} != "false" ]]; then - ./bootstrap.sh --prefix=/usr/local --with-toolset="clang-15" - # Switch the compiler from the clang-15 toolset which doesn't exist (clang-15.jam) to - # clang of version 15 when toolset clang-15 is used. - # This reconciles the project-config.jam generation with what the b2 build system allows for customization. - sed -i 's/using clang-15/using clang : 15/g' project-config.jam - ${SUDO} ./b2 "-j$(nproc)" -d0 install threading=multi toolset=clang-15 --without-python - else - ./bootstrap.sh --prefix=/usr/local - ${SUDO} ./b2 "-j$(nproc)" -d0 install threading=multi --without-python - fi - ) -} - -function install_snappy { - wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy - cmake_install snappy -DSNAPPY_BUILD_TESTS=OFF -} - -function install_fmt { - wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt - cmake_install fmt -DFMT_TEST=OFF -} - -function install_protobuf { - wget_and_untar https://github.com/protocolbuffers/protobuf/releases/download/v21.8/protobuf-all-21.8.tar.gz protobuf - ( - cd protobuf - ./configure --prefix=/usr - make "-j${NPROC}" - make install - ldconfig - ) -} - -function install_fizz { - wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz - cmake_install fizz/fizz -DBUILD_TESTS=OFF +function install_cuda { + # See https://developer.nvidia.com/cuda-downloads + dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo + dnf install -y cuda-nvcc-$(echo $1 | tr '.' '-') cuda-cudart-devel-$(echo $1 | tr '.' '-') } -function install_folly { - wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly - cmake_install folly -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON -} +function install_s3 { + dnf -y install libxml2-devel libgsasl-devel libuuid-devel krb5-devel -function install_wangle { - wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle - cmake_install wangle/wangle -DBUILD_TESTS=OFF -} + install_aws_deps -function install_fbthrift { - wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift - cmake_install fbthrift -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF + local MINIO_ARCH="amd64" + local MINIO_OS="linux" + install_minio ${MINIO_ARCH} ${MINIO_OS} } -function install_mvfst { - wget_and_untar https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz mvfst - cmake_install mvfst -DBUILD_TESTS=OFF +function install_gcs { + # Dependencies of GCS, probably a workaround until the docker image is rebuilt + dnf -y install npm curl-devel c-ares-devel + install_gcs-sdk-cpp } -function install_duckdb { - if $BUILD_DUCKDB ; then - echo 'Building DuckDB' - wget_and_untar https://github.com/duckdb/duckdb/archive/refs/tags/v0.8.1.tar.gz duckdb - cmake_install duckdb -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release - fi +function install_abfs { + # Dependencies of Azure Storage Blob cpp + dnf -y install perl-IPC-Cmd openssl + install_azure-storage-sdk-cpp } -function install_arrow { - wget_and_untar https://archive.apache.org/dist/arrow/arrow-${ARROW_VERSION}/apache-arrow-${ARROW_VERSION}.tar.gz arrow - ( - cd arrow/cpp - cmake_install \ - -DARROW_PARQUET=OFF \ - -DARROW_WITH_THRIFT=ON \ - -DARROW_WITH_LZ4=ON \ - -DARROW_WITH_SNAPPY=ON \ - -DARROW_WITH_ZLIB=ON \ - -DARROW_WITH_ZSTD=ON \ - -DARROW_JEMALLOC=OFF \ - -DARROW_SIMD_LEVEL=NONE \ - -DARROW_RUNTIME_SIMD_LEVEL=NONE \ - -DARROW_WITH_UTF8PROC=OFF \ - -DARROW_TESTING=ON \ - -DCMAKE_INSTALL_PREFIX=/usr/local \ - -DCMAKE_BUILD_TYPE=Release \ - -DARROW_BUILD_STATIC=ON \ - -DThrift_SOURCE=BUNDLED - - # Install thrift. - cd _build/thrift_ep-prefix/src/thrift_ep-build - cmake --install ./ --prefix /usr/local/ - ) +function install_hdfs { + github_checkout apache/hawq master + libhdfs3_dir=$DEPENDENCY_DIR/hawq/depends/libhdfs3 + sed -i "/FIND_PACKAGE(GoogleTest REQUIRED)/d" $libhdfs3_dir/CMakeLists.txt + sed -i "s/dumpversion/dumpfullversion/" $libhdfs3_dir/CMake/Platform.cmake + # Dependencies for Hadoop testing + wget_and_untar https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz hadoop + cp -a hadoop /usr/local/ + cmake_install ${libhdfs3_dir} } -function install_cuda { - # See https://developer.nvidia.com/cuda-downloads - dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo - dnf install -y cuda-nvcc-$(echo $1 | tr '.' '-') cuda-cudart-devel-$(echo $1 | tr '.' '-') +function install_adapters { + run_and_time install_s3 + run_and_time install_gcs + run_and_time install_abfs + run_and_time install_hdfs } function install_velox_deps { @@ -225,6 +139,8 @@ function install_velox_deps { run_and_time install_fbthrift run_and_time install_duckdb run_and_time install_arrow + run_and_time install_xsimd + run_and_time install_simdjson } (return 2> /dev/null) && return # If script was sourced, don't run commands. diff --git a/scripts/setup-common.sh b/scripts/setup-common.sh new file mode 100644 index 000000000000..5bdd991e4926 --- /dev/null +++ b/scripts/setup-common.sh @@ -0,0 +1,265 @@ +#!/bin/bash +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") +source $SCRIPTDIR/setup-helper-functions.sh +source $SCRIPTDIR/setup-versions.sh + +NPROC=$(getconf _NPROCESSORS_ONLN) +CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" +DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)} + +BUILD_DUCKDB="${BUILD_DUCKDB:-true}" + +USE_CLANG="${USE_CLANG:-false}" + +MACHINE=$(uname -m) + +function install_fmt { + wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt + cmake_install fmt -DFMT_TEST=OFF +} + +function install_folly { + wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly + cmake_install folly -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON +} + +function install_fizz { + wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz + cmake_install fizz/fizz -DBUILD_TESTS=OFF +} + +function install_wangle { + wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle + cmake_install wangle/wangle -DBUILD_TESTS=OFF +} + +function install_mvfst { + wget_and_untar https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz mvfst + cmake_install mvfst -DBUILD_TESTS=OFF +} + +function install_fbthrift { + wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift + cmake_install fbthrift -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF +} + +function install_duckdb { + if $BUILD_DUCKDB ; then + echo 'Building DuckDB' + wget_and_untar https://github.com/duckdb/duckdb/archive/refs/tags/${DUCKDB_VERSION}.tar.gz duckdb + cmake_install duckdb -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + fi +} + +function install_boost { + wget_and_untar https://github.com/boostorg/boost/releases/download/${BOOST_VERSION}/${BOOST_VERSION}.tar.gz boost + ( + cd boost + if [[ ${USE_CLANG} != "false" ]]; then + ./bootstrap.sh --prefix=/usr/local --with-toolset="clang-15" + # Switch the compiler from the clang-15 toolset which doesn't exist (clang-15.jam) to + # clang of version 15 when toolset clang-15 is used. + # This reconciles the project-config.jam generation with what the b2 build system allows for customization. + sed -i 's/using clang-15/using clang : 15/g' project-config.jam + ${SUDO} ./b2 "-j${NPROC}" -d0 install threading=multi toolset=clang-15 --without-python + else + ./bootstrap.sh --prefix=/usr/local + ${SUDO} ./b2 "-j${NPROC}" -d0 install threading=multi --without-python + fi + ) +} + +function install_protobuf { + wget_and_untar https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-all-${PROTOBUF_VERSION}.tar.gz protobuf + cmake_install protobuf -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_ABSL_PROVIDER=package +} + +function install_double_conversion { + wget_and_untar https://github.com/google/double-conversion/archive/refs/tags/${DOUBLE_CONVERSION_VERSION}.tar.gz double-conversion + cmake_install double-conversion -DBUILD_TESTING=OFF +} + +function install_ranges_v3 { + wget_and_untar https://github.com/ericniebler/range-v3/archive/refs/tags/${RANGE_V3_VERSION}.tar.gz ranges_v3 + cmake_install ranges_v3 -DRANGES_ENABLE_WERROR=OFF -DRANGE_V3_TESTS=OFF -DRANGE_V3_EXAMPLES=OFF +} + +function install_re2 { + wget_and_untar https://github.com/google/re2/archive/refs/tags/${RE2_VERSION}.tar.gz re2 + cmake_install re2 -DRE2_BUILD_TESTING=OFF +} + +function install_gflags { + # Remove an older version if present. + dnf remove -y gflags + wget_and_untar https://github.com/gflags/gflags/archive/${GFLAGS_VERSION}.tar.gz gflags + cmake_install gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 +} + +function install_glog { + wget_and_untar https://github.com/google/glog/archive/${GLOG_VERSION}.tar.gz glog + cmake_install glog -DBUILD_SHARED_LIBS=ON +} + +function install_lzo { + wget_and_untar http://www.oberhumer.com/opensource/lzo/download/lzo-${LZO_VERSION}.tar.gz lzo + ( + cd lzo + ./configure --prefix=/usr --enable-shared --disable-static --docdir=/usr/share/doc/lzo-${LZO_VERSION} + make "-j${NPROC}" + ${SUDO} make install + ) +} + +function install_snappy { + wget_and_untar https://github.com/google/snappy/archive/${SNAPPY_VERSION}.tar.gz snappy + cmake_install snappy -DSNAPPY_BUILD_TESTS=OFF +} + +function install_xsimd { + wget_and_untar https://github.com/xtensor-stack/xsimd/archive/refs/tags/${XSIMD_VERSION}.tar.gz xsimd + cmake_install xsimd +} + +function install_simdjson { + wget_and_untar https://github.com/simdjson/simdjson/archive/refs/tags/v${SIMDJSON_VERSION}.tar.gz simdjson + cmake_install simdjson +} + +function install_arrow { + wget_and_untar https://archive.apache.org/dist/arrow/arrow-${ARROW_VERSION}/apache-arrow-${ARROW_VERSION}.tar.gz arrow + ( + cd arrow/cpp + cmake_install \ + -DARROW_PARQUET=OFF \ + -DARROW_WITH_THRIFT=ON \ + -DARROW_WITH_LZ4=ON \ + -DARROW_WITH_SNAPPY=ON \ + -DARROW_WITH_ZLIB=ON \ + -DARROW_WITH_ZSTD=ON \ + -DARROW_JEMALLOC=OFF \ + -DARROW_SIMD_LEVEL=NONE \ + -DARROW_RUNTIME_SIMD_LEVEL=NONE \ + -DARROW_WITH_UTF8PROC=OFF \ + -DARROW_TESTING=ON \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ + -DARROW_BUILD_STATIC=ON \ + -DThrift_SOURCE=BUNDLED + + # Install thrift. + cd _build/thrift_ep-prefix/src/thrift_ep-build + ${SUDO} cmake --install ./ --prefix /usr/local/ + ) +} + +# Adapters that can be installed. + +function install_aws_deps { + local AWS_REPO_NAME="aws/aws-sdk-cpp" + local + + github_checkout $AWS_REPO_NAME $AWS_SDK_VERSION --depth 1 --recurse-submodules + cmake_install -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS:BOOL=OFF -DMINIMIZE_SIZE:BOOL=ON -DENABLE_TESTING:BOOL=OFF -DBUILD_ONLY:STRING="s3;identity-management" +} + +function install_minio { + local MINIO_ARCH=$1 + local MINIO_OS=$2 + wget https://dl.min.io/server/minio/release/${MINIO_OS}-${MINIO_ARCH}/archive/minio.RELEASE.${MINIO_VERSION} -O ${MINIO_BINARY_NAME} + chmod +x ./${MINIO_BINARY_NAME} + ${SUDO} mv ./${MINIO_BINARY_NAME} /usr/local/bin/ +} + +function install_gcs-sdk-cpp { + # Install gcs dependencies + # https://github.com/googleapis/google-cloud-cpp/blob/main/doc/packaging.md#required-libraries + + # abseil-cpp + github_checkout abseil/abseil-cpp ${ABSEIL_VERSION} --depth 1 + cmake_install \ + -DABSL_BUILD_TESTING=OFF \ + -DCMAKE_CXX_STANDARD=17 \ + -DABSL_PROPAGATE_CXX_STD=ON \ + -DABSL_ENABLE_INSTALL=ON + + # protobuf + github_checkout protocolbuffers/protobuf v${PROTOBUF_VERSION} --depth 1 + cmake_install \ + -Dprotobuf_BUILD_TESTS=OFF \ + -Dprotobuf_ABSL_PROVIDER=package + + # grpc + github_checkout grpc/grpc ${GRPC_VERSION} --depth 1 + cmake_install \ + -DgRPC_BUILD_TESTS=OFF \ + -DgRPC_ABSL_PROVIDER=package \ + -DgRPC_ZLIB_PROVIDER=package \ + -DgRPC_CARES_PROVIDER=package \ + -DgRPC_RE2_PROVIDER=package \ + -DgRPC_SSL_PROVIDER=package \ + -DgRPC_PROTOBUF_PROVIDER=package \ + -DgRPC_INSTALL=ON + + # crc32 + github_checkout google/crc32c ${CRC32_VERSION} --depth 1 + cmake_install \ + -DCRC32C_BUILD_TESTS=OFF \ + -DCRC32C_BUILD_BENCHMARKS=OFF \ + -DCRC32C_USE_GLOG=OFF + + # nlohmann json + github_checkout nlohmann/json ${NLOHMAN_JSON_VERSION} --depth 1 + cmake_install \ + -DJSON_BuildTests=OFF + + # google-cloud-cpp + github_checkout googleapis/google-cloud-cpp ${GOOGLE_CLOUD_CPP_VERSION} --depth 1 + cmake_install \ + -DGOOGLE_CLOUD_CPP_ENABLE_EXAMPLES=OFF \ + -DGOOGLE_CLOUD_CPP_ENABLE=storage +} + +function install_azure-storage-sdk-cpp { + # Disable VCPKG to install additional static dependencies under the VCPKG installed path + # instead of using system pre-installed dependencies. + export AZURE_SDK_DISABLE_AUTO_VCPKG=ON + vcpkg_commit_id=7a6f366cefd27210f6a8309aed10c31104436509 + github_checkout azure/azure-sdk-for-cpp azure-storage-files-datalake_${AZURE_SDK_VERSION} + sed -i "s/set(VCPKG_COMMIT_STRING .*)/set(VCPKG_COMMIT_STRING $vcpkg_commit_id)/" cmake-modules/AzureVcpkg.cmake + + azure_core_dir="sdk/core/azure-core" + if ! grep -q "baseline" $azure_core_dir/vcpkg.json; then + # build and install azure-core with the version compatible with system pre-installed openssl + openssl_version=$(openssl version -v | awk '{print $2}') + if [[ "$openssl_version" == 1.1.1* ]]; then + openssl_version="1.1.1n" + fi + sed -i "s/\"version-string\"/\"builtin-baseline\": \"$vcpkg_commit_id\",\"version-string\"/" $azure_core_dir/vcpkg.json + sed -i "s/\"version-string\"/\"overrides\": [{ \"name\": \"openssl\", \"version-string\": \"$openssl_version\" }],\"version-string\"/" $azure_core_dir/vcpkg.json + fi + cmake_install $azure_core_dir -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF + + # install azure-storage-common + cmake_install sdk/storage/azure-storage-common -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF + + # install azure-storage-blobs + cmake_install sdk/storage/azure-storage-blobs -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF + + # install azure-storage-files-datalake + cmake_install sdk/storage/azure-storage-files-datalake -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_SHARED_LIBS=OFF +} diff --git a/scripts/setup-macos.sh b/scripts/setup-macos.sh index 450324e8b19a..cf622a4cc85e 100755 --- a/scripts/setup-macos.sh +++ b/scripts/setup-macos.sh @@ -29,16 +29,13 @@ set -e # Exit on error. set -x # Print commands that are executed. SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") -source $SCRIPTDIR/setup-helper-functions.sh +source $SCRIPTDIR/setup-common.sh PYTHON_VENV=${PYHTON_VENV:-"${SCRIPTDIR}/../.venv"} -NPROC=$(getconf _NPROCESSORS_ONLN) - -DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)} -MACOS_VELOX_DEPS="bison boost double-conversion flex fmt gflags glog googletest icu4c libevent libsodium lz4 lzo openssl protobuf@21 simdjson snappy thrift xz xsimd zstd" +MACOS_VELOX_DEPS="bison double-conversion flex gflags glog googletest icu4c libevent libsodium lz4 lzo openssl protobuf@21 simdjson snappy thrift xz xsimd zstd" MACOS_BUILD_DEPS="ninja cmake ccache" -FB_OS_VERSION="v2024.05.20.00" -FMT_VERSION="10.1.1" + +SUDO="${SUDO:-""}" function update_brew { DEFAULT_BREW_PATH=/usr/local/bin/brew @@ -86,49 +83,42 @@ function install_velox_deps_from_brew { done } -function install_fmt { - wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt - cmake_install fmt -DFMT_TEST=OFF -} +function install_s3 { + brew install libxml2 gsasl -function install_folly { - wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly - cmake_install folly -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON -} + install_aws_deps -function install_fizz { - wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz - cmake_install fizz/fizz -DBUILD_TESTS=OFF -} - -function install_wangle { - wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle - cmake_install wangle/wangle -DBUILD_TESTS=OFF -} - -function install_mvfst { - wget_and_untar https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz mvfst - cmake_install mvfst -DBUILD_TESTS=OFF + local MINIO_ARCH=$MACHINE + if [[ $MACHINE == aarch64 ]]; then + MINIO_ARCH="arm64" + elif [[ $MACHINE == x86_64 ]]; then + MINIO_ARCH="amd64" + fi + local MINIO_OS="darwin" + install_minio ${MINIO_ARCH} ${MINIO_OS} } -function install_fbthrift { - wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift - cmake_install fbthrift -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF +function install_gcs { + install_gcs-sdk-cpp } -function install_double_conversion { - wget_and_untar https://github.com/google/double-conversion/archive/refs/tags/v3.1.5.tar.gz double-conversion - cmake_install double-conversion -DBUILD_TESTING=OFF +function install_abfs { + install_azure-storage-sdk-cpp } -function install_ranges_v3 { - wget_and_untar https://github.com/ericniebler/range-v3/archive/refs/tags/0.12.0.tar.gz ranges_v3 - cmake_install ranges_v3 -DRANGES_ENABLE_WERROR=OFF -DRANGE_V3_TESTS=OFF -DRANGE_V3_EXAMPLES=OFF +function install_hdfs { + github_checkout apache/hawq master + libhdfs3_dir=$DEPENDENCY_DIR/hawq/depends/libhdfs3 + sed -i '' -e "/FIND_PACKAGE(GoogleTest REQUIRED)/d" $libhdfs3_dir/CMakeLists.txt + sed -i '' -e "s/dumpversion/dumpfullversion/" $libhdfs3_dir/CMakeLists.txt + cmake_install ${libhdfs3_dir} } -function install_re2 { - wget_and_untar https://github.com/google/re2/archive/refs/tags/2022-02-01.tar.gz re2 - cmake_install re2 -DRE2_BUILD_TESTING=OFF +function install_adapters { + run_and_time install_s3 + run_and_time install_gcs + run_and_time install_abfs + run_and_time install_hdfs } function install_velox_deps { @@ -136,6 +126,7 @@ function install_velox_deps { run_and_time install_ranges_v3 run_and_time install_double_conversion run_and_time install_re2 + run_and_time install_boost run_and_time install_fmt run_and_time install_folly run_and_time install_fizz diff --git a/scripts/setup-ubuntu.sh b/scripts/setup-ubuntu.sh index e40bba1d9923..02c916575352 100755 --- a/scripts/setup-ubuntu.sh +++ b/scripts/setup-ubuntu.sh @@ -28,16 +28,8 @@ # Minimal setup for Ubuntu 22.04. set -eufx -o pipefail SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") -source $SCRIPTDIR/setup-helper-functions.sh +source $SCRIPTDIR/setup-common.sh -# Folly must be built with the same compiler flags so that some low level types -# are the same size. -COMPILER_FLAGS=$(get_cxx_flags) -export COMPILER_FLAGS -NPROC=$(getconf _NPROCESSORS_ONLN) -DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)} -BUILD_DUCKDB="${BUILD_DUCKDB:-true}" -export CMAKE_BUILD_TYPE=Release SUDO="${SUDO:-"sudo --preserve-env"}" USE_CLANG="${USE_CLANG:-false}" @@ -53,11 +45,6 @@ function install_clang15 { ${SUDO} apt install ${CLANG_PACKAGE_LIST} -y } -FB_OS_VERSION="v2024.05.20.00" -FMT_VERSION="10.1.1" -BOOST_VERSION="boost-1.84.0" -ARROW_VERSION="15.0.0" - # Install packages required for build. function install_build_prerequisites { ${SUDO} apt update @@ -94,6 +81,7 @@ function install_velox_deps_from_apt { libgoogle-glog-dev \ libbz2-dev \ libgflags-dev \ + libgtest-dev \ libgmock-dev \ libevent-dev \ liblz4-dev \ @@ -110,53 +98,6 @@ function install_velox_deps_from_apt { tzdata } -function install_fmt { - wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt - cmake_install fmt -DFMT_TEST=OFF -} - -function install_boost { - wget_and_untar https://github.com/boostorg/boost/releases/download/${BOOST_VERSION}/${BOOST_VERSION}.tar.gz boost - ( - cd boost - if [[ ${USE_CLANG} != "false" ]]; then - ./bootstrap.sh --prefix=/usr/local --with-toolset="clang-15" - # Switch the compiler from the clang-15 toolset which doesn't exist (clang-15.jam) to - # clang of version 15 when toolset clang-15 is used. - # This reconciles the project-config.jam generation with what the b2 build system allows for customization. - sed -i 's/using clang-15/using clang : 15/g' project-config.jam - ${SUDO} ./b2 "-j$(nproc)" -d0 install threading=multi toolset=clang-15 --without-python - else - ./bootstrap.sh --prefix=/usr/local - ${SUDO} ./b2 "-j$(nproc)" -d0 install threading=multi --without-python - fi - ) -} - -function install_folly { - wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly - cmake_install folly -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON -} - -function install_fizz { - wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz - cmake_install fizz/fizz -DBUILD_TESTS=OFF -} - -function install_wangle { - wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle - cmake_install wangle/wangle -DBUILD_TESTS=OFF -} - -function install_mvfst { - wget_and_untar https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz mvfst - cmake_install mvfst -DBUILD_TESTS=OFF -} - -function install_fbthrift { - wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift - cmake_install fbthrift -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF -} function install_conda { MINICONDA_PATH="${HOME:-/opt}/miniconda-for-velox" @@ -176,41 +117,6 @@ function install_conda { ) } -function install_duckdb { - if $BUILD_DUCKDB ; then - echo 'Building DuckDB' - wget_and_untar https://github.com/duckdb/duckdb/archive/refs/tags/v0.8.1.tar.gz duckdb - cmake_install duckdb -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release - fi -} - -function install_arrow { - wget_and_untar https://archive.apache.org/dist/arrow/arrow-${ARROW_VERSION}/apache-arrow-${ARROW_VERSION}.tar.gz arrow - ( - cd arrow/cpp - cmake_install \ - -DARROW_PARQUET=OFF \ - -DARROW_WITH_THRIFT=ON \ - -DARROW_WITH_LZ4=ON \ - -DARROW_WITH_SNAPPY=ON \ - -DARROW_WITH_ZLIB=ON \ - -DARROW_WITH_ZSTD=ON \ - -DARROW_JEMALLOC=OFF \ - -DARROW_SIMD_LEVEL=NONE \ - -DARROW_RUNTIME_SIMD_LEVEL=NONE \ - -DARROW_WITH_UTF8PROC=OFF \ - -DARROW_TESTING=ON \ - -DCMAKE_INSTALL_PREFIX=/usr/local \ - -DCMAKE_BUILD_TYPE=Release \ - -DARROW_BUILD_STATIC=ON \ - -DThrift_SOURCE=BUNDLED - - # Install thrift. - cd _build/thrift_ep-prefix/src/thrift_ep-build - $SUDO cmake --install ./ --prefix /usr/local/ - ) -} - function install_cuda { # See https://developer.nvidia.com/cuda-downloads if ! dpkg -l cuda-keyring 1>/dev/null; then @@ -222,10 +128,56 @@ function install_cuda { $SUDO apt install -y cuda-nvcc-$(echo $1 | tr '.' '-') cuda-cudart-dev-$(echo $1 | tr '.' '-') } +function install_s3 { + apt install -y --no-install-recommends libxml2-dev libgsasl7-dev uuid-dev + + install_aws_deps + + local MINIO_ARCH=$MACHINE + if [[ $MACHINE == aarch64 ]]; then + MINIO_ARCH="arm64" + elif [[ $MACHINE == x86_64 ]]; then + MINIO_ARCH="amd64" + fi + local MINIO_OS="linux" + install_minio ${MINIO_ARCH} ${MINIO_OS} +} + +function install_gcs { + # Dependencies of GCS, probably a workaround until the docker image is rebuilt + apt install -y --no-install-recommends libc-ares-dev libcurl4-openssl-dev + install_gcs-sdk-cpp +} + +function install_abfs { + # Dependencies of Azure Storage Blob cpp + apt install -y openssl + install_azure-storage-sdk-cpp +} + +function install_hdfs { + github_checkout apache/hawq master + libhdfs3_dir=$DEPENDENCY_DIR/hawq/depends/libhdfs3 + sed -i "/FIND_PACKAGE(GoogleTest REQUIRED)/d" $libhdfs3_dir/CMakeLists.txt + sed -i "s/dumpversion/dumpfullversion/" $libhdfs3_dir/CMake/Platform.cmake + # Dependencies for Hadoop testing + wget_and_untar https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz hadoop + ${SUDO} cp -a hadoop /usr/local/ + cmake_install ${libhdfs3_dir} +} + +function install_adapters { + run_and_time install_s3 + run_and_time install_gcs + run_and_time install_abfs + run_and_time install_hdfs +} + function install_velox_deps { run_and_time install_velox_deps_from_apt run_and_time install_fmt run_and_time install_boost + run_and_time install_protobuf run_and_time install_folly run_and_time install_fizz run_and_time install_wangle @@ -234,6 +186,8 @@ function install_velox_deps { run_and_time install_conda run_and_time install_duckdb run_and_time install_arrow + run_and_time install_xsimd + run_and_time install_simdjson } function install_apt_deps { diff --git a/scripts/setup-versions.sh b/scripts/setup-versions.sh new file mode 100644 index 000000000000..217d82c0e12f --- /dev/null +++ b/scripts/setup-versions.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The versions of the dependencies are also listed in the +# README.md velox/CMake/resolve_dependency_modules/README.md +# The versions should match the declared versions in this file. + +# Build dependencies versions. +FB_OS_VERSION="v2024.05.20.00" +FMT_VERSION="10.1.1" +BOOST_VERSION="boost-1.84.0" +ARROW_VERSION="15.0.0" +DUCKDB_VERSION="v0.8.1" +PROTOBUF_VERSION="21.8" +MINIO_VERSION="2022-05-26T05-48-41Z" +MINIO_BINARY_NAME="minio-2022-05-26" +AWS_SDK_VERSION="1.11.321" +XSIMD_VERSION="10.0.0" +SIMDJSON_VERSION="3.9.3" +CPR_VERSION="1.10.5" +DOUBLE_CONVERSION_VERSION="v3.1.5" +RANGE_V3_VERSION="0.12.0" +RE2_VERSION="2022-02-01" +GFLAGS_VERSION="v2.2.2" +GLOG_VERSION="v0.6.0" +LZO_VERSION="2.10" +SNAPPY_VERSION="1.1.8" + +# Adapter related versions. +ABSEIL_VERSION="20240116.2" +GRPC_VERSION="v1.48.1" +CRC32_VERSION="1.1.2" +NLOHMAN_JSON_VERSION="v3.11.3" +GOOGLE_CLOUD_CPP_VERSION="v2.22.0" +HADOOP_VERSION="2.10.1" +AZURE_SDK_VERSION="12.8.0"