ci: integrate install dependencies script of Interactive into gsctl (#…

…4234) ```bash # pip3 install gsctl gsctl install-deps dev-interactive # or pip3 install click cd GraphScope python3 gsctl.py install-deps dev-interactive ```
alibaba · Sep 18, 2024 · 1130f91 · 1130f91
1 parent 94e5142
commit 1130f91
Show file tree

Hide file tree

Showing 3 changed files with 86 additions and 106 deletions.
diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml
@@ -10,14 +10,14 @@ on:
     paths:
       - 'flex/**'
       - 'interactive_engine/**'
-      - '.github/workflows/hqps-db-ci.yml'
+      - '.github/workflows/interactive.yml'
   pull_request:
     branches:
       - main
     paths:
       - 'flex/**'
       - 'interactive_engine/**'
-      - '.github/workflows/hqps-db-ci.yml'
+      - '.github/workflows/interactive.yml'
 
 concurrency:
   group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }}
@@ -328,15 +328,17 @@ jobs:
 
     - name: Install dependencies
       run: |
-        cd ${GITHUB_WORKSPACE}/flex
-        bash ./scripts/install_dependencies.sh 4
+        cd ${GITHUB_WORKSPACE}
+        python3 -m pip install --upgrade pip && python3 -m pip install click
+        python3 ./gsctl.py install-deps dev-interactive
 
     - name: Setup tmate session
       if: false
       uses: mxschmitt/action-tmate@v3
 
     - name: Build
       run: |
+        source ${HOME}/.graphscope_env
         cd ${GITHUB_WORKSPACE}/flex
         git submodule update --init
         mkdir build && cd build # only test default build

diff --git a/flex/scripts/install_dependencies.sh b/flex/scripts/install_dependencies.sh
diff --git a/python/graphscope/gsctl/scripts/install_deps.sh b/python/graphscope/gsctl/scripts/install_deps.sh
@@ -347,8 +347,25 @@ install_openssl_static() {
   rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}"
 }
 
+# arrow for ubuntu and centos
+install_arrow() {
+  if [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then
+    if ! dpkg -s libarrow-dev &>/dev/null; then
+      ${SUDO} apt-get install -y lsb-release
+      # shellcheck disable=SC2046,SC2019,SC2018
+      wget -c https://apache.jfrog.io/artifactory/arrow/"$(lsb_release --id --short | tr 'A-Z' 'a-z')"/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb -P /tmp/
+      ${SUDO} apt-get install -y -V /tmp/apache-arrow-apt-source-latest-"$(lsb_release --codename --short)".deb
+      ${SUDO} apt-get update -y
+      ${SUDO} apt-get install -y libarrow-dev=${ARROW_VERSION}-1 libarrow-dataset-dev=${ARROW_VERSION}-1 libarrow-acero-dev=${ARROW_VERSION}-1 libparquet-dev=${ARROW_VERSION}-1
+      rm /tmp/apache-arrow-apt-source-latest-*.deb
+    fi
+  elif [[ "${OS_PLATFORM}" == *"CentOS"* || "${OS_PLATFORM}" == *"Aliyun"* ]]; then
+    install_arrow_centos
+  fi
+}
+
 # arrow for centos
-install_apache_arrow() {
+install_arrow_centos() {
   if [[ -f "${install_prefix}/include/arrow/api.h" ]]; then
     return 0
   fi
@@ -470,6 +487,28 @@ install_zlib() {
   rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}"
 }
 
+# opentelemetry
+install_opentelemetry() {
+  pushd "${tempdir}" || exit
+  git clone https://github.com/open-telemetry/opentelemetry-cpp -b v1.15.0
+  cd opentelemetry-cpp
+  cmake . -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_INSTALL_PREFIX="${install_prefix}" \
+    -DCMAKE_CXX_STANDARD=17 \
+    -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+    -DBUILD_SHARED_LIBS=ON \
+    -DWITH_OTLP_HTTP=ON \
+    -DWITH_OTLP_GRPC=OFF \
+    -DWITH_ABSEIL=OFF \
+    -DWITH_PROMETHEUS=OFF \
+    -DBUILD_TESTING=OFF \
+    -DWITH_EXAMPLES=OFF
+  make -j -j$(nproc)
+  make install
+  popd || exit
+  rm -rf "${tempdir:?}/opentelemetry-cpp"
+}
+
 # grpc for centos
 install_grpc() {
   if [[ -f "${install_prefix}/include/grpcpp/grpcpp.h" ]]; then
@@ -533,6 +572,24 @@ install_patchelf() {
   rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}"
 }
 
+# libgrape-lite
+install_libgrape_lite() {
+  if [[ -f "${install_prefix}/include/grape/grape.h" ]]; then
+    return 0
+  fi
+  local branch=$1
+  pushd "${tempdir}" || exit
+  git clone -b ${branch} https://github.com/alibaba/libgrape-lite.git
+  cd libgrape-lite
+  cmake . -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_INSTALL_PREFIX="${install_prefix}" \
+    -DBUILD_LIBGRAPELITE_TESTS=OFF
+  make -j$(nproc)
+  make install
+  popd || exit
+  rm -rf "${tempdir:?}/libgrape-lite"
+}
+
 # vineyard
 install_vineyard() {
   if [[ -f "${V6D_PREFIX}/include/vineyard/client/client.h" ]]; then
@@ -705,7 +762,7 @@ ANALYTICAL_CENTOS_8=("${ANALYTICAL_CENTOS_7[@]}" "boost-devel" "gflags-devel" "g
 
 install_analytical_centos_common_dependencies() {
   install_patchelf
-  install_apache_arrow
+  install_arrow_centos
   install_openmpi
   install_protobuf
   install_zlib
@@ -723,15 +780,7 @@ install_analytical_dependencies() {
     # patchelf
     install_patchelf
     # arrow
-    if ! dpkg -s libarrow-dev &>/dev/null; then
-      ${SUDO} apt-get install -y lsb-release 
-      # shellcheck disable=SC2046,SC2019,SC2018
-      wget -c https://apache.jfrog.io/artifactory/arrow/"$(lsb_release --id --short | tr 'A-Z' 'a-z')"/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb -P /tmp/
-      ${SUDO} apt-get install -y -V /tmp/apache-arrow-apt-source-latest-"$(lsb_release --codename --short)".deb
-      ${SUDO} apt-get update -y
-      ${SUDO} apt-get install -y libarrow-dev=${ARROW_VERSION}-1 libarrow-dataset-dev=${ARROW_VERSION}-1 libarrow-acero-dev=${ARROW_VERSION}-1 libparquet-dev=${ARROW_VERSION}-1
-      rm /tmp/apache-arrow-apt-source-latest-*.deb
-    fi
+    install_arrow
     # install boost >= 1.75 for leaf
     install_boost
   else
@@ -782,20 +831,29 @@ install_analytical_java_dependencies() {
   fi
 }
 
-FLEX_INTERACTIVE_UBUNTU=("rapidjson-dev")
-FLEX_INTERACTIVE_CENTOS=("rapidjson-devel")
+INTERACTIVE_MACOS=("apache-arrow" "rapidjson" "boost" "glog" "gflags")
+INTERACTIVE_UBUNTU=("rapidjson-dev" "libgoogle-glog-dev" "libgflags-dev")
+INTERACTIVE_CENTOS=("rapidjson-devel")
 
-install_packages_for_interactive() {
+install_interactive_dependencies() {
+  # dependencies package
   if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then
-    brew install ${FLEX_INTERACTIVE_MACOS[*]}
+    brew install ${INTERACTIVE_MACOS[*]}
   elif [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then
-    DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${SUDO} apt-get install -y ${FLEX_INTERACTIVE_UBUNTU[*]}
+    DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${SUDO} apt-get install -y ${INTERACTIVE_UBUNTU[*]}
+    install_arrow
+    install_boost
+    # hiactor is only supported on ubuntu
+    install_hiactor
+    ${SUDO} sh -c 'echo "fs.aio-max-nr = 1048576" >> /etc/sysctl.conf'
+    ${SUDO} sysctl -p /etc/sysctl.conf
   else
-    ${SUDO} yum install -y ${FLEX_INTERACTIVE_CENTOS[*]}
+    ${SUDO} yum install -y ${INTERACTIVE_CENTOS[*]}
+    install_arrow
+    install_boost
   fi
-}
-
-install_interactive_dependencies() {
+  # libgrape-lite
+  install_libgrape_lite "v0.3.2"
   # java
   install_java_and_maven
   # rust
@@ -806,14 +864,8 @@ install_interactive_dependencies() {
     rustup default 1.71.0
     rustc --version
   fi
-  # hiactor
-  if [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then
-    install_hiactor
-  else
-    warning "Skip installing dependencies for flex interactive on ${OS_PLATFORM}."
-  fi
-  # install rapidjson 
-  install_packages_for_interactive
+  # opentelemetry
+  install_opentelemetry
 }
 
 install_learning_dependencies() {