refactor(interactive): Unify devcontainer.json of graphscope-dev and flex

.github/workflows/build-graphscope-dev-images.yml

@@ -33,7 +33,7 @@ on:
     branches:
       - main
     paths:
-      - 'python/graphscope/gsctl/**'
+      - 'python/graphscope/gsctl/scripts/**'
 
 concurrency:
   group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }}
@@ -45,7 +45,7 @@ env:
 jobs:
   build-wheel-image-x86-64:
     runs-on: ubuntu-20.04
-    if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_wheel == 'true') || (github.pull_request == 'pull_request')
+    if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_wheel == 'true') || (github.event_name == 'pull_request')
 
     steps:
     - uses: actions/checkout@v3
@@ -82,7 +82,7 @@ jobs:
 
   build-wheel-image-aarch64:
     runs-on: [self-hosted, Linux, ARM64]
-    # if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_wheel == 'true') || (github.pull_request == 'pull_request')
+    # if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_wheel == 'true') || (github.event_name == 'pull_request')
     if: false
 
     steps:
@@ -151,7 +151,7 @@ jobs:
 
   build-graphscope-dev-image-x86-64:
     runs-on: ubuntu-20.04
-    if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_graphscope_dev == 'true') || (github.pull_request == 'pull_request')
+    if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_graphscope_dev == 'true') || (github.event_name == 'pull_request')
 
     steps:
     - uses: actions/checkout@v3
@@ -188,7 +188,7 @@ jobs:
 
   build-graphscope-dev-image-aarch64:
     runs-on: [self-hosted, Linux, ARM64]
-    # if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_graphscope_dev == 'true') || (github.pull_request == 'pull_request')
+    # if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_graphscope_dev == 'true') || (github.event_name == 'pull_request')
     if: false
 
     steps:
@@ -257,7 +257,7 @@ jobs:
 
   build-vineyard-dev-image-x86-64:
     runs-on: ubuntu-20.04
-    if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_vineyard_dev == 'true') || (github.pull_request == 'pull_request')
+    if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_vineyard_dev == 'true') || (github.event_name == 'pull_request')
 
     steps:
     - uses: actions/checkout@v3
@@ -294,7 +294,7 @@ jobs:
 
   build-vineyard-dev-image-aarch64:
     runs-on: [self-hosted, Linux, ARM64]
-    # if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_vineyard_dev == 'true') || (github.pull_request == 'pull_request')
+    # if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_vineyard_dev == 'true') || (github.event_name == 'pull_request')
     if: false
 
     steps:
@@ -365,7 +365,7 @@ jobs:
     runs-on: ubuntu-20.04
     # only trigger this step in 'workflow_dispatch' event,
     # since the 'vineyard-dev' image isn't actually pushed in 'pull_request'
-    if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }}
+    if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }} || (github.event_name == 'pull_request')
 
     needs: [build-vineyard-dev-image-x86-64]
     steps:
@@ -387,6 +387,7 @@ jobs:
         fi
 
     - name: Release Image
+      if: ${{ github.event_name == 'workflow_dispatch' }}
       env:
         docker_password: ${{ secrets.DOCKER_PASSWORD }}
         docker_username: ${{ secrets.DOCKER_USER }}
@@ -404,7 +405,7 @@ jobs:
     runs-on: [self-hosted, Linux, ARM64]
     # only trigger this step in 'workflow_dispatch' event,
     # since the 'vineyard-dev' image isn't actually pushed in 'pull_request'
-    # if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }}
+    # if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }} || (github.event_name == 'pull_request')
     if: false
 
     needs: [manifest-push-vineyard-dev-image]

.github/workflows/interactive.yml

@@ -28,7 +28,7 @@ jobs:
     runs-on: ubuntu-20.04
     if: ${{ github.repository == 'alibaba/GraphScope' }}
     container:
-      image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.10
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.23.0
     steps:
     - uses: actions/checkout@v3
 
@@ -68,6 +68,7 @@ jobs:
         GIE_HOME: ${{ github.workspace }}/interactive_engine/
         HOME: /home/graphscope/
       run: |
+        . /home/graphscope/.graphscope_env
         cd ${GITHUB_WORKSPACE}/
         git submodule update --init
         cd ${GITHUB_WORKSPACE}/flex
@@ -311,6 +312,7 @@ jobs:
     - name: Run Gremlin test on modern graph
       env:
         GS_TEST_DIR: ${{ github.workspace }}/gstest
+        HOME : /home/graphscope/
         INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
         LD_LIBRARY_PATH: /usr/local/lib
       run: |
@@ -352,7 +354,7 @@ jobs:
   test-cmake-options:
     runs-on: ubuntu-20.04
     container:
-      image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.10
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.23.0
     strategy:
       matrix:
         BUILD_HQPS: [ON, OFF]
@@ -373,15 +375,15 @@ jobs:
   test-AOCC-compilation:
     runs-on: ubuntu-20.04
     container:
-      image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.10
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.23.0
     steps:
     - uses: actions/checkout@v3
 
     - name: Download aocc compiler
       run: |
         cd ${GITHUB_WORKSPACE}/flex
         wget https://download.amd.com/developer/eula/aocc/aocc-4-1/aocc-compiler-4.1.0_1_amd64.deb
-        sudo apt-get update && sudo apt-get install libncurses-dev libncurses5-dev
+        sudo apt-get update && sudo apt-get install -y libncurses-dev libncurses5-dev
         sudo dpkg -i aocc-compiler-4.1.0_1_amd64.deb
 
     - name: Test AOCC compiler
@@ -402,7 +404,7 @@ jobs:
     runs-on: ubuntu-20.04
     if: ${{ github.repository == 'alibaba/GraphScope' }}
     container:
-      image: registry.cn-hongkong.aliyuncs.com/graphscope/hqps-server-base:v0.0.10
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.23.0
     steps:
     - uses: actions/checkout@v3
 

flex/resources/hqps/CMakeLists.txt.template

@@ -33,6 +33,12 @@ else ()
 endif()
 
 include_directories(${CMAKE_CURRENT_SOURCE_DIR})
+
+# Check if /opt/graphscope exists and include directory. Since grape headers may installed here.
+if(EXISTS "/opt/graphscope/include")
+    include_directories("/opt/graphscope/include")
+endif()
+
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp -Wl,-rpath,$ORIGIN -O0 -flto -Werror=unused-result -fPIC -no-pie")
 

python/graphscope/gsctl/scripts/install_deps_command.sh

@@ -141,7 +141,9 @@ _install_apache_arrow_ubuntu() {
       -P /tmp/
     ${SUDO} apt-get install -y -V /tmp/apache-arrow-apt-source-latest-"$(lsb_release --codename --short)".deb
     ${SUDO} apt-get update -y
-    ${SUDO} apt-get install -y libarrow-dev libarrow-dataset-dev libarrow-acero-dev libparquet-dev
+    ${SUDO} apt-get install -y libarrow-dev=15.0.2-1 libarrow-dataset-dev=15.0.2-1 libarrow-acero-dev=15.0.2-1 libparquet-dev=15.0.2-1
+    #TODO(zhanglei): Remove the arrow version constrait after new version of vineyard is release. see https://github.com/v6d-io/v6d/pull/1911 and https://github.com/apache/incubator-graphar/issues/515
+    #${SUDO} apt-get install -y libarrow-dev libarrow-dataset-dev libarrow-acero-dev libparquet-dev
     rm /tmp/apache-arrow-apt-source-latest-*.deb
   else
     log "apache-arrow (libarrow-dev) already installed, skip."
@@ -321,6 +323,10 @@ install_dependencies_analytical_universal() {
   fi
 }
 
+install_interactive_deps() {
+  install_hiactor "${install_prefix}"
+}
+
 write_env_config() {
   log "Output environments config file ${OUTPUT_ENV_FILE}"
   if [ -f "${OUTPUT_ENV_FILE}" ]; then
@@ -330,7 +336,7 @@ write_env_config() {
 
   {
     echo "export GRAPHSCOPE_HOME=${install_prefix}"
-    echo "export CMAKE_PREFIX_PATH=/opt/vineyard"
+    echo "export CMAKE_PREFIX_PATH=/opt/vineyard:/opt/graphscope/"
     echo "export PATH=${install_prefix}/bin:\$HOME/.local/bin:\$HOME/.cargo/bin:\$PATH"
     echo "export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}"
     echo "export LIBRARY_PATH=${install_prefix}/lib:${install_prefix}/lib64"
@@ -407,6 +413,13 @@ install_deps_for_dev() {
     install_llvm_universal
     install_rust_universal
     install_cppkafka "${deps_prefix}" "${install_prefix}"
+    # install dependencies for flex interactive
+    # can not install on macos since seastar can not be built on macos
+    if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then
+      warning "Skip installing dependencies for flex interactive on macOS."
+    else
+      install_interactive_deps
+    fi
   fi
 
   write_env_config

python/graphscope/gsctl/scripts/lib/install_thirdparty_dependencies.sh

@@ -434,3 +434,16 @@ install_maven() {
   popd || exit
   cleanup_files "${workdir}/${directory}" "${workdir}/${file}"
 }
+
+install_hiactor() {
+  install_prefix=$1
+  pushd /tmp
+  git clone https://github.com/alibaba/hiactor.git -b v0.1.1 --single-branch
+  cd hiactor && git submodule update --init --recursive
+  sudo bash ./seastar/seastar/install-dependencies.sh
+  mkdir build && cd build
+  cmake -DHiactor_DEMOS=OFF -DHiactor_TESTING=OFF -DHiactor_DPDK=OFF -DCMAKE_INSTALL_PREFIX="${install_prefix}" \
+        -DHiactor_CXX_DIALECT=gnu++17 -DSeastar_CXX_FLAGS="-DSEASTAR_DEFAULT_ALLOCATOR -mno-avx512" ..
+  make -j 4 && make install
+  popd && rm -rf /tmp/hiactor
+}