diff --git a/.github/workflows/flex-interactive-dummy.yml b/.github/workflows/flex-interactive-dummy.yml deleted file mode 100644 index 1d05cffc3363..000000000000 --- a/.github/workflows/flex-interactive-dummy.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Flex Interactive CI (Dummy) - -on: - pull_request: - branches: - - main - paths: - - '**' - - '!flex/**' - - '!coordinator/gscoordinator/flex/**' - - '!python/graphscope/gsctl/**' - - '!.github/workflows/flex-interactive.yml' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -jobs: - build: - runs-on: ubuntu-20.04 - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - run: 'echo "No action required" ' diff --git a/.github/workflows/gae-dummy.yml b/.github/workflows/gae-dummy.yml deleted file mode 100644 index b405f337efc3..000000000000 --- a/.github/workflows/gae-dummy.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: GraphScope GAE CI (Dummy) - -on: - pull_request: - branches: - - main - paths: - - '**' - - '!analytical_engine/**' - - '!.github/workflows/gae.yml' - - '!coordinator/gscoordinator/template/CMakeLists.template' # to verify whether java app can be successfully built when CMakeLists.template changes. - - 'analytical_engine/**.md' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -jobs: - build-gae: - runs-on: ubuntu-20.04 - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - run: 'echo "No action required" ' diff --git a/.github/workflows/gss-dummy.yml b/.github/workflows/gss-dummy.yml deleted file mode 100644 index d309323ab8fb..000000000000 --- a/.github/workflows/gss-dummy.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: GraphScope Store CI (Dummy) - -on: - pull_request: - branches: - - main - paths: - - '**' - - '!proto/**' - - '!interactive_engine/**' - - '!python/graphscope/client/**' - - '!charts/graphscope-store/**' - - '!.github/workflows/gss.yml' - - '!flex/openapi/openapi_coordinator.yaml' - - '!coordinator/gscoordinator/flex/**' - - '!python/graphscope/gsctl/**' - - 'interactive_engine/**.md' - - 'charts/graphscope-store/**.md' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -env: - GSS_IMAGE: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-store - -jobs: - gremlin-test: - # Require the host is able to run docker without sudo and - # can `ssh localhost` without password, which may need to - # be configured manually when a new self-hosted runner is added. - runs-on: ubuntu-20.04 - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - run: 'echo "No action required" ' - - helm-test: - runs-on: ubuntu-20.04 - if: ${{ github.repository == 'alibaba/GraphScope' }} - needs: [gremlin-test] - steps: - - run: 'echo "No action required" ' diff --git a/.github/workflows/k8s-ci-dummy.yml b/.github/workflows/k8s-ci-dummy.yml deleted file mode 100644 index 78857883c413..000000000000 --- a/.github/workflows/k8s-ci-dummy.yml +++ /dev/null @@ -1,115 +0,0 @@ -name: GraphScope CI on Kubernetes (Dummy) - -on: - pull_request: - branches: - - main - paths: - - '**' - - '!.github/workflows/k8s-ci.yml' - - '!Makefile' - - '!analytical_engine/**' - - '!charts/**' - - '!coordinator/**' - - '!interactive_engine/**' - - '!k8s/**' - - '!learning_engine/**' - - '!proto/**' - - '!python/**' - - '**.md' - - '**.rst' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -env: - GS_IMAGE: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope - -jobs: - # JOB to run change detection - changes: - runs-on: ubuntu-20.04 - # Set job outputs to values from filter step - outputs: - gae-python: ${{ steps.filter.outputs.gae-python }} - networkx: ${{ steps.filter.outputs.networkx }} - gie-function-test: ${{ steps.filter.outputs.gie-function-test }} - steps: - # For push it's necessary to checkout the code - - uses: actions/checkout@v4 - # For pull requests it's not necessary to checkout the code - - uses: dorny/paths-filter@v2 - id: filter - with: - base: main # Change detection against merge-base with main before push - filters: | - gae-python: - - 'proto/**' - - 'analytical_engine/**' - - 'python/graphscope/analytical/**' - - 'python/graphscope/client/**' - - 'python/graphscope/dataset/**' - - 'python/graphscope/deploy/**' - - 'python/graphscope/framework/**' - - 'python/graphscope/tests/unittest/**' - - 'coordinator/gscoordinator/**' - - '.github/workflows/ci.yml' - networkx: - - 'analytical_engine/apps/**' - - 'analytical_engine/frame/**' - - 'analytical_engine/core/**' - - 'python/graphscope/nx/**' - gie-function-test: - - 'interactive_engine/**' - - 'python/graphscope/interactive/**' - - '.github/workflows/ci.yml' - - build-analytical: - runs-on: ubuntu-20.04 - needs: [changes] - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - run: 'echo "No action required" ' - - build-analytical-java: - runs-on: ubuntu-20.04 - needs: [changes] - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - run: 'echo "No action required" ' - - build-interactive: - runs-on: ubuntu-20.04 - needs: [changes] - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - run: 'echo "No action required" ' - - build-learning: - runs-on: ubuntu-20.04 - needs: [changes] - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - run: 'echo "No action required" ' - - # build-coordinator: - # runs-on: ubuntu-20.04 - # needs: [changes] - # if: ${{ github.repository == 'alibaba/GraphScope' }} - # steps: - # - run: 'echo "No action required" ' - - k8s-test: - runs-on: ubuntu-20.04 - if: ${{ github.repository == 'alibaba/GraphScope' }} - needs: [build-analytical, build-analytical-java, build-interactive, build-learning] - steps: - - run: 'echo "No action required" ' - - gie-test: - runs-on: ubuntu-20.04 - needs: [build-analytical, build-analytical-java, build-interactive, build-learning] - if: ${{ (needs.changes.outputs.gie-function-test == 'false' || github.ref == 'refs/heads/main') && github.repository == 'alibaba/GraphScope' }} - steps: - - run: 'echo "No action required" ' diff --git a/.github/workflows/local-ci-dummy.yml b/.github/workflows/local-ci-dummy.yml deleted file mode 100644 index 0dc73235cc8c..000000000000 --- a/.github/workflows/local-ci-dummy.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: GraphScope CI on Local (Dummy) - -on: - pull_request: - branches: - - main - paths: - - '**' - - '!.github/workflows/local-ci.yml' - - '!Makefile' - - '!analytical_engine/**' - - '!charts/**' - - '!coordinator/**' - - '!interactive_engine/**' - - '!k8s/**' - - '!learning_engine/**' - - '!proto/**' - - '!python/**' - - '**.md' - - '**.rst' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -env: - GS_IMAGE: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope - -jobs: - # JOB to run change detection - changes: - runs-on: ubuntu-20.04 - # Set job outputs to values from filter step - outputs: - gae-python: ${{ steps.filter.outputs.gae-python }} - networkx: ${{ steps.filter.outputs.networkx }} - gie-function-test: ${{ steps.filter.outputs.gie-function-test }} - steps: - # For push it's necessary to checkout the code - - uses: actions/checkout@v4 - # For pull requests it's not necessary to checkout the code - - uses: dorny/paths-filter@v2 - id: filter - with: - base: main # Change detection against merge-base with main before push - filters: | - gae-python: - - 'proto/**' - - 'analytical_engine/**' - - 'python/graphscope/analytical/**' - - 'python/graphscope/client/**' - - 'python/graphscope/dataset/**' - - 'python/graphscope/deploy/**' - - 'python/graphscope/framework/**' - - 'python/graphscope/tests/unittest/**' - - 'coordinator/gscoordinator/**' - - '.github/workflows/ci.yml' - networkx: - - 'analytical_engine/apps/**' - - 'analytical_engine/frame/**' - - 'analytical_engine/core/**' - - 'python/graphscope/nx/**' - gie-function-test: - - 'interactive_engine/**' - - 'python/graphscope/interactive/**' - - '.github/workflows/ci.yml' - - build-wheels: - runs-on: ubuntu-20.04 - if: ${{ github.repository == 'alibaba/GraphScope' }} - steps: - - run: 'echo "No action required" ' - - mini-test: - runs-on: ubuntu-20.04 - if: ${{ github.repository == 'alibaba/GraphScope' }} - needs: [build-wheels] - steps: - - run: 'echo "No action required" ' - - python-unittest: - runs-on: ubuntu-20.04 - needs: [build-wheels, changes] - strategy: - matrix: - deployment: ["standalone", "distributed"] - steps: - - run: 'echo "No action required" ' - - networkx-basic-test: - runs-on: ubuntu-20.04 - needs: [build-wheels, changes] - strategy: - matrix: - deployment: ["standalone", "distributed"] - steps: - - run: 'echo "No action required" ' - - networkx-algo-and-generator-test: - runs-on: ubuntu-20.04 - needs: [build-wheels, changes] - strategy: - matrix: - deployment: ["standalone", "distributed"] - steps: - - run: 'echo "No action required" ' diff --git a/V6D_VERSION b/V6D_VERSION new file mode 100644 index 000000000000..ca222b7cf394 --- /dev/null +++ b/V6D_VERSION @@ -0,0 +1 @@ +0.23.0 diff --git a/k8s/dockerfiles/graphscope-dev-wheel.Dockerfile b/k8s/dockerfiles/graphscope-dev-wheel.Dockerfile index 1a58a42839f8..20ae2b3532cb 100644 --- a/k8s/dockerfiles/graphscope-dev-wheel.Dockerfile +++ b/k8s/dockerfiles/graphscope-dev-wheel.Dockerfile @@ -62,7 +62,7 @@ COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope ARG VINEYARD_VERSION=main RUN cd /home/graphscope/GraphScope && \ python3 -m pip install click packaging --user && \ - python3 gsctl.py install-deps dev --v6d-version=$VINEYARD_VERSION -j $(nproc) && \ + python3 gsctl.py install-deps dev --v6d-version=$VINEYARD_VERSION && \ sudo rm -rf /home/graphscope/GraphScope && \ sudo yum clean all -y && \ sudo rm -fr /var/cache/yum diff --git a/k8s/dockerfiles/graphscope-dev.Dockerfile b/k8s/dockerfiles/graphscope-dev.Dockerfile index 1c13a487033d..c99e86e0bb6f 100644 --- a/k8s/dockerfiles/graphscope-dev.Dockerfile +++ b/k8s/dockerfiles/graphscope-dev.Dockerfile @@ -29,9 +29,10 @@ WORKDIR /home/graphscope COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope ARG VINEYARD_VERSION=main + RUN cd /home/graphscope/GraphScope && \ python3 -m pip install click packaging && \ - python3 gsctl.py install-deps dev --v6d-version=$VINEYARD_VERSION --cn -j $(nproc) && \ + python3 gsctl.py install-deps dev --v6d-version=$VINEYARD_VERSION --cn && \ cd /home/graphscope && \ rm -fr GraphScope diff --git a/k8s/dockerfiles/manylinux2014.Dockerfile b/k8s/dockerfiles/manylinux2014.Dockerfile index fe2e85349658..d1e70de6a5ab 100644 --- a/k8s/dockerfiles/manylinux2014.Dockerfile +++ b/k8s/dockerfiles/manylinux2014.Dockerfile @@ -22,7 +22,7 @@ ENV LANG=en_US.utf-8 COPY . /root/GraphScope RUN cd /root/GraphScope && \ python3 -m pip install click packaging && \ - python3 gsctl.py install-deps dev --cn --for-analytical --no-v6d -j $(nproc) && \ + python3 gsctl.py install-deps dev-analytical --cn --no-v6d && \ rm -fr /root/GraphScope # install hadoop for processing hadoop data source diff --git a/k8s/dockerfiles/vineyard-dev.Dockerfile b/k8s/dockerfiles/vineyard-dev.Dockerfile index a68df0989273..d3525418efe8 100644 --- a/k8s/dockerfiles/vineyard-dev.Dockerfile +++ b/k8s/dockerfiles/vineyard-dev.Dockerfile @@ -44,7 +44,7 @@ ARG VINEYARD_VERSION=main RUN sudo chmod a+wrx /tmp && \ cd /home/graphscope/GraphScope && \ python3 -m pip install click packaging && \ - python3 gsctl.py install-deps dev --for-analytical --v6d-version=$VINEYARD_VERSION -j $(nproc) && \ + python3 gsctl.py install-deps dev-analytical --v6d-version=$VINEYARD_VERSION && \ cd /home/graphscope && sudo rm -rf /home/graphscope/GraphScope RUN python3 -m pip --no-cache install pyyaml --user diff --git a/python/graphscope/gsctl/V6D_VERSION b/python/graphscope/gsctl/V6D_VERSION new file mode 120000 index 000000000000..ba52b47717e8 --- /dev/null +++ b/python/graphscope/gsctl/V6D_VERSION @@ -0,0 +1 @@ +../../../V6D_VERSION \ No newline at end of file diff --git a/python/graphscope/gsctl/commands/dev.py b/python/graphscope/gsctl/commands/dev.py index 10edb82c1786..6be3f40cca55 100644 --- a/python/graphscope/gsctl/commands/dev.py +++ b/python/graphscope/gsctl/commands/dev.py @@ -26,27 +26,28 @@ import click from packaging import version -from graphscope.gsctl.utils import err -from graphscope.gsctl.utils import info -from graphscope.gsctl.utils import succ - +# Make sure this file doesn't depend on any graphscope directories for +# installing dependencies, so we can't `import graphscope` here. version_file_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), "..", "VERSION" ) - with open(version_file_path, "r", encoding="utf-8") as fp: sv = version.parse(fp.read().strip()) - __is_prerelease__ = sv.is_prerelease __version__ = str(sv) +v6d_version_file_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "..", "V6D_VERSION" +) +with open(v6d_version_file_path, "r", encoding="utf-8") as fp: + sv = version.parse(fp.read().strip()) + __v6d_version__ = str(sv) # Interactive docker container config INTERACTIVE_DOCKER_CONTAINER_NAME = "gs-interactive-instance" INTERACTIVE_DOCKER_CONTAINER_LABEL = "flex=interactive" - scripts_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "scripts") -install_deps_script = os.path.join(scripts_dir, "install_deps_command.sh") +install_deps_script = os.path.join(scripts_dir, "install_deps.sh") default_graphscope_repo_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), "..", @@ -58,7 +59,7 @@ def run_shell_cmd(cmd, workingdir): """wrapper function to run a shell command/scripts.""" - info(f"run a shell command on cwd={workingdir}. \ncmd=\"{' '.join(cmd)}\"") + click.secho(f"run a shell command on cwd={workingdir}. \ncmd=\"{' '.join(cmd)}\"") proc = subprocess.Popen( cmd, cwd=workingdir, @@ -69,9 +70,10 @@ def run_shell_cmd(cmd, workingdir): stdout_wrapper = io.TextIOWrapper(proc.stdout, encoding="utf-8") stderr_wrapper = io.TextIOWrapper(proc.stderr, encoding="utf-8") for line in stdout_wrapper: - info(line.rstrip()) + click.secho(line.rstrip()) for line in stderr_wrapper: - err(line.rstrip()) + click.secho("[FAILED] ", nl=False, fg="red", bold=True) + click.secho(line.rstrip()) proc.wait() return proc.returncode @@ -144,7 +146,8 @@ def interactive(app, graphscope_repo): if not os.path.exists(interactive_build_dir) or not os.path.isdir( interactive_build_dir ): - err( + click.secho("[FAILED] ", nl=False, fg="red", bold=True) + click.secho( f"No such file or directory {interactive_build_dir}, try --graphscope-repo param.", ) return @@ -280,7 +283,8 @@ def deploy( Apart from interactive sdk, you can also use neo4j native tools(like cypher-shell) to connect to cypher endpoint, and gremlin console to connect to gremlin endpoint. """ - succ(message, fg="blue") + click.secho("[SUCCESS] ", nl=False, fg="green", bold=True) + click.secho(message, bold=False, fg="blue") @instance.command @@ -308,7 +312,6 @@ def destroy(type, container_name): "-f", container_name, ] - info("Run command: {0}".format(" ".join(cmd))) result = subprocess.run(cmd, capture_output=True, text=True) if result.returncode == 0: click.secho("[SUCCESS] ", nl=False, fg="green", bold=True) @@ -349,17 +352,20 @@ def status(type): @click.argument( "type", type=click.Choice( - ["dev", "client"], + [ + "dev", + "dev-analytical", + "dev-analytical-java", + "dev-interactive", + "dev-learning", + ], case_sensitive=False, ), required=True, ) @click.option( "--graphscope-repo", - envvar="GRAPHSCOPE_REPO", - type=click.Path(), - default=os.path.abspath("."), - show_default=True, + required=False, help="GraphScope code repo location.", ) @click.option( @@ -373,50 +379,26 @@ def status(type): type=click.Path(), default="/opt/graphscope", show_default=True, - help="Install built binaries to customized location.", -) -@click.option( - "--from-local", - type=click.Path(), - default="/tmp/gs-local-deps", - show_default=True, - help="""Find raw dependencies of GraphScope from a local directory. The raw - dependencies would then be built and installed to [prefix]. If the directory - is empty or not exists, dependency files would be downloaded to [directory].""", + help="Specify the directory on disk to which the file will be installed", ) @click.option( "--v6d-version", - default="main", + default=__v6d_version__, show_default=True, - help="v6d version to clone.", -) -@click.option( - "-j", - "--jobs", - default="2", - help="Concurrent jobs in building, i.e., -j argument passed to make.", -) -@click.option( - "--for-analytical", - is_flag=True, - default=False, - help="Only install analytical engine dependencies.", + help="vineyard version", ) @click.option( "--no-v6d", is_flag=True, default=False, - help="Do not install v6d, for build base docker images, could only be used with '--for-analytical'", + help="Do not install vineyard, could be used with analytical type", ) def install_deps( type, graphscope_repo, cn, install_prefix, - from_local, v6d_version, - jobs, - for_analytical, no_v6d, ): """Install dependencies for building GraphScope.""" @@ -424,23 +406,13 @@ def install_deps( "bash", "-e", install_deps_script, - "-t", type, - "-i", + "--install-prefix", install_prefix, - "-d", - from_local, - "-v", + "--v6d-version", str(v6d_version), - "-j", - str(jobs), ] - if for_analytical: - cmd.append("--for-analytical") if no_v6d: - if not for_analytical: - # could only be used with '--for-analytical' - raise RuntimeError("Missing --for-analytical with --no-v6d parameter") cmd.append("--no-v6d") if cn: cmd.append("--cn") diff --git a/python/graphscope/gsctl/scripts/__init__.py b/python/graphscope/gsctl/scripts/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/python/graphscope/gsctl/scripts/dev_command.sh b/python/graphscope/gsctl/scripts/dev_command.sh deleted file mode 100644 index 6e137d2f9801..000000000000 --- a/python/graphscope/gsctl/scripts/dev_command.sh +++ /dev/null @@ -1,41 +0,0 @@ -script_dir="$(cd "$(dirname "$0")" && pwd)" -source ${script_dir}/lib/get_os_version.sh -source ${script_dir}/lib/log.sh -source ${script_dir}/lib/colors.sh -source ${script_dir}/lib/install_thirdparty_dependencies.sh -source ${script_dir}/lib/install_vineyard.sh -source ${script_dir}/lib/util.sh - - -while getopts ":l:" opt; do - case $opt in - l) - local="$OPTARG" - ;; - \?) - echo "Invalid options: -$OPTARG" >&2 - exit 1 - ;; - :) - echo "The option -$OPTARG requires parameters" >&2 - exit 1 - ;; - esac -done - - -mount_option="" - -if [[ -n $local ]]; then - echo "Opened a new container with $local mounted to /home/graphscope/graphscope." - mount_option="--mount type=bind,source=${local},target=/home/graphscope/graphscope" -else - echo "No local directory assigned, open a new container without mounting local directory." -fi - -# docker pull graphscope/graphscope-dev -REGISTRY="registry.cn-hongkong.aliyuncs.com" -docker run \ - -it \ - ${mount_option} \ - ${REGISTRY}/graphscope/graphscope-dev:latest diff --git a/python/graphscope/gsctl/scripts/format_command.sh b/python/graphscope/gsctl/scripts/format_command.sh deleted file mode 100644 index bfba0b2cfa68..000000000000 --- a/python/graphscope/gsctl/scripts/format_command.sh +++ /dev/null @@ -1,103 +0,0 @@ -script_dir="$(cd "$(dirname "$0")" && pwd)" -source ${script_dir}/lib/get_os_version.sh -source ${script_dir}/lib/log.sh -source ${script_dir}/lib/colors.sh -source ${script_dir}/lib/install_thirdparty_dependencies.sh -source ${script_dir}/lib/install_vineyard.sh -source ${script_dir}/lib/util.sh - - -echo "# this file is located in 'src/format_command.sh'" -echo "# code for 'gs format' goes here" -echo "# you can edit it freely and regenerate (it will not be overwritten)" - -while getopts ":l:" opt; do - case $opt in - l) - lang="$OPTARG" - ;; - \?) - echo "Invalid options: -$OPTARG" >&2 - exit 1 - ;; - :) - echo "The option -$OPTARG requires parameters" >&2 - exit 1 - ;; - esac -done - - - -GS_SOURCE_DIR="$(dirname -- "$(readlink -f "${BASH_SOURCE}")")" - -function format_cpp { - # Check if clang-format is installed - if ! [ -x "$(command -v clang-format)" ]; then - echo 'Downloading clang-format.' >&2 - curl -L https://github.com/muttleyxd/clang-tools-static-binaries/releases/download/master-22538c65/clang-format-8_linux-amd64 --output ${GRAPHSCOPE_HOME}/bin/clang-format - chmod +x ${GRAPHSCOPE_HOME}/clang-format - export PATH="${GRAPHSCOPE_HOME}/bin:${PATH}" - fi - # find all relevant files - pushd "${GS_SOURCE_DIR}"/analytical_engine || exit - files=$(find ./apps ./benchmarks ./core ./frame ./misc ./test \( -name "*.h" -o -name "*.cc" \)) - - # run format - clang-format -i --style=file $(echo $files) - popd || exit -} - -function lint_cpp { - # use cpplint.py for static analysis - pushd "${GS_SOURCE_DIR}"/analytical_engine || exit - files=$(find ./apps ./benchmarks ./core ./frame ./misc ./test \( -name "*.h" -o -name "*.cc" \)) - - ./misc/cpplint.py $(echo $files) - popd || exit -} - -function format_java { - jarfile=google-java-format-1.13.0-all-deps.jar - if [[ ! -f ${jarfile} ]]; then - wget https://github.com/google/google-java-format/releases/download/v1.13.0/${jarfile} - fi - # run formatter in-place - java -jar ${jarfile} --aosp --skip-javadoc-formatting -i $(git ls-files *.java) - -} - -function format_python { - # Install dependency - if ! [ -x "$(command -v black)" ]; then - pip3 install -r ${GS_SOURCE_DIR}/coordinator/requirements-dev.txt --user - fi - pushd python || exit - python3 -m isort --check --diff . - python3 -m black --check --diff . - python3 -m flake8 . - popd || exit - pushd coordinator || exit - python3 -m isort --check --diff . - python3 -m black --check --diff . - python3 -m flake8 . - popd || exit -} - -function format_rust { - # Use cargo fmt for formatting checks - cd "${GS_SOURCE_DIR}"/interactive_engine/executor/assembly/groot - cargo +nightly fmt -- --check - cd "${GS_SOURCE_DIR}"/interactive_engine/executor/assembly/v6d - cargo +nightly fmt -- --check - cd "${GS_SOURCE_DIR}"/interactive_engine/executor/common/dyn_type/ - cargo +nightly fmt -- --check - cd "${GS_SOURCE_DIR}"/interactive_engine/executor/engine/pegasus/ - cargo +nightly fmt -- --check - cd "${GS_SOURCE_DIR}"/interactive_engine/executor/ir/ - cargo +nightly fmt -- --check - cd "${GS_SOURCE_DIR}"/interactive_engine/executor/store/ - cargo +nightly fmt -- --check -} - -format_"${lang}" diff --git a/python/graphscope/gsctl/scripts/initialize.sh b/python/graphscope/gsctl/scripts/initialize.sh deleted file mode 100644 index d3a8414bca95..000000000000 --- a/python/graphscope/gsctl/scripts/initialize.sh +++ /dev/null @@ -1,23 +0,0 @@ -script_dir="$(cd "$(dirname "$0")" && pwd)" -source ${script_dir}/lib/get_os_version.sh -source ${script_dir}/lib/log.sh -source ${script_dir}/lib/colors.sh -source ${script_dir}/lib/install_thirdparty_dependencies.sh -source ${script_dir}/lib/install_vineyard.sh -source ${script_dir}/lib/util.sh - - -## Code here runs inside the initialize() function -## Use it for anything that you need to run before any other function, like -## setting environment variables: -## CONFIG_FILE=settings.ini -## -## Feel free to empty (but not delete) this file. - -bash_source_dir="$(dirname -- "$(readlink -f "${BASH_SOURCE}")")" - -if [ -f "$HOME/.graphscope_env" ]; then - source $HOME/.graphscope_env -fi - -log "Read the env: GRAPHSCOPE_HOME=${GRAPHSCOPE_HOME:-}" diff --git a/python/graphscope/gsctl/scripts/install_deps.sh b/python/graphscope/gsctl/scripts/install_deps.sh new file mode 100755 index 000000000000..cc654d4eda4c --- /dev/null +++ b/python/graphscope/gsctl/scripts/install_deps.sh @@ -0,0 +1,875 @@ +#!/bin/bash +set -e + +# blue +info() { + printf "\e[34m%b\e[0m\n" "$*" +} + +# red +err() { + printf "\e[31m%b\e[0m\n" "$*" +} + +# yellow +warning() { + printf "\e[1;33m%b\e[0m\n" "$*" +} + +# red +debug() { + printf "\e[31m%b\e[0m\n" "[DEBUG] $*" +} + +get_os_version() { + if [ -f /etc/centos-release ]; then + # Older Red Hat, CentOS, Alibaba Cloud Linux etc. + PLATFORM=CentOS + OS_VERSION=$(sed 's/.* \([0-9]\).*/\1/' < /etc/centos-release) + if grep -q "Alibaba Cloud Linux" /etc/centos-release; then + PLATFORM="Aliyun_based_on_CentOS" + OS_VERSION=$(rpm -E %{rhel}) + fi + elif [ -f /etc/os-release ]; then + # freedesktop.org and systemd + . /etc/os-release + PLATFORM="${NAME}" + OS_VERSION="${VERSION_ID}" + elif type lsb_release >/dev/null 2>&1; then + # linuxbase.org + PLATFORM=$(lsb_release -si) + OS_VERSION=$(lsb_release -sr) + elif [ -f /etc/lsb-release ]; then + # For some versions of Debian/Ubuntu without lsb_release command + . /etc/lsb-release + PLATFORM="${DISTRIB_ID}" + OS_VERSION="${DISTRIB_RELEASE}" + elif [ -f /etc/debian_version ]; then + # Older Debian/Ubuntu/etc. + PLATFORM=Debian + OS_VERSION=$(cat /etc/debian_version) + else + # Fall back to uname, e.g. "Linux ", also works for BSD, Darwin, etc. + PLATFORM=$(uname -s) + OS_VERSION=$(uname -r) + fi + if [[ "${PLATFORM}" != *"Ubuntu"* && "${PLATFORM}" != *"CentOS"* && "${PLATFORM}" != *"Darwin"* && "${PLATFORM}" != *"Aliyun"* ]];then + err "Only suppport on Ubuntu/CentOS/macOS/AliyunOS platform." + exit 1 + fi + if [[ "${PLATFORM}" == *"Ubuntu"* && "${OS_VERSION:0:2}" -lt "20" ]]; then + err "Ubuntu ${OS_VERSION} found, requires 20 or greater." + exit 1 + fi + if [[ "${PLATFORM}" == *"CentOS"* && "${OS_VERSION}" -lt "7" ]]; then + err "CentOS ${OS_VERSION} found, requires 8 or greater." + exit 1 + fi + if [[ "${PLATFORM}" == *"Darwin"* ]]; then + export HOMEBREW_NO_INSTALL_CLEANUP=1 + export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 + elif [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then + export DEBIAN_FRONTEND=noninteractive + fi + echo "$PLATFORM-$OS_VERSION" +} + +# default values +readonly OS=$(get_os_version) +readonly OS_PLATFORM=${OS%-*} +readonly OS_VERSION=${OS#*-} +readonly ARCH=$(uname -m) +readonly OUTPUT_ENV_FILE="${HOME}/.graphscope_env" +if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then + readonly HOMEBREW_PREFIX=$(brew --prefix) +fi +readonly ARROW_VERSION="15.0.2" +readonly MPI_PREFIX="/opt/openmpi" # fixed, related to coordinator/setup.py +readonly V6D_PREFIX="/opt/vineyard" # fixed, related to coordinator/setup.py +readonly tempdir="/tmp/gs-local-deps" +v6d_version="main" +no_v6d=false +for_analytical=false +for_analytical_java=false +for_interactive=false +for_learning=false +cn_flag=false +debug_flag=false +install_prefix="/opt/graphscope" + +# parse args +while (( "$#" )); do + case "$1" in + dev) + for_analytical=true + for_interactive=true + for_learning=true + shift + ;; + dev-analytical) + for_analytical=true + shift + ;; + dev-analytical-java) + for_analytical=true + for_analytical_java=true + shift + ;; + dev-interactive) + for_interactive=true + shift + ;; + dev-learning) + for_learning=true + shift + ;; + --install-prefix) + install_prefix="$2" + shift 2 + ;; + --v6d-version) + v6d_version="$2" + shift 2 + ;; + --no-v6d) + no_v6d=true + shift + ;; + --cn) + cn_flag=true + shift + ;; + --debug) + debug_flag=true + shift + ;; + *) + shift + ;; + esac +done + +if [[ ${for_analytical} == false && ${for_interactive} == false && ${for_learning} == false ]]; then + usage="Usage: ${0} dev/dev-analytical/dev-analytical-java/dev-interactive/dev-learning" + usage="${usage} [--cn] [--v6d-version ] [--install-prefix ] [--no-v6d] [--debug]" + err "${usage}" + exit 0 +fi + +if [[ ${debug_flag} == true ]]; then + debug "OS: ${OS}, OS_PLATFORM: ${OS_PLATFORM}, OS_VERSION: ${OS_VERSION}" + debug "install dependencies for analytical=${for_analytical}, analytical-java=${for_analytical_java}, v6d_version=${v6d_version}, no_v6d=${no_v6d}" + debug "install dependencies for interactive=${for_interactive}, learning=${for_learning}" + debug "install prefix: ${install_prefix}" +fi + +# sudo +SUDO=sudo +if [[ $(id -u) -eq 0 ]]; then + SUDO="" +fi + +# speed up +if [ "${cn_flag}" == true ]; then + export HOMEBREW_BREW_GIT_REMOTE="https://mirrors.tuna.tsinghua.edu.cn/git/homebrew/brew.git" + export HOMEBREW_CORE_GIT_REMOTE="https://mirrors.tuna.tsinghua.edu.cn/git/homebrew/homebrew-core.git" + export HOMEBREW_BOTTLE_DOMAIN="https://mirrors.tuna.tsinghua.edu.cn/homebrew-bottles" +fi + +# install functions +init_workspace_and_env() { + info "creating directory: ${install_prefix} ${MPI_PREFIX} ${tempdir}" + ${SUDO} mkdir -p ${install_prefix} ${MPI_PREFIX} ${tempdir} + ${SUDO} chown -R $(id -u):$(id -g) ${install_prefix} ${MPI_PREFIX} ${tempdir} + export PATH=${install_prefix}/bin:${PATH} + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${install_prefix}/lib:${install_prefix}/lib64 +} + +# utils functions +function set_to_cn_url() { + local url=$1 + if [[ ${cn_flag} == true ]]; then + url="https://graphscope.oss-cn-beijing.aliyuncs.com/dependencies" + fi + echo ${url} +} + +function fetch_source() { + local url=$1 + local file=$2 + curl -fsSL -o "${file}" "${url}/${file}" +} + +function download_and_untar() { + local url=$1 + local file=$2 + local directory=$3 + if [ ! -d "${directory}" ]; then + [ ! -f "${file}" ] && fetch_source "${url}" "${file}" + tar zxf "${file}" + fi +} + +function git_clone() { + local url=$1 + local file=$2 + local directory=$3 + local branch=$4 + if [ ! -d "${directory}" ]; then + if [ ! -f "${file}" ]; then + git clone --depth=1 --branch "${branch}" "${url}" "${directory}" + pushd "${directory}" || exit + git submodule update --init || true + popd || exit + else + tar zxf "${file}" + fi + fi +} + +# cmake for centos +install_cmake() { + if [[ -f "${install_prefix}/bin/cmake" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + file="cmake-3.24.3-linux-${ARCH}.sh" + url="https://github.com/Kitware/CMake/releases/download/v3.24.3" + url=$(set_to_cn_url ${url}) + [ ! -f "${file}" ] && fetch_source "${url}" "${file}" + bash "${file}" --prefix="${install_prefix}" --skip-license + popd || exit + rm -rf "${tempdir:?}/${file:?}" +} + +# gflags for centos +install_gflags() { + if [[ -f "${install_prefix}/include/gflags/gflags.h" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + directory="gflags-2.2.2" + file="v2.2.2.tar.gz" + url="https://github.com/gflags/gflags/archive" + url=$(set_to_cn_url ${url}) + download_and_untar "${url}" "${file}" "${directory}" + pushd ${directory} || exit + cmake . -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ + -DCMAKE_PREFIX_PATH="${install_prefix}" \ + -DBUILD_SHARED_LIBS=ON + make -j$(nproc) + make install + popd || exit + popd || exit + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + +# glog for centos +install_glog() { + if [[ -f "${install_prefix}/include/glog/logging.h" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + directory="glog-0.6.0" + file="v0.6.0.tar.gz" + url="https://github.com/google/glog/archive" + url=$(set_to_cn_url ${url}) + download_and_untar "${url}" "${file}" "${directory}" + pushd ${directory} || exit + cmake . -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ + -DCMAKE_PREFIX_PATH="${install_prefix}" \ + -DBUILD_SHARED_LIBS=ON + make -j$(nproc) + make install + popd || exit + popd || exit + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + +# boost for centos +install_boost() { + if [[ -f "${install_prefix}/include/boost/version.hpp" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + directory="boost_1_74_0" + file="${directory}.tar.gz" + url="https://boostorg.jfrog.io/artifactory/main/release/1.74.0/source" + url=$(set_to_cn_url ${url}) + download_and_untar "${url}" "${file}" "${directory}" + pushd ${directory} || exit + ./bootstrap.sh --prefix="${install_prefix}" \ + --with-libraries=system,filesystem,context,program_options,regex,thread,random,chrono,atomic,date_time + ./b2 install link=shared runtime-link=shared variant=release threading=multi + popd || exit + popd || exit + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + +# openssl for centos +install_openssl() { + if [[ -f "${install_prefix}/include/openssl/ssl.h" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + directory="openssl-OpenSSL_1_1_1k" + file="OpenSSL_1_1_1k.tar.gz" + url="https://github.com/openssl/openssl/archive" + url=$(set_to_cn_url ${url}) + download_and_untar "${url}" "${file}" "${directory}" + pushd ${directory} || exit + ./config --prefix="${install_prefix}" -fPIC -shared + make -j$(nproc) + make install + popd || exit + popd || exit + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + +# static openssl for centos 8 +install_openssl_static() { + if [[ -f "${install_prefix}/include/openssl/ssl.h" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + directory="openssl-OpenSSL_1_1_1k" + file="OpenSSL_1_1_1k.tar.gz" + url="https://github.com/openssl/openssl/archive" + url=$(set_to_cn_url ${url}) + download_and_untar "${url}" "${file}" "${directory}" + pushd ${directory} || exit + ./config --prefix="${install_prefix}" no-shared -fPIC + make -j$(nproc) + make install + popd || exit + popd || exit + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + +# arrow for centos +install_apache_arrow() { + if [[ -f "${install_prefix}/include/arrow/api.h" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + directory="arrow-apache-arrow-${ARROW_VERSION}" + file="apache-arrow-${ARROW_VERSION}.tar.gz" + url="https://github.com/apache/arrow/archive" + url=$(set_to_cn_url ${url}) + download_and_untar "${url}" "${file}" "${directory}" + pushd ${directory} || exit + cmake ./cpp \ + -DCMAKE_PREFIX_PATH="${install_prefix}" \ + -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ + -DARROW_COMPUTE=ON \ + -DARROW_WITH_UTF8PROC=OFF \ + -DARROW_CSV=ON \ + -DARROW_CUDA=OFF \ + -DARROW_DATASET=OFF \ + -DARROW_FILESYSTEM=ON \ + -DARROW_FLIGHT=OFF \ + -DARROW_GANDIVA=OFF \ + -DARROW_HDFS=OFF \ + -DARROW_JSON=OFF \ + -DARROW_ORC=OFF \ + -DARROW_PARQUET=OFF \ + -DARROW_PLASMA=OFF \ + -DARROW_PYTHON=OFF \ + -DARROW_S3=OFF \ + -DARROW_WITH_BZ2=OFF \ + -DARROW_WITH_ZLIB=OFF \ + -DARROW_WITH_LZ4=OFF \ + -DARROW_WITH_SNAPPY=OFF \ + -DARROW_WITH_ZSTD=OFF \ + -DARROW_WITH_BROTLI=OFF \ + -DARROW_IPC=ON \ + -DARROW_BUILD_BENCHMARKS=OFF \ + -DARROW_BUILD_EXAMPLES=OFF \ + -DARROW_BUILD_INTEGRATION=OFF \ + -DARROW_BUILD_UTILITIES=OFF \ + -DARROW_BUILD_TESTS=OFF \ + -DARROW_ENABLE_TIMING_TESTS=OFF \ + -DARROW_FUZZING=OFF \ + -DARROW_USE_ASAN=OFF \ + -DARROW_USE_TSAN=OFF \ + -DARROW_USE_UBSAN=OFF \ + -DARROW_JEMALLOC=OFF \ + -DARROW_BUILD_SHARED=ON \ + -DARROW_BUILD_STATIC=OFF + make -j$(nproc) + make install + popd || exit + popd || exit + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + +# openmpi for centos +install_openmpi() { + if [[ -f "${install_prefix}/include/mpi.h" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + directory="openmpi-4.0.5" + file="openmpi-4.0.5.tar.gz" + url="https://download.open-mpi.org/release/open-mpi/v4.0" + url=$(set_to_cn_url ${url}) + download_and_untar "${url}" "${file}" "${directory}" + pushd ${directory} || exit + ./configure --enable-mpi-cxx --disable-dlopen --prefix=${MPI_PREFIX} + make -j$(nproc) + make install + popd || exit + popd || exit + cp -rs ${MPI_PREFIX}/* "${install_prefix}" + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + +# protobuf for centos +install_protobuf() { + if [[ -f "${install_prefix}/include/google/protobuf/port.h" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + directory="protobuf-21.9" + file="protobuf-all-21.9.tar.gz" + url="https://github.com/protocolbuffers/protobuf/releases/download/v21.9" + url=$(set_to_cn_url ${url}) + download_and_untar "${url}" "${file}" "${directory}" + pushd ${directory} || exit + ./configure --prefix="${install_prefix}" --enable-shared --disable-static + make -j$(nproc) + make install + popd || exit + popd || exit + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + +# zlib for centos +install_zlib() { + if [[ -f "${install_prefix}/include/zlib.h" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + directory="zlib-1.2.11" + file="v1.2.11.tar.gz" + url="https://github.com/madler/zlib/archive" + url=$(set_to_cn_url ${url}) + download_and_untar "${url}" "${file}" "${directory}" + pushd ${directory} || exit + cmake . -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ + -DCMAKE_PREFIX_PATH="${install_prefix}" \ + -DBUILD_SHARED_LIBS=ON + make -j$(nproc) + make install + popd || exit + popd || exit + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + +# grpc for centos +install_grpc() { + if [[ -f "${install_prefix}/include/grpcpp/grpcpp.h" ]]; then + return 0 + fi + directory="grpc" + branch="v1.49.1" + file="${directory}-${branch}.tar.gz" + url="https://github.com/grpc/grpc.git" + url=$(set_to_cn_url ${url}) + log "Building and installing ${directory}." + pushd "${tempdir}" || exit + if [[ ${url} == *.git ]]; then + git_clone "${url}" "${file}" "${directory}" "${branch}" + else + download_and_untar "${url}" "${file}" "${directory}" + fi + pushd ${directory} || exit + cmake . -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ + -DCMAKE_PREFIX_PATH="${install_prefix}" \ + -DBUILD_SHARED_LIBS=ON \ + -DgRPC_INSTALL=ON \ + -DgRPC_BUILD_TESTS=OFF \ + -DgRPC_BUILD_CSHARP_EXT=OFF \ + -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF \ + -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF \ + -DgRPC_BACKWARDS_COMPATIBILITY_MODE=ON \ + -DgRPC_PROTOBUF_PROVIDER=package \ + -DgRPC_ZLIB_PROVIDER=package \ + -DgRPC_SSL_PROVIDER=package \ + -DOPENSSL_ROOT_DIR="${install_prefix}" \ + -DCMAKE_CXX_FLAGS="-fpermissive" \ + -DPNG_ARM_NEON_OPT=0 + make -j$(nproc) + make install + popd || exit + popd || exit + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + +# patchelf +install_patchelf() { + if [[ -f "${install_prefix}/bin/patchelf" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + directory="patchelf" # patchelf doesn't have a folder + file="patchelf-0.14.5-${ARCH}.tar.gz" + url="https://github.com/NixOS/patchelf/releases/download/0.14.5" + url=$(set_to_cn_url ${url}) + mkdir -p "${directory}" + pushd "${directory}" || exit + download_and_untar "${url}" "${file}" "${directory}" + mkdir -p ${install_prefix}/bin + mv bin/patchelf ${install_prefix}/bin/patchelf + popd || exit + popd || exit + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + +# vineyard +install_vineyard() { + if [[ -f "${V6D_PREFIX}/include/vineyard/client/client.h" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + python3 -m pip --no-cache-dir install pip -U --user + python3 -m pip --no-cache-dir install libclang wheel auditwheel --user + auditwheel_path=$(python3 -c "import auditwheel; print(auditwheel.__path__[0] + '/main_repair.py')") + sed -i 's/p.error/logger.warning/g' ${auditwheel_path} + if [[ "${v6d_version}" != "v"* ]]; then + directory="v6d" + file="${directory}-${v6d_version}.tar.gz" + url="https://github.com/v6d-io/v6d.git" + git_clone "${url}" "${file}" "${directory}" "${v6d_version}" + else + # remove the prefix 'v' + directory="v6d-${v6d_version:1:100}" + file="${directory}.tar.gz" + url="https://github.com/v6d-io/v6d/releases/download/${v6d_version}" + download_and_untar "${url}" "${file}" "${directory}" + fi + pushd ${directory} || exit + + # make sure it complain loudly if installing vineyard fails + set -e + cmake . -DCMAKE_PREFIX_PATH="${install_prefix}" \ + -DCMAKE_INSTALL_PREFIX="${V6D_PREFIX}" \ + -DBUILD_VINEYARD_TESTS=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_VINEYARD_PYTHON_BINDINGS=ON \ + -DBUILD_VINEYARD_GRAPH_WITH_GAR=ON + make -j$(nproc) + make install + strip "${V6D_PREFIX}"/bin/vineyard* "${V6D_PREFIX}"/lib/libvineyard* + pip3 install --no-cache -i https://pypi.org/simple -U "vineyard" "vineyard-io" + cp -rs "${V6D_PREFIX}"/* "${install_prefix}"/ + set +e + popd || exit + popd || exit + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + +# java +install_java_and_maven() { + if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then + if ! command -v javac &>/dev/null; then + brew install --ignore-dependencies openjdk@11 + fi + if ! command -v mvn &>/dev/null; then + brew install --ignore-dependencies maven + fi + elif [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then + if ! command -v javac &>/dev/null; then + ${SUDO} apt-get install default-jdk -y + fi + if ! command -v mvn &>/dev/null; then + ${SUDO} apt-get install maven -y + fi + else + if ! command -v javac &>/dev/null; then + ${SUDO} yum install java-11-openjdk-devel -y + fi + if ! command -v mvn &>/dev/null; then + install_maven + fi + fi +} + +# maven for centos +install_maven() { + if [[ -f "${install_prefix}/bin/mvn" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + directory="apache-maven-3.8.6" + file="apache-maven-3.8.6-bin.tar.gz" + url="https://archive.apache.org/dist/maven/maven-3/3.8.6/binaries" + url=$(set_to_cn_url ${url}) + download_and_untar "${url}" "${file}" "${directory}" + cp -r ${directory} "${install_prefix}"/ + mkdir -p "${install_prefix}"/bin + ln -s "${install_prefix}/${directory}/bin/mvn" "${install_prefix}/bin/mvn" + popd || exit + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + +install_hiactor() { + if [[ -f "${install_prefix}/include/hiactor/core/actor_core.hh" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + git clone https://github.com/alibaba/hiactor.git -b v0.1.1 --single-branch + cd hiactor && git submodule update --init --recursive + sudo bash ./seastar/seastar/install-dependencies.sh + mkdir build && cd build + cmake -DHiactor_DEMOS=OFF -DHiactor_TESTING=OFF -DHiactor_DPDK=OFF -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ + -DHiactor_CXX_DIALECT=gnu++17 -DSeastar_CXX_FLAGS="-DSEASTAR_DEFAULT_ALLOCATOR -mno-avx512" .. + make -j$(nproc) && make install + popd && rm -rf "${tempdir}/hiactor" +} + + +install_cppkafka() { + if [[ -f "${install_prefix}/include/cppkafka/cppkafka.h" ]]; then + return 0 + fi + pushd "${tempdir}" || exit + directory="cppkafka-0.4.0" + file="0.4.0.tar.gz" + url="https://graphscope.oss-cn-beijing.aliyuncs.com/dependencies" + url=$(set_to_cn_url ${url}) + download_and_untar "${url}" "${file}" "${directory}" + pushd ${directory} || exit + # cppkafka may not find the lib64 directory + export LIBRARY_PATH=${LIBRARY_PATH}:${install_prefix}/lib:${install_prefix}/lib64 + cmake . -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ + -DCMAKE_PREFIX_PATH="${install_prefix}" \ + -DCPPKAFKA_DISABLE_TESTS=ON \ + -DCPPKAFKA_DISABLE_EXAMPLES=ON + make -j4 + make install + popd || exit + popd || exit + rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" +} + + +BASIC_PACKAGES_LINUX=("file" "curl" "wget" "git" "sudo") +BASIC_PACKAGES_UBUNTU=("${BASIC_PACKAGES_LINUX[@]}" "build-essential" "cmake" "libunwind-dev" "python3-pip") +BASIC_PACKAGES_CENTOS_8=("${BASIC_PACKAGES_LINUX[@]}" "epel-release" "libunwind-devel" "libcurl-devel" "perl" "which") +BASIC_PACKAGES_CENTOS_7=("${BASIC_PACKAGES_CENTOS_8[@]}" "centos-release-scl-rh") +ADDITIONAL_PACKAGES_CENTOS_8=("gcc-c++" "python38-devel") +ADDITIONAL_PACKAGES_CENTOS_7=("make" "devtoolset-8-gcc-c++" "rh-python38-python-pip" "rh-python38-python-devel") + +install_basic_packages() { + if [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then + ${SUDO} apt-get update -y + DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${SUDO} apt-get install -y ${BASIC_PACKAGES_UBUNTU[*]} + elif [[ "${OS_PLATFORM}" == *"CentOS"* || "${OS_PLATFORM}" == *"Aliyun"* ]]; then + ${SUDO} yum update -y + if [[ "${OS_VERSION}" -eq "7" ]]; then + # centos7 + ${SUDO} yum install -y ${BASIC_PACKAGES_CENTOS_7[*]} + # change the source for centos-release-scl-rh + ${SUDO} sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*scl* + ${SUDO} sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*scl* + ${SUDO} sed -i 's|# baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*scl* + ${SUDO} yum install -y ${ADDITIONAL_PACKAGES_CENTOS_7[*]} + else + if [[ "${OS_PLATFORM}" == *"Aliyun"* ]]; then + ${SUDO} yum install -y 'dnf-command(config-manager)' + ${SUDO} dnf install -y epel-release --allowerasing + else + ${SUDO} sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + ${SUDO} sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + ${SUDO} yum install -y 'dnf-command(config-manager)' + ${SUDO} dnf install -y epel-release + ${SUDO} dnf config-manager --set-enabled powertools + fi + ${SUDO} dnf config-manager --set-enabled epel + ${SUDO} yum install -y ${BASIC_PACKAGES_CENTOS_8[*]} + ${SUDO} yum install -y ${ADDITIONAL_PACKAGES_CENTOS_8[*]} + fi + fi +} + +ANALYTICAL_MACOS=("apache-arrow" "boost" "gflags" "glog" "open-mpi" "openssl@1.1" "protobuf" "grpc" "rapidjson" "msgpack-cxx" "librdkafka" "patchelf") +ANALYTICAL_UBUNTU=("libboost-all-dev" "libopenmpi-dev" "libgflags-dev" "libgoogle-glog-dev" "libprotobuf-dev" "libgrpc++-dev" "libmsgpack-dev" "librdkafka-dev" "protobuf-compiler-grpc" "rapidjson-dev") +ANALYTICAL_CENTOS_7=("librdkafka-devel" "msgpack-devel" "rapidjson-devel") +ANALYTICAL_CENTOS_8=("${ANALYTICAL_CENTOS_7[@]}" "boost-devel" "gflags-devel" "glog-devel") + +install_analytical_centos_common_dependencies() { + install_patchelf + install_apache_arrow + install_openmpi + install_protobuf + install_zlib + install_grpc +} + +install_analytical_dependencies() { + if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then + brew install ${ANALYTICAL_MACOS[*]} + export LDFLAGS="${LDFLAGS} -L${HOMEBREW_PREFIX}/opt/openssl@1.1/lib" + export CPPFLAGS="${CPPFLAGS} -I${HOMEBREW_PREFIX}/opt/openssl@1.1/include" + export PKG_CONFIG_PATH="${HOMEBREW_PREFIX}/opt/openssl@1.1/lib/pkgconfig" + elif [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then + DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${SUDO} apt-get install -y ${ANALYTICAL_UBUNTU[*]} + # patchelf + install_patchelf + # arrow + if ! dpkg -s libarrow-dev &>/dev/null; then + ${SUDO} apt-get install -y lsb-release + # shellcheck disable=SC2046,SC2019,SC2018 + wget -c https://apache.jfrog.io/artifactory/arrow/"$(lsb_release --id --short | tr 'A-Z' 'a-z')"/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb -P /tmp/ + ${SUDO} apt-get install -y -V /tmp/apache-arrow-apt-source-latest-"$(lsb_release --codename --short)".deb + ${SUDO} apt-get update -y + ${SUDO} apt-get install -y libarrow-dev=${ARROW_VERSION}-1 libarrow-dataset-dev=${ARROW_VERSION}-1 libarrow-acero-dev=${ARROW_VERSION}-1 libparquet-dev=${ARROW_VERSION}-1 + rm /tmp/apache-arrow-apt-source-latest-*.deb + fi + else + if [[ "${OS_VERSION}" -eq "7" ]]; then + ${SUDO} yum install -y ${ANALYTICAL_CENTOS_7[*]} + source /opt/rh/devtoolset-8/enable + source /opt/rh/rh-python38/enable + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${install_prefix}/lib:${install_prefix}/lib64 + install_cmake + install_gflags + install_glog + install_boost + # must to install openssl before grpc + install_openssl + else + ${SUDO} yum install -y ${ANALYTICAL_CENTOS_8[*]} + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/lib/:/lib64${install_prefix}/lib:${install_prefix}/lib64 + install_cmake + install_openssl_static + fi + install_analytical_centos_common_dependencies + fi + # install java for gae-java + install_java_and_maven + # install vineyard + if [[ "${no_v6d}" != true ]]; then + if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then + brew install vineyard + else + install_vineyard + fi + fi +} + +install_analytical_java_dependencies() { + # llvm + if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then + brew install llvm || true # prevent the `brew link` failure + elif [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then + ${SUDO} apt-get install -y llvm-11-dev lld-11 clang-11 + else + if [[ "${OS_VERSION}" -eq "7" ]]; then + ${SUDO} yum install -y llvm-toolset-7.0-clang-devel + source /opt/rh/llvm-toolset-7.0/enable + else + ${SUDO} yum install -y llvm-devel clang-devel lld + fi + fi +} + +install_interactive_dependencies() { + # java + install_java_and_maven + # rust + if ! command -v rustup &>/dev/null; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + source $HOME/.cargo/env + rustup install 1.71.0 + rustup default 1.71.0 + rustc --version + fi + # hiactor + if [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then + install_hiactor + else + warning "Skip installing dependencies for flex interactive on ${OS_PLATFORM}." + fi +} + +install_learning_dependencies() { + install_cppkafka +} + +write_env_config() { + echo "" > ${OUTPUT_ENV_FILE} + # common + { + echo "export GRAPHSCOPE_HOME=${install_prefix}" + echo "export CMAKE_PREFIX_PATH=/opt/vineyard:/opt/graphscope/" + echo "export PATH=${install_prefix}/bin:\$HOME/.local/bin:\$HOME/.cargo/bin:\$PATH" + echo "export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" + echo "export LIBRARY_PATH=${install_prefix}/lib:${install_prefix}/lib64" + } >> "${OUTPUT_ENV_FILE}" + { + if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then + echo "export OPENSSL_ROOT_DIR=${homebrew_prefix}/opt/openssl" + echo "export OPENSSL_LIBRARIES=${homebrew_prefix}/opt/openssl/lib" + echo "export OPENSSL_SSL_LIBRARY=${homebrew_prefix}/opt/openssl/lib/libssl.dylib" + elif [[ "${OS_PLATFORM}" == *"CentOS"* || "${OS_PLATFORM}" == *"Aliyun"* ]]; then + if [[ "${OS_VERSION}" -eq "7" ]]; then + echo "source /opt/rh/devtoolset-8/enable" + echo "source /opt/rh/rh-python38/enable" + fi + echo "export OPENSSL_ROOT_DIR=${install_prefix}" + fi + } >> "${OUTPUT_ENV_FILE}" + # JAVA_HOME + { + if [[ "${for_analytical}" == true || "${for_interactive}" == true ]]; then + if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then + if [ -z "${JAVA_HOME}" ]; then + echo "export JAVA_HOME=\$(/usr/libexec/java_home -v11)" + fi + elif [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then + if [ -z "${JAVA_HOME}" ]; then + echo "export JAVA_HOME=/usr/lib/jvm/default-java" + fi + else + if [ -z "${JAVA_HOME}" ]; then + echo "export OPENSSL_ROOT_DIR=${install_prefix}" + fi + fi + fi + } >> "${OUTPUT_ENV_FILE}" + { + if [[ "${for_analytical_java}" == true ]]; then + if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then + echo "export CC=${homebrew_prefix}/opt/llvm/bin/clang" + echo "export CXX=${homebrew_prefix}/opt/llvm/bin/clang++" + echo "export CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=${CC}" + echo "export LDFLAGS=\"-L${homebrew_prefix}/opt/llvm/lib\"" + echo "export CPPFLAGS=\"-I${homebrew_prefix}/opt/llvm/include\"" + elif [[ "${OS_PLATFORM}" == *"CentOS"* || "${OS_PLATFORM}" == *"Aliyun"* ]]; then + echo "source /opt/rh/llvm-toolset-7.0/enable || true" + echo "export LIBCLANG_PATH=/opt/rh/llvm-toolset-7.0/root/usr/lib64/" + fi + fi + } >> "${OUTPUT_ENV_FILE}" +} + +install_deps() { + init_workspace_and_env + install_basic_packages + [[ "${for_analytical}" == true ]] && install_analytical_dependencies + [[ "${for_analytical_java}" == true ]] && install_analytical_java_dependencies + [[ "${for_interactive}" == true ]] && install_interactive_dependencies + [[ "${for_learning}" == true ]] && install_learning_dependencies + write_env_config + info "The script has installed all dependencies, don't forget to exec command:\n + $ source ${OUTPUT_ENV_FILE} + \nbefore building GraphScope." +} + +install_deps diff --git a/python/graphscope/gsctl/scripts/install_deps_command.sh b/python/graphscope/gsctl/scripts/install_deps_command.sh deleted file mode 100644 index a835b690f8cd..000000000000 --- a/python/graphscope/gsctl/scripts/install_deps_command.sh +++ /dev/null @@ -1,448 +0,0 @@ -script_dir="$(cd "$(dirname "$0")" && pwd)" -source ${script_dir}/lib/get_os_version.sh -source ${script_dir}/lib/log.sh -source ${script_dir}/lib/colors.sh -source ${script_dir}/lib/install_thirdparty_dependencies.sh -source ${script_dir}/lib/install_vineyard.sh -source ${script_dir}/lib/util.sh -source ${script_dir}/initialize.sh - -# parse args -while getopts ":t:i:d:v:j:-:" opt; do - case ${opt} in - t) - type=${OPTARG} - ;; - i) - install_prefix=${OPTARG} - ;; - d) - deps_prefix=${OPTARG} - ;; - v) - v6d_version=${OPTARG} - ;; - j) - jobs=${OPTARG} - ;; - -) - case ${OPTARG} in - for-analytical) - for_analytical=true - ;; - no-v6d) - no_v6d=true - ;; - cn) - cn=true - ;; - *) - echo "Invalid option: --${OPTARG}" - exit 1 - ;; - esac - ;; - *) - echo "Invalid option: -${OPTARG}" - exit 1 - ;; - esac -done -shift $((OPTIND -1)) - -SUDO=sudo -if [[ $(id -u) -eq 0 ]]; then - warning "Please note that I am running as root." - SUDO="" -fi - -readonly OS=$(get_os_version) -readonly OS_PLATFORM=${OS%-*} -readonly OS_VERSION=${OS#*-} - -readonly OUTPUT_ENV_FILE="${HOME}/.graphscope_env" - -log "Installing ${type} dependencies for GraphScope on ${OS}..." - -if [ "${cn}" == true ]; then - log "Set to speed up downloading for CN locations." - # export some mirror locations for CN, e.g., brew/docker... - export HOMEBREW_BREW_GIT_REMOTE="https://mirrors.tuna.tsinghua.edu.cn/git/homebrew/brew.git" - export HOMEBREW_CORE_GIT_REMOTE="https://mirrors.tuna.tsinghua.edu.cn/git/homebrew/homebrew-core.git" - export HOMEBREW_BOTTLE_DOMAIN="https://mirrors.tuna.tsinghua.edu.cn/homebrew-bottles" - export GRAPHSCOPE_DOWNLOAD_FROM_CN="true" -fi - -check_os_compatibility() { - if [[ "${OS_PLATFORM}" != *"Ubuntu"* && "${OS_PLATFORM}" != *"CentOS"* && "${OS_PLATFORM}" != *"Darwin"* && "${OS_PLATFORM}" != *"Aliyun"* ]]; then - err "The script is only support platforms of Ubuntu/CentOS/macOS/AliyunOS" - exit 1 - fi - - if [[ "${OS_PLATFORM}" == *"Ubuntu"* && "${OS_VERSION:0:2}" -lt "20" ]]; then - err "The version of Ubuntu is ${OS_VERSION}. This script requires Ubuntu 20 or greater." - exit 1 - fi - - if [[ "${OS_PLATFORM}" == *"CentOS"* && "${OS_VERSION}" -lt "7" ]]; then - err "The version of CentOS is ${OS_VERSION}. This script requires CentOS 8 or greater." - exit 1 - fi - - log "Running on ${OS_PLATFORM} ${OS_VERSION}" -} - -BASIC_PACKAGES_LINUX=("file" "curl" "wget" "git" "sudo") - -BASIC_PACKAGES_UBUNTU=("${BASIC_PACKAGES_LINUX[@]}" "build-essential" "cmake" "libunwind-dev" "python3-pip") - -BASIC_PACKAGES_CENTOS_8=("${BASIC_PACKAGES_LINUX[@]}" "epel-release" "libunwind-devel" "libcurl-devel" "perl" "which") -BASIC_PACKAGES_CENTOS_7=("${BASIC_PACKAGES_CENTOS_8[@]}" "centos-release-scl-rh") -ADDITIONAL_PACKAGES_CENTOS_8=("gcc-c++" "python38-devel") -ADDITIONAL_PACKAGES_CENTOS_7=("make" "devtoolset-8-gcc-c++" "rh-python38-python-pip" "rh-python38-python-devel") - -ANALYTICAL_UBUNTU=( - "libboost-all-dev" - "libopenmpi-dev" - "libgflags-dev" - "libgoogle-glog-dev" - "libprotobuf-dev" - "libgrpc++-dev" - "libmsgpack-dev" - "librdkafka-dev" - "protobuf-compiler-grpc" - "rapidjson-dev" -) - -ANALYTICAL_CENTOS_7=("librdkafka-devel" "msgpack-devel" "rapidjson-devel") -ANALYTICAL_CENTOS_8=("${ANALYTICAL_CENTOS_7[@]}" "boost-devel" "gflags-devel" "glog-devel") - -ANALYTICAL_MACOS=( - "apache-arrow" - "boost" - "gflags" - "glog" - "open-mpi" - "openssl@1.1" - "protobuf" - "grpc" - "rapidjson" - "msgpack-cxx" - "librdkafka" - "patchelf" -) - -_install_apache_arrow_ubuntu() { - if ! dpkg -s libarrow-dev &>/dev/null; then - log "Installing apache-arrow." - ${SUDO} apt-get install -y lsb-release - # shellcheck disable=SC2046,SC2019,SC2018 - wget -c https://apache.jfrog.io/artifactory/arrow/"$(lsb_release --id --short | tr 'A-Z' 'a-z')"/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb \ - -P /tmp/ - ${SUDO} apt-get install -y -V /tmp/apache-arrow-apt-source-latest-"$(lsb_release --codename --short)".deb - ${SUDO} apt-get update -y - ${SUDO} apt-get install -y libarrow-dev=15.0.2-1 libarrow-dataset-dev=15.0.2-1 libarrow-acero-dev=15.0.2-1 libparquet-dev=15.0.2-1 - #TODO(zhanglei): Remove the arrow version constrait after new version of vineyard is release. see https://github.com/v6d-io/v6d/pull/1911 and https://github.com/apache/incubator-graphar/issues/515 - #${SUDO} apt-get install -y libarrow-dev libarrow-dataset-dev libarrow-acero-dev libparquet-dev - rm /tmp/apache-arrow-apt-source-latest-*.deb - else - log "apache-arrow (libarrow-dev) already installed, skip." - fi -} - -_install_java_maven_ubuntu() { - if ! command -v javac &>/dev/null; then - log "Installing default-jdk" - ${SUDO} apt-get install default-jdk -y - fi - if ! command -v mvn &>/dev/null; then - log "Installing maven" - ${SUDO} apt-get install maven -y - fi -} - -_install_java_maven_centos() { - if ! command -v javac &>/dev/null; then - log "Installing java-11-openjdk-devel" - ${SUDO} yum install java-11-openjdk-devel -y - fi - if ! command -v mvn &>/dev/null; then - log "Installing maven" - install_maven "${deps_prefix}" "${install_prefix}" - fi -} - -_install_java_maven_macos() { - if ! command -v javac &>/dev/null; then - log "Installing openjdk@11" - # we need arm64-base jvm, install from brew. - brew install --ignore-dependencies openjdk@11 - fi - if ! command -v mvn &>/dev/null; then - log "Installing maven" - brew install --ignore-dependencies maven - fi -} - -_install_dependencies_analytical_ubuntu() { - DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${SUDO} apt-get install -y ${ANALYTICAL_UBUNTU[*]} - _install_apache_arrow_ubuntu -} - -_install_dependencies_analytical_centos_common() { - # the openssl must put before grpc, otherwise the grpc - # cannot find the openssl. - install_apache_arrow "${deps_prefix}" "${install_prefix}" - install_open_mpi "${deps_prefix}" "${install_prefix}" - install_protobuf "${deps_prefix}" "${install_prefix}" - install_zlib "${deps_prefix}" "${install_prefix}" - install_grpc "${deps_prefix}" "${install_prefix}" -} - -_install_dependencies_analytical_centos8() { - ${SUDO} yum install -y ${ANALYTICAL_CENTOS_8[*]} - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/lib/:/lib64${install_prefix}/lib:${install_prefix}/lib64 - install_cmake "${deps_prefix}" "${install_prefix}" - install_openssl_static "${deps_prefix}" "${install_prefix}" - _install_dependencies_analytical_centos_common -} -_install_dependencies_analytical_centos7() { - ${SUDO} yum install -y ${ANALYTICAL_CENTOS_7[*]} - source /opt/rh/devtoolset-8/enable - source /opt/rh/rh-python38/enable - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${install_prefix}/lib:${install_prefix}/lib64 - - install_cmake "${deps_prefix}" "${install_prefix}" - install_gflags "${deps_prefix}" "${install_prefix}" - install_glog "${deps_prefix}" "${install_prefix}" - install_boost "${deps_prefix}" "${install_prefix}" - install_openssl "${deps_prefix}" "${install_prefix}" - _install_dependencies_analytical_centos_common -} -_install_dependencies_analytical_macos() { - brew install ${ANALYTICAL_MACOS[*]} - homebrew_prefix=$(brew --prefix) - export LDFLAGS="${LDFLAGS} -L${homebrew_prefix}/opt/openssl@1.1/lib" - export CPPFLAGS="${CPPFLAGS} -I${homebrew_prefix}/opt/openssl@1.1/include" - export PKG_CONFIG_PATH="${homebrew_prefix}/opt/openssl@1.1/lib/pkgconfig" -} - -install_basic_packages_universal() { - if [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then - ${SUDO} apt-get update -y - DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${SUDO} apt-get install -y ${BASIC_PACKAGES_UBUNTU[*]} - elif [[ "${OS_PLATFORM}" == *"CentOS"* || "${OS_PLATFORM}" == *"Aliyun"* ]]; then - if [[ "${OS_VERSION}" -eq "7" ]]; then - ${SUDO} yum install -y ${BASIC_PACKAGES_CENTOS_7[*]} - # change the source for centos-release-scl-rh - ${SUDO} sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*scl* - ${SUDO} sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*scl* - ${SUDO} sed -i 's|# baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*scl* - ${SUDO} yum install -y ${ADDITIONAL_PACKAGES_CENTOS_7[*]} - else - if [[ "${OS_PLATFORM}" == *"Aliyun"* ]]; then - ${SUDO} yum install -y 'dnf-command(config-manager)' - ${SUDO} dnf install -y epel-release --allowerasing - else - ${SUDO} sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - ${SUDO} sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - ${SUDO} yum install -y 'dnf-command(config-manager)' - ${SUDO} dnf install -y epel-release - ${SUDO} dnf config-manager --set-enabled powertools - fi - ${SUDO} dnf config-manager --set-enabled epel - ${SUDO} yum install -y ${BASIC_PACKAGES_CENTOS_8[*]} - ${SUDO} yum install -y ${ADDITIONAL_PACKAGES_CENTOS_8[*]} - fi - fi -} - -install_grape_vineyard_universal() { - if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then - brew install vineyard - else - log "Installing python packages for vineyard codegen." - pip3 --no-cache-dir install pip -U --user - pip3 --no-cache-dir install libclang wheel auditwheel --user - install_vineyard "${deps_prefix}" "${install_prefix}" "${v6d_version}" "${jobs}" - fi -} - -install_rust_universal() { - if ! command -v rustup &>/dev/null; then - log "Installing rust with fixed version: 1.71/0." - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - source $HOME/.cargo/env - rustup install 1.71.0 - rustup default 1.71.0 - rustc --version - fi -} - -install_java_maven_universal() { - if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then - _install_java_maven_macos - elif [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then - _install_java_maven_ubuntu - else - _install_java_maven_centos - fi -} - -install_llvm_universal() { - if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then - brew install llvm || true # prevent the `brew link` failure - homebrew_prefix=$(brew --prefix) - export CC=${homebrew_prefix}/opt/llvm/bin/clang - export CXX=${homebrew_prefix}/opt/llvm/bin/clang++ - export CPPFLAGS="${CPPFLAGS} -I${homebrew_prefix}/opt/llvm/include" - export CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=${CC} - elif [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then - ${SUDO} apt-get install -y llvm-11-dev lld-11 clang-11 - else - if [[ "${OS_VERSION}" -eq "7" ]]; then - ${SUDO} yum install -y llvm-toolset-7.0-clang-devel - source /opt/rh/llvm-toolset-7.0/enable - export LIBCLANG_PATH=/opt/rh/llvm-toolset-7.0/root/usr/lib64/ - else - ${SUDO} yum install -y llvm-devel clang-devel lld - fi - fi -} - -install_dependencies_analytical_universal() { - if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then - _install_dependencies_analytical_macos - elif [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then - install_patchelf "${deps_prefix}" "${install_prefix}" - _install_dependencies_analytical_ubuntu - else - install_patchelf "${deps_prefix}" "${install_prefix}" - if [[ "${OS_VERSION}" -eq "7" ]]; then - _install_dependencies_analytical_centos7 - install_java_maven_universal - else - _install_dependencies_analytical_centos8 - fi - fi -} - -install_interactive_deps() { - # seastar can not be built on macos and centos7 - if [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then - install_hiactor "${install_prefix}" - else - warning "Skip installing dependencies for flex interactive on ${OS_PLATFORM}." - fi -} - -write_env_config() { - log "Output environments config file ${OUTPUT_ENV_FILE}" - if [ -f "${OUTPUT_ENV_FILE}" ]; then - warning "Found ${OUTPUT_ENV_FILE} exists, remove the environment config file and generate a new one." - rm -f "${OUTPUT_ENV_FILE}" - fi - - { - echo "export GRAPHSCOPE_HOME=${install_prefix}" - echo "export CMAKE_PREFIX_PATH=/opt/vineyard:/opt/graphscope/" - echo "export PATH=${install_prefix}/bin:\$HOME/.local/bin:\$HOME/.cargo/bin:\$PATH" - echo "export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" - echo "export LIBRARY_PATH=${install_prefix}/lib:${install_prefix}/lib64" - } >>"${OUTPUT_ENV_FILE}" - - if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then - homebrew_prefix=$(brew --prefix) - { - echo "export CC=${homebrew_prefix}/opt/llvm/bin/clang" - echo "export CXX=${homebrew_prefix}/opt/llvm/bin/clang++" - echo "export CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=${CC}" - if [ -z "${JAVA_HOME}" ]; then - echo "export JAVA_HOME=\$(/usr/libexec/java_home -v11)" - fi - echo "export OPENSSL_ROOT_DIR=${homebrew_prefix}/opt/openssl" - echo "export OPENSSL_LIBRARIES=${homebrew_prefix}/opt/openssl/lib" - echo "export OPENSSL_SSL_LIBRARY=${homebrew_prefix}/opt/openssl/lib/libssl.dylib" - echo "export LDFLAGS=\"-L${homebrew_prefix}/opt/llvm/lib\"" - echo "export CPPFLAGS=\"-I${homebrew_prefix}/opt/llvm/include\"" - } >>"${OUTPUT_ENV_FILE}" - - elif [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then - { - if [ -z "${JAVA_HOME}" ]; then - echo "export JAVA_HOME=/usr/lib/jvm/default-java" - fi - } >>"${OUTPUT_ENV_FILE}" - else - { - if [[ "${OS_VERSION}" -eq "7" ]]; then - echo "source /opt/rh/devtoolset-8/enable" - echo "source /opt/rh/rh-python38/enable" - echo "source /opt/rh/llvm-toolset-7.0/enable || true" - echo "export LIBCLANG_PATH=/opt/rh/llvm-toolset-7.0/root/usr/lib64/" - fi - if [ -z "${JAVA_HOME}" ]; then - echo "export JAVA_HOME=/usr/lib/jvm/jre-openjdk" - fi - echo "export OPENSSL_ROOT_DIR=${install_prefix}" - } >>"${OUTPUT_ENV_FILE}" - fi -} - -init_workspace_and_env() { - mkdir -p "${install_prefix}" - mkdir -p "${deps_prefix}" - export PATH=${install_prefix}/bin:${PATH} - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${install_prefix}/lib:${install_prefix}/lib64 -} - -install_deps_for_dev() { - # install_deps for development on local - check_os_compatibility - - init_workspace_and_env - - if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then - export HOMEBREW_NO_INSTALL_CLEANUP=1 - export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 - fi - - install_basic_packages_universal - if [ "${for_analytical}" == true ]; then - install_dependencies_analytical_universal - if [ "${no_v6d}" != true ]; then - install_grape_vineyard_universal - fi - else # for all - install_dependencies_analytical_universal - if [ "${no_v6d}" != true ]; then - install_grape_vineyard_universal - fi - install_java_maven_universal - install_llvm_universal - install_rust_universal - install_cppkafka "${deps_prefix}" "${install_prefix}" - # install dependencies for flex interactive - install_interactive_deps - fi - - write_env_config - - succ "The script has installed all dependencies for building GraphScope, use commands:\n - $ source ${OUTPUT_ENV_FILE} - $ make install - \nto build and develop GraphScope." -} - -install_deps_for_client() { - # install python.. - # TODO: refine - pip3 --no-cache-dir install -U pip --user - pip3 --no-cache-dir install auditwheel daemons etcd-distro gremlinpython \ - hdfs3 fsspec oss2 s3fs ipython kubernetes libclang networkx==2.4 numpy pandas parsec pycryptodome \ - pyorc pytest scipy scikit_learn wheel --user - pip3 --no-cache-dir install Cython --pre -U --user -} - -# run subcommand with the type -install_deps_for_"${type}" diff --git a/python/graphscope/gsctl/scripts/lib/colors.sh b/python/graphscope/gsctl/scripts/lib/colors.sh deleted file mode 100644 index 25bc07e9a3d9..000000000000 --- a/python/graphscope/gsctl/scripts/lib/colors.sh +++ /dev/null @@ -1,42 +0,0 @@ -## Color functions [@bashly-upgrade colors] -## This file is a part of Bashly standard library -## -## Usage: -## Use any of the functions below to color or format a portion of a string. -## -## echo "before $(red this is red) after" -## echo "before $(green_bold this is green_bold) after" -## -## Color output will be disabled if `NO_COLOR` environment variable is set -## in compliance with https://no-color.org/ -## -print_in_color() { - local color="$1" - shift - if [[ -z ${NO_COLOR+x} ]]; then - printf "$color%b\e[0m\n" "$*"; - else - printf "%b\n" "$*"; - fi -} - -red() { print_in_color "\e[31m" "$*"; } -green() { print_in_color "\e[32m" "$*"; } -yellow() { print_in_color "\e[33m" "$*"; } -blue() { print_in_color "\e[34m" "$*"; } -magenta() { print_in_color "\e[35m" "$*"; } -cyan() { print_in_color "\e[36m" "$*"; } -bold() { print_in_color "\e[1m" "$*"; } -underlined() { print_in_color "\e[4m" "$*"; } -red_bold() { print_in_color "\e[1;31m" "$*"; } -green_bold() { print_in_color "\e[1;32m" "$*"; } -yellow_bold() { print_in_color "\e[1;33m" "$*"; } -blue_bold() { print_in_color "\e[1;34m" "$*"; } -magenta_bold() { print_in_color "\e[1;35m" "$*"; } -cyan_bold() { print_in_color "\e[1;36m" "$*"; } -red_underlined() { print_in_color "\e[4;31m" "$*"; } -green_underlined() { print_in_color "\e[4;32m" "$*"; } -yellow_underlined() { print_in_color "\e[4;33m" "$*"; } -blue_underlined() { print_in_color "\e[4;34m" "$*"; } -magenta_underlined() { print_in_color "\e[4;35m" "$*"; } -cyan_underlined() { print_in_color "\e[4;36m" "$*"; } diff --git a/python/graphscope/gsctl/scripts/lib/get_os_version.sh b/python/graphscope/gsctl/scripts/lib/get_os_version.sh deleted file mode 100644 index 6a306b5446cf..000000000000 --- a/python/graphscope/gsctl/scripts/lib/get_os_version.sh +++ /dev/null @@ -1,34 +0,0 @@ -get_os_version() { - if [ -f /etc/centos-release ]; then - # Older Red Hat, CentOS, Alibaba Cloud Linux etc. - PLATFORM=CentOS - OS_VERSION=$(sed 's/.* \([0-9]\).*/\1/' < /etc/centos-release) - if grep -q "Alibaba Cloud Linux" /etc/centos-release; then - PLATFORM="Aliyun_based_on_CentOS" - OS_VERSION=$(rpm -E %{rhel}) - fi - elif [ -f /etc/os-release ]; then - # freedesktop.org and systemd - . /etc/os-release - PLATFORM="${NAME}" - OS_VERSION="${VERSION_ID}" - elif type lsb_release >/dev/null 2>&1; then - # linuxbase.org - PLATFORM=$(lsb_release -si) - OS_VERSION=$(lsb_release -sr) - elif [ -f /etc/lsb-release ]; then - # For some versions of Debian/Ubuntu without lsb_release command - . /etc/lsb-release - PLATFORM="${DISTRIB_ID}" - OS_VERSION="${DISTRIB_RELEASE}" - elif [ -f /etc/debian_version ]; then - # Older Debian/Ubuntu/etc. - PLATFORM=Debian - OS_VERSION=$(cat /etc/debian_version) - else - # Fall back to uname, e.g. "Linux ", also works for BSD, Darwin, etc. - PLATFORM=$(uname -s) - OS_VERSION=$(uname -r) - fi - echo "$PLATFORM-$OS_VERSION" -} diff --git a/python/graphscope/gsctl/scripts/lib/install_thirdparty_dependencies.sh b/python/graphscope/gsctl/scripts/lib/install_thirdparty_dependencies.sh deleted file mode 100644 index 704b61ec6d74..000000000000 --- a/python/graphscope/gsctl/scripts/lib/install_thirdparty_dependencies.sh +++ /dev/null @@ -1,472 +0,0 @@ -install_cmake() { - workdir=$1 - install_prefix=$2 - - if [[ -f "${install_prefix}/bin/cmake" ]]; then - log "cmake already installed, skip." - return 0 - fi - - ARCH=$(uname -m) - file="cmake-3.24.3-linux-${ARCH}.sh" - log "Building and installing ${file}." - pushd "${workdir}" || exit - - url="https://github.com/Kitware/CMake/releases/download/v3.24.3" - url=$(maybe_set_to_cn_url ${url}) - [ ! -f "${file}" ] && - fetch_source "${file}" "${url}" - bash "${file}" --prefix="${install_prefix}" --skip-license - popd || exit - cleanup_files "${workdir}/${file}" -} - -install_open_mpi() { - workdir=$1 - install_prefix=$2 - MPI_PREFIX="/opt/openmpi" # fixed, related to coordinator/setup.py - - if [[ -f "${install_prefix}/include/mpi.h" ]]; then - log "openmpi already installed, skip." - return 0 - fi - - directory="openmpi-4.0.5" - file="openmpi-4.0.5.tar.gz" - url="https://download.open-mpi.org/release/open-mpi/v4.0" - url=$(maybe_set_to_cn_url ${url}) - log "Building and installing ${directory}." - pushd "${workdir}" || exit - download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" - pushd ${directory} || exit - - ./configure --enable-mpi-cxx --disable-dlopen --prefix=${MPI_PREFIX} - make -j$(nproc) - make install - popd || exit - popd || exit - cp -rs ${MPI_PREFIX}/* "${install_prefix}" - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} - -install_gflags() { - workdir=$1 - install_prefix=$2 - - if [[ -f "${install_prefix}/include/gflags/gflags.h" ]]; then - log "gflags already installed, skip." - return 0 - fi - - directory="gflags-2.2.2" - file="v2.2.2.tar.gz" - url="https://github.com/gflags/gflags/archive" - url=$(maybe_set_to_cn_url ${url}) - log "Building and installing gflags-2.2.2." - pushd "${workdir}" || exit - download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" - pushd ${directory} || exit - - cmake . -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ - -DCMAKE_PREFIX_PATH="${install_prefix}" \ - -DBUILD_SHARED_LIBS=ON - make -j$(nproc) - make install - popd || exit - popd || exit - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} - -install_glog() { - workdir=$1 - install_prefix=$2 - - if [[ -f "${install_prefix}/include/glog/logging.h" ]]; then - log "glog already installed, skip." - return 0 - fi - - directory="glog-0.6.0" - file="v0.6.0.tar.gz" - url="https://github.com/google/glog/archive" - url=$(maybe_set_to_cn_url ${url}) - log "Building and installing ${directory}." - pushd "${workdir}" || exit - download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" - pushd ${directory} || exit - - cmake . -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ - -DCMAKE_PREFIX_PATH="${install_prefix}" \ - -DBUILD_SHARED_LIBS=ON - make -j$(nproc) - make install - popd || exit - popd || exit - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} - -install_apache_arrow() { - workdir=$1 - install_prefix=$2 - - if [[ -f "${install_prefix}/include/arrow/api.h" ]]; then - log "apache-arrow already installed, skip." - return 0 - fi - - directory="arrow-apache-arrow-10.0.1" - file="apache-arrow-10.0.1.tar.gz" - url="https://github.com/apache/arrow/archive" - url=$(maybe_set_to_cn_url ${url}) - log "Building and installing ${directory}." - pushd "${workdir}" || exit - download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" - pushd ${directory} || exit - - # temporarily fix the xsimd dependency downloading issue by manually download - mkdir -p src && wget https://graphscope.oss-cn-beijing.aliyuncs.com/dependencies/9.0.1.tar.gz -P src/ - - cmake ./cpp \ - -DCMAKE_PREFIX_PATH="${install_prefix}" \ - -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ - -DARROW_COMPUTE=ON \ - -DARROW_WITH_UTF8PROC=OFF \ - -DARROW_CSV=ON \ - -DARROW_CUDA=OFF \ - -DARROW_DATASET=OFF \ - -DARROW_FILESYSTEM=ON \ - -DARROW_FLIGHT=OFF \ - -DARROW_GANDIVA=OFF \ - -DARROW_HDFS=OFF \ - -DARROW_JSON=OFF \ - -DARROW_ORC=OFF \ - -DARROW_PARQUET=OFF \ - -DARROW_PLASMA=OFF \ - -DARROW_PYTHON=OFF \ - -DARROW_S3=OFF \ - -DARROW_WITH_BZ2=OFF \ - -DARROW_WITH_ZLIB=OFF \ - -DARROW_WITH_LZ4=OFF \ - -DARROW_WITH_SNAPPY=OFF \ - -DARROW_WITH_ZSTD=OFF \ - -DARROW_WITH_BROTLI=OFF \ - -DARROW_IPC=ON \ - -DARROW_BUILD_BENCHMARKS=OFF \ - -DARROW_BUILD_EXAMPLES=OFF \ - -DARROW_BUILD_INTEGRATION=OFF \ - -DARROW_BUILD_UTILITIES=OFF \ - -DARROW_BUILD_TESTS=OFF \ - -DARROW_ENABLE_TIMING_TESTS=OFF \ - -DARROW_FUZZING=OFF \ - -DARROW_USE_ASAN=OFF \ - -DARROW_USE_TSAN=OFF \ - -DARROW_USE_UBSAN=OFF \ - -DARROW_JEMALLOC=OFF \ - -DARROW_BUILD_SHARED=ON \ - -DARROW_BUILD_STATIC=OFF - make -j$(nproc) - make install - popd || exit - popd || exit - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} - -install_boost() { - workdir=$1 - install_prefix=$2 - - if [[ -f "${install_prefix}/include/boost/version.hpp" ]]; then - log "boost already installed, skip." - return 0 - fi - - directory="boost_1_74_0" - file="${directory}.tar.gz" - url="https://boostorg.jfrog.io/artifactory/main/release/1.74.0/source" - url=$(maybe_set_to_cn_url ${url}) - log "Building and installing ${directory}." - pushd "${workdir}" || exit - download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" - pushd ${directory} || exit - - ./bootstrap.sh --prefix="${install_prefix}" \ - --with-libraries=system,filesystem,context,program_options,regex,thread,random,chrono,atomic,date_time - ./b2 install link=shared runtime-link=shared variant=release threading=multi - popd || exit - popd || exit - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} - -install_openssl() { - workdir=$1 - install_prefix=$2 - - if [[ -f "${install_prefix}/include/openssl/ssl.h" ]]; then - log "openssl already installed, skip." - return 0 - fi - - directory="openssl-OpenSSL_1_1_1k" - file="OpenSSL_1_1_1k.tar.gz" - url="https://github.com/openssl/openssl/archive" - url=$(maybe_set_to_cn_url ${url}) - log "Building and installing ${directory}." - pushd "${workdir}" || exit - download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" - pushd ${directory} || exit - - ./config --prefix="${install_prefix}" -fPIC -shared - make -j$(nproc) - make install - popd || exit - popd || exit - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} - -install_openssl_static() { - workdir=$1 - install_prefix=$2 - - if [[ -f "${install_prefix}/include/openssl/ssl.h" ]]; then - log "openssl already installed, skip." - return 0 - fi - - directory="openssl-OpenSSL_1_1_1k" - file="OpenSSL_1_1_1k.tar.gz" - url="https://github.com/openssl/openssl/archive" - url=$(maybe_set_to_cn_url ${url}) - log "Building and installing ${directory}." - pushd "${workdir}" || exit - download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" - pushd ${directory} || exit - - ./config --prefix="${install_prefix}" no-shared -fPIC - make -j$(nproc) - make install - popd || exit - popd || exit - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} - -install_zlib() { - workdir=$1 - install_prefix=$2 - - if [[ -f "${install_prefix}/include/zlib.h" ]]; then - log "zlib already installed, skip." - return 0 - fi - - directory="zlib-1.2.11" - file="v1.2.11.tar.gz" - url="https://github.com/madler/zlib/archive" - url=$(maybe_set_to_cn_url ${url}) - log "Building and installing ${directory}." - pushd "${workdir}" || exit - download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" - pushd ${directory} || exit - - cmake . -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ - -DCMAKE_PREFIX_PATH="${install_prefix}" \ - -DBUILD_SHARED_LIBS=ON - make -j$(nproc) - make install - popd || exit - popd || exit - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} - -install_protobuf() { - workdir=$1 - install_prefix=$2 - - # Check whether protobuf is installed - if [[ -f "${install_prefix}/include/google/protobuf/port.h" ]]; then - log "protobuf already installed, skip." - return 0 - fi - - # Define the version and download link for protobuf - directory="protobuf-21.9" - file="protobuf-all-21.9.tar.gz" - url="https://github.com/protocolbuffers/protobuf/releases/download/v21.9" - url=$(maybe_set_to_cn_url ${url}) - log "Building and installing ${directory}." - pushd "${workdir}" || exit - download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" - pushd ${directory} || exit - - # Configure and compile protobuf - ./configure --prefix="${install_prefix}" --enable-shared --disable-static - make -j$(nproc) - make install - popd || exit - popd || exit - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} - -install_grpc() { - workdir=$1 - install_prefix=$2 - - # Check if grpc is installed - if [[ -f "${install_prefix}/include/grpcpp/grpcpp.h" ]]; then - log "grpc already installed, skip." - return 0 - fi - - # Define the grpc version and download link - directory="grpc" - branch="v1.49.1" - file="${directory}-${branch}.tar.gz" - url="https://github.com/grpc/grpc.git" - url=$(maybe_set_to_cn_url ${url}) - log "Building and installing ${directory}." - pushd "${workdir}" || exit - if [[ ${url} == *.git ]]; then - clone_if_not_exists ${directory} ${file} "${url}" ${branch} - else - download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" - fi - pushd ${directory} || exit - - # Configure and compile grpc - cmake . -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ - -DCMAKE_PREFIX_PATH="${install_prefix}" \ - -DBUILD_SHARED_LIBS=ON \ - -DgRPC_INSTALL=ON \ - -DgRPC_BUILD_TESTS=OFF \ - -DgRPC_BUILD_CSHARP_EXT=OFF \ - -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF \ - -DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF \ - -DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF \ - -DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF \ - -DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF \ - -DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF \ - -DgRPC_BACKWARDS_COMPATIBILITY_MODE=ON \ - -DgRPC_PROTOBUF_PROVIDER=package \ - -DgRPC_ZLIB_PROVIDER=package \ - -DgRPC_SSL_PROVIDER=package \ - -DOPENSSL_ROOT_DIR="${install_prefix}" \ - -DCMAKE_CXX_FLAGS="-fpermissive" \ - -DPNG_ARM_NEON_OPT=0 - make -j$(nproc) - make install - popd || exit - popd || exit - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} - -install_patchelf() { - workdir=$1 - install_prefix=$2 - - # Check if patchelf is installed - if [[ -f "${install_prefix}/bin/patchelf" ]]; then - log "patchelf already installed, skip." - return 0 - fi - - # Define the version and download link for patchelf - ARCH=$(uname -m) - - directory="patchelf" # patchelf doesn't have a folder - file="patchelf-0.14.5-${ARCH}.tar.gz" - url="https://github.com/NixOS/patchelf/releases/download/0.14.5" - url=$(maybe_set_to_cn_url ${url}) - - # Log and start installing patchelf - log "Building and installing ${directory}." - pushd "${workdir}" || exit - mkdir -p "${directory}" - pushd "${directory}" || exit - download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" - mkdir -p ${install_prefix}/bin - mv bin/patchelf ${install_prefix}/bin/patchelf - - # Go back to your working directory and clean up your files - popd || exit - popd || exit - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} - -install_cppkafka() { - workdir=$1 - install_prefix=$2 - # Check whether cppkafka is installed - if [[ -f "${install_prefix}/include/cppkafka/cppkafka.h" ]]; then - log "cppkafka already installed, skip." - return 0 - fi - - # Define the cppkafka version and download link - directory="cppkafka-0.4.0" - file="0.4.0.tar.gz" - url="https://graphscope.oss-cn-beijing.aliyuncs.com/dependencies" - url=$(maybe_set_to_cn_url ${url}) - - # Log and start installing cppkafka - log "Building and installing ${directory}." - pushd "${workdir}" || exit - download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" - pushd ${directory} || exit - # Configure and compile cppkafka - # cppkafka may not find the lib64 directory - export LIBRARY_PATH=${LIBRARY_PATH}:${install_prefix}/lib:${install_prefix}/lib64 - - cmake . -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ - -DCMAKE_PREFIX_PATH="${install_prefix}" \ - -DCPPKAFKA_DISABLE_TESTS=ON \ - -DCPPKAFKA_DISABLE_EXAMPLES=ON - make -j4 - make install - popd || exit - popd || exit - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} - -install_maven() { - workdir=$1 - install_prefix=$2 - - # Check if maven is installed - if [[ -f "${install_prefix}/bin/mvn" ]]; then - log "maven already installed, skip." - return 0 - fi - - # Define the maven version and download link - directory="apache-maven-3.8.6" - file="apache-maven-3.8.6-bin.tar.gz" - url="https://archive.apache.org/dist/maven/maven-3/3.8.6/binaries" - url=$(maybe_set_to_cn_url ${url}) - - # Log and start installing maven - log "Building and installing ${directory}." - pushd "${workdir}" || exit - download_tar_and_untar_if_not_exists ${directory} ${file} "${url}" - cp -r ${directory} "${install_prefix}"/ - - # Configure maven's environment variables - mkdir -p "${install_prefix}"/bin - ln -s "${install_prefix}/${directory}/bin/mvn" "${install_prefix}/bin/mvn" - popd || exit - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} - -install_hiactor() { - install_prefix=$1 - pushd /tmp - - git clone https://github.com/alibaba/hiactor.git -b v0.1.1 --single-branch - cd hiactor && git submodule update --init --recursive - - # Configure and compile hiactor - sudo bash ./seastar/seastar/install-dependencies.sh - mkdir build && cd build - cmake -DHiactor_DEMOS=OFF -DHiactor_TESTING=OFF -DHiactor_DPDK=OFF -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ - -DHiactor_CXX_DIALECT=gnu++17 -DSeastar_CXX_FLAGS="-DSEASTAR_DEFAULT_ALLOCATOR -mno-avx512" .. - make -j 4 && make install - popd && rm -rf /tmp/hiactor -} diff --git a/python/graphscope/gsctl/scripts/lib/install_vineyard.sh b/python/graphscope/gsctl/scripts/lib/install_vineyard.sh deleted file mode 100644 index d7814c1b9616..000000000000 --- a/python/graphscope/gsctl/scripts/lib/install_vineyard.sh +++ /dev/null @@ -1,55 +0,0 @@ -install_vineyard() { - workdir=$1 - install_prefix=$2 - v6d_version=$3 - jobs=${4:-4} # $4:default=4 - V6D_PREFIX="/opt/vineyard" # fixed, related to coordinator/setup.py - - if [[ -f "${V6D_PREFIX}/include/vineyard/client/client.h" ]]; then - log "vineyard already installed, skip." - return 0 - fi - - auditwheel_path=$(python3 -c "import auditwheel; print(auditwheel.__path__[0] + '/main_repair.py')") - sed -i 's/p.error/logger.warning/g' ${auditwheel_path} - - log "Building and installing v6d." - pushd "${workdir}" || exit - if [[ "${v6d_version}" != "v"* ]]; then - directory="v6d" - file="${directory}-${v6d_version}.tar.gz" - url="https://github.com/v6d-io/v6d.git" - clone_if_not_exists ${directory} "${file}" "${url}" "${v6d_version}" - else - # remove the prefix 'v' - directory="v6d-${v6d_version:1:100}" - file="${directory}.tar.gz" - url="https://github.com/v6d-io/v6d/releases/download/${v6d_version}" - cn_url=$(maybe_set_to_cn_url "${url}") - status=$(curl --head --silent "${cn_url}"/"${file}" | head -n 1) - if echo "${status}" | grep -q 404; then - download_tar_and_untar_if_not_exists ${directory} "${file}" "${url}" - else - download_tar_and_untar_if_not_exists ${directory} "${file}" "${cn_url}" - fi - fi - pushd ${directory} || exit - - # make sure it complain loudly if installing vineyard fails - set -e - cmake . -DCMAKE_PREFIX_PATH="${install_prefix}" \ - -DCMAKE_INSTALL_PREFIX="${V6D_PREFIX}" \ - -DBUILD_VINEYARD_TESTS=OFF \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_VINEYARD_PYTHON_BINDINGS=ON \ - -DBUILD_VINEYARD_GRAPH_WITH_GAR=ON - make -j"${jobs}" - make install - strip "${V6D_PREFIX}"/bin/vineyard* "${V6D_PREFIX}"/lib/libvineyard* - pip3 install --no-cache -i https://pypi.org/simple -U "vineyard" "vineyard-io" - cp -rs "${V6D_PREFIX}"/* "${install_prefix}"/ - set +e - popd || exit - popd || exit - cleanup_files "${workdir}/${directory}" "${workdir}/${file}" -} diff --git a/python/graphscope/gsctl/scripts/lib/log.sh b/python/graphscope/gsctl/scripts/lib/log.sh deleted file mode 100644 index 4a743eefca47..000000000000 --- a/python/graphscope/gsctl/scripts/lib/log.sh +++ /dev/null @@ -1,20 +0,0 @@ - -err() { - echo -e "$(red_bold "[ERROR] $*")" >&2 -} - -warning() { - echo -e "$(yellow_bold "[WARNING] $*")" >&1 -} - -log() { - echo -e "$(green "$*")" >&1 -} - -info() { - echo -e "$(green "$*")" >&1 -} - -succ() { - echo -e "$(green_bold "[SUCCESS]: $*")" >&1 -} diff --git a/python/graphscope/gsctl/scripts/lib/util.sh b/python/graphscope/gsctl/scripts/lib/util.sh deleted file mode 100644 index 213496f1b8df..000000000000 --- a/python/graphscope/gsctl/scripts/lib/util.sh +++ /dev/null @@ -1,64 +0,0 @@ -# refer from https://github.com/pypa/manylinux/blob/b4884d90c984cb17f7cb4aabe3509347698d7ee7/docker/build_scripts/build_utils.sh#L26 -function fetch_source { - local file=$1 - local url=$2 - if [ -f "${file}" ]; then - echo "${file} exists, skipping fetch" - else - curl -fsSL -o "${file}" "${url}/${file}" - # Use sock5s proxy to download files in case download fails in normal cases - # `host.docker.internal` is the localhost of host machine from a container's perspective. - # See https://docs.docker.com/desktop/networking/#i-want-to-connect-from-a-container-to-a-service-on-the-host - # curl -fsSL -o ${file} ${url}/${file} || curl -x socks5h://host.docker.internal:13659 -fsSL -o ${file} ${url}/${file} - fi -} - -function download_tar_and_untar_if_not_exists { - local directory=$1 - local file=$2 - local url=$3 - if [ ! -d "${directory}" ]; then - [ ! -f "${file}" ] && - fetch_source "${file}" "${url}" - tar zxf "${file}" - fi -} - -function clone_if_not_exists { - local directory=$1 - local file=$2 - local url=$3 - local branch=$4 - if [ ! -d "${directory}" ]; then - if [ ! -f "${file}" ]; then - git clone --depth=1 --branch "${branch}" "${url}" "${directory}" - pushd "${directory}" || exit - git submodule update --init || true - popd || exit - else - tar zxf "${file}" - fi - fi -} - -function cleanup_files { - if [ "${GRAPHSCOPE_NO_INSTALL_CLEANUP}" != "true" ]; then - log "Cleaning up intermediate files [$*]" - log "Disable this behaviour by setting GRAPHSCOPE_NO_INSTALL_CLEANUP=true." - for file in "$@" - do - log "Cleaning up ${file}" - if [[ -f "${file}" || -d "${file}" ]]; then - rm -rf "${file}" - fi - done - fi -} - -function maybe_set_to_cn_url { - local url=$1 - if [ "${GRAPHSCOPE_DOWNLOAD_FROM_CN}" == "true" ]; then - url="https://graphscope.oss-cn-beijing.aliyuncs.com/dependencies" - fi - echo ${url} -} diff --git a/python/graphscope/gsctl/scripts/make_command.sh b/python/graphscope/gsctl/scripts/make_command.sh deleted file mode 100644 index c09b5b16c0a1..000000000000 --- a/python/graphscope/gsctl/scripts/make_command.sh +++ /dev/null @@ -1,107 +0,0 @@ -script_dir="$(cd "$(dirname "$0")" && pwd)" -source ${script_dir}/lib/get_os_version.sh -source ${script_dir}/lib/log.sh -source ${script_dir}/lib/colors.sh -source ${script_dir}/lib/install_thirdparty_dependencies.sh -source ${script_dir}/lib/install_vineyard.sh -source ${script_dir}/lib/util.sh -source ${script_dir}/initialize.sh - - -echo "# this file is located in 'src/make_command.sh'" -echo "# code for 'gs make' goes here" -echo "# you can edit it freely and regenerate (it will not be overwritten)" - -while getopts ":c:i:s:" opt; do - case $opt in - c) - component="$OPTARG" - ;; - i) - install_prefix="$OPTARG" - ;; - s) - storage_type="$OPTARG" - ;; - \?) - echo "Invalid options: -$OPTARG" >&2 - exit 1 - ;; - :) - echo "The option -$OPTARG requires parameters" >&2 - exit 1 - ;; - esac -done - - -log "Making component ${component}" - - -GS_SOURCE_DIR="$(dirname -- "$(readlink -f "${BASH_SOURCE}")")" - -echo "dir $GS_SOURCE_DIR" - - -export INSTALL_PREFIX=${install_prefix} - -make_all() { - make all -} - -make_install() { - make install -} - -make_analytical() { - make analytical -} - -make_interactive() { - if [[ ${storage_type} = "experimental" ]]; then - cd "${GS_SOURCE_DIR}"/interactive_engine/compiler && make build QUIET_OPT="" - elif [[ ${storage_type} = "vineyard" ]]; then - cd "${GS_SOURCE_DIR}"/interactive_engine && mvn install -DskipTests -Drust.compile.mode=release -P graphscope,graphscope-assembly - cd "${GS_SOURCE_DIR}"/interactive_engine/assembly/target && tar xvzf graphscope.tar.gz - else - make interactive - fi -} - -make_learning() { - make learning -} - -make_analytical-install() { - make analytical-install INSTALL_PREFIX=${install_prefix} -} - -make_interactive-install() { - make interactive-install INSTALL_PREFIX=${install_prefix} -} - -make_learning-install() { - make learning-install INSTALL_PREFIX=${install_prefix} -} - -make_client() { - make client -} - -make_coordinator() { - make coordinator -} - -make_analytical-java() { - make analytical-java -} - -make_analytical-java-install() { - make analytical-java-install INSTALL_PREFIX=${install_prefix} -} - -make_clean() { - make clean -} - -make_${component} diff --git a/python/graphscope/gsctl/scripts/make_image_command.sh b/python/graphscope/gsctl/scripts/make_image_command.sh deleted file mode 100644 index d83ca19b649f..000000000000 --- a/python/graphscope/gsctl/scripts/make_image_command.sh +++ /dev/null @@ -1,95 +0,0 @@ -script_dir="$(cd "$(dirname "$0")" && pwd)" -source ${script_dir}/lib/get_os_version.sh -source ${script_dir}/lib/log.sh -source ${script_dir}/lib/colors.sh -source ${script_dir}/lib/install_thirdparty_dependencies.sh -source ${script_dir}/lib/install_vineyard.sh -source ${script_dir}/lib/util.sh -source ${script_dir}/initialize.sh - - -echo "# this file is located in 'src/make_image_command.sh'" -echo "# code for 'gs make-image' goes here" -echo "# you can edit it freely and regenerate (it will not be overwritten)" - -# allowed: [all, graphscope-dev, coordinator, analytical, analytical-java, interactive, interactive-frontend, interactive-executor, learning, vineyard-dev, vineyard-runtime] - -while getopts ":c:r:t:" opt; do - case $opt in - c) - component="$OPTARG" - ;; - r) - registry="$OPTARG" - ;; - t) - tag="$OPTARG" - ;; - \?) - echo "Invalid options: -$OPTARG" >&2 - exit 1 - ;; - :) - echo "The option -$OPTARG requires parameters" >&2 - exit 1 - ;; - esac -done - - -log "Making image ${component}" - -export INSTALL_PREFIX=${install_prefix} - -make_all() { - cd k8s - make all -} - -make_graphscope-dev() { - cd k8s - make graphscope-dev REGISTRY=${registry} VERSION=${tag} -} - -make_analytical() { - cd k8s - make analytical REGISTRY=${registry} VERSION=${tag} -} - -make_interactive() { - cd k8s - make interactive REGISTRY=${registry} VERSION=${tag} -} - -make_interactive-frontend() { - cd k8s - make interactive-frontend REGISTRY=${registry} VERSION=${tag} -} - -make_interactive-executor() { - cd k8s - make interactive-executor REGISTRY=${registry} VERSION=${tag} -} - -make_learning() { - cd k8s - make learning REGISTRY=${registry} VERSION=${tag} -} - - -make_coordinator() { - cd k8s - make coordinator REGISTRY=${registry} VERSION=${tag} -} - -make_vineyard-dev() { - cd k8s - make vineyard-dev REGISTRY=${registry} VERSION=${tag} -} - -make_vineyard-runtime() { - cd k8s - make vineyard-runtime REGISTRY=${registry} VERSION=${tag} -} - -make_${component} diff --git a/python/graphscope/gsctl/scripts/spark_classpath_command.sh b/python/graphscope/gsctl/scripts/spark_classpath_command.sh deleted file mode 100644 index b763c477a985..000000000000 --- a/python/graphscope/gsctl/scripts/spark_classpath_command.sh +++ /dev/null @@ -1,22 +0,0 @@ -script_dir="$(cd "$(dirname "$0")" && pwd)" -source ${script_dir}/lib/get_os_version.sh -source ${script_dir}/lib/log.sh -source ${script_dir}/lib/colors.sh -source ${script_dir}/lib/install_thirdparty_dependencies.sh -source ${script_dir}/lib/install_vineyard.sh -source ${script_dir}/lib/util.sh - - -echo "# this file is located in 'src/spark_classpath_command.sh'" -echo "# code for 'gs spark-classpath' goes here" -echo "# you can edit it freely and regenerate (it will not be overwritten)" - -# TODO(zhanglei): generate a file named ~/.graphscope_4spark.env - -# check if the file exists -# if yes, raise a warning -# otherwise, generate a new - -echo "$(green_bold "Generated environment variables for spark jobs in ~/.graphscope_4spark.env")" -echo -echo "export the env to your bash terminal by running: source ~/.graphscope_4spark.env" diff --git a/python/graphscope/gsctl/scripts/test_command.sh b/python/graphscope/gsctl/scripts/test_command.sh deleted file mode 100644 index 4b051c24f3df..000000000000 --- a/python/graphscope/gsctl/scripts/test_command.sh +++ /dev/null @@ -1,172 +0,0 @@ -script_dir="$(cd "$(dirname "$0")" && pwd)" -source ${script_dir}/lib/get_os_version.sh -source ${script_dir}/lib/log.sh -source ${script_dir}/lib/colors.sh -source ${script_dir}/lib/install_thirdparty_dependencies.sh -source ${script_dir}/lib/install_vineyard.sh -source ${script_dir}/lib/util.sh -source ${script_dir}/initialize.sh - - -while getopts ":t:d:l:s:k:n:" opt; do - case $opt in - t) - type=$OPTARG - ;; - d) - testdata=$OPTARG - ;; - l) - on_local=$OPTARG - ;; - s) - storage_type=$OPTARG - ;; - k) - on_k8s=$OPTARG - ;; - n) - nx=$OPTARG - ;; - :) - echo "Invalid options: -$OPTARG" >&2 - exit 1 - ;; - :) - echo "The option -$OPTARG requires parameters" >&2 - exit 1 - ;; - esac -done - -export GS_TEST_DIR=${testdata} - -# analytical, analytical-java, interactive, learning, e2e, groot - -GS_SOURCE_DIR="$(dirname -- "$(readlink -f "${BASH_SOURCE}")")" - - -function get_test_data { - if [[ ! -d ${GS_TEST_DIR} ]]; then - log "Downloading test data to ${testdata}" - git clone -b master --single-branch --depth=1 https://github.com/graphscope/gstest.git "${GS_TEST_DIR}" - fi -} - -function test_analytical { - get_test_data - info "Testing analytical on local" - "${GS_SOURCE_DIR}"/analytical_engine/test/app_tests.sh --test_dir "${GS_TEST_DIR}" -} - -function test_analytical-java { - get_test_data - info "Testing analytical-java on local" - - pushd "${GS_SOURCE_DIR}"/analytical_engine/java || exit - mvn test -Dmaven.antrun.skip=true - popd || exit - - version=$(cat "${GS_SOURCE_DIR}"/VERSION) - export RUN_JAVA_TESTS=ON - export USER_JAR_PATH="${GS_SOURCE_DIR}"/analytical_engine/java/grape-demo/target/grape-demo-${version}-shaded.jar - # for giraph test - export GIRAPH_JAR_PATH="${GS_SOURCE_DIR}"/analytical_engine/java/grape-giraph/target/grape-giraph-${version}-shaded.jar - - "${GS_SOURCE_DIR}"/analytical_engine/test/app_tests.sh --test_dir "${GS_TEST_DIR}" -} - -function test_interactive { - get_test_data - if [ "${on_local}" == "True" ]; then - if [[ ${storage_type} = "experimental" ]]; then - info "Testing interactive on local with experimental storage" - # IR unit test - cd "${GS_SOURCE_DIR}"/interactive_engine/compiler && make test - # CommonType Unit Test - cd "${GS_SOURCE_DIR}"/interactive_engine/executor/common/dyn_type && cargo test - # Store Unit test - cd "${GS_SOURCE_DIR}"/interactive_engine/executor/store/exp_store && cargo test - # IR integration test - cd "${GS_SOURCE_DIR}"/interactive_engine/compiler && ./ir_exprimental_ci.sh - # IR integration pattern test - cd "${GS_SOURCE_DIR}"/interactive_engine/compiler && ./ir_exprimental_pattern_ci.sh - elif [[ ${storage_type} = "vineyard" ]]; then - info "Testing interactive on local with vineyard storage" - # start vineyard service - export VINEYARD_IPC_SOCKET=/tmp/vineyard.sock - vineyardd --socket=${VINEYARD_IPC_SOCKET} --meta=local & - # load modern graph - export STORE_DATA_PATH="${GS_SOURCE_DIR}"/charts/gie-standalone/data - vineyard-graph-loader --config "${GS_SOURCE_DIR}"/charts/gie-standalone/config/v6d_modern_loader.json - # start gie executor && frontend - export GRAPHSCOPE_HOME="${GS_SOURCE_DIR}"/interactive_engine/assembly/target/graphscope - schema_json=$(ls /tmp/*.json | head -1) - object_id=${schema_json//[^0-9]/} - GRAPHSCOPE_HOME=${GRAPHSCOPE_HOME} ${GRAPHSCOPE_HOME}/bin/giectl create_gremlin_instance_on_local /tmp/gs/${object_id} ${object_id} ${schema_json} 1 1235 1234 8182 ${VINEYARD_IPC_SOCKET} - # IR integration test - sleep 3s - cd "${GS_SOURCE_DIR}"/interactive_engine/compiler - make gremlin_test || true - # clean - rm -rf /tmp/*.json - id=$(pgrep -f 'gaia_executor') - if [[ -n ${id} ]]; then - echo ${id} | xargs kill - fi - id=$(pgrep -f 'frontend') - if [[ -n ${id} ]]; then - echo ${id} | xargs kill - fi - id=$(pgrep -f 'vineyardd') - if [[ -n ${id} ]]; then - echo ${id} | xargs kill -9 - fi - else - info "Testing interactive on local with default storage" - fi - fi - if [ "${on_k8s}" == "True" ]; then - info "Testing interactive on k8s" - export PYTHONPATH="${GS_SOURCE_DIR}"/python:${PYTHONPATH} - cd "${GS_SOURCE_DIR}"/interactive_engine && mvn clean install --quiet -DskipTests -Drust.compile.skip=true -P graphscope,graphscope-assembly - cd "${GS_SOURCE_DIR}"/interactive_engine/tests || exit - ./function_test.sh 8112 2 - fi -} -function test_learning { - get_test_data - err "Not implemented" - exit 1 -} - -function test_e2e { - get_test_data - # Import python projects in the source directory - cd "${GS_SOURCE_DIR}"/python || exit - if [ "${on_local}" == "True" ]; then - # unittest - python3 -m pytest -s -vvv --exitfirst graphscope/tests/minitest/test_min.py - fi - if [ "${on_k8s}" == "True" ]; then - # Run tests in Kubernetes environment using pytest - python3 -m pytest -s -vvv --exitfirst ./graphscope/tests/kubernetes/test_demo_script.py - fi -} - -function test_groot { - # Used to test groot - get_test_data - if [ "${on_local}" == "True" ]; then - info "Testing groot on local" - cd "${GS_SOURCE_DIR}"/interactive_engine/groot-server - mvn test -Pgremlin-test - fi - if [ "${on_k8s}" == "True" ]; then - info "Testing groot on k8s, note you must already setup a groot cluster and necessary environment variables" - cd "${GS_SOURCE_DIR}"/python || exit - python3 -m pytest --exitfirst -s -vvv ./graphscope/tests/kubernetes/test_store_service.py - fi -} - -test_"${type}" diff --git a/python/graphscope/gsctl/version.py b/python/graphscope/gsctl/version.py index 71aa867aef68..1d2bc863ffc4 100644 --- a/python/graphscope/gsctl/version.py +++ b/python/graphscope/gsctl/version.py @@ -23,9 +23,13 @@ with open(version_file_path, "r", encoding="utf-8") as fp: sv = version.parse(fp.read().strip()) - __is_prerelease__ = sv.is_prerelease __version__ = str(sv) -__version_tuple__ = (v for v in __version__.split(".")) +version_file_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "V6D_VERSION" +) +with open(version_file_path, "r", encoding="utf-8") as fp: + sv = version.parse(fp.read().strip()) + __v6d_version__ = str(sv) del version_file_path diff --git a/python/setup_gsctl.py b/python/setup_gsctl.py index 415cf05e9f5b..3d118445204f 100644 --- a/python/setup_gsctl.py +++ b/python/setup_gsctl.py @@ -71,7 +71,7 @@ def parse_version(root, **kwargs): long_description="""\ gsctl is a command-line utility for GraphScope. It provides a set of functionalities to make it easy to use GraphScope. These functionalities include building and testing binaries, managing sessions and resources, and more. """, # noqa: E501 - package_data={"graphscope.gsctl": ["scripts/*.sh", "scripts/lib/*.sh", "VERSION"]}, + package_data={"graphscope.gsctl": ["scripts/*.sh", "VERSION", "V6D_VERSION"]}, entry_points={ "console_scripts": [ "gsctl = graphscope.gsctl.gsctl:cli",