From adf6801e8788dfbf4ac1e3ef12d2b91a7011f6c5 Mon Sep 17 00:00:00 2001 From: Cong Zhao Date: Fri, 25 Nov 2022 23:30:39 +0800 Subject: [PATCH] [improve][build] Improve package & cli & build image (#200) * improve package * Refactor computer: abstract computer-api from computer-core (#196) * address comment * improve pom and options * fix package * improve build image * improve start-computer.sh * improve log file * improve start-computer.sh * improve build image * trigger maven cache invalidation * Update ci.yml Co-authored-by: imbajin --- .github/workflows/publish_images.yml | 11 +-- computer-algorithm/Dockerfile | 23 ----- computer-algorithm/pom.xml | 1 + computer-core/pom.xml | 6 -- .../core/util/ComputerContextUtil.java | 3 +- .../computer/core/util/ShutdownHook.java | 3 - .../computer/core/worker/WorkerService.java | 4 +- computer-dist/Dockerfile | 7 +- computer-dist/pom.xml | 5 + .../src/assembly/static/bin/start-computer.sh | 97 +++++++++++-------- .../assembly/static/conf/computer.properties | 16 +++ .../src/assembly/static/conf/log4j2.xml | 66 +++++++++++++ .../src/assembly/travis/build-images.sh | 13 +-- .../computer/dist/HugeGraphComputer.java | 19 +++- .../src/main/resources/computer.properties | 0 computer-k8s-operator/Dockerfile | 2 +- .../controller/ComputerJobDeployer.java | 2 +- .../hugegraph/computer/k8s/Constants.java | 2 +- .../k8s/config/KubeDriverOptions.java | 2 +- .../src/main/resources/docker_push.sh | 2 +- computer-test/conf/images/docker_push_test.sh | 2 +- .../computer/dist/HugeGraphComputerTest.java | 6 +- .../resources/computer-worker1.properties | 15 --- ...-master.properties => computer.properties} | 2 +- .../src/main/resources/log4j2.xml | 0 pom.xml | 6 +- 26 files changed, 185 insertions(+), 130 deletions(-) delete mode 100644 computer-algorithm/Dockerfile create mode 100644 computer-dist/src/assembly/static/conf/computer.properties create mode 100644 computer-dist/src/assembly/static/conf/log4j2.xml delete mode 100644 computer-dist/src/main/resources/computer.properties delete mode 100644 computer-test/src/main/resources/computer-worker1.properties rename computer-test/src/main/resources/{computer-master.properties => computer.properties} (95%) rename {computer-core => computer-test}/src/main/resources/log4j2.xml (100%) diff --git a/.github/workflows/publish_images.yml b/.github/workflows/publish_images.yml index 608c9a74d..ba4c02b4d 100644 --- a/.github/workflows/publish_images.yml +++ b/.github/workflows/publish_images.yml @@ -19,13 +19,11 @@ jobs: map: | { "push": { - "FRAMEWORK_IMAGE_URL": "hugegraph/hugegraph-computer-framework:latest", - "ALGORITHM_IMAGE_URL": "hugegraph/hugegraph-builtin-algorithms:latest", + "COMPUTER_IMAGE_URL": "hugegraph/hugegraph-computer:latest", "OPERATOR_IMAGE_URL": "hugegraph/hugegraph-computer-operator:latest" }, "release": { - "FRAMEWORK_IMAGE_URL": "hugegraph/hugegraph-computer-framework:${{ github.ref_name }}", - "ALGORITHM_IMAGE_URL": "hugegraph/hugegraph-builtin-algorithms:${{ github.ref_name }}", + "COMPUTER_IMAGE_URL": "hugegraph/hugegraph-computer:${{ github.ref_name }}", "OPERATOR_IMAGE_URL": "hugegraph/hugegraph-computer-operator:${{ github.ref_name }}" } } @@ -58,13 +56,12 @@ jobs: - name: Build images env: TRAVIS_DIR: computer-dist/src/assembly/travis - run: $TRAVIS_DIR/build-images.sh ${{ env.FRAMEWORK_IMAGE_URL }} ${{ env.ALGORITHM_IMAGE_URL }} ${{ env.OPERATOR_IMAGE_URL }} + run: $TRAVIS_DIR/build-images.sh ${{ env.COMPUTER_IMAGE_URL }} ${{ env.OPERATOR_IMAGE_URL }} - name: Docker images run: docker images - name: Publish run: | - docker push ${{ env.FRAMEWORK_IMAGE_URL }} - docker push ${{ env.ALGORITHM_IMAGE_URL }} + docker push ${{ env.COMPUTER_IMAGE_URL }} docker push ${{ env.OPERATOR_IMAGE_URL }} diff --git a/computer-algorithm/Dockerfile b/computer-algorithm/Dockerfile deleted file mode 100644 index 26c04aefd..000000000 --- a/computer-algorithm/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -FROM hugegraph/hugegraph-computer-framework:latest - -LABEL maintainer="HugeGraph Docker Maintainers " - -ARG jarFilePath="/opt/jars/hugegraph-builtin-algorithms.jar" -COPY target/computer-algorithm-*.jar ${jarFilePath} -ENV JAR_FILE_PATH=${jarFilePath} diff --git a/computer-algorithm/pom.xml b/computer-algorithm/pom.xml index 4c641775c..b35c7eeda 100644 --- a/computer-algorithm/pom.xml +++ b/computer-algorithm/pom.xml @@ -32,6 +32,7 @@ com.baidu.hugegraph computer-api ${version} + provided org.mvel diff --git a/computer-core/pom.xml b/computer-core/pom.xml index 30f4ae567..cb19ed067 100644 --- a/computer-core/pom.xml +++ b/computer-core/pom.xml @@ -67,12 +67,6 @@ org.apache.hugegraph hugegraph-loader - - - spark-core_2.12 - org.apache.spark - - org.apache.hugegraph diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/util/ComputerContextUtil.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/util/ComputerContextUtil.java index f55af98b2..ca16ea7a0 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/util/ComputerContextUtil.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/util/ComputerContextUtil.java @@ -45,8 +45,7 @@ public static Config initContext(Map params) { algorithmParams = (AlgorithmParams) Class.forName( algorithmParamsName).newInstance(); } catch (Exception e) { - throw new ComputerException("Can't create algorithmParams, " + - "algorithmParamsName = {}", + throw new ComputerException("Can't create algorithmParams, algorithmParamsName = %s", algorithmParamsName); } algorithmParams.setAlgorithmParameters(params); diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/util/ShutdownHook.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/util/ShutdownHook.java index a31f5355c..fa11175db 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/util/ShutdownHook.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/util/ShutdownHook.java @@ -21,7 +21,6 @@ import java.io.Closeable; import org.apache.hugegraph.util.Log; -import org.apache.logging.log4j.LogManager; import org.slf4j.Logger; public class ShutdownHook { @@ -41,8 +40,6 @@ public boolean hook(Closeable hook) { } catch (Throwable e) { LOG.warn("Failed to execute shutdown hook: {}", e.getMessage(), e); - } finally { - LogManager.shutdown(); } }); Runtime.getRuntime().addShutdownHook(this.threadShutdownHook); diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerService.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerService.java index 515b34df4..aba86e31c 100644 --- a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerService.java +++ b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerService.java @@ -25,6 +25,8 @@ import java.util.List; import java.util.Map; +import org.apache.hugegraph.util.E; +import org.apache.hugegraph.util.Log; import org.slf4j.Logger; import com.baidu.hugegraph.computer.core.aggregator.Aggregator; @@ -56,8 +58,6 @@ import com.baidu.hugegraph.computer.core.sort.sorting.SortManager; import com.baidu.hugegraph.computer.core.store.FileManager; import com.baidu.hugegraph.computer.core.util.ShutdownHook; -import org.apache.hugegraph.util.E; -import org.apache.hugegraph.util.Log; public class WorkerService implements Closeable { diff --git a/computer-dist/Dockerfile b/computer-dist/Dockerfile index 6ab4d462a..d7649a64c 100644 --- a/computer-dist/Dockerfile +++ b/computer-dist/Dockerfile @@ -15,9 +15,12 @@ # limitations under the License. # FROM openjdk:11-jre -LABEL maintainer="HugeGraph Docker Maintainers " +LABEL maintainer="HugeGraph Docker Maintainers " # use ParallelGC which is more friendly to olap system ENV JAVA_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseParallelGC -XX:+UseContainerSupport -XX:MaxRAMPercentage=50 -XshowSettings:vm" -COPY . /etc/local/hugegraph-computer +COPY ./bin /etc/local/hugegraph-computer/bin +COPY ./lib /etc/local/hugegraph-computer/lib +COPY ./algorithm /etc/local/hugegraph-computer/algorithm +COPY ./conf /etc/local/hugegraph-computer/conf WORKDIR /etc/local/hugegraph-computer RUN apt-get update && apt-get -y install gettext-base && apt-get -y install wget diff --git a/computer-dist/pom.xml b/computer-dist/pom.xml index 61d7ebf45..2e4835262 100644 --- a/computer-dist/pom.xml +++ b/computer-dist/pom.xml @@ -116,6 +116,11 @@ + mkdir -p ${top.level.dir}/${final.name}/algorithm/ + cp ${top.level.dir}/computer-algorithm/target/computer-algorithm-${project.version}.jar ${top.level.dir}/${final.name}/algorithm/builtin-algorithm.jar + mkdir -p ${top.level.dir}/${final.name}/k8s-operator/ + cp ${top.level.dir}/computer-k8s-operator/target/hugegraph-computer-operator-${project.version}.jar ${top.level.dir}/${final.name}/k8s-operator/hugegraph-computer-operator.jar + tar -zcvf \ ${top.level.dir}/${final.name}.tar.gz \ ${final.name} || exit 1 diff --git a/computer-dist/src/assembly/static/bin/start-computer.sh b/computer-dist/src/assembly/static/bin/start-computer.sh index dcbff4ad0..3f119d5cd 100644 --- a/computer-dist/src/assembly/static/bin/start-computer.sh +++ b/computer-dist/src/assembly/static/bin/start-computer.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -14,12 +15,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # -#!/usr/bin/env bash -set -e + BIN_DIR=$(cd "$(dirname "$0")" && pwd -P) BASE_DIR=$(cd "${BIN_DIR}/.." && pwd -P) LIB_DIR=${BASE_DIR}/lib CONF_DIR="${BASE_DIR}/conf" +DEFAULT_ALGORITHM_DIR="${BASE_DIR}/algorithm" COMPUTER_CONF_PATH="${COMPUTER_CONF_PATH}" LOG4J_CONF_PATH="${LOG4J_CONF_PATH}" @@ -34,10 +35,12 @@ ROLE_WORKER="worker" usage() { echo "Usage:" - echo " start-computer.sh <-c|--conf conf_file_path>" - echo " <-a|--algorithm algorithm_jar_path>" + echo " start-computer.sh" + echo " [-c|--conf conf_file_path>" + echo " [-a|--algorithm algorithm_jar_path>" echo " [-l|--log4 log4_conf_path]" echo " <-d|--drive drive_type(local|k8s|yarn)>" + echo " <-r|--role role(master|worker)>" } if [ $# -lt 4 ]; then @@ -124,23 +127,13 @@ parse_opts() { parse_opts $* -echo "************************************" -echo "COMPUTER_CONF_PATH=${COMPUTER_CONF_PATH}" -echo "LOG4J_CONF_PATH=${LOG4J_CONF_PATH}" -echo "JAR_FILE_PATH=${JAR_FILE_PATH}" -echo "DRIVE=${DRIVE}" -echo "************************************" - -if [ "${JAR_FILE_PATH}" = "" ]; then - echo "graph algorithm jar file missed"; - usage; - exit 1; -fi - if [ "${COMPUTER_CONF_PATH}" = "" ]; then - echo "conf file missed"; - usage; - exit 1; + if [ "$DRIVE" = "$K8S_DRIVE" ]; then + echo "conf file missed"; + usage; + exit 1; + fi + COMPUTER_CONF_PATH=${CONF_DIR}/computer.properties fi if [ "${DRIVE}" = "" ]; then @@ -155,21 +148,37 @@ if [ "${ROLE}" = "" ]; then exit 1; fi +if [ "${LOG4J_CONF_PATH}" = "" ];then + LOG4J_CONF_PATH=${CONF_DIR}/log4j2.xml +fi + +echo "************************************" +echo "COMPUTER_CONF_PATH=${COMPUTER_CONF_PATH}" +echo "LOG4J_CONF_PATH=${LOG4J_CONF_PATH}" +echo "ALGORITHM_JAR_FILE_PATH=${JAR_FILE_PATH}" +echo "DRIVE=${DRIVE}" +echo "ROLE=${ROLE}" +echo "************************************" + CP=$(find "${LIB_DIR}" -name "*.jar" | tr "\n" ":") -CP="$JAR_FILE_PATH":${CP} +CP=${CP}:"${DEFAULT_ALGORITHM_DIR}/*" + +if [ "${JAR_FILE_PATH}" != "" ]; then + CP=${CP}:${JAR_FILE_PATH} +fi # Download remote job JAR file. -if [[ "${JOB_JAR_URI}" == http://* || "${JOB_JAR_URI}" == https://* ]]; then +if [[ "${REMOTE_JAR_URI}" == http://* || "${REMOTE_JAR_URI}" == https://* ]]; then mkdir -p "${BASE_DIR}/job" - echo "Downloading job JAR ${JOB_JAR_URI} to ${BASE_DIR}/job/" - wget -nv -P "${BASE_DIR}/job/" "${JOB_JAR_URI}" + echo "Downloading job JAR ${REMOTE_JAR_URI} to ${BASE_DIR}/job/" + wget -nv -P "${BASE_DIR}/job/" "${REMOTE_JAR_URI}" JOB_JAR=$(find "${BASE_DIR}/job" -name "*.jar" | tr "\n" ":") if [[ "$JOB_JAR" != "" ]]; then CP="${JOB_JAR}"$CP fi -elif [[ "${JOB_JAR_URI}" != "" ]]; then - echo "Unsupported protocol for ${JOB_JAR_URI}" +elif [[ "${REMOTE_JAR_URI}" != "" ]]; then + echo "Unsupported protocol for ${REMOTE_JAR_URI}" exit 1 fi @@ -195,26 +204,30 @@ if [ ! -a "${CONF_DIR}" ];then mkdir -p "${CONF_DIR}" fi -COPY_CONF_DIR="${CONF_DIR}/copy" -if [ ! -a "${COPY_CONF_DIR}" ]; then - mkdir -p "${COPY_CONF_DIR}" - chmod 777 "${COPY_CONF_DIR}" +if [ "$DRIVE" = "$K8S_DRIVE" ]; then + COPY_CONF_DIR="${CONF_DIR}/copy" + if [ ! -a "${COPY_CONF_DIR}" ]; then + mkdir -p "${COPY_CONF_DIR}" + chmod 777 "${COPY_CONF_DIR}" + fi + + NEW_COMPUTER_CONF_PATH="${COPY_CONF_DIR}/$(basename "${COMPUTER_CONF_PATH}")" + envsubst '${POD_IP},${HOSTNAME},${POD_NAME},${POD_NAMESPACE}' < "${COMPUTER_CONF_PATH}" > "${NEW_COMPUTER_CONF_PATH}" + chmod 777 "${NEW_COMPUTER_CONF_PATH}" + COMPUTER_CONF_PATH=${NEW_COMPUTER_CONF_PATH} fi -NEW_COMPUTER_CONF_PATH="${COPY_CONF_DIR}/$(basename "${COMPUTER_CONF_PATH}")" -envsubst '${POD_IP},${HOSTNAME},${POD_NAME},${POD_NAMESPACE}' < "${COMPUTER_CONF_PATH}" > "${NEW_COMPUTER_CONF_PATH}" -chmod 777 "${NEW_COMPUTER_CONF_PATH}" +LOG4j_CONF=-Dlog4j.configurationFile="${LOG4J_CONF_PATH}" -if [ "${LOG4J_CONF_PATH}" != "" ];then - LOG4j_CONF=-Dlog4j.configurationFile="${LOG4J_CONF_PATH}" +if [ "${ROLE}" = "${ROLE_MASTER}" ]; then + LOG_NAME=-Dlog.name=hugegraph-computer-master +fi + +if [ "${ROLE}" = "${ROLE_WORKER}" ]; then + LOG_NAME=-Dlog.name=hugegraph-computer-worker fi MAIN_CLASS=com.baidu.hugegraph.computer.dist.HugeGraphComputer -if [ "${LOG4j_CONF}" != "" ]; then - exec ${JAVA} -Dname="hugegraph-computer" "${LOG4j_CONF}" ${JAVA_OPTS} ${JVM_OPTIONS} \ - -cp "${CP}" ${MAIN_CLASS} "${NEW_COMPUTER_CONF_PATH}" ${ROLE} ${DRIVE} -else - exec ${JAVA} -Dname="hugegraph-computer" ${JAVA_OPTS} ${JVM_OPTIONS} -cp "${CP}" \ - ${MAIN_CLASS} "${NEW_COMPUTER_CONF_PATH}" ${ROLE} ${DRIVE} -fi +exec ${JAVA} -Dname="hugegraph-computer" "${LOG4j_CONF}" ${LOG_NAME} ${JAVA_OPTS} ${JVM_OPTIONS} \ + -cp "${CP}" ${MAIN_CLASS} "${COMPUTER_CONF_PATH}" ${ROLE} ${DRIVE} diff --git a/computer-dist/src/assembly/static/conf/computer.properties b/computer-dist/src/assembly/static/conf/computer.properties new file mode 100644 index 000000000..1e760932e --- /dev/null +++ b/computer-dist/src/assembly/static/conf/computer.properties @@ -0,0 +1,16 @@ +job.id=local_001 +job.workers_count=1 + +rpc.server_host=localhost +rpc.server_port=8190 + +bsp.etcd_endpoints=http://127.0.0.1:2379 + +hugegraph.url=http://127.0.0.1:8080 +hugegraph.name=hugegraph + +algorithm.params_class=com.baidu.hugegraph.computer.algorithm.centrality.pagerank.PageRankParams + +bsp.register_timeout=100000 +bsp.log_interval=30000 +bsp.max_super_step=2 diff --git a/computer-dist/src/assembly/static/conf/log4j2.xml b/computer-dist/src/assembly/static/conf/log4j2.xml new file mode 100644 index 000000000..250d7f6a9 --- /dev/null +++ b/computer-dist/src/assembly/static/conf/log4j2.xml @@ -0,0 +1,66 @@ + + + + + + logs + ${sys:log.name:-hugegraph-computer} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/computer-dist/src/assembly/travis/build-images.sh b/computer-dist/src/assembly/travis/build-images.sh index 1dededfe1..1cc0d6028 100755 --- a/computer-dist/src/assembly/travis/build-images.sh +++ b/computer-dist/src/assembly/travis/build-images.sh @@ -20,7 +20,6 @@ set -ev BASE_DIR=$(cd "$(dirname "$0")" && pwd -P) PROJECT_PATH="$(cd "${BASE_DIR}/../../../.." && pwd -P)" PROJECT_POM_PATH="${PROJECT_PATH}/pom.xml" -JAR_FILE_PATH=/opt/jars/hugegraph-builtin-algorithms.jar mvn -f "${PROJECT_POM_PATH}" clean package -DskipTests @@ -29,16 +28,6 @@ CONTEXT_PATH=$(mvn -f "${PROJECT_POM_PATH}" -q -N \ -Dexec.executable='echo' -Dexec.args='${final.name}') CONTEXT_PATH="${PROJECT_PATH}/${CONTEXT_PATH}" -PROJECT_VERSION=$(mvn -f "${PROJECT_POM_PATH}" -q -N \ - org.codehaus.mojo:exec-maven-plugin:1.3.1:exec \ - -Dexec.executable='echo' -Dexec.args='${project.version}') - docker build -t $1 $CONTEXT_PATH -f $PROJECT_PATH/computer-dist/Dockerfile -echo "FROM $1 -LABEL maintainer='HugeGraph Docker Maintainers ' -COPY target/computer-algorithm-*.jar $JAR_FILE_PATH -ENV JAR_FILE_PATH=$JAR_FILE_PATH" | \ -docker build -t $2 -f - $PROJECT_PATH/computer-algorithm - -docker build -t $3 -f $PROJECT_PATH/computer-k8s-operator/Dockerfile $PROJECT_PATH/computer-k8s-operator +docker build -t $2 -f $PROJECT_PATH/computer-k8s-operator/Dockerfile $PROJECT_PATH/computer-k8s-operator diff --git a/computer-dist/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputer.java b/computer-dist/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputer.java index 9b2a60f22..9b53608ea 100644 --- a/computer-dist/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputer.java +++ b/computer-dist/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputer.java @@ -25,6 +25,11 @@ import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; +import org.apache.hugegraph.config.OptionSpace; +import org.apache.hugegraph.config.RpcOptions; +import org.apache.hugegraph.util.E; +import org.apache.hugegraph.util.Log; +import org.apache.logging.log4j.LogManager; import org.slf4j.Logger; import com.baidu.hugegraph.computer.core.common.ComputerContext; @@ -34,9 +39,6 @@ import com.baidu.hugegraph.computer.core.network.message.MessageType; import com.baidu.hugegraph.computer.core.util.ComputerContextUtil; import com.baidu.hugegraph.computer.core.worker.WorkerService; -import org.apache.hugegraph.config.OptionSpace; -import org.apache.hugegraph.util.E; -import org.apache.hugegraph.util.Log; public class HugeGraphComputer { @@ -56,6 +58,8 @@ private static void loadClass() throws ClassNotFoundException { public static void main(String[] args) throws IOException, ClassNotFoundException { + Runtime.getRuntime().addShutdownHook(new Thread(LogManager::shutdown)); + E.checkArgument(ArrayUtils.getLength(args) == 3, "Argument count must be three, " + "the first is conf path;" + @@ -69,7 +73,7 @@ public static void main(String[] args) throws IOException, setUncaughtExceptionHandler(); loadClass(); registerOptions(); - ComputerContext context = parseContext(args[0]); + ComputerContext context = parseContext(args[0], role); switch (role) { case ROLE_MASTER: executeMasterService(context); @@ -124,12 +128,17 @@ private static void executeMasterService(ComputerContext context) { } } - private static ComputerContext parseContext(String conf) + private static ComputerContext parseContext(String conf, String role) throws IOException { Properties properties = new Properties(); BufferedReader bufferedReader = new BufferedReader( new FileReader(conf)); properties.load(bufferedReader); + properties.remove(RpcOptions.RPC_REMOTE_URL.name()); + if (ROLE_WORKER.equals(role)) { + properties.remove(RpcOptions.RPC_SERVER_HOST.name()); + properties.remove(RpcOptions.RPC_SERVER_PORT.name()); + } ComputerContextUtil.initContext(properties); return ComputerContext.instance(); } diff --git a/computer-dist/src/main/resources/computer.properties b/computer-dist/src/main/resources/computer.properties deleted file mode 100644 index e69de29bb..000000000 diff --git a/computer-k8s-operator/Dockerfile b/computer-k8s-operator/Dockerfile index 37172ce26..93b0fad6c 100644 --- a/computer-k8s-operator/Dockerfile +++ b/computer-k8s-operator/Dockerfile @@ -15,7 +15,7 @@ # limitations under the License. # FROM openjdk:11-jre -LABEL maintainer="HugeGraph Docker Maintainers " +LABEL maintainer="HugeGraph Docker Maintainers " WORKDIR /opt/app COPY target/hugegraph-computer-operator-*.jar hugegraph-computer-operator.jar ENTRYPOINT ["java", "-jar", "hugegraph-computer-operator.jar"] diff --git a/computer-k8s-operator/src/main/java/com/baidu/hugegraph/computer/k8s/operator/controller/ComputerJobDeployer.java b/computer-k8s-operator/src/main/java/com/baidu/hugegraph/computer/k8s/operator/controller/ComputerJobDeployer.java index c465660f1..2251ceb79 100644 --- a/computer-k8s-operator/src/main/java/com/baidu/hugegraph/computer/k8s/operator/controller/ComputerJobDeployer.java +++ b/computer-k8s-operator/src/main/java/com/baidu/hugegraph/computer/k8s/operator/controller/ComputerJobDeployer.java @@ -473,7 +473,7 @@ private Container getContainer(String name, String namespace, String remoteJarUri = spec.getRemoteJarUri(); if (StringUtils.isNotBlank(remoteJarUri)) { EnvVar jobJarURI = new EnvVarBuilder() - .withName(Constants.ENV_JOB_JAR_URI) + .withName(Constants.ENV_REMOTE_JAR_URI) .withValue(remoteJarUri) .build(); envVars.add(jobJarURI); diff --git a/computer-k8s/src/main/java/com/baidu/hugegraph/computer/k8s/Constants.java b/computer-k8s/src/main/java/com/baidu/hugegraph/computer/k8s/Constants.java index 279ed8c0a..7467b020a 100644 --- a/computer-k8s/src/main/java/com/baidu/hugegraph/computer/k8s/Constants.java +++ b/computer-k8s/src/main/java/com/baidu/hugegraph/computer/k8s/Constants.java @@ -40,7 +40,7 @@ public class Constants { public static final String ENV_COMPUTER_CONF_PATH = "COMPUTER_CONF_PATH"; public static final String ENV_LOG4J_CONF_PATH = "LOG4J_CONF_PATH"; public static final String ENV_JAR_FILE_PATH = "JAR_FILE_PATH"; - public static final String ENV_JOB_JAR_URI = "JOB_JAR_URI"; + public static final String ENV_REMOTE_JAR_URI = "REMOTE_JAR_URI"; public static final String ENV_JVM_OPTIONS = "JVM_OPTIONS"; public static final String ENV_CPU_LIMIT = "CPU_LIMIT"; public static final String ENV_MEMORY_LIMIT = "MEMORY_LIMIT"; diff --git a/computer-k8s/src/main/java/com/baidu/hugegraph/computer/k8s/config/KubeDriverOptions.java b/computer-k8s/src/main/java/com/baidu/hugegraph/computer/k8s/config/KubeDriverOptions.java index a5c2d5886..6bfd7c6e8 100644 --- a/computer-k8s/src/main/java/com/baidu/hugegraph/computer/k8s/config/KubeDriverOptions.java +++ b/computer-k8s/src/main/java/com/baidu/hugegraph/computer/k8s/config/KubeDriverOptions.java @@ -67,7 +67,7 @@ public static synchronized KubeDriverOptions instance() { "k8s.framework_image_url", "The image url of computer framework.", disallowEmpty(), - "hugegraph/hugegraph-computer-framework:latest" + "hugegraph/hugegraph-computer:latest" ); public static final ConfigOption BUILD_IMAGE_BASH_PATH = diff --git a/computer-k8s/src/main/resources/docker_push.sh b/computer-k8s/src/main/resources/docker_push.sh index 48163a18d..dbe3e6eb6 100755 --- a/computer-k8s/src/main/resources/docker_push.sh +++ b/computer-k8s/src/main/resources/docker_push.sh @@ -29,7 +29,7 @@ function build_image() { JAR_FILE="" IMG_URL="" FRAMEWORK_IMG_URL="" - MAINTAINER="HugeGraph Docker Maintainers " + MAINTAINER="HugeGraph Docker Maintainers " while getopts "r:u:p:s:j:i:f:" arg; do case ${arg} in diff --git a/computer-test/conf/images/docker_push_test.sh b/computer-test/conf/images/docker_push_test.sh index 46078ea11..5c45e8010 100644 --- a/computer-test/conf/images/docker_push_test.sh +++ b/computer-test/conf/images/docker_push_test.sh @@ -29,7 +29,7 @@ function build_image() { JAR_FILE="" IMG_URL="" FRAMEWORK_IMG_URL="" - MAINTAINER="HugeGraph Docker Maintainers " + MAINTAINER="HugeGraph Docker Maintainers " while getopts "r:u:p:s:j:i:f:" arg; do case ${arg} in diff --git a/computer-test/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputerTest.java b/computer-test/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputerTest.java index a54d54f9d..52e31fbac 100644 --- a/computer-test/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputerTest.java +++ b/computer-test/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputerTest.java @@ -41,9 +41,9 @@ public void testServiceWith1Worker() throws InterruptedException { CountDownLatch countDownLatch = new CountDownLatch(2); Throwable[] exceptions = new Throwable[2]; String masterConfPath = HugeGraphComputerTest.class.getResource( - "/computer-master.properties").getPath(); + "/computer.properties").getPath(); String work1ConfPath = HugeGraphComputerTest.class.getResource( - "/computer-worker1.properties").getPath(); + "/computer.properties").getPath(); pool.submit(() -> { try { Thread.sleep(2000L); @@ -79,7 +79,7 @@ public void testServiceWith1Worker() throws InterruptedException { @Test public void testServiceWithError() { String work1ConfPath = HugeGraphComputerTest.class.getResource( - "/computer-worker1.properties").getPath(); + "/computer.properties").getPath(); Assert.assertThrows(IllegalArgumentException.class, () -> { String[] args1 = {work1ConfPath, "worker111", diff --git a/computer-test/src/main/resources/computer-worker1.properties b/computer-test/src/main/resources/computer-worker1.properties deleted file mode 100644 index 8852515b8..000000000 --- a/computer-test/src/main/resources/computer-worker1.properties +++ /dev/null @@ -1,15 +0,0 @@ -job.id=local_002 -job.workers_count=1 - -transport.server_port=8086 - -bsp.register_timeout=100000 -bsp.log_interval=30000 -bsp.max_super_step=2 - -worker.computation_class=com.baidu.hugegraph.computer.core.worker.MockComputation - -algorithm.result_class=com.baidu.hugegraph.computer.core.graph.value.DoubleValue -algorithm.message_class=com.baidu.hugegraph.computer.core.graph.value.DoubleValue -algorithm.params_class=com.baidu.hugegraph.computer.core.worker.MockComputationParams - diff --git a/computer-test/src/main/resources/computer-master.properties b/computer-test/src/main/resources/computer.properties similarity index 95% rename from computer-test/src/main/resources/computer-master.properties rename to computer-test/src/main/resources/computer.properties index 379682633..dd52c3b6c 100644 --- a/computer-test/src/main/resources/computer-master.properties +++ b/computer-test/src/main/resources/computer.properties @@ -1,5 +1,5 @@ rpc.server_host=localhost -rpc.server_port=8090 +rpc.server_port=8190 job.id=local_002 job.workers_count=1 diff --git a/computer-core/src/main/resources/log4j2.xml b/computer-test/src/main/resources/log4j2.xml similarity index 100% rename from computer-core/src/main/resources/log4j2.xml rename to computer-test/src/main/resources/log4j2.xml diff --git a/pom.xml b/pom.xml index 7707774e3..ed586f0e3 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,7 @@ hugegraph-computer https://github.com/apache/hugegraph-computer - hugegraph-computer is a fast-speed, highly-scalable, fault-tolerance graph processing system developed by baidu. + hugegraph-computer is a fast-speed, highly-scalable, fault-tolerance graph processing system developed by apache. 2020 @@ -140,6 +140,10 @@ commons-collections commons-collections + + slf4j-log4j12 + org.slf4j +