[improve][build] Improve package & cli & build image

computer-algorithm/Dockerfile

@@ -16,7 +16,7 @@
 #
 FROM hugegraph/hugegraph-computer-framework:latest
 
-LABEL maintainer="HugeGraph Docker Maintainers <hugegraph@googlegroups.com>"
+LABEL maintainer="HugeGraph Docker Maintainers <dev@hugegraph.apache.org>"
 
 ARG jarFilePath="/opt/jars/hugegraph-builtin-algorithms.jar"
 COPY target/computer-algorithm-*.jar ${jarFilePath}

computer-core/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerService.java

@@ -25,6 +25,8 @@
 import java.util.List;
 import java.util.Map;
 
+import org.apache.hugegraph.util.E;
+import org.apache.hugegraph.util.Log;
 import org.slf4j.Logger;
 
 import com.baidu.hugegraph.computer.core.aggregator.Aggregator;
@@ -56,8 +58,6 @@
 import com.baidu.hugegraph.computer.core.sort.sorting.SortManager;
 import com.baidu.hugegraph.computer.core.store.FileManager;
 import com.baidu.hugegraph.computer.core.util.ShutdownHook;
-import org.apache.hugegraph.util.E;
-import org.apache.hugegraph.util.Log;
 
 public class WorkerService implements Closeable {
 

computer-dist/Dockerfile

@@ -15,9 +15,10 @@
 # limitations under the License.
 #
 FROM openjdk:11-jre
-LABEL maintainer="HugeGraph Docker Maintainers <hugegraph@googlegroups.com>"
+LABEL maintainer="HugeGraph Docker Maintainers <dev@hugegraph.apache.org>"
 # use ParallelGC which is more friendly to olap system
 ENV JAVA_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseParallelGC -XX:+UseContainerSupport -XX:MaxRAMPercentage=50 -XshowSettings:vm"
-COPY . /etc/local/hugegraph-computer
+COPY ./bin /etc/local/hugegraph-computer/bin
+COPY ./lib /etc/local/hugegraph-computer/lib
 WORKDIR /etc/local/hugegraph-computer
 RUN apt-get update && apt-get -y install gettext-base && apt-get -y install wget

computer-dist/pom.xml

@@ -116,6 +116,11 @@
                         <configuration>
                             <tasks>
                                 <echo file="${top.level.dir}/dist.sh">
+                                    mkdir -p ${top.level.dir}/${final.name}/algorithm/
+                                    cp ${top.level.dir}/computer-algorithm/target/computer-algorithm-${project.version}.jar ${top.level.dir}/${final.name}/algorithm/
+                                    mkdir -p ${top.level.dir}/${final.name}/k8s-operator/
+                                    cp ${top.level.dir}/computer-k8s-operator/target/hugegraph-computer-operator-${project.version}.jar ${top.level.dir}/${final.name}/k8s-operator/
+
                                     tar -zcvf \
                                     ${top.level.dir}/${final.name}.tar.gz \
                                     ${final.name} || exit 1

computer-dist/src/assembly/static/bin/start-computer.sh

@@ -1,3 +1,4 @@
+!/usr/bin/env bash
 #
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
@@ -14,8 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-#!/usr/bin/env bash
-set -e
+
 BIN_DIR=$(cd "$(dirname "$0")" && pwd -P)
 BASE_DIR=$(cd "${BIN_DIR}/.." && pwd -P)
 LIB_DIR=${BASE_DIR}/lib
@@ -38,6 +38,7 @@ usage() {
     echo "        <-a|--algorithm algorithm_jar_path>"
     echo "        [-l|--log4 log4_conf_path]"
     echo "        <-d|--drive drive_type(local|k8s|yarn)>"
+    echo "        <-r|--role role(master|worker)>"
 }
 
 if [ $# -lt 4 ]; then
@@ -138,9 +139,12 @@ if [ "${JAR_FILE_PATH}" = "" ]; then
 fi
 
 if [ "${COMPUTER_CONF_PATH}" = "" ]; then
-    echo "conf file missed";
-    usage;
-    exit 1;
+    if [ "$DRIVE" = "$K8S_DRIVE" ]; then
+      echo "conf file missed";
+      usage;
+      exit 1;
+    fi
+    COMPUTER_CONF_PATH=${CONF_DIR}/computer.properties
 fi
 
 if [ "${DRIVE}" = "" ]; then
@@ -195,26 +199,25 @@ if [ ! -a "${CONF_DIR}" ];then
     mkdir -p "${CONF_DIR}"
 fi
 
-COPY_CONF_DIR="${CONF_DIR}/copy"
-if [ ! -a "${COPY_CONF_DIR}" ]; then
-    mkdir -p "${COPY_CONF_DIR}"
-    chmod 777 "${COPY_CONF_DIR}"
-fi
-
-NEW_COMPUTER_CONF_PATH="${COPY_CONF_DIR}/$(basename "${COMPUTER_CONF_PATH}")"
-envsubst '${POD_IP},${HOSTNAME},${POD_NAME},${POD_NAMESPACE}' < "${COMPUTER_CONF_PATH}" > "${NEW_COMPUTER_CONF_PATH}"
-chmod 777 "${NEW_COMPUTER_CONF_PATH}"
+if [ "$DRIVE" = "$K8S_DRIVE" ]; then
+    COPY_CONF_DIR="${CONF_DIR}/copy"
+    if [ ! -a "${COPY_CONF_DIR}" ]; then
+        mkdir -p "${COPY_CONF_DIR}"
+        chmod 777 "${COPY_CONF_DIR}"
+    fi
 
-if [ "${LOG4J_CONF_PATH}" != "" ];then
-    LOG4j_CONF=-Dlog4j.configurationFile="${LOG4J_CONF_PATH}"
+    NEW_COMPUTER_CONF_PATH="${COPY_CONF_DIR}/$(basename "${COMPUTER_CONF_PATH}")"
+    envsubst '${POD_IP},${HOSTNAME},${POD_NAME},${POD_NAMESPACE}' < "${COMPUTER_CONF_PATH}" > "${NEW_COMPUTER_CONF_PATH}"
+    chmod 777 "${NEW_COMPUTER_CONF_PATH}"
+    COMPUTER_CONF_PATH=${NEW_COMPUTER_CONF_PATH}
 fi
 
 MAIN_CLASS=com.baidu.hugegraph.computer.dist.HugeGraphComputer
 
 if [ "${LOG4j_CONF}" != "" ]; then
     exec ${JAVA} -Dname="hugegraph-computer" "${LOG4j_CONF}" ${JAVA_OPTS} ${JVM_OPTIONS} \
-        -cp "${CP}" ${MAIN_CLASS} "${NEW_COMPUTER_CONF_PATH}" ${ROLE} ${DRIVE}
+        -cp "${CP}" ${MAIN_CLASS} "${COMPUTER_CONF_PATH}" ${ROLE} ${DRIVE}
 else
     exec ${JAVA} -Dname="hugegraph-computer" ${JAVA_OPTS} ${JVM_OPTIONS} -cp "${CP}" \
-        ${MAIN_CLASS} "${NEW_COMPUTER_CONF_PATH}" ${ROLE} ${DRIVE}
+        ${MAIN_CLASS} "${COMPUTER_CONF_PATH}" ${ROLE} ${DRIVE}
 fi

computer-dist/src/assembly/static/conf/computer.properties

@@ -0,0 +1,14 @@
+rpc.server_host=localhost
+rpc.server_port=8190
+
+job.id=local_001
+job.workers_count=1
+
+bsp.register_timeout=100000
+bsp.log_interval=30000
+bsp.max_super_step=2
+
+hugegraph.url=http://127.0.0.1:8080
+hugegraph.name=hugegraph
+
+algorithm.params_class=com.baidu.hugegraph.computer.algorithm.centrality.pagerank.PageRankParams

computer-dist/src/assembly/travis/build-images.sh

@@ -36,7 +36,7 @@ PROJECT_VERSION=$(mvn -f "${PROJECT_POM_PATH}" -q -N \
 docker build -t $1 $CONTEXT_PATH -f $PROJECT_PATH/computer-dist/Dockerfile
 
 echo "FROM $1
-LABEL maintainer='HugeGraph Docker Maintainers <hugegraph@googlegroups.com>'
+LABEL maintainer='HugeGraph Docker Maintainers <dev@hugegraph.apache.org>'
 COPY target/computer-algorithm-*.jar $JAR_FILE_PATH
 ENV JAR_FILE_PATH=$JAR_FILE_PATH" | \
 docker build -t $2 -f - $PROJECT_PATH/computer-algorithm

computer-dist/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputer.java

@@ -25,6 +25,10 @@
 
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.hugegraph.config.OptionSpace;
+import org.apache.hugegraph.config.RpcOptions;
+import org.apache.hugegraph.util.E;
+import org.apache.hugegraph.util.Log;
 import org.slf4j.Logger;
 
 import com.baidu.hugegraph.computer.core.common.ComputerContext;
@@ -34,9 +38,6 @@
 import com.baidu.hugegraph.computer.core.network.message.MessageType;
 import com.baidu.hugegraph.computer.core.util.ComputerContextUtil;
 import com.baidu.hugegraph.computer.core.worker.WorkerService;
-import org.apache.hugegraph.config.OptionSpace;
-import org.apache.hugegraph.util.E;
-import org.apache.hugegraph.util.Log;
 
 public class HugeGraphComputer {
 
@@ -69,7 +70,7 @@ public static void main(String[] args) throws IOException,
         setUncaughtExceptionHandler();
         loadClass();
         registerOptions();
-        ComputerContext context = parseContext(args[0]);
+        ComputerContext context = parseContext(args[0], role);
         switch (role) {
             case ROLE_MASTER:
                 executeMasterService(context);
@@ -124,12 +125,17 @@ private static void executeMasterService(ComputerContext context) {
         }
     }
 
-    private static ComputerContext parseContext(String conf)
+    private static ComputerContext parseContext(String conf, String role)
                                                 throws IOException {
         Properties properties = new Properties();
         BufferedReader bufferedReader = new BufferedReader(
                                             new FileReader(conf));
         properties.load(bufferedReader);
+        properties.remove(RpcOptions.RPC_REMOTE_URL.name());
+        if (ROLE_WORKER.equals(role)) {
+            properties.remove(RpcOptions.RPC_SERVER_HOST.name());
+            properties.remove(RpcOptions.RPC_SERVER_PORT.name());
+        }
         ComputerContextUtil.initContext(properties);
         return ComputerContext.instance();
     }

computer-k8s-operator/Dockerfile

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 FROM openjdk:11-jre
-LABEL maintainer="HugeGraph Docker Maintainers <hugegraph@googlegroups.com>"
+LABEL maintainer="HugeGraph Docker Maintainers <dev@hugegraph.apache.org>"
 WORKDIR /opt/app
 COPY target/hugegraph-computer-operator-*.jar hugegraph-computer-operator.jar
 ENTRYPOINT ["java", "-jar", "hugegraph-computer-operator.jar"]

computer-k8s/src/main/resources/docker_push.sh

@@ -29,7 +29,7 @@ function build_image() {
     JAR_FILE=""
     IMG_URL=""
     FRAMEWORK_IMG_URL=""
-    MAINTAINER="HugeGraph Docker Maintainers <hugegraph@googlegroups.com>"
+    MAINTAINER="HugeGraph Docker Maintainers <dev@hugegraph.apache.org>"
 
     while getopts "r:u:p:s:j:i:f:" arg; do
       case ${arg} in

computer-test/conf/images/docker_push_test.sh

@@ -29,7 +29,7 @@ function build_image() {
     JAR_FILE=""
     IMG_URL=""
     FRAMEWORK_IMG_URL=""
-    MAINTAINER="HugeGraph Docker Maintainers <hugegraph@googlegroups.com>"
+    MAINTAINER="HugeGraph Docker Maintainers <dev@hugegraph.apache.org>"
 
     while getopts "r:u:p:s:j:i:f:" arg; do
       case ${arg} in

computer-test/src/main/java/com/baidu/hugegraph/computer/dist/HugeGraphComputerTest.java

@@ -41,9 +41,9 @@ public void testServiceWith1Worker() throws InterruptedException {
         CountDownLatch countDownLatch = new CountDownLatch(2);
         Throwable[] exceptions = new Throwable[2];
         String masterConfPath = HugeGraphComputerTest.class.getResource(
-                                "/computer-master.properties").getPath();
+                                "/computer.properties").getPath();
         String work1ConfPath = HugeGraphComputerTest.class.getResource(
-                               "/computer-worker1.properties").getPath();
+                               "/computer.properties").getPath();
         pool.submit(() -> {
             try {
                 Thread.sleep(2000L);
@@ -79,7 +79,7 @@ public void testServiceWith1Worker() throws InterruptedException {
     @Test
     public void testServiceWithError() {
         String work1ConfPath = HugeGraphComputerTest.class.getResource(
-                               "/computer-worker1.properties").getPath();
+                               "/computer.properties").getPath();
         Assert.assertThrows(IllegalArgumentException.class,
                             () -> {
                                 String[] args1 = {work1ConfPath, "worker111",

computer-test/src/main/resources/computer-master.properties → computer-test/src/main/resources/computer.properties

@@ -1,5 +1,5 @@
 rpc.server_host=localhost
-rpc.server_port=8090
+rpc.server_port=8190
 
 job.id=local_002
 job.workers_count=1