[improve][build] Improve package & cli & build image (#200)

* improve package * Refactor computer: abstract computer-api from computer-core (#196) * address comment * improve pom and options * fix package * improve build image * improve start-computer.sh * improve log file * improve start-computer.sh * improve build image * trigger maven cache invalidation * Update ci.yml Co-authored-by: imbajin <jin@apache.org>
apache · Nov 25, 2022 · adf6801 · adf6801
1 parent 5320190
commit adf6801
Show file tree

Hide file tree

Showing 26 changed files with 185 additions and 130 deletions.
diff --git a/.github/workflows/publish_images.yml b/.github/workflows/publish_images.yml
@@ -19,13 +19,11 @@ jobs:
           map: |
             {
               "push": {
-                "FRAMEWORK_IMAGE_URL": "hugegraph/hugegraph-computer-framework:latest",
-                "ALGORITHM_IMAGE_URL": "hugegraph/hugegraph-builtin-algorithms:latest",
+                "COMPUTER_IMAGE_URL": "hugegraph/hugegraph-computer:latest",
                 "OPERATOR_IMAGE_URL": "hugegraph/hugegraph-computer-operator:latest"
               },
               "release": {
-                "FRAMEWORK_IMAGE_URL": "hugegraph/hugegraph-computer-framework:${{ github.ref_name }}",
-                "ALGORITHM_IMAGE_URL": "hugegraph/hugegraph-builtin-algorithms:${{ github.ref_name }}",
+                "COMPUTER_IMAGE_URL": "hugegraph/hugegraph-computer:${{ github.ref_name }}",
                 "OPERATOR_IMAGE_URL": "hugegraph/hugegraph-computer-operator:${{ github.ref_name }}"
               }
             }
@@ -58,13 +56,12 @@ jobs:
       - name: Build images
         env:
           TRAVIS_DIR: computer-dist/src/assembly/travis
-        run: $TRAVIS_DIR/build-images.sh ${{ env.FRAMEWORK_IMAGE_URL }} ${{ env.ALGORITHM_IMAGE_URL }} ${{ env.OPERATOR_IMAGE_URL }}
+        run: $TRAVIS_DIR/build-images.sh ${{ env.COMPUTER_IMAGE_URL }} ${{ env.OPERATOR_IMAGE_URL }}
 
       - name: Docker images
         run: docker images
 
       - name: Publish
         run: |
-          docker push ${{ env.FRAMEWORK_IMAGE_URL }}
-          docker push ${{ env.ALGORITHM_IMAGE_URL }}
+          docker push ${{ env.COMPUTER_IMAGE_URL }}
           docker push ${{ env.OPERATOR_IMAGE_URL }}
diff --git a/computer-algorithm/Dockerfile b/computer-algorithm/Dockerfile
diff --git a/computer-algorithm/pom.xml b/computer-algorithm/pom.xml
@@ -32,6 +32,7 @@
             <groupId>com.baidu.hugegraph</groupId>
             <artifactId>computer-api</artifactId>
             <version>${version}</version>
+            <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.mvel</groupId>

diff --git a/computer-core/pom.xml b/computer-core/pom.xml
@@ -67,12 +67,6 @@
         <dependency>
             <groupId>org.apache.hugegraph</groupId>
             <artifactId>hugegraph-loader</artifactId>
-            <exclusions>
-                <exclusion>
-                    <artifactId>spark-core_2.12</artifactId>
-                    <groupId>org.apache.spark</groupId>
-                </exclusion>
-            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.apache.hugegraph</groupId>

diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/util/ComputerContextUtil.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/util/ComputerContextUtil.java
@@ -45,8 +45,7 @@ public static Config initContext(Map<String, String> params) {
             algorithmParams = (AlgorithmParams) Class.forName(
                               algorithmParamsName).newInstance();
         } catch (Exception e) {
-            throw new ComputerException("Can't create algorithmParams, " +
-                                        "algorithmParamsName = {}",
+            throw new ComputerException("Can't create algorithmParams, algorithmParamsName = %s",
                                         algorithmParamsName);
         }
         algorithmParams.setAlgorithmParameters(params);

diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/util/ShutdownHook.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/util/ShutdownHook.java
@@ -21,7 +21,6 @@
 import java.io.Closeable;
 
 import org.apache.hugegraph.util.Log;
-import org.apache.logging.log4j.LogManager;
 import org.slf4j.Logger;
 
 public class ShutdownHook {
@@ -41,8 +40,6 @@ public boolean hook(Closeable hook) {
             } catch (Throwable e) {
                 LOG.warn("Failed to execute shutdown hook: {}",
                           e.getMessage(), e);
-            } finally {
-                LogManager.shutdown();
             }
         });
         Runtime.getRuntime().addShutdownHook(this.threadShutdownHook);

diff --git a/computer-core/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerService.java b/computer-core/src/main/java/com/baidu/hugegraph/computer/core/worker/WorkerService.java
@@ -25,6 +25,8 @@
 import java.util.List;
 import java.util.Map;
 
+import org.apache.hugegraph.util.E;
+import org.apache.hugegraph.util.Log;
 import org.slf4j.Logger;
 
 import com.baidu.hugegraph.computer.core.aggregator.Aggregator;
@@ -56,8 +58,6 @@
 import com.baidu.hugegraph.computer.core.sort.sorting.SortManager;
 import com.baidu.hugegraph.computer.core.store.FileManager;
 import com.baidu.hugegraph.computer.core.util.ShutdownHook;
-import org.apache.hugegraph.util.E;
-import org.apache.hugegraph.util.Log;
 
 public class WorkerService implements Closeable {
 

diff --git a/computer-dist/Dockerfile b/computer-dist/Dockerfile
@@ -15,9 +15,12 @@
 # limitations under the License.
 #
 FROM openjdk:11-jre
-LABEL maintainer="HugeGraph Docker Maintainers <hugegraph@googlegroups.com>"
+LABEL maintainer="HugeGraph Docker Maintainers <dev@hugegraph.apache.org>"
 # use ParallelGC which is more friendly to olap system
 ENV JAVA_OPTS="-XX:+UnlockExperimentalVMOptions -XX:+UseParallelGC -XX:+UseContainerSupport -XX:MaxRAMPercentage=50 -XshowSettings:vm"
-COPY . /etc/local/hugegraph-computer
+COPY ./bin /etc/local/hugegraph-computer/bin
+COPY ./lib /etc/local/hugegraph-computer/lib
+COPY ./algorithm /etc/local/hugegraph-computer/algorithm
+COPY ./conf /etc/local/hugegraph-computer/conf
 WORKDIR /etc/local/hugegraph-computer
 RUN apt-get update && apt-get -y install gettext-base && apt-get -y install wget
diff --git a/computer-dist/pom.xml b/computer-dist/pom.xml
@@ -116,6 +116,11 @@
                         <configuration>
                             <tasks>
                                 <echo file="${top.level.dir}/dist.sh">
+                                    mkdir -p ${top.level.dir}/${final.name}/algorithm/
+                                    cp ${top.level.dir}/computer-algorithm/target/computer-algorithm-${project.version}.jar ${top.level.dir}/${final.name}/algorithm/builtin-algorithm.jar
+                                    mkdir -p ${top.level.dir}/${final.name}/k8s-operator/
+                                    cp ${top.level.dir}/computer-k8s-operator/target/hugegraph-computer-operator-${project.version}.jar ${top.level.dir}/${final.name}/k8s-operator/hugegraph-computer-operator.jar
+
                                     tar -zcvf \
                                     ${top.level.dir}/${final.name}.tar.gz \
                                     ${final.name} || exit 1

diff --git a/computer-dist/src/assembly/static/bin/start-computer.sh b/computer-dist/src/assembly/static/bin/start-computer.sh
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
 #
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
@@ -14,12 +15,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-#!/usr/bin/env bash
-set -e
+
 BIN_DIR=$(cd "$(dirname "$0")" && pwd -P)
 BASE_DIR=$(cd "${BIN_DIR}/.." && pwd -P)
 LIB_DIR=${BASE_DIR}/lib
 CONF_DIR="${BASE_DIR}/conf"
+DEFAULT_ALGORITHM_DIR="${BASE_DIR}/algorithm"
 
 COMPUTER_CONF_PATH="${COMPUTER_CONF_PATH}"
 LOG4J_CONF_PATH="${LOG4J_CONF_PATH}"
@@ -34,10 +35,12 @@ ROLE_WORKER="worker"
 
 usage() {
     echo "Usage:"
-    echo "    start-computer.sh <-c|--conf conf_file_path>"
-    echo "        <-a|--algorithm algorithm_jar_path>"
+    echo "    start-computer.sh"
+    echo "        [-c|--conf conf_file_path>"
+    echo "        [-a|--algorithm algorithm_jar_path>"
     echo "        [-l|--log4 log4_conf_path]"
     echo "        <-d|--drive drive_type(local|k8s|yarn)>"
+    echo "        <-r|--role role(master|worker)>"
 }
 
 if [ $# -lt 4 ]; then
@@ -124,23 +127,13 @@ parse_opts() {
 
 parse_opts $*
 
-echo "************************************"
-echo "COMPUTER_CONF_PATH=${COMPUTER_CONF_PATH}"
-echo "LOG4J_CONF_PATH=${LOG4J_CONF_PATH}"
-echo "JAR_FILE_PATH=${JAR_FILE_PATH}"
-echo "DRIVE=${DRIVE}"
-echo "************************************"
-
-if [ "${JAR_FILE_PATH}" = "" ]; then
-    echo "graph algorithm jar file missed";
-    usage;
-    exit 1;
-fi
-
 if [ "${COMPUTER_CONF_PATH}" = "" ]; then
-    echo "conf file missed";
-    usage;
-    exit 1;
+    if [ "$DRIVE" = "$K8S_DRIVE" ]; then
+      echo "conf file missed";
+      usage;
+      exit 1;
+    fi
+    COMPUTER_CONF_PATH=${CONF_DIR}/computer.properties
 fi
 
 if [ "${DRIVE}" = "" ]; then
@@ -155,21 +148,37 @@ if [ "${ROLE}" = "" ]; then
     exit 1;
 fi
 
+if [ "${LOG4J_CONF_PATH}" = "" ];then
+    LOG4J_CONF_PATH=${CONF_DIR}/log4j2.xml
+fi
+
+echo "************************************"
+echo "COMPUTER_CONF_PATH=${COMPUTER_CONF_PATH}"
+echo "LOG4J_CONF_PATH=${LOG4J_CONF_PATH}"
+echo "ALGORITHM_JAR_FILE_PATH=${JAR_FILE_PATH}"
+echo "DRIVE=${DRIVE}"
+echo "ROLE=${ROLE}"
+echo "************************************"
+
 CP=$(find "${LIB_DIR}" -name "*.jar" | tr "\n" ":")
 
-CP="$JAR_FILE_PATH":${CP}
+CP=${CP}:"${DEFAULT_ALGORITHM_DIR}/*"
+
+if [ "${JAR_FILE_PATH}" != "" ]; then
+    CP=${CP}:${JAR_FILE_PATH}
+fi
 
 # Download remote job JAR file.
-if [[ "${JOB_JAR_URI}" == http://* || "${JOB_JAR_URI}" == https://* ]]; then
+if [[ "${REMOTE_JAR_URI}" == http://* || "${REMOTE_JAR_URI}" == https://* ]]; then
     mkdir -p "${BASE_DIR}/job"
-    echo "Downloading job JAR ${JOB_JAR_URI} to ${BASE_DIR}/job/"
-    wget -nv -P "${BASE_DIR}/job/" "${JOB_JAR_URI}"
+    echo "Downloading job JAR ${REMOTE_JAR_URI} to ${BASE_DIR}/job/"
+    wget -nv -P "${BASE_DIR}/job/" "${REMOTE_JAR_URI}"
     JOB_JAR=$(find "${BASE_DIR}/job" -name "*.jar" | tr "\n" ":")
     if [[ "$JOB_JAR" != "" ]]; then
         CP="${JOB_JAR}"$CP
     fi
-elif [[ "${JOB_JAR_URI}" != "" ]]; then
-    echo "Unsupported protocol for ${JOB_JAR_URI}"
+elif [[ "${REMOTE_JAR_URI}" != "" ]]; then
+    echo "Unsupported protocol for ${REMOTE_JAR_URI}"
     exit 1
 fi
 
@@ -195,26 +204,30 @@ if [ ! -a "${CONF_DIR}" ];then
     mkdir -p "${CONF_DIR}"
 fi
 
-COPY_CONF_DIR="${CONF_DIR}/copy"
-if [ ! -a "${COPY_CONF_DIR}" ]; then
-    mkdir -p "${COPY_CONF_DIR}"
-    chmod 777 "${COPY_CONF_DIR}"
+if [ "$DRIVE" = "$K8S_DRIVE" ]; then
+    COPY_CONF_DIR="${CONF_DIR}/copy"
+    if [ ! -a "${COPY_CONF_DIR}" ]; then
+        mkdir -p "${COPY_CONF_DIR}"
+        chmod 777 "${COPY_CONF_DIR}"
+    fi
+
+    NEW_COMPUTER_CONF_PATH="${COPY_CONF_DIR}/$(basename "${COMPUTER_CONF_PATH}")"
+    envsubst '${POD_IP},${HOSTNAME},${POD_NAME},${POD_NAMESPACE}' < "${COMPUTER_CONF_PATH}" > "${NEW_COMPUTER_CONF_PATH}"
+    chmod 777 "${NEW_COMPUTER_CONF_PATH}"
+    COMPUTER_CONF_PATH=${NEW_COMPUTER_CONF_PATH}
 fi
 
-NEW_COMPUTER_CONF_PATH="${COPY_CONF_DIR}/$(basename "${COMPUTER_CONF_PATH}")"
-envsubst '${POD_IP},${HOSTNAME},${POD_NAME},${POD_NAMESPACE}' < "${COMPUTER_CONF_PATH}" > "${NEW_COMPUTER_CONF_PATH}"
-chmod 777 "${NEW_COMPUTER_CONF_PATH}"
+LOG4j_CONF=-Dlog4j.configurationFile="${LOG4J_CONF_PATH}"
 
-if [ "${LOG4J_CONF_PATH}" != "" ];then
-    LOG4j_CONF=-Dlog4j.configurationFile="${LOG4J_CONF_PATH}"
+if [ "${ROLE}" = "${ROLE_MASTER}" ]; then
+    LOG_NAME=-Dlog.name=hugegraph-computer-master
+fi
+
+if [ "${ROLE}" = "${ROLE_WORKER}" ]; then
+    LOG_NAME=-Dlog.name=hugegraph-computer-worker
 fi
 
 MAIN_CLASS=com.baidu.hugegraph.computer.dist.HugeGraphComputer
 
-if [ "${LOG4j_CONF}" != "" ]; then
-    exec ${JAVA} -Dname="hugegraph-computer" "${LOG4j_CONF}" ${JAVA_OPTS} ${JVM_OPTIONS} \
-        -cp "${CP}" ${MAIN_CLASS} "${NEW_COMPUTER_CONF_PATH}" ${ROLE} ${DRIVE}
-else
-    exec ${JAVA} -Dname="hugegraph-computer" ${JAVA_OPTS} ${JVM_OPTIONS} -cp "${CP}" \
-        ${MAIN_CLASS} "${NEW_COMPUTER_CONF_PATH}" ${ROLE} ${DRIVE}
-fi
+exec ${JAVA} -Dname="hugegraph-computer" "${LOG4j_CONF}" ${LOG_NAME} ${JAVA_OPTS} ${JVM_OPTIONS} \
+        -cp "${CP}" ${MAIN_CLASS} "${COMPUTER_CONF_PATH}" ${ROLE} ${DRIVE}
diff --git a/computer-dist/src/assembly/static/conf/computer.properties b/computer-dist/src/assembly/static/conf/computer.properties
@@ -0,0 +1,16 @@
+job.id=local_001
+job.workers_count=1
+
+rpc.server_host=localhost
+rpc.server_port=8190
+
+bsp.etcd_endpoints=http://127.0.0.1:2379
+
+hugegraph.url=http://127.0.0.1:8080
+hugegraph.name=hugegraph
+
+algorithm.params_class=com.baidu.hugegraph.computer.algorithm.centrality.pagerank.PageRankParams
+
+bsp.register_timeout=100000
+bsp.log_interval=30000
+bsp.max_super_step=2
diff --git a/computer-dist/src/assembly/static/conf/log4j2.xml b/computer-dist/src/assembly/static/conf/log4j2.xml
@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+<!-- Config will be auto loaded every 60s -->
+<configuration status="error" monitorInterval="60">
+    <properties>
+        <property name="LOG_PATH">logs</property>
+        <property name="FILE_NAME">${sys:log.name:-hugegraph-computer}</property>
+    </properties>
+
+    <appenders>
+        <!-- Container standard/console log -->
+        <Console name="console" target="SYSTEM_OUT">
+            <ThresholdFilter level="DEBUG" onMatch="ACCEPT" onMismatch="DENY"/>
+            <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n"/>
+        </Console>
+
+        <!-- Container file log -->
+        <RollingRandomAccessFile name="file" fileName="${LOG_PATH}/${FILE_NAME}.log"
+                                 filePattern="${LOG_PATH}/$${date:yyyy-MM}/${FILE_NAME}-%d{yyyy-MM-dd}-%i.log"
+                                 immediateFlush="false">
+            <ThresholdFilter level="TRACE" onMatch="ACCEPT" onMismatch="DENY"/>
+            <PatternLayout pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%p] %c{1.} - %m%n"/>
+            <!-- Trigger after exceeding 1day or 50MB -->
+            <Policies>
+                <SizeBasedTriggeringPolicy size="50MB"/>
+                <TimeBasedTriggeringPolicy interval="1" modulate="true" />
+            </Policies>
+            <!-- Keep 5 files per day & auto delete after over 1GB or 50 files -->
+            <DefaultRolloverStrategy max="5">
+                <Delete basePath="${LOG_PATH}" maxDepth="2">
+                    <IfFileName glob="*/*.log"/>
+                    <!-- Limit log amount & size -->
+                    <IfAny>
+                        <IfAccumulatedFileSize exceeds="1GB" />
+                        <IfAccumulatedFileCount exceeds="50" />
+                    </IfAny>
+                </Delete>
+            </DefaultRolloverStrategy>
+        </RollingRandomAccessFile>
+    </appenders>
+    <loggers>
+        <root level="INFO">
+            <appender-ref ref="console"/>
+            <appender-ref ref="file"/>
+        </root>
+        <Logger name="com.baidu.hugegraph" level="INFO" additivity="false">
+            <appender-ref ref="console"/>
+            <appender-ref ref="file"/>
+        </Logger>
+    </loggers>
+</configuration>