Merge branch 'master' into master

det101 · Dec 28, 2019 · 5be42ad · 5be42ad
2 parents fa2d653 + 53c810a
commit 5be42ad
Show file tree

Hide file tree

Showing 37 changed files with 616 additions and 276 deletions.
diff --git a/assembly/pom.xml b/assembly/pom.xml
@@ -22,7 +22,7 @@
     <parent>
         <artifactId>dss</artifactId>
         <groupId>com.webank.wedatasphere.dss</groupId>
-        <version>0.6.0</version>
+        <version>0.7.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/bin/checkEnv.sh b/bin/checkEnv.sh
@@ -39,8 +39,7 @@ need_cmd mysql
 need_cmd unzip
 need_cmd expect
 need_cmd telnet
-need_cmd sed
 need_cmd tar
-need_cmd source
-need_cmd hostname
+need_cmd sed
+need_cmd dos2unix
 echo "<-----end to check linux cmd:yum java mysql unzip expect telnet sed tar------>"
diff --git a/bin/checkMicro.sh b/bin/checkMicro.sh
@@ -0,0 +1,50 @@
+#
+# Copyright 2019 WeBank
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#!/bin/sh
+source ~/.bash_profile
+
+shellDir=`dirname $0`
+workDir=`cd ${shellDir}/..;pwd`
+
+##load config
+source ${workDir}/conf/config.sh
+source ${workDir}/conf/db.sh
+
+MICRO_SERVICE_NAME=$1
+MICRO_SERVICE_IP=$2
+MICRO_SERVICE_PORT=$3
+echo "<--------------------------------------------------------------------------->"
+echo "Start to Check if your microservice:$MICRO_SERVICE_NAME is normal via telnet"
+echo ""
+if [ ! -d $DSS_INSTALL_HOME/$MICRO_SERVICE_NAME ];then
+  echo "$MICRO_SERVICE_NAME is not installed,the check steps will be skipped"
+  exit 0
+fi
+
+result=`echo -e "\n" | telnet $MICRO_SERVICE_IP $MICRO_SERVICE_PORT 2>/dev/null | grep Connected | wc -l`
+if [ $result -eq 1 ]; then
+      echo "$MICRO_SERVICE_NAME is ok."
+else
+      echo "<--------------------------------------------------------------------------->"
+      echo "ERROR your $MICRO_SERVICE_NAME microservice is not start successful !!! ERROR logs as follows :"
+      echo "PLEAESE CHECK  DETAIL LOG,LOCATION:$DSS_INSTALL_HOME/$MICRO_SERVICE_NAME/logs/linkis.out"
+      echo '<------------------------------------------------------------->'
+      tail -n 50 $DSS_INSTALL_HOME/$MICRO_SERVICE_NAME/logs/*.out
+      echo '<-------------------------------------------------------------->'
+      echo "PLEAESE CHECK DETAIL LOG,LOCATION:$DSS_INSTALL_HOME/$MICRO_SERVICE_NAME/logs/linkis.out"
+      exit 1
+fi
+
diff --git a/bin/install.sh b/bin/install.sh
@@ -37,7 +37,6 @@ elif [[ "$OSTYPE" == "win32" ]]; then
     echo "dss not support Windows operating system"
     exit 1
 elif [[ "$OSTYPE" == "freebsd"* ]]; then
-
     txt=""
 else
     echo "Operating system unknown, please tell us(submit issue) for better service"
@@ -46,14 +45,13 @@ fi
 
 function isSuccess(){
 if [ $? -ne 0 ]; then
-    echo "Failed to " + $1
+    echo "ERROR to " + $1
     exit 1
 else
-    echo "Succeed to" + $1
+    echo "SUCESS to" + $1
 fi
 }
 
-
 #check env
 sh ${workDir}/bin/checkEnv.sh
 isSuccess "check env"
@@ -74,63 +72,46 @@ else
 fi
 }
 
-function checkPythonAndJava(){
-	python --version
-	isSuccess "execute python --version"
-	java -version
-	isSuccess "execute java --version"
-}
-
-function checkHadoopAndHive(){
-	hdfs version
-	isSuccess "execute hdfs version"
-	hive --help
-	#isSuccess "execute hive -h"
-}
-
-function checkSpark(){
-	spark-submit --version
-	isSuccess "execute spark-submit --version"
-}
-
-##install env:expect,
-sudo yum install -y expect
-isSuccess "install expect"
-
-##install env:telnet,
-sudo yum install -y telnet
-isSuccess "install telnet"
-
 ##load config
 echo "step1:load config"
 source ${workDir}/conf/config.sh
 source ${workDir}/conf/db.sh
 isSuccess "load config"
 
-local_host="`hostname --fqdn`"
+local_host="`hostname -i`"
 
 ##env check
 echo "Please enter the mode selection such as: 1"
-echo " 1: Simple"
-echo " 2: Standard"
+echo " 1: lite"
+echo " 2: sample"
+echo " 3: Standard"
 echo ""
 
 INSTALL_MODE=1
 
 read -p "Please input the choice:"  idx
 if [[ '1' = "$idx" ]];then
   INSTALL_MODE=1
-  echo "You chose Simple installation mode"
+  echo "You chose lite installation mode"
   #check for Java
   checkJava
   #check for mysql
   SERVER_NAME=MYSQL
   EXTERNAL_SERVER_IP=$MYSQL_HOST
   EXTERNAL_SERVER_PORT=$MYSQL_PORT
   checkExternalServer
-
 elif [[ '2' = "$idx" ]];then
   INSTALL_MODE=2
+  echo "You chose sample installation mode"
+  #check for Java
+  checkJava
+  #check for mysql
+  SERVER_NAME=MYSQL
+  EXTERNAL_SERVER_IP=$MYSQL_HOST
+  EXTERNAL_SERVER_PORT=$MYSQL_PORT
+
+elif [[ '3' = "$idx" ]];then
+  INSTALL_MODE=3
   echo "You chose Standard installation mode"
   #check for Java
   checkJava
@@ -155,8 +136,8 @@ else
   exit 1
 fi
 
-##env check
-echo "Do you want to clear Dss table information in the database?"
+##init db
+echo "Do you want to clear DSS table information in the database?"
 echo " 1: Do not execute table-building statements"
 echo " 2: Dangerous! Clear all data and rebuild the tables."
 echo ""
@@ -189,18 +170,25 @@ if [[ '2' = "$MYSQL_INSTALL_MODE" ]];then
     #echo $GATEWAY_INSTALL_IP_2
     sed -i "s/GATEWAY_INSTALL_IP_2/$GATEWAY_INSTALL_IP_2/g" ${workDir}/db/dss_dml.sql
     sed -i "s/GATEWAY_PORT/$GATEWAY_PORT/g" ${workDir}/db/dss_dml.sql
-    if [ $VISUALIS_NGINX_IP == "127.0.0.1" ]||[ $VISUALIS_NGINX_IP == "0.0.0.0" ];then
+    mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source ${workDir}/db/dss_dml.sql"
+	isSuccess "source dss_dml.sql"
+
+    if [ '2' = "$INSTALL_MODE" ]||[ '3' = "$INSTALL_MODE" ];then
+	echo "visualis support "
+	    if [ $VISUALIS_NGINX_IP == "127.0.0.1" ]||[ $VISUALIS_NGINX_IP == "0.0.0.0" ];then
         echo "VISUALIS_NGINX_IP is equals $VISUALIS_NGINX_IP ,we will change it to ip address"
         VISUALIS_NGINX_IP_2=$LOCAL_IP
-    else
+        else
         VISUALIS_NGINX_IP_2=$VISUALIS_NGINX_IP
+        fi
+        #echo $VISUALIS_NGINX_IP_2
+        sed -i "s/VISUALIS_NGINX_IP_2/$VISUALIS_NGINX_IP_2/g" ${workDir}/db/visualis.sql
+        sed -i "s/VISUALIS_NGINX_PORT/$VISUALIS_NGINX_PORT/g" ${workDir}/db/visualis.sql
+	    mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source ${workDir}/db/visualis.sql"
+	    isSuccess "source visualis.sql"
     fi
-    #echo $VISUALIS_NGINX_IP_2
-    sed -i "s/VISUALIS_NGINX_IP_2/$VISUALIS_NGINX_IP_2/g" ${workDir}/db/dss_dml.sql
-    sed -i "s/VISUALIS_NGINX_PORT/$VISUALIS_NGINX_PORT/g" ${workDir}/db/dss_dml.sql
-	mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source ${workDir}/db/dss_dml.sql"
-	isSuccess "source dss_dml.sql"
-	if [[ '2' = "$INSTALL_MODE" ]];then
+
+	if [[ '3' = "$INSTALL_MODE" ]];then
 	   echo "start to update azkaban and qualitis table info "
        #azkaban
        if [ $AZKABAN_ADRESS_IP == "127.0.0.1" ];then
@@ -229,7 +217,7 @@ if [[ '2' = "$MYSQL_INSTALL_MODE" ]];then
     fi
 fi
 
-##env check
+## davinci db init
 echo "Do you want to clear davinci table information in the database ? If you have not installed davinci environment,you must input '2',if you have davinci installed,choice 1."
 echo " 1: Do not execute table-building statements"
 echo "WARN:"
@@ -252,7 +240,6 @@ else
   exit 1
 fi
 
-
 ###linkis Eurkea info
 SERVER_IP=$EUREKA_INSTALL_IP
 SERVER_PORT=$EUREKA_PORT
@@ -362,6 +349,8 @@ ssh  -p $SSH_PORT $SERVER_IP "sed -i  \"s#wds.dss.appjoint.scheduler.project.sto
 isSuccess "subsitution linkis.properties of $SERVERNAME"
 echo "<----------------$SERVERNAME:end------------------->"
 echo ""
+
+if [ '2' = "$INSTALL_MODE" ]||[ '3' = "$INSTALL_MODE" ];then
 ##Flow execution Install
 PACKAGE_DIR=dss/dss-flow-execution-entrance
 SERVERNAME=dss-flow-execution-entrance
@@ -437,7 +426,7 @@ APPJOINTNAME_CONF_PATH_PATENT=$SERVER_HOME/$APPJOINTPARENT/$APPJOINTNAME/appjoin
 ssh  -p $SSH_PORT  $SERVER_IP "sed -i  \"s#job.datachecker.jdo.option.url.*#job.datachecker.jdo.option.url=$HIVE_META_URL#g\" $APPJOINTNAME_CONF_PATH_PATENT"
 ssh  -p $SSH_PORT  $SERVER_IP "sed -i  \"s#job.datachecker.jdo.option.username.*#job.datachecker.jdo.option.username=$HIVE_META_USER#g\" $APPJOINTNAME_CONF_PATH_PATENT"
 ssh  -p $SSH_PORT  $SERVER_IP "sed -i  \"s#job.datachecker.jdo.option.password.*#job.datachecker.jdo.option.password=$HIVE_META_PASSWORD#g\" $APPJOINTNAME_CONF_PATH_PATENT"
-isSuccess "subsitution conf of $SERVERNAME"
+isSuccess "subsitution conf of datachecker"
 echo "<----------------datachecker appjoint install end------------------->"
 echo ""
 echo "<----------------eventchecker appjoint install start------------------->"
@@ -450,7 +439,7 @@ APPJOINTNAME_CONF_PATH_PATENT=$SERVER_HOME/$APPJOINTPARENT/$APPJOINTNAME/appjoin
 ssh  -p $SSH_PORT  $SERVER_IP "sed -i  \"s#msg.eventchecker.jdo.option.url.*#msg.eventchecker.jdo.option.url=jdbc:mysql://${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB}?characterEncoding=UTF-8#g\" $APPJOINTNAME_CONF_PATH_PATENT"
 ssh  -p $SSH_PORT  $SERVER_IP "sed -i  \"s#msg.eventchecker.jdo.option.username.*#msg.eventchecker.jdo.option.username=$MYSQL_USER#g\" $APPJOINTNAME_CONF_PATH_PATENT"
 ssh  -p $SSH_PORT  $SERVER_IP "sed -i  \"s#msg.eventchecker.jdo.option.password.*#msg.eventchecker.jdo.option.password=$MYSQL_PASSWORD#g\" $APPJOINTNAME_CONF_PATH_PATENT"
-isSuccess "subsitution conf of $SERVERNAME"
+isSuccess "subsitution conf of eventchecker"
 echo "<----------------$APPJOINTNAME:end------------------->"
 echo ""
 echo "<----------------visualis  appjoint install start------------------->"
@@ -459,8 +448,10 @@ APPJOINTNAME=visualis
 #visualis  appjoint install
 installAppjoints
 echo "<----------------$APPJOINTNAME:end------------------->"
-##sample version does not install qualitis APPJoint and scheduis APPJoint
-if [[ '2' = "$INSTALL_MODE" ]];then
+fi
+
+##lite and sample version does not install qualitis APPJoint and scheduis APPJoint
+if [[ '3' = "$INSTALL_MODE" ]];then
 echo ""
 echo "<----------------qualitis  appjoint install start------------------->"
 APPJOINTPARENT=dss-appjoints
@@ -469,7 +460,7 @@ APPJOINTNAME=qualitis
 installAppjoints
 APPJOINTNAME_CONF_PATH_PATENT=$SERVER_HOME/$APPJOINTPARENT/$APPJOINTNAME/appjoint.properties
 ssh  -p $SSH_PORT  $SERVER_IP "sed -i  \"s#baseUrl=http://127.0.0.1:8090#baseUrl=http://$QUALITIS_ADRESS_IP:$QUALITIS_ADRESS_PORT#g\" $APPJOINTNAME_CONF_PATH_PATENT"
-isSuccess "subsitution conf of $SERVERNAME"
+isSuccess "subsitution conf of qualitis"
 echo "<----------------$APPJOINTNAME:end------------------->"
 echo ""
 echo "<----------------schedulis  appjoint install start------------------->"
@@ -479,4 +470,4 @@ APPJOINTNAME=schedulis
 installAppjoints
 isSuccess "subsitution conf of schedulis"
 echo "<----------------$APPJOINTNAME:end------------------->"
-fi
+fi
diff --git a/bin/start-all.sh b/bin/start-all.sh
@@ -31,6 +31,31 @@ workDir=`cd "$workDir"; pwd`
 CONF_DIR="${workDir}"/../conf
 CONF_FILE=${CONF_DIR}/config.sh
 
+function isLocal(){
+    if [ "$1" == "127.0.0.1" ];then
+        return 0
+    elif [ $1 == "localhost" ]; then
+        return 0
+    elif [ $1 == $local_host ]; then
+        return 0
+    elif [ $1 == $ipaddr ]; then
+        return 0
+    fi
+        return 1
+}
+
+function executeCMD(){
+   isLocal $1
+   flag=$?
+   echo "Is local "$flag
+   if [ $flag == "0" ];then
+      eval $2
+   else
+      ssh -p $SSH_PORT $1 $2
+   fi
+
+}
+
 function isSuccess(){
 if [ $? -ne 0 ]; then
     echo "ERROR:  " + $1
@@ -60,7 +85,14 @@ function startApp(){
 echo "<-------------------------------->"
 echo "Begin to start $SERVER_NAME"
 SERVER_BIN=${DSS_INSTALL_HOME}/${SERVER_NAME}/bin
-SERVER_START_CMD="source ~/.bash_profile;cd ${SERVER_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1;sh start-${SERVER_NAME}.sh > /dev/null 2>&1 &"
+SERVER_START_CMD="source /etc/profile;source ~/.bash_profile;cd ${SERVER_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1;sh start-${SERVER_NAME}.sh > /dev/null 2>&1 &"
+
+if [ ! -d $SERVER_BIN ];then
+  echo "<-------------------------------->"
+  echo "$SERVER_NAME is not installed,the start steps will be skipped"
+  echo "<-------------------------------->"
+  return
+fi
 
 if [ -n "${SERVER_IP}"  ];then
     ssh ${SERVER_IP} "${SERVER_START_CMD}"
@@ -76,16 +108,6 @@ sleep 15 #for Eureka register
 SERVER_NAME=dss-server
 SERVER_IP=$DSS_SERVER_INSTALL_IP
 startApp
-#MICRO_SERVICE_NAME=dss-server
-#MICRO_SERVICE_IP=$DSS_SERVER_INSTALL_IP
-#MICRO_SERVICE_PORT=$DSS_SERVER_PORT
-#sh $workDir/check.sh $MICRO_SERVICE_NAME $MICRO_SERVICE_IP $MICRO_SERVICE_PORT
-#state=`echo -e "\n" | telnet $MICRO_SERVICE_IP $MICRO_SERVICE_PORT 2>/dev/null | grep Connected | wc -l`
-#if [ $state -eq 0 ]; then
-#   echo ""
-#   echo "ERROR " $MICRO_SERVICE_NAME "is a critical service and must be guaranteed to be started !!!"  
-#   exit 1
-#fi
 
 #dss-flow-execution-entrance
 SERVER_NAME=dss-flow-execution-entrance
@@ -101,15 +123,14 @@ SERVER_NAME=visualis-server
 SERVER_IP=$VISUALIS_SERVER_INSTALL_IP
 startApp
 
-
+echo ""
 echo "Start to check all dss microservice"
-
+echo ""
 #check dss-server
 MICRO_SERVICE_NAME=dss-server
 MICRO_SERVICE_IP=$DSS_SERVER_INSTALL_IP
 MICRO_SERVICE_PORT=$DSS_SERVER_PORT
 sh $workDir/checkMicro.sh $MICRO_SERVICE_NAME $MICRO_SERVICE_IP $MICRO_SERVICE_PORT
-state=`echo -e "\n" | telnet $MICRO_SERVICE_IP $MICRO_SERVICE_PORT 2>/dev/null | grep Connected | wc -l`
 isSuccess "$MICRO_SERVICE_NAME start"
 
 
@@ -118,15 +139,13 @@ MICRO_SERVICE_NAME=dss-flow-execution-entrance
 MICRO_SERVICE_IP=$FLOW_EXECUTION_INSTALL_IP
 MICRO_SERVICE_PORT=$FLOW_EXECUTION_PORT
 sh $workDir/checkMicro.sh $MICRO_SERVICE_NAME $MICRO_SERVICE_IP $MICRO_SERVICE_PORT
-state=`echo -e "\n" | telnet $MICRO_SERVICE_IP $MICRO_SERVICE_PORT 2>/dev/null | grep Connected | wc -l`
 isSuccess "$MICRO_SERVICE_NAME start"
 
 #check linkis-appjoint-entrance
 MICRO_SERVICE_NAME=linkis-appjoint-entrance
 MICRO_SERVICE_IP=$APPJOINT_ENTRANCE_INSTALL_IP
 MICRO_SERVICE_PORT=$APPJOINT_ENTRANCE_PORT
 sh $workDir/checkMicro.sh $MICRO_SERVICE_NAME $MICRO_SERVICE_IP $MICRO_SERVICE_PORT
-state=`echo -e "\n" | telnet $MICRO_SERVICE_IP $MICRO_SERVICE_PORT 2>/dev/null | grep Connected | wc -l`
 isSuccess "$MICRO_SERVICE_NAME start"
 
 
@@ -136,6 +155,5 @@ MICRO_SERVICE_NAME=visualis-server
 MICRO_SERVICE_IP=$VISUALIS_SERVER_INSTALL_IP
 MICRO_SERVICE_PORT=$VISUALIS_SERVER_PORT
 sh $workDir/checkMicro.sh $MICRO_SERVICE_NAME $MICRO_SERVICE_IP $MICRO_SERVICE_PORT
-state=`echo -e "\n" | telnet $MICRO_SERVICE_IP $MICRO_SERVICE_PORT 2>/dev/null | grep Connected | wc -l`
 isSuccess "$MICRO_SERVICE_NAME start"