diff --git a/assembly/pom.xml b/assembly/pom.xml index 4ec1c8b647..29934071c0 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -105,6 +105,51 @@ jackson-core 2.9.6 + + net.databinder.dispatch + dispatch-core_2.11 + 0.11.2 + + + net.databinder.dispatch + dispatch-json4s-jackson_2.11 + 0.11.2 + + + org.apache.htrace + htrace-core + 3.1.0-incubating + + + org.apache.commons + commons-math3 + 3.1.1 + + + org.apache.httpcomponents + httpclient + 4.5.4 + + + org.apache.httpcomponents + httpcore + 4.4.7 + + + com.ning + async-http-client + 1.8.10 + + + commons-beanutils + commons-beanutils + 1.7.0 + + + commons-beanutils + commons-beanutils-core + 1.8.0 + dss-assembly diff --git a/bin/checkEnv.sh b/bin/checkEnv.sh index bdf48659ae..d51bd5ca21 100644 --- a/bin/checkEnv.sh +++ b/bin/checkEnv.sh @@ -1,3 +1,4 @@ +#!/bin/sh # # Copyright 2019 WeBank # @@ -13,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # -#!/bin/sh say() { printf 'check command fail \n %s\n' "$1" } @@ -32,7 +32,7 @@ need_cmd() { err "need '$1' (your linux command not found)" fi } -echo "<-----start to check linux cmd:yum java mysql unzip expect telnet sed tar---->" +echo "<-----start to check used cmd---->" need_cmd yum need_cmd java need_cmd mysql @@ -42,4 +42,4 @@ need_cmd telnet need_cmd tar need_cmd sed need_cmd dos2unix -echo "<-----end to check linux cmd:yum java mysql unzip expect telnet sed tar------>" +echo "<-----end to check used cmd---->" diff --git a/bin/checkMicro.sh b/bin/checkServices.sh similarity index 50% rename from bin/checkMicro.sh rename to bin/checkServices.sh index 3f8ff95fa9..a540d525fa 100644 --- a/bin/checkMicro.sh +++ b/bin/checkServices.sh @@ -20,18 +20,59 @@ shellDir=`dirname $0` workDir=`cd ${shellDir}/..;pwd` ##load config -source ${workDir}/conf/config.sh -source ${workDir}/conf/db.sh +export LINKIS_DSS_CONF_FILE=${LINKIS_DSS_CONF_FILE:-"${workDir}/conf/config.sh"} +export DISTRIBUTION=${DISTRIBUTION:-"${workDir}/conf/config.sh"} +source ${LINKIS_DSS_CONF_FILE} +source ${DISTRIBUTION} MICRO_SERVICE_NAME=$1 MICRO_SERVICE_IP=$2 MICRO_SERVICE_PORT=$3 -echo "<--------------------------------------------------------------------------->" -echo "Start to Check if your microservice:$MICRO_SERVICE_NAME is normal via telnet" -echo "" -if [ ! -d $DSS_INSTALL_HOME/$MICRO_SERVICE_NAME ];then - echo "$MICRO_SERVICE_NAME is not installed,the check steps will be skipped" - exit 0 + +local_host="`hostname --fqdn`" + +ipaddr="`hostname -i`" + +function isLocal(){ + if [ "$1" == "127.0.0.1" ];then + return 0 + elif [ $1 == "localhost" ]; then + return 0 + elif [ $1 == $local_host ]; then + return 0 + elif [ $1 == $ipaddr ]; then + return 0 + fi + return 1 +} + +function executeCMD(){ + isLocal $1 + flag=$? + echo "Is local "$flag + if [ $flag == "0" ];then + eval $2 + else + ssh -p $SSH_PORT $1 $2 + fi + +} + +#echo "<--------------------------------------------------------------------------->" +#echo "Start to Check if your microservice:$MICRO_SERVICE_NAME is normal via telnet" +#echo "" +#if ! executeCMD $SERVER_IP "test -e $DSS_INSTALL_HOME/$MICRO_SERVICE_NAME"; then +# echo "$MICRO_SERVICE_NAME is not installed,the check steps will be skipped" +# exit 0 +#fi +echo "===========================================================" +echo $MICRO_SERVICE_NAME +echo $MICRO_SERVICE_IP +echo $MICRO_SERVICE_PORT +echo "===========================================================" + +if [ $MICRO_SERVICE_NAME == "visualis-server" ]||[ $MICRO_SERVICE_IP == "127.0.0.1" ]; then + MICRO_SERVICE_IP="`hostname -i`" fi result=`echo -e "\n" | telnet $MICRO_SERVICE_IP $MICRO_SERVICE_PORT 2>/dev/null | grep Connected | wc -l` @@ -42,7 +83,7 @@ else echo "ERROR your $MICRO_SERVICE_NAME microservice is not start successful !!! ERROR logs as follows :" echo "PLEAESE CHECK DETAIL LOG,LOCATION:$DSS_INSTALL_HOME/$MICRO_SERVICE_NAME/logs/linkis.out" echo '<------------------------------------------------------------->' - tail -n 50 $DSS_INSTALL_HOME/$MICRO_SERVICE_NAME/logs/*.out + executeCMD $MICRO_SERVICE_IP "tail -n 50 $DSS_INSTALL_HOME/$MICRO_SERVICE_NAME/logs/*.out" echo '<-------------------------------------------------------------->' echo "PLEAESE CHECK DETAIL LOG,LOCATION:$DSS_INSTALL_HOME/$MICRO_SERVICE_NAME/logs/linkis.out" exit 1 diff --git a/bin/install.sh b/bin/install.sh index 5961dfdbd8..01b8c3a04e 100644 --- a/bin/install.sh +++ b/bin/install.sh @@ -15,6 +15,7 @@ # #!/bin/sh #Actively load user env + source ~/.bash_profile shellDir=`dirname $0` @@ -45,17 +46,13 @@ fi function isSuccess(){ if [ $? -ne 0 ]; then - echo "ERROR to " + $1 + echo "Failed to " + $1 exit 1 else - echo "SUCESS to" + $1 + echo "Succeed to" + $1 fi } -#check env -sh ${workDir}/bin/checkEnv.sh -isSuccess "check env" - function checkJava(){ java -version isSuccess "execute java --version" @@ -72,46 +69,108 @@ else fi } + +say() { + printf 'check command fail \n %s\n' "$1" +} + +err() { + say "$1" >&2 + exit 1 +} + +check_cmd() { + command -v "$1" > /dev/null 2>&1 +} + +need_cmd() { + if ! check_cmd "$1"; then + err "need '$1' (command not found)" + fi +} + +#check env +sh ${workDir}/bin/checkEnv.sh +isSuccess "check env" + ##load config echo "step1:load config" -source ${workDir}/conf/config.sh -source ${workDir}/conf/db.sh +export DSS_CONFIG_PATH=${DSS_CONFIG_PATH:-"${workDir}/conf/config.sh"} +export DSS_DB_CONFIG_PATH=${DSS_DB_CONFIG_PATH:-"${workDir}/conf/db.sh"} +export DISTRIBUTION=${DISTRIBUTION:-"${workDir}/conf/config.sh"} +source ${DSS_CONFIG_PATH} +source ${DSS_DB_CONFIG_PATH} +source ${DISTRIBUTION} isSuccess "load config" -local_host="`hostname -i`" +local_host="`hostname --fqdn`" +ipaddr="`hostname -i`" + +function isLocal(){ + if [ "$1" == "127.0.0.1" ];then + return 0 + elif [ $1 == "localhost" ]; then + return 0 + elif [ $1 == $local_host ]; then + return 0 + elif [ $1 == $ipaddr ]; then + return 0 + fi + return 1 +} + +function executeCMD(){ + isLocal $1 + flag=$? + if [ $flag == "0" ];then + echo "Is local execution:$2" + eval $2 + else + echo "Is remote execution:$2" + ssh -p $SSH_PORT $1 $2 + fi +} -##env check -echo "Please enter the mode selection such as: 1" -echo " 1: lite" -echo " 2: sample" -echo " 3: Standard" -echo "" +function copyFile(){ + isLocal $1 + flag=$? + src=$2 + dest=$3 + if [ $flag == "0" ];then + echo "Is local cp " + eval "cp -r $src $dest" + else + echo "Is remote cp " + scp -r -P $SSH_PORT $src $1:$dest + fi +} -INSTALL_MODE=1 +##install mode choice +if [ "$INSTALL_MODE" == "" ];then + echo "Please enter the mode selection such as: 1" + echo " 1: Lite" + echo " 2: Simple" + echo " 3: Standard" + echo "" + read -p "Please input the choice:" idx + INSTALL_MODE=$idx +fi -read -p "Please input the choice:" idx -if [[ '1' = "$idx" ]];then - INSTALL_MODE=1 +if [[ '1' = "$INSTALL_MODE" ]];then echo "You chose lite installation mode" - #check for Java checkJava - #check for mysql SERVER_NAME=MYSQL EXTERNAL_SERVER_IP=$MYSQL_HOST EXTERNAL_SERVER_PORT=$MYSQL_PORT checkExternalServer -elif [[ '2' = "$idx" ]];then - INSTALL_MODE=2 +elif [[ '2' = "$INSTALL_MODE" ]];then echo "You chose sample installation mode" - #check for Java checkJava - #check for mysql SERVER_NAME=MYSQL EXTERNAL_SERVER_IP=$MYSQL_HOST EXTERNAL_SERVER_PORT=$MYSQL_PORT - -elif [[ '3' = "$idx" ]];then - INSTALL_MODE=3 + checkExternalServer +elif [[ '3' = "$INSTALL_MODE" ]];then echo "You chose Standard installation mode" #check for Java checkJava @@ -124,13 +183,16 @@ elif [[ '3' = "$idx" ]];then SERVER_NAME=Qualitis EXTERNAL_SERVER_IP=$QUALITIS_ADRESS_IP EXTERNAL_SERVER_PORT=$QUALITIS_ADRESS_PORT + if [[ $IGNORECHECK = "" ]];then checkExternalServer + fi #check azkaban serivice SERVER_NAME=AZKABAN EXTERNAL_SERVER_IP=$AZKABAN_ADRESS_IP EXTERNAL_SERVER_PORT=$AZKABAN_ADRESS_PORT + if [[ $IGNORECHECK = "" ]];then checkExternalServer - + fi else echo "no choice,exit!" exit 1 @@ -156,40 +218,97 @@ else exit 1 fi +echo "create hdfs directory and local directory" +if [ "$WORKSPACE_USER_ROOT_PATH" != "" ] +then + localRootDir=$WORKSPACE_USER_ROOT_PATH + if [[ $WORKSPACE_USER_ROOT_PATH == file://* ]];then + localRootDir=${WORKSPACE_USER_ROOT_PATH#file://} + mkdir -p $localRootDir/$deployUser + sudo chmod -R 775 $localRootDir/$deployUser + elif [[ $WORKSPACE_USER_ROOT_PATH == hdfs://* ]];then + localRootDir=${WORKSPACE_USER_ROOT_PATH#hdfs://} + hdfs dfs -mkdir -p $localRootDir/$deployUser + hdfs dfs -chmod -R 775 $localRootDir/$deployUser + else + echo "does not support $WORKSPACE_USER_ROOT_PATH filesystem types" + fi +isSuccess "create $WORKSPACE_USER_ROOT_PATH directory" +fi + + +if [ "$RESULT_SET_ROOT_PATH" != "" ] +then + localRootDir=$RESULT_SET_ROOT_PATH + if [[ $RESULT_SET_ROOT_PATH == file://* ]];then + localRootDir=${RESULT_SET_ROOT_PATH#file://} + mkdir -p $localRootDir/$deployUser + sudo chmod -R 775 $localRootDir/$deployUser + elif [[ $RESULT_SET_ROOT_PATH == hdfs://* ]];then + localRootDir=${RESULT_SET_ROOT_PATH#hdfs://} + hdfs dfs -mkdir -p $localRootDir/$deployUser + hdfs dfs -chmod -R 775 $localRootDir/$deployUser + else + echo "does not support $RESULT_SET_ROOT_PATH filesystem types" + fi +isSuccess "create $RESULT_SET_ROOT_PATH directory" +fi + + +if [ "$WDS_SCHEDULER_PATH" != "" ] +then + localRootDir=$WDS_SCHEDULER_PATH + if [[ $WDS_SCHEDULER_PATH == file://* ]];then + localRootDir=${WDS_SCHEDULER_PATH#file://} + mkdir -p $localRootDir + sudo chmod -R 775 $localRootDir + elif [[ $WDS_SCHEDULER_PATH == hdfs://* ]];then + localRootDir=${WDS_SCHEDULER_PATH#hdfs://} + hdfs dfs -mkdir -p $localRootDir + hdfs dfs -chmod -R 775 $localRootDir + else + echo "does not support $WDS_SCHEDULER_PATH filesystem types" + fi +isSuccess "create $WDS_SCHEDULER_PATH directory" +fi + + ##init db if [[ '2' = "$MYSQL_INSTALL_MODE" ]];then - mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source ${workDir}/db/dss_ddl.sql" - isSuccess "source dss_ddl.sql" - LOCAL_IP="`hostname -i`" - if [ $GATEWAY_INSTALL_IP == "127.0.0.1" ];then - echo "GATEWAY_INSTALL_IP is equals 127.0.0.1 ,we will change it to ip address" - GATEWAY_INSTALL_IP_2=$LOCAL_IP + mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source ${workDir}/db/dss_ddl.sql" + isSuccess "source dss_ddl.sql" + LOCAL_IP="`hostname -i`" + if [ $GATEWAY_INSTALL_IP == "127.0.0.1" ];then + echo "GATEWAY_INSTALL_IP is equals 127.0.0.1 ,we will change it to ip address" + GATEWAY_INSTALL_IP_2=$LOCAL_IP else - GATEWAY_INSTALL_IP_2=$GATEWAY_INSTALL_IP + GATEWAY_INSTALL_IP_2=$GATEWAY_INSTALL_IP fi #echo $GATEWAY_INSTALL_IP_2 sed -i "s/GATEWAY_INSTALL_IP_2/$GATEWAY_INSTALL_IP_2/g" ${workDir}/db/dss_dml.sql sed -i "s/GATEWAY_PORT/$GATEWAY_PORT/g" ${workDir}/db/dss_dml.sql mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source ${workDir}/db/dss_dml.sql" - isSuccess "source dss_dml.sql" + isSuccess "source dss_dml.sql" - if [ '2' = "$INSTALL_MODE" ]||[ '3' = "$INSTALL_MODE" ];then - echo "visualis support " - if [ $VISUALIS_NGINX_IP == "127.0.0.1" ]||[ $VISUALIS_NGINX_IP == "0.0.0.0" ];then - echo "VISUALIS_NGINX_IP is equals $VISUALIS_NGINX_IP ,we will change it to ip address" - VISUALIS_NGINX_IP_2=$LOCAL_IP + if [[ '2' = "$INSTALL_MODE" ]] || [[ '3' = "$INSTALL_MODE" ]];then + echo "visualis support,visualis database will be initialized !" + if [ $VISUALIS_NGINX_IP == "127.0.0.1" ]||[ $VISUALIS_NGINX_IP == "0.0.0.0" ];then + echo "VISUALIS_NGINX_IP is equals $VISUALIS_NGINX_IP ,we will change it to ip address" + VISUALIS_NGINX_IP_2=$LOCAL_IP else - VISUALIS_NGINX_IP_2=$VISUALIS_NGINX_IP + VISUALIS_NGINX_IP_2=$VISUALIS_NGINX_IP fi #echo $VISUALIS_NGINX_IP_2 sed -i "s/VISUALIS_NGINX_IP_2/$VISUALIS_NGINX_IP_2/g" ${workDir}/db/visualis.sql sed -i "s/VISUALIS_NGINX_PORT/$VISUALIS_NGINX_PORT/g" ${workDir}/db/visualis.sql - mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source ${workDir}/db/visualis.sql" - isSuccess "source visualis.sql" + mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source ${workDir}/db/visualis.sql" + isSuccess "source visualis.sql" + mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source ${workDir}/db/davinci.sql" + isSuccess "source davinci.sql" fi - if [[ '3' = "$INSTALL_MODE" ]];then - echo "start to update azkaban and qualitis table info " + if [[ '3' = "$INSTALL_MODE" ]];then + echo "azkaban and qualitis support, azkaban and qualitis database will be initialized !" #azkaban if [ $AZKABAN_ADRESS_IP == "127.0.0.1" ];then echo "AZKABAN_ADRESS_IP is equals 127.0.0.1 ,we will change it to ip address" @@ -217,28 +336,9 @@ if [[ '2' = "$MYSQL_INSTALL_MODE" ]];then fi fi -## davinci db init -echo "Do you want to clear davinci table information in the database ? If you have not installed davinci environment,you must input '2',if you have davinci installed,choice 1." -echo " 1: Do not execute table-building statements" -echo "WARN:" -echo " 2: Dangerous! Clear all data and rebuild the tables." -echo "" -DAVINCI_INSTALL_MODE=1 -read -p "Please input the choice:" idx -if [[ '2' = "$idx" ]];then - DAVINCI_INSTALL_MODE=2 - echo "You chose rebuild davinci's table !!! start rebuild all tables" - mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source ${workDir}/db/davinci.sql" - isSuccess "source davinci.sql" - echo "" -elif [[ '1' = "$idx" ]];then - DAVINCI_INSTALL_MODE=1 - echo "You chose not execute table-building statements" - echo "" -else - echo "no choice,exit!" - exit 1 -fi +##Deal special symbol '#' +HIVE_META_PASSWORD=$(echo ${HIVE_META_PASSWORD//'#'/'\#'}) +MYSQL_PASSWORD=$(echo ${MYSQL_PASSWORD//'#'/'\#'}) ###linkis Eurkea info SERVER_IP=$EUREKA_INSTALL_IP @@ -260,24 +360,30 @@ then SERVER_IP=$local_host fi -if ! ssh -p $SSH_PORT $SERVER_IP test -e $SERVER_HOME; then - ssh -p $SSH_PORT $SERVER_IP "sudo mkdir -p $SERVER_HOME;sudo chown -R $deployUser:$deployUser $SERVER_HOME" +if ! executeCMD $SERVER_IP "test -e $SERVER_HOME"; then + executeCMD $SERVER_IP "sudo mkdir -p $SERVER_HOME;sudo chown -R $deployUser:$deployUser $SERVER_HOME" isSuccess "create the dir of $SERVERNAME" fi echo "$SERVERNAME-step2:copy install package" -scp -P $SSH_PORT ${workDir}/share/$PACKAGE_DIR/$SERVERNAME.zip $SERVER_IP:$SERVER_HOME +copyFile $SERVER_IP ${workDir}/share/$PACKAGE_DIR/$SERVERNAME.zip $SERVER_HOME + +if ! executeCMD $SERVER_IP "test -e $SERVER_HOME/lib"; then + copyFile $SERVER_IP ${workDir}/lib $SERVER_HOME +fi + +#copyFile $SERVER_IP ${workDir}/lib $SERVER_HOME isSuccess "copy ${SERVERNAME}.zip" -ssh -p $SSH_PORT $SERVER_IP "cd $SERVER_HOME/;rm -rf $SERVERNAME-bak; mv -f $SERVERNAME $SERVERNAME-bak" -ssh -p $SSH_PORT $SERVER_IP "cd $SERVER_HOME/;unzip $SERVERNAME.zip > /dev/null" -ssh -p $SSH_PORT $SERVER_IP "cd $workDir/;scp -r lib/* $SERVER_HOME/$SERVERNAME/lib" +executeCMD $SERVER_IP "cd $SERVER_HOME/;rm -rf $SERVERNAME-bak; mv -f $SERVERNAME $SERVERNAME-bak" +executeCMD $SERVER_IP "cd $SERVER_HOME/;unzip $SERVERNAME.zip > /dev/null" +executeCMD $SERVER_IP "cd $SERVER_HOME/;scp -r lib/* $SERVER_HOME/$SERVERNAME/lib" isSuccess "unzip ${SERVERNAME}.zip" echo "$SERVERNAME-step3:subsitution conf" SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/application.yml -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#port:.*#port: $SERVER_PORT#g\" $SERVER_CONF_PATH" -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#defaultZone:.*#defaultZone: $EUREKA_URL#g\" $SERVER_CONF_PATH" -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#hostname:.*#hostname: $SERVER_IP#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#port:.*#port: $SERVER_PORT#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#defaultZone:.*#defaultZone: $EUREKA_URL#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#hostname:.*#hostname: $SERVER_IP#g\" $SERVER_CONF_PATH" isSuccess "subsitution conf of $SERVERNAME" } ##function end @@ -291,16 +397,16 @@ then SERVER_IP=$local_host fi -if ! ssh -p $SSH_PORT $SERVER_IP test -e $SERVER_HOME; then - ssh -p $SSH_PORT $SERVER_IP "sudo mkdir -p $SERVER_HOME;sudo chown -R $deployUser:$deployUser $SERVER_HOME" +if ! executeCMD $SERVER_IP "test -e $SERVER_HOME"; then + executeCMD $SERVER_IP "sudo mkdir -p $SERVER_HOME;sudo chown -R $deployUser:$deployUser $SERVER_HOME" isSuccess "create the dir of $SERVERNAME" fi echo "$SERVERNAME-step2:copy install package" -scp -P $SSH_PORT ${workDir}/share/$PACKAGE_DIR/$SERVERNAME.zip $SERVER_IP:$SERVER_HOME +copyFile $SERVER_IP ${workDir}/share/$PACKAGE_DIR/$SERVERNAME.zip $SERVER_HOME isSuccess "copy ${SERVERNAME}.zip" -ssh -p $SSH_PORT $SERVER_IP "cd $SERVER_HOME/;rm -rf $SERVERNAME-bak; mv -f $SERVERNAME $SERVERNAME-bak" -ssh -p $SSH_PORT $SERVER_IP "cd $SERVER_HOME/;unzip $SERVERNAME.zip > /dev/null" +executeCMD $SERVER_IP "cd $SERVER_HOME/;rm -rf $SERVERNAME-bak; mv -f $SERVERNAME $SERVERNAME-bak" +executeCMD $SERVER_IP "cd $SERVER_HOME/;unzip $SERVERNAME.zip > /dev/null" isSuccess "unzip ${SERVERNAME}.zip" } ##function end @@ -315,15 +421,15 @@ then SERVER_IP=$local_host fi -if ! ssh -p $SSH_PORT $SERVER_IP test -e $SERVER_HOME/$APPJOINTPARENT; then - ssh -p $SSH_PORT $SERVER_IP "sudo mkdir -p $SERVER_HOME/$APPJOINTPARENT;sudo chown -R $deployUser:$deployUser $SERVER_HOME/$APPJOINTPARENT" +if ! executeCMD $SERVER_IP "test -e $SERVER_HOME/$APPJOINTPARENT"; then + executeCMD $SERVER_IP "sudo mkdir -p $SERVER_HOME/$APPJOINTPARENT;sudo chown -R $deployUser:$deployUser $SERVER_HOME/$APPJOINTPARENT" isSuccess "create the dir of $SERVER_HOME/$APPJOINTPARENT;" fi echo "$APPJOINTNAME-step2:copy install package" -scp -P $SSH_PORT $workDir/share/appjoints/$APPJOINTNAME/*.zip $SERVER_IP:$SERVER_HOME/$APPJOINTPARENT +copyFile $SERVER_IP $workDir/share/appjoints/$APPJOINTNAME/*.zip $SERVER_HOME/$APPJOINTPARENT isSuccess "copy ${APPJOINTNAME}.zip" -ssh -p $SSH_PORT $SERVER_IP "cd $SERVER_HOME/$APPJOINTPARENT/;unzip -o dss-*-appjoint.zip > /dev/null;rm -rf dss-*-appjoint.zip" +executeCMD $SERVER_IP "cd $SERVER_HOME/$APPJOINTPARENT/;unzip -o dss-*-appjoint.zip > /dev/null;rm -rf dss-*-appjoint.zip" isSuccess "install ${APPJOINTNAME}.zip" } ##function end @@ -339,18 +445,18 @@ installPackage ###update Dss-Server linkis.properties echo "$SERVERNAME-step4:update linkis.properties" SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/linkis.properties -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#wds.linkis.server.mybatis.datasource.url.*#wds.linkis.server.mybatis.datasource.url=jdbc:mysql://${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB}?characterEncoding=UTF-8#g\" $SERVER_CONF_PATH" -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#wds.linkis.server.mybatis.datasource.username.*#wds.linkis.server.mybatis.datasource.username=$MYSQL_USER#g\" $SERVER_CONF_PATH" -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#wds.linkis.server.mybatis.datasource.password.*#wds.linkis.server.mybatis.datasource.password=$MYSQL_PASSWORD#g\" $SERVER_CONF_PATH" -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#wds.dss.appjoint.scheduler.azkaban.address.*#wds.dss.appjoint.scheduler.azkaban.address=http://${AZKABAN_ADRESS_IP}:${AZKABAN_ADRESS_PORT}#g\" $SERVER_CONF_PATH" -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#wds.linkis.gateway.ip.*#wds.linkis.gateway.ip=$GATEWAY_INSTALL_IP#g\" $SERVER_CONF_PATH" -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#wds.linkis.gateway.port.*#wds.linkis.gateway.port=$GATEWAY_PORT#g\" $SERVER_CONF_PATH" -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#wds.dss.appjoint.scheduler.project.store.dir.*#wds.dss.appjoint.scheduler.project.store.dir=$WDS_SCHEDULER_PATH#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#wds.linkis.server.mybatis.datasource.url.*#wds.linkis.server.mybatis.datasource.url=jdbc:mysql://${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB}?characterEncoding=UTF-8#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#wds.linkis.server.mybatis.datasource.username.*#wds.linkis.server.mybatis.datasource.username=$MYSQL_USER#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#wds.linkis.server.mybatis.datasource.password.*#wds.linkis.server.mybatis.datasource.password=$MYSQL_PASSWORD#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#wds.dss.appjoint.scheduler.azkaban.address.*#wds.dss.appjoint.scheduler.azkaban.address=http://${AZKABAN_ADRESS_IP}:${AZKABAN_ADRESS_PORT}#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#wds.linkis.gateway.ip.*#wds.linkis.gateway.ip=$GATEWAY_INSTALL_IP#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#wds.linkis.gateway.port.*#wds.linkis.gateway.port=$GATEWAY_PORT#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#wds.dss.appjoint.scheduler.project.store.dir.*#wds.dss.appjoint.scheduler.project.store.dir=$WDS_SCHEDULER_PATH#g\" $SERVER_CONF_PATH" isSuccess "subsitution linkis.properties of $SERVERNAME" echo "<----------------$SERVERNAME:end------------------->" echo "" -if [ '2' = "$INSTALL_MODE" ]||[ '3' = "$INSTALL_MODE" ];then +if [[ '2' = "$INSTALL_MODE" ]]||[[ '3' = "$INSTALL_MODE" ]];then ##Flow execution Install PACKAGE_DIR=dss/dss-flow-execution-entrance SERVERNAME=dss-flow-execution-entrance @@ -362,9 +468,9 @@ installPackage ###Update flow execution linkis.properties echo "$SERVERNAME-step4:update linkis.properties" SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/linkis.properties -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#wds.linkis.entrance.config.logPath.*#wds.linkis.entrance.config.logPath=$WORKSPACE_USER_ROOT_PATH#g\" $SERVER_CONF_PATH" -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#wds.linkis.resultSet.store.path.*#wds.linkis.resultSet.store.path=$RESULT_SET_ROOT_PATH#g\" $SERVER_CONF_PATH" -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#wds.linkis.gateway.url.*#wds.linkis.gateway.url=http://${GATEWAY_INSTALL_IP}:${GATEWAY_PORT}#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#wds.linkis.entrance.config.logPath.*#wds.linkis.entrance.config.logPath=$WORKSPACE_USER_ROOT_PATH#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#wds.linkis.resultSet.store.path.*#wds.linkis.resultSet.store.path=$RESULT_SET_ROOT_PATH#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#wds.linkis.gateway.url.*#wds.linkis.gateway.url=http://${GATEWAY_INSTALL_IP}:${GATEWAY_PORT}#g\" $SERVER_CONF_PATH" isSuccess "subsitution linkis.properties of $SERVERNAME" echo "<----------------$SERVERNAME:end------------------->" echo "" @@ -379,8 +485,8 @@ installPackage ###Update appjoint entrance linkis.properties echo "$SERVERNAME-step4:update linkis.properties" SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/linkis.properties -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#wds.linkis.entrance.config.logPath.*#wds.linkis.entrance.config.logPath=$WORKSPACE_USER_ROOT_PATH#g\" $SERVER_CONF_PATH" -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#wds.linkis.resultSet.store.path.*#wds.linkis.resultSet.store.path=$RESULT_SET_ROOT_PATH#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#wds.linkis.entrance.config.logPath.*#wds.linkis.entrance.config.logPath=$WORKSPACE_USER_ROOT_PATH#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#wds.linkis.resultSet.store.path.*#wds.linkis.resultSet.store.path=$RESULT_SET_ROOT_PATH#g\" $SERVER_CONF_PATH" isSuccess "subsitution linkis.properties of $SERVERNAME" echo "<----------------$SERVERNAME:end------------------->" echo "" @@ -396,22 +502,25 @@ installVisualis echo "$SERVERNAME-step4:update linkis.properties" SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/linkis.properties if [ $VISUALIS_NGINX_IP == "127.0.0.1" ]||[ $VISUALIS_NGINX_IP == "0.0.0.0" ]; then - VISUALIS_NGINX_IP=$local_host + VISUALIS_NGINX_IP=$ipaddr +fi +if [ $VISUALIS_SERVER_INSTALL_IP == "127.0.0.1" ]||[ $VISUALIS_SERVER_INSTALL_IP == "0.0.0.0" ]; then + VISUALIS_SERVER_INSTALL_IP=$ipaddr fi -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#wds.linkis.entrance.config.logPath.*#wds.linkis.entrance.config.logPath=$WORKSPACE_USER_ROOT_PATH#g\" $SERVER_CONF_PATH" -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#wds.linkis.resultSet.store.path.*#wds.linkis.resultSet.store.path=$RESULT_SET_ROOT_PATH#g\" $SERVER_CONF_PATH" -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#wds.dss.visualis.gateway.ip.*#wds.dss.visualis.gateway.ip=$GATEWAY_INSTALL_IP#g\" $SERVER_CONF_PATH" -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#wds.dss.visualis.gateway.port.*#wds.dss.visualis.gateway.port=$GATEWAY_PORT#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#wds.linkis.entrance.config.logPath.*#wds.linkis.entrance.config.logPath=$WORKSPACE_USER_ROOT_PATH#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#wds.linkis.resultSet.store.path.*#wds.linkis.resultSet.store.path=$RESULT_SET_ROOT_PATH#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#wds.dss.visualis.gateway.ip.*#wds.dss.visualis.gateway.ip=$GATEWAY_INSTALL_IP#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#wds.dss.visualis.gateway.port.*#wds.dss.visualis.gateway.port=$GATEWAY_PORT#g\" $SERVER_CONF_PATH" SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/application.yml -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#address: 127.0.0.1#address: $VISUALIS_SERVER_INSTALL_IP#g\" $SERVER_CONF_PATH" -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#port: 9007#port: $VISUALIS_SERVER_PORT#g\" $SERVER_CONF_PATH" -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#url: http://0.0.0.0:0000/dss/visualis#url: http://$VISUALIS_NGINX_IP:$VISUALIS_NGINX_PORT/dss/visualis#g\" $SERVER_CONF_PATH" -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#address: 0.0.0.0#address: $VISUALIS_NGINX_IP#g\" $SERVER_CONF_PATH" -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#port: 0000#port: $VISUALIS_NGINX_PORT#g\" $SERVER_CONF_PATH" -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#defaultZone: http://127.0.0.1:20303/eureka/#defaultZone: http://$EUREKA_INSTALL_IP:$EUREKA_PORT/eureka/#g\" $SERVER_CONF_PATH" -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#url: jdbc:mysql://127.0.0.1:3306/xxx?characterEncoding=UTF-8#url: jdbc:mysql://$MYSQL_HOST:$MYSQL_PORT/$MYSQL_DB?characterEncoding=UTF-8#g\" $SERVER_CONF_PATH" -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#username: xxx#username: $MYSQL_USER#g\" $SERVER_CONF_PATH" -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#password: xxx#password: $MYSQL_PASSWORD#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#address: 127.0.0.1#address: $VISUALIS_SERVER_INSTALL_IP#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#port: 9007#port: $VISUALIS_SERVER_PORT#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#url: http://0.0.0.0:0000/dss/visualis#url: http://$VISUALIS_NGINX_IP:$VISUALIS_NGINX_PORT/dss/visualis#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#address: 0.0.0.0#address: $VISUALIS_NGINX_IP#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#port: 0000#port: $VISUALIS_NGINX_PORT#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#defaultZone: http://127.0.0.1:20303/eureka/#defaultZone: http://$EUREKA_INSTALL_IP:$EUREKA_PORT/eureka/#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#url: jdbc:mysql://127.0.0.1:3306/xxx?characterEncoding=UTF-8#url: jdbc:mysql://$MYSQL_HOST:$MYSQL_PORT/$MYSQL_DB?characterEncoding=UTF-8#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#username: xxx#username: $MYSQL_USER#g\" $SERVER_CONF_PATH" +executeCMD $SERVER_IP "sed -i \"s#password: xxx#password: $MYSQL_PASSWORD#g\" $SERVER_CONF_PATH" isSuccess "subsitution linkis.properties of $SERVERNAME" echo "<----------------$SERVERNAME:end------------------->" echo "" @@ -423,9 +532,9 @@ APPJOINTNAME=datachecker installAppjoints echo "$APPJOINTNAME:subsitution conf" APPJOINTNAME_CONF_PATH_PATENT=$SERVER_HOME/$APPJOINTPARENT/$APPJOINTNAME/appjoint.properties -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#job.datachecker.jdo.option.url.*#job.datachecker.jdo.option.url=$HIVE_META_URL#g\" $APPJOINTNAME_CONF_PATH_PATENT" -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#job.datachecker.jdo.option.username.*#job.datachecker.jdo.option.username=$HIVE_META_USER#g\" $APPJOINTNAME_CONF_PATH_PATENT" -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#job.datachecker.jdo.option.password.*#job.datachecker.jdo.option.password=$HIVE_META_PASSWORD#g\" $APPJOINTNAME_CONF_PATH_PATENT" +executeCMD $SERVER_IP "sed -i \"s#job.datachecker.jdo.option.url.*#job.datachecker.jdo.option.url=$HIVE_META_URL#g\" $APPJOINTNAME_CONF_PATH_PATENT" +executeCMD $SERVER_IP "sed -i \"s#job.datachecker.jdo.option.username.*#job.datachecker.jdo.option.username=$HIVE_META_USER#g\" $APPJOINTNAME_CONF_PATH_PATENT" +executeCMD $SERVER_IP "sed -i \"s#job.datachecker.jdo.option.password.*#job.datachecker.jdo.option.password=$HIVE_META_PASSWORD#g\" $APPJOINTNAME_CONF_PATH_PATENT" isSuccess "subsitution conf of datachecker" echo "<----------------datachecker appjoint install end------------------->" echo "" @@ -436,9 +545,9 @@ APPJOINTNAME=eventchecker installAppjoints echo "$APPJOINTNAME:subsitution conf" APPJOINTNAME_CONF_PATH_PATENT=$SERVER_HOME/$APPJOINTPARENT/$APPJOINTNAME/appjoint.properties -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#msg.eventchecker.jdo.option.url.*#msg.eventchecker.jdo.option.url=jdbc:mysql://${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB}?characterEncoding=UTF-8#g\" $APPJOINTNAME_CONF_PATH_PATENT" -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#msg.eventchecker.jdo.option.username.*#msg.eventchecker.jdo.option.username=$MYSQL_USER#g\" $APPJOINTNAME_CONF_PATH_PATENT" -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#msg.eventchecker.jdo.option.password.*#msg.eventchecker.jdo.option.password=$MYSQL_PASSWORD#g\" $APPJOINTNAME_CONF_PATH_PATENT" +executeCMD $SERVER_IP "sed -i \"s#msg.eventchecker.jdo.option.url.*#msg.eventchecker.jdo.option.url=jdbc:mysql://${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB}?characterEncoding=UTF-8#g\" $APPJOINTNAME_CONF_PATH_PATENT" +executeCMD $SERVER_IP "sed -i \"s#msg.eventchecker.jdo.option.username.*#msg.eventchecker.jdo.option.username=$MYSQL_USER#g\" $APPJOINTNAME_CONF_PATH_PATENT" +executeCMD $SERVER_IP "sed -i \"s#msg.eventchecker.jdo.option.password.*#msg.eventchecker.jdo.option.password=$MYSQL_PASSWORD#g\" $APPJOINTNAME_CONF_PATH_PATENT" isSuccess "subsitution conf of eventchecker" echo "<----------------$APPJOINTNAME:end------------------->" echo "" @@ -459,7 +568,7 @@ APPJOINTNAME=qualitis #qualitis appjoint install installAppjoints APPJOINTNAME_CONF_PATH_PATENT=$SERVER_HOME/$APPJOINTPARENT/$APPJOINTNAME/appjoint.properties -ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#baseUrl=http://127.0.0.1:8090#baseUrl=http://$QUALITIS_ADRESS_IP:$QUALITIS_ADRESS_PORT#g\" $APPJOINTNAME_CONF_PATH_PATENT" +executeCMD $SERVER_IP "sed -i \"s#baseUrl=http://127.0.0.1:8090#baseUrl=http://$QUALITIS_ADRESS_IP:$QUALITIS_ADRESS_PORT#g\" $APPJOINTNAME_CONF_PATH_PATENT" isSuccess "subsitution conf of qualitis" echo "<----------------$APPJOINTNAME:end------------------->" echo "" diff --git a/bin/start-all.sh b/bin/start-all.sh index 8f5b04e2ca..14016d0200 100644 --- a/bin/start-all.sh +++ b/bin/start-all.sh @@ -15,21 +15,33 @@ # limitations under the License. # - - # Start all dss applications info="We will start all dss applications, it will take some time, please wait" echo ${info} #Actively load user env +source /etc/profile source ~/.bash_profile workDir=`dirname "${BASH_SOURCE-$0}"` workDir=`cd "$workDir"; pwd` - CONF_DIR="${workDir}"/../conf -CONF_FILE=${CONF_DIR}/config.sh +export LINKIS_DSS_CONF_FILE=${LINKIS_DSS_CONF_FILE:-"${CONF_DIR}/config.sh"} +export DISTRIBUTION=${DISTRIBUTION:-"${CONF_DIR}/config.sh"} +source $LINKIS_DSS_CONF_FILE +source ${DISTRIBUTION} +function isSuccess(){ +if [ $? -ne 0 ]; then + echo "ERROR: " + $1 + exit 1 +else + echo "INFO:" + $1 +fi +} +local_host="`hostname --fqdn`" + +ipaddr="`hostname -i`" function isLocal(){ if [ "$1" == "127.0.0.1" ];then @@ -56,28 +68,14 @@ function executeCMD(){ } -function isSuccess(){ -if [ $? -ne 0 ]; then - echo "ERROR: " + $1 - exit 1 -else - echo "INFO:" + $1 -fi -} - -sudo yum -y install dos2unix - - -local_host="`hostname --fqdn`" - #if there is no LINKIS_INSTALL_HOME，we need to source config again if [ -z ${DSS_INSTALL_HOME} ];then echo "Warning: DSS_INSTALL_HOME does not exist, we will source config" - if [ ! -f "${CONF_FILE}" ];then + if [ ! -f "${LINKIS_DSS_CONF_FILE}" ];then echo "Error: can not find config file, start applications failed" exit 1 else - source ${CONF_FILE} + source ${LINKIS_DSS_CONF_FILE} fi fi @@ -85,19 +83,29 @@ function startApp(){ echo "<-------------------------------->" echo "Begin to start $SERVER_NAME" SERVER_BIN=${DSS_INSTALL_HOME}/${SERVER_NAME}/bin -SERVER_START_CMD="source /etc/profile;source ~/.bash_profile;cd ${SERVER_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1;sh start-${SERVER_NAME}.sh > /dev/null 2>&1 &" +#echo $SERVER_BIN +SERVER_LOCAL_START_CMD="dos2unix ${SERVER_BIN}/* > /dev/null 2>&1; dos2unix ${SERVER_BIN}/../conf/* > /dev/null 2>&1;sh ${SERVER_BIN}/start-${SERVER_NAME}.sh > /dev/null 2>&1 &" +SERVER_REMOTE_START_CMD="source /etc/profile;source ~/.bash_profile;cd ${SERVER_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh start-${SERVER_NAME}.sh > /dev/null 2>&1" + +if test -z "$SERVER_IP" +then + SERVER_IP=$local_host +fi -if [ ! -d $SERVER_BIN ];then +if ! executeCMD $SERVER_IP "test -e $SERVER_BIN"; then echo "<-------------------------------->" echo "$SERVER_NAME is not installed,the start steps will be skipped" echo "<-------------------------------->" return fi -if [ -n "${SERVER_IP}" ];then - ssh ${SERVER_IP} "${SERVER_START_CMD}" +isLocal $SERVER_IP +flag=$? +echo "Is local "$flag +if [ $flag == "0" ];then + eval $SERVER_LOCAL_START_CMD else - ssh ${local_host} "${SERVER_START_CMD}" + ssh -p $SSH_PORT $SERVER_IP $SERVER_REMOTE_START_CMD fi isSuccess "End to start $SERVER_NAME" echo "<-------------------------------->" @@ -119,6 +127,7 @@ SERVER_NAME=linkis-appjoint-entrance SERVER_IP=$APPJOINT_ENTRANCE_INSTALL_IP startApp +#visualis-server SERVER_NAME=visualis-server SERVER_IP=$VISUALIS_SERVER_INSTALL_IP startApp @@ -126,34 +135,53 @@ startApp echo "" echo "Start to check all dss microservice" echo "" + +function checkServer(){ +echo "<-------------------------------->" +echo "Begin to check $SERVER_NAME" +if test -z "$SERVER_IP" +then + SERVER_IP=$local_host +fi + +SERVER_BIN=${SERVER_HOME}/${SERVER_NAME}/bin + +if ! executeCMD $SERVER_IP "test -e ${DSS_INSTALL_HOME}/${SERVER_NAME}"; then + echo "$SERVER_NAME is not installed,the checkServer steps will be skipped" + return +fi + +sh $workDir/checkServices.sh $SERVER_NAME $SERVER_IP $SERVER_PORT +isSuccess "start $SERVER_NAME " +sleep 3 +echo "<-------------------------------->" +} + #check dss-server -MICRO_SERVICE_NAME=dss-server -MICRO_SERVICE_IP=$DSS_SERVER_INSTALL_IP -MICRO_SERVICE_PORT=$DSS_SERVER_PORT -sh $workDir/checkMicro.sh $MICRO_SERVICE_NAME $MICRO_SERVICE_IP $MICRO_SERVICE_PORT -isSuccess "$MICRO_SERVICE_NAME start" +SERVER_NAME=dss-server +SERVER_IP=$DSS_SERVER_INSTALL_IP +SERVER_PORT=$DSS_SERVER_PORT +checkServer #check dss-flow-execution-entrance -MICRO_SERVICE_NAME=dss-flow-execution-entrance -MICRO_SERVICE_IP=$FLOW_EXECUTION_INSTALL_IP -MICRO_SERVICE_PORT=$FLOW_EXECUTION_PORT -sh $workDir/checkMicro.sh $MICRO_SERVICE_NAME $MICRO_SERVICE_IP $MICRO_SERVICE_PORT -isSuccess "$MICRO_SERVICE_NAME start" +SERVER_NAME=dss-flow-execution-entrance +SERVER_IP=$FLOW_EXECUTION_INSTALL_IP +SERVER_PORT=$FLOW_EXECUTION_PORT +checkServer #check linkis-appjoint-entrance -MICRO_SERVICE_NAME=linkis-appjoint-entrance -MICRO_SERVICE_IP=$APPJOINT_ENTRANCE_INSTALL_IP -MICRO_SERVICE_PORT=$APPJOINT_ENTRANCE_PORT -sh $workDir/checkMicro.sh $MICRO_SERVICE_NAME $MICRO_SERVICE_IP $MICRO_SERVICE_PORT -isSuccess "$MICRO_SERVICE_NAME start" +SERVER_NAME=linkis-appjoint-entrance +SERVER_IP=$APPJOINT_ENTRANCE_INSTALL_IP +SERVER_PORT=$APPJOINT_ENTRANCE_PORT +checkServer #check visualis-server -sleep 10 #for visualis-server -MICRO_SERVICE_NAME=visualis-server -MICRO_SERVICE_IP=$VISUALIS_SERVER_INSTALL_IP -MICRO_SERVICE_PORT=$VISUALIS_SERVER_PORT -sh $workDir/checkMicro.sh $MICRO_SERVICE_NAME $MICRO_SERVICE_IP $MICRO_SERVICE_PORT -isSuccess "$MICRO_SERVICE_NAME start" +sleep 10 #visualis service need more time to register +SERVER_NAME=visualis-server +SERVER_IP=$VISUALIS_SERVER_INSTALL_IP +SERVER_PORT=$VISUALIS_SERVER_PORT +checkServer +echo "DSS started successfully" diff --git a/bin/stop-all.sh b/bin/stop-all.sh index 82252c7b55..af158c9b78 100644 --- a/bin/stop-all.sh +++ b/bin/stop-all.sh @@ -29,7 +29,12 @@ workDir=`cd "$workDir"; pwd` CONF_DIR="${workDir}"/../conf -CONF_FILE=${CONF_DIR}/config.sh +export LINKIS_DSS_CONF_FILE=${LINKIS_DSS_CONF_FILE:-"${CONF_DIR}/config.sh"} +export DISTRIBUTION=${DISTRIBUTION:-"${CONF_DIR}/config.sh"} +source ${DISTRIBUTION} + +local_host="`hostname --fqdn`" +ipaddr="`hostname -i`" function isSuccess(){ if [ $? -ne 0 ]; then @@ -40,18 +45,40 @@ else fi } +function isLocal(){ + if [ "$1" == "127.0.0.1" ];then + return 0 + elif [ $1 == "localhost" ]; then + return 0 + elif [ $1 == $local_host ]; then + return 0 + elif [ $1 == $ipaddr ]; then + return 0 + fi + return 1 +} +function executeCMD(){ + isLocal $1 + flag=$? + echo "Is local "$flag + if [ $flag == "0" ];then + eval $2 + else + ssh -p $SSH_PORT $1 $2 + fi + +} -local_host="`hostname --fqdn`" #if there is no LINKIS_INSTALL_HOME，we need to source config again if [ -z ${DSS_INSTALL_HOME} ];then echo "Warning: DSS_INSTALL_HOME does not exist, we will source config" - if [ ! -f "${CONF_FILE}" ];then + if [ ! -f "${LINKIS_DSS_CONF_FILE}" ];then echo "Error: can not find config file, stop applications failed" exit 1 else - source ${CONF_FILE} + source ${LINKIS_DSS_CONF_FILE} fi fi @@ -59,18 +86,26 @@ function stopAPP(){ echo "<-------------------------------->" echo "Begin to stop $SERVER_NAME" SERVER_BIN=${DSS_INSTALL_HOME}/${SERVER_NAME}/bin -SERVER_STOP_CMD="source ~/.bash_profile;cd ${SERVER_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh stop-${SERVER_NAME}.sh" -if [ ! -d ${DSS_INSTALL_HOME}/${SERVER_NAME} ];then +SERVER_LOCAL_STOP_CMD="sh ${SERVER_BIN}/stop-${SERVER_NAME}.sh" +SERVER_REMOTE_STOP_CMD="source /etc/profile;source ~/.bash_profile;cd ${SERVER_BIN}; sh stop-${SERVER_NAME}.sh " +if test -z "$SERVER_IP" +then + SERVER_IP=$local_host +fi + +if ! executeCMD $SERVER_IP "test -e ${DSS_INSTALL_HOME}/${SERVER_NAME}"; then echo "$SERVER_NAME is not installed,the stop steps will be skipped" return fi -if [ -n "${SERVER_IP}" ];then - ssh -p $SSH_PORT ${SERVER_IP} "${SERVER_STOP_CMD}" +isLocal $SERVER_IP +flag=$? +echo "Is local "$flag +if [ $flag == "0" ];then + eval $SERVER_LOCAL_STOP_CMD else - ssh -p $SSH_PORT ${local_host} "${SERVER_STOP_CMD}" + ssh -p $SSH_PORT $SERVER_IP $SERVER_REMOTE_STOP_CMD fi -isSuccess "End to stop $SERVER_NAME" echo "<-------------------------------->" sleep 3 } @@ -89,7 +124,10 @@ stopAPP SERVER_NAME=linkis-appjoint-entrance SERVER_IP=$APPJOINT_ENTRANCE_INSTALL_IP stopAPP + #visualis-server SERVER_NAME=visualis-server SERVER_IP=$VISUALIS_SERVER_INSTALL_IP stopAPP + +echo "stop-all shell script executed completely" diff --git a/conf/config.sh b/conf/config.sh index 2d0172d23a..5499f42c1d 100644 --- a/conf/config.sh +++ b/conf/config.sh @@ -1,8 +1,13 @@ +#!/bin/sh + +shellDir=`dirname $0` +workDir=`cd ${shellDir}/..;pwd` + ### deploy user deployUser=hadoop ### The install home path of DSS，Must provided -DSS_INSTALL_HOME=/appcom/Install/DSS +DSS_INSTALL_HOME=$workDir ### Specifies the user workspace, which is used to store the user's script files and log files. ### Generally local directory @@ -72,4 +77,4 @@ AZKABAN_ADRESS_PORT=8091 QUALITIS_ADRESS_IP=127.0.0.1 QUALITIS_ADRESS_PORT=8090 -DSS_VERSION=0.7.0 \ No newline at end of file +DSS_VERSION=0.7.0 diff --git a/db/dss_dml.sql b/db/dss_dml.sql index 1d48c18b20..79de16e9af 100644 --- a/db/dss_dml.sql +++ b/db/dss_dml.sql @@ -9,7 +9,7 @@ INSERT INTO `dss_workflow_node` (`id`, `icon`, `node_type`, `application_id`, `s INSERT INTO `dss_workflow_node` (`id`, `icon`, `node_type`, `application_id`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `support_jump`, `jump_url`) VALUES (NULL, NULL, 'linkis.spark.sql', @linkis_appid, '1', '1', '0', '1', NULL); INSERT INTO `dss_workflow_node` (`id`, `icon`, `node_type`, `application_id`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `support_jump`, `jump_url`) VALUES (NULL, NULL, 'linkis.spark.scala', @linkis_appid, '1', '1', '0', '1', NULL); INSERT INTO `dss_workflow_node` (`id`, `icon`, `node_type`, `application_id`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `support_jump`, `jump_url`) VALUES (NULL, NULL, 'linkis.hive.hql', @linkis_appid, '1', '1', '0', '1', NULL); -INSERT INTO `dss_workflow_node` (`id`, `icon`, `node_type`, `application_id`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `support_jump`, `jump_url`) VALUES (NULL, NULL, 'linkis.jdbc', @linkis_appid, '1', '1', '0', '1', NULL); +INSERT INTO `dss_workflow_node` (`id`, `icon`, `node_type`, `application_id`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `support_jump`, `jump_url`) VALUES (NULL, NULL, 'linkis.jdbc.jdbc', @linkis_appid, '1', '1', '0', '1', NULL); INSERT INTO `dss_workflow_node` (`id`, `icon`, `node_type`, `application_id`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `support_jump`, `jump_url`) VALUES (NULL, NULL, 'linkis.control.empty', @linkis_appid, '1', '1', '0', '0', NULL); INSERT INTO `dss_workflow_node` (`id`, `icon`, `node_type`, `application_id`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `support_jump`, `jump_url`) VALUES (NULL, NULL, 'linkis.appjoint.sendemail', @linkis_appid, '1', '1', '0', '0', NULL); INSERT INTO `dss_workflow_node` (`id`, `icon`, `node_type`, `application_id`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `support_jump`, `jump_url`) VALUES (NULL, NULL, 'linkis.appjoint.eventchecker.eventsender', @linkis_appid, '1', '1', '0', '0', NULL); diff --git a/docs/en_US/ch1/DataSphereStudio_Compile_Manual.md b/docs/en_US/ch1/DataSphereStudio_Compile_Manual.md index d3dde44ae7..6a24df612d 100644 --- a/docs/en_US/ch1/DataSphereStudio_Compile_Manual.md +++ b/docs/en_US/ch1/DataSphereStudio_Compile_Manual.md @@ -7,7 +7,7 @@ ```xml 0.7.0 - 0.9.1 + 0.9.3 2.11.8 1.8 3.3.3 diff --git a/docs/en_US/ch2/DSS Quick Installation Guide.md b/docs/en_US/ch2/DSS Quick Installation Guide.md index f8837393dc..f4b8cd1595 100644 --- a/docs/en_US/ch2/DSS Quick Installation Guide.md +++ b/docs/en_US/ch2/DSS Quick Installation Guide.md @@ -17,7 +17,7 @@ DSS also implements the integration of many external systems, such as [Qualitis] DSS environment configuration can be divided into three steps, including basic software installation, backend environment configuration, and frontend environment configuration. The details are as below: ### 2.1 Frontend and backend basic software installation -Linkis standard version (above 0.9.1). How to install [Linkis](https://github.com/WeBankFinTech/Linkis/blob/master/docs/en_US/ch1/deploy.md) +Linkis standard version (above 0.9.3). How to install [Linkis](https://github.com/WeBankFinTech/Linkis/blob/master/docs/en_US/ch1/deploy.md) JDK (above 1.8.0_141). How to install [JDK](https://www.runoob.com/java/java-environment-setup.html) @@ -111,7 +111,7 @@ The environment is ready, click me to enter ****[4. Installation and use](https: ## Three Standard DSS environment configuration preparation The standard DSS environment preparation is also divided into three parts, the frontEnd-end and back-end basic software installation, back-end environment preparation, and frontEnd-end environment preparation. The details are as follows: ### 3.1 frontEnd and BackEnd basic software installation -Linkis standard version (above 0.9.1), [How to install Linkis](https://github.com/WeBankFinTech/Linkis/blob/master/docs/en_US/ch1/deploy.md) +Linkis standard version (above 0.9.3), [How to install Linkis](https://github.com/WeBankFinTech/Linkis/blob/master/docs/en_US/ch1/deploy.md) JDK (above 1.8.0_141), How to install [JDK](https://www.runoob.com/java/java-environment-setup.html) diff --git "a/docs/zh_CN/ch1/DSS\347\274\226\350\257\221\346\226\207\346\241\243.md" "b/docs/zh_CN/ch1/DSS\347\274\226\350\257\221\346\226\207\346\241\243.md" index 26f89d3dc4..0b4d82f0b8 100644 --- "a/docs/zh_CN/ch1/DSS\347\274\226\350\257\221\346\226\207\346\241\243.md" +++ "b/docs/zh_CN/ch1/DSS\347\274\226\350\257\221\346\226\207\346\241\243.md" @@ -7,7 +7,7 @@ ```xml 0.7.0 - 0.9.1 + 0.9.3 2.11.8 1.8 3.3.3 diff --git "a/docs/zh_CN/ch2/DSS\345\277\253\351\200\237\345\256\211\350\243\205\344\275\277\347\224\250\346\226\207\346\241\243.md" "b/docs/zh_CN/ch2/DSS\345\277\253\351\200\237\345\256\211\350\243\205\344\275\277\347\224\250\346\226\207\346\241\243.md" index d3d96b212a..23c3008bc2 100644 --- "a/docs/zh_CN/ch2/DSS\345\277\253\351\200\237\345\256\211\350\243\205\344\275\277\347\224\250\346\226\207\346\241\243.md" +++ "b/docs/zh_CN/ch2/DSS\345\277\253\351\200\237\345\256\211\350\243\205\344\275\277\347\224\250\346\226\207\346\241\243.md" @@ -32,7 +32,7 @@ ## 二、精简版DSS环境配置准备 DSS环境配置准备分为三部分，前后端基础软件安装、后端环境配置准备和前端环配置境准备，详细介绍如下： ### 2.1 前后端基础软件安装 -Linkis简单版(0.9.1及以上)，[如何安装Linkis](https://github.com/WeBankFinTech/Linkis/wiki/%E5%A6%82%E4%BD%95%E5%BF%AB%E9%80%9F%E5%AE%89%E8%A3%85%E4%BD%BF%E7%94%A8Linkis) +Linkis简单版(0.9.3及以上)，[如何安装Linkis](https://github.com/WeBankFinTech/Linkis/wiki/%E5%A6%82%E4%BD%95%E5%BF%AB%E9%80%9F%E5%AE%89%E8%A3%85%E4%BD%BF%E7%94%A8Linkis) JDK (1.8.0_141以上)，[如何安装JDK](https://www.runoob.com/java/java-environment-setup.html) @@ -143,7 +143,7 @@ dss_ipaddr=$(ip addr | awk '/^[0-9]+: / {}; /inet.*global/ {print gensub(/(.*)\/ ## 三、简单版DSS环境配置准备 DSS环境配置准备分为三部分，前后端基础软件安装、后端环境配置准备和前端环配置境准备，详细介绍如下： ### 3.1 前后端基础软件安装 -Linkis简单版(0.9.1及以上)，[如何安装Linkis](https://github.com/WeBankFinTech/Linkis/wiki/%E5%A6%82%E4%BD%95%E5%BF%AB%E9%80%9F%E5%AE%89%E8%A3%85%E4%BD%BF%E7%94%A8Linkis) +Linkis简单版(0.9.3及以上)，[如何安装Linkis](https://github.com/WeBankFinTech/Linkis/wiki/%E5%A6%82%E4%BD%95%E5%BF%AB%E9%80%9F%E5%AE%89%E8%A3%85%E4%BD%BF%E7%94%A8Linkis) JDK (1.8.0_141以上)，[如何安装JDK](https://www.runoob.com/java/java-environment-setup.html) @@ -251,7 +251,7 @@ dss_ipaddr=$(ip addr | awk '/^[0-9]+: / {}; /inet.*global/ {print gensub(/(.*)\/ ## 四、标准版DSS环境配置准备 标准版DSS环境准备也分为三部分，前后端基础软件安装、后端环境准备和前端环境准备，详细介绍如下： ### 4.1 前后端基础软件安装 -Linkis简单版(0.9.1及以上)，[如何安装Linkis](https://github.com/WeBankFinTech/Linkis/wiki/%E5%A6%82%E4%BD%95%E5%BF%AB%E9%80%9F%E5%AE%89%E8%A3%85%E4%BD%BF%E7%94%A8Linkis) +Linkis简单版(0.9.3及以上)，[如何安装Linkis](https://github.com/WeBankFinTech/Linkis/wiki/%E5%A6%82%E4%BD%95%E5%BF%AB%E9%80%9F%E5%AE%89%E8%A3%85%E4%BD%BF%E7%94%A8Linkis) JDK (1.8.0_141以上)，[如何安装JDK](https://www.runoob.com/java/java-environment-setup.html) diff --git a/dss-appjoint-auth/pom.xml b/dss-appjoint-auth/pom.xml index 04144de5c6..595ad36f0b 100644 --- a/dss-appjoint-auth/pom.xml +++ b/dss-appjoint-auth/pom.xml @@ -33,6 +33,11 @@ linkis-gateway-httpclient-support ${linkis.version} + + com.webank.wedatasphere.linkis + linkis-common + ${linkis.version} + diff --git a/dss-flow-execution-entrance/pom.xml b/dss-flow-execution-entrance/pom.xml index 12218e310e..97d12b24be 100644 --- a/dss-flow-execution-entrance/pom.xml +++ b/dss-flow-execution-entrance/pom.xml @@ -33,12 +33,28 @@ com.webank.wedatasphere.linkis linkis-ujes-entrance ${linkis.version} + + + org.apache.poi + ooxml-schemas + + + + + com.webank.wedatasphere.linkis + linkis-cloudRPC + ${linkis.version} - com.webank.wedatasphere.dss dss-linkis-node-execution ${dss.version} + + + com.ibm.icu + icu4j + + diff --git a/dss-flow-execution-entrance/src/main/assembly/distribution.xml b/dss-flow-execution-entrance/src/main/assembly/distribution.xml index c080c0c09f..d59e84970f 100644 --- a/dss-flow-execution-entrance/src/main/assembly/distribution.xml +++ b/dss-flow-execution-entrance/src/main/assembly/distribution.xml @@ -84,7 +84,6 @@ com.google.code.gson:gson:jar com.google.guava:guava:jar com.google.inject:guice:jar - com.google.protobuf:protobuf-java:jar com.netflix.archaius:archaius-core:jar com.netflix.eureka:eureka-client:jar com.netflix.eureka:eureka-core:jar @@ -100,7 +99,6 @@ com.netflix.ribbon:ribbon-loadbalancer:jar com.netflix.ribbon:ribbon-transport:jar com.netflix.servo:servo-core:jar - com.ning:async-http-client:jar com.sun.jersey.contribs:jersey-apache-client4:jar com.sun.jersey:jersey-client:jar com.sun.jersey:jersey-core:jar @@ -113,15 +111,10 @@ com.webank.wedatasphere.linkis:linkis-common:jar com.webank.wedatasphere.linkis:linkis-module:jar commons-beanutils:commons-beanutils:jar - commons-beanutils:commons-beanutils-core:jar - commons-cli:commons-cli:jar commons-codec:commons-codec:jar commons-collections:commons-collections:jar commons-configuration:commons-configuration:jar - commons-daemon:commons-daemon:jar commons-dbcp:commons-dbcp:jar - commons-digester:commons-digester:jar - commons-httpclient:commons-httpclient:jar commons-io:commons-io:jar commons-jxpath:commons-jxpath:jar commons-lang:commons-lang:jar @@ -129,7 +122,6 @@ commons-net:commons-net:jar commons-pool:commons-pool:jar io.micrometer:micrometer-core:jar - io.netty:netty:jar io.netty:netty-all:jar io.netty:netty-buffer:jar io.netty:netty-codec:jar @@ -146,41 +138,21 @@ javax.annotation:javax.annotation-api:jar javax.inject:javax.inject:jar javax.servlet:javax.servlet-api:jar - javax.servlet.jsp:jsp-api:jar javax.validation:validation-api:jar javax.websocket:javax.websocket-api:jar javax.ws.rs:javax.ws.rs-api:jar javax.xml.bind:jaxb-api:jar javax.xml.stream:stax-api:jar joda-time:joda-time:jar - log4j:log4j:jar mysql:mysql-connector-java:jar - net.databinder.dispatch:dispatch-core_2.11:jar - net.databinder.dispatch:dispatch-json4s-jackson_2.11:jar org.antlr:antlr-runtime:jar org.antlr:stringtemplate:jar - org.apache.commons:commons-compress:jar org.apache.commons:commons-math:jar - org.apache.commons:commons-math3:jar - org.apache.curator:curator-client:jar - org.apache.curator:curator-framework:jar - org.apache.curator:curator-recipes:jar - org.apache.directory.api:api-asn1-api:jar - org.apache.directory.api:api-util:jar - org.apache.directory.server:apacheds-i18n:jar - org.apache.directory.server:apacheds-kerberos-codec:jar - org.apache.hadoop:hadoop-annotations:jar - org.apache.hadoop:hadoop-auth:jar - org.apache.hadoop:hadoop-common:jar - org.apache.hadoop:hadoop-hdfs:jar - org.apache.htrace:htrace-core:jar org.apache.httpcomponents:httpclient:jar - org.apache.httpcomponents:httpcore:jar org.apache.logging.log4j:log4j-api:jar org.apache.logging.log4j:log4j-core:jar org.apache.logging.log4j:log4j-jul:jar org.apache.logging.log4j:log4j-slf4j-impl:jar - org.apache.zookeeper:zookeeper:jar org.aspectj:aspectjweaver:jar org.bouncycastle:bcpkix-jdk15on:jar org.bouncycastle:bcprov-jdk15on:jar @@ -194,7 +166,6 @@ org.eclipse.jetty:jetty-continuation:jar org.eclipse.jetty:jetty-http:jar org.eclipse.jetty:jetty-io:jar - org.eclipse.jetty:jetty-jndi:jar org.eclipse.jetty:jetty-plus:jar org.eclipse.jetty:jetty-security:jar org.eclipse.jetty:jetty-server:jar @@ -210,7 +181,6 @@ org.eclipse.jetty.websocket:websocket-common:jar org.eclipse.jetty.websocket:websocket-server:jar org.eclipse.jetty.websocket:websocket-servlet:jar - org.fusesource.leveldbjni:leveldbjni-all:jar org.glassfish.hk2:class-model:jar org.glassfish.hk2:config-types:jar org.glassfish.hk2.external:aopalliance-repackaged:jar @@ -243,13 +213,10 @@ org.json4s:json4s-ast_2.11:jar org.json4s:json4s-core_2.11:jar org.json4s:json4s-jackson_2.11:jar - org.jsoup:jsoup:jar org.jvnet.mimepull:mimepull:jar org.jvnet:tiger-types:jar org.latencyutils:LatencyUtils:jar org.mortbay.jasper:apache-el:jar - org.mortbay.jetty:jetty:jar - org.mortbay.jetty:jetty-util:jar org.ow2.asm:asm-analysis:jar org.ow2.asm:asm-commons:jar org.ow2.asm:asm-tree:jar @@ -296,11 +263,8 @@ org.springframework:spring-jcl:jar org.springframework:spring-web:jar org.springframework:spring-webmvc:jar - org.tukaani:xz:jar org.yaml:snakeyaml:jar software.amazon.ion:ion-java:jar - xerces:xercesImpl:jar - xmlenc:xmlenc:jar xmlpull:xmlpull:jar xpp3:xpp3_min:jar diff --git a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionAppJointSignalSharedJob.java b/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionAppJointSignalSharedJob.java index 8d0735e581..6cfcf877be 100644 --- a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionAppJointSignalSharedJob.java +++ b/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionAppJointSignalSharedJob.java @@ -19,15 +19,27 @@ import com.webank.wedatasphere.dss.common.entity.node.DWSNode; import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration; +import com.webank.wedatasphere.dss.linkis.node.execution.job.JobSignalKeyCreator; import com.webank.wedatasphere.dss.linkis.node.execution.job.SignalSharedJob; import java.util.Map; /** - * Created by peacewong on 2019/11/14. + * Created by johnnwang on 2019/11/14. */ -public class FlowExecutionAppJointSignalSharedJob extends FlowExecutionAppJointLinkisSharedJob implements SignalSharedJob { +public class FlowExecutionAppJointSignalSharedJob extends FlowExecutionAppJointLinkisJob implements SignalSharedJob { + private JobSignalKeyCreator signalKeyCreator; + + @Override + public JobSignalKeyCreator getSignalKeyCreator() { + return this.signalKeyCreator; + } + + @Override + public void setSignalKeyCreator(JobSignalKeyCreator signalKeyCreator) { + this.signalKeyCreator = signalKeyCreator; + } @Override public String getMsgSaveKey() { diff --git a/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionJobSignalKeyCreator.java b/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionJobSignalKeyCreator.java new file mode 100644 index 0000000000..e284b44d8e --- /dev/null +++ b/dss-flow-execution-entrance/src/main/java/com/webank/wedatasphere/dss/flow/execution/entrance/job/FlowExecutionJobSignalKeyCreator.java @@ -0,0 +1,39 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.flow.execution.entrance.job; + +import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration; +import com.webank.wedatasphere.dss.linkis.node.execution.job.Job; +import com.webank.wedatasphere.dss.linkis.node.execution.job.JobSignalKeyCreator; +import com.webank.wedatasphere.dss.linkis.node.execution.job.SignalSharedJob; + +public class FlowExecutionJobSignalKeyCreator implements JobSignalKeyCreator { + + @Override + public String getSignalKeyByJob(Job job) { + String projectId = job.getJobProps().get(FlowExecutionEntranceConfiguration.PROJECT_NAME()); + String flowId = job.getJobProps().get(FlowExecutionEntranceConfiguration.FLOW_NAME()); + String flowExecId = job.getJobProps().get(FlowExecutionEntranceConfiguration.FLOW_EXEC_ID()); + return projectId + "." + flowId + "." + flowExecId; + } + + @Override + public String getSignalKeyBySignalSharedJob(SignalSharedJob job) { + return getSignalKeyByJob((Job)job); + } +} diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/conf/FlowExecutionEntranceConfiguration.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/conf/FlowExecutionEntranceConfiguration.scala index 91cdeedd6d..245a23687f 100644 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/conf/FlowExecutionEntranceConfiguration.scala +++ b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/conf/FlowExecutionEntranceConfiguration.scala @@ -44,9 +44,10 @@ object FlowExecutionEntranceConfiguration { val NODE_STATUS_POLLER_THREAD_SIZE = CommonVars("wds.dds.flow.node.status.poller.thread.size", 20) - val NODE_STATUS_POLLER_SCHEDULER_TIME = CommonVars("wds.dds.flow.node.status.poller.scheduler.time", 2) + val NODE_STATUS_POLLER_SCHEDULER_TIME = CommonVars("wds.dds.flow.node.status.poller.scheduler.time", 5) val FLOW_EXECUTION_SCHEDULER_POOL_SIZE = CommonVars("wds.linkis.flow.execution.pool.size", 30) + val NODE_STATUS_INTERVAL = CommonVars("wds.dds.flow.node.status.poller.interval.time", 3000) val COMMAND = "command" diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/DefaultFlowExecution.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/DefaultFlowExecution.scala index 04b56936e8..d5a9cbf797 100644 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/DefaultFlowExecution.scala +++ b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/execution/DefaultFlowExecution.scala @@ -16,7 +16,7 @@ */ package com.webank.wedatasphere.dss.flow.execution.entrance.execution -import java.util + import java.util.concurrent.{Executors, LinkedBlockingQueue, TimeUnit} import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration @@ -31,13 +31,11 @@ import scala.collection.mutable.ArrayBuffer /** - * Created by peacewong on 2019/11/5. - */ + * Created by johnnwang on 2019/11/5. + */ @Service class DefaultFlowExecution extends FlowExecution with Logging { - private val executeService = Utils.newCachedThreadPool(FlowExecutionEntranceConfiguration.FLOW_EXECUTION_POOL_SIZE.getValue, - "DefaultFlowExecution",true) private val nodeRunnerQueue: LinkedBlockingQueue[NodeRunner] = new LinkedBlockingQueue[NodeRunner]() @@ -74,6 +72,7 @@ class DefaultFlowExecution extends FlowExecution with Logging { if (pollerCount < FlowExecutionEntranceConfiguration.NODE_STATUS_POLLER_THREAD_SIZE.getValue){ scheduledThreadPool.scheduleAtFixedRate(new NodeExecutionStatusPoller(nodeRunnerQueue), 1, FlowExecutionEntranceConfiguration.NODE_STATUS_POLLER_SCHEDULER_TIME.getValue ,TimeUnit.SECONDS) + pollerCount = pollerCount + 1 } } } diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowJobNodeParser.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowJobNodeParser.scala index 158a8aebde..6447e45eb2 100644 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowJobNodeParser.scala +++ b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/job/parser/FlowJobNodeParser.scala @@ -19,6 +19,7 @@ package com.webank.wedatasphere.dss.flow.execution.entrance.job.parser import java.util +import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration import com.webank.wedatasphere.dss.flow.execution.entrance.conf.FlowExecutionEntranceConfiguration._ import com.webank.wedatasphere.dss.flow.execution.entrance.exception.FlowExecutionErrorException import com.webank.wedatasphere.dss.flow.execution.entrance.job.FlowEntranceJob @@ -34,8 +35,8 @@ import org.springframework.core.annotation.Order import org.springframework.stereotype.Component /** - * Created by peacewong on 2019/11/6. - */ + * Created by johnnwang on 2019/11/6. + */ @Order(2) @Component @@ -76,6 +77,8 @@ class FlowJobNodeParser extends FlowEntranceJobParser with Logging{ } } + propsMap.put(FlowExecutionEntranceConfiguration.FLOW_EXEC_ID, flowEntranceJob.getId) + params.put(PROPS_MAP, propsMap) params.put(FLOW_VAR_MAP, flowVar) params.put(PROJECT_RESOURCES, project.getProjectResources) diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/AppJointJobBuilder.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/AppJointJobBuilder.scala index 02505695a8..440e6a4aa4 100644 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/AppJointJobBuilder.scala +++ b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/AppJointJobBuilder.scala @@ -26,13 +26,28 @@ import com.webank.wedatasphere.dss.flow.execution.entrance.job._ import com.webank.wedatasphere.dss.flow.execution.entrance.utils.FlowExecutionUtils import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration import com.webank.wedatasphere.dss.linkis.node.execution.entity.BMLResource +import com.webank.wedatasphere.dss.linkis.node.execution.execution.impl.LinkisNodeExecutionImpl import com.webank.wedatasphere.dss.linkis.node.execution.job._ +import com.webank.wedatasphere.dss.linkis.node.execution.parser.JobParamsParser import org.apache.commons.lang.StringUtils /** - * Created by peacewong on 2019/11/5. - */ + * Created by johnnwang on 2019/11/5. + */ object AppJointJobBuilder { + + val signalKeyCreator = new FlowExecutionJobSignalKeyCreator + + init() + + def init(): Unit ={ + val jobParamsParser = new JobParamsParser + + jobParamsParser.setSignalKeyCreator(signalKeyCreator) + + LinkisNodeExecutionImpl.getLinkisNodeExecution.asInstanceOf[LinkisNodeExecutionImpl].registerJobParser(jobParamsParser) + } + def builder():FlowBuilder = new FlowBuilder class FlowBuilder extends Builder { @@ -95,9 +110,10 @@ object AppJointJobBuilder { override protected def createSignalSharedJob(isLinkisType: Boolean): SignalSharedJob = { if(isLinkisType){ - null + null } else { val signalJob = new FlowExecutionAppJointSignalSharedJob + signalJob.setSignalKeyCreator(signalKeyCreator) signalJob.setJobProps(this.jobProps) signalJob } diff --git a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/DefaultNodeRunner.scala b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/DefaultNodeRunner.scala index a9462dda8a..89ddc910a4 100644 --- a/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/DefaultNodeRunner.scala +++ b/dss-flow-execution-entrance/src/main/scala/com/webank/wedatasphere/dss/flow/execution/entrance/node/DefaultNodeRunner.scala @@ -27,11 +27,12 @@ import com.webank.wedatasphere.dss.flow.execution.entrance.log.FlowExecutionLog import com.webank.wedatasphere.dss.flow.execution.entrance.node.NodeExecutionState.NodeExecutionState import com.webank.wedatasphere.dss.linkis.node.execution.execution.impl.LinkisNodeExecutionImpl import com.webank.wedatasphere.dss.linkis.node.execution.listener.LinkisExecutionListener +import com.webank.wedatasphere.linkis.common.exception.ErrorException import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} /** - * Created by peacewong on 2019/11/5. - */ + * Created by johnnwang on 2019/11/5. + */ class DefaultNodeRunner extends NodeRunner with Logging { private var node: SchedulerNode = _ @@ -48,6 +49,8 @@ class DefaultNodeRunner extends NodeRunner with Logging { private var startTime: Long = _ + private var lastGetStatusTime: Long = 0 + override def getNode: SchedulerNode = this.node def setNode(schedulerNode: SchedulerNode): Unit = { @@ -64,6 +67,15 @@ class DefaultNodeRunner extends NodeRunner with Logging { } override def isLinkisJobCompleted: Boolean = Utils.tryCatch{ + + val interval = System.currentTimeMillis() - lastGetStatusTime + + if ( interval < FlowExecutionEntranceConfiguration.NODE_STATUS_INTERVAL.getValue){ + return false + } + + lastGetStatusTime = System.currentTimeMillis() + if(NodeExecutionState.isCompleted(getStatus)) return true val toState = NodeExecutionState.withName(LinkisNodeExecutionImpl.getLinkisNodeExecution.getState(this.linkisJob)) if (NodeExecutionState.isCompleted(toState)) { @@ -75,9 +87,11 @@ class DefaultNodeRunner extends NodeRunner with Logging { } else { false } - }{ t => - warn(s"Failed to get ${this.node.getName} linkis job states", t) - false + }{ + case e:ErrorException => logger.warn(s"failed to get ${this.node.getName} state", e) + false + case t :Throwable => logger.error(s"failed to get ${this.node.getName} state", t) + true } override def setNodeRunnerListener(nodeRunnerListener: NodeRunnerListener): Unit = this.nodeRunnerListener = nodeRunnerListener @@ -102,7 +116,7 @@ class DefaultNodeRunner extends NodeRunner with Logging { } LinkisNodeExecutionImpl.getLinkisNodeExecution.runJob(this.linkisJob) - info(s"Finished to run node of ${node.getName}") + info(s"start to run node of ${node.getName}") /*LinkisNodeExecutionImpl.getLinkisNodeExecution.waitForComplete(this.linkisJob) val listener = LinkisNodeExecutionImpl.getLinkisNodeExecution.asInstanceOf[LinkisExecutionListener] val toState = LinkisNodeExecutionImpl.getLinkisNodeExecution.getState(this.linkisJob) diff --git a/dss-linkis-node-execution/pom.xml b/dss-linkis-node-execution/pom.xml index ecea3e8bff..b98c33987a 100644 --- a/dss-linkis-node-execution/pom.xml +++ b/dss-linkis-node-execution/pom.xml @@ -33,13 +33,13 @@ com.webank.wedatasphere.linkis linkis-ujes-client - 0.9.1 + 0.9.3 com.webank.wedatasphere.linkis linkis-workspace-httpclient - 0.9.1 + 0.9.3 diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/WorkflowContextImpl.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/WorkflowContextImpl.java index e928813569..8f2ff84fac 100644 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/WorkflowContextImpl.java +++ b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/WorkflowContextImpl.java @@ -21,12 +21,13 @@ import com.google.common.cache.CacheBuilder; import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration; import com.webank.wedatasphere.dss.linkis.node.execution.entity.ContextInfo; +import org.apache.commons.lang.StringUtils; import java.util.*; import java.util.concurrent.TimeUnit; /** - * Created by peacewong on 2019/9/26. + * Created by johnnwang on 2019/9/26. */ public class WorkflowContextImpl implements WorkflowContext { @@ -79,7 +80,7 @@ public Map getSubMapByPrefix(String keyPrefix) { while (keys.hasNext()) { String key = keys.next(); if (key.startsWith(keyPrefix)) { - map.put(key, getValue(key)); + map.put(StringUtils.substringAfter(key, keyPrefix), getValue(key)); } } return map; diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/conf/LinkisJobExecutionConfiguration.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/conf/LinkisJobExecutionConfiguration.java index 852262bb57..9b75d73a01 100644 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/conf/LinkisJobExecutionConfiguration.java +++ b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/conf/LinkisJobExecutionConfiguration.java @@ -68,7 +68,7 @@ public class LinkisJobExecutionConfiguration { public final static CommonVars LINKIS_CONNECTION_TIMEOUT = CommonVars.apply("wds.linkis.flow.connection.timeout",30000); - public final static CommonVars LINKIS_JOB_REQUEST_STATUS_TIME = CommonVars.apply("wds.linkis.flow.connection.timeout",1000); + public final static CommonVars LINKIS_JOB_REQUEST_STATUS_TIME = CommonVars.apply("wds.linkis.flow.connection.timeout",3000); public final static CommonVars LINKIS_ADMIN_USER = CommonVars.apply("wds.linkis.client.flow.adminuser","ws"); diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/execution/impl/LinkisNodeExecutionImpl.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/execution/impl/LinkisNodeExecutionImpl.java index 635ce6ac03..264e8288c4 100644 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/execution/impl/LinkisNodeExecutionImpl.java +++ b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/execution/impl/LinkisNodeExecutionImpl.java @@ -23,7 +23,6 @@ import com.webank.wedatasphere.dss.linkis.node.execution.job.SharedJob; import com.webank.wedatasphere.dss.linkis.node.execution.job.SignalSharedJob; import com.webank.wedatasphere.dss.linkis.node.execution.listener.LinkisExecutionListener; -import com.webank.wedatasphere.dss.linkis.node.execution.parser.JobParamsParser; import com.webank.wedatasphere.dss.linkis.node.execution.parser.JobRuntimeParamsParser; import com.webank.wedatasphere.dss.linkis.node.execution.service.impl.BuildJobActionImpl; import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration; @@ -51,7 +50,7 @@ public class LinkisNodeExecutionImpl implements LinkisNodeExecution , LinkisExec private LinkisNodeExecutionImpl() { registerJobParser(new CodeParser()); - registerJobParser(new JobParamsParser()); + /*registerJobParser(new JobParamsParser());*/ registerJobParser(new JobRuntimeParamsParser()); } @@ -107,7 +106,7 @@ public String getLog(Job job) { job.getLogFromLine(), LinkisJobExecutionConfiguration.LOG_SIZE.getValue()); - job.setLogFromLint(jobLogResult.fromLine()); + job.setLogFromLine(jobLogResult.fromLine()); ArrayList logArray = jobLogResult.getLog(); @@ -191,12 +190,7 @@ public void onStatusChanged(String fromState, String toState, Job job) { if (job instanceof SignalSharedJob){ SignalSharedJob signalSharedJob = (SignalSharedJob) job; String result = getResult(job, 0, -1); - String msgSaveKey = signalSharedJob.getMsgSaveKey(); - String key = SignalSharedJob.PREFIX ; - if (StringUtils.isNotEmpty(msgSaveKey)){ - key = key + msgSaveKey; - } - WorkflowContext.getAppJointContext().setValue(key, result , -1); + WorkflowContext.getAppJointContext().setValue(signalSharedJob.getSharedKey(), result , -1); } else if(job instanceof SharedJob){ String taskId = job.getJobExecuteResult().getTaskID(); job.getLogObj().info("Set shared info:" + taskId); diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AbstractAppJointLinkisJob.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AbstractAppJointLinkisJob.java index 1bce73c8af..7d9e47f069 100644 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AbstractAppJointLinkisJob.java +++ b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AbstractAppJointLinkisJob.java @@ -146,7 +146,7 @@ public int getLogFromLine() { } @Override - public void setLogFromLint(int index) { + public void setLogFromLine(int index) { this.logFromLine = index; } diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AbstractCommonLinkisJob.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AbstractCommonLinkisJob.java index 2d836eaacc..c7f74f2348 100644 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AbstractCommonLinkisJob.java +++ b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/AbstractCommonLinkisJob.java @@ -154,7 +154,7 @@ public int getLogFromLine() { } @Override - public void setLogFromLint(int index) { + public void setLogFromLine(int index) { this.logFromLine = index; } diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/Job.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/Job.java index ac25c724b4..1811ad105e 100644 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/Job.java +++ b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/Job.java @@ -63,6 +63,6 @@ public interface Job { int getLogFromLine(); - void setLogFromLint(int index); + void setLogFromLine(int index); } diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/JobSignalKeyCreator.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/JobSignalKeyCreator.java new file mode 100644 index 0000000000..ce64bf9017 --- /dev/null +++ b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/JobSignalKeyCreator.java @@ -0,0 +1,25 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.linkis.node.execution.job; + +public interface JobSignalKeyCreator { + + String getSignalKeyByJob(Job job); + + String getSignalKeyBySignalSharedJob(SignalSharedJob job); +} diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/SignalSharedJob.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/SignalSharedJob.java index a71a12555a..13a9b8b444 100644 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/SignalSharedJob.java +++ b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/job/SignalSharedJob.java @@ -24,6 +24,10 @@ public interface SignalSharedJob extends SharedJob { String PREFIX = "signal."; + JobSignalKeyCreator getSignalKeyCreator(); + + void setSignalKeyCreator(JobSignalKeyCreator signalKeyCreator); + @Override default int getSharedNum() { return -1; @@ -31,7 +35,7 @@ default int getSharedNum() { @Override default String getSharedKey() { - return PREFIX + getMsgSaveKey(); + return PREFIX + getSignalKeyCreator().getSignalKeyBySignalSharedJob(this) + "." + getMsgSaveKey(); } String getMsgSaveKey(); diff --git a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/JobParamsParser.java b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/JobParamsParser.java index fca769d8a6..5c9e013c6b 100644 --- a/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/JobParamsParser.java +++ b/dss-linkis-node-execution/src/main/java/com/webank/wedatasphere/dss/linkis/node/execution/parser/JobParamsParser.java @@ -19,19 +19,31 @@ import com.google.gson.reflect.TypeToken; import com.webank.wedatasphere.dss.linkis.node.execution.WorkflowContext; +import com.webank.wedatasphere.dss.linkis.node.execution.job.JobSignalKeyCreator; import com.webank.wedatasphere.dss.linkis.node.execution.job.LinkisJob; import com.webank.wedatasphere.dss.linkis.node.execution.job.Job; import com.webank.wedatasphere.dss.linkis.node.execution.job.SignalSharedJob; import com.webank.wedatasphere.dss.linkis.node.execution.utils.LinkisJobExecutionUtils; +import java.util.Collection; import java.util.List; import java.util.Map; /** - * Created by peacewong on 2019/11/3. + * Created by johnnwang on 2019/11/3. */ public class JobParamsParser implements JobParser { + private JobSignalKeyCreator signalKeyCreator; + + public JobSignalKeyCreator getSignalKeyCreator() { + return signalKeyCreator; + } + + public void setSignalKeyCreator(JobSignalKeyCreator signalKeyCreator) { + this.signalKeyCreator = signalKeyCreator; + } + @Override public void parseJob(Job job) throws Exception { @@ -44,11 +56,15 @@ public void parseJob(Job job) throws Exception { Map flowVariables = linkisJob.getVariables(); putParamsMap(job.getParams(), "variable", flowVariables); //put signal info - Map sharedValue = WorkflowContext.getAppJointContext().getSubMapByPrefix(SignalSharedJob.PREFIX); + Map sharedValue = WorkflowContext.getAppJointContext() + .getSubMapByPrefix(SignalSharedJob.PREFIX + this.getSignalKeyCreator().getSignalKeyByJob(job)); if (sharedValue != null) { - putParamsMap(job.getParams(), "variable", sharedValue); + Collection values = sharedValue.values(); + for(Object value : values){ + Map variableMap = LinkisJobExecutionUtils.gson.fromJson(value.toString(), new TypeToken>() {}.getType()); + putParamsMap(job.getParams(), "variable", variableMap); + } } - // put configuration Map configuration = linkisJob.getConfiguration(); putParamsMap(job.getParams(), "configuration", configuration); diff --git a/dss-scheduler-appjoint-core/pom.xml b/dss-scheduler-appjoint-core/pom.xml index 1eaba100fd..b09213fc83 100644 --- a/dss-scheduler-appjoint-core/pom.xml +++ b/dss-scheduler-appjoint-core/pom.xml @@ -48,6 +48,11 @@ ${dss.version} + + com.google.code.gson + gson + 2.8.5 + diff --git a/dss-server/pom.xml b/dss-server/pom.xml index f42dbb0a44..b0627677bc 100644 --- a/dss-server/pom.xml +++ b/dss-server/pom.xml @@ -117,7 +117,7 @@ com.webank.wedatasphere.linkis - 0.9.1 + 0.9.3 diff --git a/dss-server/src/main/assembly/distribution.xml b/dss-server/src/main/assembly/distribution.xml index ffa6656d8c..560fc34e34 100644 --- a/dss-server/src/main/assembly/distribution.xml +++ b/dss-server/src/main/assembly/distribution.xml @@ -176,7 +176,9 @@ org.eclipse.jetty:jetty-continuation:jar org.eclipse.jetty:jetty-http:jar org.eclipse.jetty:jetty-io:jar + org.eclipse.jetty:jetty-plus:jar org.eclipse.jetty:jetty-security:jar org.eclipse.jetty:jetty-server:jar @@ -225,7 +227,9 @@ org.json4s:json4s-ast_2.11:jar org.json4s:json4s-core_2.11:jar org.json4s:json4s-jackson_2.11:jar + org.jvnet.mimepull:mimepull:jar org.jvnet:tiger-types:jar org.latencyutils:LatencyUtils:jar diff --git a/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/job/AzkabanAppJointSignalSharedJob.java b/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/job/AzkabanAppJointSignalSharedJob.java index 01ebbf0b69..a24adcb69d 100644 --- a/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/job/AzkabanAppJointSignalSharedJob.java +++ b/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/job/AzkabanAppJointSignalSharedJob.java @@ -17,17 +17,30 @@ package com.webank.wedatasphere.dss.plugins.azkaban.linkis.jobtype.job; +import com.webank.wedatasphere.dss.linkis.node.execution.job.JobSignalKeyCreator; import com.webank.wedatasphere.dss.linkis.node.execution.job.SignalSharedJob; import com.webank.wedatasphere.dss.plugins.azkaban.linkis.jobtype.conf.LinkisJobTypeConf; import java.util.Map; /** - * Created by peacewong on 2019/11/14. + * Created by johnnwang on 2019/11/14. */ -public class AzkabanAppJointSignalSharedJob extends AzkabanAppJointLinkisSharedJob implements SignalSharedJob { +public class AzkabanAppJointSignalSharedJob extends AzkabanAppJointLinkisJob implements SignalSharedJob { + private JobSignalKeyCreator signalKeyCreator; + + @Override + public JobSignalKeyCreator getSignalKeyCreator() { + return this.signalKeyCreator; + } + + @Override + public void setSignalKeyCreator(JobSignalKeyCreator signalKeyCreator) { + this.signalKeyCreator = signalKeyCreator; + } + @Override public String getMsgSaveKey() { Map configuration = this.getConfiguration(); diff --git a/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/job/AzkabanJobSignalKeyCreator.java b/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/job/AzkabanJobSignalKeyCreator.java new file mode 100644 index 0000000000..30c6a6415f --- /dev/null +++ b/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/job/AzkabanJobSignalKeyCreator.java @@ -0,0 +1,39 @@ +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.dss.plugins.azkaban.linkis.jobtype.job; + +import com.webank.wedatasphere.dss.linkis.node.execution.job.Job; +import com.webank.wedatasphere.dss.linkis.node.execution.job.JobSignalKeyCreator; +import com.webank.wedatasphere.dss.linkis.node.execution.job.SignalSharedJob; +import com.webank.wedatasphere.dss.plugins.azkaban.linkis.jobtype.conf.LinkisJobTypeConf; + +public class AzkabanJobSignalKeyCreator implements JobSignalKeyCreator { + + @Override + public String getSignalKeyByJob(Job job) { + String projectId = job.getJobProps().get(LinkisJobTypeConf.PROJECT_ID); + String flowId = job.getJobProps().get(LinkisJobTypeConf.FLOW_NAME); + String flowExecId = job.getJobProps().get(LinkisJobTypeConf.FLOW_EXEC_ID); + return projectId + "." + flowId + "." + flowExecId ; + } + + @Override + public String getSignalKeyBySignalSharedJob(SignalSharedJob job) { + return getSignalKeyByJob((Job)job); + } +} diff --git a/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/job/AzkanbanBuilder.java b/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/job/AzkanbanBuilder.java index bcd88097bd..c38fac2440 100644 --- a/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/job/AzkanbanBuilder.java +++ b/plugins/azkaban/linkis-jobtype/src/main/java/com/webank/wedatasphere/dss/plugins/azkaban/linkis/jobtype/job/AzkanbanBuilder.java @@ -20,7 +20,10 @@ import com.webank.wedatasphere.dss.linkis.node.execution.conf.LinkisJobExecutionConfiguration; import com.webank.wedatasphere.dss.linkis.node.execution.entity.BMLResource; import com.webank.wedatasphere.dss.linkis.node.execution.exception.LinkisJobExecutionErrorException; +import com.webank.wedatasphere.dss.linkis.node.execution.execution.LinkisNodeExecution; +import com.webank.wedatasphere.dss.linkis.node.execution.execution.impl.LinkisNodeExecutionImpl; import com.webank.wedatasphere.dss.linkis.node.execution.job.*; +import com.webank.wedatasphere.dss.linkis.node.execution.parser.JobParamsParser; import com.webank.wedatasphere.dss.linkis.node.execution.utils.LinkisJobExecutionUtils; import com.webank.wedatasphere.dss.plugins.azkaban.linkis.jobtype.conf.LinkisJobTypeConf; import com.webank.wedatasphere.dss.plugins.azkaban.linkis.jobtype.utils.LinkisJobTypeUtils; @@ -29,17 +32,29 @@ import java.util.*; /** - * Created by peacewong on 2019/11/3. + * Created by johnnwang on 2019/11/3. */ public class AzkanbanBuilder extends Builder{ private Map jobProps; + private JobSignalKeyCreator jobSignalKeyCreator = new AzkabanJobSignalKeyCreator(); + public AzkanbanBuilder setJobProps(Map jobProps) { this.jobProps = jobProps; return this; } + { + init(); + } + + private void init(){ + JobParamsParser jobParamsParser = new JobParamsParser(); + jobParamsParser.setSignalKeyCreator(jobSignalKeyCreator); + LinkisNodeExecutionImpl linkisNodeExecution = (LinkisNodeExecutionImpl)LinkisNodeExecutionImpl.getLinkisNodeExecution(); + linkisNodeExecution.registerJobParser(jobParamsParser); + } @Override protected String getJobType() { @@ -99,6 +114,7 @@ protected SignalSharedJob createSignalSharedJob(boolean isLinkisType) { return null; } else { AzkabanAppJointSignalSharedJob signalSharedJob = new AzkabanAppJointSignalSharedJob(); + signalSharedJob.setSignalKeyCreator(jobSignalKeyCreator); signalSharedJob.setJobProps(this.jobProps); return signalSharedJob; } diff --git a/plugins/linkis/linkis-appjoint-entrance/pom.xml b/plugins/linkis/linkis-appjoint-entrance/pom.xml index 26b80a750c..0651c396ef 100644 --- a/plugins/linkis/linkis-appjoint-entrance/pom.xml +++ b/plugins/linkis/linkis-appjoint-entrance/pom.xml @@ -54,8 +54,21 @@ - - + + com.webank.wedatasphere.linkis + linkis-cloudRPC + ${linkis.version} + + + com.webank.wedatasphere.linkis + linkis-storage + ${linkis.version} + + + com.webank.wedatasphere.linkis + linkis-httpclient + ${linkis.version} + diff --git a/plugins/linkis/linkis-appjoint-entrance/src/main/assembly/distribution.xml b/plugins/linkis/linkis-appjoint-entrance/src/main/assembly/distribution.xml index d4a200502a..83c6ee1a07 100644 --- a/plugins/linkis/linkis-appjoint-entrance/src/main/assembly/distribution.xml +++ b/plugins/linkis/linkis-appjoint-entrance/src/main/assembly/distribution.xml @@ -84,7 +84,7 @@ com.google.code.gson:gson:jar com.google.guava:guava:jar com.google.inject:guice:jar - com.google.protobuf:protobuf-java:jar + com.netflix.archaius:archaius-core:jar com.netflix.eureka:eureka-client:jar com.netflix.eureka:eureka-core:jar @@ -100,7 +100,6 @@ com.netflix.ribbon:ribbon-loadbalancer:jar com.netflix.ribbon:ribbon-transport:jar com.netflix.servo:servo-core:jar - com.ning:async-http-client:jar com.sun.jersey.contribs:jersey-apache-client4:jar com.sun.jersey:jersey-client:jar com.sun.jersey:jersey-core:jar @@ -112,15 +111,15 @@ com.thoughtworks.xstream:xstream:jar com.webank.wedatasphere.linkis:linkis-common:jar com.webank.wedatasphere.linkis:linkis-module:jar - commons-beanutils:commons-beanutils:jar - commons-beanutils:commons-beanutils-core:jar - commons-cli:commons-cli:jar - commons-codec:commons-codec:jar - commons-collections:commons-collections:jar - commons-configuration:commons-configuration:jar - commons-daemon:commons-daemon:jar - commons-dbcp:commons-dbcp:jar - commons-digester:commons-digester:jar + + + + + + + + + commons-httpclient:commons-httpclient:jar commons-io:commons-io:jar commons-jxpath:commons-jxpath:jar @@ -155,11 +154,9 @@ joda-time:joda-time:jar log4j:log4j:jar mysql:mysql-connector-java:jar - net.databinder.dispatch:dispatch-core_2.11:jar - net.databinder.dispatch:dispatch-json4s-jackson_2.11:jar org.antlr:antlr-runtime:jar org.antlr:stringtemplate:jar - org.apache.commons:commons-compress:jar + org.apache.commons:commons-math:jar org.apache.commons:commons-math3:jar org.apache.curator:curator-client:jar @@ -169,13 +166,13 @@ org.apache.directory.api:api-util:jar org.apache.directory.server:apacheds-i18n:jar org.apache.directory.server:apacheds-kerberos-codec:jar - org.apache.hadoop:hadoop-annotations:jar - org.apache.hadoop:hadoop-auth:jar - org.apache.hadoop:hadoop-common:jar - org.apache.hadoop:hadoop-hdfs:jar - org.apache.htrace:htrace-core:jar - org.apache.httpcomponents:httpclient:jar - org.apache.httpcomponents:httpcore:jar + + + + + + + org.apache.logging.log4j:log4j-api:jar org.apache.logging.log4j:log4j-core:jar org.apache.logging.log4j:log4j-jul:jar @@ -194,7 +191,6 @@ org.eclipse.jetty:jetty-continuation:jar org.eclipse.jetty:jetty-http:jar org.eclipse.jetty:jetty-io:jar - org.eclipse.jetty:jetty-jndi:jar org.eclipse.jetty:jetty-plus:jar org.eclipse.jetty:jetty-security:jar org.eclipse.jetty:jetty-server:jar @@ -243,7 +239,6 @@ org.json4s:json4s-ast_2.11:jar org.json4s:json4s-core_2.11:jar org.json4s:json4s-jackson_2.11:jar - org.jsoup:jsoup:jar org.jvnet.mimepull:mimepull:jar org.jvnet:tiger-types:jar org.latencyutils:LatencyUtils:jar diff --git a/pom.xml b/pom.xml index c93448dc71..76737321d2 100644 --- a/pom.xml +++ b/pom.xml @@ -48,7 +48,7 @@ 0.7.0 - 0.9.1 + 0.9.3 2.11.8 1.8 3.3.3 diff --git a/visualis-appjoint/appjoint/pom.xml b/visualis-appjoint/appjoint/pom.xml index 9c6016de16..ab48a6148b 100644 --- a/visualis-appjoint/appjoint/pom.xml +++ b/visualis-appjoint/appjoint/pom.xml @@ -45,6 +45,14 @@ true + + com.webank.wedatasphere.linkis + linkis-httpclient + ${linkis.version} + provided + true + + diff --git a/visualis-appjoint/appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/visualis/execution/VisualisNodeExecution.scala b/visualis-appjoint/appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/visualis/execution/VisualisNodeExecution.scala index 9de32921cf..fa25757263 100644 --- a/visualis-appjoint/appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/visualis/execution/VisualisNodeExecution.scala +++ b/visualis-appjoint/appjoint/src/main/scala/com/webank/wedatasphere/dss/appjoint/visualis/execution/VisualisNodeExecution.scala @@ -17,7 +17,7 @@ package com.webank.wedatasphere.dss.appjoint.visualis.execution -import java.io.ByteArrayOutputStream +import java.io.{ByteArrayOutputStream, InputStream} import java.util import java.util.Base64 @@ -28,25 +28,34 @@ import com.webank.wedatasphere.dss.appjoint.service.session.Session import com.webank.wedatasphere.dss.appjoint.visualis.execution.VisualisNodeExecutionConfiguration._ import com.webank.wedatasphere.linkis.common.exception.ErrorException import com.webank.wedatasphere.linkis.common.log.LogUtils -import com.webank.wedatasphere.linkis.common.utils.{HttpClient, Logging, Utils} +import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord} import org.apache.commons.io.IOUtils import scala.collection.JavaConversions.mapAsScalaMap -import scala.concurrent.ExecutionContext +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, ExecutionContext} +import dispatch._ +import org.json4s.{DefaultFormats, Formats} /** * Created by enjoyyin on 2019/10/12. */ -class VisualisNodeExecution extends NodeExecution with HttpClient with Logging { - - override protected implicit val executors: ExecutionContext = Utils.newCachedExecutionContext(VISUALIS_THREAD_MAX.getValue, getName + "-NodeExecution-Thread", true) +class VisualisNodeExecution extends NodeExecution with Logging { private val DISPLAY = "display" private val DASHBOARD = "dashboard" var basicUrl:String = _ + protected implicit val executors: ExecutionContext = Utils.newCachedExecutionContext(VISUALIS_THREAD_MAX.getValue, getName + "-NodeExecution-Thread", true) + protected implicit val formats: Formats = DefaultFormats + + private implicit def svc(url: String): Req = + dispatch.url(url) + + + override def getBaseUrl: String = this.basicUrl override def setBaseUrl(basicUrl: String): Unit = this.basicUrl = basicUrl @@ -95,6 +104,17 @@ class VisualisNodeExecution extends NodeExecution with HttpClient with Logging { appJointResponse } + def download(url: String, queryParams: Map[String, String], headerParams: Map[String, String], + write: InputStream => Unit, + paths: String*): Unit = { + var req = url.GET + if(headerParams != null && headerParams.nonEmpty) req = req <:< headerParams + if(queryParams != null) queryParams.foreach{ case (k, v) => req = req.addQueryParameter(k, v)} + if(paths != null) paths.filter(_ != null).foreach(p => req = req / p) + val response = Http(req OK as.Response(_.getResponseBodyAsStream)).map(write) + Await.result(response, Duration.Inf) + } + private def getRealId(displayId:String):Int = { Utils.tryCatch{ val f = java.lang.Float.parseFloat(displayId)