Skip to content

Commit

Permalink
Merge pull request #4 from WeBankFinTech/master
Browse files Browse the repository at this point in the history 
拉代码
  • Loading branch information
@zwx-master
zwx-master authored Jan 7, 2020
2 parents 5d03317 + a6b036d commit e79e5ad
Show file tree
Hide file tree
Showing 36 changed files with 410 additions and 218 deletions.
11 changes: 11 additions & 0 deletions README-ZH.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,17 @@ DSS主要特点：

           **根据需要，用户可以简单快速替换DSS已集成的各种功能组件，或新增功能组件。**

## Demo试用环境

       由于DataSphereStudio支持执行脚本风险较高，WeDataSphere Demo环境的隔离没有做完，考虑到大家都在咨询Demo环境，决定向社区先定向发放邀请码，接受企业和组织的试用申请。

       如果您想试用Demo环境，请加入DataSphere Studio社区用户群（**加群方式请翻到本文档末尾处**），联系团队成员获取邀请码。

       WeDataSphere Demo环境用户注册页面：https://sandbox.webank.com/wds/dss/#/register

       WeDataSphere Demo环境登录页面：https://sandbox.webank.com/wds/dss/

       我们会尽快解决环境隔离问题，争取早日向社区完全开放WeDataSphere Demo环境。

## 与类似系统对比

Expand Down
2 changes: 1 addition & 1 deletion assembly/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<parent>
<artifactId>dss</artifactId>
<groupId>com.webank.wedatasphere.dss</groupId>
<version>0.6.0</version>
<version>0.7.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
5 changes: 2 additions & 3 deletions bin/checkEnv.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,7 @@ need_cmd mysql
need_cmd unzip
need_cmd expect
need_cmd telnet
need_cmd sed
need_cmd tar
need_cmd source
need_cmd hostname
need_cmd sed
need_cmd dos2unix
echo "<-----end to check linux cmd:yum java mysql unzip expect telnet sed tar------>"
50 changes: 50 additions & 0 deletions bin/checkMicro.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
#
# Copyright 2019 WeBank
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#!/bin/sh
source ~/.bash_profile

shellDir=`dirname $0`
workDir=`cd ${shellDir}/..;pwd`

##load config
source ${workDir}/conf/config.sh
source ${workDir}/conf/db.sh

MICRO_SERVICE_NAME=$1
MICRO_SERVICE_IP=$2
MICRO_SERVICE_PORT=$3
echo "<--------------------------------------------------------------------------->"
echo "Start to Check if your microservice:$MICRO_SERVICE_NAME is normal via telnet"
echo ""
if [ ! -d $DSS_INSTALL_HOME/$MICRO_SERVICE_NAME ];then
echo "$MICRO_SERVICE_NAME is not installed,the check steps will be skipped"
exit 0
fi

result=`echo -e "\n" | telnet $MICRO_SERVICE_IP $MICRO_SERVICE_PORT 2>/dev/null | grep Connected | wc -l`
if [ $result -eq 1 ]; then
echo "$MICRO_SERVICE_NAME is ok."
else
echo "<--------------------------------------------------------------------------->"
echo "ERROR your $MICRO_SERVICE_NAME microservice is not start successful !!! ERROR logs as follows :"
echo "PLEAESE CHECK DETAIL LOG,LOCATION:$DSS_INSTALL_HOME/$MICRO_SERVICE_NAME/logs/linkis.out"
echo '<------------------------------------------------------------->'
tail -n 50 $DSS_INSTALL_HOME/$MICRO_SERVICE_NAME/logs/*.out
echo '<-------------------------------------------------------------->'
echo "PLEAESE CHECK DETAIL LOG,LOCATION:$DSS_INSTALL_HOME/$MICRO_SERVICE_NAME/logs/linkis.out"
exit 1
fi

99 changes: 45 additions & 54 deletions bin/install.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@ elif [[ "$OSTYPE" == "win32" ]]; then
echo "dss not support Windows operating system"
exit 1
elif [[ "$OSTYPE" == "freebsd"* ]]; then

txt=""
else
echo "Operating system unknown, please tell us(submit issue) for better service"
Expand All @@ -46,14 +45,13 @@ fi

function isSuccess(){
if [ $? -ne 0 ]; then
echo "Failed to " + $1
echo "ERROR to " + $1
exit 1
else
echo "Succeed to" + $1
echo "SUCESS to" + $1
fi
}


#check env
sh ${workDir}/bin/checkEnv.sh
isSuccess "check env"
Expand All @@ -74,63 +72,46 @@ else
fi
}

function checkPythonAndJava(){
python --version
isSuccess "execute python --version"
java -version
isSuccess "execute java --version"
}

function checkHadoopAndHive(){
hdfs version
isSuccess "execute hdfs version"
hive --help
#isSuccess "execute hive -h"
}

function checkSpark(){
spark-submit --version
isSuccess "execute spark-submit --version"
}

##install env:expect,
sudo yum install -y expect
isSuccess "install expect"

##install env:telnet,
sudo yum install -y telnet
isSuccess "install telnet"

##load config
echo "step1:load config"
source ${workDir}/conf/config.sh
source ${workDir}/conf/db.sh
isSuccess "load config"

local_host="`hostname --fqdn`"
local_host="`hostname -i`"

##env check
echo "Please enter the mode selection such as: 1"
echo " 1: Simple"
echo " 2: Standard"
echo " 1: lite"
echo " 2: sample"
echo " 3: Standard"
echo ""

INSTALL_MODE=1

read -p "Please input the choice:" idx
if [[ '1' = "$idx" ]];then
INSTALL_MODE=1
echo "You chose Simple installation mode"
echo "You chose lite installation mode"
#check for Java
checkJava
#check for mysql
SERVER_NAME=MYSQL
EXTERNAL_SERVER_IP=$MYSQL_HOST
EXTERNAL_SERVER_PORT=$MYSQL_PORT
checkExternalServer

elif [[ '2' = "$idx" ]];then
INSTALL_MODE=2
echo "You chose sample installation mode"
#check for Java
checkJava
#check for mysql
SERVER_NAME=MYSQL
EXTERNAL_SERVER_IP=$MYSQL_HOST
EXTERNAL_SERVER_PORT=$MYSQL_PORT

elif [[ '3' = "$idx" ]];then
INSTALL_MODE=3
echo "You chose Standard installation mode"
#check for Java
checkJava
Expand All @@ -155,8 +136,8 @@ else
exit 1
fi

##env check
echo "Do you want to clear Dss table information in the database?"
##init db
echo "Do you want to clear DSS table information in the database?"
echo " 1: Do not execute table-building statements"
echo " 2: Dangerous! Clear all data and rebuild the tables."
echo ""
Expand Down Expand Up @@ -189,18 +170,25 @@ if [[ '2' = "$MYSQL_INSTALL_MODE" ]];then
#echo $GATEWAY_INSTALL_IP_2
sed -i "s/GATEWAY_INSTALL_IP_2/$GATEWAY_INSTALL_IP_2/g" ${workDir}/db/dss_dml.sql
sed -i "s/GATEWAY_PORT/$GATEWAY_PORT/g" ${workDir}/db/dss_dml.sql
if [ $VISUALIS_NGINX_IP == "127.0.0.1" ]||[ $VISUALIS_NGINX_IP == "0.0.0.0" ];then
mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source ${workDir}/db/dss_dml.sql"
isSuccess "source dss_dml.sql"

if [ '2' = "$INSTALL_MODE" ]||[ '3' = "$INSTALL_MODE" ];then
echo "visualis support "
if [ $VISUALIS_NGINX_IP == "127.0.0.1" ]||[ $VISUALIS_NGINX_IP == "0.0.0.0" ];then
echo "VISUALIS_NGINX_IP is equals $VISUALIS_NGINX_IP ,we will change it to ip address"
VISUALIS_NGINX_IP_2=$LOCAL_IP
else
else
VISUALIS_NGINX_IP_2=$VISUALIS_NGINX_IP
fi
#echo $VISUALIS_NGINX_IP_2
sed -i "s/VISUALIS_NGINX_IP_2/$VISUALIS_NGINX_IP_2/g" ${workDir}/db/visualis.sql
sed -i "s/VISUALIS_NGINX_PORT/$VISUALIS_NGINX_PORT/g" ${workDir}/db/visualis.sql
mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source ${workDir}/db/visualis.sql"
isSuccess "source visualis.sql"
fi
#echo $VISUALIS_NGINX_IP_2
sed -i "s/VISUALIS_NGINX_IP_2/$VISUALIS_NGINX_IP_2/g" ${workDir}/db/dss_dml.sql
sed -i "s/VISUALIS_NGINX_PORT/$VISUALIS_NGINX_PORT/g" ${workDir}/db/dss_dml.sql
mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source ${workDir}/db/dss_dml.sql"
isSuccess "source dss_dml.sql"
if [[ '2' = "$INSTALL_MODE" ]];then

if [[ '3' = "$INSTALL_MODE" ]];then
echo "start to update azkaban and qualitis table info "
#azkaban
if [ $AZKABAN_ADRESS_IP == "127.0.0.1" ];then
Expand Down Expand Up @@ -229,7 +217,7 @@ if [[ '2' = "$MYSQL_INSTALL_MODE" ]];then
fi
fi

##env check
## davinci db init
echo "Do you want to clear davinci table information in the database ? If you have not installed davinci environment,you must input '2',if you have davinci installed,choice 1."
echo " 1: Do not execute table-building statements"
echo "WARN:"
Expand All @@ -252,7 +240,6 @@ else
exit 1
fi


###linkis Eurkea info
SERVER_IP=$EUREKA_INSTALL_IP
SERVER_PORT=$EUREKA_PORT
Expand Down Expand Up @@ -362,6 +349,8 @@ ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#wds.dss.appjoint.scheduler.project.sto
isSuccess "subsitution linkis.properties of $SERVERNAME"
echo "<----------------$SERVERNAME:end------------------->"
echo ""

if [ '2' = "$INSTALL_MODE" ]||[ '3' = "$INSTALL_MODE" ];then
##Flow execution Install
PACKAGE_DIR=dss/dss-flow-execution-entrance
SERVERNAME=dss-flow-execution-entrance
Expand Down Expand Up @@ -437,7 +426,7 @@ APPJOINTNAME_CONF_PATH_PATENT=$SERVER_HOME/$APPJOINTPARENT/$APPJOINTNAME/appjoin
ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#job.datachecker.jdo.option.url.*#job.datachecker.jdo.option.url=$HIVE_META_URL#g\" $APPJOINTNAME_CONF_PATH_PATENT"
ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#job.datachecker.jdo.option.username.*#job.datachecker.jdo.option.username=$HIVE_META_USER#g\" $APPJOINTNAME_CONF_PATH_PATENT"
ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#job.datachecker.jdo.option.password.*#job.datachecker.jdo.option.password=$HIVE_META_PASSWORD#g\" $APPJOINTNAME_CONF_PATH_PATENT"
isSuccess "subsitution conf of $SERVERNAME"
isSuccess "subsitution conf of datachecker"
echo "<----------------datachecker appjoint install end------------------->"
echo ""
echo "<----------------eventchecker appjoint install start------------------->"
Expand All @@ -450,7 +439,7 @@ APPJOINTNAME_CONF_PATH_PATENT=$SERVER_HOME/$APPJOINTPARENT/$APPJOINTNAME/appjoin
ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#msg.eventchecker.jdo.option.url.*#msg.eventchecker.jdo.option.url=jdbc:mysql://${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB}?characterEncoding=UTF-8#g\" $APPJOINTNAME_CONF_PATH_PATENT"
ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#msg.eventchecker.jdo.option.username.*#msg.eventchecker.jdo.option.username=$MYSQL_USER#g\" $APPJOINTNAME_CONF_PATH_PATENT"
ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#msg.eventchecker.jdo.option.password.*#msg.eventchecker.jdo.option.password=$MYSQL_PASSWORD#g\" $APPJOINTNAME_CONF_PATH_PATENT"
isSuccess "subsitution conf of $SERVERNAME"
isSuccess "subsitution conf of eventchecker"
echo "<----------------$APPJOINTNAME:end------------------->"
echo ""
echo "<----------------visualis appjoint install start------------------->"
Expand All @@ -459,8 +448,10 @@ APPJOINTNAME=visualis
#visualis appjoint install
installAppjoints
echo "<----------------$APPJOINTNAME:end------------------->"
##sample version does not install qualitis APPJoint and scheduis APPJoint
if [[ '2' = "$INSTALL_MODE" ]];then
fi

##lite and sample version does not install qualitis APPJoint and scheduis APPJoint
if [[ '3' = "$INSTALL_MODE" ]];then
echo ""
echo "<----------------qualitis appjoint install start------------------->"
APPJOINTPARENT=dss-appjoints
Expand All @@ -469,7 +460,7 @@ APPJOINTNAME=qualitis
installAppjoints
APPJOINTNAME_CONF_PATH_PATENT=$SERVER_HOME/$APPJOINTPARENT/$APPJOINTNAME/appjoint.properties
ssh -p $SSH_PORT $SERVER_IP "sed -i \"s#baseUrl=http://127.0.0.1:8090#baseUrl=http://$QUALITIS_ADRESS_IP:$QUALITIS_ADRESS_PORT#g\" $APPJOINTNAME_CONF_PATH_PATENT"
isSuccess "subsitution conf of $SERVERNAME"
isSuccess "subsitution conf of qualitis"
echo "<----------------$APPJOINTNAME:end------------------->"
echo ""
echo "<----------------schedulis appjoint install start------------------->"
Expand All @@ -479,4 +470,4 @@ APPJOINTNAME=schedulis
installAppjoints
isSuccess "subsitution conf of schedulis"
echo "<----------------$APPJOINTNAME:end------------------->"
fi
fi
Loading

0 comments on commit e79e5ad

Please sign in to comment.