Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Linkis1.2.0[datamodel][dataassets][datawarehouse] #952

Merged
merged 7 commits into from
Dec 30, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
4 changes: 4 additions & 0 deletions assembly/bin/executeSQL.sh
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,10 @@ function executeSQL() {
isSuccess "source dss_dataapi_ddl.sql"
mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source $DB_CONF_PATH/apps/dss_guide_ddl.sql"
isSuccess "source dss_guide_ddl.sql"
mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source $DB_CONF_PATH/apps/datamodel.sql"
isSuccess "source datamodel.sql"
mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source $DB_CONF_PATH/apps/datawarehouse.sql"
isSuccess "source datawarehouse.sql"
mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source $DB_DML_PATH"
isSuccess "source dss_dml_real.sql"
echo "Rebuild the table"
Expand Down
54 changes: 43 additions & 11 deletions assembly/bin/install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -121,11 +121,18 @@ function replaceCommonIp() {
DSS_DATA_API_SERVER_PORT=9208
fi

if [ -z "$DSS_DATA_GOVERNANCE_SERVER_INSTALL_IP" ]; then
DSS_DATA_GOVERNANCE_SERVER_INSTALL_IP=$LOCAL_IP
if [ -z "$DSS_DATA_ASSETS_SERVER_INSTALL_IP" ]; then
DSS_DATA_ASSETS_SERVER_INSTALL_IP=$LOCAL_IP
fi
if [ -z "$DSS_DATA_GOVERNANCE_SERVER_PORT" ]; then
DSS_DATA_GOVERNANCE_SERVER_PORT=9209
if [ -z "$DSS_DATA_ASSETS_SERVER_INSTALL_IP" ]; then
DSS_DATA_ASSETS_SERVER_INSTALL_IP=9300
fi
if [ -z "$DSS_DATAMODEL_CENTER_SERVER_INSTALL_IP" ]; then
DSS_DATAMODEL_CENTER_SERVER_INSTALL_IP=9400
fi

if [ -z "$DSS_DATA_WAREHOUSE_SERVER_INSTALL_IP" ]; then
DSS_DATA_WAREHOUSE_SERVER_INSTALL_IP=9500
fi

if [ -z "$DSS_GUIDE_SERVER_INSTALL_IP" ]; then
Expand Down Expand Up @@ -343,13 +350,38 @@ function installDssProject() {
CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml
installPackage

###install dss-data-governance-server
SERVER_NAME=dss-data-governance-server
SERVER_IP=$DSS_DATA_GOVERNANCE_SERVER_INSTALL_IP
SERVER_PORT=$DSS_DATA_GOVERNANCE_SERVER_PORT
UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-data-governance/$SERVER_NAME
LIB_PATH=$SERVER_HOME/lib/dss-data-governance
LOG_PATH=$SERVER_HOME/logs/dss-data-governance/$SERVER_NAME
###install dss-data-assets-server
SERVER_NAME=dss-data-assets-server
SERVER_IP=$DSS_DATA_ASSETS_SERVER_INSTALL_IP
SERVER_PORT=$DSS_DATA_ASSETS_SERVER_PORT
UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-data-assets-server/$SERVER_NAME
LIB_PATH=$SERVER_HOME/lib/dss-data-assets-server
LIB_PATH=$SERVER_HOME/lib/atlas-application.properties
LOG_PATH=$SERVER_HOME/logs/dss-data-assets-server/$SERVER_NAME
CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties
CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties
CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml
installPackage

###install dss-data-warehouse-server
SERVER_NAME=dss-data-warehouse-server
SERVER_IP=$DSS_DATA_WAREHOUSE_SERVER_INSTALL_IP
SERVER_PORT=$DSS_DATA_WAREHOUSE_SERVER_PORT
UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-data-warehouse-server/$SERVER_NAME
LIB_PATH=$SERVER_HOME/lib/dss-data-warehouse-server
LOG_PATH=$SERVER_HOME/logs/dss-data-warehouse-server/$SERVER_NAME
CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties
CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties
CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml
installPackage

###install dss-datamodel-center-server
SERVER_NAME=dss-datamodel-center-server
SERVER_IP=$DSS_DATAMODEL_CENTER_SERVER_INSTALL_IP
SERVER_PORT=$DSS_DATAMODEL_CENTER_SERVER_PORT
UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-datamodel-center-server/$SERVER_NAME
LIB_PATH=$SERVER_HOME/lib/dss-datamodel-center-server
LOG_PATH=$SERVER_HOME/logs/dss-datamodel-center-server/$SERVER_NAME
CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties
CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties
CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml
Expand Down
15 changes: 10 additions & 5 deletions assembly/config/config.sh
Original file line number Diff line number Diff line change
Expand Up @@ -52,13 +52,19 @@ DSS_FLOW_EXECUTION_SERVER_PORT=9006
###dss-scriptis-server
DSS_SCRIPTIS_SERVER_INSTALL_IP=127.0.0.1
DSS_SCRIPTIS_SERVER_PORT=9008
###dss-data-assets-server
DSS_DATA_ASSETS_SERVER_INSTALL_IP=127.0.0.1
DSS_DATA_ASSETS_SERVER_PORT=9300
###dss-datamodel-center-server
DSS_DATAMODEL_CENTER_SERVER_INSTALL_IP=127.0.0.1
DSS_DATAMODEL_CENTER_SERVER_PORT=9400
###dss-data-warehouse-server
DSS_DATA_WAREHOUSE_SERVER_INSTALL_IP=127.0.0.1
DSS_DATA_WAREHOUSE_SERVER_PORT=9500

###dss-data-api-server
DSS_DATA_API_SERVER_INSTALL_IP=127.0.0.1
DSS_DATA_API_SERVER_PORT=9208
###dss-data-governance-server
DSS_DATA_GOVERNANCE_SERVER_INSTALL_IP=127.0.0.1
DSS_DATA_GOVERNANCE_SERVER_PORT=9209
###dss-guide-server
DSS_GUIDE_SERVER_INSTALL_IP=127.0.0.1
DSS_GUIDE_SERVER_PORT=9210
Expand All @@ -83,5 +89,4 @@ EMAIL_PORT=25
EMAIL_USERNAME=xxx@163.com
EMAIL_PASSWORD=xxxxx
EMAIL_PROTOCOL=smtp
############## ############## dss_appconn_instance configuration end ############## ##############

############## ############## dss_appconn_instance configuration end ############## ##############
2 changes: 1 addition & 1 deletion assembly/dss-package/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
<parent>
<artifactId>dss</artifactId>
<groupId>com.webank.wedatasphere.dss</groupId>
<version>1.1.0</version>
<version>1.2.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dss-package</artifactId>
Expand Down
30 changes: 25 additions & 5 deletions assembly/dss-package/src/main/assembly/distribution.xml
Original file line number Diff line number Diff line change
Expand Up @@ -287,20 +287,40 @@
<!--dss-data-api-server-->
<fileSet>
<directory>
${basedir}/../../dss-data-api/dss-data-api-server/target/out/dss-data-api-server/lib/
${basedir}/../../dss-apps/dss-data-api/dss-data-api-server/target/out/dss-data-api-server/lib/
</directory>
<outputDirectory>lib/dss-data-api/dss-data-api-server</outputDirectory>
<outputDirectory>lib/dss-apps/dss-data-api-server</outputDirectory>
<includes>
<include>**/*</include>
</includes>
</fileSet>


<fileSet>
<directory>
${basedir}/../../dss-apps/dss-dataasset-management/dss-data-assets-server/target/out/dss-data-assets-server/lib/
</directory>
<outputDirectory>lib/dss-apps/dss-data-assets-server</outputDirectory>
<includes>
<include>**/*</include>
</includes>
</fileSet>

<fileSet>
<directory>
${basedir}/../../dss-apps/dss-datamodel-center/dss-datamodel-center-server/target/out/dss-datamodel-center-server/lib/
</directory>
<outputDirectory>lib/dss-apps/dss-datamodel-center-server</outputDirectory>
<includes>
<include>**/*</include>
</includes>
</fileSet>

<!--dss-data-governance-server-->
<fileSet>
<directory>
${basedir}/../../dss-data-governance/dss-data-governance-server/target/out/dss-data-governance-server/lib/
${basedir}/../../dss-apps/dss-datawarehouse-design/dss-data-warehouse-server/target/out/dss-data-warehouse-server/lib/
</directory>
<outputDirectory>lib/dss-data-governance/dss-data-governance-server</outputDirectory>
<outputDirectory>lib/dss-apps/dss-data-warehouse-server</outputDirectory>
<includes>
<include>**/*</include>
</includes>
Expand Down
2 changes: 1 addition & 1 deletion assembly/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<parent>
<artifactId>dss</artifactId>
<groupId>com.webank.wedatasphere.dss</groupId>
<version>1.1.0</version>
<version>1.2.0</version>
</parent>
<packaging>pom</packaging>
<modelVersion>4.0.0</modelVersion>
Expand Down
67 changes: 67 additions & 0 deletions conf/atlas-application.properties
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
# Generated by Apache Ambari. Sun Feb 27 10:44:18 2022

atlas.audit.hbase.tablename=ATLAS_ENTITY_AUDIT_EVENTS
atlas.audit.hbase.zookeeper.quorum=hdp
atlas.audit.zookeeper.session.timeout.ms=60000
atlas.authentication.keytab=/etc/security/keytabs/atlas.service.keytab
atlas.authentication.method.file=true
atlas.authentication.method.file.filename=/home/hadoop/software/apache-atlas-2.0.0/conf/users-credentials.properties
atlas.authentication.method.kerberos=false
atlas.authentication.method.ldap=false
atlas.authentication.method.ldap.ad.base.dn=
atlas.authentication.method.ldap.ad.bind.dn=
atlas.authentication.method.ldap.ad.bind.password=
atlas.authentication.method.ldap.ad.default.role=ROLE_USER
atlas.authentication.method.ldap.ad.domain=
atlas.authentication.method.ldap.ad.referral=ignore
atlas.authentication.method.ldap.ad.url=
atlas.authentication.method.ldap.ad.user.searchfilter=(sAMAccountName={0})
atlas.authentication.method.ldap.base.dn=
atlas.authentication.method.ldap.bind.dn=
atlas.authentication.method.ldap.bind.password=
atlas.authentication.method.ldap.default.role=ROLE_USER
atlas.authentication.method.ldap.groupRoleAttribute=cn
atlas.authentication.method.ldap.groupSearchBase=
atlas.authentication.method.ldap.groupSearchFilter=
atlas.authentication.method.ldap.referral=ignore
atlas.authentication.method.ldap.type=ldap
atlas.authentication.method.ldap.url=
atlas.authentication.method.ldap.user.searchfilter=
atlas.authentication.method.ldap.userDNpattern=uid=
atlas.authentication.principal=atlas
atlas.authorizer.impl=ranger
atlas.cluster.name=bigdata_all_in_one
atlas.enableTLS=false
atlas.graph.index.search.solr.mode=cloud
atlas.graph.index.search.solr.wait-searcher=true
atlas.graph.index.search.solr.zookeeper-url=hadoop:2181/infra-solr
atlas.graph.storage.hbase.table=atlas_janus
atlas.graph.storage.hostname=hdp
atlas.kafka.auto.commit.enable=false
atlas.kafka.bootstrap.servers=hadoop:6667
atlas.kafka.hook.group.id=atlas
atlas.kafka.zookeeper.connect=hadoop:2181
atlas.kafka.zookeeper.connection.timeout.ms=30000
atlas.kafka.zookeeper.session.timeout.ms=60000
atlas.kafka.zookeeper.sync.time.ms=20
atlas.lineage.schema.query.hive_table=hive_table where __guid='%s'\, columns
atlas.lineage.schema.query.Table=Table where __guid='%s'\, columns
atlas.notification.create.topics=true
atlas.notification.embedded=false
atlas.notification.replicas=1
atlas.notification.topics=ATLAS_HOOK,ATLAS_ENTITIES
atlas.proxyusers=
atlas.rest.address=http://hadoop:21000
atlas.server.address.id1=hadoop:21000
atlas.server.bind.address=0.0.0.0
atlas.server.ha.enabled=false
atlas.server.http.port=21000
atlas.server.https.port=21443
atlas.server.ids=id1
atlas.simple.authz.policy.file=/home/hadoop/software/apache-atlas-2.0.0/conf/atlas-simple-authz-policy.json
atlas.solr.kerberos.enable=false
atlas.ssl.exclude.protocols=TLSv1.2
atlas.sso.knox.browser.useragent=
atlas.sso.knox.enabled=false
atlas.sso.knox.providerurl=
atlas.sso.knox.publicKey=
88 changes: 88 additions & 0 deletions conf/dss-data-assets-server.properties
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
#
# /*
# * Copyright 2019 WeBank
# *
# * Licensed under the Apache License, Version 2.0 (the "License");
# * you may not use this file except in compliance with the License.
# * You may obtain a copy of the License at
# *
# * http://www.apache.org/licenses/LICENSE-2.0
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# */
#
# Spring configurations
spring.server.port=9300
spring.spring.application.name=dss-data-assets-server
wds.linkis.test.mode=true
wds.linkis.log.clear=true
wds.linkis.server.version=v1


wds.linkis.server.mybatis.datasource.url=jdbc:mysql://localhost:3306/hive?useSSL=false&characterEncoding=UTF-8

wds.linkis.server.mybatis.datasource.username=root

wds.linkis.server.mybatis.datasource.password=root





wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.data.governance.restful

##mybatis
wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/data/governance/dao/impl/*.xml

wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.data.governance.entity

wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.data.governance.dao


# atlas config
atlas.rest.address=http://localhost:21000
atlas.username=admin
atlas.password=admin123
atlas.client.readTimeoutMSecs=60000
atlas.client.connectTimeoutMSecs=60000

# create root classification
atlas.root.indicator=datamodel_indicator
atlas.root.measure=datamodel_measure
atlas.root.dimension=datamodel_dimension
atlas.root.layer=datamodel_layer
atlas.root.theme=datamodel_theme

# create root glossary
atlas.root.label=datamodel_label
atlas.root.collection=datamodel_collection


# hive metadata config
metastore.datasource.driver=com.mysql.jdbc.Driver
metastore.datasource.url=jdbc:mysql://localhost:3306/hive?characterEncoding=UTF-8
metastore.datasource.username=root
metastore.datasource.password=root


wds.workspace.client.serverurl=http://localhost:9001

wds.workspace.client.authtoken.key=hadoop

wds.workspace.client.authenticationStrategy=org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy

wds.workspace.client.discovery.enabled=false

wds.workspace.client.authtoken.value=hadoop

spring.spring.mvc.servlet.path=/api/rest_j/v1

wds.workspace.client.dws.version=v1

wds.wedatasphere.data.assert.client.strategy=static
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

recommend wds.wedatasphere.data.assert.client.strategy=token



53 changes: 0 additions & 53 deletions conf/dss-data-governance-server.properties

This file was deleted.

Loading