Merge pull request WeBankFinTech#95 from 77954309/dw-dev-1.0.0

Dw dev 1.0.0 project list + data source management

.gitignore

@@ -26,3 +26,4 @@ target
 .mvn/wrapper/maven-wrapper.jar
 .mvn/wrapper/maven-wrapper.properties
 /packages/
+exchangis-server/exchangis-extds

db/exchangis_ddl.sql

@@ -0,0 +1,63 @@
+DROP TABLE IF EXISTS `exchangis_job_param_config`;
+CREATE TABLE `exchangis_job_param_config`  (
+    id bigint auto_increment,
+    config_key varchar(64) not null unique,
+    config_name varchar(64) not null,
+    type varchar(32) not null,
+    ui_type varchar(32),
+    ui_field varchar(32),
+    ui_label varchar(32),
+    value_type varchar(32),
+    value_range varchar(255),
+    default_value varchar(255),
+    validate_type varchar(64),
+    validate_range varchar(64),
+    is_hidden bit,
+    is_advanced bit,
+    level tinyint,
+    treename varchar(32),
+    sort int,
+    description varchar(255),
+    status tinyint,
+    primary key (id)
+) ENGINE = InnoDB CHARACTER SET = utf8;
+
+DROP TABLE IF EXISTS `exchangis_job_info`;
+CREATE TABLE `exchangis_job_info`  (
+    id bigint auto_increment,
+    job_name varchar(64) not null unique,
+    job_desc varchar(255),
+    engine_type varchar(32) not null,
+    alarm_user varchar(32),
+    alarm_level int,
+    content text,
+    label_map varchar(255),
+    create_time datetime,
+    modify_time datetime,
+    create_user varchar(64),
+    modify_user varchar(64),
+    to_proxy_user varchar(64),
+    exec_nodes varchar(255),
+    last_launch_time datetime,
+    project_id int,
+    parent_id bigint,
+    primary key (id)
+) ENGINE = InnoDB CHARACTER SET = utf8;
+
+DROP TABLE IF EXISTS `exchangis_project`;
+CREATE TABLE `exchangis_project`  (
+    id bigint auto_increment,
+    dss_project_id bigint,
+    workspace_name varchar(64),
+    name varchar(64) not null,
+    description varchar(255),
+    create_time datetime,
+    last_update_time datetime,
+    create_by varchar(64),
+    last_update_by varchar(64),
+    tags varchar(255),
+    exec_users varchar(255),
+    view_users varchar(255),
+    edit_users varchar(255),
+    primary key (id)
+) ENGINE = InnoDB CHARACTER SET = utf8;

db/exchangis_dml.sql

@@ -0,0 +1,10 @@
+INSERT INTO `exchangis_job_param_config`(config_key, config_name, type, ui_type, ui_field, ui_label, value_type, value_range, default_value, validate_type, validate_range, is_hidden, is_advanced, level, treename, sort, description, status) VALUES ('exchangis.job.ds.params.hive.transform_type', '传输方式', 'HIVE', 'OPTION', 'transform', '传输方式', 'ARRAY', '[\"Record\", \"二进制\"]', 'Record', '', '', 0, 0, 1, '', 1, '', 1);
+
+INSERT INTO `exchangis_job_param_config`(config_key, config_name, type, ui_type, ui_field, ui_label, value_type, value_range, default_value, validate_type, validate_range, is_hidden, is_advanced, level, treename, sort, description, status) VALUES ('exchangis.datax.setting.speed.channel', '并发数', 'DATAX', 'INPUT', 'channel', '并发数', 'NUMBER', '1', '1', '', '', 0, 0, 1, '', 1, '', 1);
+INSERT INTO `exchangis_job_param_config`(config_key, config_name, type, ui_type, ui_field, ui_label, value_type, value_range, default_value, validate_type, validate_range, is_hidden, is_advanced, level, treename, sort, description, status) VALUES ('exchangis.datax.setting.speed.byte', '传输速率', 'DATAX', 'INPUT', 'channel', '传输速率', 'NUMBER', '', '', '', '', 0, 0, 1, '', 1, '', 1);
+INSERT INTO `exchangis_job_param_config`(config_key, config_name, type, ui_type, ui_field, ui_label, value_type, value_range, default_value, validate_type, validate_range, is_hidden, is_advanced, level, treename, sort, description, status) VALUES ('exchangis.datax.setting.errorlimit.record', '脏数据最大记录数', 'DATAX', 'INPUT', 'errorlimit_record', '脏数据最大记录数', 'NUMBER', '', '', '', '', 0, 0, 1, '', 1, '', 1);
+INSERT INTO `exchangis_job_param_config`(config_key, config_name, type, ui_type, ui_field, ui_label, value_type, value_range, default_value, validate_type, validate_range, is_hidden, is_advanced, level, treename, sort, description, status) VALUES ('exchangis.datax.setting.errorlimit.percentage', '脏数据占比阈值', 'DATAX', 'INPUT', 'errorlimit_percentage', '脏数据占比阈值', 'DOUBLE', '', '', '', '', 0, 0, 1, '', 1, '', 1);
+
+
+
+

db/job_content_example.json

@@ -0,0 +1,76 @@
+{
+  "dataSources": {
+    "source_id": "HIVE.10001.test_db.test_table",
+    "sink_id": "MYSQL.10002.mask_db.mask_table"
+  },
+  "params": {
+    "sources": [
+      {
+        "config_key": "exchangis.job.ds.params.hive.transform_type",
+        "config_name": "传输方式",
+        "config_value": "二进制",
+        "sort": 1
+      },
+      {
+        "config_key": "exchangis.job.ds.params.hive.partitioned_by",
+        "config_name": "分区信息",
+        "config_value": "2021-07-30",
+        "sort": 2
+      },
+      {
+        "config_key": "exchangis.job.ds.params.hive.empty_string",
+        "config_name": "空值字符",
+        "config_value": "",
+        "sort": 3
+      }
+    ],
+    "sinks": [
+      {
+        "config_key": "exchangis.job.ds.params.mysql.write_type",
+        "config_name": "写入方式",
+        "config_value": "insert",
+        "sort": 1
+      },
+      {
+        "config_key": "exchangis.job.ds.params.mysql.batch_size",
+        "config_name": "批量大小",
+        "config_value": 1000,
+        "sort": 2
+      }
+    ]
+  },
+  "transforms": [
+    {
+      "source_field_name": "name",
+      "source_field_type": "VARCHAR",
+      "sink_field_name": "c_name",
+      "sink_field_type": "VARCHAR"
+    },
+    {
+      "source_field_name": "year",
+      "source_field_type": "VARCHAR",
+      "sink_field_name": "d_year",
+      "sink_field_type": "VARCHAR"
+    }
+  ],
+  "settings": [
+    {
+      "config_key": "rate_limit",
+      "config_name": "作业速率限制",
+      "config_value": 102400,
+      "sort": 1
+    },
+    {
+      "config_key": "record_limit",
+      "config_name": "作业记录数限制",
+      "config_value": 10000,
+      "sort": 2
+    },
+    {
+      "config_key": "max_errors",
+      "config_name": "最多错误记录数",
+      "config_value": 100,
+      "sort": 3
+    }
+  ]
+}

exchangis-dao/pom.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>exchangis</artifactId>
+        <groupId>com.webank.wedatasphere.exchangis</groupId>
+        <version>1.0.0-RC1</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>exchangis-dao</artifactId>
+
+    <properties>
+        <maven.compiler.source>8</maven.compiler.source>
+        <maven.compiler.target>8</maven.compiler.target>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.webank.wedatasphere.linkis</groupId>
+            <artifactId>linkis-mybatis</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-context</artifactId>
+            <version>5.2.12.RELEASE</version>
+            <scope>provided</scope>
+        </dependency>
+    </dependencies>
+</project>

exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/dao/domain/ExchangisJobInfo.java

@@ -0,0 +1,200 @@
+package com.webank.wedatasphere.exchangis.dao.domain;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+
+import java.util.Date;
+
+@TableName("exchangis_job_info")
+public class ExchangisJobInfo {
+
+    @TableId(value = "id", type = IdType.AUTO)
+    private Long id;
+
+    @TableField(value = "job_name")
+    private String jobName;
+
+    @TableField(value = "job_desc")
+    private String jobDesc;
+
+    @TableField(value = "engine_type")
+    private String engineType;
+
+    @TableField(value = "alarm_user")
+    private String alarmUser;
+
+    @TableField(value = "alarm_level")
+    private Integer alarmLevel;
+
+    @TableField(value = "content")
+    private String content;
+
+    @TableField(value = "label_map")
+    private String labelMap;
+
+    @TableField(value = "create_time")
+    private Date createTime;
+
+    @TableField(value = "modify_time")
+    private Date modifyTime;
+
+    @TableField(value = "create_user")
+    private String createUser;
+
+    @TableField(value = "modify_user")
+    private String modifyUser;
+
+    @TableField(value = "to_proxy_user")
+    private String toProxyUser;
+
+    @TableField(value = "exec_nodes")
+    private String execNodes;
+
+    @TableField(value = "last_launch_time")
+    private Date lastLaunchTime;
+
+    @TableField(value = "project_id")
+    private Integer projectId;
+
+    @TableField(value = "parent_id")
+    private Long parentId;
+
+    public Long getId() {
+        return id;
+    }
+
+    public void setId(Long id) {
+        this.id = id;
+    }
+
+    public String getJobName() {
+        return jobName;
+    }
+
+    public void setJobName(String jobName) {
+        this.jobName = jobName;
+    }
+
+    public String getJobDesc() {
+        return jobDesc;
+    }
+
+    public void setJobDesc(String jobDesc) {
+        this.jobDesc = jobDesc;
+    }
+
+    public String getEngineType() {
+        return engineType;
+    }
+
+    public void setEngineType(String engineType) {
+        this.engineType = engineType;
+    }
+
+    public String getAlarmUser() {
+        return alarmUser;
+    }
+
+    public void setAlarmUser(String alarmUser) {
+        this.alarmUser = alarmUser;
+    }
+
+    public Integer getAlarmLevel() {
+        return alarmLevel;
+    }
+
+    public void setAlarmLevel(Integer alarmLevel) {
+        this.alarmLevel = alarmLevel;
+    }
+
+    public String getContent() {
+        return content;
+    }
+
+    public void setContent(String content) {
+        this.content = content;
+    }
+
+    public String getLabelMap() {
+        return labelMap;
+    }
+
+    public void setLabelMap(String labelMap) {
+        this.labelMap = labelMap;
+    }
+
+    public Date getCreateTime() {
+        return createTime;
+    }
+
+    public void setCreateTime(Date createTime) {
+        this.createTime = createTime;
+    }
+
+    public Date getModifyTime() {
+        return modifyTime;
+    }
+
+    public void setModifyTime(Date modifyTime) {
+        this.modifyTime = modifyTime;
+    }
+
+    public String getCreateUser() {
+        return createUser;
+    }
+
+    public void setCreateUser(String createUser) {
+        this.createUser = createUser;
+    }
+
+    public String getModifyUser() {
+        return modifyUser;
+    }
+
+    public void setModifyUser(String modifyUser) {
+        this.modifyUser = modifyUser;
+    }
+
+    public String getToProxyUser() {
+        return toProxyUser;
+    }
+
+    public void setToProxyUser(String toProxyUser) {
+        this.toProxyUser = toProxyUser;
+    }
+
+    public String getExecNodes() {
+        return execNodes;
+    }
+
+    public void setExecNodes(String execNodes) {
+        this.execNodes = execNodes;
+    }
+
+    public Date getLastLaunchTime() {
+        return lastLaunchTime;
+    }
+
+    public void setLastLaunchTime(Date lastLaunchTime) {
+        this.lastLaunchTime = lastLaunchTime;
+    }
+
+    public Integer getProjectId() {
+        return projectId;
+    }
+
+    public void setProjectId(Integer projectId) {
+        this.projectId = projectId;
+    }
+
+    public Long getParentId() {
+        return parentId;
+    }
+
+    public void setParentId(Long parentId) {
+        this.parentId = parentId;
+    }
+
+}