Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Feature][Data Quality] Data Quality Support Choose Database #14406

Merged
merged 14 commits into from
Jul 12, 2023
Merged

[Feature][Data Quality] Data Quality Support Choose Database #14406

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
3ea9f85
add dataquality database api
qingwli Jun 26, 2023
0481dad
change ui
qingwli Jun 26, 2023
3910676
api change
qingwli Jun 26, 2023
49d9323
update
qingwli Jun 26, 2023
267ec67
fix spotless
qingwli Jun 26, 2023
3e6f4d0
fix h2
qingwli Jun 26, 2023
55459cb
fix pg
qingwli Jun 27, 2023
ba88a2d
fix-dead-line
qingwli Jun 27, 2023
f503e4e
update
qingwli Jun 28, 2023
269efdd
fix-spotless
qingwli Jun 28, 2023
4f5af23
update pg sql
qingwli Jun 29, 2023
5e17117
add ut
qingwli Jul 4, 2023
1f47876
Merge branch 'dev' into dataquality-support-choose-database
qingwli Jul 5, 2023
15132b0
fix ut
qingwli Jul 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 22 additions & 5 deletions ...er-api/src/main/java/org/apache/dolphinscheduler/api/controller/DataSourceController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import static org.apache.dolphinscheduler.api.enums.Status.CONNECT_DATASOURCE_FAILURE;
import static org.apache.dolphinscheduler.api.enums.Status.CREATE_DATASOURCE_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.DELETE_DATA_SOURCE_FAILURE;
import static org.apache.dolphinscheduler.api.enums.Status.GET_DATASOURCE_DATABASES_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.GET_DATASOURCE_TABLES_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.GET_DATASOURCE_TABLE_COLUMNS_ERROR;
import static org.apache.dolphinscheduler.api.enums.Status.KERBEROS_STARTUP_STATE;
Expand Down Expand Up @@ -342,27 +343,43 @@ public Result getKerberosStartupState(@Parameter(hidden = true) @RequestAttribut

@Operation(summary = "tables", description = "GET_DATASOURCE_TABLES_NOTES")
@Parameters({
@Parameter(name = "datasourceId", description = "DATA_SOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "1"))
@Parameter(name = "datasourceId", description = "DATA_SOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "1")),
qingwli marked this conversation as resolved.
Show resolved Hide resolved
@Parameter(name = "database", description = "DATABASE", required = true, schema = @Schema(implementation = String.class, example = "test"))
})
@GetMapping(value = "/tables")
@ResponseStatus(HttpStatus.OK)
@ApiException(GET_DATASOURCE_TABLES_ERROR)
public Result getTables(@RequestParam("datasourceId") Integer datasourceId) {
Map<String, Object> result = dataSourceService.getTables(datasourceId);
public Result getTables(@RequestParam("datasourceId") Integer datasourceId,
@RequestParam(value = "database") String database) {
Map<String, Object> result = dataSourceService.getTables(datasourceId, database);
return returnDataList(result);
}

@Operation(summary = "tableColumns", description = "GET_DATASOURCE_TABLE_COLUMNS_NOTES")
@Parameters({
@Parameter(name = "datasourceId", description = "DATA_SOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "1")),
@Parameter(name = "database", description = "DATABASE", required = true, schema = @Schema(implementation = String.class, example = "test")),
@Parameter(name = "tableName", description = "TABLE_NAME", required = true, schema = @Schema(implementation = String.class, example = "test"))
Radeity marked this conversation as resolved.
Show resolved Hide resolved
})
@GetMapping(value = "/tableColumns")
@ResponseStatus(HttpStatus.OK)
@ApiException(GET_DATASOURCE_TABLE_COLUMNS_ERROR)
public Result getTableColumns(@RequestParam("datasourceId") Integer datasourceId,
@RequestParam("tableName") String tableName) {
Map<String, Object> result = dataSourceService.getTableColumns(datasourceId, tableName);
@RequestParam("tableName") String tableName,
@RequestParam(value = "database") String database) {
Map<String, Object> result = dataSourceService.getTableColumns(datasourceId, database, tableName);
return returnDataList(result);
}

@Operation(summary = "databases", description = "GET_DATASOURCE_DATABASE_NOTES")
@Parameters({
@Parameter(name = "datasourceId", description = "DATA_SOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "1"))
})
@GetMapping(value = "/databases")
@ResponseStatus(HttpStatus.OK)
@ApiException(GET_DATASOURCE_DATABASES_ERROR)
public Result getDatabases(@RequestParam("datasourceId") Integer datasourceId) {
Map<String, Object> result = dataSourceService.getDatabases(datasourceId);
return returnDataList(result);
}
}
1 change: 1 addition & 0 deletions dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -479,6 +479,7 @@ public enum Status {
GET_DATASOURCE_OPTIONS_ERROR(1200017, "get datasource options error", "获取数据源Options错误"),
GET_DATASOURCE_TABLES_ERROR(1200018, "get datasource tables error", "获取数据源表列表错误"),
GET_DATASOURCE_TABLE_COLUMNS_ERROR(1200019, "get datasource table columns error", "获取数据源表列名错误"),
GET_DATASOURCE_DATABASES_ERROR(1200035, "get datasource databases error", "获取数据库列表错误"),

CREATE_CLUSTER_ERROR(120020, "create cluster error", "创建集群失败"),
CLUSTER_NAME_EXISTS(120021, "this cluster name [{0}] already exists", "集群名称[{0}]已经存在"),
Expand Down
13 changes: 11 additions & 2 deletions ...cheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,15 +132,24 @@ public interface DataSourceService {
/**
* get tables
* @param datasourceId
* @param database
* @return
*/
Map<String, Object> getTables(Integer datasourceId);
Map<String, Object> getTables(Integer datasourceId, String database);

/**
* get table columns
* @param datasourceId
* @param database
* @param tableName
* @return
*/
Map<String, Object> getTableColumns(Integer datasourceId, String tableName);
Map<String, Object> getTableColumns(Integer datasourceId, String database, String tableName);

/**
* get databases
* @param datasourceId
* @return
*/
Map<String, Object> getDatabases(Integer datasourceId);
}
57 changes: 52 additions & 5 deletions ...api/src/main/java/org/apache/dolphinscheduler/api/service/impl/DataSourceServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -525,7 +525,7 @@ public Map<String, Object> authedDatasource(User loginUser, Integer userId) {
}

@Override
public Map<String, Object> getTables(Integer datasourceId) {
public Map<String, Object> getTables(Integer datasourceId, String database) {
Map<String, Object> result = new HashMap<>();

DataSource dataSource = dataSourceMapper.selectById(datasourceId);
Expand Down Expand Up @@ -561,7 +561,7 @@ public Map<String, Object> getTables(Integer datasourceId) {
}

tables = metaData.getTables(
connectionParam.getDatabase(),
database,
getDbSchemaPattern(dataSource.getType(), schema, connectionParam),
"%", TABLE_TYPES);
if (null == tables) {
Expand Down Expand Up @@ -593,7 +593,7 @@ public Map<String, Object> getTables(Integer datasourceId) {
}

@Override
public Map<String, Object> getTableColumns(Integer datasourceId, String tableName) {
public Map<String, Object> getTableColumns(Integer datasourceId, String database, String tableName) {
Map<String, Object> result = new HashMap<>();

DataSource dataSource = dataSourceMapper.selectById(datasourceId);
Expand All @@ -613,8 +613,6 @@ public Map<String, Object> getTableColumns(Integer datasourceId, String tableNam
ResultSet rs = null;

try {

String database = connectionParam.getDatabase();
if (null == connection) {
return result;
}
Expand Down Expand Up @@ -645,6 +643,55 @@ public Map<String, Object> getTableColumns(Integer datasourceId, String tableNam
return result;
}

@Override
public Map<String, Object> getDatabases(Integer datasourceId) {
qingwli marked this conversation as resolved.
Show resolved Hide resolved
Map<String, Object> result = new HashMap<>();

DataSource dataSource = dataSourceMapper.selectById(datasourceId);

List<String> tableList;
BaseConnectionParam connectionParam =
(BaseConnectionParam) DataSourceUtils.buildConnectionParams(
dataSource.getType(),
dataSource.getConnectionParams());

if (null == connectionParam) {
putMsg(result, Status.DATASOURCE_CONNECT_FAILED);
return result;
}

Connection connection =
DataSourceUtils.getConnection(dataSource.getType(), connectionParam);
ResultSet rs = null;

try {
if (null == connection) {
putMsg(result, Status.DATASOURCE_CONNECT_FAILED);
return result;
}

rs = connection.createStatement().executeQuery(Constants.DATABASES_QUERY);
tableList = new ArrayList<>();
while (rs.next()) {
String name = rs.getString(1);
tableList.add(name);
}
} catch (Exception e) {
log.error("Get databases error, datasourceId:{}.", datasourceId, e);
putMsg(result, Status.GET_DATASOURCE_TABLES_ERROR);
return result;
} finally {
closeResult(rs);
releaseConnection(connection);
}

List<ParamsOptions> options = getParamsOptions(tableList);

result.put(Constants.DATA_LIST, options);
putMsg(result, Status.SUCCESS);
return result;
}

private List<ParamsOptions> getParamsOptions(List<String> columnList) {
List<ParamsOptions> options = null;
if (CollectionUtils.isNotEmpty(columnList)) {
Expand Down
5 changes: 5 additions & 0 deletions ...cheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -850,4 +850,9 @@ private Constants() {
public static final String REMOTE_LOGGING_GCS_CREDENTIAL = "remote.logging.google.cloud.storage.credential";

public static final String REMOTE_LOGGING_GCS_BUCKET_NAME = "remote.logging.google.cloud.storage.bucket.name";

/**
* data quality
*/
public static final String DATABASES_QUERY = "show databases";
qingwli marked this conversation as resolved.
Show resolved Hide resolved
}
46 changes: 44 additions & 2 deletions dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1370,6 +1370,12 @@ VALUES(28, 'enum_list', 'input', '$t(enum_list)', NULL, NULL, 'Please enter enum
INSERT INTO `t_ds_dq_rule_input_entry`
(`id`, `field`, `type`, `title`, `value`, `options`, `placeholder`, `option_source_type`, `value_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`)
VALUES(29, 'begin_time', 'input', '$t(begin_time)', NULL, NULL, 'Please enter begin time', 0, 0, 0, 1, 1, 0, 0, '2021-03-03 11:31:24.0', '2021-03-03 11:31:24.0');
INSERT INTO `t_ds_dq_rule_input_entry`
(`id`, `field`, `type`, `title`, `value`, `options`, `placeholder`, `option_source_type`, `value_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`)
VALUES(30, 'src_database', 'select', '$t(src_database)', NULL, NULL, 'Please select source database', 0, 0, 0, 1, 1, 1, 1, '2021-03-03 11:31:24.0', '2021-03-03 11:31:24.0');
INSERT INTO `t_ds_dq_rule_input_entry`
(`id`, `field`, `type`, `title`, `value`, `options`, `placeholder`, `option_source_type`, `value_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`)
VALUES(31, 'target_database', 'select', '$t(target_database)', NULL, NULL, 'Please select target database', 0, 0, 0, 1, 1, 1, 1, '2021-03-03 11:31:24.0', '2021-03-03 11:31:24.0');
qingwli marked this conversation as resolved.
Show resolved Hide resolved

--
-- Table structure for table `t_ds_dq_task_statistics_value`
Expand Down Expand Up @@ -1853,9 +1859,45 @@ VALUES(148, 10, 17, NULL, 11, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.00
INSERT INTO `t_ds_relation_rule_input_entry`
(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`, `update_time`)
VALUES(149, 10, 19, NULL, 12, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
INSERT INTO t_ds_relation_rule_input_entry
INSERT INTO `t_ds_relation_rule_input_entry`
(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`, `update_time`)
VALUES(150, 8, 29, NULL, 7, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
INSERT INTO `t_ds_relation_rule_input_entry`
(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`, `update_time`)
VALUES(151, 1, 30, NULL, 2, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
INSERT INTO `t_ds_relation_rule_input_entry`
(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`, `update_time`)
VALUES(152, 2, 30, NULL, 2, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
INSERT INTO `t_ds_relation_rule_input_entry`
(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`, `update_time`)
VALUES(153, 3, 30, NULL, 2, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
INSERT INTO `t_ds_relation_rule_input_entry`
(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`, `update_time`)
VALUES(154, 4, 30, NULL, 2, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
INSERT INTO `t_ds_relation_rule_input_entry`
(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`, `update_time`)
VALUES(155, 5, 30, NULL, 2, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
INSERT INTO `t_ds_relation_rule_input_entry`
(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`, `update_time`)
VALUES(156, 6, 30, NULL, 2, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
INSERT INTO `t_ds_relation_rule_input_entry`
(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`, `update_time`)
VALUES(157, 7, 30, NULL, 2, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
INSERT INTO `t_ds_relation_rule_input_entry`
(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`, `update_time`)
VALUES(158, 8, 30, NULL, 2, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
INSERT INTO `t_ds_relation_rule_input_entry`
(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`, `update_time`)
VALUES(159, 9, 30, NULL, 2, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
INSERT INTO `t_ds_relation_rule_input_entry`
(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`, `update_time`)
VALUES(160, 10, 30, NULL, 2, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
INSERT INTO `t_ds_relation_rule_input_entry`
(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`, `update_time`)
VALUES(161, 3, 31, NULL, 6, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');
INSERT INTO `t_ds_relation_rule_input_entry`
(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`, `update_time`)
VALUES(150, 8, 29, NULL, 7, '2021-03-03 11:31:24.0', '2021-03-03 11:31:24.0');
VALUES(162, 4, 31, NULL, 7, '2021-03-03 11:31:24.000', '2021-03-03 11:31:24.000');

--
-- Table structure for table t_ds_environment
Expand Down
42 changes: 42 additions & 0 deletions dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1359,6 +1359,12 @@ VALUES(28, 'enum_list', 'input', '$t(enum_list)', NULL, NULL, 'Please enter enum
INSERT IGNORE INTO `t_ds_dq_rule_input_entry`
(`id`, `field`, `type`, `title`, `value`, `options`, `placeholder`, `option_source_type`, `value_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`)
VALUES(29, 'begin_time', 'input', '$t(begin_time)', NULL, NULL, 'Please enter begin time', 0, 0, 0, 1, 1, 0, 0, current_timestamp, current_timestamp);
INSERT IGNORE INTO `t_ds_dq_rule_input_entry`
(`id`, `field`, `type`, `title`, `value`, `options`, `placeholder`, `option_source_type`, `value_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`)
VALUES(30, 'src_database', 'select', '$t(src_database)', NULL, NULL, 'Please select source database', 0, 0, 0, 1, 1, 1, 1, current_timestamp, current_timestamp);
INSERT IGNORE INTO `t_ds_dq_rule_input_entry`
(`id`, `field`, `type`, `title`, `value`, `options`, `placeholder`, `option_source_type`, `value_type`, `input_type`, `is_show`, `can_edit`, `is_emit`, `is_validate`, `create_time`, `update_time`)
VALUES(31, 'target_database', 'select', '$t(target_database)', NULL, NULL, 'Please select target database', 0, 0, 0, 1, 1, 1, 1, current_timestamp, current_timestamp);

--
-- Table structure for table `t_ds_dq_task_statistics_value`
Expand Down Expand Up @@ -1846,6 +1852,42 @@ INSERT IGNORE INTO t_ds_relation_rule_input_entry
(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`, `update_time`)
VALUES(150, 8, 29, NULL, 7, current_timestamp, current_timestamp);

INSERT IGNORE INTO `t_ds_relation_rule_input_entry`
(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`, `update_time`)
VALUES(151, 1, 30, NULL, 2, current_timestamp, current_timestamp);
INSERT IGNORE INTO `t_ds_relation_rule_input_entry`
(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`, `update_time`)
VALUES(152, 2, 30, NULL, 2, current_timestamp, current_timestamp);
INSERT IGNORE INTO `t_ds_relation_rule_input_entry`
(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`, `update_time`)
VALUES(153, 3, 30, NULL, 2, current_timestamp, current_timestamp);
INSERT IGNORE INTO `t_ds_relation_rule_input_entry`
(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`, `update_time`)
VALUES(154, 4, 30, NULL, 2, current_timestamp, current_timestamp);
INSERT IGNORE INTO `t_ds_relation_rule_input_entry`
(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`, `update_time`)
VALUES(155, 5, 30, NULL, 2, current_timestamp, current_timestamp);
INSERT IGNORE INTO `t_ds_relation_rule_input_entry`
(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`, `update_time`)
VALUES(156, 6, 30, NULL, 2, current_timestamp, current_timestamp);
INSERT IGNORE INTO `t_ds_relation_rule_input_entry`
(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`, `update_time`)
VALUES(157, 7, 30, NULL, 2, current_timestamp, current_timestamp);
INSERT IGNORE INTO `t_ds_relation_rule_input_entry`
(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`, `update_time`)
VALUES(158, 8, 30, NULL, 2, current_timestamp, current_timestamp);
INSERT IGNORE INTO `t_ds_relation_rule_input_entry`
(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`, `update_time`)
VALUES(159, 9, 30, NULL, 2, current_timestamp, current_timestamp);
INSERT IGNORE INTO `t_ds_relation_rule_input_entry`
(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`, `update_time`)
VALUES(160, 10, 30, NULL, 2, current_timestamp, current_timestamp);
INSERT IGNORE INTO `t_ds_relation_rule_input_entry`
(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`, `update_time`)
VALUES(161, 3, 31, NULL, 6, current_timestamp, current_timestamp);
INSERT IGNORE INTO `t_ds_relation_rule_input_entry`
(`id`, `rule_id`, `rule_input_entry_id`, `values_map`, `index`, `create_time`, `update_time`)
VALUES(162, 4, 31, NULL, 7, current_timestamp, current_timestamp);
-- ----------------------------
-- Table structure for t_ds_environment
-- ----------------------------
Expand Down
Loading