diff --git a/assembly-package/config/application-exchangis.yml b/assembly-package/config/application-exchangis.yml index 222fecbef..59caa497d 100644 --- a/assembly-package/config/application-exchangis.yml +++ b/assembly-package/config/application-exchangis.yml @@ -1,12 +1,12 @@ server: - port: 9321 + port: 9500 spring: application: name: exchangis-server eureka: client: serviceUrl: - defaultZone: http://127.0.0.1:20303/eureka/ + defaultZone: http://127.0.0.1:20503/eureka/ instance: metadata-map: test: wedatasphere @@ -17,4 +17,4 @@ management: exposure: include: refresh,info logging: - config: classpath:log4j2.xml \ No newline at end of file + config: classpath:log4j2.xml diff --git a/assembly-package/config/exchangis-server.properties b/assembly-package/config/exchangis-server.properties index 39de4fecc..07a162edc 100644 --- a/assembly-package/config/exchangis-server.properties +++ b/assembly-package/config/exchangis-server.properties @@ -15,8 +15,6 @@ # # -#wds.linkis.test.mode=false - wds.linkis.server.mybatis.datasource.url=jdbc:mysql://localhost:3306/database?useSSL=false&characterEncoding=UTF-8&allowMultiQueries=true wds.linkis.server.mybatis.datasource.username=username @@ -27,30 +25,27 @@ wds.linkis.log.clear=true wds.linkis.server.version=v1 -## datasource client +# datasource client wds.exchangis.datasource.client.serverurl= wds.exchangis.datasource.client.authtoken.key=DATASOURCE-AUTH wds.exchangis.datasource.client.authtoken.value=DATASOURCE-AUTH wds.exchangis.datasource.client.dws.version=v1 +# launcher client +wds.exchangis.client.linkis.server-url= +wds.exchangis.client.linkis.token.value=DATASOURCE-AUTH wds.exchangis.datasource.extension.dir=exchangis-extds -wds.exchangis.linkis.serverurl= - ##restful wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.exchangis.datasource.server.restful.api,\ com.webank.wedatasphere.exchangis.project.server.restful,\ com.webank.wedatasphere.exchangis.job.server.restful -wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/framework/appconn/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/workflow/dao/impl/*.xml,classpath*:com/webank/wedatasphere/exchangis/job/server/mapper/impl/*.xml +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/exchangis/job/server/mapper/impl/*.xml:\ + ,classpath*:com/webank/wedatasphere/exchangis/project/server/mapper/impl/*.xml wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.exchangis.dao,\ - com.webank.wedatasphere.exchangis.project.server.dao,\ - com.webank.wedatasphere.linkis.configuration.dao,\ - com.webank.wedatasphere.dss.framework.appconn.dao,\ - com.webank.wedatasphere.dss.workflow.dao,\ - com.webank.wedatasphere.linkis.metadata.dao,\ - com.webank.wedatasphere.exchangis.job.server.mapper,\ - com.webank.wedatasphere.exchangis.job.server.dao + com.webank.wedatasphere.exchangis.project.server.mapper,\ + com.webank.wedatasphere.exchangis.job.server.mapper diff --git a/assembly-package/config/log4j2.xml b/assembly-package/config/log4j2.xml index bcda90df7..70da2f238 100644 --- a/assembly-package/config/log4j2.xml +++ b/assembly-package/config/log4j2.xml @@ -25,16 +25,16 @@ - - - - - - + + + + + - + @@ -42,4 +42,3 @@ - diff --git a/db/exchangis_ddl.sql b/db/exchangis_ddl.sql index 05ecdaf2a..916523046 100644 --- a/db/exchangis_ddl.sql +++ b/db/exchangis_ddl.sql @@ -1,166 +1,170 @@ --- exchangis_v3.exchangis_job_ds_bind definition - -CREATE TABLE `exchangis_job_ds_bind` -( - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `job_id` bigint(20) NOT NULL, - `task_index` int(11) NOT NULL, - `source_ds_id` bigint(20) NOT NULL, - `sink_ds_id` bigint(20) NOT NULL, - PRIMARY KEY (`id`) -) ENGINE = InnoDB - DEFAULT CHARSET = utf8 - COLLATE = utf8_bin; - - --- exchangis_v3.exchangis_job_info definition - -CREATE TABLE `exchangis_job_info` -( - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `project_id` bigint(20) DEFAULT NULL, - `dss_project_id` bigint(20) DEFAULT NULL, - `dss_project_name` varchar(64) DEFAULT NULL, - `node_id` varchar(64) DEFAULT NULL, - `node_name` varchar(64) DEFAULT NULL, - `job_name` varchar(100) DEFAULT NULL, - `job_type` varchar(50) DEFAULT NULL, - `task_name` varchar(255) DEFAULT NULL, - `engine_type` varchar(50) DEFAULT NULL, - `job_labels` varchar(255) DEFAULT NULL, - `job_desc` varchar(255) DEFAULT NULL, - `content` text, - `alarm_user` varchar(50) DEFAULT NULL, - `alarm_level` int(255) DEFAULT NULL, - `proxy_user` varchar(50) DEFAULT NULL, - `execute_node` varchar(255) DEFAULT NULL, - `sync_type` varchar(50) DEFAULT NULL, - `job_params` text, - `domain` varchar(32) DEFAULT NULL, - `create_time` datetime DEFAULT NULL, - `create_user` varchar(50) DEFAULT NULL, - `modify_time` datetime DEFAULT NULL, - `modify_user` varchar(50) DEFAULT NULL, - PRIMARY KEY (`id`), - UNIQUE KEY `uk_job_info_node_id` (`node_id`) -) ENGINE = InnoDB - DEFAULT CHARSET = utf8; - - --- exchangis_v3.exchangis_job_param_config definition - -CREATE TABLE `exchangis_job_param_config` -( - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `config_key` varchar(64) NOT NULL, - `config_name` varchar(64) NOT NULL, - `config_direction` varchar(16) DEFAULT NULL, - `type` varchar(32) NOT NULL, - `ui_type` varchar(32) DEFAULT NULL, - `ui_field` varchar(64) DEFAULT NULL, - `ui_label` varchar(32) DEFAULT NULL, - `unit` varchar(32) DEFAULT NULL, - `required` bit(1) DEFAULT b'0', - `value_type` varchar(32) DEFAULT NULL, - `value_range` varchar(255) DEFAULT NULL, - `default_value` varchar(255) DEFAULT NULL, - `validate_type` varchar(64) DEFAULT NULL, - `validate_range` varchar(64) DEFAULT NULL, - `validate_msg` varchar(255) DEFAULT NULL, - `is_hidden` bit(1) DEFAULT NULL, - `is_advanced` bit(1) DEFAULT NULL, - `level` tinyint(4) DEFAULT NULL, - `treename` varchar(32) DEFAULT NULL, - `sort` int(11) DEFAULT NULL, - `description` varchar(255) DEFAULT NULL, - `status` tinyint(4) DEFAULT NULL, - PRIMARY KEY (`id`) --- UNIQUE KEY `config_key` (`config_key`) -) ENGINE = InnoDB - DEFAULT CHARSET = utf8; - - --- exchangis_v3.exchangis_launch_task definition - -CREATE TABLE `exchangis_launch_task` -( - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `task_name` varchar(100) DEFAULT NULL COMMENT '子任务名称', - `job_id` bigint(20) DEFAULT NULL COMMENT '所属任务id', - `job_name` varchar(100) DEFAULT NULL COMMENT '所属任务名称', - `content` text COMMENT '子任务执行内容', - `execute_node` varchar(255) DEFAULT NULL COMMENT '执行节点', - `create_time` datetime DEFAULT NULL COMMENT '子任务创建时间', - `create_user` varchar(50) DEFAULT NULL COMMENT '创建用户', - `launch_time` datetime DEFAULT NULL COMMENT '触发时间', - `proxy_user` varchar(50) DEFAULT NULL COMMENT '执行/代理用户', - `params_json` text COMMENT '作业参数', - `launch_id` varchar(64) DEFAULT NULL, - `status` varchar(50) DEFAULT NULL COMMENT '状态：SUCCESS | FAILED | RUNNING | BUSY | IDLE | UNLOCK', - `complete_time` datetime DEFAULT NULL COMMENT '完成/中止时间', - `engine_type` varchar(64) DEFAULT NULL, - PRIMARY KEY (`id`) -) ENGINE = InnoDB - DEFAULT CHARSET = utf8; - - --- exchangis_v3.exchangis_metric definition - -CREATE TABLE `exchangis_metric` -( - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `title` varchar(128) DEFAULT NULL COMMENT '指标名称', - `norm` varchar(128) DEFAULT NULL COMMENT '指标key', - `value` text COMMENT '指标值', - `ts` datetime DEFAULT NULL COMMENT '采集时间', - `version` bigint(20) NOT NULL DEFAULT '1', - PRIMARY KEY (`id`), - UNIQUE KEY `norm` (`norm`) -) ENGINE = InnoDB - DEFAULT CHARSET = utf8; - - --- exchangis_v3.exchangis_project definition - -CREATE TABLE `exchangis_project` -( - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `dss_project_id` bigint(20) DEFAULT NULL, - `workspace_name` varchar(64) DEFAULT NULL, - `dss_name` varchar(64) DEFAULT NULL, - `name` varchar(64) NOT NULL, - `description` varchar(255) DEFAULT NULL, - `create_time` datetime DEFAULT NULL, - `last_update_time` datetime DEFAULT NULL, - `create_by` varchar(64) DEFAULT NULL, - `last_update_by` varchar(64) DEFAULT NULL, - `tags` varchar(255) DEFAULT NULL, - `domain` varchar(32) DEFAULT NULL, - `exec_users` varchar(255) DEFAULT NULL, - `view_users` varchar(255) DEFAULT NULL, - `edit_users` varchar(255) DEFAULT NULL, - PRIMARY KEY (`id`), - UNIQUE KEY `name_UNIQUE` (`name`), - UNIQUE KEY `uk_project_dss_project_id` (`dss_project_id`), - UNIQUE KEY `workspace_name_UNIQUE` (`workspace_name`) -) ENGINE = InnoDB - DEFAULT CHARSET = utf8; - - --- exchangis_v3.exchangis_project_relation definition - -CREATE TABLE `exchangis_project_relation` -( - `id` int(11) NOT NULL AUTO_INCREMENT, - `project_id` bigint(20) DEFAULT NULL, - `node_id` bigint(20) DEFAULT NULL, - `project_version` varchar(32) COLLATE utf8_bin DEFAULT NULL, - `flow_version` varchar(32) COLLATE utf8_bin DEFAULT NULL, - `resource_id` bigint(20) DEFAULT NULL, - `version` varchar(32) COLLATE utf8_bin DEFAULT NULL, - PRIMARY KEY (`id`), - UNIQUE KEY `uq_project_relation_project_id_node_id` (`project_id`, `node_id`), - UNIQUE KEY `uq_project_relation_node_id` (`node_id`) -) ENGINE = InnoDB - DEFAULT CHARSET = utf8 - COLLATE = utf8_bin; \ No newline at end of file +-- exchangis_v4.exchangis_job_ds_bind definition + +CREATE TABLE `exchangis_job_ds_bind` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `job_id` bigint(20) NOT NULL, + `task_index` int(11) NOT NULL, + `source_ds_id` bigint(20) NOT NULL, + `sink_ds_id` bigint(20) NOT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=5043 DEFAULT CHARSET=utf8 COLLATE=utf8_bin + + +-- exchangis_v4.exchangis_job_entity definition + +CREATE TABLE `exchangis_job_entity` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `name` varchar(100) NOT NULL, + `create_time` datetime DEFAULT NULL, + `last_update_time` datetime DEFAULT NULL, + `engine_type` varchar(45) DEFAULT '', + `job_labels` varchar(255) DEFAULT NULL, + `create_user` varchar(100) DEFAULT NULL, + `job_content` mediumtext, + `execute_user` varchar(100) DEFAULT '', + `job_params` text, + `job_desc` varchar(255) DEFAULT NULL, + `job_type` varchar(50) DEFAULT NULL, + `project_id` bigint(13) DEFAULT NULL, + `source` text, + `modify_user` varchar(50) DEFAULT NULL COMMENT '修改用户', + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=129 DEFAULT CHARSET=utf8 + + +-- exchangis_v4.exchangis_job_param_config definition + +CREATE TABLE `exchangis_job_param_config` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `config_key` varchar(64) NOT NULL, + `config_name` varchar(64) NOT NULL, + `config_direction` varchar(16) DEFAULT NULL, + `type` varchar(32) NOT NULL, + `ui_type` varchar(32) DEFAULT NULL, + `ui_field` varchar(64) DEFAULT NULL, + `ui_label` varchar(32) DEFAULT NULL, + `unit` varchar(32) DEFAULT NULL, + `required` bit(1) DEFAULT b'0', + `value_type` varchar(32) DEFAULT NULL, + `value_range` varchar(255) DEFAULT NULL, + `default_value` varchar(255) DEFAULT NULL, + `validate_type` varchar(64) DEFAULT NULL, + `validate_range` varchar(64) DEFAULT NULL, + `validate_msg` varchar(255) DEFAULT NULL, + `is_hidden` bit(1) DEFAULT NULL, + `is_advanced` bit(1) DEFAULT NULL, + `source` varchar(255) DEFAULT NULL, + `level` tinyint(4) DEFAULT NULL, + `treename` varchar(32) DEFAULT NULL, + `sort` int(11) DEFAULT NULL, + `description` varchar(255) DEFAULT NULL, + `status` tinyint(4) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=32 DEFAULT CHARSET=utf8 + +-- exchangis_v4.exchangis_project_info definition + +CREATE TABLE `exchangis_project_info` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `name` varchar(64) NOT NULL, + `description` varchar(255) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `last_update_time` datetime DEFAULT NULL, + `create_user` varchar(64) DEFAULT NULL, + `last_update_user` varchar(64) DEFAULT NULL, + `project_labels` varchar(255) DEFAULT NULL, + `domain` varchar(32) DEFAULT NULL, + `exec_users` varchar(255) DEFAULT NULL, + `view_users` varchar(255) DEFAULT NULL, + `edit_users` varchar(255) DEFAULT NULL, + `source` text, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=1497870871035973652 DEFAULT CHARSET=utf8 + +-- exchangis_v4.exchangis_job_entity definition + + +CREATE TABLE `exchangis_job_entity` ( + `id` bigint(20) NOT NULL, + `name` varchar(100) NOT NULL, + `create_time` datetime DEFAULT NULL, + `last_update_time` datetime DEFAULT NULL, + `engine_type` varchar(45) DEFAULT '', + `job_labels` varchar(64) DEFAULT NULL, + `create_user` varchar(100) DEFAULT NULL, + `job_content` text NOT NULL, + `execute_user` varchar(100) DEFAULT '', + `job_params` text NOT NULL, + `project_id` bigint(13) DEFAULT NULL, + `source` varchar(255) NOT NULL, + `modify_user` varchar(50) DEFAULT NULL COMMENT '修改用户', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 + +-- exchangis_v4.exchangis_launchable_task definition + +CREATE TABLE `exchangis_launchable_task` ( + `id` bigint(13) NOT NULL, + `name` varchar(100) NOT NULL, + `job_execution_id` varchar(64) DEFAULT NULL, + `create_time` datetime DEFAULT NULL, + `last_update_time` datetime DEFAULT NULL, + `engine_type` varchar(45) DEFAULT '', + `execute_user` varchar(50) DEFAULT '', + `linkis_job_name` varchar(100) NOT NULL, + `linkis_job_content` text NOT NULL, + `linkis_params` varchar(255) DEFAULT NULL, + `linkis_source` varchar(64) DEFAULT NULL, + `labels` varchar(64) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 + +-- exchangis_v4.exchangis_launched_job_entity definition + +CREATE TABLE `exchangis_launched_job_entity` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `name` varchar(100) NOT NULL, + `create_time` datetime DEFAULT NULL, + `last_update_time` datetime DEFAULT NULL, + `job_id` bigint(20) DEFAULT NULL, + `launchable_task_num` int(20) DEFAULT '0', + `engine_type` varchar(100) DEFAULT NULL, + `execute_user` varchar(100) DEFAULT NULL, + `job_name` varchar(100) DEFAULT NULL, + `status` varchar(100) DEFAULT NULL, + `progress` varchar(100) DEFAULT NULL, + `error_code` varchar(64) DEFAULT NULL, + `error_msg` varchar(255) DEFAULT NULL, + `retry_num` bigint(10) DEFAULT NULL, + `job_execution_id` varchar(255) DEFAULT NULL, + `log_path` varchar(255) DEFAULT NULL, + `create_user` varchar(100) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `job_execution_id_UNIQUE` (`job_execution_id`) +) ENGINE=InnoDB AUTO_INCREMENT=708 DEFAULT CHARSET=utf8 + +-- exchangis_v4.exchangis_launched_task_entity definition + +CREATE TABLE `exchangis_launched_task_entity` ( + `id` bigint(20) NOT NULL, + `name` varchar(100) NOT NULL, + `create_time` datetime DEFAULT NULL, + `last_update_time` datetime DEFAULT NULL, + `job_id` bigint(20) DEFAULT NULL, + `engine_type` varchar(100) DEFAULT NULL, + `execute_user` varchar(100) DEFAULT NULL, + `job_name` varchar(100) DEFAULT NULL, + `progress` varchar(64) DEFAULT NULL, + `error_code` varchar(64) DEFAULT NULL, + `error_msg` varchar(255) DEFAULT NULL, + `retry_num` bigint(10) DEFAULT NULL, + `task_id` varchar(64) DEFAULT NULL, + `linkis_job_id` varchar(200) DEFAULT NULL, + `linkis_job_info` varchar(1000) DEFAULT NULL, + `job_execution_id` varchar(100) DEFAULT NULL, + `launch_time` datetime DEFAULT NULL, + `running_time` datetime DEFAULT NULL, + `metrics` text, + `status` varchar(64) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 \ No newline at end of file diff --git a/db/exchangis_dml.sql b/db/exchangis_dml.sql index c264cdd11..a7172b2ee 100644 --- a/db/exchangis_dml.sql +++ b/db/exchangis_dml.sql @@ -1,29 +1,15 @@ -- 插入 job_param_config 记录 -INSERT INTO `exchangis_job_param_config` (`id`, `config_key`, `config_name`, `config_direction`, `type`, `ui_type`, `ui_field`, `ui_label`, `unit`, `required`, `value_type`, `value_range`, `default_value`, `validate_type`, `validate_range`, `validate_msg`, `is_hidden`, `is_advanced`, `source`, `level`, `treename`, `sort`, `description`, `status`) -VALUES -(1,'exchangis.datax.setting.speed.bytes','作业速率限制','','DATAX','INPUT','exchangis.datax.setting.speed.bytes','作业速率限制','Mb/s',b'1','NUMBER','','','REGEX','^[1-9]\\d*$','作业速率限制输入错误',b'0',b'0','', 1,'',1,'',1), -(2,'exchangis.datax.setting.speed.records','作业记录数限制','','DATAX','INPUT','exchangis.datax.setting.speed.records','作业记录数限制','条/s',b'1','NUMBER','','','REGEX','^[1-9]\\d*$','作业记录数限制输入错误',b'0',b'0','', 1,'',2,'',1), -(3,'exchangis.datax.setting.max.parallelism','作业最大并行度','','DATAX','INPUT','exchangis.datax.setting.max.parallelism','作业最大并行度','个',b'1','NUMBER','','1','REGEX','^[1-9]\\d*$','作业最大并行度输入错误',b'0',b'0','', 1,'',3,'',1), -(4,'exchangis.datax.setting.max.memory','作业最大使用内存','','DATAX','INPUT','exchangis.datax.setting.max.memory','作业最大使用内存','Mb',b'1','NUMBER','','1000','REGEX','^[1-9]\\d*$','作业最大使用内存输入错误',b'0',b'0','', 1,'',4,'',1), -(5,'exchangis.datax.setting.errorlimit.record','最多错误记录数','','DATAX','INPUT','exchangis.datax.setting.errorlimit.record','最多错误记录数','条',b'1','NUMBER','','','REGEX','^[1-9]\\d*$','最多错误记录数输入错误',b'0',b'0','', 1,'',5,'',1), -(6,'exchangis.sqoop.setting.max.parallelism','作业最大并行数','','SQOOP','INPUT','exchangis.sqoop.setting.max.parallelism','作业最大并行数','个',b'1','NUMBER','','1','REGEX','^[1-9]\\d*$','作业最大并行数输入错误',b'0',b'0','', 1,'',1,'',1), -(7,'exchangis.sqoop.setting.max.memory','作业最大内存','','SQOOP','INPUT','exchangis.sqoop.setting.max.memory','作业最大内存','Mb',b'1','NUMBER','','1000','REGEX','^[1-9]\\d*$','作业最大内存输入错误',b'0',b'0','', 1,'',2,'',1), -(8,'exchangis.job.ds.params.datax.mysql.r.join_condition','连接条件','DATAX-SOURCE','MYSQL','INPUT','exchangis.job.ds.params.datax.mysql.r.join_condition','连接条件','',b'0','VARCHAR','','','','','连接条件输入错误',b'0',b'0','', 1,'',1,'',1), -(9,'exchangis.job.ds.params.datax.mysql.r.where_condition','WHERE条件','DATAX-SOURCE','MYSQL','INPUT','exchangis.job.ds.params.datax.mysql.r.where_condition','WHERE条件','',b'0','VARCHAR','','','','','WHERE条件输入错误',b'0',b'0','', 1,'',2,'',1), -(10,'exchangis.job.ds.params.datax.hive.r.trans_proto','传输方式','DATAX-SOURCE','HIVE','OPTION','exchangis.job.ds.params.datax.hive.r.trans_proto','传输方式','',b'1','OPTION','[\"记录\",\"二进制\"]','记录','','','传输方式输入错误',b'0',b'0','', 1,'',1,'',1), -(11,'exchangis.job.ds.params.datax.hive.r.partition','分区信息','DATAX-SOURCE','HIVE','INPUT','exchangis.job.ds.params.datax.hive.r.partition','分区信息','',b'0','VARCHAR','','','','','分区信息输入错误',b'0',b'0','', 1,'',2,'',1), -(12,'exchangis.job.ds.params.datax.hive.r.row_format','字段格式','DATAX-SOURCE','HIVE','INPUT','exchangis.job.ds.params.datax.hive.r.row_format','字段格式','',b'0','VARCHAR','','','','','字段格式输入错误',b'0',b'0','', 1,'',3,'',1), -(13,'exchangis.job.ds.params.datax.mysql.w.write_type','写入方式','DATAX-SINK','MYSQL','OPTION','exchangis.job.ds.params.datax.mysql.w.write_type','写入方式','',b'1','OPTION','[\"INSERT\",\"REPLACE\",\"UPDATE\"]','INSERT','','','写入方式输入错误',b'0',b'0','', 1,'',1,'',1), -(14,'exchangis.job.ds.params.datax.mysql.w.batch_size','批量大小','DATAX-SINK','MYSQL','INPUT','exchangis.job.ds.params.datax.mysql.w.batch_size','批量大小','',b'0','NUMBER','','1000','REGEX','^[1-9]\\d*$','批量大小输入错误',b'0',b'0','', 1,'',2,'',1), -(16,'exchangis.job.ds.params.datax.hive.w.partition','分区信息','DATAX-SINK','HIVE','INPUT','exchangis.job.ds.params.datax.hive.w.partition','分区信息','',b'0','VARCHAR','','','','','分股信息输入错误',b'0',b'0','', 1,'',2,'',1), -(17,'exchangis.job.ds.params.datax.hive.w.row_format','字段格式','DATAX-SINK','HIVE','INPUT','exchangis.job.ds.params.datax.hive.w.row_format','字段格式','',b'0','VARCHAR','','','','','字段格式输入错误',b'0',b'0','', 1,'',3,'',1), -(18,'exchangis.job.ds.params.datax.hive.w.write_type','写入策略','DATAX-SINK','HIVE','OPTION','exchangis.job.ds.params.datax.hive.w.write_type','写入策略','',b'1','OPTION','[\"清空目录\",\"追加数据\"]','清空目录','','','写入策略输入错误',b'0',b'0','', 1,'',4,'',1), -(19,'exchangis.job.ds.params.datax.hive.w.sync_meta','同步元数据','DATAX-SINK','HIVE','OPTION','exchangis.job.ds.params.datax.hive.w.sync_meta','同步元数据','',b'1','OPTION','[\"是\",\"否\"]','是','','','同步元数据输入错误',b'0',b'0','', 1,'',5,'',1), -(20,'exchangis.job.ds.params.sqoop.mysql.r.join_condition','连接条件','SQOOP-SOURCE','MYSQL','INPUT','exchangis.job.ds.params.sqoop.mysql.r.join_condition','连接条件','',b'0','VARCHAR','','','','','连接条件输入错误',b'0',b'0','', 1,'',1,'',1), -(21,'exchangis.job.ds.params.sqoop.mysql.r.where_condition','WHERE条件','SQOOP-SOURCE','MYSQL','INPUT','exchangis.job.ds.params.sqoop.mysql.r.where_condition','WHERE条件','',b'0','VARCHAR','','','','','WHERE条件输入错误',b'0',b'0','', 1,'',2,'',1), -(22,'exchangis.job.ds.params.sqoop.mysql.w.write_type','写入方式','SQOOP-SINK','MYSQL','OPTION','exchangis.job.ds.params.sqoop.mysql.w.write_type','写入方式','',b'1','OPTION','[\"UPDATEONLY\",\"ALLOWINSERT\"]','UPDATEONLY','','','写入方式输入错误',b'0',b'0','', 1,'',1,'',1), -(23,'exchangis.job.ds.params.sqoop.mysql.w.batch_size','批量大小','SQOOP-SINK','MYSQL','INPUT','exchangis.job.ds.params.sqoop.mysql.w.batch_size','批量大小','',b'0','NUMBER','','1000','REGEX','^[1-9]\\d*$','批量大小输入错误',b'0',b'0','', 1,'',2,'',1), -(25,'exchangis.job.ds.params.sqoop.hive.r.partition','分区信息','SQOOP-SOURCE','HIVE','INPUT','exchangis.job.ds.params.sqoop.hive.r.partition','分区信息','',b'0','VARCHAR','','','','','分区信息输入错误',b'0',b'0','/api/rest_j/v1/exchangis/job/partitionInfo?dataSourceType=&dbname&table=', 1,'',2,'',1), -(26,'exchangis.job.ds.params.sqoop.hive.r.row_format','字段格式','SQOOP-SOURCE','HIVE','INPUT','exchangis.job.ds.params.sqoop.hive.r.row_format','字段格式','',b'0','VARCHAR','','','','','字段格式输入错误',b'0',b'0','', 1,'',3,'',1), -(28,'exchangis.job.ds.params.sqoop.hive.w.partition','分区信息','SQOOP-SINK','HIVE','INPUT','exchangis.job.ds.params.sqoop.hive.w.partition','分区信息','',b'0','VARCHAR','','','','','分区信息输入错误',b'0',b'0','', 1,'',2,'',1), -(29,'exchangis.job.ds.params.sqoop.hive.w.row_format','字段格式','SQOOP-SINK','HIVE','INPUT','exchangis.job.ds.params.sqoop.hive.w.row_format','字段格式','',b'0','VARCHAR','','','','','字段格式输入错误',b'0',b'0','', 1,'',3,'',1); \ No newline at end of file +INSERT INTO udes_gzpc_pub_sit_01.exchangis_job_param_config (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status) VALUES +('setting.speed.bytes','作业速率限制','','DATAX','INPUT','setting.speed.bytes','作业速率限制','Mb/s',1,'NUMBER','','','REGEX','^[1-9]\\d*$','作业速率限制输入错误',0,0,'',1,'',1,'',1) +,('setting.speed.records','作业记录数限制','','DATAX','INPUT','setting.speed.records','作业记录数限制','条/s',1,'NUMBER','','','REGEX','^[1-9]\\d*$','作业记录数限制输入错误',0,0,'',1,'',2,'',1) +,('setting.max.parallelism','作业最大并行度','','DATAX','INPUT','setting.max.parallelism','作业最大并行度','个',1,'NUMBER','','1','REGEX','^[1-9]\\d*$','作业最大并行度输入错误',0,0,'',1,'',3,'',1) +,('setting.max.memory','作业最大使用内存','','DATAX','INPUT','setting.max.memory','作业最大使用内存','Mb',1,'NUMBER','','1024','REGEX','^[1-9]\\d*$','作业最大使用内存输入错误',0,0,'',1,'',4,'',1) +,('setting.errorlimit.record','最多错误记录数','','DATAX','INPUT','setting.errorlimit.record','最多错误记录数','条',1,'NUMBER','','','REGEX','^[1-9]\\d*$','最多错误记录数输入错误',0,0,'',1,'',5,'',1) +,('setting.max.parallelism','作业最大并行数','','SQOOP','INPUT','setting.max.parallelism','作业最大并行数','个',1,'NUMBER','','1','REGEX','^[1-9]\\d*$','作业最大并行数输入错误',0,0,'',1,'',1,'',1) +,('setting.max.memory','作业最大内存','','SQOOP','INPUT','setting.max.memory','作业最大内存','Mb',1,'NUMBER','','1024','REGEX','^[1-9]\\d*$','作业最大内存输入错误',0,0,'',1,'',2,'',1) +,('where','WHERE条件','SOURCE','MYSQL','INPUT','where','WHERE条件','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,500}$','WHERE条件输入过长',0,0,'',1,'',2,'',1) +,('writeMode','写入方式','SQOOP-SINK','HIVE','OPTION','writeMode','写入方式(OVERWRITE只对TEXT类型表生效)','',1,'OPTION','["OVERWRITE","APPEND"]','OVERWRITE','','','写入方式输入错误',0,0,'',1,'',1,'',1) +,('partition','分区信息','SINK','HIVE','MAP','partition','分区信息(文本)','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','分区信息过长',0,0,'/api/rest_j/v1/exchangis/datasources/render/partition/element/map',1,'',2,'',1) +,('partition','分区信息','SOURCE','HIVE','MAP','partition','分区信息(文本)',NULL,0,'VARCHAR',NULL,NULL,'REGEX','^[\\s\\S]{0,50}$','分区信息过长',0,0,'/api/rest_j/v1/exchangis/datasources/render/partition/element/map',1,NULL,1,NULL,1) +,('writeMode','写入方式','SQOOP-SINK','MYSQL','OPTION','writeMode','写入方式',NULL,1,'OPTION','["INSERT","UPDATE"]','INSERT',NULL,NULL,'写入方式输入错误',0,0,NULL,1,NULL,1,NULL,1) +; diff --git a/exchangis-dao/pom.xml b/exchangis-dao/pom.xml index 43262ec70..a8117fafc 100644 --- a/exchangis-dao/pom.xml +++ b/exchangis-dao/pom.xml @@ -14,6 +14,7 @@ 8 8 + 6.0.15.Final @@ -26,11 +27,17 @@ org.apache.linkis linkis-module ${linkis.version} + + + validation-api + javax.validation + + - org.apache.linkis - linkis-common - ${linkis.version} + org.hibernate + hibernate-validator + ${hibernate.validator} diff --git a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/ValidatorConfiguration.java b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/ValidatorConfiguration.java new file mode 100644 index 000000000..882e71f50 --- /dev/null +++ b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/ValidatorConfiguration.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.exchangis; + +import org.hibernate.validator.HibernateValidator; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import javax.validation.Validation; +import javax.validation.Validator; + +/** + * Bean validator + */ +@Configuration +public class ValidatorConfiguration { + @Bean + public Validator validator(){ + return Validation.byProvider(HibernateValidator.class) + .configure().failFast(true) + .buildValidatorFactory().getValidator(); + } +} diff --git a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/pager/PageQuery.java b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/pager/PageQuery.java new file mode 100644 index 000000000..ba95c3d80 --- /dev/null +++ b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/pager/PageQuery.java @@ -0,0 +1,49 @@ +package com.webank.wedatasphere.exchangis.common.pager; + +import java.util.Objects; + +/** + * Query Vo + */ +public class PageQuery { + + protected Integer current = 1; + + protected Integer size = 10; + + protected Integer page; + + protected Integer pageSize; + + public Integer getCurrent() { + return current; + } + + public void setCurrent(Integer current) { + this.current = current; + } + + public Integer getSize() { + return size; + } + + public void setSize(Integer size) { + this.size = size; + } + + public Integer getPage() { + return Objects.nonNull(page) ? page : current; + } + + public void setPage(Integer page) { + this.page = page; + } + + public Integer getPageSize() { + return Objects.nonNull(pageSize) ? pageSize : size; + } + + public void setPageSize(Integer pageSize) { + this.pageSize = pageSize; + } +} diff --git a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/pager/PageResult.java b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/pager/PageResult.java new file mode 100644 index 000000000..8774b54b4 --- /dev/null +++ b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/pager/PageResult.java @@ -0,0 +1,61 @@ +package com.webank.wedatasphere.exchangis.common.pager; + +import com.github.pagehelper.PageInfo; +import org.apache.linkis.server.Message; + +import java.util.List; + +/** + * Page result + * @param + */ +public class PageResult{ + /** + * Total + */ + private Long total; + + /** + * List + */ + private List list; + + public PageResult(){ + + } + + public PageResult(PageInfo pageInfo){ + this.total = pageInfo.getTotal(); + this.list = pageInfo.getList(); + } + /** + * To Message(in linkis-common) + * @return message + */ + public Message toMessage(String info){ + Message message = Message.ok(info); + message.data("total", total); + message.data("list", list); + return message; + } + + public Message toMessage(){ + return toMessage(""); + } + + public Long getTotal() { + return total; + } + + public void setTotal(Long total) { + this.total = total; + } + + public List getList() { + return list; + } + + public void setList(List list) { + this.list = list; + } +} diff --git a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/validator/groups/InsertGroup.java b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/validator/groups/InsertGroup.java new file mode 100644 index 000000000..1c05bc04d --- /dev/null +++ b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/validator/groups/InsertGroup.java @@ -0,0 +1,10 @@ +package com.webank.wedatasphere.exchangis.common.validator.groups; + +import javax.validation.groups.Default; + +/** + * Insert group for validator + */ +public interface InsertGroup extends Default { + +} diff --git a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/validator/groups/UpdateGroup.java b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/validator/groups/UpdateGroup.java new file mode 100644 index 000000000..ce05e3f7e --- /dev/null +++ b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/validator/groups/UpdateGroup.java @@ -0,0 +1,9 @@ +package com.webank.wedatasphere.exchangis.common.validator.groups; + +import javax.validation.groups.Default; + +/** + * Update group for validator + */ +public interface UpdateGroup extends Default { +} diff --git a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/dao/domain/ExchangisJobEntity.java b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/dao/domain/ExchangisJobEntity.java deleted file mode 100644 index e28dee307..000000000 --- a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/dao/domain/ExchangisJobEntity.java +++ /dev/null @@ -1,221 +0,0 @@ -package com.webank.wedatasphere.exchangis.dao.domain; - -import java.util.Date; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableField; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; - -@TableName("exchangis_job_info") -public class ExchangisJobEntity { - - @TableId(type = IdType.AUTO) - private Long id; - - @TableField("project_id") - private Long projectId; - - @TableField("dss_project_id") - private Long dssProjectId; - - @TableField("node_id") - private String nodeId; - - @TableField("node_name") - private String nodeName; - - private String jobName; - - private String jobType; - - private String engineType; - - private String jobLabels; - - private String jobDesc; - - private String content; - - private String alarmUser; - - private Integer alarmLevel; - - private String proxyUser; - - private String executeNode; - - private String syncType; - - private String jobParams; - - private Date createTime; - - private String createUser; - - private Date modifyTime; - - private String modifyUser; - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public Long getProjectId() { - return projectId; - } - - public void setProjectId(Long projectId) { - this.projectId = projectId; - } - - public Long getDssProjectId() { - return dssProjectId; - } - - public void setDssProjectId(Long dssProjectId) { this.dssProjectId = dssProjectId; } - - public String getNodeId() { return nodeId; } - - public void setNodeId(String nodeId) { this.nodeId = nodeId; } - - public String getNodeName() { return nodeName; } - - public void setNodeName(String nodeName) { this.nodeName = nodeName; } - - public String getJobName() { - return jobName; - } - - public void setJobName(String jobName) { - this.jobName = jobName; - } - - public String getJobType() { - return jobType; - } - - public void setJobType(String jobType) { - this.jobType = jobType; - } - - public String getEngineType() { - return engineType; - } - - public void setEngineType(String engineType) { - this.engineType = engineType; - } - - public String getJobLabels() { - return jobLabels; - } - - public void setJobLabels(String jobLabels) { - this.jobLabels = jobLabels; - } - - public String getJobDesc() { - return jobDesc; - } - - public void setJobDesc(String jobDesc) { - this.jobDesc = jobDesc; - } - - public String getContent() { - return content; - } - - public void setContent(String content) { - this.content = content; - } - - public String getAlarmUser() { - return alarmUser; - } - - public void setAlarmUser(String alarmUser) { - this.alarmUser = alarmUser; - } - - public Integer getAlarmLevel() { - return alarmLevel; - } - - public void setAlarmLevel(Integer alarmLevel) { - this.alarmLevel = alarmLevel; - } - - public String getProxyUser() { - return proxyUser; - } - - public void setProxyUser(String proxyUser) { - this.proxyUser = proxyUser; - } - - public String getExecuteNode() { - return executeNode; - } - - public void setExecuteNode(String executeNode) { - this.executeNode = executeNode; - } - - public String getSyncType() { - return syncType; - } - - public void setSyncType(String syncType) { - this.syncType = syncType; - } - - public String getJobParams() { - return jobParams; - } - - public void setJobParams(String jobParams) { - this.jobParams = jobParams; - } - - public Date getCreateTime() { - return createTime; - } - - public void setCreateTime(Date createTime) { - this.createTime = createTime; - } - - public String getCreateUser() { - return createUser; - } - - public void setCreateUser(String createUser) { - this.createUser = createUser; - } - - public Date getModifyTime() { - return modifyTime; - } - - public void setModifyTime(Date modifyTime) { - this.modifyTime = modifyTime; - } - - public String getModifyUser() { - return modifyUser; - } - - public void setModifyUser(String modifyUser) { - this.modifyUser = modifyUser; - } - -} \ No newline at end of file diff --git a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/dao/domain/ExchangisJobParamConfig.java b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/dao/domain/ExchangisJobParamConfig.java index b35fe0934..40ce6e138 100644 --- a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/dao/domain/ExchangisJobParamConfig.java +++ b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/dao/domain/ExchangisJobParamConfig.java @@ -64,7 +64,7 @@ public class ExchangisJobParamConfig { private Boolean advanced; /** - * store url exa. http://127.0.0.1/api/v1/exchangis/xxx + * store url exa. http://127.0.0.1/api/v1/dss/exchangis/main/xxx */ private String source; diff --git a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/dao/hook/MapperHook.java b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/dao/hook/MapperHook.java index 89a21334f..ba0de2947 100644 --- a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/dao/hook/MapperHook.java +++ b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/dao/hook/MapperHook.java @@ -1,6 +1,5 @@ package com.webank.wedatasphere.exchangis.dao.hook; -import com.webank.wedatasphere.exchangis.dao.mapper.ExchangisJobInfoMapper; import com.webank.wedatasphere.exchangis.dao.mapper.ExchangisJobParamConfigMapper; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -8,23 +7,13 @@ @Component public class MapperHook { - private ExchangisJobInfoMapper exchangisJobInfoMapper; private ExchangisJobParamConfigMapper exchangisJobParamConfigMapper; @Autowired - public MapperHook(ExchangisJobInfoMapper exchangisJobInfoMapper, ExchangisJobParamConfigMapper exchangisJobParamConfigMapper) { - this.exchangisJobInfoMapper = exchangisJobInfoMapper; + public MapperHook(ExchangisJobParamConfigMapper exchangisJobParamConfigMapper) { this.exchangisJobParamConfigMapper = exchangisJobParamConfigMapper; } - public ExchangisJobInfoMapper getExchangisJobInfoMapper() { - return exchangisJobInfoMapper; - } - - public void setExchangisJobInfoMapper(ExchangisJobInfoMapper exchangisJobInfoMapper) { - this.exchangisJobInfoMapper = exchangisJobInfoMapper; - } - public ExchangisJobParamConfigMapper getExchangisJobParamConfigMapper() { return exchangisJobParamConfigMapper; } diff --git a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/dao/mapper/ExchangisJobInfoMapper.java b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/dao/mapper/ExchangisJobInfoMapper.java deleted file mode 100644 index 0dc563635..000000000 --- a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/dao/mapper/ExchangisJobInfoMapper.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.webank.wedatasphere.exchangis.dao.mapper; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobEntity; -import org.apache.ibatis.annotations.Mapper; - -@Mapper -public interface ExchangisJobInfoMapper extends BaseMapper { -} diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/domain/MetaColumn.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/domain/MetaColumn.java new file mode 100644 index 000000000..297a0333c --- /dev/null +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/domain/MetaColumn.java @@ -0,0 +1,69 @@ +package com.webank.wedatasphere.exchangis.datasource.core.domain; + +/** + * Meta column + */ +public class MetaColumn { + + /** + * Column index + */ + private int index = -1; + + /** + * Is primary key + */ + private boolean primaryKey; + + /** + * Name + */ + private String name; + + /** + * Type symbol + */ + private String type; + + public MetaColumn(){ + + } + + public MetaColumn(int index, String name, String type, boolean primaryKey){ + this.index = index; + this.name = name; + this.type = type; + this.primaryKey = primaryKey; + } + public int getIndex() { + return index; + } + + public void setIndex(int index) { + this.index = index; + } + + public boolean isPrimaryKey() { + return primaryKey; + } + + public void setPrimaryKey(boolean primaryKey) { + this.primaryKey = primaryKey; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } +} diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/exception/ExchangisDataSourceException.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/exception/ExchangisDataSourceException.java index e755366b5..2486a3bb6 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/exception/ExchangisDataSourceException.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/exception/ExchangisDataSourceException.java @@ -2,6 +2,8 @@ import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.common.exception.ExceptionLevel; +import org.apache.linkis.common.exception.LinkisRuntimeException; public class ExchangisDataSourceException extends ErrorException { @@ -17,4 +19,16 @@ public ExchangisDataSourceException(int errCode, String desc, String ip, int por super(errCode, desc, ip, port, serviceKind); } + public static class Runtime extends LinkisRuntimeException { + + public Runtime(int errCode, String desc, Throwable t) { + super(errCode, desc); + super.initCause(t); + } + + @Override + public ExceptionLevel getLevel() { + return ExceptionLevel.ERROR; + } + } } \ No newline at end of file diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/service/MetadataInfoService.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/service/MetadataInfoService.java index 58d380cea..04b7d78f3 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/service/MetadataInfoService.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/service/MetadataInfoService.java @@ -1,5 +1,6 @@ package com.webank.wedatasphere.exchangis.datasource.core.service; +import com.webank.wedatasphere.exchangis.datasource.core.domain.MetaColumn; import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisDataSourceException; import com.webank.wedatasphere.exchangis.datasource.core.service.rpc.ServiceRpcClient; @@ -45,4 +46,15 @@ Map getTableProps(ServiceRpcClient rpcClient, String userName * @throws ExchangisDataSourceException */ List getPartitionKeys(String userName, Long dataSourceId, String database, String table) throws ExchangisDataSourceException; + + /** + * Get columns + * @param userName userName + * @param dataSourceId data source id + * @param database database + * @param table table + * @return + * @throws ExchangisDataSourceException + */ + List getColumns(String userName, Long dataSourceId, String database, String table) throws ExchangisDataSourceException; } diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/utils/Json.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/utils/Json.java index 68d7b81ac..a59822cec 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/utils/Json.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/utils/Json.java @@ -70,7 +70,7 @@ public static T fromJson(String json, Class tClass, Class... parameter } catch (Exception e) { logger.warn("Fail to process method 'fromJson(" + (json.length() > 5 ? json.substring(0, 5) + "..." : json) + ": " + json.getClass() + - ", " + tClass.getSimpleName() + ": "+ Class.class + ", ...: " + Class.class + ")"); + ", " + tClass.getSimpleName() + ": "+ Class.class + ", ...: " + Class.class + ")", e); return null; } } diff --git a/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/service/LinkisDataSourceInfoService.java b/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/service/LinkisDataSourceInfoService.java index cb0d658a0..e11bff3b6 100644 --- a/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/service/LinkisDataSourceInfoService.java +++ b/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/service/LinkisDataSourceInfoService.java @@ -1,3 +1,20 @@ +/* + Copyright 2022 WeBank + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + + package com.webank.wedatasphere.exchangis.datasource.linkis.service; import com.webank.wedatasphere.exchangis.datasource.core.service.DataSourceInfoService; diff --git a/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/service/LinkisMetadataInfoService.java b/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/service/LinkisMetadataInfoService.java index ecf9bd872..610e063ba 100644 --- a/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/service/LinkisMetadataInfoService.java +++ b/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/service/LinkisMetadataInfoService.java @@ -1,22 +1,29 @@ package com.webank.wedatasphere.exchangis.datasource.linkis.service; +import com.webank.wedatasphere.exchangis.datasource.core.domain.MetaColumn; import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisDataSourceException; import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisServiceRpcException; import com.webank.wedatasphere.exchangis.datasource.core.service.MetadataInfoService; import com.webank.wedatasphere.exchangis.datasource.core.service.rpc.ServiceRpcClient; import com.webank.wedatasphere.exchangis.datasource.linkis.ExchangisLinkisRemoteClient; +//import com.webank.wedatasphere.exchangis.datasource.linkis.partition.MetadataGetPartitionsResult; import com.webank.wedatasphere.exchangis.datasource.linkis.request.MetadataGetPartitionPropsAction; import com.webank.wedatasphere.exchangis.datasource.linkis.response.MetadataGetPartitionPropsResult; import com.webank.wedatasphere.exchangis.datasource.linkis.service.rpc.LinkisDataSourceServiceOperation; import com.webank.wedatasphere.exchangis.datasource.linkis.service.rpc.LinkisDataSourceServiceRpcDispatcher; import org.apache.linkis.datasource.client.impl.LinkisMetaDataRemoteClient; +import org.apache.linkis.datasource.client.request.MetadataGetColumnsAction; import org.apache.linkis.datasource.client.request.MetadataGetPartitionsAction; import org.apache.linkis.datasource.client.request.MetadataGetTablePropsAction; +import org.apache.linkis.datasource.client.response.MetadataGetColumnsResult; import org.apache.linkis.datasource.client.response.MetadataGetPartitionsResult; import org.apache.linkis.datasource.client.response.MetadataGetTablePropsResult; +import org.apache.linkis.metadatamanager.common.domain.MetaColumnInfo; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import static com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisDataSourceExceptionCode.*; @@ -73,10 +80,23 @@ public Map getTableProps(ServiceRpcClient rpcClient, String u @Override public List getPartitionKeys(String userName, Long dataSourceId, String database, String table) throws ExchangisDataSourceException { MetadataGetPartitionsResult result = dispatch(getDefaultRemoteClient(), new LinkisDataSourceServiceOperation(() -> MetadataGetPartitionsAction.builder() - .setDataSourceId(String.valueOf(dataSourceId)).setDatabase(database).setTable(table) + .setDataSourceId(dataSourceId).setDatabase(database).setTable(table) .setUser(userName).setSystem(LINKIS_RPC_CLIENT_SYSTEM.getValue()).build()), CLIENT_METADATA_GET_PARTITION.getCode(), "getPartitionKeys"); return result.getPartitionInfo().getPartKeys(); } + @Override + public List getColumns(String userName, Long dataSourceId, String database, String table) throws ExchangisDataSourceException { + MetadataGetColumnsResult result = dispatch(getDefaultRemoteClient(), new LinkisDataSourceServiceOperation(() -> MetadataGetColumnsAction.builder() + .setSystem(LINKIS_RPC_CLIENT_SYSTEM.getValue()) + .setDataSourceId(dataSourceId).setDatabase(database).setTable(table) + .setUser(userName).build()),CLIENT_METADATA_GET_PARTITION.getCode(), "getColumns"); + List columnInfoList = result.getAllColumns(); + List columns = new ArrayList<>(); + Optional.ofNullable(columnInfoList).ifPresent(infoList -> infoList.forEach(info -> + columns.add(new MetaColumn(info.getIndex(), info.getName(), info.getType(), info.isPrimaryKey())))); + return columns; + } + } diff --git a/exchangis-datasource/exchangis-datasource-linkis/src/main/scala/com/webank/wedatasphere/exchangis/datasource/linkis/request/ParamsTestConnectAction.scala b/exchangis-datasource/exchangis-datasource-linkis/src/main/scala/com/webank/wedatasphere/exchangis/datasource/linkis/request/ParamsTestConnectAction.scala new file mode 100644 index 000000000..7461f63fb --- /dev/null +++ b/exchangis-datasource/exchangis-datasource-linkis/src/main/scala/com/webank/wedatasphere/exchangis/datasource/linkis/request/ParamsTestConnectAction.scala @@ -0,0 +1,39 @@ +package com.webank.wedatasphere.exchangis.datasource.linkis.request + +import org.apache.linkis.datasource.client.config.DatasourceClientConfig.DATA_SOURCE_SERVICE_MODULE +import org.apache.linkis.datasource.client.request.DataSourceAction +import org.apache.linkis.httpclient.dws.DWSHttpClient +import org.apache.linkis.httpclient.request.POSTAction + +import java.util +import scala.collection.JavaConverters.mapAsScalaMapConverter + +/** + * Connect test for the data source params + */ +class ParamsTestConnectAction extends POSTAction with DataSourceAction{ + + private var user: String = _ + + override def getRequestPayload: String = DWSHttpClient.jacksonJson.writeValueAsString(getRequestPayloads) + + override def suffixURLs: Array[String] = Array(DATA_SOURCE_SERVICE_MODULE.getValue, "op", "connect", "json") + + override def setUser(user: String): Unit = this.user = user + + override def getUser: String = this.user + + /** + * + * @param dataSource data source map + * @param user user + */ + def this(dataSource: util.Map[String, Any], user: String){ + this() + dataSource.asScala.foreach{ + case (key, value) => + this.addRequestPayload(key, value) + } + this.user = user + } +} diff --git a/exchangis-datasource/exchangis-datasource-linkis/src/main/scala/com/webank/wedatasphere/exchangis/datasource/linkis/response/ParamsTestConnectResult.scala b/exchangis-datasource/exchangis-datasource-linkis/src/main/scala/com/webank/wedatasphere/exchangis/datasource/linkis/response/ParamsTestConnectResult.scala new file mode 100644 index 000000000..1dc447082 --- /dev/null +++ b/exchangis-datasource/exchangis-datasource-linkis/src/main/scala/com/webank/wedatasphere/exchangis/datasource/linkis/response/ParamsTestConnectResult.scala @@ -0,0 +1,12 @@ +package com.webank.wedatasphere.exchangis.datasource.linkis.response + +import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult +import org.apache.linkis.httpclient.dws.response.DWSResult + +import scala.beans.BeanProperty + +@DWSHttpMessageResult("/api/rest_j/v\\d+/data-source-manager/op/connect/json") +class ParamsTestConnectResult extends DWSResult{ + @BeanProperty var ok: Boolean = _ + +} diff --git a/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/restful/api/ExchangisDataSourceRenderRestfulApi.java b/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/restful/api/ExchangisDataSourceRenderRestfulApi.java index e64ba041b..00e6b5c2d 100644 --- a/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/restful/api/ExchangisDataSourceRenderRestfulApi.java +++ b/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/restful/api/ExchangisDataSourceRenderRestfulApi.java @@ -20,7 +20,7 @@ * Expose the ui interface to front-end rendering */ @RestController -@RequestMapping(value = "exchangis/datasources/render", produces = {"application/json;charset=utf-8"}) +@RequestMapping(value = "dss/exchangis/main/datasources/render", produces = {"application/json;charset=utf-8"}) public class ExchangisDataSourceRenderRestfulApi { private static final Logger LOG = LoggerFactory.getLogger(ExchangisDataSourceRenderRestfulApi.class); @@ -52,11 +52,8 @@ public Message partition(@PathVariable("elementType") String type, LOG.error(uiMessage + ", reason: " + e.getMessage(), e); result = Message.error(uiMessage); } - result.setMethod("/api/rest_j/v1/exchangis/datasources/render/partition/element/" + type); + result.setMethod("/api/rest_j/v1/dss/exchangis/main/datasources/render/partition/element/" + type); return result; } - public static void main(String[] args){ - ElementUI.Type.valueOf("map".toUpperCase(Locale.ROOT)); - } } diff --git a/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/restful/api/ExchangisDataSourceRestfulApi.java b/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/restful/api/ExchangisDataSourceRestfulApi.java index 6ac152306..26803afb6 100644 --- a/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/restful/api/ExchangisDataSourceRestfulApi.java +++ b/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/restful/api/ExchangisDataSourceRestfulApi.java @@ -3,27 +3,36 @@ import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisDataSourceException; import com.webank.wedatasphere.exchangis.datasource.core.ui.ElementUI; import com.webank.wedatasphere.exchangis.datasource.service.ExchangisDataSourceService; +import com.webank.wedatasphere.exchangis.datasource.vo.DataSourceCreateVO; import com.webank.wedatasphere.exchangis.datasource.vo.DataSourceQueryVO; import com.webank.wedatasphere.exchangis.datasource.vo.FieldMappingVO; import org.apache.linkis.server.Message; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.validation.BindingResult; +import org.springframework.validation.FieldError; +import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.*; import javax.servlet.http.HttpServletRequest; +import javax.validation.Valid; import javax.ws.rs.QueryParam; import java.util.List; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; @RestController -@RequestMapping(value = "exchangis/datasources", produces = {"application/json;charset=utf-8"}) +@RequestMapping(value = "dss/exchangis/main/datasources", produces = {"application/json;charset=utf-8"}) public class ExchangisDataSourceRestfulApi { private static final Logger LOG = LoggerFactory.getLogger(ExchangisDataSourceRestfulApi.class); private final ExchangisDataSourceService exchangisDataSourceService; + private static Pattern p = Pattern.compile("(?<=\\[)[^]]+"); + @Autowired public ExchangisDataSourceRestfulApi(ExchangisDataSourceService exchangisDataSourceService) { this.exchangisDataSourceService = exchangisDataSourceService; @@ -32,13 +41,39 @@ public ExchangisDataSourceRestfulApi(ExchangisDataSourceService exchangisDataSou // list all datasource types @RequestMapping( value = "/type", method = RequestMethod.GET) public Message listDataSourceTypes(HttpServletRequest request) throws Exception { - return this.exchangisDataSourceService.listDataSources(request); + Message message = null; + try{ + message = exchangisDataSourceService.listDataSources(request); + } catch (ExchangisDataSourceException e) { + String errorMessage = "Error occur while list datasource type"; + LOG.error(errorMessage, e); + + String errorNote = e.getMessage(); + Matcher matcher = p.matcher(errorNote); + if (matcher.find()) { + message = Message.error(matcher.group()); + } + else{ + message = Message.error("Getting datasource type list fail (获取数据源类型列表失败)"); + } + } + return message; + } // query paged datasource @RequestMapping( value = "/query", method = {RequestMethod.GET,RequestMethod.POST}) public Message create(HttpServletRequest request, @RequestBody DataSourceQueryVO vo) throws Exception { - return this.exchangisDataSourceService.queryDataSources(request, vo); + Message message = null; + try{ + message = exchangisDataSourceService.queryDataSources(request, vo); + } catch (ExchangisDataSourceException e) { + String errorMessage = "Error occur while query datasource"; + LOG.error(errorMessage, e); + message = Message.error("查询数据源失败"); + } + return message; + } // list all datasources @@ -51,7 +86,24 @@ public Message listAllDataSources( @RequestParam(value = "page", required = false) Integer page, @RequestParam(value = "size", required = false) Integer size ) throws Exception { - return this.exchangisDataSourceService.listAllDataSources(request, typeName, typeId, page, size); + Message message = null; + try{ + message = exchangisDataSourceService.listAllDataSources(request, typeName, typeId, page, size); + } catch (ExchangisDataSourceException e) { + String errorMessage = "Error occur while getting datasource list"; + LOG.error(errorMessage, e); + + String errorNote = e.getMessage(); + Matcher matcher = p.matcher(errorNote); + if (matcher.find()) { + message = Message.error(matcher.group()); + } + else{ + message = Message.error("Getting datasource list fail (获取数据源列表失败)"); + } + } + return message; + } // get datasource key define @@ -60,59 +112,181 @@ public Message getDataSourceKeyDefine( HttpServletRequest request, @PathVariable("dataSourceTypeId") Long dataSourceTypeId ) throws Exception { - return this.exchangisDataSourceService.getDataSourceKeyDefine(request, dataSourceTypeId); + Message message = null; + try{ + message = exchangisDataSourceService.getDataSourceKeyDefine(request, dataSourceTypeId); + } catch (ExchangisDataSourceException e) { + String errorMessage = "Error occur while getting datasource key define"; + LOG.error(errorMessage, e); + message = Message.error("获取数据源主键定义失败"); + } + return message; + } // get datasource version list @RequestMapping( value = "/{id}/versions", method = RequestMethod.GET) public Message getDataSourceVersionsById(HttpServletRequest request, @PathVariable("id") Long id) throws Exception { - return this.exchangisDataSourceService.getDataSourceVersionsById(request, id); - } - - // create datasource - @RequestMapping( value = "", method = RequestMethod.POST) - public Message create(HttpServletRequest request, /*@PathParam("type") String type, */@RequestBody Map json) throws Exception { Message message = null; try{ - message = exchangisDataSourceService.create(request, json); + message = exchangisDataSourceService.getDataSourceVersionsById(request, id); } catch (ExchangisDataSourceException e) { - String errorMessage = "Error occur while create datasource"; + String errorMessage = "Error occur while getting datasource version"; LOG.error(errorMessage, e); - message = Message.error("已存在同名任务"); + + String errorNote = e.getMessage(); + Matcher matcher = p.matcher(errorNote); + if (matcher.find()) { + message = Message.error(matcher.group()); + } + else{ + message = Message.error("Getting datasource version fail (获取数据源版本失败)"); + } + } + return message; + + } + + // create datasource + @RequestMapping( value = "", method = RequestMethod.POST) + public Message create(/*@PathParam("type") String type, */@Valid @RequestBody DataSourceCreateVO dataSourceCreateVO, BindingResult bindingResult, HttpServletRequest request ) throws Exception { + Message message = new Message(); + LOG.info("dataSourceName: " + dataSourceCreateVO.getDataSourceName() + "dataSourceDesc: " + dataSourceCreateVO.getDataSourceDesc() + "label: " + dataSourceCreateVO.getLabels()); + if(bindingResult.hasErrors()){ + List fieldErrors = bindingResult.getFieldErrors(); + for(int i=0;i json) throws Exception { - return this.exchangisDataSourceService.updateDataSource(request, /*type, */id, json); + public Message update(HttpServletRequest request,/* @PathParam("type") String type, */@PathVariable("id") Long id, @Valid @RequestBody DataSourceCreateVO dataSourceCreateVO, BindingResult bindingResult) throws Exception { + Message message = new Message(); + + LOG.info("dataSourceName: " + dataSourceCreateVO.getDataSourceName() + "dataSourceDesc: " + dataSourceCreateVO.getDataSourceDesc() + "label: " + dataSourceCreateVO.getLabels()); + if(bindingResult.hasErrors()){ + List fieldErrors = bindingResult.getFieldErrors(); + for(int i=0;i> jobSettingsUI = this.exchangisDataSourceService.getJobEngineSettingsUI(engineType); return Message.ok().data("ui", jobSettingsUI); } diff --git a/exchangis-datasource/exchangis-datasource-service/pom.xml b/exchangis-datasource/exchangis-datasource-service/pom.xml index 68a6191fe..0c692d791 100644 --- a/exchangis-datasource/exchangis-datasource-service/pom.xml +++ b/exchangis-datasource/exchangis-datasource-service/pom.xml @@ -24,6 +24,12 @@ 1.0.0-RC1 + + + com.webank.wedatasphere.exchangis + exchangis-job-common + 1.0.0-RC1 + diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/CreateDataSourceSuccessResultDTO.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/CreateDataSourceSuccessResultDTO.java index fa0210be1..1398ecd32 100644 --- a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/CreateDataSourceSuccessResultDTO.java +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/CreateDataSourceSuccessResultDTO.java @@ -15,15 +15,15 @@ public void setData(InsertIdDTO data) { } public static class InsertIdDTO { - @JsonProperty(value = "insert_id") - private Long id; + @JsonProperty(value = "insertId") + private Long insertId; public Long getId() { - return id; + return insertId; } - public void setId(Long id) { - this.id = id; + public void setId(Long insertId) { + this.insertId = insertId; } } } diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/AbstractDataSourceService.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/AbstractDataSourceService.java index 4c519c9cf..75faf4f12 100644 --- a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/AbstractDataSourceService.java +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/AbstractDataSourceService.java @@ -5,25 +5,23 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Strings; -import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobEntity; import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; -import com.webank.wedatasphere.exchangis.dao.mapper.ExchangisJobInfoMapper; import com.webank.wedatasphere.exchangis.dao.mapper.ExchangisJobParamConfigMapper; import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSource; import com.webank.wedatasphere.exchangis.datasource.core.context.ExchangisDataSourceContext; import com.webank.wedatasphere.exchangis.datasource.core.ui.*; import com.webank.wedatasphere.exchangis.datasource.core.ui.viewer.DefaultDataSourceUIViewer; import com.webank.wedatasphere.exchangis.datasource.core.ui.viewer.ExchangisDataSourceUIViewer; +import com.webank.wedatasphere.exchangis.datasource.core.utils.Json; import com.webank.wedatasphere.exchangis.datasource.core.vo.ExchangisJobDataSourcesContent; import com.webank.wedatasphere.exchangis.datasource.core.vo.ExchangisJobInfoContent; import com.webank.wedatasphere.exchangis.datasource.core.vo.ExchangisJobParamsContent; import com.webank.wedatasphere.exchangis.datasource.core.vo.ExchangisJobTransformsContent; import com.webank.wedatasphere.exchangis.datasource.dto.GetDataSourceInfoResultDTO; +import com.webank.wedatasphere.exchangis.job.domain.ExchangisJobEntity; import org.apache.commons.lang.StringUtils; import org.apache.linkis.datasource.client.impl.LinkisDataSourceRemoteClient; import org.apache.linkis.datasource.client.request.GetInfoByDataSourceIdAction; -import org.apache.linkis.datasourcemanager.common.exception.JsonErrorException; -import org.apache.linkis.datasourcemanager.common.util.json.Json; import org.apache.linkis.httpclient.response.Result; import org.apache.linkis.server.security.SecurityFilter; import org.slf4j.Logger; @@ -37,15 +35,13 @@ public class AbstractDataSourceService { protected final ObjectMapper mapper = new ObjectMapper(); protected final ExchangisDataSourceContext context; protected final ExchangisJobParamConfigMapper exchangisJobParamConfigMapper; - protected final ExchangisJobInfoMapper exchangisJobInfoMapper; private final static Logger LOG = LoggerFactory.getLogger(AbstractDataSourceService.class); - public AbstractDataSourceService(ExchangisDataSourceContext context, ExchangisJobParamConfigMapper exchangisJobParamConfigMapper, ExchangisJobInfoMapper exchangisJobInfoMapper) { + public AbstractDataSourceService(ExchangisDataSourceContext context, ExchangisJobParamConfigMapper exchangisJobParamConfigMapper) { this.context = context; this.exchangisJobParamConfigMapper = exchangisJobParamConfigMapper; - this.exchangisJobInfoMapper = exchangisJobInfoMapper; } protected List parseJobContent(String content) { @@ -99,12 +95,9 @@ private ExchangisDataSourceIdsUI buildDataSourceIdsUI(HttpServletRequest request Result execute = dsClient.execute(action); String responseBody = execute.getResponseBody(); GetDataSourceInfoResultDTO dsInfo = null; - try { - dsInfo = Json.fromJson(responseBody, GetDataSourceInfoResultDTO.class); - source.setDs(dsInfo.getData().getInfo().getDataSourceName()); - } catch (JsonErrorException e) { - //TODO throws Exception - } + dsInfo = Json.fromJson(responseBody, GetDataSourceInfoResultDTO.class); + assert dsInfo != null; + source.setDs(dsInfo.getData().getInfo().getDataSourceName()); }); }); source.setDb(split[2]); @@ -129,12 +122,9 @@ private ExchangisDataSourceIdsUI buildDataSourceIdsUI(HttpServletRequest request Result execute = dsClient.execute(action); String responseBody = execute.getResponseBody(); GetDataSourceInfoResultDTO dsInfo = null; - try { - dsInfo = Json.fromJson(responseBody, GetDataSourceInfoResultDTO.class); - sink.setDs(dsInfo.getData().getInfo().getDataSourceName()); - } catch (JsonErrorException e) { - //TODO throw Exception - } + dsInfo = Json.fromJson(responseBody, GetDataSourceInfoResultDTO.class); + assert dsInfo != null; + sink.setDs(dsInfo.getData().getInfo().getDataSourceName()); }); }); @@ -293,7 +283,8 @@ private ElementUI fillElementUIValue(ExchangisJobParamConfig config, Object v case MAP: Map mapElement = null; try { - mapElement = Json.fromJson(String.valueOf(value), Map.class); + mapElement = Json.fromJson(Json.toJson(value, null), + Map.class, String.class, Object.class); } catch (Exception e) { LOG.info("Exception happened while parse json"+ "Config value: " + value + "message: " + e.getMessage(), e); } @@ -359,4 +350,5 @@ private MapElementUI fillMapElementUIValue(ExchangisJobParamConfig config, Map getJobDataSourceUIs(HttpServletRequest request, Long jobId) { if (Objects.isNull(jobId)) { return null; } - ExchangisJobEntity job = this.exchangisJobInfoMapper.selectById(jobId); + ExchangisJobEntity job; + try { + job = this.jobOpenService.getJobById(jobId, false); + } catch (ExchangisJobException e) { + throw new ExchangisDataSourceException + .Runtime(CONTEXT_GET_DATASOURCE_NULL.getCode(), "Fail to get job entity (获得任务信息失败)", e); + } if (Objects.isNull(job)) { return null; } - List jobInfoContents = this.parseJobContent(job.getContent()); + List jobInfoContents = this.parseJobContent(job.getJobContent()); List uis = new ArrayList<>(); for (ExchangisJobInfoContent cnt : jobInfoContents) { cnt.setEngine(job.getEngineType()); @@ -121,7 +132,7 @@ public Message listDataSources(HttpServletRequest request) throws Exception { List dtos = new ArrayList<>(); String userName = SecurityFilter.getLoginUsername(request); - LOGGER.info("listDataSources userName:" + userName); + LOGGER.info("listDataSources userName: {}" + userName); // 通过 datasourcemanager 获取的数据源类型和context中的数据源通过 type 和 name 比较 // 以 exchangis 中注册了的数据源集合为准 @@ -168,14 +179,15 @@ public Message listDataSources(HttpServletRequest request) throws Exception { } @Transactional - public Message create(HttpServletRequest request, /*String type, */Map json) throws Exception { - DataSourceCreateVO vo; + public Message create(HttpServletRequest request, /*String type, */DataSourceCreateVO vo) throws Exception { + //DataSourceCreateVO vo; + Map json; try { - vo = mapper.readValue(mapper.writeValueAsString(json), DataSourceCreateVO.class); + json = mapper.readValue(mapper.writeValueAsString(vo), Map.class); + json.put("labels",json.get("label")); } catch (JsonProcessingException e) { throw new ExchangisDataSourceException(ExchangisDataSourceExceptionCode.PARSE_JSON_ERROR.getCode(), e.getMessage()); } - String comment = vo.getComment(); String createSystem = vo.getCreateSystem(); if (Objects.isNull(comment)) { @@ -192,21 +204,10 @@ public Message create(HttpServletRequest request, /*String type, */Map connectParams = vo.getConnectParams(); -// if (!Objects.isNull(connectParams)) { -// // 如果是 hive 类型，需要处理成连接字符串 TODO -// Object host = connectParams.get("host"); -// Object port = connectParams.get("port"); -// if (!Objects.isNull(host) && !Objects.isNull(port)) { -// String uris = "thrift://" + connectParams.get("host") + ":" + connectParams.get("port"); -// connectParams.put("uris", uris); -// } -// json.put("parameter", mapper.writeValueAsString(connectParams)); -// } LOGGER.info("create datasource json as follows"); Set> entries = json.entrySet(); for (Map.Entry entry : entries) { @@ -215,11 +216,6 @@ public Message create(HttpServletRequest request, /*String type, */Map json) throws Exception { - DataSourceUpdateVO vo; + public Message updateDataSource(HttpServletRequest request,/* String type,*/ Long id, DataSourceCreateVO vo) throws Exception { + + Map json; try { - vo = mapper.readValue(mapper.writeValueAsString(json), DataSourceUpdateVO.class); + json = mapper.readValue(mapper.writeValueAsString(vo), Map.class); + json.put("labels",json.get("label")); } catch (JsonProcessingException e) { - throw new ExchangisDataSourceException(30401, e.getMessage()); + throw new ExchangisDataSourceException(ExchangisDataSourceExceptionCode.PARSE_JSON_ERROR.getCode(), e.getMessage()); } - String comment = vo.getComment(); String createSystem = vo.getCreateSystem(); if (Objects.isNull(comment)) { @@ -293,31 +290,13 @@ public Message updateDataSource(HttpServletRequest request,/* String type,*/ Lon throw new ExchangisDataSourceException(30401, "exchangis.datasource.null"); } -// Map connectParams = vo.getConnectParams(); -// if (!Objects.isNull(connectParams)) { -// // 如果是 hive 类型，需要处理成连接字符串 TODO -// Object host = connectParams.get("host"); -// Object port = connectParams.get("port"); -// if (!Objects.isNull(host) && !Objects.isNull(port)) { -// String uris = "thrift://" + connectParams.get("host") + ":" + connectParams.get("port"); -// connectParams.put("uris", uris); -// } -// json.put("parameter", mapper.writeValueAsString(connectParams)); -// } - LinkisDataSourceRemoteClient client = exchangisDataSource.getDataSourceRemoteClient(); // UpdateDataSourceResult updateDataSourceResult; String responseBody; try { -// updateDataSourceResult = client.updateDataSource(UpdateDataSourceAction.builder() -// .setUser(user) -// .setDataSourceId(id+"") -// .addRequestPayloads(json) -// .build() -// ); Result execute = client.execute(UpdateDataSourceAction.builder() .setUser(user) - .setDataSourceId(id + "") + .setDataSourceId(Long.parseLong(id + "")) .addRequestPayloads(json) .build() ); @@ -341,7 +320,7 @@ public Message updateDataSource(HttpServletRequest request,/* String type,*/ Lon try { updateDataSourceParameterResult = client.updateDataSourceParameter( UpdateDataSourceParameterAction.builder() - .setDataSourceId(id + "") + .setDataSourceId(Long.parseLong(id + "")) .setUser(user) .addRequestPayloads(json) .build() @@ -383,7 +362,7 @@ public Message deleteDataSource(HttpServletRequest request, /*String type,*/ Lon // ); Result execute = dataSourceRemoteClient.execute( - new DeleteDataSourceAction.Builder().setUser(user).setResourceId(id + "").builder() + new DeleteDataSourceAction.Builder().setUser(user).setDataSourceId(Long.parseLong(id + "")).builder() ); responseBody = execute.getResponseBody(); @@ -465,12 +444,19 @@ public Message getJobDataSourceParamsUI(Long jobId) { return null; } - ExchangisJobEntity job = this.exchangisJobInfoMapper.selectById(jobId); + ExchangisJobEntity job; + try { + job = this.jobOpenService.getJobById(jobId, false); + } catch (ExchangisJobException e) { + throw new ExchangisDataSourceException + .Runtime(CONTEXT_GET_DATASOURCE_NULL.getCode(), "Fail to get job entity (获得任务信息失败)", e); + } + if (Objects.isNull(job)) { return null; } - List jobInfoContents = this.parseJobContent(job.getContent()); + List jobInfoContents = this.parseJobContent(job.getJobContent()); List uis = new ArrayList<>(); for (ExchangisJobInfoContent cnt : jobInfoContents) { uis.add(this.buildDataSourceParamsUI(cnt)); @@ -484,12 +470,18 @@ public Message getJobDataSourceTransformsUI(Long jobId) { return null; } - ExchangisJobEntity job = this.exchangisJobInfoMapper.selectById(jobId); + ExchangisJobEntity job; + try { + job = this.jobOpenService.getJobById(jobId, false); + } catch (ExchangisJobException e) { + throw new ExchangisDataSourceException + .Runtime(CONTEXT_GET_DATASOURCE_NULL.getCode(), "Fail to get job entity (获得任务信息失败)", e); + } if (Objects.isNull(job)) { return null; } - String jobContent = job.getContent(); + String jobContent = job.getJobContent(); ExchangisJobInfoContent content; // 转换 content if (Strings.isNullOrEmpty(jobContent)) { @@ -513,12 +505,18 @@ public Message getJobDataSourceSettingsUI(Long jobId, String jobName) throws Exc return null; } - ExchangisJobEntity job = this.exchangisJobInfoMapper.selectById(jobId); + ExchangisJobEntity job; + try { + job = this.jobOpenService.getJobById(jobId, false); + } catch (ExchangisJobException e) { + throw new ExchangisDataSourceException + .Runtime(CONTEXT_GET_DATASOURCE_NULL.getCode(), "Fail to get job entity (获得任务信息失败)", e); + } if (Objects.isNull(job)) { return null; } - List contents = this.parseJobContent(job.getContent()); + List contents = this.parseJobContent(job.getJobContent()); for (ExchangisJobInfoContent content : contents) { if (content.getSubJobName().equalsIgnoreCase(jobName)) { @@ -636,7 +634,7 @@ public Message queryDataSources(HttpServletRequest request, DataSourceQueryVO vo Message message = Message.ok(); message.data("list", dataSources); - message.data("total", result.getTotalPage() * pageSize); + message.data("total", result.getTotalPage()); return message; //return Message.ok().data("list", dataSources); } @@ -858,8 +856,8 @@ private MetadataGetColumnsResultDTO getDatasourceColumns(String username, Long i throw new ExchangisDataSourceException(result.getStatus(), result.getMessage()); } } catch (JsonErrorException e) { - throw new ExchangisDataSourceException(CLIENT_METADATA_GET_COLUMNS_ERROR.getCode(), - "Fail to deserialize the columns resultSet", e); + throw new ExchangisDataSourceException(CLIENT_METADATA_GET_COLUMNS_ERROR.getCode(), + "Fail to deserialize the columns resultSet", e); } return result; @@ -923,9 +921,9 @@ public Message getDataSourceVersionsById(HttpServletRequest request, Long id) th } catch (Exception e) { if (e instanceof ErrorException) { ErrorException ee = (ErrorException) e; - throw new ExchangisDataSourceException(ExchangisDataSourceExceptionCode.CLIENT_GET_DATASOURCE_ERROR.getCode(), e.getMessage(), ee.getIp(), ee.getPort(), ee.getServiceKind()); + throw new ExchangisDataSourceException(CLIENT_GET_DATASOURCE_ERROR.getCode(), e.getMessage(), ee.getIp(), ee.getPort(), ee.getServiceKind()); } else { - throw new ExchangisDataSourceException(ExchangisDataSourceExceptionCode.CLIENT_GET_DATASOURCE_ERROR.getCode(), e.getMessage()); + throw new ExchangisDataSourceException(CLIENT_GET_DATASOURCE_ERROR.getCode(), e.getMessage()); } } // if (Objects.isNull(result)) { @@ -948,18 +946,18 @@ public Message getDataSourceVersionsById(HttpServletRequest request, Long id) th GetDataSourceVersionsResult versionsResult; try { versionsResult = linkisDataSourceRemoteClient.getDataSourceVersions( - new GetDataSourceVersionsAction.Builder().setUser(userName).setResourceId(id + "").build() + new GetDataSourceVersionsAction.Builder().setUser(userName).setDataSourceId(Long.parseLong(id + "")).build() ); } catch (Exception e) { if (e instanceof ErrorException) { ErrorException ee = (ErrorException) e; - throw new ExchangisDataSourceException(ExchangisDataSourceExceptionCode.CLIENT_GET_DATASOURCE_VERSION_ERROR.getCode(), e.getMessage(), ee.getIp(), ee.getPort(), ee.getServiceKind()); + throw new ExchangisDataSourceException(CLIENT_GET_DATASOURCE_VERSION_ERROR.getCode(), e.getMessage(), ee.getIp(), ee.getPort(), ee.getServiceKind()); } else { - throw new ExchangisDataSourceException(ExchangisDataSourceExceptionCode.CLIENT_GET_DATASOURCE_VERSION_ERROR.getCode(), e.getMessage()); + throw new ExchangisDataSourceException(CLIENT_GET_DATASOURCE_VERSION_ERROR.getCode(), e.getMessage()); } } if (Objects.isNull(versionsResult)) { - throw new ExchangisDataSourceException(ExchangisDataSourceExceptionCode.CLIENT_GET_DATASOURCE_VERSION_ERROR.getCode(), "datasource version response body null or empty"); + throw new ExchangisDataSourceException(CLIENT_GET_DATASOURCE_VERSION_ERROR.getCode(), "datasource version response body null or empty"); } if (versionsResult.getStatus() != 0) { @@ -1004,7 +1002,44 @@ public Message testConnect(HttpServletRequest request, Long id, Long version) th DataSourceTestConnectResult result; try { result = linkisDataSourceRemoteClient.getDataSourceTestConnect( - new DataSourceTestConnectAction.Builder().setUser(userName).setDataSourceId(id + "").setVersion(version + "").build() + new DataSourceTestConnectAction.Builder().setUser(userName).setDataSourceId(Long.parseLong(id + "")).setVersion(version + "").build() + ); + } catch (Exception e) { + if (e instanceof ErrorException) { + ErrorException ee = (ErrorException) e; + throw new ExchangisDataSourceException(ExchangisDataSourceExceptionCode.CLIENT_DATASOURCE_TEST_CONNECTION_ERROR.getCode(), e.getMessage(), ee.getIp(), ee.getPort(), ee.getServiceKind()); + } else { + throw new ExchangisDataSourceException(ExchangisDataSourceExceptionCode.CLIENT_DATASOURCE_TEST_CONNECTION_ERROR.getCode(), e.getMessage()); + } + } + + if (Objects.isNull(result)) { + throw new ExchangisDataSourceException(ExchangisDataSourceExceptionCode.CLIENT_DATASOURCE_TEST_CONNECTION_ERROR.getCode(), "datasource test connection response body null or empty"); + } + + if (result.getStatus() != 0) { + throw new ExchangisDataSourceException(result.getStatus(), result.getMessage()); + } + + return Message.ok(); + } + + public Message testConnectByVo(HttpServletRequest request, DataSourceCreateVO vo) throws ErrorException { + LinkisDataSourceRemoteClient linkisDataSourceRemoteClient = ExchangisLinkisRemoteClient.getLinkisDataSourceRemoteClient(); + String userName = SecurityFilter.getLoginUsername(request); + LOGGER.info("testConnect userName:" + userName); + + Map json; + try { + json = mapper.readValue(mapper.writeValueAsString(vo), Map.class); + json.put("labels",json.get("label")); + } catch (JsonProcessingException e) { + throw new ExchangisDataSourceException(ExchangisDataSourceExceptionCode.PARSE_JSON_ERROR.getCode(), e.getMessage()); + } + ParamsTestConnectResult result; + try { + result = (ParamsTestConnectResult) linkisDataSourceRemoteClient.execute( + new ParamsTestConnectAction(json, userName) ); } catch (Exception e) { if (e instanceof ErrorException) { @@ -1034,7 +1069,7 @@ public Message publishDataSource(HttpServletRequest request, Long id, Long versi PublishDataSourceVersionResult result; try { result = linkisDataSourceRemoteClient.publishDataSourceVersion( - new PublishDataSourceVersionAction.Builder().setUser(userName).setDataSourceId(id + "").setVersion(version + "").build() + new PublishDataSourceVersionAction.Builder().setUser(userName).setDataSourceId(Long.parseLong(id + "")).setVersion(Long.parseLong(version + "")).build() ); } catch (Exception e) { if (e instanceof ErrorException) { @@ -1076,7 +1111,7 @@ public Message expireDataSource(HttpServletRequest request, Long id) throws Erro // ); Result execute = linkisDataSourceRemoteClient.execute( - new ExpireDataSourceAction.Builder().setUser(userName).setDataSourceId(id + "").build() + new ExpireDataSourceAction.Builder().setUser(userName).setDataSourceId(Long.parseLong(id + "")).build() ); responseBody = execute.getResponseBody(); } catch (Exception e) { @@ -1155,7 +1190,7 @@ public Message getDataSourceKeyDefine(HttpServletRequest request, Long dataSourc throw new ExchangisDataSourceException(result.getStatus(), result.getMessage()); } - return Message.ok().data("list", Objects.isNull(result.getKey_define()) ? null : result.getKey_define()); + return Message.ok().data("list", Objects.isNull(result.getKeyDefine()) ? null : result.getKeyDefine()); } public void checkDSSupportDegree(String engine, String sourceDsType, String sinkDsType) throws ExchangisDataSourceException { @@ -1184,6 +1219,14 @@ private void checkDataXDSSupportDegree(String sourceDsType, String sinkDsType) t } + /** + * TODO: the mapping function is defined by the rule of Hive directly, we should abstract to support all the types + * @param request + * @param vo + * @return + * @throws Exception + */ + @SuppressWarnings("unchecked") public Message queryDataSourceDBTableFieldsMapping(HttpServletRequest request, FieldMappingVO vo) throws Exception { this.checkDSSupportDegree(vo.getEngine(), vo.getSourceTypeId(), vo.getSinkTypeId()); @@ -1209,39 +1252,23 @@ public Message queryDataSourceDBTableFieldsMapping(HttpServletRequest request, F field.setFieldEditable(!"HIVE".equals(vo.getSinkTypeId())); } message.data("sinkFields", sinkFields); - - // field mapping deduction List> deductions = new ArrayList<>(); -// boolean[] matchedIndex = new boolean[sinkFields.size()]; List left = sourceFields; List right = sinkFields; boolean exchanged = false; if (containHive && "HIVE".equals(vo.getSinkTypeId())) { left = sinkFields; right = sourceFields; - exchanged = !exchanged; - } -// for (DataSourceDbTableColumnDTO l : left) { -// String lname = l.getName(); -// for (DataSourceDbTableColumnDTO r : right) { -// String rname = r.getName(); -// if (lname.equals(rname)) { -// Map deduction = new HashMap<>(); -// deduction.put("source", exchanged ? r : l); -// deduction.put("sink", exchanged ? l : r); -// deduction.put("deleteEnable", !containHive); -// deductions.add(deduction); -// } -// } -// } + exchanged = true; + } for (int i = 0; i < left.size(); i ++){ DataSourceDbTableColumnDTO leftElement = left.get(i); DataSourceDbTableColumnDTO rightElement = right.get(i % right.size()); Map deduction = new HashMap<>(); deduction.put("source", exchanged ? rightElement : leftElement); deduction.put("sink", exchanged ? leftElement : rightElement); - deduction.put("deleteEnable", !containHive); + deduction.put("deleteEnable", true); deductions.add(deduction); } message.data("deductions", deductions); diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/vo/DataSourceCreateVO.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/vo/DataSourceCreateVO.java index 692f80e32..69625f6d1 100644 --- a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/vo/DataSourceCreateVO.java +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/vo/DataSourceCreateVO.java @@ -1,16 +1,22 @@ package com.webank.wedatasphere.exchangis.datasource.vo; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Size; import java.util.Date; import java.util.Map; @JsonIgnoreProperties(ignoreUnknown = true) public class DataSourceCreateVO { + @Size(min=0,max=100,message="Length of dataSource name should between 0 and 100(数据源名字的长度应该在0和100之间)") private String dataSourceName; private Long dataSourceTypeId; + @Size(min=0,max=200,message="Length of dataSource description should between 0 and 200(数据源描述的长度应该在0和200之间)") private String dataSourceDesc; private String createIdentify; @@ -27,7 +33,10 @@ public class DataSourceCreateVO { private Long versionId; - private String labels; + @Size(min=0,max=200,message="Length of labels should between 0 and 200(标签的长度应该在0和200之间)") + private String label; + + private Map labels; private Long publishedVersionId; @@ -127,12 +136,12 @@ public void setVersionId(Long versionId) { this.versionId = versionId; } - public String getLabels() { - return labels; + public String getLabel() { + return label; } - public void setLabels(String labels) { - this.labels = labels; + public void setLabel(String label) { + this.label = label; } public Long getPublishedVersionId() { @@ -168,4 +177,12 @@ public String getComment() { public void setComment(String comment) { this.comment = comment; } + + public Map getLabels() { + return labels; + } + + public void setLabels(Map labels) { + this.labels = labels; + } } diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/vo/DataSourceQueryVO.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/vo/DataSourceQueryVO.java index 980542912..f603609bc 100644 --- a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/vo/DataSourceQueryVO.java +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/vo/DataSourceQueryVO.java @@ -1,11 +1,22 @@ package com.webank.wedatasphere.exchangis.datasource.vo; +import java.util.Map; + public class DataSourceQueryVO { private Integer page; private Integer pageSize; private Long typeId; private String typeName; private String name; + private Map labels; + + public Map getLabels() { + return labels; + } + + public void setLabels(Map labels) { + this.labels = labels; + } public String getName() { return name; diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/vo/FieldMappingVO.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/vo/FieldMappingVO.java index 0fefd89a7..1170df09c 100644 --- a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/vo/FieldMappingVO.java +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/vo/FieldMappingVO.java @@ -1,5 +1,7 @@ package com.webank.wedatasphere.exchangis.datasource.vo; +import java.util.Map; + public class FieldMappingVO { private String engine; @@ -13,6 +15,15 @@ public class FieldMappingVO { private Long sinkDataSourceId; private String sinkDataBase; private String sinkTable; + private Map labels; + + public Map getLabels() { + return labels; + } + + public void setLabels(Map labels) { + this.labels = labels; + } public String getEngine() { return engine; diff --git a/exchangis-job/exchangis-job-builder/src/main/java/com/webank/wedatasphere/exchangis/job/builder/ExchangisJobBuilderContext.java b/exchangis-job/exchangis-job-builder/src/main/java/com/webank/wedatasphere/exchangis/job/builder/ExchangisJobBuilderContext.java index af221df16..e6aee9968 100644 --- a/exchangis-job/exchangis-job-builder/src/main/java/com/webank/wedatasphere/exchangis/job/builder/ExchangisJobBuilderContext.java +++ b/exchangis-job/exchangis-job-builder/src/main/java/com/webank/wedatasphere/exchangis/job/builder/ExchangisJobBuilderContext.java @@ -1,9 +1,8 @@ package com.webank.wedatasphere.exchangis.job.builder; -import com.webank.wedatasphere.exchangis.datasource.core.service.MetadataInfoService; +import com.webank.wedatasphere.exchangis.job.builder.api.ExchangisJobBuilder; import com.webank.wedatasphere.exchangis.job.domain.ExchangisJobInfo; -import com.webank.wedatasphere.exchangis.job.listener.JobLogListener; import java.util.HashMap; import java.util.Map; @@ -16,35 +15,25 @@ public class ExchangisJobBuilderContext { /** * Origin job */ - private ExchangisJobInfo originalJob; + protected ExchangisJobInfo originalJob; /** - * Listen the log event + * Current builder */ - private JobLogListener jobLogListener; + protected ExchangisJobBuilder currentBuilder; private Map env = new HashMap<>(); private Map> datasourceParams = new HashMap<>(); - private MetadataInfoService metadataInfoService; - public ExchangisJobBuilderContext() { } - public ExchangisJobBuilderContext(ExchangisJobInfo originalJob, JobLogListener jobLogListener){ + public ExchangisJobBuilderContext(ExchangisJobInfo originalJob){ this.originalJob = originalJob; - this.jobLogListener = jobLogListener; - } - - public ExchangisJobBuilderContext(ExchangisJobInfo originalJob) { - this(originalJob, null); } - public JobLogListener getJobLogListener() { - return jobLogListener; - } public ExchangisJobInfo getOriginalJob() { return originalJob; @@ -73,11 +62,11 @@ public boolean containsEnv(String name) { return this.env.containsKey(name); } - public MetadataInfoService getMetadataInfoService() { - return metadataInfoService; + public ExchangisJobBuilder getCurrentBuilder() { + return currentBuilder; } - public void setMetadataInfoService(MetadataInfoService metadataInfoService) { - this.metadataInfoService = metadataInfoService; + public void setCurrentBuilder(ExchangisJobBuilder currentBuilder) { + this.currentBuilder = currentBuilder; } } diff --git a/exchangis-job/exchangis-job-builder/src/main/java/com/webank/wedatasphere/exchangis/job/builder/api/AbstractExchangisJobBuilder.java b/exchangis-job/exchangis-job-builder/src/main/java/com/webank/wedatasphere/exchangis/job/builder/api/AbstractExchangisJobBuilder.java index 37e833011..9f8fa61d8 100644 --- a/exchangis-job/exchangis-job-builder/src/main/java/com/webank/wedatasphere/exchangis/job/builder/api/AbstractExchangisJobBuilder.java +++ b/exchangis-job/exchangis-job-builder/src/main/java/com/webank/wedatasphere/exchangis/job/builder/api/AbstractExchangisJobBuilder.java @@ -42,13 +42,16 @@ public boolean canBuild(T inputJob) { @Override public E build(T inputJob, E expectOut, ExchangisJobBuilderContext ctx) throws ExchangisJobException { + ExchangisJobBuilder beforeJoBuilder = ctx.getCurrentBuilder(); JobParamDefine.defaultParam.set(new JobParamSet()); contextThreadLocal.set(ctx); + ctx.setCurrentBuilder(this); try { return buildJob(inputJob, expectOut, ctx); } finally{ - JobParamDefine.defaultParam.remove(); + ctx.setCurrentBuilder(beforeJoBuilder); contextThreadLocal.remove(); + JobParamDefine.defaultParam.remove(); } } diff --git a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/api/ExchangisJobOpenService.java b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/api/ExchangisJobOpenService.java new file mode 100644 index 000000000..484fc58ce --- /dev/null +++ b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/api/ExchangisJobOpenService.java @@ -0,0 +1,35 @@ +package com.webank.wedatasphere.exchangis.job.api; + +import com.webank.wedatasphere.exchangis.job.domain.ExchangisJobEntity; +import com.webank.wedatasphere.exchangis.job.exception.ExchangisJobException; +import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobQueryVo; + +import java.util.List; + +/** + * Open for the other module to invoke + */ +public interface ExchangisJobOpenService { + + /** + * Get job entity by id + * @param id + * @return + */ + ExchangisJobEntity getJobById(Long id, boolean basic) throws ExchangisJobException; + + /** + * Query job entity + * @param queryVo query vo + * @param inPage if in page + * @return + */ + List queryJobs(ExchangisJobQueryVo queryVo, boolean inPage) + throws ExchangisJobException; + + /** + * Delete the job entities + * @param idList id list + */ + void deleteJobBatch(List idList) throws ExchangisJobException; +} diff --git a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisEngineJob.java b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisEngineJob.java index 6c700ae7c..2354018ce 100644 --- a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisEngineJob.java +++ b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisEngineJob.java @@ -1,6 +1,8 @@ package com.webank.wedatasphere.exchangis.job.domain; +import com.webank.wedatasphere.exchangis.job.utils.MemUtils; + import java.util.HashMap; import java.util.Map; @@ -20,6 +22,18 @@ public class ExchangisEngineJob extends GenericExchangisJob { */ private Map runtimeParams = new HashMap<>(); + /** + * Memory used in engine job + */ + private Long memoryUsed; + + private String memoryUnit = MemUtils.StoreUnit.MB.name(); + + /** + * Cpu used in engine job + */ + private Long cpuUsed; + public Map getJobContent() { return jobContent; } @@ -35,4 +49,28 @@ public Map getRuntimeParams() { public void setRuntimeParams(Map runtimeParams) { this.runtimeParams = runtimeParams; } + + public Long getMemoryUsed() { + return memoryUsed; + } + + public void setMemoryUsed(Long memoryUsed) { + this.memoryUsed = memoryUsed; + } + + public Long getCpuUsed() { + return cpuUsed; + } + + public void setCpuUsed(Long cpuUsed) { + this.cpuUsed = cpuUsed; + } + + public String getMemoryUnit() { + return memoryUnit; + } + + public void setMemoryUnit(String memoryUnit) { + this.memoryUnit = memoryUnit; + } } diff --git a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisJob.java b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisJob.java index 0f9d8eddf..3107b81ed 100644 --- a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisJob.java +++ b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisJob.java @@ -47,6 +47,5 @@ public interface ExchangisJob extends ExchangisBase{ */ String getCreateUser(); - @Override void setCreateUser(String createUser); } diff --git a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisJobInfo.java b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisJobInfo.java index af474cfe8..ffb854643 100644 --- a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisJobInfo.java +++ b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisJobInfo.java @@ -1,6 +1,6 @@ package com.webank.wedatasphere.exchangis.job.domain; -import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobVO; +import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobVo; /** * Contain the content and parameters @@ -21,11 +21,20 @@ public class ExchangisJobInfo extends GenericExchangisJob { */ protected String jobParams; + /** + * Job description + */ + protected String jobDesc; + + /** + * Job type + */ + protected String jobType; /** * Convert from view object * @param jobVo vo */ - public ExchangisJobInfo(ExchangisJobVO jobVo){ + public ExchangisJobInfo(ExchangisJobVo jobVo){ this.id = jobVo.getId(); this.name = jobVo.getJobName(); this.engineType = jobVo.getEngineType(); @@ -34,7 +43,7 @@ public ExchangisJobInfo(ExchangisJobVO jobVo){ this.createUser = jobVo.getCreateUser(); this.lastUpdateTime = jobVo.getModifyTime(); this.jobContent = jobVo.getContent(); - this.executeUser = jobVo.getProxyUser(); + this.executeUser = jobVo.getExecuteUser(); this.jobParams = jobVo.getJobParams(); } @@ -64,4 +73,20 @@ public String getJobParams() { public void setJobParams(String jobParams) { this.jobParams = jobParams; } + + public String getJobDesc() { + return jobDesc; + } + + public void setJobDesc(String jobDesc) { + this.jobDesc = jobDesc; + } + + public String getJobType() { + return jobType; + } + + public void setJobType(String jobType) { + this.jobType = jobType; + } } diff --git a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisJobPageQuery.java b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisJobPageQuery.java new file mode 100644 index 000000000..babfe84bc --- /dev/null +++ b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisJobPageQuery.java @@ -0,0 +1,50 @@ +package com.webank.wedatasphere.exchangis.job.domain; + +import com.webank.wedatasphere.exchangis.common.pager.PageQuery; + +/** + * For querying page + */ +public class ExchangisJobPageQuery extends PageQuery { + + /** + * Project id + */ + protected Long projectId; + + /** + * Job type + */ + protected String jobType; + + /** + * Job name + */ + protected String jobName; + + protected String createUser; + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public String getJobType() { + return jobType; + } + + public void setJobType(String jobType) { + this.jobType = jobType; + } + + public String getCreateUser() { + return createUser; + } + + public void setCreateUser(String createUser) { + this.createUser = createUser; + } +} diff --git a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/params/JobParamSet.java b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/params/JobParamSet.java index 87bb1ff8f..e81470598 100644 --- a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/params/JobParamSet.java +++ b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/params/JobParamSet.java @@ -1,9 +1,6 @@ package com.webank.wedatasphere.exchangis.job.domain.params; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -55,12 +52,13 @@ public JobParam load(JobParamDefine jobParamDefine){ @SuppressWarnings("unchecked") public JobParam load(JobParamDefine jobParamDefine, Object source){ - return (JobParam) jobParamStore.compute(jobParamDefine.getKey(), (key, value) -> { - if (Objects.isNull(value)) { - value = prepare(jobParamDefine, source); - } - return value; - }); + // Avoid the deadlock problem in nested call, we should not use compute/computeIfAbsent method + JobParam jobParam = this.jobParamStore.get(jobParamDefine.getKey()); + if (Objects.isNull(jobParam)){ + jobParam = prepare(jobParamDefine, source); + this.jobParamStore.put(jobParamDefine.getKey(),jobParam); + } + return (JobParam) jobParam; } public JobParamSet combine(JobParamSet paramSet){ diff --git a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/vo/ExchangisJobQueryVo.java b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/vo/ExchangisJobQueryVo.java new file mode 100644 index 000000000..d0ba5d4f9 --- /dev/null +++ b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/vo/ExchangisJobQueryVo.java @@ -0,0 +1,19 @@ +package com.webank.wedatasphere.exchangis.job.vo; + +import com.webank.wedatasphere.exchangis.job.domain.ExchangisJobPageQuery; + +public class ExchangisJobQueryVo extends ExchangisJobPageQuery { + + public ExchangisJobQueryVo(){ + + } + + public ExchangisJobQueryVo(Long projectId, String jobType, + String name, Integer current, Integer size){ + this.projectId = projectId; + this.jobType = jobType; + this.jobName = name; + this.current = current; + this.size = size; + } +} diff --git a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/vo/ExchangisJobVO.java b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/vo/ExchangisJobVO.java deleted file mode 100644 index d0bd6607a..000000000 --- a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/vo/ExchangisJobVO.java +++ /dev/null @@ -1,268 +0,0 @@ -package com.webank.wedatasphere.exchangis.job.vo; - - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; - -import java.util.Date; -import java.util.HashMap; -import java.util.Map; - -/** - *

- * 任务表. - *

- * - * @author yuxin.yuan - * @since 2021-08-10 - */ -@TableName("exchangis_job_info") -public class ExchangisJobVO { - - - @TableId(type = IdType.AUTO) - private Long id; - - private Long projectId; - - private Long dssProjectId; - - private String dssProjectName; - - private String nodeId; - - private String nodeName; - - private String jobType; - - private String engineType; - - private String jobLabels; - - private String jobName; - - private String jobDesc; - - private String content; - - private String alarmUser; - - private Integer alarmLevel; - - private String proxyUser; - - private String executeNode; - - private String syncType; - - private String jobParams; - - private Date createTime; - - private String createUser; - - private Date modifyTime; - - private String modifyUser; - - /*private Map source = new HashMap<>(); - - public Map getSource() { - return source; - } - - public void setSource(Map source) { - this.source = source; - }*/ - - public Long getId() { return id; } - - public void setId(Long id) { this.id = id; } - - public String getJobName() { - return jobName; - } - - public void setJobName(String jobName) { - this.jobName = jobName; - } - - public String getJobDesc() { - return jobDesc; - } - - public void setJobDesc(String jobDesc) { - //source.put("jobDesc", jobDesc); - this.jobDesc = jobDesc; - } - - public Date getCreateTime() { - return createTime; - } - - public void setCreateTime(Date createTime) { - this.createTime = createTime; - } - - public Date getModifyTime() { - return modifyTime; - } - - public void setModifyTime(Date modifyTime) { - //source.put("modifyTime", modifyTime); - this.modifyTime = modifyTime; - } - - public Long getProjectId() { - return projectId; - } - - public void setProjectId(Long projectId) { - this.projectId = projectId; - } - - public Long getDssProjectId() { return dssProjectId; } - - public void setDssProjectId(Long dssProjectId) { - //source.put("dssProjectId", dssProjectId); - this.dssProjectId = dssProjectId; } - - - public String getNodeName() { return nodeName; } - - public void setNodeName(String nodeName) { - //source.put("nodeName", nodeName); - this.nodeName = nodeName; } - - public String getJobType() { - return jobType; - } - - public void setJobType(String jobType) { - //source.put("jobType", jobType); - this.jobType = jobType; - } - - public String getEngineType() { - return engineType; - } - - public void setEngineType(String engineType) { - this.engineType = engineType; - } - - public String getJobLabels() { - return jobLabels; - } - - public void setJobLabels(String jobLabel) { - this.jobLabels = jobLabel; - } - - public String getContent() { - return content; - } - - public void setContent(String content) { - this.content = content; - } - - public String getAlarmUser() { - return alarmUser; - } - - public void setAlarmUser(String alarmUser) { - //source.put("alarmUser", alarmUser); - this.alarmUser = alarmUser; - } - - public Integer getAlarmLevel() { - return alarmLevel; - } - - public void setAlarmLevel(Integer alarmLevel) { - //source.put("alarmLevel", alarmLevel); - this.alarmLevel = alarmLevel; - } - - public String getProxyUser() { - return proxyUser; - } - - public void setProxyUser(String proxyUser) { - //source.put("proxyUser", proxyUser); - this.proxyUser = proxyUser; - } - - public String getExecuteNode() { - return executeNode; - } - - public void setExecuteNode(String executeNode) { - //source.put("executeNode", executeNode); - this.executeNode = executeNode; - } - - public String getSyncType() { - return syncType; - } - - public void setSyncType(String syncType) { - //source.put("syncType", syncType); - this.syncType = syncType; - } - - public String getJobParams() { - return jobParams; - } - - public void setJobParams(String jobParams) { - this.jobParams = jobParams; - } - - public String getCreateUser() { - return createUser; - } - - public void setCreateUser(String createUser) { - this.createUser = createUser; - } - - public String getModifyUser() { - return modifyUser; - } - - public void setModifyUser(String modifyUser) { - this.modifyUser = modifyUser; - } - - public String getDssProjectName() { - return dssProjectName; - } - - public void setDssProjectName(String dssProjectName) { - //source.put("dssProjectName", dssProjectName); - this.dssProjectName = dssProjectName; - } - - public void setNodeId(String nodeId) { - //source.put("nodeId", nodeId); - this.nodeId = nodeId; - } - - public String getNodeId() { - return nodeId; - } - - @Override - public String toString() { - return "ExchangisJob{" + "id=" + id + ", projectId=" + projectId + ", jobName=" + jobName + ", jobType=" - + jobType - + ", engineType=" + engineType + ", jobLabels=" + jobLabels + ", jobDesc=" + jobDesc + ", content=" - + content + ", alarmUser=" + alarmUser + ", alarmLevel=" + alarmLevel + ", proxyUser=" + proxyUser - + ", executeNode=" + executeNode + ", syncType=" + syncType + ", jobParams=" + jobParams + ", createTime=" - + createTime + ", createUser=" + createUser + ", modifyTime=" + modifyTime + ", modifyUser=" + modifyUser - + "}"; - } -} diff --git a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/vo/ExchangisJobVo.java b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/vo/ExchangisJobVo.java new file mode 100644 index 000000000..d965b728e --- /dev/null +++ b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/vo/ExchangisJobVo.java @@ -0,0 +1,287 @@ +package com.webank.wedatasphere.exchangis.job.vo; + + + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.webank.wedatasphere.exchangis.common.validator.groups.InsertGroup; +import com.webank.wedatasphere.exchangis.job.domain.ExchangisJobInfo; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Size; +import java.util.*; + +/** + * + */ +//@JsonInclude(JsonInclude.Include.NON_EMPTY) +public class ExchangisJobVo { + + /** + * Job id + */ + private Long id; + + /** + * Project id + */ + @NotNull(groups = InsertGroup.class, message = "Project id cannot be null (工程ID不能为空)") + private Long projectId; + + /** + * Job type + */ + @NotBlank(message = "Job type cannot be null (任务类型不能为空)") + @Size(max = 50, message= "Length of job type should be less than 50 (任务类型长度不超过50)") + private String jobType; + + /** + * Engine type + */ + @NotBlank(message = "Engine type cannot be null (引擎类型不能为空)") + @Size(max = 50, message = "Length of engine type should be less than 50 (引擎类型长度不超过50)") + private String engineType; + + /** + * Job labels + */ + @Size(max = 200, message = "Length of labels should be less than 200 (标签长度不能超过200)") + private String jobLabels; + + /** + * Job name + */ + @Size(max = 100, message = "Length of name should be less than 100 (名称长度不超过100)") + @NotBlank(message = "Job name cannot be null (任务名不能为空)") + private String jobName; + + /** + * Job desc + */ + @Size(max = 200, message = "Length of desc should be less than 200 (描述长度不超过200)") + private String jobDesc; + + /** + * Content + */ + private String content; + + /** + * Execute user + */ + //@JsonProperty("proxyUser") + private String executeUser; + + /** + * Execute node + */ + @Deprecated + private String executeNode; + + /** + * Job params + */ + private String jobParams; + + /** + * Create time + */ + private Date createTime; + + /** + * Create user + */ + private String createUser; + + /** + * Modify time (last_update_time) + */ + private Date modifyTime; + + /** + * Modify user + */ + private String modifyUser; + + /** + * Source map + */ + private Map source = new HashMap(); + + private Map labels; + + private String proxyUser; + + public ExchangisJobVo(){ + + } + + public ExchangisJobVo(ExchangisJobInfo jobInfo){ + if (Objects.nonNull(jobInfo)) { + this.id = jobInfo.getId(); + this.engineType = jobInfo.getEngineType(); + this.jobDesc = jobInfo.getJobDesc(); + this.jobLabels = jobInfo.getJobLabel(); + this.jobName = jobInfo.getName(); + this.jobType = jobInfo.getJobType(); + this.createTime = jobInfo.getCreateTime(); + this.createUser = jobInfo.getCreateUser(); + this.modifyTime = jobInfo.getLastUpdateTime(); + this.jobParams = jobInfo.getJobParams(); + this.executeUser = jobInfo.getExecuteUser(); + this.proxyUser = jobInfo.getExecuteUser(); + } + } + + public String getProxyUser() { + return proxyUser; + } + + public void setProxyUser(String proxyUser) { + this.proxyUser = proxyUser; + } + + public Long getId() { return id; } + + public void setId(Long id) { this.id = id; } + + public String getJobName() { + return jobName; + } + + public void setJobName(String jobName) { + this.jobName = jobName; + } + + public String getJobDesc() { + return jobDesc; + } + + public void setJobDesc(String jobDesc) { + this.jobDesc = jobDesc; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getModifyTime() { + return modifyTime; + } + + public void setModifyTime(Date modifyTime) { + this.modifyTime = modifyTime; + } + + public Long getProjectId() { + return projectId; + } + + public void setProjectId(Long projectId) { + this.projectId = projectId; + } + + public String getJobType() { + return jobType; + } + + public void setJobType(String jobType) { + this.jobType = jobType; + } + + public String getEngineType() { + return engineType; + } + + public void setEngineType(String engineType) { + this.engineType = engineType; + } + + public String getJobLabels() { + return jobLabels; + } + + public void setJobLabels(String jobLabel) { + this.jobLabels = jobLabel; + } + + public String getContent() { + return content; + } + + public void setContent(String content) { + this.content = content; + } + + public String getExecuteNode() { + Object executeNode = source.get("executeNode"); + return Objects.nonNull(executeNode)? String.valueOf(executeNode) : null; + } + + + public void setExecuteNode(String executeNode) { + source.put("executeNode", executeNode); + } + + public String getSyncType() { + Object syncType = source.get("syncType"); + return Objects.nonNull(syncType)? String.valueOf(syncType) : null; + } + + public void setSyncType(String syncType) { + source.put("syncType", syncType); + } + + public String getJobParams() { + return jobParams; + } + + public void setJobParams(String jobParams) { + this.jobParams = jobParams; + } + + public String getCreateUser() { + return createUser; + } + + public void setCreateUser(String createUser) { + this.createUser = createUser; + } + + public String getModifyUser() { + return modifyUser; + } + + public void setModifyUser(String modifyUser) { + this.modifyUser = modifyUser; + } + + public String getExecuteUser() { + return executeUser; + } + + public void setExecuteUser(String executeUser) { + this.executeUser = executeUser; + } + + public Map getSource() { + return source; + } + + public void setSource(Map source) { + this.source.putAll(source); + } + + public Map getLabels() { + return labels; + } + + public void setLabels(Map labels) { + this.labels = labels; + } +} diff --git a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/ExchangisLauncherConfiguration.java b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/ExchangisLauncherConfiguration.java index 9279ccb16..3ba907a48 100644 --- a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/ExchangisLauncherConfiguration.java +++ b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/ExchangisLauncherConfiguration.java @@ -8,6 +8,12 @@ public class ExchangisLauncherConfiguration { public static final String TASK_NOT_EXIST = "Not exists EngineConn"; + public static final String LAUNCHER_LINKIS_RUNTIME_PARAM_NAME = "runtimeParams"; + + public static final String LAUNCHER_LINKIS_STARTUP_PARAM_NAME = "startUpParams"; + + public static final String LAUNCHER_LINKIS_REQUEST_MEMORY = "wds.linkis.engineconn.java.driver.memory"; + public static final CommonVars LAUNCHER_LINKIS_CREATOR = CommonVars.apply("wds.exchangis.job.task.launcher.linkis.creator", "exchangis"); public static final CommonVars LAUNCHER_LINKIS_ENGINE_CONN_MODE = CommonVars.apply("wds.exchangis.job.task.launcher.linkis.engineConn.mode", "once"); diff --git a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/builder/LinkisExchangisLauncherJobBuilder.java b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/builder/LinkisExchangisLauncherJobBuilder.java index 51e2ef89d..8b6fda944 100644 --- a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/builder/LinkisExchangisLauncherJobBuilder.java +++ b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/builder/LinkisExchangisLauncherJobBuilder.java @@ -5,6 +5,13 @@ import com.webank.wedatasphere.exchangis.job.domain.ExchangisEngineJob; import com.webank.wedatasphere.exchangis.job.exception.ExchangisJobException; import com.webank.wedatasphere.exchangis.job.launcher.domain.LaunchableExchangisTask; +import com.webank.wedatasphere.exchangis.job.utils.MemUtils; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import static com.webank.wedatasphere.exchangis.job.launcher.ExchangisLauncherConfiguration.*; /** * Launcher job builder @@ -13,6 +20,7 @@ public class LinkisExchangisLauncherJobBuilder extends AbstractExchangisJobBuilder { private static final String LAUNCHER_NAME = "Linkis"; + @Override public LaunchableExchangisTask buildJob(ExchangisEngineJob inputJob, LaunchableExchangisTask expectOut, ExchangisJobBuilderContext ctx) throws ExchangisJobException { LaunchableExchangisTask launchableTask = new LaunchableExchangisTask(); @@ -21,7 +29,14 @@ public LaunchableExchangisTask buildJob(ExchangisEngineJob inputJob, LaunchableE launchableTask.setExecuteUser(inputJob.getCreateUser()); // launcherJob.setExecuteNode(exchangisJob.getExecuteNode()); launchableTask.setLinkisContentMap(inputJob.getJobContent()); - launchableTask.setLinkisParamsMap(inputJob.getRuntimeParams()); + Map linkisParams = new HashMap<>(); + Map startUpParams = new HashMap<>(); + linkisParams.put(LAUNCHER_LINKIS_RUNTIME_PARAM_NAME, inputJob.getRuntimeParams()); + linkisParams.put(LAUNCHER_LINKIS_STARTUP_PARAM_NAME, startUpParams); + long memoryUsed = Objects.nonNull(inputJob.getMemoryUsed())? MemUtils.convertToGB(inputJob.getMemoryUsed(), + inputJob.getMemoryUnit()) : 0; + startUpParams.put(LAUNCHER_LINKIS_REQUEST_MEMORY, String.valueOf(memoryUsed <= 0 ? 1 : memoryUsed)); + launchableTask.setLinkisParamsMap(linkisParams); launchableTask.setEngineType(inputJob.getEngineType()); launchableTask.setLabels(inputJob.getJobLabel()); launchableTask.setName(inputJob.getName()); diff --git a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/entity/LaunchedExchangisJobEntity.java b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/entity/LaunchedExchangisJobEntity.java index 1aac7a1f0..59b991cd5 100644 --- a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/entity/LaunchedExchangisJobEntity.java +++ b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/entity/LaunchedExchangisJobEntity.java @@ -3,8 +3,11 @@ import com.webank.wedatasphere.exchangis.job.domain.ExchangisJobEntity; import com.webank.wedatasphere.exchangis.job.launcher.domain.LaunchableExchangisJob; +import org.apache.commons.io.IOUtils; +import java.text.SimpleDateFormat; import java.util.Calendar; +import java.util.Date; /** * Entity to persist the launched job @@ -54,7 +57,9 @@ public LaunchedExchangisJobEntity(LaunchableExchangisJob job){ this.createTime = job.getCreateTime(); this.lastUpdateTime = job.getLastUpdateTime(); this.jobExecutionId = job.getJobExecutionId(); - this.logPath = this.executeUser + "/" + this.jobExecutionId; + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd"); + this.logPath = this.executeUser + IOUtils.DIR_SEPARATOR_UNIX + + simpleDateFormat.format(new Date()) + IOUtils.DIR_SEPARATOR_UNIX + this.jobExecutionId; } public String getJobExecutionId() { return jobExecutionId; diff --git a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisLauncherTask.java b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisLauncherTask.java index b946091fd..19c0e3a5f 100644 --- a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisLauncherTask.java +++ b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisLauncherTask.java @@ -251,6 +251,7 @@ public synchronized void submit() throws ExchangisTaskLaunchException { * @param task task * @return once job */ + @SuppressWarnings("unchecked") private SimpleOnceJob toSubmittableJob(LaunchableExchangisTask task){ //TODO deal the start up params LinkisJobBuilder jobBuilder = LinkisJobClient.once().simple().builder().setCreateService(LAUNCHER_LINKIS_CREATOR.getValue()) @@ -261,7 +262,16 @@ private SimpleOnceJob toSubmittableJob(LaunchableExchangisTask task){ .addLabel(LabelKeyUtils.ENGINE_CONN_MODE_LABEL_KEY(), LAUNCHER_LINKIS_ENGINE_CONN_MODE.getValue()) .addExecuteUser(task.getExecuteUser()); Optional.ofNullable(task.getLinkisContentMap()).ifPresent(params -> params.forEach(jobBuilder::addJobContent)); - Optional.ofNullable(task.getLinkisParamsMap()).ifPresent(params -> params.forEach(jobBuilder::addRuntimeParam)); + Optional.ofNullable(task.getLinkisParamsMap()).ifPresent(params -> { + Object runtimeParams = params.get(LAUNCHER_LINKIS_RUNTIME_PARAM_NAME); + if (Objects.nonNull(runtimeParams) && runtimeParams instanceof Map){ + jobBuilder.setRuntimeParams((Map) runtimeParams); + } + Object startupParams = params.get(LAUNCHER_LINKIS_STARTUP_PARAM_NAME); + if (Objects.nonNull(startupParams) && startupParams instanceof Map){ + jobBuilder.setStartupParams((Map) startupParams); + } + }); return jobBuilder.build(); } diff --git a/exchangis-job/exchangis-job-metrics/src/main/java/com/webank/wedatasphere/exchangis/metrics/service/impl/ExchangisMetricsServiceImpl.java b/exchangis-job/exchangis-job-metrics/src/main/java/com/webank/wedatasphere/exchangis/metrics/service/impl/ExchangisMetricsServiceImpl.java index 59d721d40..8ce15f72e 100644 --- a/exchangis-job/exchangis-job-metrics/src/main/java/com/webank/wedatasphere/exchangis/metrics/service/impl/ExchangisMetricsServiceImpl.java +++ b/exchangis-job/exchangis-job-metrics/src/main/java/com/webank/wedatasphere/exchangis/metrics/service/impl/ExchangisMetricsServiceImpl.java @@ -49,7 +49,7 @@ public class ExchangisMetricsServiceImpl implements ExchangisMetricsService { // Optional.ofNullable(unlock).ifPresent(metrices::add); // // Message message = Message.ok(); -// message.setMethod("/exchangis/metrics/taskstate"); +// message.setMethod("/dss/exchangis/main/metrics/taskstate"); // message.data("metrices", metrices); // return message; // } @@ -82,7 +82,7 @@ public Message getTaskStatusMetrics(HttpServletRequest request) { metrices.add(unlock); Message message = Message.ok(); - message.setMethod("/exchangis/metrics/taskstate"); + message.setMethod("/dss/exchangis/main/metrics/taskstate"); message.data("metrices", metrices); return message; } @@ -133,7 +133,7 @@ public Message getTaskProcessMetrics(HttpServletRequest request) { list.add(fps); Message message = Message.ok(); - message.setMethod("/exchangis/metrics/taskprocess"); + message.setMethod("/dss/exchangis/main/metrics/taskprocess"); message.data("list", list); return message; } @@ -213,7 +213,7 @@ public Message getDataSourceFlowMetrics(HttpServletRequest request) { // dataset.add(ds2Data); // dataset.add(ds3Data); Message message = Message.ok(); - message.setMethod("/exchangis/metrics/datasourceflow"); + message.setMethod("/dss/exchangis/main/metrics/datasourceflow"); message.data("dataset", dataset); return message; } @@ -272,7 +272,7 @@ public Message getEngineResourceCpuMetrics(HttpServletRequest request) { // dataset.add(ds3Data); // dataset.add(realData); Message message = Message.ok(); - message.setMethod("/exchangis/metrics/engineresourcecpu"); + message.setMethod("/dss/exchangis/main/metrics/engineresourcecpu"); message.data("dataset", dataset); return message; @@ -351,7 +351,7 @@ public Message getEngineResourceMemMetrics(HttpServletRequest request) { // dataset.add(ds2Data); // dataset.add(ds3Data); Message message = Message.ok(); - message.setMethod("/exchangis/metrics/engineresourcemem"); + message.setMethod("/dss/exchangis/main/metrics/engineresourcemem"); message.data("dataset", dataset); return message; } @@ -378,7 +378,7 @@ public Message getEngineResourceMetrics(HttpServletRequest request) { list.add(linkis); Message message = Message.ok(); - message.setMethod("/exchangis/metrics/engineresource"); + message.setMethod("/dss/exchangis/main/metrics/engineresource"); message.data("list", list); return message; } diff --git a/exchangis-job/exchangis-job-server/pom.xml b/exchangis-job/exchangis-job-server/pom.xml index f63f1e5e2..9885c5574 100644 --- a/exchangis-job/exchangis-job-server/pom.xml +++ b/exchangis-job/exchangis-job-server/pom.xml @@ -46,6 +46,12 @@ mysql-connector-java 5.1.49 + + com.webank.wedatasphere.exchangis + exchangis-project-server + 1.0.0-RC1 + compile + diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/ExchangisJobExecuteAutoConfiguration.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/ExchangisJobExecuteAutoConfiguration.java index 4805e1c5a..6faa97ceb 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/ExchangisJobExecuteAutoConfiguration.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/ExchangisJobExecuteAutoConfiguration.java @@ -171,7 +171,7 @@ public AbstractTaskExecution taskExecution(Scheduler scheduler, ExchangisTaskLau ConsumerManager consumerManager = scheduler.getSchedulerContext().getOrCreateConsumerManager(); if (consumerManager instanceof TenancyParallelConsumerManager){ ((TenancyParallelConsumerManager) consumerManager).setInitResidentThreads(observers.size() + - (Objects.nonNull(loadBalancer)? 1: 0) + 1); + (Objects.nonNull(loadBalancer)? 1: 0)); } Optional.ofNullable(executionListeners).ifPresent(listeners -> listeners.forEach(taskExecution::addListener)); return taskExecution; diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/ServiceInExchangisJobBuilderContext.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/ServiceInExchangisJobBuilderContext.java new file mode 100644 index 000000000..2f73f7ca0 --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/ServiceInExchangisJobBuilderContext.java @@ -0,0 +1,76 @@ +package com.webank.wedatasphere.exchangis.job.server.builder; + +import com.webank.wedatasphere.exchangis.datasource.core.service.MetadataInfoService; +import com.webank.wedatasphere.exchangis.job.builder.ExchangisJobBuilderContext; +import com.webank.wedatasphere.exchangis.job.builder.api.ExchangisJobBuilder; +import com.webank.wedatasphere.exchangis.job.domain.ExchangisJobInfo; +import com.webank.wedatasphere.exchangis.job.listener.JobLogListener; +import com.webank.wedatasphere.exchangis.job.listener.events.JobLogEvent; +import com.webank.wedatasphere.exchangis.job.server.log.JobServerLogging; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Objects; + +/** + * Service in job builder context + */ +public class ServiceInExchangisJobBuilderContext extends ExchangisJobBuilderContext { + + /** + * Meta info service + */ + private MetadataInfoService metadataInfoService; + + /** + * Job execution id + */ + private String jobExecutionId; + + /** + * Logging + */ + private JobServerLogging> logging; + + public ServiceInExchangisJobBuilderContext(ExchangisJobInfo originalJob, + JobLogListener jobLogListener) { + super(originalJob); + this.logging = new JobServerLogging>() { + @Override + public Logger getLogger() { + return Objects.nonNull(currentBuilder)? + LoggerFactory.getLogger(currentBuilder.getClass()) : null; + } + + @Override + public JobLogListener getJobLogListener() { + return jobLogListener; + } + + @Override + public JobLogEvent getJobLogEvent(JobLogEvent.Level level, ExchangisJobBuilder builder, String message, Object... args) { + return new JobLogEvent(level, originalJob.getExecuteUser(), jobExecutionId, message, args); + } + }; + } + + public String getJobExecutionId() { + return jobExecutionId; + } + + public void setJobExecutionId(String jobExecutionId) { + this.jobExecutionId = jobExecutionId; + } + + public MetadataInfoService getMetadataInfoService() { + return metadataInfoService; + } + + public void setMetadataInfoService(MetadataInfoService metadataInfoService) { + this.metadataInfoService = metadataInfoService; + } + + public JobServerLogging> getLogging() { + return logging; + } +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/engine/AbstractLoggingExchangisJobBuilder.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/engine/AbstractLoggingExchangisJobBuilder.java new file mode 100644 index 000000000..26b1bc672 --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/engine/AbstractLoggingExchangisJobBuilder.java @@ -0,0 +1,52 @@ +package com.webank.wedatasphere.exchangis.job.server.builder.engine; + +import com.webank.wedatasphere.exchangis.job.builder.ExchangisJobBuilderContext; +import com.webank.wedatasphere.exchangis.job.builder.api.AbstractExchangisJobBuilder; +import com.webank.wedatasphere.exchangis.job.domain.ExchangisBase; +import com.webank.wedatasphere.exchangis.job.domain.ExchangisJob; +import com.webank.wedatasphere.exchangis.job.exception.ExchangisJobException; +import com.webank.wedatasphere.exchangis.job.server.builder.ServiceInExchangisJobBuilderContext; + +/** + * Abstract implement for engine job builder + */ +public abstract class AbstractLoggingExchangisJobBuilder extends + AbstractExchangisJobBuilder { + + /** + * Get builder context + * @return context + * @throws ExchangisJobException.Runtime exception + */ + protected static ServiceInExchangisJobBuilderContext getServiceInBuilderContext() throws ExchangisJobException.Runtime{ + ExchangisJobBuilderContext context = getCurrentBuilderContext(); + if (!(context instanceof ServiceInExchangisJobBuilderContext)) { + throw new ExchangisJobException.Runtime(-1, "The job builder context cannot not be casted to " + ServiceInExchangisJobBuilderContext.class.getCanonicalName(), null); + } + return (ServiceInExchangisJobBuilderContext)context; + } + + /** + * Warn message + * @param message message + */ + public static void warn(String message, Object... args){ + getServiceInBuilderContext().getLogging().warn(null, message, args); + } + + public static void warn(String message, Throwable t){ + getServiceInBuilderContext().getLogging().warn(null, message, t); + } + + /** + * Info message + * @param message message + */ + public static void info(String message, Object... args){ + getServiceInBuilderContext().getLogging().info(null, message, args); + } + + public static void info(String message, Throwable t){ + getServiceInBuilderContext().getLogging().info(null, message, t); + } +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/engine/DefaultExchangisEngineJobBuilder.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/engine/DefaultExchangisEngineJobBuilder.java index 6de52bee5..152e058bd 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/engine/DefaultExchangisEngineJobBuilder.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/engine/DefaultExchangisEngineJobBuilder.java @@ -11,12 +11,14 @@ import java.util.Map; import java.util.Objects; +import java.util.Optional; /** * Default implements */ public class DefaultExchangisEngineJobBuilder extends AbstractExchangisJobBuilder { + private static final String ENGINE_JOB_MEMORY_USED = "setting.max.memory"; @Override public int priority() { return Integer.MIN_VALUE; @@ -32,6 +34,9 @@ public ExchangisEngineJob buildJob(SubExchangisJob inputJob, ExchangisEngineJob exchangisEngineJob.getJobContent().putAll(jobParams); } } + Map settings = inputJob.getParamsToMap(SubExchangisJob.REALM_JOB_SETTINGS, false); + Optional.ofNullable(settings.get(ENGINE_JOB_MEMORY_USED)).ifPresent(memoryUsed -> exchangisEngineJob.setMemoryUsed(Long.valueOf(String.valueOf(memoryUsed)))); + exchangisEngineJob.setRuntimeParams(settings); exchangisEngineJob.setEngineType(ctx.getOriginalJob().getEngineType()); exchangisEngineJob.setName(inputJob.getName()); exchangisEngineJob.setJobLabel(ctx.getOriginalJob().getJobLabel()); diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/engine/SqoopExchangisEngineJob.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/engine/SqoopExchangisEngineJob.java index b37c87049..1ab5241d9 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/engine/SqoopExchangisEngineJob.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/engine/SqoopExchangisEngineJob.java @@ -2,9 +2,25 @@ import com.webank.wedatasphere.exchangis.job.domain.ExchangisEngineJob; +import java.util.Objects; + /** * Sqoop engine job */ public class SqoopExchangisEngineJob extends ExchangisEngineJob { //Empty + + public SqoopExchangisEngineJob(){ + + } + + public SqoopExchangisEngineJob(ExchangisEngineJob engineJob){ + if (Objects.nonNull(engineJob)) { + setName(engineJob.getName()); + setEngineType(engineJob.getEngineType()); + getJobContent().putAll(engineJob.getJobContent()); + getRuntimeParams().putAll(engineJob.getRuntimeParams()); + setMemoryUsed(engineJob.getMemoryUsed()); + } + } } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/engine/SqoopExchangisEngineJobBuilder.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/engine/SqoopExchangisEngineJobBuilder.java index 786b7e4c8..7ab5e2fd7 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/engine/SqoopExchangisEngineJobBuilder.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/engine/SqoopExchangisEngineJobBuilder.java @@ -1,9 +1,11 @@ package com.webank.wedatasphere.exchangis.job.server.builder.engine; +import com.webank.wedatasphere.exchangis.datasource.core.domain.MetaColumn; +import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisDataSourceException; import com.webank.wedatasphere.exchangis.datasource.core.utils.Json; import com.webank.wedatasphere.exchangis.job.builder.ExchangisJobBuilderContext; -import com.webank.wedatasphere.exchangis.job.builder.api.AbstractExchangisJobBuilder; import com.webank.wedatasphere.exchangis.job.domain.ExchangisEngineJob; +import com.webank.wedatasphere.exchangis.job.domain.ExchangisJobInfo; import com.webank.wedatasphere.exchangis.job.domain.SubExchangisJob; import com.webank.wedatasphere.exchangis.job.domain.params.JobParam; import com.webank.wedatasphere.exchangis.job.domain.params.JobParamDefine; @@ -12,6 +14,7 @@ import com.webank.wedatasphere.exchangis.job.exception.ExchangisJobException; import com.webank.wedatasphere.exchangis.job.exception.ExchangisJobExceptionCode; import com.webank.wedatasphere.exchangis.job.server.builder.JobParamConstraints; +import com.webank.wedatasphere.exchangis.job.server.builder.ServiceInExchangisJobBuilderContext; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,7 +32,7 @@ import static com.webank.wedatasphere.exchangis.job.server.builder.engine.SqoopExchangisEngineJobBuilder.MODE_TYPE.EXPORT; import static com.webank.wedatasphere.exchangis.job.server.builder.engine.SqoopExchangisEngineJobBuilder.MODE_TYPE.IMPORT; -public class SqoopExchangisEngineJobBuilder extends AbstractExchangisJobBuilder { +public class SqoopExchangisEngineJobBuilder extends AbstractLoggingExchangisJobBuilder { private static final Logger LOG = LoggerFactory.getLogger(SqoopExchangisEngineJobBuilder.class); @@ -37,6 +40,20 @@ public class SqoopExchangisEngineJobBuilder extends AbstractExchangisJobBuilder< private static final List SUPPORT_RDBMS_TYPES = Arrays.asList("MYSQL", "ORACLE"); + private static final String META_INPUT_FORMAT = "file.inputformat"; + + private static final String META_OUTPUT_FORMAT = "file.outputformat"; + + private static final String META_FIELD_DELIMITER = "field.delim"; + + /** + * //TODO To support different hadoop version + */ + private static final List HADOOP_TEXT_INPUT_FORMAT = Collections.singletonList("org.apache.hadoop.mapred.TextInputFormat"); + + private static final List HADOOP_TEXT_OUTPUT_FORMAT = Arrays.asList("org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat", + "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"); + public enum MODE_TYPE { IMPORT, EXPORT} /** * Verbose, default null (means not open verbose) @@ -63,15 +80,103 @@ public enum MODE_TYPE { IMPORT, EXPORT} MODE_TYPE modeParam = MODE_ENUM.getValue(job); return modeParam.equals(IMPORT)? job.getRealmParams(REALM_JOB_CONTENT_SINK) : job.getRealmParams(REALM_JOB_CONTENT_SOURCE); }); + /** - * //TODO Get the file type from service - * Whether hcatalog + * Hive-partition-map */ - private static final JobParamDefine IS_USE_HCATALOG = JobParams.define("sqoop.use.hcatalog", (BiFunction)(k, job) ->{ -// getCurrentBuilderContext().getMetadataInfoService().getTableProps() - return true; + @SuppressWarnings("unchecked") + private static final JobParamDefine> PARTITION_MAP = JobParams.define("sqoop.partition.map", (BiFunction>) (k, job) -> { + if ("hive".equalsIgnoreCase(job.getSinkType()) || "hive".equalsIgnoreCase(job.getSourceType())){ + JobParam partitionParam = MODE_HADOOP_PARAMS.getValue(job).get(JobParamConstraints.PARTITION); + if (Objects.nonNull(partitionParam)) { + Object partition = partitionParam.getValue(); + if (partition instanceof Map) { + return (Map) partition; + } else { + String partitionStr = String.valueOf(partition); + if (StringUtils.isNotBlank(partitionStr)){ + Map partitionMap = new HashMap<>(); + String[] partValues = partitionStr.split(","); + for (String partValue : partValues){ + String[] parts = partValue.split("="); + if (parts.length == 2){ + partitionMap.put(parts[0], parts[1]); + } + } + return partitionMap; + } + } + } + } + return null; + }); + + /** + * Meta columns + */ + private static final JobParamDefine> META_COLUMNS = JobParams.define("sqoop.meta.table.columns", (BiFunction>) (k, paramSet) -> { + ServiceInExchangisJobBuilderContext context = getServiceInBuilderContext(); + JobParam dataSourceId = paramSet.get(JobParamConstraints.DATA_SOURCE_ID); + JobParam database = paramSet.get(JobParamConstraints.DATABASE, String.class); + JobParam table = paramSet.get(JobParamConstraints.TABLE, String.class); + try { + return context.getMetadataInfoService().getColumns(context.getOriginalJob().getCreateUser(), + Long.valueOf(dataSourceId.getValue()), database.getValue(), table.getValue()); + } catch (ExchangisDataSourceException e) { + throw new ExchangisJobException.Runtime(e.getErrCode(), e.getMessage(), e.getCause()); + } + }); + + /** + * Meta hadoop columns + */ + private static final JobParamDefine> META_HADOOP_COLUMNS = JobParams.define("sqoop.meta.hadoop.table.columns", (BiFunction>) (k, job) -> META_COLUMNS.newValue(MODE_HADOOP_PARAMS.getValue(job))); + + /** + * Meta rdbms columns + */ + private static final JobParamDefine> META_RDBMS_COLUMNS = JobParams.define("sqoop.meta.rdbms.table.columns", (BiFunction>) (k, job) -> META_COLUMNS.newValue(MODE_RDBMS_PARAMS.getValue(job))); + /** + * Meta table/partition props + */ + private static final JobParamDefine> META_HADOOP_TABLE_PROPS = JobParams.define("sqoop.meta.hadoop.table.props", (BiFunction>) (k, job) ->{ + ServiceInExchangisJobBuilderContext context = getServiceInBuilderContext(); + ExchangisJobInfo jobInfo = context.getOriginalJob(); + // Use the creator as userName + String userName = jobInfo.getCreateUser(); + JobParamSet hadoopParamSet = MODE_HADOOP_PARAMS.getValue(job); + JobParam dataSourceId = hadoopParamSet.get(JobParamConstraints.DATA_SOURCE_ID); + JobParam database = hadoopParamSet.get(JobParamConstraints.DATABASE, String.class); + JobParam table = hadoopParamSet.get(JobParamConstraints.TABLE, String.class); + Map partition = PARTITION_MAP.getValue(job); + try { + if (Objects.isNull(partition)) { + return context.getMetadataInfoService().getTableProps(userName, Long.valueOf(dataSourceId.getValue()), + database.getValue(), table.getValue()); + } else { + return context.getMetadataInfoService().getPartitionProps(userName, Long.valueOf(dataSourceId.getValue()), + database.getValue(), table.getValue(), URLEncoder.encode(partition.entrySet().stream().map(entry -> + entry.getKey() + "=" + entry.getValue() + ).collect(Collectors.joining(",")), "UTF-8")); + } + } catch (ExchangisDataSourceException e) { + throw new ExchangisJobException.Runtime(e.getErrCode(), e.getMessage(), e.getCause()); + } catch (UnsupportedEncodingException e) { + throw new ExchangisJobException.Runtime(-1, e.getMessage(), e); + } }); + private static final JobParamDefine IS_TEXT_FILE_TYPE = JobParams.define("sqoop.file.is.text", (BiFunction)(k, job) -> { + Map tableProps = META_HADOOP_TABLE_PROPS.getValue(job); + return HADOOP_TEXT_INPUT_FORMAT.contains(tableProps.getOrDefault(META_INPUT_FORMAT, "")) || + HADOOP_TEXT_OUTPUT_FORMAT.contains(tableProps.getOrDefault(META_OUTPUT_FORMAT, "")); + }); + /** + * + * Whether hcatalog + */ + private static final JobParamDefine IS_USE_HCATALOG = JobParams.define("sqoop.use.hcatalog", (BiFunction)(k, job) -> MODE_ENUM.getValue(job) == EXPORT || !IS_TEXT_FILE_TYPE.getValue(job)); + /** * Driver default 'com.mysql.jdbc.Driver' */ @@ -189,42 +294,14 @@ public enum MODE_TYPE { IMPORT, EXPORT} private static final JobParamDefine HIVE_OVERWRITE = JobParams.define("sqoop.args.hive.overwrite", (BiFunction) (k, job) -> { if (Objects.nonNull(HIVE_IMPORT.getValue(job))){ JobParam writeMode = MODE_HADOOP_PARAMS.getValue(job).get(JobParamConstraints.WRITE_MODE); - if (Objects.nonNull(writeMode) && "overwrite".equals(writeMode.getValue())){ + if (Objects.nonNull(writeMode) && "overwrite".equalsIgnoreCase(writeMode.getValue())){ return ""; } } return null; }); - /** - * Hive-partition-map - */ - @SuppressWarnings("unchecked") - private static final JobParamDefine> PARTITION_MAP = JobParams.define("sqoop.partition.map", (BiFunction>) (k, job) -> { - if ("hive".equalsIgnoreCase(job.getSinkType()) || "hive".equalsIgnoreCase(job.getSourceType())){ - JobParam partitionParam = MODE_HADOOP_PARAMS.getValue(job).get(JobParamConstraints.PARTITION); - if (Objects.nonNull(partitionParam)) { - Object partition = partitionParam.getValue(); - if (partition instanceof Map) { - return (Map) partition; - } else { - String partitionStr = String.valueOf(partition); - if (StringUtils.isNotBlank(partitionStr)){ - Map partitionMap = new HashMap<>(); - String[] partValues = partitionStr.split(","); - for (String partValue : partValues){ - String[] parts = partValue.split("="); - if (parts.length == 2){ - partitionMap.put(parts[0], parts[1]); - } - } - return partitionMap; - } - } - } - } - return null; - }); + /** * Import: Hive-database @@ -271,16 +348,16 @@ public enum MODE_TYPE { IMPORT, EXPORT} * Import: Hive-append */ private static final JobParamDefine HIVE_APPEND = JobParams.define("sqoop.args.append", (BiFunction) (k, job) -> { - if (Objects.nonNull(HIVE_IMPORT.getValue(job))){ - JobParam writeMode = MODE_HADOOP_PARAMS.getValue(job).get(JobParamConstraints.WRITE_MODE); - if (Objects.nonNull(writeMode) && "append".equalsIgnoreCase(writeMode.getValue())){ - return ""; - } - } +// if (Objects.nonNull(HIVE_IMPORT.getValue(job))){ +// JobParam writeMode = MODE_HADOOP_PARAMS.getValue(job).get(JobParamConstraints.WRITE_MODE); +// if (Objects.nonNull(writeMode) && "append".equalsIgnoreCase(writeMode.getValue())){ +// return ""; +// } +// } return null; }); /** - * Import: Hive-target-dir + * Import: Hive-target-dir\] */ private static final JobParamDefine HIVE_TARGET_DIR = JobParams.define("sqoop.args.target.dir", (BiFunction) (k, job) -> { if (Objects.nonNull(HIVE_IMPORT.getValue(job)) && Objects.nonNull(QUERY_STRING.getValue(job))){ @@ -293,19 +370,18 @@ public enum MODE_TYPE { IMPORT, EXPORT} * Import: Hive-delete-target-dir */ private static final JobParamDefine HIVE_DELETE_TARGET = JobParams.define("sqoop.args.delete.target.dir", (BiFunction) (k, job) -> { - if (Objects.nonNull(HIVE_IMPORT.getValue(job)) && Objects.isNull(HIVE_APPEND.getValue(job))){ + if (Objects.nonNull(HIVE_IMPORT.getValue(job))){ return ""; } return null; }); /** - * TODO get the properties from hive * Import: Hive-fields-terminated-by */ private static final JobParamDefine HIVE_FIELDS_TERMINATED_BY = JobParams.define("sqoop.args.fields.terminated.by", (BiFunction) (k, job) -> { if (MODE_ENUM.getValue(job) == IMPORT && "hive".equalsIgnoreCase(job.getSinkType())){ - return "\u0001"; + return META_HADOOP_TABLE_PROPS.getValue(job).getOrDefault(META_FIELD_DELIMITER, "\u0001"); } return null; }); @@ -344,14 +420,14 @@ public enum MODE_TYPE { IMPORT, EXPORT} }); /** - * TODO get the primary keys from RDBMS * Export: Update-key */ private static final JobParamDefine UPDATE_KEY = JobParams.define("sqoop.args.update.key", (BiFunction) (k, job) -> { if (MODE_ENUM.getValue(job) == EXPORT ){ - JobParam writeMode = MODE_HADOOP_PARAMS.getValue(job).get(JobParamConstraints.WRITE_MODE, String.class); + JobParam writeMode = MODE_RDBMS_PARAMS.getValue(job).get(JobParamConstraints.WRITE_MODE, String.class); if (Objects.nonNull(writeMode) && StringUtils.isNotBlank(writeMode.getValue()) && !"insert".equalsIgnoreCase(writeMode.getValue())){ - + return META_RDBMS_COLUMNS.getValue(job).stream().filter(MetaColumn::isPrimaryKey) + .map(MetaColumn::getName).collect(Collectors.joining(",")); } } return null; @@ -361,8 +437,8 @@ public enum MODE_TYPE { IMPORT, EXPORT} * Export: Update mode */ private static final JobParamDefine UPDATE_MODE = JobParams.define("sqoop.args.update.mode", (BiFunction) (k, job) -> { - if (Objects.nonNull(UPDATE_KEY.getValue(job))){ - JobParam writeMode = MODE_HADOOP_PARAMS.getValue(job).get(JobParamConstraints.WRITE_MODE, String.class); + if (StringUtils.isNotBlank(UPDATE_KEY.getValue(job))){ + JobParam writeMode = MODE_RDBMS_PARAMS.getValue(job).get(JobParamConstraints.WRITE_MODE, String.class); return "update".equals(writeMode.getValue())? "allowinsert" : "updateonly"; } return null; @@ -415,7 +491,7 @@ public enum MODE_TYPE { IMPORT, EXPORT} */ private static final JobParamDefine HIVE_INPUT_FIELDS_TERMINATED_KEY = JobParams.define("sqoop.args.input.fields.terminated.by", (BiFunction) (k, job) -> { if (MODE_ENUM.getValue(job) == EXPORT && "hive".equalsIgnoreCase(job.getSourceType())){ - return "\u0001"; + return META_HADOOP_TABLE_PROPS.getValue(job).getOrDefault(META_FIELD_DELIMITER, "\u0001"); } return null; }); @@ -472,6 +548,25 @@ public enum MODE_TYPE { IMPORT, EXPORT} return StringUtils.join(columnSerial, ","); }); + /** + * Inspection of the definitions above + */ + private static final JobParamDefine DEFINE_INSPECTION = JobParams.define("", (BiFunction) (key, job) -> { + List rdbmsColumns = new ArrayList<>(Arrays.asList(COLUMN_SERIAL.getValue(job).split(","))); + List hadoopColumns = META_HADOOP_COLUMNS.getValue(job).stream().map(MetaColumn::getName) + .collect(Collectors.toList()); + if (IS_USE_HCATALOG.getValue(job)){ + rdbmsColumns.removeAll(hadoopColumns); + if (!rdbmsColumns.isEmpty()){ + warn("NOTE: task:[name:{}, id:{}] 在使用Hcatalog方式下，关系型数据库字段 [" + StringUtils.join(rdbmsColumns, ",") + "] 在hive/hbase表中未查询到对应字段", + job.getName(), job.getId()); + } + }else { + warn("NOTE: task:[name: {}, id:{}] 将使用非Hcatalog方式(原生)导数, 将顺序匹配关系型数据库字段和hive/hbase字段，否则请改变写入方式为APPEND追加", + job.getName(), job.getId()); + } + return null; + }); @Override public int priority() { return 1; @@ -480,7 +575,7 @@ public int priority() { @Override public ExchangisEngineJob buildJob(SubExchangisJob inputJob, ExchangisEngineJob expectOut, ExchangisJobBuilderContext ctx) throws ExchangisJobException { try { - SqoopExchangisEngineJob engineJob = new SqoopExchangisEngineJob(); + SqoopExchangisEngineJob engineJob = new SqoopExchangisEngineJob(expectOut); engineJob.setId(inputJob.getId()); JobParamDefine[] definitions = getParamDefinitions(); Map jobContent = engineJob.getJobContent(); @@ -490,13 +585,7 @@ public ExchangisEngineJob buildJob(SubExchangisJob inputJob, ExchangisEngineJob jobContent.put(definition.getKey(), String.valueOf(paramValue)); } } - engineJob.setRuntimeParams(inputJob.getParamsToMap(SubExchangisJob.REALM_JOB_SETTINGS, false)); engineJob.setName(inputJob.getName()); - if (Objects.nonNull(expectOut)) { - engineJob.setName(expectOut.getName()); - engineJob.setEngineType(expectOut.getEngineType()); - jobContent.putAll(expectOut.getJobContent()); - } engineJob.setCreateUser(inputJob.getCreateUser()); return engineJob; } catch (Exception e) { @@ -523,7 +612,9 @@ private JobParamDefine[] getParamDefinitions(){ EXPORT_DIR, UPDATE_KEY, UPDATE_MODE, HCATALOG_DATABASE, HCATALOG_TABLE, HCATALOG_PARTITION_KEY, HCATALOG_PARTITION_VALUE, HIVE_INPUT_FIELDS_TERMINATED_KEY, HIVE_INPUT_NULL_STRING, HIVE_INPUT_NULL_NON_STRING, - COLUMN_SERIAL + COLUMN_SERIAL,DEFINE_INSPECTION }; } + + } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/GenericExchangisTransformJobBuilder.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/GenericExchangisTransformJobBuilder.java index 4c7aadb51..b9f66b5c5 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/GenericExchangisTransformJobBuilder.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/GenericExchangisTransformJobBuilder.java @@ -177,4 +177,11 @@ public void handleSink(SubExchangisJob subExchangisJob, ExchangisJobBuilderConte } } } + + public static void main(String[] args) { + String code = "[{\"subJobs\":[{\"subJobName\":\"Copy ID\",\"dataSourceIds\":{\"source\":{\"type\":\"MYSQL\",\"id\":\"111\",\"ds\":\"MYSQL_LIU\",\"db\":\"ide_gz_bdap_sit_01\",\"table\":\"dss_project_publish_history\"},\"sink\":{\"type\":\"HIVE\",\"id\":\"113\",\"ds\":\"hive_uat\",\"db\":\"hduser05db\",\"table\":\"dss_project_publish_history\"}},\"params\":{\"sources\":[{\"key\":\"where\",\"field\":\"where\",\"label\":\"WHERE条件\",\"sort\":2,\"value\":\"\",\"defaultValue\":\"\",\"unit\":\"\",\"required\":false,\"validateType\":\"REGEX\",\"validateRange\":\"^[\\\\s\\\\S]{0,500}$\",\"validateMsg\":\"WHERE条件输入过长\",\"source\":\"\",\"type\":\"INPUT\"}],\"sinks\":[{\"key\":\"writeMode\",\"field\":\"writeMode\",\"label\":\"写入方式(OVERWRITE只对TEXT类型表生效)\",\"values\":[\"OVERWRITE\",\"APPEND\"],\"value\":\"OVERWRITE\",\"defaultValue\":\"OVERWRITE\",\"sort\":1,\"unit\":\"\",\"required\":true,\"type\":\"OPTION\"},{\"key\":\"partition\",\"field\":\"partition\",\"label\":\"分区信息(文本)\",\"sort\":2,\"value\":null,\"defaultValue\":null,\"unit\":\"\",\"required\":false,\"validateType\":\"REGEX\",\"validateRange\":\"^[\\\\s\\\\S]{0,50}$\",\"validateMsg\":\"分区信息过长\",\"source\":\"/api/rest_j/v1/exchangis/datasources/render/partition/element/map\",\"type\":\"MAP\"}]},\"transforms\":{\"addEnable\":false,\"type\":\"MAPPING\",\"sql\":null,\"mapping\":[{\"validator\":null,\"transformer\":null,\"source_field_name\":\"id\",\"source_field_type\":\"BIGINT\",\"sink_field_name\":\"id\",\"sink_field_type\":\"int\",\"deleteEnable\":false,\"source_field_index\":0,\"sink_field_index\":0,\"source_field_editable\":true,\"sink_field_editable\":false},{\"validator\":null,\"transformer\":null,\"source_field_name\":\"project_version_id\",\"source_field_type\":\"BIGINT\",\"sink_field_name\":\"project_version_id\",\"sink_field_type\":\"int\",\"deleteEnable\":false,\"source_field_index\":1,\"sink_field_index\":0,\"source_field_editable\":true,\"sink_field_editable\":false},{\"validator\":null,\"transformer\":null,\"source_field_name\":\"creator_id\",\"source_field_type\":\"BIGINT\",\"sink_field_name\":\"creator_id\",\"sink_field_type\":\"int\",\"deleteEnable\":false,\"source_field_index\":2,\"sink_field_index\":0,\"source_field_editable\":true,\"sink_field_editable\":false},{\"validator\":null,\"transformer\":null,\"source_field_name\":\"create_time\",\"source_field_type\":\"DATETIME\",\"sink_field_name\":\"create_time\",\"sink_field_type\":\"date\",\"deleteEnable\":false,\"source_field_index\":3,\"sink_field_index\":0,\"source_field_editable\":true,\"sink_field_editable\":false},{\"validator\":null,\"transformer\":null,\"source_field_name\":\"state\",\"source_field_type\":\"TINYINT\",\"sink_field_name\":\"state\",\"sink_field_type\":\"boolean\",\"deleteEnable\":false,\"source_field_index\":4,\"sink_field_index\":0,\"source_field_editable\":true,\"sink_field_editable\":false}]},\"settings\":[{\"key\":\"setting.max.parallelism\",\"field\":\"setting.max.parallelism\",\"label\":\"作业最大并行数\",\"sort\":1,\"value\":\"1\",\"defaultValue\":\"1\",\"unit\":\"个\",\"required\":true,\"validateType\":\"REGEX\",\"validateRange\":\"^[1-9]\\\\d*$\",\"validateMsg\":\"作业最大并行数输入错误\",\"source\":\"\",\"type\":\"INPUT\"},{\"key\":\"setting.max.memory\",\"field\":\"setting.max.memory\",\"label\":\"作业最大内存\",\"sort\":2,\"value\":\"1048\",\"defaultValue\":\"1024\",\"unit\":\"Mb\",\"required\":true,\"validateType\":\"REGEX\",\"validateRange\":\"^[1-9]\\\\d*$\",\"validateMsg\":\"作业最大内存输入错误\",\"source\":\"\",\"type\":\"INPUT\"}]},{\"subJobName\":\"tTFeeaPBfbZJ\",\"dataSourceIds\":{\"source\":{\"type\":\"MYSQL\",\"id\":\"111\",\"ds\":\"MYSQL_LIU\",\"db\":\"ide_gz_bdap_sit_01\",\"table\":\"dss_project_publish_history\"},\"sink\":{\"type\":\"HIVE\",\"id\":\"113\",\"ds\":\"hive_uat\",\"db\":\"hduser05db\",\"table\":\"dss_project_publish_history_text\"}},\"params\":{\"sources\":[{\"key\":\"where\",\"field\":\"where\",\"label\":\"WHERE条件\",\"sort\":2,\"value\":\"\",\"defaultValue\":\"\",\"unit\":\"\",\"required\":false,\"validateType\":\"REGEX\",\"validateRange\":\"^[\\\\s\\\\S]{0,500}$\",\"validateMsg\":\"WHERE条件输入过长\",\"source\":\"\",\"type\":\"INPUT\"}],\"sinks\":[{\"key\":\"writeMode\",\"field\":\"writeMode\",\"label\":\"写入方式(OVERWRITE只对TEXT类型表生效)\",\"values\":[\"OVERWRITE\",\"APPEND\"],\"value\":\"OVERWRITE\",\"defaultValue\":\"OVERWRITE\",\"sort\":1,\"unit\":\"\",\"required\":true,\"type\":\"OPTION\"},{\"key\":\"partition\",\"field\":\"partition\",\"label\":\"分区信息(文本)\",\"sort\":2,\"value\":null,\"defaultValue\":null,\"unit\":\"\",\"required\":false,\"validateType\":\"REGEX\",\"validateRange\":\"^[\\\\s\\\\S]{0,50}$\",\"validateMsg\":\"分区信息过长\",\"source\":\"/api/rest_j/v1/exchangis/datasources/render/partition/element/map\",\"type\":\"MAP\"}]},\"transforms\":{\"addEnable\":false,\"type\":\"MAPPING\",\"sql\":null,\"mapping\":[{\"validator\":null,\"transformer\":null,\"source_field_name\":\"id\",\"source_field_type\":\"BIGINT\",\"sink_field_name\":\"id\",\"sink_field_type\":\"int\",\"deleteEnable\":true,\"source_field_index\":0,\"sink_field_index\":0,\"source_field_editable\":true,\"sink_field_editable\":false},{\"validator\":null,\"transformer\":null,\"source_field_name\":\"project_version_id\",\"source_field_type\":\"BIGINT\",\"sink_field_name\":\"project_version_id\",\"sink_field_type\":\"int\",\"deleteEnable\":true,\"source_field_index\":1,\"sink_field_index\":0,\"source_field_editable\":true,\"sink_field_editable\":false},{\"validator\":null,\"transformer\":null,\"source_field_name\":\"create_time\",\"source_field_type\":\"DATETIME\",\"sink_field_name\":\"creator_id\",\"sink_field_type\":\"int\",\"deleteEnable\":true,\"source_field_index\":2,\"sink_field_index\":0,\"source_field_editable\":true,\"sink_field_editable\":false},{\"validator\":null,\"transformer\":null,\"source_field_name\":\"creator_id\",\"source_field_type\":\"BIGINT\",\"sink_field_name\":\"create_time\",\"sink_field_type\":\"string\",\"deleteEnable\":true,\"source_field_index\":3,\"sink_field_index\":0,\"source_field_editable\":true,\"sink_field_editable\":false},{\"validator\":null,\"transformer\":null,\"source_field_name\":\"update_time\",\"source_field_type\":\"DATETIME\",\"sink_field_name\":\"state\",\"sink_field_type\":\"string\",\"deleteEnable\":true,\"source_field_index\":4,\"sink_field_index\":0,\"source_field_editable\":true,\"sink_field_editable\":false}]},\"settings\":[{\"key\":\"setting.max.parallelism\",\"field\":\"setting.max.parallelism\",\"label\":\"作业最大并行数\",\"sort\":1,\"value\":\"1\",\"defaultValue\":\"1\",\"unit\":\"个\",\"required\":true,\"validateType\":\"REGEX\",\"validateRange\":\"^[1-9]\\\\d*$\",\"validateMsg\":\"作业最大并行数输入错误\",\"source\":\"\",\"type\":\"INPUT\"},{\"key\":\"setting.max.memory\",\"field\":\"setting.max.memory\",\"label\":\"作业最大内存\",\"sort\":2,\"value\":\"1024\",\"defaultValue\":\"1024\",\"unit\":\"Mb\",\"required\":true,\"validateType\":\"REGEX\",\"validateRange\":\"^[1-9]\\\\d*$\",\"validateMsg\":\"作业最大内存输入错误\",\"source\":\"\",\"type\":\"INPUT\"}]}]}]"; + List contents = Json.fromJson(code, List.class, ExchangisJobInfoContent.class); + System.out.println(contents.get(0).getSubJobName()); + System.out.println(contents); + } } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/TransformExchangisJob.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/TransformExchangisJob.java index fe9806975..7d1d24d6c 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/TransformExchangisJob.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/TransformExchangisJob.java @@ -9,7 +9,9 @@ import com.webank.wedatasphere.exchangis.job.domain.params.JobParamSet; import com.webank.wedatasphere.exchangis.job.domain.params.JobParams; import com.webank.wedatasphere.exchangis.job.server.builder.JobParamConstraints; +import com.webank.wedatasphere.exchangis.job.server.builder.engine.DataxExchangisEngineJob; import com.webank.wedatasphere.exchangis.job.server.builder.transform.handlers.GenericSubExchangisJobHandler; +import com.webank.wedatasphere.exchangis.job.server.utils.JobUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -80,8 +82,12 @@ private void convertContentToParams(ExchangisJobInfoContent content){ setIntoParams(REALM_JOB_COLUMN_MAPPING, () -> Json.convert(content.getTransforms(), Map.class, String.class, Object.class)); if(Objects.nonNull(content.getParams())){ if(Objects.nonNull(content.getParams().getSources())) { + List items = content.getParams().getSources(); + + timePlaceHolderConvert(items); + JobParamSet paramSet = setIntoParams(REALM_JOB_CONTENT_SOURCE, () -> { - List items = content.getParams().getSources(); + //List items = content.getParams().getSources(); return items.stream().filter(item -> StringUtils.isNotBlank(item.getConfigKey()) && Objects.nonNull(item.getConfigValue())).collect (Collectors.toMap(ExchangisJobParamsContent.ExchangisJobParamsItem::getConfigKey, ExchangisJobParamsContent.ExchangisJobParamsItem::getConfigValue)); @@ -90,9 +96,13 @@ private void convertContentToParams(ExchangisJobInfoContent content){ this.sourceType = resolveDataSourceId(content.getDataSources().getSourceId(), paramSet); } } + if(Objects.nonNull(content.getParams().getSinks())) { + List items = content.getParams().getSinks(); + timePlaceHolderConvert(items); + JobParamSet paramSet = setIntoParams(REALM_JOB_CONTENT_SINK, () -> { - List items = content.getParams().getSinks(); + //List items = content.getParams().getSinks(); return items.stream().filter(item -> StringUtils.isNotBlank(item.getConfigKey()) && Objects.nonNull(item.getConfigValue())).collect (Collectors.toMap(ExchangisJobParamsContent.ExchangisJobParamsItem::getConfigKey, ExchangisJobParamsContent.ExchangisJobParamsItem::getConfigValue)); @@ -145,5 +155,34 @@ private String resolveDataSourceId(String dataSourceId, JobParamSet paramSet){ return null; } + /** + * + * @param items + * @return 用于转换时间分区 + */ + + private void timePlaceHolderConvert(List items) { + for (ExchangisJobParamsContent.ExchangisJobParamsItem exchangisJobParamsItem : items) { + if (("partition".equals(exchangisJobParamsItem.getConfigKey()) ) && exchangisJobParamsItem.getConfigValue() != null) { + Map partitionValue = (Map) exchangisJobParamsItem.getConfigValue(); + assert partitionValue != null; + Calendar calendar = Calendar.getInstance(); + if (!partitionValue.get("ds").isEmpty()) { + partitionValue.put("ds", JobUtils.renderDt(partitionValue.get("ds"), calendar)); + } + LOG.info("Time placeholder transform value: {}", partitionValue.get("ds")); + exchangisJobParamsItem.setConfigValue(partitionValue); + } + else if ("where".equals(exchangisJobParamsItem.getConfigKey())) { + String partitionValue = exchangisJobParamsItem.getConfigValue().toString(); + assert partitionValue != null; + Calendar calendar = Calendar.getInstance(); + partitionValue = JobUtils.renderDt(partitionValue, calendar); + LOG.info("Time placeholder transform value: {}", partitionValue); + exchangisJobParamsItem.setConfigValue(partitionValue); + } + } + } + } } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/dao/ExchangisJobEntityDao.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/dao/ExchangisJobEntityDao.java deleted file mode 100644 index 4ea63847a..000000000 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/dao/ExchangisJobEntityDao.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.webank.wedatasphere.exchangis.job.server.dao; - -import com.webank.wedatasphere.exchangis.job.domain.ExchangisJobEntity; -import com.webank.wedatasphere.exchangis.job.launcher.domain.LaunchableExchangisJob; -import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Param; - -import java.util.List; - -/** - * - */ - -public interface ExchangisJobEntityDao { - - /** - * Add new JobEntity - * @param exchangisJobEntity - */ - void addJobEntity(ExchangisJobEntity exchangisJobEntity); - - /** - * Delete launchableJob - * @param jobId - */ - void deleteJobEntity(@Param("jobId") Long jobId); - - /** - * upgradeLaunchableJob - * @param exchangisJobEntity - */ - void upgradeJobEntity(ExchangisJobEntity exchangisJobEntity); - - /** - * Get launchableJob - * @param jobId - */ - ExchangisJobEntity getJobEntity(@Param("jobId") Long jobId); - - /** - * Get launchableJobByFactor - * @param projectId - * @param name - */ - List getJobEntityByFactor(@Param("projectId") long projectId, @Param("name") String name); - - /** - * Get launchableJobByName - * @param name - */ - List getJobEntityByName(@Param("name") String name); - -} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/dto/ExchangisJobBasicInfoDTO.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/dto/ExchangisJobBasicInfoDTO.java index 2ef0f749f..ece45b5e8 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/dto/ExchangisJobBasicInfoDTO.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/dto/ExchangisJobBasicInfoDTO.java @@ -1,6 +1,6 @@ package com.webank.wedatasphere.exchangis.job.server.dto; -import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.webank.wedatasphere.exchangis.job.enums.JobTypeEnum; diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/dto/ExportedProject.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/dto/ExportedProject.java new file mode 100644 index 000000000..71b2fda58 --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/dto/ExportedProject.java @@ -0,0 +1,39 @@ +package com.webank.wedatasphere.exchangis.job.server.dto; + +import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobVo; + +import java.util.List; + +/** + * @author tikazhang + * @Date 2022/3/14 12:22 + */ +public class ExportedProject { + String name; + List sqoops; + List dataxes; + + public List getSqoops() { + return sqoops; + } + + public void setSqoops(List sqoops) { + this.sqoops = sqoops; + } + + public List getDataxes() { + return dataxes; + } + + public void setDataxes(List dataxes) { + this.dataxes = dataxes; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/dto/IdCatalog.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/dto/IdCatalog.java new file mode 100644 index 000000000..8ba12b551 --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/dto/IdCatalog.java @@ -0,0 +1,32 @@ +package com.webank.wedatasphere.exchangis.job.server.dto; + +import com.google.common.collect.Maps; + +import java.util.Map; + +/** + * @author tikazhang + * @Date 2022/3/15 10:33 + */ +public class IdCatalog { + + private Map sqoop = Maps.newHashMap(); + + private Map datax = Maps.newHashMap(); + + public Map getSqoop() { + return sqoop; + } + + public void setSqoop(Map sqoop) { + this.sqoop = sqoop; + } + + public Map getDatax() { + return datax; + } + + public void setDatax(Map datax) { + this.datax = datax; + } +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/generator/DefaultTaskGenerator.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/generator/DefaultTaskGenerator.java index 7dcfc6e54..8248bae5a 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/generator/DefaultTaskGenerator.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/generator/DefaultTaskGenerator.java @@ -9,6 +9,7 @@ import com.webank.wedatasphere.exchangis.job.exception.ExchangisJobException; import com.webank.wedatasphere.exchangis.job.launcher.domain.LaunchableExchangisJob; import com.webank.wedatasphere.exchangis.job.launcher.domain.LaunchableExchangisTask; +import com.webank.wedatasphere.exchangis.job.server.builder.ServiceInExchangisJobBuilderContext; import com.webank.wedatasphere.exchangis.job.server.builder.transform.TransformExchangisJob; import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisTaskGenerateException; import com.webank.wedatasphere.exchangis.job.server.execution.generator.events.TaskGenerateErrorEvent; @@ -74,9 +75,10 @@ protected void execute(LaunchableExchangisJob launchableExchangisJob, throw throwable; } ExchangisJobBuilderManager jobBuilderManager = getExchangisJobBuilderManager(); - ExchangisJobBuilderContext ctx = new ExchangisJobBuilderContext(jobInfo, generatorContext.getJobLogListener()); + ServiceInExchangisJobBuilderContext ctx = new ServiceInExchangisJobBuilderContext(jobInfo, generatorContext.getJobLogListener()); // Set the metadata service ctx.setMetadataInfoService(generatorContext.getMetadataInfoService()); + ctx.setJobExecutionId(launchableExchangisJob.getJobExecutionId()); ctx.putEnv("USER_NAME", tenancy); // ExchangisJobInfo -> TransformExchangisJob(SubExchangisJob) try { diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/loadbalance/FlexibleTenancyLoadBalancer.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/loadbalance/FlexibleTenancyLoadBalancer.java index 78a02188c..6ed5f556d 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/loadbalance/FlexibleTenancyLoadBalancer.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/loadbalance/FlexibleTenancyLoadBalancer.java @@ -212,33 +212,36 @@ private void loop(Map tenancyExecutorServices, int resi int adjustSegmentNum = 0; int coreSize = pool.getCorePoolSize(); // Must more than residentThreads - if (coreSize > residentThreads){ +// if (TenancyParallelGroupFactory.DEFAULT_TENANCY.equals(tenancy) || coreSize > residentThreads){ int segments = loopCounter.segments.get(); - int restSize = coreSize - residentThreads; - int activeThreads = pool.getActiveCount(); - if (activeThreads >= coreSize){ - // All threads is active, should reduce the number of segments - adjustSegmentNum = Math.min((int) Math.floor((double)restSize * Constraints.SEGMENT_MIN_OCCUPY.getValue()), segments); - } else { - adjustSegmentNum = Math.max((int) Math.floor((double)restSize * Constraints.SEGMENT_MAX_OCCUPY.getValue()), segments); - } - adjustSegmentNum = adjustSegmentNum > segments ? segments + Math.min(adjustStep, adjustSegmentNum - segments) - : segments - Math.min(adjustStep, segments - adjustSegmentNum); - if (segments != adjustSegmentNum) { - // Div the number of container - int average = adjustSegmentNum / loopCounter.containers.get(); - LOG.info("Adjust total number of load balance scheduler task segments for tenancy: [{}] to [{}], average [{}]", - tenancy, adjustSegmentNum, average); - for (int i = 0; i < loopCounter.containers.get(); i++) { - if (i == loopCounter.containers.get() - 1) { - loopCounter.taskContainers.get(i).adjustSegment(adjustSegmentNum); - } else { - loopCounter.taskContainers.get(i).adjustSegment(average); - adjustSegmentNum = adjustSegmentNum - average; + // TODO fix the problem that the value of residentThreads always equal 1 for not default consumer + int restSize = TenancyParallelGroupFactory.DEFAULT_TENANCY.equals(tenancy)? coreSize - residentThreads - 1: coreSize - 1; + if (restSize > 0) { + int activeThreads = pool.getActiveCount(); + if (activeThreads >= coreSize) { + // All threads is active, should reduce the number of segments + adjustSegmentNum = Math.min((int) Math.floor((double) restSize * Constraints.SEGMENT_MIN_OCCUPY.getValue()), segments); + } else { + adjustSegmentNum = Math.max((int) Math.floor((double) restSize * Constraints.SEGMENT_MAX_OCCUPY.getValue()), segments); + } + adjustSegmentNum = adjustSegmentNum > segments ? segments + Math.min(adjustStep, adjustSegmentNum - segments) + : segments - Math.min(adjustStep, segments - adjustSegmentNum); + if (segments != adjustSegmentNum) { + // Div the number of container + int average = adjustSegmentNum / loopCounter.containers.get(); + LOG.info("Adjust total number of load balance scheduler task segments for tenancy: [{}] from {} to {}, average {}", + tenancy, segments, adjustSegmentNum, average); + for (int i = 0; i < loopCounter.containers.get(); i++) { + if (i == loopCounter.containers.get() - 1) { + loopCounter.taskContainers.get(i).adjustSegment(adjustSegmentNum); + } else { + loopCounter.taskContainers.get(i).adjustSegment(average); + adjustSegmentNum = adjustSegmentNum - average; + } } } } - } +// } } }); LOG.trace("End to auto scale-in/out segments of load balance scheduler task"); @@ -329,8 +332,13 @@ private void adjustSegment(int adjustNum){ * @param scaleOut */ private void scaleOutSegment(int scaleOut){ - LOG.info("Scale-out segments for tenancy: [{}]，decr_segment_size: [{}], scheduler task: [{}]", tenancy, scaleOut, taskName); int newSize = segments.length - scaleOut; + LOG.info("Scale-out segments for tenancy: [{}]，scaleOut: [{}], newSize: [{}], scheduler task: [{}]", + tenancy, scaleOut, newSize, taskName); + if (newSize <= 0){ + LOG.warn("Scale-out fail, the newSize cannot <= 0"); + return; + } SchedulerTaskSegment[] newSegments = new SchedulerTaskSegment[newSize]; System.arraycopy(segments, 0, newSegments, 0, newSize); int offset = 0; @@ -355,7 +363,8 @@ private void scaleOutSegment(int scaleOut){ * @param scaleIn */ private void scaleInSegment(int scaleIn){ - LOG.info("Scale-in segments for tenancy: [{}]，incr_segment_size: [{}], scheduler task: [{}]", tenancy, scaleIn, taskName); + LOG.info("Scale-in segments for tenancy: [{}]，scaleIn: [{}], newSize: [{}], scheduler task: [{}]", + tenancy, scaleIn, segments.length + scaleIn, taskName); SchedulerTaskSegment[] newSegments = new SchedulerTaskSegment[segments.length + scaleIn]; System.arraycopy(segments, 0, newSegments, 0, segments.length); for(int i = segments.length; i < segments.length + scaleIn; i ++){ diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/TenancyParallelConsumerManager.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/TenancyParallelConsumerManager.java index d0bfbc78a..da52b214b 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/TenancyParallelConsumerManager.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/TenancyParallelConsumerManager.java @@ -128,8 +128,7 @@ protected ExecutorService getOrCreateExecutorService(String groupName){ if (StringUtils.isNotBlank(tenancy)){ return tenancyExecutorServices.computeIfAbsent(tenancy, tenancyName -> { // Use the default value of max running jobs - return Utils.newCachedThreadPool(parallelGroupFactory.getDefaultMaxRunningJobs() + parallelGroupFactory.getParallelPerTenancy() - + 1, + return Utils.newCachedThreadPool(parallelGroupFactory.getDefaultMaxRunningJobs() + parallelGroupFactory.getParallelPerTenancy(), TenancyParallelGroupFactory.GROUP_NAME_PREFIX + tenancy + "-Executor-", true); }); } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/subscriber/AbstractTaskObserver.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/subscriber/AbstractTaskObserver.java index b36110806..84fb7586b 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/subscriber/AbstractTaskObserver.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/subscriber/AbstractTaskObserver.java @@ -147,7 +147,7 @@ public synchronized void stop() { private void sleepOrWaitIfNeed(List publishedTasks){ long observerWait = this.lastPublishTime + publishInterval - System.currentTimeMillis(); if (publishedTasks.isEmpty() || observerWait > 0) { - observerWait = observerWait > 0? observerWait : publishBatch; + observerWait = observerWait > 0? observerWait : publishInterval; boolean hasLock = observerLock.tryLock(); if (hasLock) { try { diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/subscriber/MaxUsageTaskChooseRuler.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/subscriber/MaxUsageTaskChooseRuler.java index 8169b4344..4ccbeb3d5 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/subscriber/MaxUsageTaskChooseRuler.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/subscriber/MaxUsageTaskChooseRuler.java @@ -8,11 +8,14 @@ import org.apache.linkis.scheduler.queue.ConsumerManager; import org.springframework.stereotype.Component; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; /** @@ -26,11 +29,14 @@ public List choose(List candid if (consumerManager instanceof TenancyParallelConsumerManager){ TenancyParallelConsumerManager tenancyConsumerManager = (TenancyParallelConsumerManager)consumerManager; Map tenancyExecutorServices = tenancyConsumerManager.getTenancyExecutorServices(); - int batchSize = candidates.size(); + Map candidateCounter = new HashMap<>(); return candidates.stream().filter( task -> { - ExecutorService executorService = tenancyExecutorServices.get(StringUtils.isNotBlank(task.getExecuteUser())? - task.getExecuteUser(): TenancyParallelGroupFactory.DEFAULT_TENANCY); - return Objects.isNull(executorService) || ((ThreadPoolExecutor)executorService).getQueue().remainingCapacity() > batchSize; + String tenancy = StringUtils.isNotBlank(task.getExecuteUser())? + task.getExecuteUser(): TenancyParallelGroupFactory.DEFAULT_TENANCY; + ExecutorService executorService = tenancyExecutorServices.get(tenancy); + AtomicInteger counter = candidateCounter.computeIfAbsent(tenancy, (key) -> new AtomicInteger(0)); + // TODO complete the choose rule + return Objects.isNull(executorService) || ((ThreadPoolExecutor)executorService).getQueue().remainingCapacity() >= counter.incrementAndGet(); }).collect(Collectors.toList()); } return candidates; diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/JobServerLogging.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/JobServerLogging.java index 5002891fd..2b4e11dba 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/JobServerLogging.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/JobServerLogging.java @@ -5,6 +5,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Objects; import java.util.Optional; public interface JobServerLogging { @@ -14,7 +15,7 @@ default Logger getLogger(){ default void trace(T entity, String message, Object... args) { Logger logger = getLogger(); - if (logger.isTraceEnabled()){ + if (Objects.nonNull(logger) && logger.isTraceEnabled()){ logger.trace(message, args); } Optional.ofNullable(getJobLogListener()).ifPresent(listener -> @@ -23,14 +24,14 @@ default void trace(T entity, String message, Object... args) { default void debug(T entity, String message){ Logger logger = getLogger(); - if (logger.isDebugEnabled()){ + if (Objects.nonNull(logger) && logger.isDebugEnabled()){ logger.debug(message); } } default void info(T entity, String message, Object... args){ Logger logger = getLogger(); - if (logger.isInfoEnabled()){ + if (Objects.nonNull(logger) && logger.isInfoEnabled()){ logger.info(message, args); } Optional.ofNullable(getJobLogListener()).ifPresent(listener -> @@ -39,25 +40,25 @@ default void info(T entity, String message, Object... args){ default void info(T entity, String message, Throwable t){ Logger logger = getLogger(); - if (logger.isInfoEnabled()){ + if (Objects.nonNull(logger) && logger.isInfoEnabled()){ logger.info(message, t); } Optional.ofNullable(getJobLogListener()).ifPresent(listener -> listener.onAsyncEvent(getJobLogEvent(JobLogEvent.Level.INFO, entity, message, t))); } - default void warn(T entity, String message){ + default void warn(T entity, String message, Object... args){ Logger logger = getLogger(); - if (logger.isWarnEnabled()){ - logger.warn(message); + if (Objects.nonNull(logger) && logger.isWarnEnabled()){ + logger.warn(message, args); } Optional.ofNullable(getJobLogListener()).ifPresent(listener -> - listener.onAsyncEvent(getJobLogEvent(JobLogEvent.Level.WARN, entity, message))); + listener.onAsyncEvent(getJobLogEvent(JobLogEvent.Level.WARN, entity, message, args))); } default void warn(T entity, String message, Throwable t){ Logger logger = getLogger(); - if (logger.isWarnEnabled()){ + if (Objects.nonNull(logger) && logger.isWarnEnabled()){ logger.warn(message, t); } Optional.ofNullable(getJobLogListener()).ifPresent(listener -> @@ -65,7 +66,7 @@ default void warn(T entity, String message, Throwable t){ } default void error(T entity, String message, Object... args){ - getLogger().error(message, args); + Optional.ofNullable(getLogger()).ifPresent(logger -> logger.error(message, args)); Optional.ofNullable(getJobLogListener()).ifPresent(listener -> listener.onAsyncEvent(getJobLogEvent(JobLogEvent.Level.ERROR, entity, message, args))); } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/LocalSimpleJobLogService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/LocalSimpleJobLogService.java index 6058344ce..5b4a991dc 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/LocalSimpleJobLogService.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/LocalSimpleJobLogService.java @@ -4,7 +4,7 @@ import com.webank.wedatasphere.exchangis.job.launcher.entity.LaunchedExchangisJobEntity; import com.webank.wedatasphere.exchangis.job.log.LogQuery; import com.webank.wedatasphere.exchangis.job.log.LogResult; -import com.webank.wedatasphere.exchangis.job.server.dao.LaunchedJobDao; +import com.webank.wedatasphere.exchangis.job.server.mapper.LaunchedJobDao; import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; import com.webank.wedatasphere.exchangis.job.server.execution.scheduler.AbstractExchangisSchedulerTask; import com.webank.wedatasphere.exchangis.job.server.log.JobLogService; @@ -17,7 +17,6 @@ import org.apache.linkis.common.utils.Utils; import org.apache.linkis.scheduler.Scheduler; import org.apache.linkis.scheduler.queue.JobInfo; -import org.jvnet.hk2.annotations.Service; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,7 +50,7 @@ public class LocalSimpleJobLogService implements JobLogService { private volatile boolean cleanerOn; private static class Constraints{ - public static final CommonVars LOG_LOCAL_PATH = CommonVars.apply("wds.exchangis.job.log.local.path", "/data/bdp/exchangis/logs"); + public static final CommonVars LOG_LOCAL_PATH = CommonVars.apply("wds.exchangis.job.log.local.path", "/data/bdp/dss/exchangis/main/logs"); public static final CommonVars lOG_CACHE_SIZE = CommonVars.apply("wds.exchangis.job.log.cache.size", 15); diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/ExchangisJobEntityDao.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/ExchangisJobEntityDao.java new file mode 100644 index 000000000..2368096cc --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/ExchangisJobEntityDao.java @@ -0,0 +1,73 @@ +package com.webank.wedatasphere.exchangis.job.server.mapper; + +import com.webank.wedatasphere.exchangis.job.domain.ExchangisJobEntity; +import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobQueryVo; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +/** + * + */ + +public interface ExchangisJobEntityDao { + + /** + * Add new JobEntity + * @param exchangisJobEntity job entity + */ + void addJobEntity(ExchangisJobEntity exchangisJobEntity); + + /** + * Delete jobEntity + * @param jobId job id + */ + void deleteJobEntity(@Param("jobId") Long jobId); + + /** + * upgrade jobEntity basic info + * @param jobEntity job entity + */ + void upgradeBasicInfo(ExchangisJobEntity jobEntity); + + /** + * Upgrade config + * @param jobEntity job entity + */ + void upgradeConfig(ExchangisJobEntity jobEntity); + + /** + * Upgrade content + * @param jobEntity job entity + */ + void upgradeContent(ExchangisJobEntity jobEntity); + /** + * Get jobEntity + * @param jobId job id + */ + ExchangisJobEntity getDetail(@Param("jobId") Long jobId); + + /** + * Get jobEntity list + * @param projectId + * @return + */ + List getDetailList(@Param("projectId") Long projectId); + + ExchangisJobEntity getBasicInfo(@Param("jobId") Long jobId); + + /** + * Query page list + * @param queryVo query vo + * @return list + */ + List queryPageList(ExchangisJobQueryVo queryVo); + + /** + * Delete batch + * @param ids id list + */ + void deleteBatch(@Param("ids") List ids); + + List getByNameWithProjectId(@Param("jobName") String jobName, @Param("projectId") Long projectId); +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/ExchangisJobMapper.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/ExchangisJobMapper.java deleted file mode 100644 index ad2b4b60a..000000000 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/ExchangisJobMapper.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.webank.wedatasphere.exchangis.job.server.mapper; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobVO; - -/** - *

- * Mapper 接口 - *

- * - * @author yuxin.yuan - * @since 2021-08-10 - */ -public interface ExchangisJobMapper extends BaseMapper { - -} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/dao/LaunchableTaskDao.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/LaunchableTaskDao.java similarity index 94% rename from exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/dao/LaunchableTaskDao.java rename to exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/LaunchableTaskDao.java index 2ba903f68..4002a6e41 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/dao/LaunchableTaskDao.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/LaunchableTaskDao.java @@ -1,4 +1,4 @@ -package com.webank.wedatasphere.exchangis.job.server.dao; +package com.webank.wedatasphere.exchangis.job.server.mapper; import com.webank.wedatasphere.exchangis.job.launcher.domain.LaunchableExchangisTask; import org.apache.ibatis.annotations.Param; diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/dao/LaunchedJobDao.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/LaunchedJobDao.java similarity index 98% rename from exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/dao/LaunchedJobDao.java rename to exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/LaunchedJobDao.java index 14c2eb802..53baaef33 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/dao/LaunchedJobDao.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/LaunchedJobDao.java @@ -1,4 +1,4 @@ -package com.webank.wedatasphere.exchangis.job.server.dao; +package com.webank.wedatasphere.exchangis.job.server.mapper; import com.webank.wedatasphere.exchangis.job.launcher.entity.LaunchedExchangisJobEntity; import org.apache.ibatis.annotations.Param; diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/dao/LaunchedTaskDao.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/LaunchedTaskDao.java similarity index 98% rename from exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/dao/LaunchedTaskDao.java rename to exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/LaunchedTaskDao.java index ef7be71dd..b78acee21 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/dao/LaunchedTaskDao.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/LaunchedTaskDao.java @@ -1,4 +1,4 @@ -package com.webank.wedatasphere.exchangis.job.server.dao; +package com.webank.wedatasphere.exchangis.job.server.mapper; import com.webank.wedatasphere.exchangis.job.launcher.domain.LaunchableExchangisTask; import com.webank.wedatasphere.exchangis.job.launcher.entity.LaunchedExchangisJobEntity; diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/ExchangisJobEntityDao.xml b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/ExchangisJobEntityDao.xml deleted file mode 100644 index 47b45b6c1..000000000 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/ExchangisJobEntityDao.xml +++ /dev/null @@ -1,107 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - exchangis_job_entity - - - - - id, - name, create_time, engine_type, job_labels, create_user, job_content, execute_user, job_params, project_id, source, modify_user - - - - insert into - - (id, name, create_time, last_update_time, engine_type, job_labels, create_user, job_content, execute_user, job_params, project_id, source, modify_user) - values( - #{id}, - #{name}, - #{createTime}, - #{lastUpdateTime}, - #{engineType}, - #{jobLabel}, - #{createUser}, - #{jobContent}, - #{executeUser}, - #{jobParams}, - #{projectId}, - #{source}, - #{modifyUser} - ); - - - - delete * from - - where id = #{jobId} - - - - UPDATE - SET - id = #{id}, - name = #{name}, - create_time = #{createTime}, - last_update_time = #{lastUpdateTime}, - engine_type = #{engineType}, - job_labels = #{jobLabel}, - create_user = #{createUser}, - job_content = #{jobContent}, - execute_user = #{executeUser}, - job_params = #{jobParams}, - project_id = #{projectId}, - source = #{source}, - modify_user = #{modifyUser} - - - - - - - - - diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/ExchangisJobEntityMapper.xml b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/ExchangisJobEntityMapper.xml new file mode 100644 index 000000000..db6f4afbd --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/ExchangisJobEntityMapper.xml @@ -0,0 +1,191 @@ + + + + + + + + + + + + + + + + + + + + + + + + + exchangis_job_entity + + + + id, + name, create_time, last_update_time, engine_type, job_labels, create_user, job_content, execute_user, + job_params, job_desc, job_type, project_id, source, modify_user + + + + id, + name, create_time, last_update_time, engine_type, job_labels, create_user, execute_user, + job_params, job_desc, job_type, project_id, source, modify_user + + + + insert into + + (name, create_time, last_update_time, engine_type, job_labels, create_user, job_content, execute_user, job_params, + job_desc, job_type, project_id, source, modify_user) + values( + #{name}, + #{createTime}, + #{lastUpdateTime}, + #{engineType}, + #{jobLabel}, + #{createUser}, + #{jobContent}, + #{executeUser}, + #{jobParams}, + #{jobDesc}, + #{jobType}, + #{projectId}, + #{source}, + #{modifyUser} + ); + + + + DELETE FROM WHERE + id = #{jobId} + + + + UPDATE + SET + name = #{name}, + last_update_time = #{lastUpdateTime}, + engine_type = #{engineType}, + job_labels = #{jobLabel}, + job_type = #{jobType}, + job_desc = #{jobDesc}, + modify_user = #{modifyUser} + WHERE id = #{id} + + + + UPDATE + SET + execute_user = #{executeUser}, + job_params = #{jobParams}, + source = #{source}, + last_update_time = #{lastUpdateTime}, + modify_user = #{modifyUser} + WHERE id = #{id} + + + + UPDATE + SET + job_content = #{jobContent}, + last_update_time = #{lastUpdateTime}, + modify_user = #{modifyUser} + WHERE id = #{id} + + + + + + + + + + + + + + + + + + + + DELETE FROM + + WHERE id in + + #{item} + + + diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/ExchangisJobMapper.xml b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/ExchangisJobMapper.xml deleted file mode 100644 index 9beb91308..000000000 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/ExchangisJobMapper.xml +++ /dev/null @@ -1,36 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - exchangis_job_info - - - - - id, - project_id, job_name, job_type, engine_type, job_labels, job_desc, content, alarm_user, alarm_level, proxy_user, execute_node, sync_type, job_params, create_time, create_user, modify_time, modify_user - - diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/LaunchableTaskMapper.xml b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/LaunchableTaskMapper.xml index 0fb8e4f0c..684bdab1c 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/LaunchableTaskMapper.xml +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/LaunchableTaskMapper.xml @@ -1,6 +1,6 @@ - + diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/LaunchedJobMapper.xml b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/LaunchedJobMapper.xml index 17db5dbd9..df0e29ef3 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/LaunchedJobMapper.xml +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/LaunchedJobMapper.xml @@ -1,6 +1,6 @@ - + diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/LaunchedTaskMapper.xml b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/LaunchedTaskMapper.xml index af039b978..fa50df759 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/LaunchedTaskMapper.xml +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/mapper/impl/LaunchedTaskMapper.xml @@ -1,6 +1,6 @@ - + diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisJobRestfulApi.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisJobRestfulApi.java index 9b2027bb9..1ae961eca 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisJobRestfulApi.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisJobRestfulApi.java @@ -1,170 +1,309 @@ package com.webank.wedatasphere.exchangis.job.server.restful; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; +import com.webank.wedatasphere.exchangis.common.pager.PageResult; +import com.webank.wedatasphere.exchangis.common.validator.groups.InsertGroup; import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisDataSourceException; -import com.webank.wedatasphere.exchangis.datasource.core.ui.ElementUI; -import com.webank.wedatasphere.exchangis.datasource.service.ExchangisDataSourceService; -import com.webank.wedatasphere.exchangis.job.builder.manager.ExchangisJobBuilderManager; -import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobVO; +import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobQueryVo; +import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobVo; import com.webank.wedatasphere.exchangis.job.enums.EngineTypeEnum; -import com.webank.wedatasphere.exchangis.job.server.dto.ExchangisJobBasicInfoDTO; -import com.webank.wedatasphere.exchangis.job.server.dto.ExchangisJobContentDTO; import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; -import com.webank.wedatasphere.exchangis.job.server.mapper.ExchangisLaunchTaskMapper; -import com.webank.wedatasphere.exchangis.job.server.service.ExchangisJobService; -import com.webank.wedatasphere.exchangis.job.server.service.ExchangisLaunchTaskService; -import com.webank.wedatasphere.exchangis.job.server.vo.ExchangisJobBasicInfoVO; -import com.webank.wedatasphere.exchangis.job.server.vo.ExchangisTaskSpeedLimitVO; +import com.webank.wedatasphere.exchangis.job.server.service.JobInfoService; import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.validation.BindingResult; +import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.*; +import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; +import javax.validation.groups.Default; import java.util.*; /** * The basic controller of Exchangis job - * */ @RestController -@RequestMapping(value = "exchangis/job", produces = {"application/json;charset=utf-8"}) +@RequestMapping(value = "dss/exchangis/main/job", produces = {"application/json;charset=utf-8"}) public class ExchangisJobRestfulApi { - private static final Logger LOG = LoggerFactory.getLogger(ExchangisJobRestfulApi.class); - @Autowired - private ExchangisJobService exchangisJobService; - - @Autowired - private ExchangisJobBuilderManager jobBuilderManager; - - @Autowired - private ExchangisDataSourceService exchangisDataSourceService; - @Autowired - private ExchangisLaunchTaskService exchangisLaunchTaskService; + private static final Logger LOG = LoggerFactory.getLogger(ExchangisJobRestfulApi.class); - @Autowired - private ExchangisLaunchTaskMapper launchTaskMapper; + /** + * Job service + */ + @Resource + private JobInfoService jobInfoService; - @RequestMapping( value = "", method = RequestMethod.GET) + /** + * Query job in page + * + * @param projectId project id + * @param jobType job type + * @param name name + * @param current current + * @param size size + * @param request request + * @return message + */ + @RequestMapping(value = "", method = RequestMethod.GET) public Message getJobList(@RequestParam(value = "projectId") Long projectId, @RequestParam(value = "jobType", required = false) String jobType, @RequestParam(value = "name", required = false) String name, @RequestParam(value = "current", required = false) int current, - @RequestParam(value = "size", required = false) int size) { - List joblist = exchangisJobService.getJobList(projectId, jobType, name, current, size); - int total = exchangisJobService.count(projectId, jobType, name); - //List joblist = exchangisJobService.getJobList(projectId, jobType, name); - Message message = Message.ok(); - message.data("total", total); - message.data("result", joblist); - return message; - //return Message.ok().data("result", joblist); - } - - @RequestMapping( value = "/dss", method = RequestMethod.GET) - public Message getJobListByDss(@RequestParam(value = "dssProjectId") Long dssProjectId, - @RequestParam(value = "jobType", required = false) String jobType, - @RequestParam(value = "name", required = false) String name) { - List joblist = exchangisJobService.getJobListByDssProject(dssProjectId, jobType, name); - return Message.ok().data("result", joblist); + @RequestParam(value = "size", required = false) int size, + HttpServletRequest request) { + ExchangisJobQueryVo queryVo = new ExchangisJobQueryVo( + projectId, jobType, name, current, size + ); + String userName = SecurityFilter.getLoginUsername(request); + queryVo.setCreateUser(userName); + try { + PageResult pageResult = jobInfoService.queryJobList(queryVo); + return Message.ok().data("total", pageResult.getTotal()).data("result", pageResult.getList()); + } catch (Exception e) { + LOG.error("Fail to query job list for user {}", userName, e); + return Message.error("Failed to query job list (获取任务列表失败)"); + } } - @RequestMapping( value = "/engineType", method = RequestMethod.GET) + /** + * Engine list + * + * @return message + */ + @RequestMapping(value = "/engineType", method = RequestMethod.GET) public Message getEngineList() { // TODO limit the engine type in exchangis // return Message.ok().data("result", EngineTypeEnum.values()); return Message.ok().data("result", new EngineTypeEnum[]{EngineTypeEnum.SQOOP}); } - @RequestMapping( value = "", method = RequestMethod.POST) - public Message createJob(HttpServletRequest request, @RequestBody ExchangisJobBasicInfoDTO exchangisJobBasicInfoDTO) { - ExchangisJobBasicInfoVO job = exchangisJobService.createJob(request, exchangisJobBasicInfoDTO); - return Message.ok().data("result", job); + /** + * Create job + * + * @param request http request + * @param exchangisJobVo exchangis job vo + * @return message + */ + @RequestMapping(value = "", method = RequestMethod.POST) + public Message createJob( + @Validated({InsertGroup.class, Default.class}) @RequestBody ExchangisJobVo exchangisJobVo, + BindingResult result, + HttpServletRequest request) { + if (result.hasErrors()) { + return Message.error(result.getFieldErrors().get(0).getDefaultMessage()); + } + String userName = SecurityFilter.getLoginUsername(request); + exchangisJobVo.setCreateUser(userName); + Message response = Message.ok(); + try { + response.data("result", jobInfoService.createJob(exchangisJobVo)); + } catch (Exception e) { + String message = "Fail to create job: " + exchangisJobVo.getJobName() + " (创建任务失败)"; + LOG.error(message, e); + response = Message.error(message); + } + return response; } - @RequestMapping( value = "/{sourceJobId}/copy", method = RequestMethod.POST) + /** + * Copy job + * + * @param sourceJobId source job id + * @param exchangisJobVo job vo + * @return message + */ + @RequestMapping(value = "/{sourceJobId:\\d+}/copy", method = RequestMethod.POST) public Message copyJob(@PathVariable("sourceJobId") Long sourceJobId, - @RequestBody ExchangisJobBasicInfoDTO exchangisJobBasicInfoDTO) { - ExchangisJobBasicInfoVO job = exchangisJobService.copyJob(exchangisJobBasicInfoDTO, sourceJobId); - return Message.ok().data("result", job); + @RequestBody ExchangisJobVo exchangisJobVo) { + return Message.error("Function will be supported in next version (该功能将在下版本支持)"); } - @RequestMapping( value = "/{id}", method = RequestMethod.PUT) - public Message updateJob(@PathVariable("id") Long id, @RequestBody ExchangisJobBasicInfoDTO exchangisJobBasicInfoDTO) { - ExchangisJobBasicInfoVO job = exchangisJobService.updateJob(exchangisJobBasicInfoDTO, id); - return Message.ok().data("result", job); + /** + * Update job + * + * @param id job id + * @param exchangisJobVo job vo + * @return message + */ + @RequestMapping(value = "/{id:\\d+}", method = RequestMethod.PUT) + public Message updateJob(@PathVariable("id") Long id, + @Validated @RequestBody ExchangisJobVo exchangisJobVo, + BindingResult result, HttpServletRequest request) { + if (result.hasErrors()) { + return Message.error(result.getFieldErrors().get(0).getDefaultMessage()); + } + String userName = SecurityFilter.getLoginUsername(request); + exchangisJobVo.setId(id); + exchangisJobVo.setModifyUser(userName); + Message response = Message.ok(); + try { + if (!hasAuthority(userName, jobInfoService.getJob(id, true))) { + return Message.error("You have no permission to update (没有更新权限)"); + } + response.data("result", jobInfoService.updateJob(exchangisJobVo)); + } catch (Exception e) { + String message = "Fail to update job: " + exchangisJobVo.getJobName() + " (更新任务失败)"; + LOG.error(message, e); + response = Message.error(message); + } + return response; } -// @RequestMapping( value = "/dss/{nodeId}", method = RequestMethod.PUT) -// public Message updateJobByDss(@PathVariable("nodeId") String nodeId, @RequestBody ExchangisJobBasicInfoDTO exchangisJobBasicInfoDTO) { -// LOGGER.info("updateJobByDss nodeId {},ExchangisJobBasicInfoDTO {}", nodeId,exchangisJobBasicInfoDTO.toString()); -// ExchangisJobBasicInfoVO job = exchangisJobService.updateJobByDss(exchangisJobBasicInfoDTO, nodeId); -// return Message.ok().data("result", job); -// } - -// @RequestMapping( value = "/import", method = RequestMethod.POST, headers = {"content-type=multipart/form-data"}) -// public Message importSingleJob(@RequestPart("multipartFile") MultipartFile multipartFile) { -// ExchangisJobBasicInfoVO job = exchangisJobService.importSingleJob(multipartFile); -// return Message.ok().data("result", job); -// } - - @RequestMapping( value = "/{id}", method = RequestMethod.DELETE) - public Message deleteJob(@PathVariable("id") Long id) { - exchangisJobService.deleteJob(id); - return Message.ok("job deleted"); + /** + * Delete job + * + * @param id id + * @param request http request + * @return message + */ + @RequestMapping(value = "/{id}", method = RequestMethod.DELETE) + public Message deleteJob(@PathVariable("id") Long id, HttpServletRequest request) { + String userName = SecurityFilter.getLoginUsername(request); + Message response = Message.ok("job deleted"); + try { + if (!hasAuthority(userName, jobInfoService.getJob(id, true))) { + return Message.error("You have no permission to update ()"); + } + jobInfoService.deleteJob(id); + } catch (Exception e) { + String message = "Fail to delete job [ id: " + id + "] (删除任务失败)"; + LOG.error(message, e); + response = Message.error(message); + } + return response; } -// @RequestMapping( value = "/dss/{nodeId}", method = RequestMethod.DELETE) -// public Message deleteJobByDss(@PathVariable("nodeId") String nodeId) { -// LOGGER.info("deleteJobByDss nodeId {}", nodeId); -// return Message.ok("job deleted"); -// } - @RequestMapping( value = "/{id}", method = RequestMethod.GET) - public Message getJob(HttpServletRequest request, @PathVariable("id") Long id) throws ExchangisJobServerException { - ExchangisJobVO job = exchangisJobService.getJob(request, id); - return Message.ok().data("result", job); + /** + * Get job + * + * @param request http request + * @param id id + * @return message + */ + @RequestMapping(value = "/{id}", method = RequestMethod.GET) + public Message getJob(HttpServletRequest request, @PathVariable("id") Long id) { + Message response = Message.ok(); + try { + LOG.info("Request88888: {}", request); + ExchangisJobVo job = jobInfoService.getDecoratedJob(request, id); + response.data("result", job); + } catch (Exception e) { + String message = "Fail to get job detail (查询任务失败)"; + if (e.getCause() instanceof ExchangisJobServerException) { + message += ", reason: " + e.getCause().getMessage(); + } + LOG.error(message, e); + response = Message.error(message); + } + return response; } -// @RequestMapping( value = "/dss/{nodeId}", method = RequestMethod.GET) -// public Message getJobByDssProject(HttpServletRequest request, @PathVariable("nodeId") String nodeId) throws ExchangisJobServerException { -// LOGGER.info("getJobByDssProject nodeId {}", nodeId); -// ExchangisJobVO job = exchangisJobService.getJobByDss(request, nodeId); -// return Message.ok().data("result", job); -// } - - - - @RequestMapping( value = "/{id}/config", method = RequestMethod.PUT) - public Message saveJobConfig(@PathVariable("id") Long id, - @RequestBody ExchangisJobContentDTO exchangisJobContentDTO) throws ExchangisJobServerException { - ExchangisJobVO exchangisJob = exchangisJobService.updateJobConfig(exchangisJobContentDTO, id); - return Message.ok().data("result", exchangisJob); + /** + * Get all sub job list + * @param request + * @param projectId + * @return + */ + @RequestMapping(value = "/subJob/list", method = RequestMethod.GET) + public Message getSubJobList(HttpServletRequest request, @RequestParam(value = "projectId") Long projectId) { + Message response = Message.ok(); + try { + List jobList = jobInfoService.getSubJobList(request, projectId); + response.data("result", jobList); + } catch (Exception e) { + String message = "Fail to get job detail (查询所有子任务列表失败)"; + if (e.getCause() instanceof ExchangisJobServerException) { + message += ", reason: " + e.getCause().getMessage(); + } + LOG.error(message, e); + response = Message.error(message); + } + return response; } - @RequestMapping( value = "/{id}/content", method = RequestMethod.PUT) - public Message saveSubjobs(@PathVariable("id") Long id, - @RequestBody ExchangisJobContentDTO exchangisJobContentDTO) throws ExchangisJobServerException, ExchangisDataSourceException { - ExchangisJobVO exchangisJob = exchangisJobService.updateJobContent(exchangisJobContentDTO, id); - return Message.ok().data("result", exchangisJob); + /** + * Get job list + * @param projectId + * @param jobName + * @return + */ + @RequestMapping(value = "/getJob/list", method = RequestMethod.GET) + public Message getByNameWithProjectId(@RequestParam(value = "projectId") Long projectId, + @RequestParam(value = "jobName", required = false) String jobName) { + Message response = Message.ok(); + try { + List jobs = jobInfoService.getByNameWithProjectId(jobName, projectId); + response.data("result", jobs); + } catch (Exception e) { + String message = "Fail to get job detail (查询任务失败)"; + if (e.getCause() instanceof ExchangisJobServerException) { + message += ", reason: " + e.getCause().getMessage(); + } + LOG.error(message, e); + response = Message.error(message); + } + return response; } - @RequestMapping( value = "{id}/speedlimit/{task_name}/params/ui", method = RequestMethod.GET) - public Message getSpeedLimitSettings(@PathVariable("id") Long id, @PathVariable("task_name") String taskName) { - List> speedLimitSettings = this.exchangisJobService.getSpeedLimitSettings(id, taskName); - return Message.ok().data("ui", speedLimitSettings); + /** + * Save the job configuration + * + * @param id id + * @param jobVo job vo + * @return message + */ + @RequestMapping(value = "/{id}/config", method = RequestMethod.PUT) + public Message saveJobConfig(@PathVariable("id") Long id, + @RequestBody ExchangisJobVo jobVo, HttpServletRequest request) { + jobVo.setId(id); + jobVo.setModifyUser(SecurityFilter.getLoginUsername(request)); + Message response = Message.ok(); + try { + ExchangisJobVo exchangisJob = jobInfoService.updateJobConfig(jobVo); + response.data("id", exchangisJob.getId()); + } catch (Exception e) { + String message = "Fail to save the job configuration (保存任务配置失败)"; + LOG.error(message, e); + response = Message.error(message); + } + return response; } - @RequestMapping( value = "{id}/speedlimit/{task_name}", method = RequestMethod.PUT) - public Message setSpeedLimitSettings(@PathVariable("id") Long id, @PathVariable("task_name") String taskName, - @RequestBody ExchangisTaskSpeedLimitVO settings) { - this.exchangisJobService.setSpeedLimitSettings(id, taskName, settings); - return Message.ok(); + @RequestMapping(value = "/{id}/content", method = RequestMethod.PUT) + public Message saveSubJobs(@PathVariable("id") Long id, + @RequestBody ExchangisJobVo jobVo, HttpServletRequest request) { + jobVo.setId(id); + jobVo.setModifyUser(SecurityFilter.getLoginUsername(request)); + Message response = Message.ok(); + try { + ExchangisJobVo exchangisJob = jobInfoService.updateJobContent(jobVo); + response.data("id", exchangisJob.getId()); + } catch (Exception e) { + String message = "Fail to save the job content (保存任务内容失败)"; + if (e.getCause() instanceof ExchangisJobServerException + || e.getCause() instanceof ExchangisDataSourceException) { + message += ", reason: " + e.getCause().getMessage(); + } + LOG.error(message, e); + response = Message.error(message); + } + return response; } + /** + * TODO complete the authority strategy + * + * @param username username + * @param job job + * @return + */ + private boolean hasAuthority(String username, ExchangisJobVo job) { + return Objects.nonNull(job) && username.equals(job.getCreateUser()); + } } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisMetricsRestfulApi.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisMetricsRestfulApi.java index 69d1c299d..dd1698aab 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisMetricsRestfulApi.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisMetricsRestfulApi.java @@ -10,7 +10,7 @@ import javax.servlet.http.HttpServletRequest; @RestController -@RequestMapping(value = "exchangis", produces = {"application/json;charset=utf-8"}) +@RequestMapping(value = "dss/exchangis/main", produces = {"application/json;charset=utf-8"}) public class ExchangisMetricsRestfulApi { private final ExchangisMetricsService exchangisMetricsService; diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisTaskRestfulApi.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisTaskRestfulApi.java index a5c2293d4..7c4aab099 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisTaskRestfulApi.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/ExchangisTaskRestfulApi.java @@ -1,65 +1,16 @@ package com.webank.wedatasphere.exchangis.job.server.restful; -import java.util.List; - -import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; -import org.apache.linkis.server.Message; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import com.webank.wedatasphere.exchangis.job.server.service.ExchangisLaunchTaskService; -import com.webank.wedatasphere.exchangis.job.server.vo.ExchangisTaskInfoVO; import org.springframework.web.bind.annotation.*; /** * The type Exchangis task controller. * - * @author ruiyang.qin * @date 2021/10/13 */ @RestController -@RequestMapping(value = "exchangis/tasks", produces = {"application/json;charset=utf-8"}) +@RequestMapping(value = "dss/exchangis/main/tasks", produces = {"application/json;charset=utf-8"}) public class ExchangisTaskRestfulApi { private static final Logger LOG = LoggerFactory.getLogger(ExchangisTaskRestfulApi.class); - @Autowired - private ExchangisLaunchTaskService exchangisLaunchTaskService; - - @RequestMapping(value = "", method = RequestMethod.GET) - public Message listTasks(@RequestParam(value = "taskId", required = false) Long taskId, - @RequestParam(value = "taskName", required = false) String taskName, - @RequestParam(value = "status", required = false) String status, - @RequestParam(value = "launchStartTime", required = false) Long launchStartTime, - @RequestParam(value = "launchEndTime", required = false) Long launchEndTime, - @RequestParam(value = "current", required = false) int current, - @RequestParam(value = "size", required = false) int size) { - Message message = Message.ok("获取所有任务的列表"); - try { - List taskList = exchangisLaunchTaskService.listTasks(taskId, taskName, status, - launchStartTime, launchEndTime, current, size); - int total = exchangisLaunchTaskService.count(taskId, taskName, status, launchStartTime, launchEndTime); - message.data("result", taskList); - message.data("total", total); - } catch (ExchangisJobServerException e) { - String errorMessage = "Error occur while getting taskList: [taskId: " + taskId + "taskName" + taskName + "]"; - LOG.error(errorMessage, e); - message = Message.error(message + ", reason: " + e.getMessage()); - } - return message; - //return Message.ok().data("result", taskList).data("total", total); - } - - @RequestMapping( value = "{taskId}", method = RequestMethod.DELETE) - public Message deleteTask(@PathVariable("taskId") Long taskId) throws Exception { - Message message = Message.ok("删除任务"); - try { - this.exchangisLaunchTaskService.delete(taskId); - } catch (ExchangisJobServerException e) { - String errorMessage = "Error occur while delete task: [taskId: " + taskId + "]"; - LOG.error(errorMessage, e); - message = Message.error(message + ", reason: " + e.getMessage()); - } - return message; - //return Message.ok(); - } - } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java index 954c51e82..b4d8bc430 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java @@ -5,13 +5,13 @@ import com.webank.wedatasphere.exchangis.job.launcher.domain.task.TaskStatus; import com.webank.wedatasphere.exchangis.job.log.LogQuery; import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; -import com.webank.wedatasphere.exchangis.job.server.service.ExchangisJobService; +import com.webank.wedatasphere.exchangis.job.server.service.JobInfoService; import com.webank.wedatasphere.exchangis.job.server.service.impl.DefaultJobExecuteService; import com.webank.wedatasphere.exchangis.job.server.vo.ExchangisCategoryLogVo; import com.webank.wedatasphere.exchangis.job.server.vo.ExchangisJobProgressVo; import com.webank.wedatasphere.exchangis.job.server.vo.ExchangisJobTaskVo; -import com.webank.wedatasphere.exchangis.job.server.vo.ExchangisLaunchedJobListVO; -import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobVO; +import com.webank.wedatasphere.exchangis.job.server.vo.ExchangisLaunchedJobListVo; +import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobVo; import org.apache.commons.lang.StringUtils; import org.apache.linkis.server.Message; import org.apache.linkis.server.security.SecurityFilter; @@ -29,11 +29,11 @@ * @Date 2022/1/8 15:25 */ @RestController -@RequestMapping(value = "exchangis/job", produces = {"application/json;charset=utf-8"}) +@RequestMapping(value = "dss/exchangis/main/job", produces = {"application/json;charset=utf-8"}) public class ExchangisJobExecuteRestfulApi { private static final Logger LOG = LoggerFactory.getLogger(ExchangisJobExecuteRestfulApi.class); @Autowired - private ExchangisJobService exchangisJobService; + private JobInfoService jobInfoService; @Resource private DefaultJobExecuteService executeService; @@ -45,31 +45,38 @@ public class ExchangisJobExecuteRestfulApi { * @return message */ @RequestMapping( value = "/{id}/execute", method = RequestMethod.POST) - public Message executeJob(@RequestBody(required = false) Map permitPartialFailures, + public Message executeJob(@RequestBody(required = false) Map permitPartialFailures, @PathVariable("id") Long id, HttpServletRequest request) { - // First to find the job from the old table. TODO use the job entity - ExchangisJobVO jobVo = exchangisJobService.getById(id); - if (Objects.isNull(jobVo)){ - return Message.error("Job related the id: [" + id + "] is Empty(关联的任务不存在)"); - } - // Convert to the job info - ExchangisJobInfo jobInfo = new ExchangisJobInfo(jobVo); String loginUser = SecurityFilter.getLoginUsername(request); - if (!hasAuthority(loginUser, jobInfo)){ - return Message.error("You have no permission to execute job (没有执行任务权限)"); - } Message result = Message.ok("Submitted succeed(提交成功)！"); + ExchangisJobInfo jobInfo = null; try { + // First to find the job from the old table. + ExchangisJobVo jobVo = jobInfoService.getJob(id, false); + if (Objects.isNull(jobVo)){ + return Message.error("Job related the id: [" + id + "] is Empty(关联的任务不存在)"); + } + // Convert to the job info + jobInfo = new ExchangisJobInfo(jobVo); + if (!hasAuthority(loginUser, jobInfo)){ + return Message.error("You have no permission to execute job (没有执行任务权限)"); + } // Send to execute service String jobExecutionId = executeService.executeJob(jobInfo, StringUtils.isNotBlank(jobInfo.getExecuteUser()) ? jobInfo.getExecuteUser() : loginUser); result.data("jobExecutionId", jobExecutionId); - } catch (ExchangisJobServerException e) { - String message = "Error occur while executing job: [id: " + jobInfo.getId() + " name: " + jobInfo.getName() +"]"; + } catch (Exception e) { + String message; + if (Objects.nonNull(jobInfo)) { + message = "Error occur while executing job: [id: " + jobInfo.getId() + " name: " + jobInfo.getName() + "]"; + result = Message.error(message + "(执行任务出错), reason: " + e.getMessage()); + } else { + message = "Error to get the job detail (获取任务信息出错)"; + result = Message.error(message); + } LOG.error(message, e); - result = Message.error(message + "(执行任务出错), reason: " + e.getMessage()); } - result.setMethod("/api/rest_j/v1/exchangis/job/{id}/execute"); + result.setMethod("/api/rest_j/v1/dss/exchangis/main/job/{id}/execute"); return result; } @@ -84,7 +91,7 @@ public Message getExecutedJobTaskList(@PathVariable(value = "jobExecutionId") St LOG.error(errorMessage, e); message = Message.error(message + "(执行任务出错), reason: " + e.getMessage()); } - message.setMethod("/api/rest_j/v1/exchangis/job/execution/" + jobExecutionId + "/taskList"); + message.setMethod("/api/rest_j/v1/dss/exchangis/main/job/execution/" + jobExecutionId + "/taskList"); return message; } @@ -98,7 +105,7 @@ public Message getExecutedJobAndTaskStatus(@PathVariable(value = "jobExecutionId return Message.error("Fail to get progress info (获取任务执行状态失败), reason: [" + e.getMessage() + "]"); } Message message = Message.ok("Submitted succeed(提交成功)！"); - message.setMethod("/api/rest_j/v1/exchangis/job/execution/" +jobExecutionId +"/progress"); + message.setMethod("/api/rest_j/v1/dss/exchangis/main/job/execution/" +jobExecutionId +"/progress"); message.data("job", jobAndTaskStatus); return message; } @@ -108,7 +115,7 @@ public Message getExecutedJobStatus(@PathVariable(value = "jobExecutionId") Stri Message message = Message.ok("Submitted succeed(提交成功)！"); try { ExchangisJobProgressVo jobStatus = executeService.getJobStatus(jobExecutionId); - message.setMethod("/api/rest_j/v1/exchangis/job/execution/" + jobExecutionId + "/status"); + message.setMethod("/api/rest_j/v1/dss/exchangis/main/job/execution/" + jobExecutionId + "/status"); message.data("status", jobStatus.getStatus()); message.data("progress", jobStatus.getProgress()); message.data("allTaskStatus", jobStatus.getAllTaskStatus()); @@ -140,7 +147,7 @@ public Message getJobExecutionLogs(@PathVariable(value = "jobExecutionId") Strin LOG.error(message, e); result = Message.error(message + ", reason: " + e.getMessage()); } - result.setMethod("/api/rest_j/v1/exchangis/job/execution/{jobExecutionId}/log"); + result.setMethod("/api/rest_j/v1/dss/exchangis/main/job/execution/{jobExecutionId}/log"); return result; } @@ -163,7 +170,7 @@ public Message ExecutedJobKill(@PathVariable(value = "jobExecutionId") String jo else { message = Message.error("Kill failed(停止失败)！,job 已经到终态"); } - message.setMethod("/api/rest_j/v1/exchangis/job/execution/" + jobExecutionId + "/kill"); + message.setMethod("/api/rest_j/v1/dss/exchangis/main/job/execution/" + jobExecutionId + "/kill"); return message; } @@ -178,7 +185,7 @@ public Message listJobs(@RequestParam(value = "jobExecutionId", required = false HttpServletRequest request) { Message message = Message.ok("Submitted succeed(提交成功)！"); try { - List jobList = executeService.getExecutedJobList(jobExecutionId, jobName, status, + List jobList = executeService.getExecutedJobList(jobExecutionId, jobName, status, launchStartTime, launchEndTime, current, size, request); int total = executeService.count(jobExecutionId, jobName, status, launchStartTime, launchEndTime, request); message.data("jobList", jobList); @@ -188,29 +195,7 @@ public Message listJobs(@RequestParam(value = "jobExecutionId", required = false LOG.error(errorMessage, e); message = Message.error(message + ", reason: " + e.getMessage()); } - message.setMethod("/api/rest_j/v1/exchangis/job/execution/listJobs"); - return message; - } - - @RequestMapping(value = "/partitionInfo", method = RequestMethod.GET) - public Message partitionInfo(@RequestParam(value = "dataSourceType", required = false) String dataSourceTpe, - @RequestParam(value = "dbname", required = false) String dbname, - @RequestParam(value = "table", required = false) String table) { - Map render = new HashMap<>(); - List partitionList = new ArrayList<>(); - List partitionEmpty = new ArrayList<>(); - partitionList.add("$yyyy-MM-dd"); - partitionList.add("${run_date-1}"); - partitionList.add("${run_date-7}"); - partitionList.add("${run_month_begin-1}"); - render.put("key1", ""); - render.put("key2", "${yyyyMMdd}"); - render.put("key3", partitionList); - render.put("key4", partitionEmpty); - Message message = Message.ok("Submitted succeed(提交成功)！"); - message.setMethod("/api/rest_j/v1/exchangis/job/partionInfo/listJobs"); - message.data("type", "Map"); - message.data("render", render); + message.setMethod("/api/rest_j/v1/dss/exchangis/main/job/execution/listJobs"); return message; } @@ -236,7 +221,7 @@ public Message ExecutedJobDelete(@PathVariable(value = "jobExecutionId") String LOG.error(errorMessage, e); message = Message.error(message + ", reason: " + e.getMessage()); } - message.setMethod("/api/rest_j/v1/exchangis/job/" + jobExecutionId + "/deleteJob"); + message.setMethod("/api/rest_j/v1/dss/exchangis/main/job/" + jobExecutionId + "/deleteJob"); return message; } @@ -253,7 +238,7 @@ public Message allTaskStatus(@PathVariable(value = "jobExecutionId") String jobE LOG.error(errorMessage, e); message = Message.error(message + ", reason: " + e.getMessage()); } - message.setMethod("/api/rest_j/v1/exchangis/job/" + jobExecutionId + "/allTaskStatus"); + message.setMethod("/api/rest_j/v1/dss/exchangis/main/job/" + jobExecutionId + "/allTaskStatus"); return message; } } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisTaskExecuteRestfulApi.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisTaskExecuteRestfulApi.java index a2682e713..6a597d59b 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisTaskExecuteRestfulApi.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisTaskExecuteRestfulApi.java @@ -3,10 +3,10 @@ import com.webank.wedatasphere.exchangis.datasource.core.utils.Json; import com.webank.wedatasphere.exchangis.job.log.LogQuery; import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; -import com.webank.wedatasphere.exchangis.job.server.service.ExchangisJobService; +import com.webank.wedatasphere.exchangis.job.server.service.JobInfoService; import com.webank.wedatasphere.exchangis.job.server.service.JobExecuteService; import com.webank.wedatasphere.exchangis.job.server.vo.ExchangisCategoryLogVo; -import com.webank.wedatasphere.exchangis.job.server.vo.ExchangisLaunchedTaskMetricsVO; +import com.webank.wedatasphere.exchangis.job.server.vo.ExchangisLaunchedTaskMetricsVo; import org.apache.commons.lang.StringUtils; import org.apache.linkis.server.Message; import org.apache.linkis.server.security.SecurityFilter; @@ -25,25 +25,29 @@ */ @RestController -@RequestMapping(value = "exchangis/task", produces = {"application/json;charset=utf-8"}) +@RequestMapping(value = "dss/exchangis/main/task", produces = {"application/json;charset=utf-8"}) public class ExchangisTaskExecuteRestfulApi { private static final Logger LOG = LoggerFactory.getLogger(ExchangisTaskExecuteRestfulApi.class); @Autowired - private ExchangisJobService exchangisJobService; + private JobInfoService jobInfoService; @Resource private JobExecuteService jobExecuteService; @RequestMapping( value = "/execution/{taskId}/metrics", method = RequestMethod.POST) public Message getTaskMetrics(@PathVariable("taskId") String taskId, - @RequestBody Map json, HttpServletRequest request) throws ExchangisJobServerException { + @RequestBody Map json, HttpServletRequest request) throws ExchangisJobServerException { Message result = Message.ok("Submitted succeed(提交成功)！"); - String jobExecutionId = json.get("jobExecutionId"); + String jobExecutionId = null; + + if(null!=json.get("jobExecutionId")){ + jobExecutionId = (String) json.get("jobExecutionId"); + } if (StringUtils.isBlank(jobExecutionId)){ return Message.error("Required params 'jobExecutionId' is missing"); } try{ - ExchangisLaunchedTaskMetricsVO taskMetrics = this.jobExecuteService + ExchangisLaunchedTaskMetricsVo taskMetrics = this.jobExecuteService .getLaunchedTaskMetrics(taskId, jobExecutionId, SecurityFilter.getLoginUsername(request)); result.data("task", taskMetrics); }catch(Exception e){ @@ -51,7 +55,7 @@ public Message getTaskMetrics(@PathVariable("taskId") String taskId, LOG.error(message, e); result = Message.error(message + ", reason: " + e.getMessage()); } - result.setMethod("/api/rest_j/v1/exchangis/task/execution/{taskId}/metrics"); + result.setMethod("/api/rest_j/v1/dss/exchangis/main/task/execution/{taskId}/metrics"); return result; } @@ -74,7 +78,7 @@ public Message getTaskExecutionLogs(@PathVariable(value = "taskId") String taskI LOG.error(message, e); result = Message.error(message + ", reason: " + e.getMessage()); } - result.setMethod("/api/rest_j/v1/exchangis/job/execution/{taskId}/log"); + result.setMethod("/api/rest_j/v1/dss/exchangis/main/job/execution/{taskId}/log"); return result; } } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/external/ExchangisJobDssAppConnRestfulApi.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/external/ExchangisJobDssAppConnRestfulApi.java index aeafa728b..54daa0a6d 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/external/ExchangisJobDssAppConnRestfulApi.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/external/ExchangisJobDssAppConnRestfulApi.java @@ -1,8 +1,269 @@ package com.webank.wedatasphere.exchangis.job.server.restful.external; +import com.webank.wedatasphere.exchangis.common.validator.groups.InsertGroup; +import com.webank.wedatasphere.exchangis.job.domain.ExchangisJobInfo; +import com.webank.wedatasphere.exchangis.job.launcher.domain.task.TaskStatus; +import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; +import com.webank.wedatasphere.exchangis.job.server.service.IProjectImportService; +import com.webank.wedatasphere.exchangis.job.server.service.JobInfoService; +import com.webank.wedatasphere.exchangis.job.server.service.impl.DefaultJobExecuteService; +import com.webank.wedatasphere.exchangis.job.server.service.impl.ProjectImportServerImpl; +import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobVo; +import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; +import com.webank.wedatasphere.exchangis.project.server.mapper.ProjectMapper; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.validation.BindingResult; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; + +import javax.annotation.Resource; +import javax.servlet.http.HttpServletRequest; +import javax.validation.groups.Default; +import javax.ws.rs.core.Context; +import java.rmi.ServerException; +import java.util.Map; +import java.util.Objects; + /** * Define to support the app conn, in order to distinguish from the inner api */ +@RestController +@RequestMapping(value = "/dss/exchangis/main/appJob", produces = {"application/json;charset=utf-8"}) public class ExchangisJobDssAppConnRestfulApi { + private static final Logger LOG = LoggerFactory.getLogger(ExchangisJobDssAppConnRestfulApi.class); + /** + * Job service + */ + @Resource + private JobInfoService jobInfoService; + + /** + * Job execute service + */ + @Resource + private DefaultJobExecuteService executeService; + + @Resource + private ProjectImportServerImpl projectImportServer; + + @Autowired + private ProjectMapper projectMapper; + + /** + * Create job + * @param request http request + * @param exchangisJobVo exchangis job vo + * @return message + */ + @RequestMapping( value = "/create", method = RequestMethod.POST) + public Message createJob( + @Validated({InsertGroup.class, Default.class}) @RequestBody ExchangisJobVo exchangisJobVo, + BindingResult result, + HttpServletRequest request) { + if (result.hasErrors()){ + return Message.error(result.getFieldErrors().get(0).getDefaultMessage()); + } + String userName = SecurityFilter.getLoginUsername(request); + exchangisJobVo.setCreateUser(userName); + Message response = Message.ok(); + try{ + Long id = null; + id = jobInfoService.createJob(exchangisJobVo).getId(); + response.data("id", id); + LOG.info("id: {}", id); + } catch (Exception e){ + String message = "Fail to create dss job: " + exchangisJobVo.getJobName() +" (创建DSS任务失败)"; + LOG.error(message, e); + response = Message.error(message); + } + return response; + } + + /** + * Delete job + * @param id id + * @param request http request + * @return message + */ + @RequestMapping( value = "/{id:\\d+}", method = RequestMethod.POST) + public Message deleteJob(@PathVariable("id") Long id, HttpServletRequest request) { + String userName = SecurityFilter.getLoginUsername(request); + Message response = Message.ok("dss job deleted"); + try { + LOG.info("delete job bean: {}, jobid: {}", jobInfoService.getJob(id, true), jobInfoService.getJob(id, true).getId()); + if (Objects.isNull(jobInfoService.getJob(id, true)) || jobInfoService.getJob(id, true).getId() == null){ + return response; + } + else if (!hasAuthority(userName, jobInfoService.getJob(id, true))){ + return Message.error("You have no permission to update (没有删除权限)"); + } + jobInfoService.deleteJob(id); + } catch (Exception e){ + String message = "Fail to delete dss job [ id: " + id + "] (删除DSS任务失败)"; + LOG.error(message, e); + response = Message.error(message); + } + return response; + } + + /** + * Update job + * @param id job id + * @param exchangisJobVo job vo + * @return message + */ + @RequestMapping( value = "/{id:\\d+}", method = RequestMethod.PUT) + public Message updateJob(@PathVariable("id") Long id, + @Validated @RequestBody ExchangisJobVo exchangisJobVo, + BindingResult result, HttpServletRequest request) { + if (result.hasErrors()){ + return Message.error(result.getFieldErrors().get(0).getDefaultMessage()); + } + String userName = SecurityFilter.getLoginUsername(request); + exchangisJobVo.setId(id); + exchangisJobVo.setModifyUser(userName); + Message response = Message.ok(); + try{ + LOG.info("update job bean: {}, jobid: {}", jobInfoService.getJob(id, true), jobInfoService.getJob(id, true).getId()); + if (Objects.isNull(jobInfoService.getJob(id, true)) || jobInfoService.getJob(id, true).getId() == null){ + return Message.error("You have no job in exchangis,please delete this job (该节点在exchangis端不存在，请删除该节点)"); + } + else if (!hasAuthority(userName, jobInfoService.getJob(id , true))){ + return Message.error("You have no permission to update (没有更新权限)"); + } + response.data("id", jobInfoService.updateJob(exchangisJobVo).getId()); + } catch (Exception e){ + String message = "Fail to update dss job: " + exchangisJobVo.getJobName() +" (更新DSS任务失败)"; + LOG.error(message, e); + response = Message.error(message); + } + return response; + } + + /** + * Execute job + * @param id id + * @return message + */ + @RequestMapping( value = "/execute/{id}", method = RequestMethod.POST) + public Message executeJob(@PathVariable("id") Long id, HttpServletRequest request, @RequestBody Map params) { + String submitUser = params.get("submitUser").toString(); + String loginUser = SecurityFilter.getLoginUsername(request); + Message result = Message.ok(); + ExchangisJobInfo jobInfo = null; + try { + // First to find the job from the old table. + ExchangisJobVo jobVo = jobInfoService.getJob(id, false); + if (Objects.isNull(jobVo)){ + return Message.error("Job related the id: [" + id + "] is Empty(关联的DSS任务不存在)"); + } + // Convert to the job info + jobInfo = new ExchangisJobInfo(jobVo); + jobInfo.setName(jobVo.getJobName()); + jobInfo.setId(jobVo.getId()); + LOG.info("jobInfo: name{},executerUser{},createUser{},id{}",jobInfo.getName(),jobInfo.getExecuteUser(),jobInfo.getCreateUser(),jobInfo.getId()); + LOG.info("loginUser: {}, jobVo:{}",loginUser,jobVo); + //find project info + ExchangisProject project = projectMapper.getDetailById(jobVo.getProjectId()); + LOG.info("project: {}, getProjectId:{}",project,jobVo.getProjectId()); + //find project user authority + if (!hasExecuteAuthority(submitUser, project)){ + return Message.error("You have no permission to execute job (没有执行DSS任务权限)"); + } + // Send to execute service + String jobExecutionId = executeService.executeJob(jobInfo, StringUtils.isNotBlank(jobInfo.getExecuteUser()) ? + jobInfo.getExecuteUser() : loginUser); + result.data("jobExecutionId", jobExecutionId); + + while (true) { + TaskStatus jobStatus = executeService.getJobStatus(jobExecutionId).getStatus(); + LOG.info("Taskstatus is: {}", jobStatus.name()); + if (jobStatus == TaskStatus.Success ) { + result.data("jobStatus", jobStatus.name()); + LOG.info("Execute task success"); + break; + } else if (jobStatus == TaskStatus.Cancelled || jobStatus == TaskStatus.Failed || jobStatus == TaskStatus.Undefined || jobStatus == TaskStatus.Timeout) { + result.data("jobStatus", jobStatus.name()); + LOG.info("Execute task faild"); + throw new Exception(); + } + } + } catch (Exception e) { + String message; + if (Objects.nonNull(jobInfo)) { + message = "Error occur while executing job: [id: " + jobInfo.getId() + " name: " + jobInfo.getName() + "]"; + result = Message.error(message + "(执行任务出错), reason: " + e.getMessage()); + } else { + message = "Error to get the job detail (获取任务信息出错)"; + result = Message.error(message); + } + LOG.error(message, e); + } + return result; + } + + @RequestMapping( value = "/import", method = RequestMethod.POST) + public Message importJob(@Context HttpServletRequest request, @RequestBody Map params) throws ServerException, ExchangisJobServerException{ + + Message response = null; + try { + LOG.info("param: {}", params); + response = projectImportServer.importProject(request, params); + LOG.info("import job success"); + } catch (ExchangisJobServerException e){ + String message = "Fail import job [ id: " + params + "] (导入任务失败)"; + LOG.error(message, e); + response = Message.error(message); + } + return response; + + } + + @RequestMapping( value = "/export", method = RequestMethod.POST) + public Message exportJob(@Context HttpServletRequest request, @RequestBody Map params) throws ServerException, ExchangisJobServerException { + String userName = SecurityFilter.getLoginUsername(request); + + LOG.info("export function params: {}", params); + Message response = null; + try { + response = jobInfoService.exportProject(params, userName, request); + LOG.info("export job success"); + } catch (Exception e){ + String message = "Fail Export job [ id: " + params + "] (导出任务失败)"; + LOG.error(message, e); + response = Message.error(message); + } + return response; + + //return jobInfoService.exportProject(params, userName, request); + + } + + /** + * TODO complete the authority strategy + * @param username username + * @param job job + * @return + */ + private boolean hasAuthority(String username, ExchangisJobVo job){ + return Objects.nonNull(job) && username.equals(job.getCreateUser()); + } + + /** + * @param username username + * @param project project + * @return + */ + private boolean hasExecuteAuthority(String username, ExchangisProject project){ + if(project.getEditUsers().contains(username)||project.getExecUsers().contains(username)){ + return true; + } + return false; + } } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/external/ModuleEnum.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/external/ModuleEnum.java new file mode 100644 index 000000000..ebd971548 --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/external/ModuleEnum.java @@ -0,0 +1,30 @@ +package com.webank.wedatasphere.exchangis.job.server.restful.external; + +import java.util.Arrays; + +/** + * @author tikazhang + * @Date 2022/3/14 20:37 + */ +public enum ModuleEnum { + + SQOOP_IDS("sqoopIds", "SQOOP ids"), + + DATAX_IDS("dataxIds", "DATAX ids"); + + private String name; + private String desc; + + ModuleEnum(String name, String desc) { + this.name = name; + this.desc = desc; + } + + public static ModuleEnum getEnum(String name) { + return Arrays.stream(ModuleEnum.values()).filter(e -> e.getName().equals(name)).findFirst().orElseThrow(NullPointerException::new); + } + + public String getName() { + return name; + } +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/ExchangisJobService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/ExchangisJobService.java deleted file mode 100644 index c3b26dd47..000000000 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/ExchangisJobService.java +++ /dev/null @@ -1,102 +0,0 @@ -package com.webank.wedatasphere.exchangis.job.server.service; - -import com.baomidou.mybatisplus.extension.service.IService; -import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisDataSourceException; -import com.webank.wedatasphere.exchangis.datasource.core.ui.ElementUI; -import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobVO; -import com.webank.wedatasphere.exchangis.job.server.dto.ExchangisJobBasicInfoDTO; -import com.webank.wedatasphere.exchangis.job.server.dto.ExchangisJobContentDTO; -import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; -import com.webank.wedatasphere.exchangis.job.server.vo.ExchangisJobBasicInfoVO; -import com.webank.wedatasphere.exchangis.job.server.vo.ExchangisTaskSpeedLimitVO; -import org.springframework.web.multipart.MultipartFile; -import scala.Int; - -import javax.servlet.http.HttpServletRequest; -import java.util.List; - -/** - * The interface Exchangis job service. - * - * @author yuxin.yuan - * @since 2021-08-10 - */ -public interface ExchangisJobService extends IService { - - /** - * Create job. - * - * @param exchangisJobBasicInfoDTO the exchangis job basic info dto - * @return the exchangis job basic info vo - */ - public ExchangisJobBasicInfoVO createJob(HttpServletRequest request, ExchangisJobBasicInfoDTO exchangisJobBasicInfoDTO); - - /** - * Gets job list(return job basic info). - * - * @param projectId the project id - * @param type the type - * @param name the name - * @return the job list - */ - public List getJobList(long projectId, String type, String name, int current, int size); - - int count(long projectId, String type, String name); - - public List getJobListByDssProject(long dssProjectId, String type, String name); - - public ExchangisJobBasicInfoVO copyJob(ExchangisJobBasicInfoDTO exchangisJobBasicInfoDTO, Long sourceJobId); - - /** - * Update job exchangis job basic info. - * - * @param exchangisJobBasicInfoDTO the exchangis job basic info dto - * @param id the id - * @return the exchangis job basic info vo - */ - public ExchangisJobBasicInfoVO updateJob(ExchangisJobBasicInfoDTO exchangisJobBasicInfoDTO, Long id); - - public ExchangisJobBasicInfoVO updateJobByDss(ExchangisJobBasicInfoDTO exchangisJobBasicInfoDTO, String nodeId); - - public ExchangisJobBasicInfoVO importSingleJob(MultipartFile multipartFile); - - public void deleteJob(Long id); - - public void deleteJobByDss(String nodeId); - - public ExchangisJobVO getJob(Long id) throws ExchangisJobServerException; - /** - * Get exchangis job by id. - * - * @param id the id - * @return the job - * @throws ExchangisJobServerException the exchangis job error exception - */ - public ExchangisJobVO getJob(HttpServletRequest request, Long id) throws ExchangisJobServerException; - - public ExchangisJobVO getJobByDss(HttpServletRequest request, String nodeId) throws ExchangisJobServerException; - - /** - * Update exchangis job config. - * - * @param exchangisJobContentDTO the exchangis job content dto - * @param id the id - * @return the exchangis job - */ - public ExchangisJobVO updateJobConfig(ExchangisJobContentDTO exchangisJobContentDTO, Long id) - throws ExchangisJobServerException; - - /** - * Update exchangis job content. - * - * @param exchangisJobContentDTO the exchangis job content dto - * @param id the id - * @return the exchangis job - */ - public ExchangisJobVO updateJobContent(ExchangisJobContentDTO exchangisJobContentDTO, Long id) - throws ExchangisJobServerException, ExchangisDataSourceException; - - public List> getSpeedLimitSettings(Long id, String taskName); - - public void setSpeedLimitSettings(Long id, String taskName, ExchangisTaskSpeedLimitVO settings); -} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/ExchangisLaunchTaskService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/ExchangisLaunchTaskService.java deleted file mode 100644 index 54f554fa7..000000000 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/ExchangisLaunchTaskService.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.webank.wedatasphere.exchangis.job.server.service; - -import java.util.List; - -import com.baomidou.mybatisplus.extension.service.IService; -import com.webank.wedatasphere.exchangis.job.launcher.entity.ExchangisLaunchTask; -import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; -import com.webank.wedatasphere.exchangis.job.server.vo.ExchangisTaskInfoVO; - -public interface ExchangisLaunchTaskService extends IService { - - /** - * Gets task list(return task basic info). - * - * @param taskId the task id - * @param taskName the task name - * @param status the status - * @param launchStartTime the launch start time - * @param launchEndTime the launch end time - * @param current the current - * @param size the size - * @return the task list - */ - List listTasks(Long taskId, String taskName, String status, Long launchStartTime, - Long launchEndTime, int current, int size) throws ExchangisJobServerException; - - /** - * Count int. - *d - * @param taskId the task id - * @param taskName the task name - * @param status the status - * @param launchStartTime the launch start time - * @param launchEndTime the launch end time - * @return the int - */ - int count(Long taskId, String taskName, String status, Long launchStartTime, Long launchEndTime); - - void delete(Long historyId) throws Exception; -} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/IProjectExportService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/IProjectExportService.java new file mode 100644 index 000000000..feea880c0 --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/IProjectExportService.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.exchangis.job.server.service; + +import com.webank.wedatasphere.exchangis.job.server.dto.ExportedProject; + +import javax.servlet.http.HttpServletRequest; +import java.util.Map; +import java.util.Set; + +/** + * @author tikazhang + * @Date 2022/3/15 9:30 + */ +public interface IProjectExportService { + String exportProject(Map params, String username, HttpServletRequest request) throws Exception; + + ExportedProject export(Long projectId, Map> moduleIdsMap, boolean partial); + + Map> getModuleIdsMap(Map params); + + Long getProjectId(Map> moduleIdsMap); +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/IProjectImportService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/IProjectImportService.java new file mode 100644 index 000000000..abccb0d88 --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/IProjectImportService.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.exchangis.job.server.service; + +import com.webank.wedatasphere.exchangis.job.server.dto.IdCatalog; +import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; +import com.webank.wedatasphere.exchangis.project.server.exception.ExchangisProjectErrorException; +import org.apache.linkis.server.Message; + +import javax.servlet.http.HttpServletRequest; +import java.rmi.ServerException; +import java.util.Map; + +/** + * @author tikazhang + * @Date 2022/3/15 10:01 + */ +public interface IProjectImportService { + Message importProject(HttpServletRequest req, Map params) throws ExchangisJobServerException, ServerException; + + IdCatalog importOpt(String projectJson, Long projectId, String versionSuffix, String userName) throws ExchangisJobServerException; +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/JobExecuteService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/JobExecuteService.java index c9de33c7b..026d5a3c8 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/JobExecuteService.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/JobExecuteService.java @@ -2,15 +2,12 @@ import com.webank.wedatasphere.exchangis.job.domain.ExchangisJobInfo; -import com.webank.wedatasphere.exchangis.job.launcher.entity.LaunchedExchangisJobEntity; import com.webank.wedatasphere.exchangis.job.launcher.exception.ExchangisTaskLaunchException; import com.webank.wedatasphere.exchangis.job.log.LogQuery; import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; import com.webank.wedatasphere.exchangis.job.server.vo.*; -import org.apache.linkis.server.Message; import javax.servlet.http.HttpServletRequest; -import java.util.Date; import java.util.List; public interface JobExecuteService { @@ -34,7 +31,7 @@ public interface JobExecuteService { * @param jobExecutionId the job ExecutionId * @return the task launched metrics */ - ExchangisLaunchedTaskMetricsVO getLaunchedTaskMetrics(String taskid, String jobExecutionId, String userName) throws ExchangisJobServerException; + ExchangisLaunchedTaskMetricsVo getLaunchedTaskMetrics(String taskid, String jobExecutionId, String userName) throws ExchangisJobServerException; /** * Gets job progress info @@ -61,7 +58,7 @@ public interface JobExecuteService { * Gets Executed job list * @return the launched jobList */ - List getExecutedJobList(String jobExecutionId, String jobName, String status, + List getExecutedJobList(String jobExecutionId, String jobName, String status, Long launchStartTime, Long launchEndTime, int current, int size, HttpServletRequest request) throws ExchangisJobServerException; /** diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/JobInfoService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/JobInfoService.java index 74ba38c06..b2e153a0a 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/JobInfoService.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/JobInfoService.java @@ -1,4 +1,113 @@ package com.webank.wedatasphere.exchangis.job.server.service; +import com.webank.wedatasphere.exchangis.common.pager.PageResult; +import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisDataSourceException; +import com.webank.wedatasphere.exchangis.job.server.dto.ExportedProject; +import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobQueryVo; +import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobVo; +import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; +import org.apache.linkis.server.Message; + +import javax.servlet.http.HttpServletRequest; + +import java.util.List; + +import java.rmi.ServerException; +import java.util.Map; +import java.util.Set; + + +/** + * The interface Exchangis job service. + * + * @author yuxin.yuan + * @since 2021-08-10 + */ public interface JobInfoService { + + /** + * Create job. + * + * @param jobVo the exchangis job basic info + * @return the exchangis job basic info vo + */ + ExchangisJobVo createJob(ExchangisJobVo jobVo); + + /** + * Update job exchangis job basic info. + * + * @param jobVo the exchangis job basic info dto + * @return the exchangis job basic info vo + */ + ExchangisJobVo updateJob(ExchangisJobVo jobVo); + + /** + * Gets job list(return job basic info). + * + * @param queryVo query vo + * @return the job page result + */ + PageResult queryJobList(ExchangisJobQueryVo queryVo); + + /** + * Delete job + * @param id job id + */ + void deleteJob(Long id); + + /** + * Get exchangis job by id. + * TODO remove the request + * @param id the id + * @throws ExchangisJobServerException the exchangis job error exception + */ + ExchangisJobVo getJob(Long id, boolean basic); + + /** + * Get job by name and projectId + * @param jobName + * @param projectId + * @return + */ + List getByNameWithProjectId(String jobName, Long projectId); + + ExchangisJobVo getDecoratedJob(HttpServletRequest request, Long id) throws ExchangisJobServerException; + + /** + * Get all subJob list + * @param request + * @param projectId + * @return + * @throws ExchangisJobServerException + */ + List getSubJobList(HttpServletRequest request, Long projectId) throws ExchangisJobServerException; + + /** + * Update exchangis job config. + * + * @param jobVo the exchangis job config + * @return the exchangis job + */ + ExchangisJobVo updateJobConfig(ExchangisJobVo jobVo) throws ExchangisJobServerException; + + /** + * Update exchangis job content. + * + * @param jobVo the exchangis job content + * @return the exchangis job + */ + ExchangisJobVo updateJobContent(ExchangisJobVo jobVo) throws ExchangisJobServerException, ExchangisDataSourceException; + + /** + * Export exchangis job to BML. + * + * @param username params + * @return + */ + Message exportProject(Map params, String username, HttpServletRequest request) throws ExchangisJobServerException, ServerException; + + ExportedProject export(Long projectId, Map> moduleIdsMap, boolean partial, HttpServletRequest request) throws ExchangisJobServerException; + + Map> getModuleIdsMap(Map params); + } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobExecuteService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobExecuteService.java index 3da3e30f5..3fea2972e 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobExecuteService.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobExecuteService.java @@ -1,7 +1,6 @@ package com.webank.wedatasphere.exchangis.job.server.service.impl; import com.webank.wedatasphere.exchangis.datasource.core.utils.Json; -import com.webank.wedatasphere.exchangis.datasource.core.vo.ExchangisJobParamsContent; import com.webank.wedatasphere.exchangis.job.domain.ExchangisJobInfo; import com.webank.wedatasphere.exchangis.job.launcher.AccessibleLauncherTask; import com.webank.wedatasphere.exchangis.job.launcher.ExchangisTaskLaunchManager; @@ -15,9 +14,9 @@ import com.webank.wedatasphere.exchangis.job.launcher.exception.ExchangisTaskLaunchException; import com.webank.wedatasphere.exchangis.job.log.LogQuery; import com.webank.wedatasphere.exchangis.job.log.LogResult; -import com.webank.wedatasphere.exchangis.job.server.dao.LaunchableTaskDao; -import com.webank.wedatasphere.exchangis.job.server.dao.LaunchedJobDao; -import com.webank.wedatasphere.exchangis.job.server.dao.LaunchedTaskDao; +import com.webank.wedatasphere.exchangis.job.server.mapper.LaunchableTaskDao; +import com.webank.wedatasphere.exchangis.job.server.mapper.LaunchedJobDao; +import com.webank.wedatasphere.exchangis.job.server.mapper.LaunchedTaskDao; import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisSchedulerException; import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisTaskGenerateException; @@ -162,12 +161,12 @@ public void killJob(String jobExecutionId) throws ExchangisJobServerException{ } @Override - public ExchangisLaunchedTaskMetricsVO getLaunchedTaskMetrics(String taskId, String jobExecutionId, String userName) throws ExchangisJobServerException { + public ExchangisLaunchedTaskMetricsVo getLaunchedTaskMetrics(String taskId, String jobExecutionId, String userName) throws ExchangisJobServerException { LaunchedExchangisTaskEntity launchedExchangisTaskEntity = launchedTaskDao.getLaunchedTaskMetrics(jobExecutionId, taskId); if (Objects.isNull(launchedExchangisTaskEntity) || !hasExecuteJobAuthority(jobExecutionId, userName)) { throw new ExchangisJobServerException(METRICS_OP_ERROR.getCode(), "Unable to find the launched job by [" + jobExecutionId + "]", null); } - ExchangisLaunchedTaskMetricsVO exchangisLaunchedTaskVo = new ExchangisLaunchedTaskMetricsVO(); + ExchangisLaunchedTaskMetricsVo exchangisLaunchedTaskVo = new ExchangisLaunchedTaskMetricsVo(); exchangisLaunchedTaskVo.setTaskId(launchedExchangisTaskEntity.getTaskId()); exchangisLaunchedTaskVo.setName(launchedExchangisTaskEntity.getName()); exchangisLaunchedTaskVo.setStatus(launchedExchangisTaskEntity.getStatus().name()); @@ -242,7 +241,7 @@ public ExchangisCategoryLogVo getTaskLogInfo(String taskId, String jobExecutionI } @Override - public List getExecutedJobList(String jobExecutionId, String jobName, String status, + public List getExecutedJobList(String jobExecutionId, String jobName, String status, Long launchStartTime, Long launchEndTime, int current, int size, HttpServletRequest request) throws ExchangisJobServerException{ if (current <= 0) { current = 1; @@ -251,7 +250,7 @@ public List getExecutedJobList(String jobExecutionId size = 10; } int start = (current - 1) * size; - List jobList = new ArrayList<>(); + List jobList = new ArrayList<>(); Date startTime = launchStartTime == null ? null : new Date(launchStartTime); Date endTime = launchEndTime == null ? null : new Date(launchEndTime); List jobEntitylist = launchedJobDao.getAllLaunchedJob(jobExecutionId, jobName, status, startTime, endTime, start, size, SecurityFilter.getLoginUsername(request)); @@ -259,12 +258,15 @@ public List getExecutedJobList(String jobExecutionId if(jobEntitylist != null) { try { for (LaunchedExchangisJobEntity launchedExchangisJobEntity : jobEntitylist) { - ExchangisLaunchedJobListVO exchangisJobVo = modelMapper.map(launchedExchangisJobEntity, ExchangisLaunchedJobListVO.class); + ExchangisLaunchedJobListVo exchangisJobVo = modelMapper.map(launchedExchangisJobEntity, ExchangisLaunchedJobListVo.class); if (launchedExchangisJobEntity.getExchangisJobEntity() == null || launchedExchangisJobEntity.getExchangisJobEntity().getSource() == null) { exchangisJobVo.setExecuteNode("-"); } else { Map sourceObject = Json.fromJson(launchedExchangisJobEntity.getExchangisJobEntity().getSource(), Map.class); - exchangisJobVo.setExecuteNode(sourceObject.get("execute_node").toString()); + if (Objects.nonNull(sourceObject)){ + exchangisJobVo.setExecuteNode(String.valueOf(sourceObject + .getOrDefault("executeNode", "-"))); + } } List launchedExchangisTaskEntities = launchedTaskDao.selectTaskListByJobExecutionId(launchedExchangisJobEntity.getJobExecutionId()); if (launchedExchangisTaskEntities == null) { @@ -277,7 +279,7 @@ public List getExecutedJobList(String jobExecutionId flows += 0; continue; } - Map flowMap = Json.fromJson(launchedExchangisTaskEntity.getMetricsMap().get("traffic").toString(), Map.class); + Map flowMap = (Map) launchedExchangisTaskEntity.getMetricsMap().get("traffic"); flows += flowMap == null ? 0 : Integer.parseInt(flowMap.get("flow").toString()); } exchangisJobVo.setFlow(taskNum == 0 ? 0 : (long) (flows / taskNum)); diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobInfoService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobInfoService.java index 98d00de22..d28076ea8 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobInfoService.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultJobInfoService.java @@ -1,4 +1,443 @@ package com.webank.wedatasphere.exchangis.job.server.service.impl; -public class DefaultJobInfoService { +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.github.pagehelper.PageHelper; +import com.github.pagehelper.PageInfo; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.webank.wedatasphere.exchangis.common.pager.PageResult; +import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobDsBind; +import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisDataSourceException; +import com.webank.wedatasphere.exchangis.datasource.core.ui.viewer.ExchangisDataSourceUIViewer; +import com.webank.wedatasphere.exchangis.datasource.core.utils.Json; +import com.webank.wedatasphere.exchangis.datasource.core.vo.ExchangisJobInfoContent; +import com.webank.wedatasphere.exchangis.datasource.service.ExchangisDataSourceService; +import com.webank.wedatasphere.exchangis.job.domain.ExchangisJobEntity; +import com.webank.wedatasphere.exchangis.job.server.dto.ExportedProject; +import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; +import com.webank.wedatasphere.exchangis.job.server.mapper.ExchangisJobEntityDao; +import com.webank.wedatasphere.exchangis.job.server.restful.external.ModuleEnum; +import com.webank.wedatasphere.exchangis.job.server.service.JobInfoService; +import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobQueryVo; +import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobVo; +import com.webank.wedatasphere.exchangis.project.server.service.ProjectService; +import com.webank.wedatasphere.exchangis.project.server.vo.ExchangisProjectInfo; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.bml.client.BmlClient; +import org.apache.linkis.bml.client.BmlClientFactory; +import org.apache.linkis.bml.protocol.BmlUploadResponse; +import org.apache.linkis.common.utils.JsonUtils; +import org.apache.linkis.manager.label.utils.LabelUtils; +import org.apache.linkis.server.BDPJettyServerHelper; +import org.apache.linkis.server.Message; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import javax.annotation.Resource; +import javax.servlet.http.HttpServletRequest; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.rmi.ServerException; +import java.util.*; +import java.util.stream.Collectors; + +/** + * Default implement + */ +@Service +public class DefaultJobInfoService implements JobInfoService { + + private static final Logger LOG = LoggerFactory.getLogger(DefaultJobInfoService.class); + + /** + * Project service + */ + @Resource + private ProjectService projectService; + + @Autowired + private ExchangisJobDsBindServiceImpl exchangisJobDsBindService; + + @Autowired + private ExchangisDataSourceService exchangisDataSourceService; + + @Resource + private ExchangisJobEntityDao jobEntityDao; + + @Resource + private JobInfoService jobInfoService; + + @Override + @Transactional(rollbackFor = Exception.class) + public ExchangisJobVo createJob(ExchangisJobVo jobVo) { + // Define the entity + /*ExchangisJobEntity jobEntity = new ExchangisJobEntity(); + jobEntity.setProjectId(jobVo.getProjectId()); + jobEntity.setJobType(jobVo.getJobType()); + jobEntity.setEngineType(jobVo.getEngineType()); + jobEntity.setJobLabels(jobVo.getJobLabels()); + jobEntity.setName(jobVo.getJobName()); + jobEntity.setJobDesc(jobVo.getJobDesc()); + jobEntity.setExecuteUser(jobVo.getExecuteUser()); + jobEntity.setJobParams(jobVo.getJobParams()); + jobEntity.setCreateUser(jobVo.getCreateUser()); + jobEntity.setCreateTime(Calendar.getInstance().getTime()); + jobEntity.setSource(Json.toJson(jobVo.getSource(), null)); + jobEntityDao.addJobEntity(jobEntity); + jobVo.setId(jobEntity.getId()); + jobVo.setCreateTime(jobEntity.getCreateTime()); + return jobVo;*/ + LOG.info("00005Sqoop job labels is: {}", jobVo.getJobLabels()); + ExchangisJobEntity jobEntity = new ExchangisJobEntity(); + jobEntity.setProjectId(jobVo.getProjectId()); + jobEntity.setJobType(jobVo.getJobType()); + jobEntity.setEngineType(jobVo.getEngineType()); + jobEntity.setJobLabel(jobVo.getJobLabels()); + jobEntity.setJobLabels(jobVo.getJobLabels()); + jobEntity.setName(jobVo.getJobName()); + jobEntity.setJobDesc(jobVo.getJobDesc()); + jobEntity.setExecuteUser(jobVo.getProxyUser()); + jobEntity.setJobParams(jobVo.getJobParams()); + jobEntity.setCreateUser(jobVo.getCreateUser()); + jobEntity.setCreateTime(Calendar.getInstance().getTime()); + jobEntity.setSource(Json.toJson(jobVo.getSource(), null)); + //jobEntity.setJobContent(jobVo.getContent()); + jobEntity.setModifyUser(jobVo.getModifyUser()); + LOG.info("Sqoop job Entity labels is: {}", jobEntity.getJobLabel()); + //Map contentVo = BDPJettyServerHelper.gson().fromJson(jobVo.getContent(), Map.class); + LOG.info("Sqoop job content is: {}, Modify user is: {}, jobType is: {}", jobVo.getContent(), jobEntity.getExecuteUser(), jobEntity.getJobType()); + if(jobVo.getContent() != null) { + jobEntity.setJobContent(jobVo.getContent()); + LOG.info("Sqoop job content is: {}, executor: {}", jobEntity.getJobContent(), jobEntity.getExecuteUser()); + } + jobEntityDao.addJobEntity(jobEntity); + jobVo.setId(jobEntity.getId()); + jobVo.setCreateTime(jobEntity.getCreateTime()); + return jobVo; + } + + @Override + @Transactional(rollbackFor = Exception.class) + public ExchangisJobVo updateJob(ExchangisJobVo jobVo) { + ExchangisJobEntity jobEntity = new ExchangisJobEntity(); + jobEntity.setId(jobVo.getId()); + jobEntity.setJobType(jobVo.getJobType()); + jobEntity.setEngineType(jobVo.getEngineType()); + jobEntity.setJobLabel(jobVo.getJobLabels()); + jobEntity.setName(jobVo.getJobName()); + jobEntity.setJobLabels(jobVo.getJobLabels()); + jobEntity.setJobDesc(jobVo.getJobDesc()); + jobEntity.setLastUpdateTime(Calendar.getInstance().getTime()); + jobEntity.setModifyUser(jobVo.getModifyUser()); + jobEntityDao.upgradeBasicInfo(jobEntity); + return jobVo; + } + + @Override + public PageResult queryJobList(ExchangisJobQueryVo queryVo){ + PageHelper.startPage(queryVo.getPage(), queryVo.getPageSize()); + try{ + List jobEntities = this.jobEntityDao.queryPageList(queryVo); + PageInfo pageInfo = new PageInfo<>(jobEntities); + List infoList = jobEntities + .stream().map(ExchangisJobVo::new).collect(Collectors.toList()); + PageResult pageResult = new PageResult<>(); + pageResult.setList(infoList); + pageResult.setTotal(pageInfo.getTotal()); + return pageResult; + }finally { + PageHelper.clearPage(); + } + } + + @Override + @Transactional(rollbackFor = Exception.class) + public void deleteJob(Long id) { + this.jobEntityDao.deleteJobEntity(id); + this.exchangisJobDsBindService.updateJobDsBind(id, new ArrayList<>()); + } + + @Override + public ExchangisJobVo getJob(Long id, boolean basic) { + ExchangisJobEntity exchangisJob = basic ? this.jobEntityDao.getBasicInfo(id) : this.jobEntityDao.getDetail(id); + ExchangisJobVo jobVo = new ExchangisJobVo(exchangisJob); + jobVo.setProjectId(exchangisJob.getProjectId()); + if (exchangisJob != null && StringUtils.isNotBlank(exchangisJob.getJobContent())) { + jobVo.setContent(exchangisJob.getJobContent()); + jobVo.setSource(Objects.nonNull(exchangisJob.getSource())? + Json.fromJson(exchangisJob.getSource(), Map.class, String.class, Object.class) : new HashMap<>()); + } + return jobVo; + } + + @Override + public List getByNameWithProjectId(String jobName, Long projectId) { + List exchangisJobs = this.jobEntityDao.getByNameWithProjectId(jobName, projectId); + List exchangisJobVos = new ArrayList<>(); + for(ExchangisJobEntity exchangisJob : exchangisJobs){ + ExchangisJobVo jobVo = new ExchangisJobVo(exchangisJob); + if (exchangisJob != null && StringUtils.isNotBlank(exchangisJob.getJobContent())) { + jobVo.setContent(exchangisJob.getJobContent()); + jobVo.setSource(Objects.nonNull(exchangisJob.getSource())? + Json.fromJson(exchangisJob.getSource(), Map.class, String.class, Object.class) : new HashMap<>()); + } + exchangisJobVos.add(jobVo); + } + + return exchangisJobVos; + } + @Override + public ExchangisJobVo getDecoratedJob(HttpServletRequest request, Long id) throws ExchangisJobServerException { + ExchangisJobEntity exchangisJob = this.jobEntityDao.getDetail(id); + ExchangisJobVo jobVo = new ExchangisJobVo(exchangisJob); + if (exchangisJob != null && StringUtils.isNotBlank(exchangisJob.getJobContent())) { + // Rebuild the job content with ui configuration + List jobDataSourceUIs = exchangisDataSourceService.getJobDataSourceUIs(request, id); + ObjectMapper objectMapper = JsonUtils.jackson(); + try { + String content = objectMapper.writeValueAsString(jobDataSourceUIs); + JsonNode contentJsonNode = objectMapper.readTree(content); + ObjectNode objectNode = objectMapper.createObjectNode(); + objectNode.set("subJobs", contentJsonNode); + jobVo.setContent(objectNode.toString()); + jobVo.setSource(Objects.nonNull(exchangisJob.getSource())? + Json.fromJson(exchangisJob.getSource(), Map.class, String.class, Object.class) : new HashMap<>()); + } catch (JsonProcessingException e) { + throw new ExchangisJobServerException(31100, + "Fail to rebuild the job content with ui (渲染任务内容失败)", e); + } + } + return jobVo; + } + + @Override + public List getSubJobList(HttpServletRequest request, Long projectId) throws ExchangisJobServerException{ + List exchangisJobList = this.jobEntityDao.getDetailList(projectId); + List exchangisJobVos = new ArrayList<>(); + if(!exchangisJobList.isEmpty()) { + for(ExchangisJobEntity exchangisJob : exchangisJobList){ + ExchangisJobVo jobVo = new ExchangisJobVo(exchangisJob); + if(StringUtils.isNotBlank(exchangisJob.getJobContent())){ + List jobDataSourceUIs = exchangisDataSourceService.getJobDataSourceUIs(request, exchangisJob.getId()); + ObjectMapper objectMapper = JsonUtils.jackson(); + try { + String content = objectMapper.writeValueAsString(jobDataSourceUIs); + JsonNode contentJsonNode = objectMapper.readTree(content); + ObjectNode objectNode = objectMapper.createObjectNode(); + objectNode.set("subJobs", contentJsonNode); + jobVo.setContent(objectNode.toString()); + jobVo.setSource(Objects.nonNull(exchangisJob.getSource())? + Json.fromJson(exchangisJob.getSource(), Map.class, String.class, Object.class) : new HashMap<>()); + } catch (JsonProcessingException e) { + throw new ExchangisJobServerException(31100, + "Fail to rebuild the job content with ui (渲染任务内容失败)", e); + } + } + exchangisJobVos.add(jobVo); + } + } + return exchangisJobVos; + } + + @Override + @Transactional(rollbackFor = Exception.class) + public ExchangisJobVo updateJobConfig(ExchangisJobVo jobVo) { + ExchangisJobEntity jobEntity = this.jobEntityDao.getBasicInfo(jobVo.getId()); + Map sourceMap = StringUtils.isNotBlank(jobEntity.getSource())? + Json.fromJson(jobEntity.getSource(), Map.class, String.class, Object.class) : null; + jobEntity.setExecuteUser(jobVo.getProxyUser()); + jobEntity.setJobParams(jobVo.getJobParams()); + if (Objects.isNull(sourceMap)){ + sourceMap = new HashMap<>(); + } + sourceMap.putAll(jobVo.getSource()); + jobEntity.setSource(Json.toJson(sourceMap, null)); + jobEntity.setModifyUser(jobVo.getModifyUser()); + jobEntity.setLastUpdateTime(Calendar.getInstance().getTime()); + this.jobEntityDao.upgradeConfig(jobEntity); + return jobVo; + } + + @Override + @Transactional(rollbackFor = Exception.class) + public ExchangisJobVo updateJobContent(ExchangisJobVo jobVo) throws ExchangisJobServerException, ExchangisDataSourceException { + Long jobId = jobVo.getId(); + ExchangisJobEntity exchangisJob = this.jobEntityDao.getDetail(jobId); + exchangisJob.setJobContent(jobVo.getContent()); + final String engine = exchangisJob.getEngineType(); + // 校验是否有重复子任务名 + List content = LabelUtils.Jackson.fromJson(exchangisJob.getJobContent(), + List.class, ExchangisJobInfoContent.class); + long count = content.stream().map(ExchangisJobInfoContent::getSubJobName).distinct().count(); + if (count < content.size()) { + throw new ExchangisJobServerException(31101, "Already exits duplicated job name(存在重复子任务名)"); + } + List dsBinds = new ArrayList<>(content.size()); + // 校验引擎是否支持该数据通道 + for (int i = 0; i < content.size(); i++) { + ExchangisJobInfoContent task = content.get(i); + String sourceType = task.getDataSources().getSourceId().split("\\.")[0]; + String sinkType = task.getDataSources().getSinkId().split("\\.")[0]; + this.exchangisDataSourceService.checkDSSupportDegree(engine, sourceType, sinkType); + ExchangisJobDsBind dsBind = new ExchangisJobDsBind(); + dsBind.setJobId(jobVo.getId()); + dsBind.setTaskIndex(i); + dsBind.setSourceDsId(Long.parseLong(task.getDataSources().getSourceId().split("\\.")[1])); + dsBind.setSinkDsId(Long.parseLong(task.getDataSources().getSinkId().split("\\.")[1])); + dsBinds.add(dsBind); + } + exchangisJob.setModifyUser(jobVo.getModifyUser()); + exchangisJob.setLastUpdateTime(jobVo.getModifyTime()); + this.exchangisJobDsBindService.updateJobDsBind(jobId, dsBinds); + this.jobEntityDao.upgradeContent(exchangisJob); + return jobVo; + } + + @Override + public Message exportProject(Map params, String userName, HttpServletRequest request) throws ExchangisJobServerException, ServerException { + ExportedProject exportedProject = null; + Long projectId = Long.parseLong(params.get("projectId").toString()); + Boolean partial = (Boolean) params.get("partial"); + Map> moduleIdsMap = getModuleIdsMap(params); + + LOG.info("export project, user: {}, project: {}, partial:{}", userName, projectId, partial); + exportedProject = export(projectId, moduleIdsMap, partial, request); + String exported = null; + try { + exported = BDPJettyServerHelper.jacksonJson().writeValueAsString(exportedProject); + } catch (JsonProcessingException e) { + LOG.error("Occur error while tranform class", e.getMessage()); + } + + LOG.info("projectName: {}, exported:{}", exportedProject.getName(), exported); + BmlClient bmlClient = BmlClientFactory.createBmlClient(userName); + BmlUploadResponse bmlUploadResponse = bmlClient.uploadShareResource(userName, exportedProject.getName(), + "exchangis_exported_" + UUID.randomUUID(), new ByteArrayInputStream(exported.getBytes(StandardCharsets.UTF_8))); + + if (bmlUploadResponse == null || !bmlUploadResponse.isSuccess()) { + throw new ServerException("cannot upload exported data to BML"); + } + + LOG.info("{} is exporting the project, uploaded to BML the resourceID is {} and the version is {}", + userName, bmlUploadResponse.resourceId(), bmlUploadResponse.version()); + + Message message = Message.ok("export job") + .data("resourceId", bmlUploadResponse.resourceId()) + .data("version", bmlUploadResponse.version()); + return message; + } + + @Override + public ExportedProject export(Long projectId, Map> moduleIdsMap, boolean partial, HttpServletRequest request) throws ExchangisJobServerException { + ExportedProject exportedProject = new ExportedProject(); + ExchangisProjectInfo project = projectService.getProjectDetailById(projectId); + + LOG.info("execute export method! export project is {}.", project.getName()); + exportedProject.setName(project.getName()); + + setSqoop(projectId, moduleIdsMap, partial, exportedProject, request); + + setDatax(projectId, moduleIdsMap, partial, exportedProject, request); + + return exportedProject; + } + + private void setSqoop(Long projectId, Map> moduleIdsMap, boolean partial, ExportedProject exportedProject, HttpServletRequest request) throws ExchangisJobServerException { + List sqoops = new ArrayList<>(); + LOG.info("Request: {}", request); + if (partial) { + Set longs = moduleIdsMap.get(ModuleEnum.SQOOP_IDS.getName()); + if (longs.size() > 0) { + for (Long id : longs) { + LOG.info("id: {}", id); + ExchangisJobVo job = jobInfoService.getJob(id, false); + + String sqoopStr = null; + try { + sqoopStr = BDPJettyServerHelper.jacksonJson().writeValueAsString(job); + } catch (JsonProcessingException e) { + LOG.error("Occur error while tranform class", e.getMessage()); + } + + LOG.info("sqoopStr99999:{}", sqoopStr); + LOG.info("ExchangisJobVo sqoop: {}", job.getContent()); + LOG.info("getCreateTimep: {}", job.getId()); + LOG.info("executorUser999: {}", job.getExecuteUser()); + sqoops.add(job); + } + exportedProject.setSqoops(sqoops); + } + + } else { + LOG.info("Through request {} and projectId {} get Sqoopjob", request, projectId); + sqoops = jobInfoService.getSubJobList(request, projectId); + exportedProject.setSqoops(sqoops); + //exportedProject.setSqoops(jobInfoService.getByProject(request, projectId)); + } + LOG.info("exporting project, export sqoopJob: {}", exportedProject); + } + + private void setDatax(Long projectId, Map> moduleIdsMap, boolean partial, ExportedProject exportedProject, HttpServletRequest request) throws ExchangisJobServerException { + List datax = new ArrayList<>(); + if (partial) { + Set longs = moduleIdsMap.get(ModuleEnum.DATAX_IDS.getName()); + if (longs.size() > 0) { + for (Long id : longs) { + ExchangisJobVo job = jobInfoService.getDecoratedJob(request, id); + datax.add(job); + } + exportedProject.setDataxes(datax); + } + + } else { + LOG.info("Through request {} and projectId {} get datax", request, projectId); + datax = jobInfoService.getSubJobList(request, projectId); + exportedProject.setSqoops(datax); + //exportedProject.setDataxes(jobInfoService.getByProject(request, projectId)); + } + LOG.info("exporting project, export datax: {}", exportedProject); + } + + /** + * 获取需要导出对象集合 + * + * @param params + * @return + */ + @Override + public Map> getModuleIdsMap(Map params) { + + Map> map = Maps.newHashMap(); + String sqoopIdsStr = null; + if(params.get("sqoopIds") != null) { + sqoopIdsStr = params.get("sqoopIds").toString(); + } + String dataxIdsStr = null; + if(params.get("dataxIds") != null) { + dataxIdsStr = params.get("dataxIds").toString(); + } + + Set sqoopIds = Sets.newHashSet(); + Set dataxIds = Sets.newHashSet(); + + if (StringUtils.isNotEmpty(sqoopIdsStr)) { + sqoopIds = Arrays.stream(StringUtils.split(sqoopIdsStr, ",")) + .map(Long::parseLong).collect(Collectors.toSet()); + } + if (StringUtils.isNotEmpty(dataxIdsStr)) { + dataxIds = Arrays.stream(StringUtils.split(dataxIdsStr, ",")) + .map(Long::parseLong).collect(Collectors.toSet()); + } + map.put("sqoopIds", sqoopIds); + map.put("dataxIds", dataxIds); + LOG.info("The objects to be exported are: {}", map); + return map; + } } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultTaskExecuteService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultTaskExecuteService.java index 491e3191c..d32248478 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultTaskExecuteService.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultTaskExecuteService.java @@ -6,8 +6,8 @@ import com.webank.wedatasphere.exchangis.job.launcher.domain.LaunchedExchangisTask; import com.webank.wedatasphere.exchangis.job.launcher.domain.task.TaskStatus; import com.webank.wedatasphere.exchangis.job.launcher.entity.LaunchedExchangisJobEntity; -import com.webank.wedatasphere.exchangis.job.server.dao.LaunchedJobDao; -import com.webank.wedatasphere.exchangis.job.server.dao.LaunchedTaskDao; +import com.webank.wedatasphere.exchangis.job.server.mapper.LaunchedJobDao; +import com.webank.wedatasphere.exchangis.job.server.mapper.LaunchedTaskDao; import com.webank.wedatasphere.exchangis.job.server.execution.events.*; import com.webank.wedatasphere.exchangis.job.server.log.cache.JobLogCacheUtils; import com.webank.wedatasphere.exchangis.job.server.service.TaskExecuteService; @@ -15,7 +15,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; import javax.annotation.Resource; diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultTaskGenerateService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultTaskGenerateService.java index f7aed439f..f4a6aa44d 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultTaskGenerateService.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultTaskGenerateService.java @@ -4,19 +4,17 @@ import com.webank.wedatasphere.exchangis.job.launcher.domain.LaunchableExchangisTask; import com.webank.wedatasphere.exchangis.job.launcher.domain.task.TaskStatus; import com.webank.wedatasphere.exchangis.job.launcher.entity.LaunchedExchangisJobEntity; -import com.webank.wedatasphere.exchangis.job.server.dao.LaunchableTaskDao; -import com.webank.wedatasphere.exchangis.job.server.dao.LaunchedJobDao; +import com.webank.wedatasphere.exchangis.job.server.mapper.LaunchableTaskDao; +import com.webank.wedatasphere.exchangis.job.server.mapper.LaunchedJobDao; import com.webank.wedatasphere.exchangis.job.server.execution.generator.events.TaskGenerateErrorEvent; import com.webank.wedatasphere.exchangis.job.server.execution.generator.events.TaskGenerateInitEvent; import com.webank.wedatasphere.exchangis.job.server.execution.generator.events.TaskGenerateSuccessEvent; import com.webank.wedatasphere.exchangis.job.server.execution.subscriber.NewInTaskObserver; import com.webank.wedatasphere.exchangis.job.server.log.cache.JobLogCacheUtils; import com.webank.wedatasphere.exchangis.job.server.service.TaskGenerateService; -import com.webank.wedatasphere.exchangis.job.utils.SnowFlake; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import javax.annotation.PostConstruct; import javax.annotation.Resource; import java.util.Calendar; import java.util.List; diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultTaskObserverService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultTaskObserverService.java index bc8a1665f..6c9a8b60e 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultTaskObserverService.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/DefaultTaskObserverService.java @@ -2,8 +2,8 @@ import com.webank.wedatasphere.exchangis.job.launcher.domain.LaunchableExchangisTask; import com.webank.wedatasphere.exchangis.job.launcher.entity.LaunchedExchangisTaskEntity; -import com.webank.wedatasphere.exchangis.job.server.dao.LaunchableTaskDao; -import com.webank.wedatasphere.exchangis.job.server.dao.LaunchedTaskDao; +import com.webank.wedatasphere.exchangis.job.server.mapper.LaunchableTaskDao; +import com.webank.wedatasphere.exchangis.job.server.mapper.LaunchedTaskDao; import com.webank.wedatasphere.exchangis.job.server.service.TaskObserverService; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ExchangisJobServiceImpl.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ExchangisJobServiceImpl.java deleted file mode 100644 index 3d3eb285d..000000000 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ExchangisJobServiceImpl.java +++ /dev/null @@ -1,372 +0,0 @@ -package com.webank.wedatasphere.exchangis.job.server.service.impl; - -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.conditions.query.LambdaQueryChainWrapper; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.base.Strings; -import com.google.gson.JsonArray; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobDsBind; -import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisDataSourceException; -import com.webank.wedatasphere.exchangis.datasource.core.ui.ElementUI; -import com.webank.wedatasphere.exchangis.datasource.core.ui.InputElementUI; -import com.webank.wedatasphere.exchangis.datasource.core.ui.OptionElementUI; -import com.webank.wedatasphere.exchangis.datasource.core.ui.viewer.ExchangisDataSourceUIViewer; -import com.webank.wedatasphere.exchangis.datasource.core.vo.ExchangisJobInfoContent; -import com.webank.wedatasphere.exchangis.datasource.service.ExchangisDataSourceService; -import com.webank.wedatasphere.exchangis.job.server.dto.ExchangisJobBasicInfoDTO; -import com.webank.wedatasphere.exchangis.job.server.dto.ExchangisJobContentDTO; -import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; -import com.webank.wedatasphere.exchangis.job.server.mapper.ExchangisJobMapper; -import com.webank.wedatasphere.exchangis.job.server.service.ExchangisJobService; -import com.webank.wedatasphere.exchangis.job.server.vo.ExchangisJobBasicInfoVO; -import com.webank.wedatasphere.exchangis.job.server.vo.ExchangisTaskSpeedLimitVO; -import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobVO; -import org.apache.commons.lang.StringUtils; -import org.apache.linkis.common.utils.JsonUtils; -import org.apache.linkis.manager.label.utils.LabelUtils; -import org.apache.linkis.server.security.SecurityFilter; -import org.modelmapper.ModelMapper; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.web.multipart.MultipartFile; - -import javax.servlet.http.HttpServletRequest; -import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - *

- * 服务实现类 - *

- * - * @author yuxin.yuan - * @since 2021-08-10 - */ -@Service -public class ExchangisJobServiceImpl extends ServiceImpl implements ExchangisJobService { - - @Autowired - private ModelMapper modelMapper; - - @Autowired - private ExchangisJobService exchangisJobService; - - @Autowired - private ExchangisJobDsBindServiceImpl exchangisJobDsBindService; - - @Autowired - private ExchangisDataSourceService exchangisDataSourceService; - - @Override - public ExchangisJobBasicInfoVO createJob(HttpServletRequest request, ExchangisJobBasicInfoDTO exchangisJobBasicInfoDTO) { - ExchangisJobVO exchangisJob = modelMapper.map(exchangisJobBasicInfoDTO, ExchangisJobVO.class); - Calendar calendar = Calendar.getInstance(); - String createUser = ""; - try { - createUser = SecurityFilter.getLoginUsername(request); - } catch (Exception e) { - log.error("Get create user error.", e); - } - exchangisJob.setCreateUser(createUser); - exchangisJob.setCreateTime(calendar.getTime()); - exchangisJobService.save(exchangisJob); - return modelMapper.map(exchangisJob, ExchangisJobBasicInfoVO.class); - } - - @Override - public List getJobList(long projectId, String type, String name, int current, int size) { - LambdaQueryChainWrapper query = - exchangisJobService.lambdaQuery().eq(ExchangisJobVO::getProjectId, projectId); - if (StringUtils.isNotBlank(type)) { - query.eq(ExchangisJobVO::getJobType, type); - } - if (StringUtils.isNotBlank(name)) { - query.like(ExchangisJobVO::getJobName, name.trim()); - } - List exchangisJobs = query.list(); - - Stream returnlist = Optional.ofNullable(exchangisJobs).orElse(new ArrayList<>()).stream() - .map(job -> modelMapper.map(job, ExchangisJobBasicInfoVO.class)); - - if (current <= 0) { - current = 1; - } - if (size <= 0) { - size = 10; - } - int start = (current - 1) * size; - int last = 0; - if(current * size > exchangisJobs.size()) { - last = exchangisJobs.size(); - } - else { - last = current * size; - } - return returnlist.collect(Collectors.toList()).subList(start, last); - } - - @Override - public int count(long projectId, String type, String name) { - LambdaQueryChainWrapper query = - exchangisJobService.lambdaQuery().eq(ExchangisJobVO::getProjectId, projectId); - if (StringUtils.isNotBlank(type)) { - query.eq(ExchangisJobVO::getJobType, type); - } - if (StringUtils.isNotBlank(name)) { - query.like(ExchangisJobVO::getJobName, name.trim()); - } - return query.count(); - } - - @Override - public List getJobListByDssProject(long dssProjectId, String type, String name) { - LambdaQueryChainWrapper query = - exchangisJobService.lambdaQuery().eq(ExchangisJobVO::getDssProjectId, dssProjectId); - if (StringUtils.isNotBlank(type)) { - query.eq(ExchangisJobVO::getJobType, type); - } - if (StringUtils.isNotBlank(name)) { - query.like(ExchangisJobVO::getJobName, name.trim()); - } - List exchangisJobs = query.list(); - - Stream returnlist = Optional.ofNullable(exchangisJobs).orElse(new ArrayList<>()).stream() - .map(job -> modelMapper.map(job, ExchangisJobBasicInfoVO.class)); - return returnlist.collect(Collectors.toList()); - } - - @Override - public ExchangisJobBasicInfoVO copyJob(ExchangisJobBasicInfoDTO exchangisJobBasicInfoDTO, Long sourceJobId) { - ExchangisJobVO oldJob = exchangisJobService.getById(sourceJobId); - ExchangisJobVO newJob = modelMapper.map(exchangisJobBasicInfoDTO, ExchangisJobVO.class); - newJob.setProjectId(oldJob.getProjectId()); - newJob.setJobType(oldJob.getJobType()); - newJob.setEngineType(oldJob.getEngineType()); - exchangisJobService.save(newJob); - return modelMapper.map(newJob, ExchangisJobBasicInfoVO.class); - } - - @Override - public ExchangisJobBasicInfoVO updateJob(ExchangisJobBasicInfoDTO exchangisJobBasicInfoDTO, Long id) { - ExchangisJobVO job = exchangisJobService.getById(id); - job.setJobName(exchangisJobBasicInfoDTO.getJobName()); - job.setJobLabels(exchangisJobBasicInfoDTO.getJobLabels()); - job.setJobDesc(exchangisJobBasicInfoDTO.getJobDesc()); - job.setJobType(exchangisJobBasicInfoDTO.getJobType().toString()); - job.setEngineType(exchangisJobBasicInfoDTO.getEngineType()); - exchangisJobService.updateById(job); - - return modelMapper.map(job, ExchangisJobBasicInfoVO.class); - } - - @Override - public ExchangisJobBasicInfoVO updateJobByDss(ExchangisJobBasicInfoDTO exchangisJobBasicInfoDTO, String nodeId) { - Optional optional = this.getByNodeId(nodeId); - if (optional.isPresent()) { - ExchangisJobVO job = optional.get(); - job.setJobName(exchangisJobBasicInfoDTO.getJobName()); - job.setJobLabels(exchangisJobBasicInfoDTO.getJobLabels()); - job.setJobDesc(exchangisJobBasicInfoDTO.getJobDesc()); - exchangisJobService.updateById(job); - return modelMapper.map(job, ExchangisJobBasicInfoVO.class); - } - return null; - } - - private Optional getByNodeId(String nodeId) { - ExchangisJobVO job = null; - LambdaQueryChainWrapper query = - exchangisJobService.lambdaQuery().eq(ExchangisJobVO::getNodeId, nodeId); - List exchangisJobs = query.list(); - if (null != exchangisJobs && exchangisJobs.size() == 1) { - job = exchangisJobs.get(0); - } - return Optional.ofNullable(job); - } - - @Override - public ExchangisJobBasicInfoVO importSingleJob(MultipartFile multipartFile) { - - return null; - } - - @Override - public void deleteJob(Long id) { - exchangisJobService.removeById(id); - this.exchangisJobDsBindService.updateJobDsBind(id, new ArrayList<>()); - } - - @Override - public void deleteJobByDss(String nodeId) { - this.getByNodeId(nodeId).ifPresent(job -> { - exchangisJobService.removeById(job.getId()); - exchangisJobDsBindService.updateJobDsBind(job.getId(), new ArrayList<>()); - }); - } - - public ExchangisJobVO getJob(Long id) throws ExchangisJobServerException { - return this.getJob(null, id); - } - - @Override - public ExchangisJobVO getJob(HttpServletRequest request, Long id) throws ExchangisJobServerException { - ExchangisJobVO exchangisJob = exchangisJobService.getById(id); - if (exchangisJob != null) { - // generate subjobs ui content - List jobDataSourceUIs = exchangisDataSourceService.getJobDataSourceUIs(request, id); - ObjectMapper objectMapper = JsonUtils.jackson(); - try { - String content = objectMapper.writeValueAsString(jobDataSourceUIs); - JsonNode contentJsonNode = objectMapper.readTree(content); - ObjectNode objectNode = objectMapper.createObjectNode(); - objectNode.set("subJobs", contentJsonNode); - exchangisJob.setContent(objectNode.toString()); - } catch (JsonProcessingException e) { - throw new ExchangisJobServerException(31100, "exchangis.subjob.ui.create.error", e); - } - } - return exchangisJob; - } - - @Override - public ExchangisJobVO getJobByDss(HttpServletRequest request, String nodeId) throws ExchangisJobServerException { - - Optional optional = this.getByNodeId(nodeId); - if (optional.isPresent()) { - ExchangisJobVO exchangisJob = optional.get(); - List jobDataSourceUIs = exchangisDataSourceService.getJobDataSourceUIs(request, exchangisJob.getId()); - ObjectMapper objectMapper = JsonUtils.jackson(); - try { - String content = objectMapper.writeValueAsString(jobDataSourceUIs); - JsonNode contentJsonNode = objectMapper.readTree(content); - ObjectNode objectNode = objectMapper.createObjectNode(); - objectNode.set("subJobs", contentJsonNode); - exchangisJob.setContent(objectNode.toString()); - } catch (JsonProcessingException e) { - throw new ExchangisJobServerException(31100, "exchangis.subjob.ui.create.error", e); - } - return exchangisJob; - } - - return null; - } - - - - @Override - public ExchangisJobVO updateJobConfig(ExchangisJobContentDTO exchangisJobContentDTO, Long id) - throws ExchangisJobServerException { - ExchangisJobVO exchangisJob = exchangisJobService.getById(id); - exchangisJob.setProxyUser(exchangisJobContentDTO.getProxyUser()); - exchangisJob.setExecuteNode(exchangisJobContentDTO.getExecuteNode()); - exchangisJob.setSyncType(exchangisJobContentDTO.getSyncType()); - exchangisJob.setJobParams(exchangisJobContentDTO.getJobParams()); - exchangisJobService.updateById(exchangisJob); - - return this.getJob(id); - } - - @Override - public ExchangisJobVO updateJobContent(ExchangisJobContentDTO exchangisJobContentDTO, Long id) - throws ExchangisJobServerException, ExchangisDataSourceException { - ExchangisJobVO exchangisJob = exchangisJobService.getById(id); - final String engine = exchangisJob.getEngineType(); - - // 校验是否有重复子任务名 - List content = LabelUtils.Jackson.fromJson(exchangisJobContentDTO.getContent(), List.class, ExchangisJobInfoContent.class); - long count = content.stream().map(ExchangisJobInfoContent::getSubJobName).distinct().count(); - if (count < content.size()) { - throw new ExchangisJobServerException(31101, "存在重复子任务名"); - } - - List dsBinds = new ArrayList<>(content.size()); - - // 校验引擎是否支持该数据通道 - - for (int i = 0; i < content.size(); i++) { - ExchangisJobInfoContent task = content.get(i); - String sourceType = task.getDataSources().getSourceId().split("\\.")[0]; - String sinkType = task.getDataSources().getSinkId().split("\\.")[0]; - this.exchangisDataSourceService.checkDSSupportDegree(engine, sourceType, sinkType); - ExchangisJobDsBind dsBind = new ExchangisJobDsBind(); - dsBind.setJobId(id); - dsBind.setTaskIndex(i); - dsBind.setSourceDsId(Long.parseLong(task.getDataSources().getSourceId().split("\\.")[1])); - dsBind.setSinkDsId(Long.parseLong(task.getDataSources().getSinkId().split("\\.")[1])); - dsBinds.add(dsBind); - } - - this.exchangisJobDsBindService.updateJobDsBind(id, dsBinds); - - exchangisJob.setContent(exchangisJobContentDTO.getContent()); - exchangisJobService.updateById(exchangisJob); - return this.getJob(id); - } - - @Override - public List> getSpeedLimitSettings(Long id, String taskName) { - ExchangisJobVO exchangisJob = exchangisJobService.getById(id); - Map values = new HashMap<>(); - if (null != exchangisJob && null != exchangisJob.getContent() && !"".equals(exchangisJob.getContent())) { - List o = LabelUtils.Jackson.fromJson(exchangisJob.getContent(), List.class, ExchangisJobInfoContent.class); - o.forEach(task -> { - if (task.getSubJobName().equals(taskName)) { - Optional.ofNullable(task.getSettings()).orElse(new ArrayList<>()).forEach(setting -> { - values.put(setting.getConfigKey(), setting.getConfigValue()); - }); - } - }); - } - - List> jobEngineSettingsUI = this.exchangisDataSourceService.getJobEngineSettingsUI(exchangisJob.getEngineType()); - jobEngineSettingsUI.forEach(s -> { - if (values.containsKey(s.getField())) { - if (s instanceof InputElementUI) { - InputElementUI e = (InputElementUI) s; - e.setValue(String.valueOf(values.get(s.getField()))); - } - if (s instanceof OptionElementUI) { - OptionElementUI e = (OptionElementUI) s; - e.setValue(String.valueOf(values.get(s.getField()))); - } - } - }); - - return jobEngineSettingsUI; - - } - - @Override - public void setSpeedLimitSettings(Long id, String taskName, ExchangisTaskSpeedLimitVO settings) { - ExchangisJobVO exchangisJob = exchangisJobService.getById(id); - JsonArray content = new JsonParser().parse(exchangisJob.getContent()).getAsJsonArray(); - JsonArray newSet = new JsonArray(); - settings.getSettings().forEach(s -> { - JsonObject json = new JsonObject(); - json.addProperty("config_key", s.getConfig_key()); - json.addProperty("config_name", s.getConfig_name()); - json.addProperty("config_value", s.getConfig_value()); - json.addProperty("sort", s.getSort()); - newSet.add(json); - }); - content.forEach(c -> { - JsonObject task = c.getAsJsonObject(); - if (task.get("subJobName").getAsString().equals(taskName)) { - task.remove("settings"); - task.add("settings", newSet); - } - }); - exchangisJob.setContent(content.toString()); - exchangisJobService.updateById(exchangisJob); - } - -} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ExchangisLaunchTaskServiceImpl.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ExchangisLaunchTaskServiceImpl.java deleted file mode 100644 index 45f32812f..000000000 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ExchangisLaunchTaskServiceImpl.java +++ /dev/null @@ -1,71 +0,0 @@ -package com.webank.wedatasphere.exchangis.job.server.service.impl; - -import java.util.Date; -import java.util.List; -import java.util.stream.Collectors; - -import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisDataSourceExceptionCode; -import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; -import org.modelmapper.ModelMapper; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; -import com.webank.wedatasphere.exchangis.job.launcher.entity.ExchangisLaunchTask; -import com.webank.wedatasphere.exchangis.job.server.mapper.ExchangisLaunchTaskMapper; -import com.webank.wedatasphere.exchangis.job.server.service.ExchangisLaunchTaskService; -import com.webank.wedatasphere.exchangis.job.server.vo.ExchangisTaskInfoVO; - -@Service -public class ExchangisLaunchTaskServiceImpl extends ServiceImpl - implements ExchangisLaunchTaskService { - - @Autowired - private ModelMapper modelMapper; - - @Autowired - private ExchangisLaunchTaskService exchangisLaunchTaskService; - - @Override - public List listTasks(Long taskId, String taskName, String status, Long launchStartTime, - Long launchEndTime, int current, int size) throws ExchangisJobServerException{ - if (current <= 0) { - current = 1; - } - if (size <= 0) { - size = 10; - } - int start = (current - 1) * size; - - Date startTime = launchStartTime == null ? null : new Date(launchStartTime); - Date endTime = launchEndTime == null ? null : new Date(launchEndTime); - List taskList = - this.baseMapper.listTasks(taskId, taskName, status, startTime, endTime, start, size); - - return taskList.stream().map(task -> { - return modelMapper.map(task, ExchangisTaskInfoVO.class); - }).collect(Collectors.toList()); - } - - @Override - public int count(Long taskId, String taskName, String status, Long launchStartTime, Long launchEndTime) { - Date startTime = launchStartTime == null ? null : new Date(launchStartTime); - Date endTime = launchEndTime == null ? null : new Date(launchEndTime); - - return this.baseMapper.count(taskId, taskName, status, startTime, endTime); - } - - @Override - public void delete(Long historyId) throws Exception { - ExchangisLaunchTask task = this.baseMapper.selectById(historyId); - if (null == task) { - throw new ExchangisJobServerException(ExchangisDataSourceExceptionCode.DELETE_HISTORY_ERROR.getCode(), "Task " + historyId + " not exists."); - } - if (task.getStatus().equals("SUCCESS") || task.getStatus().equals("FAILED")) { - this.baseMapper.deleteById(historyId); - } else { - throw new ExchangisJobServerException(ExchangisDataSourceExceptionCode.DELETE_HISTORY_ERROR.getCode(), "The status of task " + historyId + " is " + task.getStatus() + ", " + - "only 'SUCCESS' or 'FAILED' status can be deleted."); - } - } -} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ExchangisMetricsServiceImpl.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ExchangisMetricsServiceImpl.java index 8160a3ede..a29c9bc71 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ExchangisMetricsServiceImpl.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ExchangisMetricsServiceImpl.java @@ -43,7 +43,7 @@ public Message getTaskStateMetrics(HttpServletRequest request) { Optional.ofNullable(unlock).ifPresent(metrices::add); Message message = Message.ok(); - message.setMethod("/exchangis/metrics/taskstate"); + message.setMethod("/dss/exchangis/main/metrics/taskstate"); message.data("metrices", metrices); return message; } @@ -94,7 +94,7 @@ public Message getTaskProcessMetrics(HttpServletRequest request) { list.add(fps); Message message = Message.ok(); - message.setMethod("/exchangis/metrics/taskprocess"); + message.setMethod("/dss/exchangis/main/metrics/taskprocess"); message.data("list", list); return message; } @@ -174,7 +174,7 @@ public Message getDataSourceFlowMetrics(HttpServletRequest request) { // dataset.add(ds2Data); // dataset.add(ds3Data); Message message = Message.ok(); - message.setMethod("/exchangis/metrics/datasourceflow"); + message.setMethod("/dss/exchangis/main/metrics/datasourceflow"); message.data("dataset", dataset); return message; } @@ -233,7 +233,7 @@ public Message getEngineResourceCpuMetrics(HttpServletRequest request) { // dataset.add(ds3Data); // dataset.add(realData); Message message = Message.ok(); - message.setMethod("/exchangis/metrics/engineresourcecpu"); + message.setMethod("/dss/exchangis/main/metrics/engineresourcecpu"); message.data("dataset", dataset); return message; @@ -312,7 +312,7 @@ public Message getEngineResourceMemMetrics(HttpServletRequest request) { // dataset.add(ds2Data); // dataset.add(ds3Data); Message message = Message.ok(); - message.setMethod("/exchangis/metrics/engineresourcemem"); + message.setMethod("/dss/exchangis/main/metrics/engineresourcemem"); message.data("dataset", dataset); return message; } @@ -339,7 +339,7 @@ public Message getEngineResourceMetrics(HttpServletRequest request) { list.add(linkis); Message message = Message.ok(); - message.setMethod("/exchangis/metrics/engineresource"); + message.setMethod("/dss/exchangis/main/metrics/engineresource"); message.data("list", list); return message; } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ProjectImportServerImpl.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ProjectImportServerImpl.java new file mode 100644 index 000000000..126ec250a --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ProjectImportServerImpl.java @@ -0,0 +1,197 @@ +package com.webank.wedatasphere.exchangis.job.server.service.impl; + +import com.webank.wedatasphere.exchangis.job.server.dto.ExportedProject; +import com.webank.wedatasphere.exchangis.job.server.dto.IdCatalog; +import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; +import com.webank.wedatasphere.exchangis.job.server.service.IProjectImportService; +import com.webank.wedatasphere.exchangis.job.server.service.JobInfoService; +import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobVo; +import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; +import com.webank.wedatasphere.exchangis.project.server.exception.ExchangisProjectErrorException; +import com.webank.wedatasphere.exchangis.project.server.mapper.ProjectMapper; +import com.webank.wedatasphere.exchangis.project.server.service.ProjectService; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.bml.client.BmlClient; +import org.apache.linkis.bml.client.BmlClientFactory; +import org.apache.linkis.bml.protocol.BmlDownloadResponse; +import org.apache.linkis.server.BDPJettyServerHelper; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import javax.annotation.Resource; +import javax.servlet.http.HttpServletRequest; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.rmi.ServerException; +import java.util.Calendar; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * @author tikazhang + * @Date 2022/3/15 9:58 + */ + +@Service +public class ProjectImportServerImpl implements IProjectImportService { + + private static final Logger LOG = LoggerFactory.getLogger(ProjectImportServerImpl.class); + + Pattern pattern1 = Pattern.compile("([a-zA-Z]+_\\d+).*"); + + Pattern pattern2 = Pattern.compile("(\\S+)_v1\\S+"); + + @Resource + private JobInfoService jobInfoService; + + @Autowired + private ProjectMapper projectMapper; + + @Override + public Message importProject(HttpServletRequest req, Map params) throws ExchangisJobServerException, ServerException { + String userName = SecurityFilter.getLoginUsername(req); + //String resourceId = "99763d27-a35e-43f2-829b-100830bca538"; + String resourceId = (String) params.get("resourceId"); + String version = (String) params.get("flowVersion"); + //Long projectId = Long.parseLong("1497870871035973669"); + Long projectId = Long.parseLong("111111"); + String projectVersion = (String) params.get("projectVersion"); + String flowVersion = (String) params.get("flowVersion"); + String versionSuffix = projectVersion; + LOG.info("resourceId: {}, projectId: {}, versionSuffix: {}, version: {}, userName: {}", resourceId, projectId, versionSuffix, version, userName); + BmlClient bmlClient = BmlClientFactory.createBmlClient(userName); + BmlDownloadResponse bmlDownloadResponse = bmlClient.downloadShareResource(userName, resourceId, version); + LOG.info("bmlDownloadResponse: {}", bmlDownloadResponse); + Message message = null; + if (bmlDownloadResponse == null || !bmlDownloadResponse.isSuccess()) { + throw new ServerException("cannot download exported data from BML"); + } + try { + String projectJson = IOUtils.toString(bmlDownloadResponse.inputStream(), StandardCharsets.UTF_8); + LOG.info("projectJson: {}", projectJson); + IdCatalog idCatalog = importOpt(projectJson, projectId, versionSuffix, userName); + message = Message.ok("import Job ok") + .data("sqoop", idCatalog.getSqoop()) + .data("datax", idCatalog.getDatax()); + + return message; + } catch (IOException | ExchangisJobServerException e) { + LOG.error("Error occur while import option: {}", e.getMessage()); + message = Message.error("Error occur while import option: {}"); + //throw new ExchangisJobServerException(31101, "导入出现错误:" + "[" + e.getMessage() + "]"); + } + finally { + IOUtils.closeQuietly(bmlDownloadResponse.inputStream()); + } + + return message; + } + + @Override + public IdCatalog importOpt(String projectJson, Long projectId, String versionSuffix, String userName) throws ExchangisJobServerException { + ExportedProject exportedProject = BDPJettyServerHelper.gson().fromJson(projectJson, ExportedProject.class); + IdCatalog idCatalog = new IdCatalog(); + + importSqoop(projectId, versionSuffix, exportedProject, idCatalog, userName); + + importDatax(projectId, versionSuffix, exportedProject, idCatalog, userName); + + return idCatalog; + } + + private void importSqoop(Long projectId, String versionSuffix, ExportedProject exportedProject, IdCatalog idCatalog, String userName) throws ExchangisJobServerException { + List sqoops = exportedProject.getSqoops(); + if (sqoops == null) { + return; + } + List projects = projectMapper.getDetailByName(exportedProject.getName()); + if (projects.size() == 0) { + ExchangisProject project = new ExchangisProject(); + project.setName(exportedProject.getName()); + project.setCreateTime(Calendar.getInstance().getTime()); + project.setCreateUser(userName); + Long newProjectId = projectMapper.insertOne(project); + List newProjects = projectMapper.getDetailByName(exportedProject.getName()); + addSqoopTask (sqoops, newProjects, versionSuffix, idCatalog, projectId); + } + else if (projects.size() == 1) { + addSqoopTask (sqoops, projects, versionSuffix, idCatalog, projectId); + } + else { + throw new ExchangisJobServerException(31101, "Already exits duplicated project name(存在重复项目名称) projectName is:" + "[" + exportedProject.getName() + "]"); + } + } + + private void importDatax(Long projectId, String versionSuffix, ExportedProject exportedProject, IdCatalog idCatalog, String userName) { + List dataxes = exportedProject.getDataxes(); + if (dataxes == null) { + return; + } + for (ExchangisJobVo datax : dataxes) { + Long oldId = datax.getId(); + datax.setProjectId(projectId); + datax.setJobName(updateName(datax.getJobName(), versionSuffix)); + //Long existingId = (long) 66; + Long existingId = jobInfoService.getByNameWithProjectId(datax.getJobName(), projectId).get(0).getId(); + //Long existingId = jobInfoService.getByNameWithProjectId(datax.getJobName(), projectId); + if (existingId != null) { + idCatalog.getSqoop().put(oldId, existingId); + } else { + jobInfoService.createJob(datax); + idCatalog.getSqoop().put(oldId, datax.getId()); + } + } + } + + public void addSqoopTask (List sqoops, List projects, String versionSuffix, IdCatalog idCatalog, Long projectId) throws ExchangisJobServerException { + for (ExchangisJobVo sqoop : sqoops) { + Long projectIdProd = projects.get(0).getId(); + Long oldId = sqoop.getId(); + sqoop.setProjectId(projectIdProd); + sqoop.setJobName(updateName(sqoop.getJobName(), versionSuffix)); + //Long existingId = (long) 55; + LOG.info("oldId: {}, projectid: {}, jobName: {}", sqoop.getId(), sqoop.getProjectId(), sqoop.getJobName()); + LOG.info("jobByNameWithProjectId: {}", jobInfoService.getByNameWithProjectId(sqoop.getJobName(), projectIdProd)); + Long existingId; + if (jobInfoService.getByNameWithProjectId(sqoop.getJobName(), projectIdProd) == null || jobInfoService.getByNameWithProjectId(sqoop.getJobName(), projectId).size() == 0) { + existingId = null; + } else { + existingId = jobInfoService.getByNameWithProjectId(sqoop.getJobName(), projectIdProd).get(0).getId(); + } + //Long existingId = jobInfoService.getByNameWithProjectId(sqoop.getJobName(), projectId); + if (existingId != null) { + idCatalog.getSqoop().put(oldId, existingId); + throw new ExchangisJobServerException(31101, "Already exits duplicated job name(存在重复任务名称) jobName is:" + "[" + sqoop.getJobName() + "]"); + } else { + //sqoop.setJobName("hahaha"); + LOG.info("Sqoop job content is: {}, Modify user is: {}, jobType is: {}", sqoop.getContent(), sqoop.getExecuteUser(), sqoop.getJobType()); + jobInfoService.createJob(sqoop); + idCatalog.getSqoop().put(oldId, sqoop.getId()); + } + } + } + + private String updateName(String name, String versionSuffix) { + if (StringUtils.isBlank(versionSuffix)) { + return name; + } + + Matcher matcher = pattern1.matcher(name); + if (matcher.find()) { + return matcher.group(1) + "_" + versionSuffix; + } else { + Matcher matcher2 = pattern2.matcher(name); + if (matcher2.find()) { + return matcher2.group(1) + "_" + versionSuffix; + } + } + return name + "_" + versionSuffix; + } +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/open/DefaultJobOpenService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/open/DefaultJobOpenService.java new file mode 100644 index 000000000..81ec15408 --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/open/DefaultJobOpenService.java @@ -0,0 +1,66 @@ +package com.webank.wedatasphere.exchangis.job.server.service.open; + +import com.github.pagehelper.PageHelper; +import com.webank.wedatasphere.exchangis.job.api.ExchangisJobOpenService; +import com.webank.wedatasphere.exchangis.job.domain.ExchangisJobEntity; +import com.webank.wedatasphere.exchangis.job.exception.ExchangisJobException; +import com.webank.wedatasphere.exchangis.job.server.mapper.ExchangisJobEntityDao; +import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobQueryVo; +import org.apache.commons.lang.StringUtils; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import javax.annotation.Resource; +import java.util.List; + +import static com.webank.wedatasphere.exchangis.job.exception.ExchangisJobExceptionCode.JOB_EXCEPTION_CODE; + + +/** + * Default implement + */ +@Service +public class DefaultJobOpenService implements ExchangisJobOpenService { + + @Resource + private ExchangisJobEntityDao jobEntityDao; + + @Override + public ExchangisJobEntity getJobById(Long id, boolean basic) throws ExchangisJobException { + try { + return basic ? this.jobEntityDao.getBasicInfo(id) : this.jobEntityDao.getDetail(id); + } catch (Exception e){ + throw new ExchangisJobException(JOB_EXCEPTION_CODE.getCode(), + "Fail to the information of job [id: " + id + "]", e); + } + } + + @Override + public List queryJobs(ExchangisJobQueryVo queryVo, boolean inPage) throws ExchangisJobException { + try { + if (inPage) { + PageHelper.startPage(queryVo.getPage(), queryVo.getPageSize()); + try { + return this.jobEntityDao.queryPageList(queryVo); + } finally { + PageHelper.clearPage(); + } + } + return this.jobEntityDao.queryPageList(queryVo); + } catch (Exception e){ + throw new ExchangisJobException(JOB_EXCEPTION_CODE.getCode(), + "Fail to query job list", e); + } + } + + @Override + @Transactional(rollbackFor = Exception.class) + public void deleteJobBatch(List idList) throws ExchangisJobException { + try { + this.jobEntityDao.deleteBatch(idList); + } catch (Exception e){ + throw new ExchangisJobException(JOB_EXCEPTION_CODE.getCode(), + "Fail to delete batch job ids, id list: [" + StringUtils.join(idList,",") + "]", e); + } + } +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/utils/DateTool.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/utils/DateTool.java new file mode 100644 index 000000000..d6fa86083 --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/utils/DateTool.java @@ -0,0 +1,225 @@ +package com.webank.wedatasphere.exchangis.job.server.utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.regex.Pattern; + + +public class DateTool { + private static final String TIME_PLACEHOLDER_DATE_0 = "${yyyyMMdd}"; + private static final String TIME_PLACEHOLDER_DATE_1 = "${yyyy-MM-dd}"; + public static final String TIME_PLACEHOLDER_DATE_TIME = "${yyyy-MM-dd HH:mm:ss}"; + /** + * yyyyMMddHH + * yyyy-MM-dd-HH + * HH + */ + public static final String TIME_PLACEHOLDER_DATE_HOUR_0 = "${yyyyMMddHH}"; + public static final String TIME_PLACEHOLDER_DATE_HOUR_1 = "${yyyy-MM-dd-HH}"; + public static final String TIME_PLACEHOLDER_DATE_HOUR_2 = "${HH}"; + + private static final String TIME_PLACEHOLDER_TIMESTAMP = "${timestamp}"; + static final String MONTH_BEGIN_SYMBOL = "run_month_begin"; + static final String MONTH_END_SYMBOL = "run_month_end"; + static final String TIME_PLACEHOLDER_SYMBOL = "timestamp"; + static final String MONTH_BEGIN_LAST_SYMBOL = "run_month_begin-1"; + static final String[] HOUR_SPEC_SYMBOLS = new String[]{"yyyyMMdd", "yyyy-MM-dd", "HH"}; + static final String LINE_SYMBOL = "_std"; + public static final String[] TIME_PLACEHOLDER = new String[]{ + TIME_PLACEHOLDER_DATE_0, TIME_PLACEHOLDER_DATE_1, TIME_PLACEHOLDER_DATE_TIME, TIME_PLACEHOLDER_TIMESTAMP, + TIME_PLACEHOLDER_DATE_HOUR_0, TIME_PLACEHOLDER_DATE_HOUR_1, TIME_PLACEHOLDER_DATE_HOUR_2}; + + private static final String TIME_REGULAR_EXPRESSION = "\\$\\{(run_date|run_month_begin|run_month_end|HH|yyyyMMddHH|yyyy-MM-dd-HH)(_std)?\\s*([+-])?\\s*([0-9])?\\}"; + public static final Pattern TIME_REGULAR_PATTERN = Pattern.compile(TIME_REGULAR_EXPRESSION); + + private static Logger log = LoggerFactory.getLogger(DateTool.class); + private Calendar calendar=Calendar.getInstance(); + + public DateTool(Date date){ + this.calendar.setTime(date); + } + + public DateTool(long timeInMillis){ + this.calendar.setTimeInMillis(timeInMillis); + } + + public DateTool(){ + } + + public DateTool set(int field, int value) { + calendar.set(field, value); + return this; + } + + public DateTool getDate(String date) { + SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd"); + try { + calendar.setTime(format.parse(date)); + } catch (ParseException e) { + log.error("Parse exception.",e); + } + return this; + } + + /** + * + * @return + */ + public DateTool getHalfBeg(int amount) { + calendar.add(Calendar.MONTH, amount * 6); + int currentMonth = calendar.get(Calendar.MONTH) + 1; + + if (currentMonth >= 1 && currentMonth <= 6) { + calendar.set(Calendar.MONTH, 0); + } else if (currentMonth >= 7 && currentMonth <= 12) { + calendar.set(Calendar.MONTH, 6); + } + calendar.set(Calendar.DATE, 1); + return this; + } + + /** + * + * @return + */ + public DateTool getHalfEnd(int amount) { + calendar.add(Calendar.MONTH, amount * 6); + int currentMonth = calendar.get(Calendar.MONTH) + 1; + + if (currentMonth >= 1 && currentMonth <= 6) { + calendar.set(Calendar.MONTH, 5); + calendar.set(Calendar.DATE, 30); + } else if (currentMonth >= 7 && currentMonth <= 12) { + calendar.set(Calendar.MONTH, 11); + calendar.set(Calendar.DATE, 31); + } + return this; + } + + /** + * Begin of quarter + * + * @return + */ + public DateTool getQuarterBeg(int amount) { + calendar.add(Calendar.MONTH, amount * 3); + + int currentMonth = calendar.get(Calendar.MONTH) + 1; + + if (currentMonth >= 1 && currentMonth <= 3) { + calendar.set(Calendar.MONTH, 0); + } else if (currentMonth >= 4 && currentMonth <= 6) { + calendar.set(Calendar.MONTH, 3); + } else if (currentMonth >= 7 && currentMonth <= 9) { + calendar.set(Calendar.MONTH, 6); + } else if (currentMonth >= 10 && currentMonth <= 12) { + calendar.set(Calendar.MONTH, 9); + } + calendar.set(Calendar.DATE, 1); + return this; + } + + /** + * End of quarter + * + * @return + */ + public DateTool getQuarterEnd(int amount) { + calendar.add(Calendar.MONTH, amount * 3); + int currentMonth = calendar.get(Calendar.MONTH) + 1; + if (currentMonth >= 1 && currentMonth <= 3) { + calendar.set(Calendar.MONTH, 2); + calendar.set(Calendar.DATE, 31); + } else if (currentMonth >= 4 && currentMonth <= 6) { + calendar.set(Calendar.MONTH, 5); + calendar.set(Calendar.DATE, 30); + } else if (currentMonth >= 7 && currentMonth <= 9) { + calendar.set(Calendar.MONTH, 8); + calendar.set(Calendar.DATE, 30); + } else if (currentMonth >= 10 && currentMonth <= 12) { + calendar.set(Calendar.MONTH, 11); + calendar.set(Calendar.DATE, 31); + } + return this; + } + + public DateTool addDay(int amount){ + calendar.add(Calendar.DAY_OF_YEAR, amount); + return this; + } + + public DateTool addHour(int amount){ + calendar.add(Calendar.HOUR_OF_DAY, amount); + return this; + } + public DateTool add(int field, int amount){ + calendar.add(field, amount); + return this; + } + + public DateTool addMonth(int amount){ + calendar.add(Calendar.MONTH, amount); + return this; + } + + public DateTool addYesterdayMonth(int amount){ + calendar.add(Calendar.DAY_OF_MONTH, -1); + calendar.add(Calendar.MONTH, amount); + return this; + } + public DateTool getMonthEnd(int amount){ + calendar.set(Calendar.DATE, 1); + calendar.add(Calendar.MONTH, amount+1); + calendar.add(Calendar.DAY_OF_MONTH, -1); + return this; + } + + public DateTool getMonthBegin(int amount){ + calendar.set(Calendar.DATE, 1); + calendar.add(Calendar.MONTH, amount); + return this; + } + + public DateTool getMonthBeginLastDay(int amount){ + calendar.set(Calendar.DATE, 1); + calendar.add(Calendar.MONTH, amount); + calendar.add(Calendar.DAY_OF_MONTH, -1); + return this; + } + + public String format(String pattern){ + SimpleDateFormat format=new SimpleDateFormat(pattern); + return format.format(calendar.getTime()); + } + + public String format(String pattern, long time){ + SimpleDateFormat format = new SimpleDateFormat(pattern); + return format.format(new Date(time)); + } + public String currentTimestamp(){ + return String.valueOf(System.currentTimeMillis()/1000); + } + + + public static Date stringToDate(String dateStr, String formatStr){ + DateFormat sdf=new SimpleDateFormat(formatStr); + Date date=null; + try { + date = sdf.parse(dateStr); + } catch (ParseException e) { + log.error("Parse exception.",e); + } + return date; + } + + public DateTool truncate(long mills) { + calendar.setTimeInMillis(mills * (calendar.getTimeInMillis() / mills)); + return this; + } + +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/utils/JobUtils.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/utils/JobUtils.java new file mode 100644 index 000000000..ba088e970 --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/utils/JobUtils.java @@ -0,0 +1,140 @@ +package com.webank.wedatasphere.exchangis.job.server.utils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.StringWriter; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.regex.Matcher; + + +public class JobUtils { + private static Logger logger = LoggerFactory.getLogger(JobUtils.class); + + /** + * Replace the parameter in variables + * @param map + * @param key + * @param subValue + */ + private static void replaceParameters(Map map, String key, + StringBuffer subValue) { + StringBuilder subKey = new StringBuilder(); + char[] chars = key.toCharArray(); + int i=0; + boolean isParamter = false; + int count = 0; + while(i 0){ + subKey.append(chars[i]).append(chars[i+1]); + } + count++; + i=i+2; + continue; + }else if(chars[i] == '}'){ + count--; + if(count == 0){ + String parameter = subKey.toString(); + if(parameter.contains("${") && parameter.contains("}")){ + StringBuffer sb = new StringBuffer(); + replaceParameters(map, parameter, sb); + parameter = sb.toString(); + } + String v = map.get(parameter); + isParamter = false; + subKey.delete(0, subKey.length()); + if(null != v && !"".equals(v)){ + if(v.contains("${") && v.contains("}")){ + replaceParameters(map, v, subValue); + }else{ + subValue.append(v); + } + }else{ + subValue.append("${").append(parameter).append("}"); + } + i=i+1; + continue; + } + + } + if(isParamter){ + subKey.append(chars[i]); + }else{ + subValue.append(chars[i]); + } + i=i+1; + } + } + + + public static String renderDt(String template, Calendar calendar){ + long time = calendar.getTimeInMillis(); + if(template==null){ + return null; + } + Date date =new Date(); + Matcher matcher= DateTool.TIME_REGULAR_PATTERN.matcher(template); + while(matcher.find()){ + try { + String m = template.substring(matcher.start(), matcher.end()); + StringWriter sw = new StringWriter(); + DateTool dataTool = new DateTool(time); + String symbol = matcher.group(1); + boolean spec = false; + if (null != symbol) { + String startTime = null; + String tempTime = null; + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + if ("run_date".equals(symbol)) { + int n = 1; + if (m.split("-").length > 1) { + List days = Arrays.asList(m.split("-")); + n = Integer.parseInt(days.get(1).substring(0, days.get(1).length()-1)); + } + calendar.setTime(date); + calendar.add(Calendar.DAY_OF_MONTH, -n); + tempTime = format.format(calendar.getTime()).substring(0, 10) + " 00:00:00"; + startTime = template.replace(m, tempTime); + return startTime; + } + + for(String specSymbol : DateTool.HOUR_SPEC_SYMBOLS){ + if(specSymbol.equals(symbol)){ + tempTime = dataTool.format(specSymbol); + startTime = template.replace(m, tempTime); + return startTime; + } + } + if (DateTool.MONTH_BEGIN_SYMBOL.equals(symbol)) { + dataTool.getMonthBegin(0); + tempTime = dataTool.format("yyyy-MM-dd HH:mm:ss"); + startTime = template.replace(m, tempTime); + return startTime; + } else if (DateTool.MONTH_BEGIN_LAST_SYMBOL.equals(symbol)) { + dataTool.getMonthBeginLastDay(0); + tempTime = dataTool.format("yyyy-MM-dd HH:mm:ss"); + startTime = template.replace(m, tempTime); + return startTime; + } else if (DateTool.TIME_PLACEHOLDER_SYMBOL.equals(symbol)){ + calendar.setTime(date); + calendar.add(Calendar.DAY_OF_MONTH, -1); + tempTime = String.valueOf(calendar.getTimeInMillis()); + startTime = template.replace(m, tempTime); + return startTime; + } + } + + }catch(Exception e){ + logger.error("TASK_ERROR, cannot render job's configuration, message: {}", e.getMessage(), e); + break; + } + } + //${yesterday} + return template.replace("${yesterday}",new DateTool(time).addDay(-1).format("yyyyMMdd")); + } + +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisJobExportVo.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisJobExportVo.java new file mode 100644 index 000000000..030e71808 --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisJobExportVo.java @@ -0,0 +1,4 @@ +package com.webank.wedatasphere.exchangis.job.server.vo; + +public class ExchangisJobExportVo { +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisJobImportVo.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisJobImportVo.java new file mode 100644 index 000000000..ed0433750 --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisJobImportVo.java @@ -0,0 +1,4 @@ +package com.webank.wedatasphere.exchangis.job.server.vo; + +public class ExchangisJobImportVo { +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisJobLogQuery.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisJobLogQuery.java deleted file mode 100644 index 8620565af..000000000 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisJobLogQuery.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.webank.wedatasphere.exchangis.job.server.vo; - -/** - * - * @Date 2022/1/18 21:10 - */ -public class ExchangisJobLogQuery{ - -} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisLaunchedJobListVO.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisLaunchedJobListVo.java similarity index 94% rename from exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisLaunchedJobListVO.java rename to exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisLaunchedJobListVo.java index 1e1bed7b6..f48c9f8b2 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisLaunchedJobListVO.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisLaunchedJobListVo.java @@ -6,7 +6,7 @@ * * @Date 2022/1/16 10:21 */ -public class ExchangisLaunchedJobListVO { +public class ExchangisLaunchedJobListVo { private String jobExecutionId; private String executeNode; @@ -27,11 +27,11 @@ public class ExchangisLaunchedJobListVO { private Date lastUpdateTime; - public ExchangisLaunchedJobListVO(){ + public ExchangisLaunchedJobListVo(){ } - public ExchangisLaunchedJobListVO(String jobExecutionId, String executeNode, String name, Date createTime, Long flow, String createUser, String executeUser, String status, double progress, Date lastUpdateTime){ + public ExchangisLaunchedJobListVo(String jobExecutionId, String executeNode, String name, Date createTime, Long flow, String createUser, String executeUser, String status, double progress, Date lastUpdateTime){ this.jobExecutionId = jobExecutionId; this.executeNode = executeNode; this.name = name; diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisLaunchedTaskMetricsVO.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisLaunchedTaskMetricsVo.java similarity index 87% rename from exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisLaunchedTaskMetricsVO.java rename to exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisLaunchedTaskMetricsVo.java index debc1e167..006863f20 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisLaunchedTaskMetricsVO.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisLaunchedTaskMetricsVo.java @@ -6,7 +6,7 @@ * * @Date 2022/1/12 23:00 */ -public class ExchangisLaunchedTaskMetricsVO { +public class ExchangisLaunchedTaskMetricsVo { private String taskId; private String name; @@ -16,11 +16,11 @@ public class ExchangisLaunchedTaskMetricsVO { private ExchangisMetricsVo metrics; //private Map metrics; - public ExchangisLaunchedTaskMetricsVO(){ + public ExchangisLaunchedTaskMetricsVo(){ } - public ExchangisLaunchedTaskMetricsVO(String taskId, String name, String status, ExchangisMetricsVo metrics){ + public ExchangisLaunchedTaskMetricsVo(String taskId, String name, String status, ExchangisMetricsVo metrics){ this.taskId = taskId; this.name = name; this.status = status; diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisLaunchedTaskVO.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisLaunchedTaskVO.java deleted file mode 100644 index 25bbf5ac2..000000000 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisLaunchedTaskVO.java +++ /dev/null @@ -1,57 +0,0 @@ -package com.webank.wedatasphere.exchangis.job.server.vo; - -import com.webank.wedatasphere.exchangis.job.server.dto.ExchangisTaskMetricsDTO; - -/** - * - * @Date 2022/1/12 22:50 - */ -public class ExchangisLaunchedTaskVO { - - private Long taskId; - - private String name; - - private String status; - - private Long progress; - - public ExchangisLaunchedTaskVO(Long taskId, String name, String status, Long progress){ - this.taskId = taskId; - this.name = name; - this.status = status; - this.progress = progress; - } - - public Long getTaskId() { - return taskId; - } - - public void setTaskId(Long taskId) { - this.taskId = taskId; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getStatus() { - return status; - } - - public void setStatus(String status) { - this.status = status; - } - - public Long getProgress() { - return progress; - } - - public void setProgress(Long progress) { - this.progress = progress; - } -} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisTaskInfoVO.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisTaskInfoVO.java deleted file mode 100644 index 374348374..000000000 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisTaskInfoVO.java +++ /dev/null @@ -1,76 +0,0 @@ -package com.webank.wedatasphere.exchangis.job.server.vo; - -import java.util.Date; - -public class ExchangisTaskInfoVO { - - private Long id; - - private Long jobId; - - private String taskName; - - private Date launchTime; - - private String createUser; - - private String status; - - private Date completeTime; - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public String getTaskName() { - return taskName; - } - - public void setTaskName(String taskName) { - this.taskName = taskName; - } - - public Date getLaunchTime() { - return launchTime; - } - - public void setLaunchTime(Date launchTime) { - this.launchTime = launchTime; - } - - public String getCreateUser() { - return createUser; - } - - public void setCreateUser(String createUser) { - this.createUser = createUser; - } - - public String getStatus() { - return status; - } - - public void setStatus(String status) { - this.status = status; - } - - public Date getCompleteTime() { - return completeTime; - } - - public void setCompleteTime(Date completeTime) { - this.completeTime = completeTime; - } - - public Long getJobId() { - return jobId; - } - - public void setJobId(Long jobId) { - this.jobId = jobId; - } -} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisTaskSpeedLimitVO.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisTaskSpeedLimitVO.java deleted file mode 100644 index edbce8093..000000000 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/vo/ExchangisTaskSpeedLimitVO.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.webank.wedatasphere.exchangis.job.server.vo; - -import java.util.List; - -public class ExchangisTaskSpeedLimitVO { - - private List settings; - - public List getSettings() { - return settings; - } - - public void setSettings(List settings) { - this.settings = settings; - } - - public static class Settings { - private String config_key; - private String config_name; - private String config_value; - private Integer sort; - - public String getConfig_key() { - return config_key; - } - - public void setConfig_key(String config_key) { - this.config_key = config_key; - } - - public String getConfig_name() { - return config_name; - } - - public void setConfig_name(String config_name) { - this.config_name = config_name; - } - - public String getConfig_value() { - return config_value; - } - - public void setConfig_value(String config_value) { - this.config_value = config_value; - } - - public Integer getSort() { - return sort; - } - - public void setSort(Integer sort) { - this.sort = sort; - } - } - -} diff --git a/exchangis-job/exchangis-job-server/src/main/scala/com/webank/wedatasphere/exchangis/job/server/log/DefaultRpcJobLogger.scala b/exchangis-job/exchangis-job-server/src/main/scala/com/webank/wedatasphere/exchangis/job/server/log/DefaultRpcJobLogger.scala index 2fb087e7c..62cc242d3 100644 --- a/exchangis-job/exchangis-job-server/src/main/scala/com/webank/wedatasphere/exchangis/job/server/log/DefaultRpcJobLogger.scala +++ b/exchangis-job/exchangis-job-server/src/main/scala/com/webank/wedatasphere/exchangis/job/server/log/DefaultRpcJobLogger.scala @@ -22,6 +22,7 @@ class DefaultRpcJobLogger extends JobLogListener{ event.getLevel match { case Level.INFO => getLogger.info(message, event.getArgs: _*) case Level.ERROR => getLogger.error(message, event.getArgs: _*) + case Level.WARN => getLogger.warn(message, event.getArgs: _*) case _ => getLogger.trace(message, event.getArgs) } } diff --git a/exchangis-plugins/engine/datax/datax-assembly/package.xml b/exchangis-plugins/engine/datax/datax-assembly/package.xml index 154b4cb5e..d88b54980 100644 --- a/exchangis-plugins/engine/datax/datax-assembly/package.xml +++ b/exchangis-plugins/engine/datax/datax-assembly/package.xml @@ -3,7 +3,7 @@ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd"> core - dir + zip false diff --git a/exchangis-plugins/engine/datax/datax-assembly/pom.xml b/exchangis-plugins/engine/datax/datax-assembly/pom.xml index 3f97a5aa6..c243fc656 100644 --- a/exchangis-plugins/engine/datax/datax-assembly/pom.xml +++ b/exchangis-plugins/engine/datax/datax-assembly/pom.xml @@ -5,7 +5,7 @@ exchangis-executor-engine-datax com.webank.wedatasphere.exchangis - 0.5.0.RELEASE + 1.0.0-RC1 ../pom.xml 4.0.0 diff --git a/exchangis-plugins/engine/datax/datax-core/pom.xml b/exchangis-plugins/engine/datax/datax-core/pom.xml index de3e80015..ae737a761 100644 --- a/exchangis-plugins/engine/datax/datax-core/pom.xml +++ b/exchangis-plugins/engine/datax/datax-core/pom.xml @@ -5,7 +5,7 @@ exchangis-executor-engine-datax com.webank.wedatasphere.exchangis - 0.5.0.RELEASE + 1.0.0-RC1 ../pom.xml 4.0.0 @@ -94,6 +94,7 @@ junit junit + 4.12 test diff --git a/exchangis-plugins/engine/datax/datax-elasticsearchwriter/pom.xml b/exchangis-plugins/engine/datax/datax-elasticsearchwriter/pom.xml index a8c830f35..944169751 100644 --- a/exchangis-plugins/engine/datax/datax-elasticsearchwriter/pom.xml +++ b/exchangis-plugins/engine/datax/datax-elasticsearchwriter/pom.xml @@ -5,7 +5,7 @@ exchangis-executor-engine-datax com.webank.wedatasphere.exchangis - 0.5.0.RELEASE + 1.0.0-RC1 ../pom.xml 4.0.0 diff --git a/exchangis-plugins/engine/datax/datax-elasticsearchwriter/src/main/java/com/webank/wedatasphere/exchangis/datax/plugin/writer/elasticsearchwriter/v6/ElasticRestClient.java b/exchangis-plugins/engine/datax/datax-elasticsearchwriter/src/main/java/com/webank/wedatasphere/exchangis/datax/plugin/writer/elasticsearchwriter/v6/ElasticRestClient.java index 59dc6db80..7866336c7 100644 --- a/exchangis-plugins/engine/datax/datax-elasticsearchwriter/src/main/java/com/webank/wedatasphere/exchangis/datax/plugin/writer/elasticsearchwriter/v6/ElasticRestClient.java +++ b/exchangis-plugins/engine/datax/datax-elasticsearchwriter/src/main/java/com/webank/wedatasphere/exchangis/datax/plugin/writer/elasticsearchwriter/v6/ElasticRestClient.java @@ -351,7 +351,7 @@ private void initialClient(String[] endPoints, CredentialsProvider credentialsPr ); restClientBuilder.setRequestConfigCallback( requestConfigBuilder -> requestConfigBuilder - .setContentCompressionEnabled(true) +// .setContentCompressionEnabled(true) .setConnectTimeout(Integer.parseInt( String.valueOf(finalClientConfig.getOrDefault(ElasticKey.CLIENT_CONFIG_CONN_TIMEOUT, CONN_TIMEOUT_IN_MILLISECONDS)))) diff --git a/exchangis-plugins/engine/datax/datax-ftpreader/pom.xml b/exchangis-plugins/engine/datax/datax-ftpreader/pom.xml index 341f31c9e..c80dd8e83 100644 --- a/exchangis-plugins/engine/datax/datax-ftpreader/pom.xml +++ b/exchangis-plugins/engine/datax/datax-ftpreader/pom.xml @@ -5,7 +5,7 @@ exchangis-executor-engine-datax com.webank.wedatasphere.exchangis - 0.5.0.RELEASE + 1.0.0-RC1 ../pom.xml 4.0.0 @@ -43,6 +43,7 @@ commons-codec commons-codec + ${commons-codec} provided diff --git a/exchangis-plugins/engine/datax/datax-ftpwriter/pom.xml b/exchangis-plugins/engine/datax/datax-ftpwriter/pom.xml index 353f56d73..a4966c9c7 100644 --- a/exchangis-plugins/engine/datax/datax-ftpwriter/pom.xml +++ b/exchangis-plugins/engine/datax/datax-ftpwriter/pom.xml @@ -5,7 +5,7 @@ exchangis-executor-engine-datax com.webank.wedatasphere.exchangis - 0.5.0.RELEASE + 1.0.0-RC1 ../pom.xml 4.0.0 @@ -48,6 +48,7 @@ commons-codec commons-codec + ${commons-codec} provided diff --git a/exchangis-plugins/engine/datax/datax-hdfsreader/pom.xml b/exchangis-plugins/engine/datax/datax-hdfsreader/pom.xml index 7459d68fd..dfc58f3db 100644 --- a/exchangis-plugins/engine/datax/datax-hdfsreader/pom.xml +++ b/exchangis-plugins/engine/datax/datax-hdfsreader/pom.xml @@ -5,7 +5,7 @@ exchangis-executor-engine-datax com.webank.wedatasphere.exchangis - 0.5.0.RELEASE + 1.0.0-RC1 ../pom.xml 4.0.0 diff --git a/exchangis-plugins/engine/datax/datax-hdfswriter/pom.xml b/exchangis-plugins/engine/datax/datax-hdfswriter/pom.xml index cfad21165..5e09ca5d7 100644 --- a/exchangis-plugins/engine/datax/datax-hdfswriter/pom.xml +++ b/exchangis-plugins/engine/datax/datax-hdfswriter/pom.xml @@ -5,7 +5,7 @@ exchangis-executor-engine-datax com.webank.wedatasphere.exchangis - 0.5.0.RELEASE + 1.0.0-RC1 ../pom.xml 4.0.0 diff --git a/exchangis-plugins/engine/datax/datax-mysqlreader/pom.xml b/exchangis-plugins/engine/datax/datax-mysqlreader/pom.xml index fc483064f..f66e7486a 100644 --- a/exchangis-plugins/engine/datax/datax-mysqlreader/pom.xml +++ b/exchangis-plugins/engine/datax/datax-mysqlreader/pom.xml @@ -5,7 +5,7 @@ exchangis-executor-engine-datax com.webank.wedatasphere.exchangis - 0.5.0.RELEASE + 1.0.0-RC1 ../pom.xml 4.0.0 diff --git a/exchangis-plugins/engine/datax/datax-mysqlwriter/pom.xml b/exchangis-plugins/engine/datax/datax-mysqlwriter/pom.xml index 3d180f5a3..d901cc033 100644 --- a/exchangis-plugins/engine/datax/datax-mysqlwriter/pom.xml +++ b/exchangis-plugins/engine/datax/datax-mysqlwriter/pom.xml @@ -5,7 +5,7 @@ exchangis-executor-engine-datax com.webank.wedatasphere.exchangis - 0.5.0.RELEASE + 1.0.0-RC1 ../pom.xml 4.0.0 diff --git a/exchangis-plugins/engine/datax/datax-oraclereader/pom.xml b/exchangis-plugins/engine/datax/datax-oraclereader/pom.xml index 3c86ffa58..a98114677 100644 --- a/exchangis-plugins/engine/datax/datax-oraclereader/pom.xml +++ b/exchangis-plugins/engine/datax/datax-oraclereader/pom.xml @@ -5,7 +5,7 @@ exchangis-executor-engine-datax com.webank.wedatasphere.exchangis - 0.5.0.RELEASE + 1.0.0-RC1 ../pom.xml 4.0.0 diff --git a/exchangis-plugins/engine/datax/datax-oraclewriter/pom.xml b/exchangis-plugins/engine/datax/datax-oraclewriter/pom.xml index 09beff175..f926cd8b0 100644 --- a/exchangis-plugins/engine/datax/datax-oraclewriter/pom.xml +++ b/exchangis-plugins/engine/datax/datax-oraclewriter/pom.xml @@ -5,7 +5,7 @@ exchangis-executor-engine-datax com.webank.wedatasphere.exchangis - 0.5.0.RELEASE + 1.0.0-RC1 ../pom.xml 4.0.0 diff --git a/exchangis-plugins/engine/datax/datax-textfilereader/pom.xml b/exchangis-plugins/engine/datax/datax-textfilereader/pom.xml index d0ada3034..de9f2479f 100644 --- a/exchangis-plugins/engine/datax/datax-textfilereader/pom.xml +++ b/exchangis-plugins/engine/datax/datax-textfilereader/pom.xml @@ -5,7 +5,7 @@ exchangis-executor-engine-datax com.webank.wedatasphere.exchangis - 0.5.0.RELEASE + 1.0.0-RC1 ../pom.xml 4.0.0 @@ -48,6 +48,7 @@ commons-codec commons-codec + ${commons-codec} provided diff --git a/exchangis-plugins/engine/datax/datax-textfilewriter/pom.xml b/exchangis-plugins/engine/datax/datax-textfilewriter/pom.xml index c4f25904a..5b9794f0c 100644 --- a/exchangis-plugins/engine/datax/datax-textfilewriter/pom.xml +++ b/exchangis-plugins/engine/datax/datax-textfilewriter/pom.xml @@ -5,7 +5,7 @@ exchangis-executor-engine-datax com.webank.wedatasphere.exchangis - 0.5.0.RELEASE + 1.0.0-RC1 ../pom.xml 4.0.0 @@ -48,6 +48,7 @@ commons-codec commons-codec + ${commons-codec} provided diff --git a/exchangis-plugins/engine/datax/pom.xml b/exchangis-plugins/engine/datax/pom.xml index 5d73e5559..9150add55 100644 --- a/exchangis-plugins/engine/datax/pom.xml +++ b/exchangis-plugins/engine/datax/pom.xml @@ -7,6 +7,7 @@ com.webank.wedatasphere.exchangis 1.0.0-RC1 + 4.0.0 exchangis-executor-engine-datax @@ -24,6 +25,10 @@ 1.10 1.2 1.9.4 + 2.7.2 + 1.2.1 + 6.7.1 + 1.11 3.0.0-Plus-2 diff --git a/exchangis-plugins/exchangis-appconn/pom.xml b/exchangis-plugins/exchangis-appconn/pom.xml index e738afdc1..7658a56c7 100644 --- a/exchangis-plugins/exchangis-appconn/pom.xml +++ b/exchangis-plugins/exchangis-appconn/pom.xml @@ -6,6 +6,7 @@ exchangis-plugins com.webank.wedatasphere.exchangis 1.0.0-RC1 + ../pom.xml 4.0.0 @@ -41,18 +42,15 @@ ${dss.version} - com.webank.wedatasphere.dss - dss-origin-sso-integration-standard - ${dss.version} + org.apache.linkis + linkis-storage + ${linkis.version} + provided linkis-common org.apache.linkis - - json4s-jackson_2.11 - org.json4s - @@ -70,6 +68,12 @@ dss-development-process-standard-execution ${dss.version} + + com.webank.wedatasphere.dss + dss-common + ${dss.version} + provided + diff --git a/exchangis-plugins/exchangis-appconn/src/main/assembly/distribution.xml b/exchangis-plugins/exchangis-appconn/src/main/assembly/distribution.xml index e51825ca8..8383286c3 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/assembly/distribution.xml +++ b/exchangis-plugins/exchangis-appconn/src/main/assembly/distribution.xml @@ -24,7 +24,8 @@ zip - + true + exchangis @@ -59,6 +60,15 @@ conf unix + + ${basedir}/src/main/resources + + init.sql + + 0777 + db + unix + diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ExchangisAppConn.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ExchangisAppConn.java index 7240d7864..4fdde8cd4 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ExchangisAppConn.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ExchangisAppConn.java @@ -4,6 +4,7 @@ import com.webank.wedatasphere.dss.appconn.core.impl.AbstractOnlySSOAppConn; import com.webank.wedatasphere.dss.standard.app.development.standard.DevelopmentIntegrationStandard; import com.webank.wedatasphere.dss.standard.app.structure.StructureIntegrationStandard; +import org.apache.linkis.common.conf.CommonVars; /** * Exchangis AppConn top implement diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/constraints/Constraints.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/constraints/Constraints.java index b17e6aaf6..95c0c64ac 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/constraints/Constraints.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/constraints/Constraints.java @@ -11,6 +11,8 @@ public class Constraints { // AppConn name public final static String EXCHANGIS_APPCONN_NAME = CommonVars.apply("wds.dss.appconn.exchangis.name", "Exchangis").getValue(); + public final static String API_REQUEST_PREFIX = CommonVars.apply("wds.dss.appconn.exchangis.api.request-prefix", "api/rest_j/v1/dss/exchangis/main").getValue(); + public final static String DOMAIN_NAME = CommonVars.apply("wds.dss.appconn.exchangis.domain.name", "DSS").getValue(); // Constraint in Project operation @@ -30,7 +32,8 @@ public class Constraints { // Constraint in Ref operation public final static String REF_JOB_DESC = "desc"; public final static String REF_JOB_ID = "id"; - public final static String REF_JUMP_URL_FORMAT = "dss/exchangis/#/childJobManagement"; + public final static String REF_JOB_LABELS = "labels"; + public final static String REF_JUMP_URL_FORMAT = CommonVars.apply("wds.dss.appconn.exchangis.ref.jump","#/childJobManagement").getValue(); } diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/AbstractExchangisOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/AbstractExchangisOperation.java index d607cc131..2ed669883 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/AbstractExchangisOperation.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/AbstractExchangisOperation.java @@ -8,6 +8,7 @@ import com.webank.wedatasphere.dss.standard.common.entity.ref.RequestRef; import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; import com.webank.wedatasphere.exchangis.dss.appconn.constraints.Constraints; +import com.webank.wedatasphere.exchangis.dss.appconn.operation.ref.ExchangisRefExecutionOperation; import com.webank.wedatasphere.exchangis.dss.appconn.request.action.HttpExtAction; import com.webank.wedatasphere.exchangis.dss.appconn.response.result.ExchangisEntityRespResult; import org.apache.commons.io.IOUtils; @@ -18,16 +19,21 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.function.Function; import java.util.function.Supplier; +import static com.webank.wedatasphere.exchangis.dss.appconn.constraints.Constraints.API_REQUEST_PREFIX; + /** * Abstract implement, contains the method to create sso request */ public abstract class AbstractExchangisOperation { + private final static Logger LOG = LoggerFactory.getLogger(AbstractExchangisOperation.class); + private SSORequestService ssoRequestService; private String uri; @@ -37,6 +43,8 @@ public abstract class AbstractExchangisOperation { */ private String baseURL; + private String redirectUrl; + public AbstractExchangisOperation(){ } @@ -61,6 +69,10 @@ protected void setSSORequestService(AppIntegrationService app this.ssoRequestService = appIntegrationService.getSSORequestService(); // Also upgrade the base url this.baseURL = appIntegrationService.getAppInstance().getBaseUrl(); + // Append the api prefix + this.baseURL = this.baseURL.endsWith(IOUtils.DIR_SEPARATOR_UNIX + "")? + baseURL + API_REQUEST_PREFIX: baseURL + IOUtils.DIR_SEPARATOR_UNIX + API_REQUEST_PREFIX; + this.redirectUrl = String.valueOf(appIntegrationService.getAppInstance().getConfig().get("redirectUrl")); } } @@ -86,7 +98,8 @@ protected SSORequestOperation getOrCreateSSORequestOperation() thro */ protected SSOUrlBuilderOperation getSSOUrlBuilderOperation(Workspace workspace, String appName, String url){ - return workspace.getSSOUrlBuilderOperation().copy().setAppName(appName).setReqUrl(requestURL()) + LOG.info("requestURL555555: {}", requestURL()); + return workspace.getSSOUrlBuilderOperation().copy().setAppName(appName).setReqUrl(url) .setWorkspace(workspace.getWorkspaceName()); } @@ -115,6 +128,11 @@ public String requestURL(String customUri){ baseURL + customUri : baseURL + IOUtils.DIR_SEPARATOR_UNIX + customUri; } + public String pageUrl(String customUri){ + return redirectUrl.endsWith(IOUtils.DIR_SEPARATOR_UNIX + "") ? + redirectUrl + customUri : redirectUrl + IOUtils.DIR_SEPARATOR_UNIX + customUri; + } + /** * Send request and get response entity * @param url url @@ -131,11 +149,14 @@ public String requestURL(String customUri){ protected ExchangisEntityRespResult.BasicMessageEntity requestToGetEntity(String url, Workspace workspace, T requestRef, HttpActionBuilder httpActionBuilder, Class entityClass, Class... entityParameters) throws ExternalOperationFailedException { + LOG.info("Create job url{}: ", url); HttpExtAction action = httpActionBuilder.build(requestRef); + LOG.info("Action123456{}: ", action.getRequestBody()); if (Objects.nonNull(action)){ SSOUrlBuilderOperation ssoUrlBuilderOperation = getSSOUrlBuilderOperation(workspace, getAppName(), url); ExchangisEntityRespResult.BasicMessageEntity entity; try { + LOG.info("ssoUrlBuilderOperation666: {}", ssoUrlBuilderOperation.getBuiltUrl()); action.setUrl(ssoUrlBuilderOperation.getBuiltUrl()); SSORequestOperation ssoRequestOperation = getOrCreateSSORequestOperation(); ExchangisEntityRespResult httpResult = new ExchangisEntityRespResult(ssoRequestOperation.requestWithSSO(ssoUrlBuilderOperation, action)); diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/AbstractExchangisProjectOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/AbstractExchangisProjectOperation.java index 567ee804b..b9579bc72 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/AbstractExchangisProjectOperation.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/AbstractExchangisProjectOperation.java @@ -7,6 +7,7 @@ import com.webank.wedatasphere.exchangis.dss.appconn.request.entity.ProjectReqEntity; import com.webank.wedatasphere.exchangis.dss.appconn.utils.AppConnUtils; import com.webank.wedatasphere.exchangis.dss.appconn.utils.JsonExtension; +import org.apache.commons.lang.StringUtils; import java.util.HashMap; import java.util.Map; @@ -30,10 +31,12 @@ public AbstractExchangisProjectOperation(){ protected ProjectReqEntity getProjectEntity(ProjectRequestRef projectRequestRef){ // Build project request entity Map source = new HashMap<>(); - String owner = projectRequestRef.getCreateBy(); - ProjectReqEntity projectReqEntity = new ProjectReqEntity(owner, + String editUsers= StringUtils.join( projectRequestRef.getEditUsers(),","); + String viewUsers = StringUtils.join( projectRequestRef.getAccessUsers(),","); + String execUsers = StringUtils.join( projectRequestRef.getReleaseUsers(),","); + ProjectReqEntity projectReqEntity = new ProjectReqEntity(editUsers,viewUsers,execUsers, projectRequestRef.getName(), projectRequestRef.getDescription(), source); - projectReqEntity.setTags(AppConnUtils.serializeDssLabel(projectRequestRef.getDSSLabels())); + projectReqEntity.setLabels(AppConnUtils.serializeDssLabel(projectRequestRef.getDSSLabels())); // Try to set the project request ref into the source map try { Map requestRefMap = JsonExtension.convert(projectReqEntity, Map.class, String.class, Object.class); diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectCreationOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectCreationOperation.java index bd6b77dc3..1baed0bc5 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectCreationOperation.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectCreationOperation.java @@ -34,7 +34,7 @@ public class ExchangisProjectCreationOperation extends AbstractExchangisProjectO private StructureService structureService; public ExchangisProjectCreationOperation(StructureService structureService) { - super(new String[]{"project"}); + super(new String[]{"appProject"}); setStructureService(structureService); } diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectDeletionOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectDeletionOperation.java index 4a4cd2bb3..609408baf 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectDeletionOperation.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectDeletionOperation.java @@ -9,12 +9,14 @@ import com.webank.wedatasphere.exchangis.dss.appconn.constraints.Constraints; import com.webank.wedatasphere.exchangis.dss.appconn.request.action.ExchangisDeleteAction; import com.webank.wedatasphere.exchangis.dss.appconn.ref.ExchangisProjectResponseRef; +import com.webank.wedatasphere.exchangis.dss.appconn.request.action.ExchangisEntityPostAction; import com.webank.wedatasphere.exchangis.dss.appconn.response.result.ExchangisEntityRespResult; import org.apache.linkis.httpclient.request.HttpAction; import org.apache.linkis.httpclient.response.HttpResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -39,15 +41,14 @@ protected Logger getLogger() { @Override public ProjectResponseRef deleteProject(ProjectRequestRef projectRequestRef) throws ExternalOperationFailedException { - // TODO Get the project id - long projectId = 0l; - String url = requestURL("/project/" + projectId); LOG.info("delete project request => dss_projectId:{}, name:{}, createName:{}", projectRequestRef.getId(), projectRequestRef.getName(), projectRequestRef.getCreateBy()); - ExchangisEntityRespResult.BasicMessageEntity> entity = requestToGetEntity(url, projectRequestRef.getWorkspace(), projectRequestRef, - (requestRef) -> { - // Build project delete action - return new ExchangisDeleteAction(requestRef.getCreateBy()); + String url = requestURL("appProject/" + projectRequestRef.getName()+"/"); + ExchangisEntityRespResult.BasicMessageEntity> entity = requestToGetEntity(url,projectRequestRef.getWorkspace(), projectRequestRef, + (requestRef) ->{ + ExchangisEntityPostAction exchangisEntityPostAction = new ExchangisEntityPostAction(); + exchangisEntityPostAction.setUser(requestRef.getCreateBy()); + return exchangisEntityPostAction; }, Map.class); if (Objects.isNull(entity)){ throw new ExternalOperationFailedException(31020, "The response entity cannot be empty", null); diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectGetOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectGetOperation.java new file mode 100644 index 000000000..13d7a7162 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectGetOperation.java @@ -0,0 +1,89 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.operation.project; + +import com.webank.wedatasphere.dss.common.entity.project.DSSProject; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; +import com.webank.wedatasphere.dss.standard.app.structure.StructureService; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectGetOperation; +import com.webank.wedatasphere.dss.standard.app.structure.project.ProjectRequestRef; +import com.webank.wedatasphere.dss.standard.common.exception.AppStandardErrorException; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.exchangis.dss.appconn.ref.ExchangisProjectResponseRef; +import com.webank.wedatasphere.exchangis.dss.appconn.request.action.ExchangisEntityPostAction; +import com.webank.wedatasphere.exchangis.dss.appconn.request.action.ExchangisGetAction; +import com.webank.wedatasphere.exchangis.dss.appconn.request.action.HttpExtAction; +import com.webank.wedatasphere.exchangis.dss.appconn.response.result.ExchangisEntityRespResult; +import com.webank.wedatasphere.exchangis.dss.appconn.service.ExchangisProjectService; +import com.webank.wedatasphere.exchangis.dss.appconn.utils.AppConnUtils; +import org.apache.linkis.httpclient.request.HttpAction; +import org.apache.linkis.httpclient.response.HttpResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * @author tikazhang + * @Date 2022/3/22 0:48 + */ +public class ExchangisProjectGetOperation extends AbstractExchangisProjectOperation implements ProjectGetOperation { + private static Logger LOG = LoggerFactory.getLogger(ExchangisProjectDeletionOperation.class); + + private SSORequestOperation ssoRequestOperation; + private StructureService structureService; + + public ExchangisProjectGetOperation(StructureService structureService) { + setStructureService(structureService); + } + + @Override + protected Logger getLogger() { + return LOG; + } + + @Override + public DSSProject getProject(ProjectRequestRef projectRequestRef) throws ExternalOperationFailedException { + LOG.info("get project request => dss_projectId:{}, name:{}, createName:{}", projectRequestRef.getId(), + projectRequestRef.getName(), projectRequestRef.getCreateBy()); + String url = requestURL("appProject/check/" + projectRequestRef.getName() + "/"); + ExchangisEntityRespResult.BasicMessageEntity> entity = requestToGetEntity(url, projectRequestRef.getWorkspace(), projectRequestRef, + (requestRef) -> { + ExchangisEntityPostAction exchangisEntityPostAction = new ExchangisEntityPostAction(); + exchangisEntityPostAction.setUser(requestRef.getCreateBy()); + return exchangisEntityPostAction; + }, Map.class); + if (Objects.isNull(entity)) { + throw new ExternalOperationFailedException(31020, "The response entity cannot be empty", null); + } + try { + ExchangisEntityRespResult httpResult = entity.getResult(); + httpResult.getResponseBody(); + Map resMap = AppConnUtils.getResponseMap(httpResult); + DSSProject dssProject = new DSSProject(); + Map dataMap = (Map) resMap.get("data"); + Map projectInfo = (Map) dataMap.get("projectInfo"); + //如果查询不到project，则无重复项目可以直接返回 + if (projectInfo == null) { + return null; + } + dssProject.setId(Long.parseLong(projectInfo.get("id").toString())); + return dssProject; + } catch (AppStandardErrorException e) { + throw new ExternalOperationFailedException(90176, "search Exchangis Project failed when get HttpResult", e); + } catch (Exception e) { + throw new ExternalOperationFailedException(90176, "search Exchangis project failed when parse response json", e); + } + } + + @Override + public void init() { + + } + + @Override + public void setStructureService(StructureService structureService) { + this.structureService = structureService; + setSSORequestService(this.structureService); + } +} \ No newline at end of file diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectUpdateOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectUpdateOperation.java index 7a504431f..eac1d3c92 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectUpdateOperation.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/project/ExchangisProjectUpdateOperation.java @@ -27,17 +27,15 @@ public class ExchangisProjectUpdateOperation extends AbstractExchangisProjectOpe private StructureService structureService; public ExchangisProjectUpdateOperation(StructureService structureService) { + super(new String[]{"appProject"}); setStructureService(structureService); } @Override public ProjectResponseRef updateProject(ProjectRequestRef projectRequestRef) throws ExternalOperationFailedException { - // TODO Get the project id - long projectId = 0l; - String url = requestURL("/project/" + projectId); LOG.info("update project request => dss_projectId:{}, name:{}, createName:{}", projectRequestRef.getId(), projectRequestRef.getName(),projectRequestRef.getCreateBy()); - ExchangisEntityRespResult.BasicMessageEntity> entity = requestToGetEntity(url, projectRequestRef.getWorkspace(), projectRequestRef, + ExchangisEntityRespResult.BasicMessageEntity> entity = requestToGetEntity(projectRequestRef.getWorkspace(), projectRequestRef, (requestRef) -> { // Build project put(update) action return new ExchangisEntityPutAction<>(getProjectEntity(requestRef), requestRef.getCreateBy()); @@ -47,7 +45,7 @@ public ProjectResponseRef updateProject(ProjectRequestRef projectRequestRef) thr } ExchangisEntityRespResult httpResult = entity.getResult(); LOG.info("update project response => status {}, response {}", httpResult.getStatusCode(), httpResult.getResponseBody()); - AtomicLong newProjectId = new AtomicLong(projectId); + AtomicLong newProjectId = new AtomicLong(projectRequestRef.getId()); try { Optional.ofNullable(entity.getData()).ifPresent( data -> newProjectId .set(Long.parseLong(String.valueOf(data.getOrDefault(Constraints.PROJECT_ID, newProjectId.get()))))); diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/publish/ExchangisExportRequestRef.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/publish/ExchangisExportRequestRef.java new file mode 100644 index 000000000..fbe71be63 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/publish/ExchangisExportRequestRef.java @@ -0,0 +1,52 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.operation.publish; + +import com.google.common.collect.Maps; +import com.webank.wedatasphere.dss.standard.app.development.ref.ExportRequestRef; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; + +import java.util.Map; + +/** + * @author tikazhang + * @Date 2022/3/13 16:33 + */ +public class ExchangisExportRequestRef implements ExportRequestRef { + + Map parameters = Maps.newHashMap(); + Workspace workspace; + + @Override + public Object getParameter(String key) { + return parameters.get(key); + } + + @Override + public void setParameter(String key, Object value) { + parameters.put(key, value); + } + + @Override + public Map getParameters() { + return parameters; + } + + @Override + public String getName() { + return parameters.get("name").toString(); + } + + @Override + public String getType() { + return parameters.get("type").toString(); + } + + @Override + public Workspace getWorkspace() { + return workspace; + } + + @Override + public void setWorkspace(Workspace workspace) { + this.workspace = workspace; + } +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/publish/ExchangisImportRequestRef.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/publish/ExchangisImportRequestRef.java new file mode 100644 index 000000000..58be4e6fa --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/publish/ExchangisImportRequestRef.java @@ -0,0 +1,74 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.operation.publish; + +import com.google.common.collect.Maps; +import com.webank.wedatasphere.dss.standard.app.development.operation.RefImportOperation; +import com.webank.wedatasphere.dss.standard.app.development.ref.ImportRequestRef; +import com.webank.wedatasphere.dss.standard.app.development.service.DevelopmentService; +import com.webank.wedatasphere.dss.standard.app.sso.Workspace; +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; +import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.exchangis.dss.appconn.ref.ExchangisExportResponseRef; +import org.apache.linkis.httpclient.request.HttpAction; +import org.apache.linkis.httpclient.response.HttpResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +/** + * @author tikazhang + * @Date 2022/3/13 16:45 + */ +public class ExchangisImportRequestRef implements ImportRequestRef { + + Map parameters = Maps.newHashMap(); + Workspace workspace; + + @Override + public Object getParameter(String key) { + return parameters.get(key); + } + + @Override + public void setParameter(String key, Object value) { + parameters.put(key, value); + } + + @Override + public Map getParameters() { + return parameters; + } + + @Override + public String getName() { + return parameters.get("name").toString(); + } + + @Override + public String getType() { + return parameters.get("type").toString(); + } + + @Override + public boolean equals(Object ref) { + return false; + } + + @Override + public String toString() { + return parameters.toString(); + } + + @Override + public Workspace getWorkspace() { + return workspace; + } + + @Override + public void setWorkspace(Workspace workspace) { + this.workspace = workspace; + } + +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/AbstractExchangisRefOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/AbstractExchangisRefOperation.java index 7d0260bf4..b4b9cb831 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/AbstractExchangisRefOperation.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/AbstractExchangisRefOperation.java @@ -18,7 +18,7 @@ public AbstractExchangisRefOperation() { } public AbstractExchangisRefOperation(String[] uriParts) { - super(uriParts); + super(uriParts); } /** diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisExportOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisExportOperation.java index 9768b9714..ccc0e422a 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisExportOperation.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisExportOperation.java @@ -1,14 +1,26 @@ package com.webank.wedatasphere.exchangis.dss.appconn.operation.ref; +import com.fasterxml.jackson.core.JsonProcessingException; import com.webank.wedatasphere.dss.standard.app.development.operation.RefExportOperation; import com.webank.wedatasphere.dss.standard.app.development.ref.ExportRequestRef; import com.webank.wedatasphere.dss.standard.app.development.service.DevelopmentService; +import com.webank.wedatasphere.dss.standard.app.sso.builder.SSOUrlBuilderOperation; +import com.webank.wedatasphere.dss.standard.app.sso.request.SSORequestOperation; import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; import com.webank.wedatasphere.exchangis.dss.appconn.constraints.Constraints; +import com.webank.wedatasphere.exchangis.dss.appconn.ref.ExchangisExportResponseRef; +import com.webank.wedatasphere.exchangis.dss.appconn.ref.ExchangisOpenResponseRef; +import com.webank.wedatasphere.exchangis.dss.appconn.request.action.ExchangisEntityPostAction; +import org.apache.linkis.httpclient.request.HttpAction; +import org.apache.linkis.httpclient.response.HttpResult; +import org.apache.linkis.server.BDPJettyServerHelper; +import org.apache.linkis.server.conf.ServerConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.HashMap; + /** * Ref export operation */ @@ -16,16 +28,69 @@ public class ExchangisExportOperation extends AbstractExchangisRefOperation impl private final static Logger LOG = LoggerFactory.getLogger(ExchangisExportOperation.class); private DevelopmentService developmentService; + private SSORequestOperation ssoRequestOperation; public ExchangisExportOperation(DevelopmentService developmentService){ this.developmentService = developmentService; + this.ssoRequestOperation = this.developmentService.getSSORequestService().createSSORequestOperation(getAppName()); setSSORequestService(this.developmentService); } @Override public ResponseRef exportRef(ExportRequestRef exportRequestRef) throws ExternalOperationFailedException { - return null; + String url = developmentService.getAppInstance().getBaseUrl() + "api/rest_j/" + ServerConfiguration.BDP_SERVER_VERSION() + "/dss/exchangis/main/appJob" + "/export"; + ExchangisEntityPostAction exchangisEntityPostAction = new ExchangisEntityPostAction(); + exchangisEntityPostAction.setUser(exportRequestRef.getParameter("user").toString()); + exchangisEntityPostAction.addRequestPayload("projectId", exportRequestRef.getParameter("projectId")); + exchangisEntityPostAction.addRequestPayload("partial", true); + HashMap labels = new HashMap<>(); + labels.put("route", "dev"); + exchangisEntityPostAction.addRequestPayload("labels", labels); + String nodeType = exportRequestRef.getParameter("nodeType").toString(); + String externalContent = null; + + String exporrtContent = null; + try { + exporrtContent = BDPJettyServerHelper.jacksonJson().writeValueAsString(exportRequestRef); + } catch (JsonProcessingException e) { + LOG.error("Occur error while tranform class", e.getMessage()); + } + LOG.info("url: {}", url); + LOG.info("exportRequestRef: {}", exporrtContent); + try { + LOG.info("jobContent: {}", exportRequestRef.getParameter("jobContent")); + externalContent = BDPJettyServerHelper.jacksonJson().writeValueAsString(exportRequestRef.getParameter("jobContent")); + if(Constraints.NODE_TYPE_SQOOP.equalsIgnoreCase(nodeType)) { + ExchangisOpenResponseRef exchangisOpenResponseRef = new ExchangisOpenResponseRef(externalContent, 0); + exchangisEntityPostAction.addRequestPayload("sqoopIds", ((Double) Double.parseDouble(exchangisOpenResponseRef.getSqoopId())).longValue()); + LOG.info("sqoopIds: {}", exchangisEntityPostAction.getRequestPayload()); + } else if(Constraints.NODE_TYPE_DATAX.equalsIgnoreCase(nodeType)) { + ExchangisOpenResponseRef exchangisOpenResponseRef = new ExchangisOpenResponseRef(externalContent, 0); + exchangisEntityPostAction.addRequestPayload("dataXIds", ((Double) Double.parseDouble(exchangisOpenResponseRef.getSqoopId())).longValue()); + } else { + throw new ExternalOperationFailedException(90177, "Unknown task type " + exportRequestRef.getType(), null); + } + } catch (Exception e) { + LOG.error("Failed to create export request", e); + } + SSOUrlBuilderOperation ssoUrlBuilderOperation = exportRequestRef.getWorkspace().getSSOUrlBuilderOperation().copy(); + ssoUrlBuilderOperation.setAppName(getAppName()); + ssoUrlBuilderOperation.setReqUrl(url); + ssoUrlBuilderOperation.setWorkspace(exportRequestRef.getWorkspace().getWorkspaceName()); + ResponseRef responseRef; + try{ + LOG.info("getBuiltUrl: {}", ssoUrlBuilderOperation.getBuiltUrl()); + LOG.info("postAction: {}", exchangisEntityPostAction); + exchangisEntityPostAction.setUrl(ssoUrlBuilderOperation.getBuiltUrl()); + HttpResult httpResult = this.ssoRequestOperation.requestWithSSO(ssoUrlBuilderOperation, exchangisEntityPostAction); + LOG.info("httpResult: {}", httpResult.getResponseBody()); + responseRef = new ExchangisExportResponseRef(httpResult.getResponseBody()); + LOG.info("responseRef: {}", responseRef.getResponseBody()); + } catch (Exception e){ + throw new ExternalOperationFailedException(90176, "Export Exchangis Exception", e); + } + return responseRef; } @Override diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisImportOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisImportOperation.java index a0bae1b41..4a4180330 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisImportOperation.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisImportOperation.java @@ -8,12 +8,19 @@ import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef; import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; import com.webank.wedatasphere.exchangis.dss.appconn.constraints.Constraints; +import com.webank.wedatasphere.exchangis.dss.appconn.ref.ExchangisImportResponseRef; +import com.webank.wedatasphere.exchangis.dss.appconn.request.action.ExchangisEntityPostAction; import com.webank.wedatasphere.exchangis.dss.appconn.request.action.ExchangisPostAction; import com.webank.wedatasphere.exchangis.dss.appconn.ref.AbstractExchangisResponseRef; +import org.apache.linkis.httpclient.request.HttpAction; import org.apache.linkis.httpclient.response.HttpResult; +import org.apache.linkis.server.conf.ServerConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.HashMap; +import java.util.Map; + /** * Ref import operation @@ -22,15 +29,44 @@ public class ExchangisImportOperation extends AbstractExchangisRefOperation impl private final static Logger LOG = LoggerFactory.getLogger(ExchangisImportOperation.class); private DevelopmentService developmentService; + private SSORequestOperation ssoRequestOperation; public ExchangisImportOperation(DevelopmentService developmentService){ this.developmentService = developmentService; setSSORequestService(this.developmentService); + this.ssoRequestOperation = this.developmentService.getSSORequestService().createSSORequestOperation(getAppName()); } @Override public ResponseRef importRef(ImportRequestRef importRequestRef) throws ExternalOperationFailedException { - return null; + String url = developmentService.getAppInstance().getBaseUrl() + "api/rest_j/" + ServerConfiguration.BDP_SERVER_VERSION() + "/dss/exchangis/main/appJob" + "/import"; + ExchangisEntityPostAction exchangisEntityPostAction = new ExchangisEntityPostAction(); + exchangisEntityPostAction.setUser(importRequestRef.getParameter("user").toString()); + exchangisEntityPostAction.addRequestPayload("projectId", importRequestRef.getParameter("projectId")); + exchangisEntityPostAction.addRequestPayload("projectVersion", importRequestRef.getParameter("orcVersion")); + exchangisEntityPostAction.addRequestPayload("flowVersion", importRequestRef.getParameter("version")); + exchangisEntityPostAction.addRequestPayload("resourceId", importRequestRef.getParameter("resourceId").toString()); + exchangisEntityPostAction.addRequestPayload("version", importRequestRef.getParameter("version").toString()); + exchangisEntityPostAction.addRequestPayload("user", importRequestRef.getParameter("user").toString()); + //exchangisEntityPostAction.addRequestPayload("labels", "{\"route\":\"prod\"}"); + HashMap labels = new HashMap<>(); + labels.put("route", "prod"); + exchangisEntityPostAction.addRequestPayload("labels", labels); + LOG.info("resourceId: {}", importRequestRef.getParameter("resourceId")); + LOG.info("exchangisEntityPostAction: {}", exchangisEntityPostAction.getRequestPayload()); + SSOUrlBuilderOperation ssoUrlBuilderOperation = importRequestRef.getWorkspace().getSSOUrlBuilderOperation().copy(); + ssoUrlBuilderOperation.setAppName(getAppName()); + ssoUrlBuilderOperation.setReqUrl(url); + ssoUrlBuilderOperation.setWorkspace(importRequestRef.getWorkspace().getWorkspaceName()); + ResponseRef responseRef; + try { + exchangisEntityPostAction.setUrl(ssoUrlBuilderOperation.getBuiltUrl()); + HttpResult httpResult = this.ssoRequestOperation.requestWithSSO(ssoUrlBuilderOperation, exchangisEntityPostAction); + responseRef = new ExchangisImportResponseRef((Map) importRequestRef.getParameter("jobContent"), httpResult.getResponseBody(), importRequestRef.getParameter("nodeType").toString()); + } catch (Exception e) { + throw new ExternalOperationFailedException(90176, "Import Exchangis Exception", e); + } + return responseRef; } @Override diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefCreationOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefCreationOperation.java index 9f1cc5a84..343b94709 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefCreationOperation.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefCreationOperation.java @@ -26,7 +26,7 @@ public class ExchangisRefCreationOperation extends AbstractExchangisRefOperation DevelopmentService developmentService; public ExchangisRefCreationOperation(DevelopmentService developmentService){ - super(new String[]{"job"}); + super(new String[]{"appJob/create"}); this.developmentService = developmentService; setSSORequestService(developmentService); } @@ -38,6 +38,7 @@ public ResponseRef createRef(CreateRequestRef createRequestRef) throws ExternalO ResponseRef responseRef = null; if(Constraints.NODE_TYPE_SQOOP.equalsIgnoreCase(exchangisCreateRequestRef.getNodeType())){ responseRef = sendOffLineRequest(exchangisCreateRequestRef, Constraints.ENGINE_TYPE_SQOOP_NAME); + LOG.info("responseRef: {}", responseRef.toMap()); }else if(Constraints.NODE_TYPE_DATAX.equalsIgnoreCase(exchangisCreateRequestRef.getNodeType())){ responseRef = sendOffLineRequest(exchangisCreateRequestRef, Constraints.ENGINE_TYPE_DATAX_NAME); } diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefDeletionOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefDeletionOperation.java index d06c9279f..b5e51895c 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefDeletionOperation.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefDeletionOperation.java @@ -10,12 +10,14 @@ import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; import com.webank.wedatasphere.exchangis.dss.appconn.constraints.Constraints; import com.webank.wedatasphere.exchangis.dss.appconn.request.action.ExchangisDeleteAction; +import com.webank.wedatasphere.exchangis.dss.appconn.request.action.ExchangisEntityPostAction; import com.webank.wedatasphere.exchangis.dss.appconn.response.result.ExchangisEntityRespResult; import com.webank.wedatasphere.exchangis.dss.appconn.utils.AppConnUtils; import org.apache.linkis.httpclient.response.HttpResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -39,14 +41,21 @@ public void deleteRef(RequestRef deleteRequestRef) throws ExternalOperationFaile } private void deleteJob(NodeRequestRef nodeRequestRef) throws ExternalOperationFailedException{ - Long id = AppConnUtils.resolveParam(nodeRequestRef.getJobContent(), Constraints.REF_JOB_ID, Long.class); + Integer id = AppConnUtils.resolveParam(nodeRequestRef.getJobContent(), Constraints.REF_JOB_ID, Integer.class); LOG.info("delete job request => id: {}, jobContext:{}", id, nodeRequestRef.getJobContent().toString()); - String url = requestURL("/job/" + id); + //TODO delete job have bug while get id. + String url = requestURL("appJob/" + id); ExchangisEntityRespResult.BasicMessageEntity> entity = requestToGetEntity(url, nodeRequestRef.getWorkspace(), nodeRequestRef, (requestRef) ->{ // Build ref delete action - return new ExchangisDeleteAction(requestRef.getUserName()); + ExchangisEntityPostAction exchangisEntityPostAction = new ExchangisEntityPostAction(); + exchangisEntityPostAction.setUser(requestRef.getUserName()); + HashMap labels = new HashMap<>(); + labels.put("route", "dev"); + exchangisEntityPostAction.addRequestPayload("labels", labels); + return exchangisEntityPostAction; + //return new ExchangisDeleteAction(requestRef.getUserName()); }, Map.class); if (Objects.isNull(entity)){ throw new ExternalOperationFailedException(31020, "The response entity cannot be empty", null); diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefExecutionOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefExecutionOperation.java index 1bf442df7..20b2b53dc 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefExecutionOperation.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefExecutionOperation.java @@ -1,5 +1,6 @@ package com.webank.wedatasphere.exchangis.dss.appconn.operation.ref; +import com.fasterxml.jackson.core.JsonProcessingException; import com.webank.wedatasphere.dss.standard.app.development.listener.common.AsyncExecutionRequestRef; import com.webank.wedatasphere.dss.standard.app.development.operation.RefExecutionOperation; import com.webank.wedatasphere.dss.standard.app.development.ref.ExecutionRequestRef; @@ -15,11 +16,16 @@ import com.webank.wedatasphere.exchangis.dss.appconn.ref.AbstractExchangisResponseRef; import com.webank.wedatasphere.exchangis.dss.appconn.response.result.ExchangisEntityRespResult; import com.webank.wedatasphere.exchangis.dss.appconn.utils.AppConnUtils; +import org.apache.commons.io.IOUtils; import org.apache.linkis.httpclient.request.HttpAction; import org.apache.linkis.httpclient.response.HttpResult; +import org.apache.linkis.server.BDPJettyServerHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.Map; import java.util.Objects; @@ -39,16 +45,16 @@ public ExchangisRefExecutionOperation(DevelopmentService service) { @Override public ResponseRef execute(ExecutionRequestRef executionRequestRef) throws ExternalOperationFailedException { AsyncExecutionRequestRef nodeRequestRef = (AsyncExecutionRequestRef) executionRequestRef; - Long id = AppConnUtils.resolveParam(nodeRequestRef.getJobContent(), Constraints.REF_JOB_ID, Long.class); - LOG.info("execute job request => id: {}, name: {}, user: {}, jobContent: {}", - id, nodeRequestRef.getName(), nodeRequestRef.getExecutionRequestRefContext().getUser(), - nodeRequestRef.getJobContent().toString()); - String url = requestURL("/job/execute/" + id); - ExchangisEntityRespResult.BasicMessageEntity> entity = requestToGetEntity(nodeRequestRef.getWorkspace(), nodeRequestRef, + LOG.info("execute job request => jobcontent: {}", nodeRequestRef.getJobContent()); + Long id = AppConnUtils.resolveParam(nodeRequestRef.getJobContent(), Constraints.REF_JOB_ID, Double.class).longValue(); + String url = requestURL("/appJob/execute/" + id); + String submitUser = nodeRequestRef.getExecutionRequestRefContext().getRuntimeMap().get("wds.dss.workflow.submit.user").toString(); + ExchangisEntityRespResult.BasicMessageEntity> entity = requestToGetEntity(url, nodeRequestRef.getWorkspace(), nodeRequestRef, (requestRef) -> { // Build ref execution action - return new ExchangisEntityPostAction<>(null, - requestRef.getExecutionRequestRefContext().getUser()); + ExchangisEntityPostAction exchangisEntityPostAction = new ExchangisEntityPostAction(); + exchangisEntityPostAction.addRequestPayload("submitUser",submitUser); + return exchangisEntityPostAction; }, Map.class); if (Objects.isNull(entity)){ throw new ExternalOperationFailedException(31020, "The response entity cannot be empty", null); diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefQueryOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefQueryOperation.java index 9049d6449..bf4cd6084 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefQueryOperation.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefQueryOperation.java @@ -33,11 +33,14 @@ public ResponseRef query(OpenRequestRef openRequestRef) throws ExternalOperation // Note: dss will scan the AppConn package to new the ExchangisOpenRequestRef ExchangisOpenRequestRef exchangisOpenRequestRef = (ExchangisOpenRequestRef) openRequestRef; try { - Long id = AppConnUtils.resolveParam(exchangisOpenRequestRef.getJobContent(), Constraints.REF_JOB_ID, Long.class); - String jumpUrl = requestURL(Constraints.REF_JUMP_URL_FORMAT + "?id=" + id); + LOG.info("ExchangisOpenRequestRef getJobContent: {}", exchangisOpenRequestRef.getJobContent()); + Integer id = AppConnUtils.resolveParam(exchangisOpenRequestRef.getJobContent(), Constraints.REF_JOB_ID, Integer.class); + String labels = (String) exchangisOpenRequestRef.getJobContent().get(Constraints.REF_JOB_LABELS); + LOG.info("Job labels is : {}", labels); + String jumpUrl = pageUrl(Constraints.REF_JUMP_URL_FORMAT + "?id=" + id + "&labels=" + labels); Map retMap = new HashMap<>(); LOG.info("ExchangisOpenResponseRef jump url: {}", jumpUrl); - retMap.put("jumpUrl",jumpUrl); + retMap.put("jumpUrl", jumpUrl); return new ExchangisOpenResponseRef(DSSCommonUtils.COMMON_GSON.toJson(retMap),0); } catch (Exception e) { throw new ExternalOperationFailedException(31022, "Failed to parse jobContent ", e); diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefUpdateOperation.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefUpdateOperation.java index e2ea49858..4e886f16c 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefUpdateOperation.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/ref/ExchangisRefUpdateOperation.java @@ -49,14 +49,14 @@ public ResponseRef updateRef(UpdateRequestRef updateRequestRef) throws ExternalO } private ResponseRef updateOffLineRequest(NodeRequestRef nodeRequestRef, String engineType) throws ExternalOperationFailedException{ - Long id = AppConnUtils.resolveParam(nodeRequestRef.getJobContent(), Constraints.REF_JOB_ID, Long.class); + Integer id = AppConnUtils.resolveParam(nodeRequestRef.getJobContent(), Constraints.REF_JOB_ID, Integer.class); LOG.info("update {} job request => id: {}, jobContent:{}", id, engineType, nodeRequestRef.getJobContent()); - String url = requestURL("/job/" + id); + String url = requestURL("/appJob/" + id); ExchangisEntityRespResult.BasicMessageEntity> entity = requestToGetEntity(url, nodeRequestRef.getWorkspace(), nodeRequestRef, (requestRef) -> { // Build ref update action RefJobReqEntity jobReqEntity = getRefJobReqEntity(requestRef, engineType); - jobReqEntity.setId(id); + jobReqEntity.setId(Long.valueOf(id)); return new ExchangisEntityPutAction<>(jobReqEntity, requestRef.getUserName()); }, Map.class); if (Objects.isNull(entity)){ diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisExportResponseRef.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisExportResponseRef.java new file mode 100644 index 000000000..017e1d610 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisExportResponseRef.java @@ -0,0 +1,29 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.ref; + +import com.webank.wedatasphere.dss.standard.common.entity.ref.CommonResponseRef; +import com.webank.wedatasphere.exchangis.dss.appconn.operation.ref.ExchangisExportOperation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +/** + * @author tikazhang + * @Date 2022/3/9 17:39 + */ +public class ExchangisExportResponseRef extends CommonResponseRef { + + private final static Logger LOG = LoggerFactory.getLogger(ExchangisExportResponseRef.class); + + Map bmlResource; + + public ExchangisExportResponseRef(String responseBody) throws Exception { + super(responseBody, 0); + LOG.info("responseBody123: {}", responseBody); + LOG.info("responseMap123: {}", responseMap.toString()); + bmlResource = ((Map) responseMap.get("data")); + } + + @Override + public Map toMap() {return bmlResource;} +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisImportResponseRef.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisImportResponseRef.java new file mode 100644 index 000000000..cec7c68e5 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisImportResponseRef.java @@ -0,0 +1,29 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.ref; + +import com.google.common.collect.Maps; +import com.webank.wedatasphere.dss.standard.common.entity.ref.CommonResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.exchangis.dss.appconn.constraints.Constraints; + +import java.util.Map; + +/** + * @author tikazhang + * @Date 2022/3/9 23:01 + */ +public class ExchangisImportResponseRef extends CommonResponseRef { + + Map newJobContent = Maps.newHashMap(); + + public ExchangisImportResponseRef(Map jobContent, String responseBody, String nodeType) throws Exception { + super(responseBody, 0); + if (!Constraints.NODE_TYPE_SQOOP.equalsIgnoreCase(nodeType) && !Constraints.NODE_TYPE_DATAX.equalsIgnoreCase(nodeType)) { + throw new ExternalOperationFailedException(90177, "Unknown task type" + nodeType, null); + } + this.newJobContent = jobContent; + } + + @Override + public Map toMap() {return newJobContent;} + +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisOpenResponseRef.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisOpenResponseRef.java index 7e23b682d..a1ee802cf 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisOpenResponseRef.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisOpenResponseRef.java @@ -1,6 +1,8 @@ package com.webank.wedatasphere.exchangis.dss.appconn.ref; import com.webank.wedatasphere.dss.standard.app.development.ref.CommonResponseRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.exchangis.dss.appconn.utils.ExchangisNodeUtils; /** * Ref open response @@ -9,4 +11,12 @@ public class ExchangisOpenResponseRef extends CommonResponseRef { public ExchangisOpenResponseRef(String responseBody, int status) { super(responseBody, status); } + + public String getSqoopId() throws ExternalOperationFailedException { + return ExchangisNodeUtils.getSqoopId(responseBody); + } + + public String getDataxId() throws ExternalOperationFailedException { + return ExchangisNodeUtils.getDataxId(responseBody); + } } diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisProjectResponseRef.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisProjectResponseRef.java index 19336b9ba..4f50d9b0a 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisProjectResponseRef.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/ref/ExchangisProjectResponseRef.java @@ -23,6 +23,11 @@ public ExchangisProjectResponseRef(ExchangisEntityRespResult result, this.projectRefId = projectId; } + public ExchangisProjectResponseRef(String responseBody, + int status){ + super(responseBody, status); + } + @Override public Long getProjectRefId() { return projectRefId; diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/action/ExchangisEntityPostAction.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/action/ExchangisEntityPostAction.java index 36cd12805..746e08548 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/action/ExchangisEntityPostAction.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/action/ExchangisEntityPostAction.java @@ -7,6 +7,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.HashMap; import java.util.Objects; /** @@ -23,6 +24,8 @@ public class ExchangisEntityPostAction extends POSTAction implements HttpExtA private String url; private String user; + + private HashMap queryParams; /** * Entity to post request */ @@ -41,6 +44,14 @@ public ExchangisEntityPostAction(T postEntity, String user){ this.user = user; } + public HashMap getQueryParams() { + return queryParams; + } + + public void setQueryParams(HashMap queryParams) { + this.queryParams = queryParams; + } + @Override public String getURL() { return url; diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/entity/ProjectReqEntity.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/entity/ProjectReqEntity.java index ef7647bfe..d9c6eb1c9 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/entity/ProjectReqEntity.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/request/entity/ProjectReqEntity.java @@ -38,9 +38,9 @@ public class ProjectReqEntity { private String execUsers; /** - * Tags + * labels */ - private String tags; + private String labels; public ProjectReqEntity(){ @@ -54,6 +54,16 @@ public ProjectReqEntity(String owner, String projectName, String description, Ma setViewUsers(owner); setExecUsers(owner); } + + public ProjectReqEntity(String editUsers, String viewUsers, String execUsers, String projectName, String description, Map source){ + this.projectName = projectName; + this.description = description; + this.source = source; + setEditUsers(editUsers); + setViewUsers(viewUsers); + setExecUsers(execUsers); + } + public String getProjectName() { return projectName; } @@ -102,11 +112,15 @@ public void setExecUsers(String execUsers) { this.execUsers = execUsers; } - public String getTags() { - return tags; + public String getDomain() { + return domain; + } + + public String getLabels() { + return labels; } - public void setTags(String tags) { - this.tags = tags; + public void setLabels(String labels) { + this.labels = labels; } } diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/response/result/ExchangisEntityRespResult.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/response/result/ExchangisEntityRespResult.java index fdbb23c73..a97b84e74 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/response/result/ExchangisEntityRespResult.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/response/result/ExchangisEntityRespResult.java @@ -109,6 +109,46 @@ public static class BasicMessageEntity{ */ private T data; + private String title; + + private String desc; + + private String labels; + + private Object workspace; + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getDesc() { + return desc; + } + + public void setDesc(String desc) { + this.desc = desc; + } + + public String getLabels() { + return labels; + } + + public void setLabels(String labels) { + this.labels = labels; + } + + public Object getWorkspace() { + return workspace; + } + + public void setWorkspace(Object workspace) { + this.workspace = workspace; + } + private ExchangisEntityRespResult result; public ExchangisEntityRespResult getResult() { diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/service/ExchangisProjectService.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/service/ExchangisProjectService.java index a26fa1ae9..8a7bcc687 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/service/ExchangisProjectService.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/service/ExchangisProjectService.java @@ -2,6 +2,9 @@ import com.webank.wedatasphere.dss.standard.app.structure.project.*; import com.webank.wedatasphere.exchangis.dss.appconn.operation.project.ExchangisProjectCreationOperation; +import com.webank.wedatasphere.exchangis.dss.appconn.operation.project.ExchangisProjectDeletionOperation; +import com.webank.wedatasphere.exchangis.dss.appconn.operation.project.ExchangisProjectGetOperation; +import com.webank.wedatasphere.exchangis.dss.appconn.operation.project.ExchangisProjectUpdateOperation; /** * Project service implement @@ -25,12 +28,18 @@ protected ProjectCreationOperation createProjectCreationOperation() { @Override protected ProjectUpdateOperation createProjectUpdateOperation() { - return null; + return new ExchangisProjectUpdateOperation(this); } @Override protected ProjectDeletionOperation createProjectDeletionOperation() { + return new ExchangisProjectDeletionOperation(this); + } + + @Override + protected ProjectGetOperation createProjectGetOperation() { return null; + //return new ExchangisProjectGetOperation(this); } @Override diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/utils/AppConnUtils.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/utils/AppConnUtils.java index 2755ede81..7a7a2fbc3 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/utils/AppConnUtils.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/utils/AppConnUtils.java @@ -3,7 +3,9 @@ import com.webank.wedatasphere.dss.common.label.DSSLabel; import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; import com.webank.wedatasphere.exchangis.dss.appconn.response.result.ExchangisEntityRespResult; +import org.apache.linkis.httpclient.response.HttpResult; import org.apache.linkis.manager.label.entity.SerializableLabel; +import org.apache.linkis.server.BDPJettyServerHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,4 +49,12 @@ public static T resolveParam(Map responseMap, String key, Cla return null; } + public static Map getResponseMap(HttpResult httpResult) throws ExternalOperationFailedException { + try { + return BDPJettyServerHelper.jacksonJson().readValue(httpResult.getResponseBody(), Map.class); + } catch (Exception e) { + throw new ExternalOperationFailedException(90176, "search visualis project exception", e); + } + } + } diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/utils/ExchangisNodeUtils.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/utils/ExchangisNodeUtils.java new file mode 100644 index 000000000..b29ef48dc --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/utils/ExchangisNodeUtils.java @@ -0,0 +1,53 @@ +package com.webank.wedatasphere.exchangis.dss.appconn.utils; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.webank.wedatasphere.dss.standard.app.development.ref.NodeRequestRef; +import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import org.apache.linkis.server.BDPJettyServerHelper; + +import java.util.Map; + +/** + * @author tikazhang + * @Date 2022/3/9 16:14 + */ +public class ExchangisNodeUtils { + public static String getId(NodeRequestRef nodeRequestRef) throws Exception { + String externalContent = BDPJettyServerHelper.jacksonJson().writeValueAsString(nodeRequestRef.getJobContent()); + if ("linkis.appconn.exchangis.sqoop".equalsIgnoreCase(nodeRequestRef.getNodeType())) { + return NumberUtils.parseDoubleString(getSqoopId(externalContent)); + } else if ("linkis.appconn.exchangis.datax".equalsIgnoreCase(nodeRequestRef.getNodeType())) { + return NumberUtils.parseDoubleString(getDataxId(externalContent)); + } + return null; + } + + + public static String getSqoopId(String responseBody) throws ExternalOperationFailedException { + String sqoopId = null; + try { + Map responseMap = BDPJettyServerHelper.jacksonJson().readValue(responseBody, Map.class); + sqoopId = ((Map) responseMap.get("data")).get("id").toString(); + } catch (JsonMappingException e) { + throw new ExternalOperationFailedException(1000054, "Get sqoop Id failed!", e); + } catch (JsonProcessingException e) { + throw new ExternalOperationFailedException(1000054, "Get sqoop Id failed!", e); + } + + return sqoopId; + } + + public static String getDataxId(String responseBody) throws ExternalOperationFailedException { + String dataxId = null; + try { + Map responseMap = BDPJettyServerHelper.jacksonJson().readValue(responseBody, Map.class); + dataxId = ((Map) responseMap.get("data")).get("dataxId").toString(); + } catch (JsonMappingException e) { + throw new ExternalOperationFailedException(1000055, "Get datax Id failed!", e); + } catch (JsonProcessingException e) { + throw new ExternalOperationFailedException(1000055, "Get datax Id failed!", e); + } + return dataxId; + } +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/utils/JsonExtension.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/utils/JsonExtension.java index 0bde26625..8f6f4960f 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/utils/JsonExtension.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/utils/JsonExtension.java @@ -2,12 +2,17 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationFailedException; +import com.webank.wedatasphere.exchangis.dss.appconn.operation.ref.ExchangisRefQueryOperation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Extends the function from the mapper "BDPJettyServerHelper.jacksonJson()" */ public class JsonExtension { + private final static Logger LOG = LoggerFactory.getLogger(JsonExtension.class); + /** * Convert object using serialization and deserialization * diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/utils/NumberUtils.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/utils/NumberUtils.java new file mode 100644 index 000000000..8f71dbfd3 --- /dev/null +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/utils/NumberUtils.java @@ -0,0 +1,34 @@ +/* + * Copyright 2019 WeBank + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.webank.wedatasphere.exchangis.dss.appconn.utils; + +public class NumberUtils { + + public static Integer getInt(Object original){ + if(original instanceof Double){ + return ((Double) original).intValue(); + } + return (Integer) original; + } + + public static String parseDoubleString(String doubleString) { + Double doubleValue = Double.parseDouble(doubleString); + Integer intValue = doubleValue.intValue(); + return intValue.toString(); + } + +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql b/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql index 5430367d5..db228dcc2 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql +++ b/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql @@ -3,7 +3,7 @@ delete from `dss_application` WHERE `name` ='exchangis'; INSERT INTO `dss_application`(`name`,`url`,`is_user_need_init`,`level`,`user_init_url`,`exists_project_service`,`project_url`,`enhance_json`,`if_iframe`,`homepage_url`,`redirect_url`) VALUES ('exchangis','',0,1,NULL,0,'','{\"watermark\":false,\"rsDownload\":true}',1,'',NULL); UPDATE `dss_application` SET url = 'http://APPCONN_INSTALL_IP:APPCONN_INSTALL_PORT' WHERE `name` ='exchangis'; -UPDATE `dss_application` SET project_url = 'http://APPCONN_INSTALL_IP:APPCONN_INSTALL_PORT/dss/exchangis/',homepage_url = 'http://APPCONN_INSTALL_IP:APPCONN_INSTALL_PORT/dss/exchangis/' WHERE `name` in('exchangis'); +UPDATE `dss_application` SET project_url = 'http://APPCONN_INSTALL_IP:APPCONN_INSTALL_PORT/#/projectManage',homepage_url = 'http://APPCONN_INSTALL_IP:APPCONN_INSTALL_PORT/#/projectManage' WHERE `name` in('exchangis'); select @dss_exchangis_applicationId:=id from `dss_application` WHERE `name` ='exchangis'; @@ -15,22 +15,22 @@ INSERT INTO `dss_onestop_menu_application` (`application_id`, `onestop_menu_id` delete from `dss_appconn` where `appconn_name`='exchangis'; -INSERT INTO `dss_appconn` (`appconn_name`, `is_user_need_init`, `level`, `if_iframe`, `is_external`, `reference`, `class_name`, `appconn_class_path`, `resource`) VALUES ('exchangis', 0, 1, NULL, 0, NULL, 'com.webank.wedatasphere.exchangis.appconn.ExchangisAppConn', 'DSS_INSTALL_HOME_VAL/dss-appconns/exchangis/lib', ''); +INSERT INTO `dss_appconn` (`appconn_name`, `is_user_need_init`, `level`, `if_iframe`, `is_external`, `reference`, `class_name`, `appconn_class_path`, `resource`) VALUES ('exchangis', 0, 1, NULL, 0, NULL, 'com.webank.wedatasphere.exchangis.dss.appconn.ExchangisAppConn', 'DSS_INSTALL_HOME_VAL/dss-appconns/exchangis/lib', ''); select @dss_appconn_exchangisId:=id from `dss_appconn` where `appconn_name` = 'exchangis'; delete from `dss_appconn_instance` where `homepage_url` like '%exchangis%'; -INSERT INTO `dss_appconn_instance` (`appconn_id`, `label`, `url`, `enhance_json`, `homepage_url`, `redirect_url`) VALUES (@dss_appconn_exchangisId, 'DEV', 'http://APPCONN_INSTALL_IP:APPCONN_INSTALL_PORT/', '', 'http://APPCONN_INSTALL_IP:APPCONN_INSTALL_PORT/dss/exchangis/', 'http://APPCONN_INSTALL_IP:APPCONN_INSTALL_PORT/'); +INSERT INTO `dss_appconn_instance` (`appconn_id`, `label`, `url`, `enhance_json`, `homepage_url`, `redirect_url`) VALUES (@dss_appconn_exchangisId, 'DEV', 'http://APPCONN_INSTALL_IP:APPCONN_INSTALL_PORT/', '', 'http://APPCONN_INSTALL_IP:APPCONN_INSTALL_PORT/#/projectManage', 'http://APPCONN_INSTALL_IP:APPCONN_INSTALL_PORT/'); delete from `dss_workflow_node` where `node_type` like '%exchangis%'; -insert into `dss_workflow_node` (`name`, `appconn_name`, `node_type`, `jump_url`, `support_jump`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `icon`) values('sqoop','exchangis','wds.dss.appconn.exchangis.sqoop','http://APPCONN_INSTALL_IP:APPCONN_INSTALL_PORT/dss/exchangis/#/childJobManagement','1','1','0','1',''); -insert into `dss_workflow_node` (`name`, `appconn_name`, `node_type`, `jump_url`, `support_jump`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `icon`) values('datax','exchangis','wds.dss.appconn.exchangis.datax','http://APPCONN_INSTALL_IP:APPCONN_INSTALL_PORT/dss/exchangis/#/childJobManagement','1','1','0','1',''); +insert into `dss_workflow_node` (`name`, `appconn_name`, `node_type`, `jump_url`, `support_jump`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `icon`) values('sqoop','exchangis','linkis.appconn.exchangis.sqoop','http://APPCONN_INSTALL_IP:APPCONN_INSTALL_PORT/#/childJobManagement','1','1','0','1',''); +-- insert into `dss_workflow_node` (`name`, `appconn_name`, `node_type`, `jump_url`, `support_jump`, `submit_to_scheduler`, `enable_copy`, `should_creation_before_node`, `icon`) values('datax','exchangis','linkis.appconn.exchangis.datax','http://APPCONN_INSTALL_IP:APPCONN_INSTALL_PORT/dss/exchangis/#/childJobManagement','1','1','0','1',''); -select @dss_exchangis_sqoopId:=id from `dss_workflow_node` where `node_type` = 'wds.dss.appconn.exchangis.sqoop'; -select @dss_exchangis_dataxId:=id from `dss_workflow_node` where `node_type` = 'wds.dss.appconn.exchangis.datax'; +select @dss_exchangis_sqoopId:=id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.sqoop'; +-- select @dss_exchangis_dataxId:=id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.datax'; delete from `dss_workflow_node_to_group` where `node_id`=@dss_exchangis_sqoopId; @@ -41,11 +41,11 @@ delete from `dss_workflow_node_to_ui` where `workflow_node_id`=@dss_exchangis_sq delete from `dss_workflow_node_to_ui` where `workflow_node_id`=@dss_exchangis_dataxId; -select @dss_exchangis_sqoopId:=id from `dss_workflow_node` where `node_type` = 'wds.dss.appconn.exchangis.sqoop'; -select @dss_exchangis_dataxId:=id from `dss_workflow_node` where `node_type` = 'wds.dss.appconn.exchangis.datax'; +select @dss_exchangis_sqoopId:=id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.sqoop'; +-- select @dss_exchangis_dataxId:=id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.datax'; INSERT INTO `dss_workflow_node_to_group`(`node_id`,`group_id`) values (@dss_exchangis_sqoopId,1); -INSERT INTO `dss_workflow_node_to_group`(`node_id`,`group_id`) values (@dss_exchangis_dataxId,1); +-- INSERT INTO `dss_workflow_node_to_group`(`node_id`,`group_id`) values (@dss_exchangis_dataxId,1); INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_sqoopId,1); @@ -55,11 +55,11 @@ INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_e INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_sqoopId,5); -INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_dataxId,1); -INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_dataxId,2); -INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_dataxId,3); -INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_dataxId,4); -INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_dataxId,5); +-- INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_dataxId,1); +-- INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_dataxId,2); +-- INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_dataxId,3); +-- INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_dataxId,4); +-- INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values (@dss_exchangis_dataxId,5); diff --git a/exchangis-plugins/exchangis-appconn/src/test/java/com/webank/wedatasphere/exchangis/appconn/TestAppConn.java b/exchangis-plugins/exchangis-appconn/src/test/java/com/webank/wedatasphere/exchangis/appconn/TestAppConn.java index 413eb339f..4347f6ea9 100644 --- a/exchangis-plugins/exchangis-appconn/src/test/java/com/webank/wedatasphere/exchangis/appconn/TestAppConn.java +++ b/exchangis-plugins/exchangis-appconn/src/test/java/com/webank/wedatasphere/exchangis/appconn/TestAppConn.java @@ -4,16 +4,21 @@ import com.google.gson.Gson; import org.apache.linkis.server.BDPJettyServerHelper; +import java.util.HashMap; import java.util.Map; public class TestAppConn { private final static Gson gson = new Gson(); public static void main(String[] args) throws JsonProcessingException { - String str="{\"method\":null,\"status\":0,\"message\":\"OK\",\"data\":{\"item\":{\"id\":\"1469200683600183298\",\"dssProjectId\":null,\"name\":\"DWExchangis06\",\"workspaceName\":\"DWExchangis06\",\"description\":\"测试不要删除\",\"tags\":\"\",\"editUsers\":\"\",\"viewUsers\":\"\",\"execUsers\":\"\",\"domain\":\"STANDALONE\"}}}"; - + //String str="{\"method\":null,\"status\":0,\"message\":\"OK\",\"data\":{\"item\":{\"id\":\"1469200683600183298\",\"dssProjectId\":null,\"name\":\"DWExchangis06\",\"workspaceName\":\"DWExchangis06\",\"description\":\"测试不要删除\",\"tags\":\"\",\"editUsers\":\"\",\"viewUsers\":\"\",\"execUsers\":\"\",\"domain\":\"STANDALONE\"}}}"; + String str = "{\"route\":\"prod\"}"; + Map labels = new HashMap<>(); Map responseMap = BDPJettyServerHelper.jacksonJson().readValue(str, Map.class); System.out.println(responseMap); + labels.put("labels", responseMap); + System.out.println(labels); Map item = (Map) ((Map) responseMap.get("data")).get("item"); System.out.println(item.get("id")); + } } diff --git a/exchangis-plugins/pom.xml b/exchangis-plugins/pom.xml index 8d19aee4b..55444b3d8 100644 --- a/exchangis-plugins/pom.xml +++ b/exchangis-plugins/pom.xml @@ -21,5 +21,19 @@ 8 8 - + + + + org.apache.maven.plugins + maven-assembly-plugin + 2.2.1 + + + assemble + none + + + + + \ No newline at end of file diff --git a/exchangis-project/exchangis-project-server/pom.xml b/exchangis-project/exchangis-project-server/pom.xml index 8d33f2bf4..6a0b22106 100644 --- a/exchangis-project/exchangis-project-server/pom.xml +++ b/exchangis-project/exchangis-project-server/pom.xml @@ -40,6 +40,12 @@ 1.0.0-RC1 compile + + + com.webank.wedatasphere.exchangis + exchangis-job-common + 1.0.0-RC1 + diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/dao/ExchangisProjectMapper.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/dao/ExchangisProjectMapper.java deleted file mode 100644 index c20047547..000000000 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/dao/ExchangisProjectMapper.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.webank.wedatasphere.exchangis.project.server.dao; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; -import org.apache.ibatis.annotations.Mapper; - -@Mapper -public interface ExchangisProjectMapper extends BaseMapper { - -} diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/dao/ExchangisProjectRelationMapper.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/dao/ExchangisProjectRelationMapper.java deleted file mode 100644 index b618f9e4f..000000000 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/dao/ExchangisProjectRelationMapper.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.webank.wedatasphere.exchangis.project.server.dao; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProjectRelation; -import org.apache.ibatis.annotations.Mapper; - -@Mapper -public interface ExchangisProjectRelationMapper extends BaseMapper { -} diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/dao/impl/ExchangisProjectMapper.xml b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/dao/impl/ExchangisProjectMapper.xml deleted file mode 100644 index ce101e917..000000000 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/dao/impl/ExchangisProjectMapper.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/domain/ProjectPageQuery.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/domain/ProjectPageQuery.java new file mode 100644 index 000000000..9a4a012f3 --- /dev/null +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/domain/ProjectPageQuery.java @@ -0,0 +1,42 @@ +package com.webank.wedatasphere.exchangis.project.server.domain; + +import com.webank.wedatasphere.exchangis.common.pager.PageQuery; +import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; + +/** + * For querying page + */ +public class ProjectPageQuery extends PageQuery { + /** + * Project name + */ + protected String name; + + protected String domain; + + protected String createUser; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDomain() { + return domain; + } + + public void setDomain(String domain) { + this.domain = domain; + } + + public String getCreateUser() { + return createUser; + } + + public void setCreateUser(String createUser) { + this.createUser = createUser; + } +} diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/dto/ExchangisProjectDTO.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/dto/ExchangisProjectDTO.java deleted file mode 100644 index ea6a958d0..000000000 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/dto/ExchangisProjectDTO.java +++ /dev/null @@ -1,78 +0,0 @@ -package com.webank.wedatasphere.exchangis.project.server.dto; - - -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; - -public class ExchangisProjectDTO { - - public ExchangisProjectDTO() {} - - public ExchangisProjectDTO(ExchangisProject project) { - - this.setId(String.valueOf(project.getId())); - Optional.ofNullable(project.getDssProjectId()).ifPresent(dssProjectId -> { - this.setDssProjectId(String.valueOf(dssProjectId)); - }); - this.setName(project.getName()); - this.setWorkspaceName(project.getWorkspaceName()); - this.setTags(project.getTags()); - this.setDescription(project.getDescription()); - this.setDomain(project.getDomain()); - } - - private String id; - private String dssProjectId; - private String name; - private String workspaceName; - private String description; - private String tags; - private String domain; - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getDssProjectId() { return dssProjectId; } - - public void setDssProjectId(String dssProjectId) { this.dssProjectId = dssProjectId; } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getWorkspaceName() { return workspaceName; } - - public void setWorkspaceName(String workspaceName) { this.workspaceName = workspaceName; } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getTags() { - return tags; - } - - public void setTags(String tags) { - this.tags = tags; - } - - public String getDomain() { return domain; } - - public void setDomain(String domain) { this.domain = domain; } -} diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/dto/ExchangisProjectGetDTO.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/dto/ExchangisProjectGetDTO.java deleted file mode 100644 index 14ad59a46..000000000 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/dto/ExchangisProjectGetDTO.java +++ /dev/null @@ -1,109 +0,0 @@ -package com.webank.wedatasphere.exchangis.project.server.dto; - -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; - -import java.util.Optional; - -public class ExchangisProjectGetDTO { - - public ExchangisProjectGetDTO() { - - } - - public ExchangisProjectGetDTO(ExchangisProject project) { - if (null == project) { - return; - } - this.setId(String.valueOf(project.getId())); - Optional.ofNullable(project.getDssProjectId()).ifPresent(dssProjectId -> { - this.setDssProjectId(String.valueOf(project.getDssProjectId())); - }); - this.setName(project.getName()); - this.setWorkspaceName(project.getWorkspaceName()); - this.setDescription(project.getDescription()); - this.setTags(project.getTags()); - this.setEditUsers(project.getEditUsers()); - this.setViewUsers(project.getViewUsers()); - this.setExecUsers(project.getExecUsers()); - this.setDomain(project.getDomain()); - } - - private String id; - private String dssProjectId; - private String name; - private String workspaceName; - private String description; - private String tags; - private String editUsers; - private String viewUsers; - private String execUsers; - private String domain; - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getDssProjectId() { return dssProjectId; } - - public void setDssProjectId(String dssProjectId) { this.dssProjectId = dssProjectId; } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getWorkspaceName() { return workspaceName; } - - public void setWorkspaceName(String workspaceName) { this.workspaceName = workspaceName; } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getTags() { - return tags; - } - - public void setTags(String tags) { - this.tags = tags; - } - - public String getEditUsers() { - return editUsers; - } - - public void setEditUsers(String editUsers) { - this.editUsers = editUsers; - } - - public String getViewUsers() { - return viewUsers; - } - - public void setViewUsers(String viewUsers) { - this.viewUsers = viewUsers; - } - - public String getExecUsers() { - return execUsers; - } - - public void setExecUsers(String execUsers) { - this.execUsers = execUsers; - } - - public String getDomain() { return domain; } - - public void setDomain(String domain) { this.domain = domain; } -} diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/dto/ExchangisProjectRelationDTO.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/dto/ExchangisProjectRelationDTO.java deleted file mode 100644 index b6156acd6..000000000 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/dto/ExchangisProjectRelationDTO.java +++ /dev/null @@ -1,77 +0,0 @@ -package com.webank.wedatasphere.exchangis.project.server.dto; - - -import javax.validation.constraints.NotEmpty; -import javax.validation.constraints.NotNull; - -public class ExchangisProjectRelationDTO { - - private String id; - @NotNull - @NotEmpty - private String projectId; - @NotNull - @NotEmpty - private String nodeId; - private String projectVersion; - private String flowVersion; - private String resourceId; - private String version; - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getProjectId() { - return projectId; - } - - public void setProjectId(String projectId) { - this.projectId = projectId; - } - - public String getNodeId() { - return nodeId; - } - - public void setNodeId(String nodeId) { - this.nodeId = nodeId; - } - - public String getProjectVersion() { - return projectVersion; - } - - public void setProjectVersion(String projectVersion) { - this.projectVersion = projectVersion; - } - - public String getFlowVersion() { - return flowVersion; - } - - public void setFlowVersion(String flowVersion) { - this.flowVersion = flowVersion; - } - - public String getResourceId() { - return resourceId; - } - - public void setResourceId(String resourceId) { - this.resourceId = resourceId; - } - - public String getVersion() { - return version; - } - - public void setVersion(String version) { - this.version = version; - } - -} diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/entity/ExchangisProject.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/entity/ExchangisProject.java index 7744e235a..924be5031 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/entity/ExchangisProject.java +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/entity/ExchangisProject.java @@ -1,34 +1,88 @@ package com.webank.wedatasphere.exchangis.project.server.entity; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.linkis.common.utils.JsonUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; public class ExchangisProject { - public static enum Domain { - WORKSPACE, STANDALONE + private static final Logger LOG = LoggerFactory.getLogger(ExchangisProject.class); + + public enum Domain { + DSS, STANDALONE } + /** + * Id (Long value) + */ private Long id; - private Long dssProjectId; + /** + * Project name + */ private String name; - private String dssName; + /** + * Description + */ private String description; - private String createBy; + + /** + * Create user + */ + private String createUser; + + /** + * Create time + */ private Date createTime; - private String lastUpdateBy; + + /** + * Last update user + */ + private String lastUpdateUser; + + /** + * Last update time + */ private Date lastUpdateTime; - private String tags; + + /** + * Labels + */ + private String labels; + + /** + * User has editing permission + */ private String editUsers; + + /** + * User has viewing permission + */ private String viewUsers; + + /** + * User has executing permission + */ private String execUsers; - private String workspaceName; + + /** + * Domain + */ private String domain; -// public ExchangisProject(String name, String description, String workspaceName){ -// this.name = name; -// this.description = description; -// this.workspaceName = workspaceName; -// } + /** + * Source map + */ + private Map sourceMap = new HashMap<>(); + + private String source; public Long getId() { return id; @@ -46,10 +100,6 @@ public void setName(String name) { this.name = name; } - public String getDssName() { return dssName; } - - public void setDssName(String dssName) { this.dssName = dssName; } - public String getDescription() { return description; } @@ -58,14 +108,6 @@ public void setDescription(String description) { this.description = description; } - public String getCreateBy() { - return createBy; - } - - public void setCreateBy(String createBy) { - this.createBy = createBy; - } - public Date getCreateTime() { return createTime; } @@ -74,14 +116,6 @@ public void setCreateTime(Date createTime) { this.createTime = createTime; } - public String getLastUpdateBy() { - return lastUpdateBy; - } - - public void setLastUpdateBy(String lastUpdateBy) { - this.lastUpdateBy = lastUpdateBy; - } - public Date getLastUpdateTime() { return lastUpdateTime; } @@ -90,30 +124,6 @@ public void setLastUpdateTime(Date lastUpdateTime) { this.lastUpdateTime = lastUpdateTime; } - public String getTags() { - return tags; - } - - public void setTags(String tags) { - this.tags = tags; - } - - public String getWorkspaceName() { - return workspaceName; - } - - public void setWorkspaceName(String workspaceName) { - this.workspaceName = workspaceName; - } - - public Long getDssProjectId() { - return dssProjectId; - } - - public void setDssProjectId(Long dssProjectId) { - this.dssProjectId = dssProjectId; - } - public String getEditUsers() { return editUsers; } @@ -141,4 +151,60 @@ public void setExecUsers(String execUsers) { public String getDomain() { return domain; } public void setDomain(String domain) { this.domain = domain; } + + public String getCreateUser() { + return createUser; + } + + public void setCreateUser(String createUser) { + this.createUser = createUser; + } + + public String getLastUpdateUser() { + return lastUpdateUser; + } + + public void setLastUpdateUser(String lastUpdateUser) { + this.lastUpdateUser = lastUpdateUser; + } + + public String getLabels() { + return labels; + } + + public void setLabels(String labels) { + this.labels = labels; + } + + // TODO use the common Json util + public Map getSourceMap() { + if (Objects.isNull(this.sourceMap) && Objects.nonNull(this.source)){ + try { + ObjectMapper mapper = JsonUtils.jackson(); + this.sourceMap = mapper.readValue(this.source, mapper.getTypeFactory() + .constructParametricType(Map.class, String.class, Object.class)); + } catch (JsonProcessingException e) { + //Ignore + LOG.warn("Cannot deserialize the source string: {}", this.source, e); + } + } + return sourceMap; + } + + public void setSourceMap(Map sourceMap) { + this.sourceMap = sourceMap; + } + + // TODO use the common Json util + public String getSource() { + if (Objects.isNull(this.source) && Objects.nonNull(this.sourceMap)){ + try { + this.source = JsonUtils.jackson().writeValueAsString(this.sourceMap); + } catch (JsonProcessingException e) { + // Ignore + LOG.warn("Cannot serialize the source map", e); + } + } + return source; + } } diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/entity/ExchangisProjectRelation.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/entity/ExchangisProjectRelation.java deleted file mode 100644 index 632bfdfc8..000000000 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/entity/ExchangisProjectRelation.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.webank.wedatasphere.exchangis.project.server.entity; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableField; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; - -@TableName("exchangis_project_relation") -public class ExchangisProjectRelation { - - @TableId(value = "id", type = IdType.AUTO) - private Long id; - - @TableField("project_id") - private Long projectId; - @TableField("node_id") - private Long nodeId; - @TableField("project_version") - private String projectVersion; - @TableField("flow_version") - private String flowVersion; - @TableField("resource_id") - private Long resourceId; - @TableField("version") - private String version; - - public Long getId() { return id; } - - public void setId(Long id) { this.id = id; } - - public Long getProjectId() { return projectId; } - - public void setProjectId(Long projectId) { this.projectId = projectId; } - - public Long getNodeId() { return nodeId; } - - public void setNodeId(Long nodeId) { this.nodeId = nodeId; } - - public String getProjectVersion() { return projectVersion; } - - public void setProjectVersion(String projectVersion) { this.projectVersion = projectVersion; } - - public String getFlowVersion() { return flowVersion; } - - public void setFlowVersion(String flowVersion) { this.flowVersion = flowVersion; } - - public Long getResourceId() { return resourceId; } - - public void setResourceId(Long resourceId) { this.resourceId = resourceId; } - - public String getVersion() { return version; } - - public void setVersion(String version) { this.version = version; } -} diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/ProjectMapper.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/ProjectMapper.java new file mode 100644 index 000000000..7eeb19428 --- /dev/null +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/ProjectMapper.java @@ -0,0 +1,75 @@ +package com.webank.wedatasphere.exchangis.project.server.mapper; + +import com.webank.wedatasphere.exchangis.project.server.domain.ProjectPageQuery; +import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +/** + * Project mapper + */ +public interface ProjectMapper { + + /** + * Query the page list + * @param pageQuery page query + * @return page list + */ + List queryPageList(ProjectPageQuery pageQuery); + + /** + * Get detail by id + * @param projectId project id + * @return project entity + */ + ExchangisProject getDetailById(Long projectId); + + /** + * Get basic info by id + * @param projectId project id + * @return project entity + */ + ExchangisProject getBasicById(Long projectId); + /** + * Insert project + * @param project project entity + * @return project id + */ + long insertOne(ExchangisProject project); + + /** + * If exists + * @param projectId project id + * @param projectName project name + * @return int + */ + Integer existsOne(@Param("projectId")Long projectId, @Param("projectName")String projectName); + + /** + * Update one + * @param project project entity + */ + void updateOne(ExchangisProject project); + + /** + * Delete project + * @param projectId + */ + void deleteOne(Long projectId); + + /** + * Delete project by name + * @param name + */ + void deleteByName(String name); + + /** + * get projects by name + * @param projectName + * @return List + */ + List getDetailByName(@Param("projectName") String projectName); + + ExchangisProject selectByName(String name); +} diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/impl/ProjectMapper.xml b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/impl/ProjectMapper.xml new file mode 100644 index 000000000..9871e1094 --- /dev/null +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/mapper/impl/ProjectMapper.xml @@ -0,0 +1,134 @@ + + + + + + + + + + + + + + + + + + + + + + + + id, name, description, create_user, create_time, last_update_user, + last_update_time, project_labels, edit_users, view_users, exec_users, domain, source + + + id, name, description, create_user, create_time, last_update_user, + last_update_time, project_labels, domain + + + + + + + AND name like concat('%', #{name}, '%') + + + AND create_user = #{createUser} + + + AND domain = #{domain} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + `id` = #{id}; + + + + + + + + + + + + DELETE FROM exchangis_project_info WHERE name = #{name} + + + + + + + + diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/request/CreateProjectRequest.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/request/CreateProjectRequest.java deleted file mode 100644 index 3c8b598ad..000000000 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/request/CreateProjectRequest.java +++ /dev/null @@ -1,90 +0,0 @@ -package com.webank.wedatasphere.exchangis.project.server.request; - -public class CreateProjectRequest { - - private String dssProjectId; - - private String workspaceName; - - private String projectName; - - private String dssProjectName; - - private String description; - - private String tags; - - private String editUsers; - - private String viewUsers; - - private String execUsers; - - public String getDssProjectId() { - return dssProjectId; - } - - public void setDssProjectId(String dssProjectId) { - this.dssProjectId = dssProjectId; - } - - public String getProjectName() { - return projectName; - } - - public void setProjectName(String projectName) { - this.projectName = projectName; - } - - public String getDssProjectName() { return dssProjectName; } - - public void setDssProjectName(String dssProjectName) { this.dssProjectName = dssProjectName; } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getTags() { - return tags; - } - - public void setTags(String tags) { - this.tags = tags; - } - - public String getWorkspaceName() { - return workspaceName; - } - - public void setWorkspaceName(String workspaceName) { - this.workspaceName = workspaceName; - } - - public String getEditUsers() { - return editUsers; - } - - public void setEditUsers(String editUsers) { - this.editUsers = editUsers; - } - - public String getViewUsers() { - return viewUsers; - } - - public void setViewUsers(String viewUsers) { - this.viewUsers = viewUsers; - } - - public String getExecUsers() { - return execUsers; - } - - public void setExecUsers(String execUsers) { - this.execUsers = execUsers; - } -} diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/request/ProjectQueryRequest.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/request/ProjectQueryRequest.java deleted file mode 100644 index af61e8165..000000000 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/request/ProjectQueryRequest.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.webank.wedatasphere.exchangis.project.server.request; - -import java.io.Serializable; - -public class ProjectQueryRequest implements Serializable { - - private static final long serialVersionUID=1L; - - private String username; - - private String name; - - private String domain; - - public String getUsername() { - return username; - } - - public void setUsername(String username) { - this.username = username; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getDomain() { return domain; } - - public void setDomain(String domain) { this.domain = domain; } -} diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/request/UpdateProjectRequest.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/request/UpdateProjectRequest.java deleted file mode 100644 index 0d129837e..000000000 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/request/UpdateProjectRequest.java +++ /dev/null @@ -1,92 +0,0 @@ -package com.webank.wedatasphere.exchangis.project.server.request; - -public class UpdateProjectRequest { - - private Long id; - - private Long dssProjectId; - - private String workspaceName; - - private String projectName; - - private String dssProjectName; - - private String description; - - private String tags; - - private String editUsers; - - private String viewUsers; - - private String execUsers; - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public Long getDssProjectId() { return dssProjectId; } - - public void setDssProjectId(Long dssProjectId) { this.dssProjectId = dssProjectId; } - - public String getWorkspaceName() { return workspaceName; } - - public void setWorkspaceName(String workspaceName) { this.workspaceName = workspaceName; } - - public String getProjectName() { - return projectName; - } - - public void setProjectName(String projectName) { - this.projectName = projectName; - } - - public String getDssProjectName() { return dssProjectName; } - - public void setDssProjectName(String dssProjectName) { this.dssProjectName = dssProjectName; } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getTags() { - return tags; - } - - public void setTags(String tags) { - this.tags = tags; - } - - public String getEditUsers() { - return editUsers; - } - - public void setEditUsers(String editUsers) { - this.editUsers = editUsers; - } - - public String getViewUsers() { - return viewUsers; - } - - public void setViewUsers(String viewUsers) { - this.viewUsers = viewUsers; - } - - public String getExecUsers() { - return execUsers; - } - - public void setExecUsers(String execUsers) { - this.execUsers = execUsers; - } -} diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRelationRestful.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRelationRestful.java deleted file mode 100644 index b46975b0d..000000000 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRelationRestful.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.webank.wedatasphere.exchangis.project.server.restful; - -import com.webank.wedatasphere.exchangis.project.server.dto.ExchangisProjectRelationDTO; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProjectRelation; -import com.webank.wedatasphere.exchangis.project.server.service.ExchangisProjectRelationService; -import org.apache.linkis.server.Message; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.*; - -import javax.validation.Valid; -import java.util.Optional; - -@RestController -@RequestMapping(value = "/exchangis/relation", produces = {"application/json;charset=utf-8"}) -public class ExchangisProjectRelationRestful { - - private static final Logger LOGGER = LoggerFactory.getLogger(ExchangisProjectRelationRestful.class); - - @Autowired - private ExchangisProjectRelationService projectRelationService; - - @RequestMapping( value = "", method = RequestMethod.POST) - public Message save(@Valid @RequestBody ExchangisProjectRelationDTO dto) { - this.projectRelationService.save(dto); - return Message.ok(); - } - - @RequestMapping( value = "/nodeid/{nodeId}", method = RequestMethod.GET) - public Message getByNodeId(@PathVariable("nodeId") Long nodeId) { - Optional optional = this.projectRelationService.getByNodeId(nodeId); - ExchangisProjectRelationDTO dto = null; - if (optional.isPresent()) { - ExchangisProjectRelation r = optional.get(); - dto = new ExchangisProjectRelationDTO(); - dto.setId(String.valueOf(r.getId())); - dto.setProjectId(String.valueOf(r.getProjectId())); - dto.setNodeId(String.valueOf(r.getNodeId())); - dto.setProjectVersion(r.getProjectVersion()); - dto.setResourceId(String.valueOf(r.getResourceId())); - dto.setFlowVersion(r.getFlowVersion()); - dto.setVersion(r.getVersion()); - } - return Message.ok().data("data", dto); - } - - -} diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java index 1e5c20356..3c3b1a9f8 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/ExchangisProjectRestfulApi.java @@ -1,148 +1,217 @@ package com.webank.wedatasphere.exchangis.project.server.restful; -import com.webank.wedatasphere.exchangis.project.server.dto.ExchangisProjectDTO; -import com.webank.wedatasphere.exchangis.project.server.dto.ExchangisProjectGetDTO; +import com.webank.wedatasphere.exchangis.common.pager.PageResult; +import com.webank.wedatasphere.exchangis.common.validator.groups.UpdateGroup; import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; -import com.webank.wedatasphere.exchangis.project.server.request.CreateProjectRequest; -import com.webank.wedatasphere.exchangis.project.server.request.ProjectQueryRequest; -import com.webank.wedatasphere.exchangis.project.server.request.UpdateProjectRequest; -import com.webank.wedatasphere.exchangis.project.server.service.ExchangisProjectService; +import com.webank.wedatasphere.exchangis.project.server.service.ProjectService; import com.webank.wedatasphere.exchangis.project.server.utils.ExchangisProjectRestfulUtils; +import com.webank.wedatasphere.exchangis.project.server.vo.ExchangisProjectInfo; +import com.webank.wedatasphere.exchangis.project.server.vo.ProjectQueryVo; +import org.apache.commons.lang.StringUtils; import org.apache.commons.math3.util.Pair; +import org.apache.linkis.common.utils.JsonUtils; import org.apache.linkis.server.Message; import org.apache.linkis.server.security.SecurityFilter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.validation.BindingResult; +import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.*; +import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; -import javax.validation.Valid; -import java.util.List; +import javax.validation.groups.Default; +import java.util.Objects; +import java.util.Optional; /** - * this is the restful class for exchangis project + * This is the restful class for exchangis project */ @RestController -@RequestMapping(value = "/exchangis", produces = {"application/json;charset=utf-8"}) +@RequestMapping(value = "/dss/exchangis/main", produces = {"application/json;charset=utf-8"}) public class ExchangisProjectRestfulApi { - private static final Logger LOGGER = LoggerFactory.getLogger(ExchangisProjectRestfulApi.class); + private static final Logger LOG = LoggerFactory.getLogger(ExchangisProjectRestfulApi.class); - @Autowired - private ExchangisProjectService projectService; + /** + * Project service + */ + @Resource + private ProjectService projectService; - @RequestMapping( value = "projects", method = RequestMethod.POST) + /** + * Project query + * @param request http request + * @param queryVo query vo + * @param current current page + * @param size size + * @param name name + * @return message + */ + @RequestMapping( value = "projects", method = {RequestMethod.POST, RequestMethod.GET}) public Message queryProjects(HttpServletRequest request, - @Valid @RequestBody ProjectQueryRequest projectQueryRequest, - @RequestParam(value = "current", required = false) int current, - @RequestParam(value = "size", required = false) int size, + @RequestBody ProjectQueryVo queryVo, + @RequestParam(value = "current", required = false) Integer current, + @RequestParam(value = "size", required = false) Integer size, @RequestParam(value = "name", required = false) String name) { String username = SecurityFilter.getLoginUsername(request); - if (null == projectQueryRequest) { - projectQueryRequest = new ProjectQueryRequest(); - } - projectQueryRequest.setUsername(username); + Optional.ofNullable(current).ifPresent(queryVo::setCurrent); + Optional.ofNullable(size).ifPresent(queryVo::setSize); + Optional.ofNullable(name).ifPresent(queryVo::setName); + queryVo.setCreateUser(username); try { - List projects = projectService.queryProjects(projectQueryRequest, current, size, name); - int total = projectService.count(projectQueryRequest, name); - Message message = Message.ok(); - message.data("total", total); - message.data("list", projects); - return message; - //return Message.ok().data("list", projects); - } catch (final Throwable t) { - LOGGER.error("failed to create project for user {}", username, t); - return Message.error("获取工程列表失败,原因是:" + t.getMessage()); + PageResult pageResult = projectService.queryProjects(queryVo); + return pageResult.toMessage(); + } catch (Exception t) { + LOG.error("Failed to query project list for user {}", username, t); + return Message.error("Failed to query project list (获取工程列表失败)"); } } - @RequestMapping( value = "projects/{projectId}", method = RequestMethod.GET) - public Message queryProjects(HttpServletRequest request, - @PathVariable("projectId") String projectId) { + /** + * Project query detail by id + * @param request http request + * @param projectId project id + * @return + */ + @RequestMapping( value = "projects/{projectId:\\d+}", method = RequestMethod.GET) + public Message queryProjectDetail(HttpServletRequest request, + @PathVariable("projectId") Long projectId) { String username = SecurityFilter.getLoginUsername(request); try { - ExchangisProjectGetDTO dto = projectService.getProjectById(projectId); - return Message.ok().data("item", dto); - } catch (final Throwable t) { - LOGGER.error("failed to create project for user {}", username, t); - return Message.error("获取工程列表失败,原因是:" + t.getMessage()); + ExchangisProjectInfo project = projectService.getProjectDetailById(projectId); + if (Objects.isNull(project)){ + return Message.error("Not found the project (找不到对应工程)"); + } + if (!hasAuthority(username, project)){ + return Message.error("You have no permission to query (没有工程查看权限)"); + } + return Message.ok().data("item", project); + } catch (Exception t) { + LOG.error("failed to get project detail for user {}", username, t); + return Message.error("Fail to get project detail (获取工程详情失败)"); } } - @RequestMapping(value = "projects/dss/{dssName}", method = RequestMethod.GET) - public Message queryProjectsByDss(HttpServletRequest request, - @PathVariable("dssName") String dssName) { + /** + * Create project + * @param request request + * @param projectVo project vo + * @param result result + * @return + */ + @RequestMapping(value = "createProject", method = RequestMethod.POST) + public Message createProject(@Validated @RequestBody ExchangisProjectInfo projectVo, + BindingResult result, HttpServletRequest request) { + if (result.hasErrors()){ + return Message.error(result.getFieldErrors().get(0).getDefaultMessage()); + } String username = SecurityFilter.getLoginUsername(request); try { - LOGGER.info("queryProjectsByDss workspaceName {}", dssName); - ExchangisProjectGetDTO dto = projectService.getProjectByDssName(dssName); - return Message.ok().data("item", dto); - } catch (final Throwable t) { - LOGGER.error("failed to create project for user {}", username, t); - return Message.error("获取工程列表失败,原因是:" + t.getMessage()); + if (projectService.existsProject(null, projectVo.getName())){ + return Message.error("Have the same name project (存在同名工程)"); + } + LOG.info("CreateProject vo: {}, userName: {}", JsonUtils.jackson().writeValueAsString(projectVo), username); + long projectId = projectService.createProject(projectVo, username); + return ExchangisProjectRestfulUtils.dealOk("创建工程成功", + new Pair<>("projectName", projectVo.getName()), + new Pair<>("projectId", projectId)); + } catch (Exception t) { + LOG.error("Failed to create project for user {}", username, t); + return Message.error("Fail to create project (创建工程失败)"); } } - - @RequestMapping(value = "createProject", method = RequestMethod.POST) - public Message createProject(HttpServletRequest request, - @Valid @RequestBody CreateProjectRequest createProjectRequest) { + /** + * check project name + * @param request http request + * @param name project name + * @return + */ + @RequestMapping( value = "/check/{name}", method = RequestMethod.POST) + public Message getProjectByName(HttpServletRequest request, @PathVariable("name") String name) { String username = SecurityFilter.getLoginUsername(request); - if (createProjectRequest.getProjectName().length() > 64) { - return Message.error("数据源名称输入过长"); - } - else { - try { - LOGGER.info("createProject createProjectRequest {}", createProjectRequest.toString()); - ExchangisProject exchangisProject = projectService.createProject(username, createProjectRequest); - return ExchangisProjectRestfulUtils.dealOk("创建工程成功", - new Pair<>("projectName", exchangisProject.getName()), new Pair<>("projectId", exchangisProject.getId())); - } catch (final Throwable t) { - LOGGER.error("failed to create project for user {}", username, t); - return Message.error("创建工程失败,存在同名项目"); - //return Message.error("创建工程失败,原因是:" + t.getMessage()); - } + try { + ExchangisProjectInfo projectInfo = projectService.selectByName(name); + return ExchangisProjectRestfulUtils.dealOk("根据名字获取工程成功", + new Pair<>("projectInfo",projectInfo)); + } catch (Exception t) { + LOG.error("Failed to delete project for user {}", username, t); + return Message.error("Failed to delete project (根据名字获取工程失败)"); } + + } + + /** + * Update project + * @param request request + * @param projectVo project vo + * @return + */ @RequestMapping( value = "updateProject", method = RequestMethod.PUT) - public Message updateProject(HttpServletRequest request, - @Valid @RequestBody UpdateProjectRequest updateProjectRequest) { + public Message updateProject(@Validated({UpdateGroup.class, Default.class}) @RequestBody ExchangisProjectInfo projectVo + , BindingResult result, HttpServletRequest request) { + if (result.hasErrors()){ + return Message.error(result.getFieldErrors().get(0).getDefaultMessage()); + } String username = SecurityFilter.getLoginUsername(request); try { - LOGGER.info("updateProject updateProjectRequest {}", updateProjectRequest.toString()); - ExchangisProject exchangisProject = projectService.updateProject(username, updateProjectRequest); + ExchangisProjectInfo projectStored = projectService.getProjectById(Long.valueOf(projectVo.getId())); + if (!hasAuthority(username, projectStored)){ + return Message.error("You have no permission to update (没有项目的更新权限)"); + } + String domain = projectStored.getDomain(); + if (StringUtils.isNotBlank(domain) && !ExchangisProject.Domain.STANDALONE.name() + .equalsIgnoreCase(domain)){ + return Message.error("Cannot update the outer project (无法更新来自 " + domain + " 的外部项目)"); + } + LOG.info("UpdateProject vo: {}, userName: {}", JsonUtils.jackson().writeValueAsString(projectVo), username); + projectService.updateProject(projectVo, username); return ExchangisProjectRestfulUtils.dealOk("更新工程成功", - new Pair<>("projectName", exchangisProject.getName()), new Pair<>("projectId", exchangisProject.getId())); - } catch (final Throwable t) { - LOGGER.error("failed to update project for user {}", username, t); - return Message.error("更新工程失败,原因是:" + t.getMessage()); + new Pair<>("projectName", projectVo.getName()), + new Pair<>("projectId", projectVo.getId())); + } catch (Exception t) { + LOG.error("Failed to update project for user {}", username, t); + return Message.error("Fail to update project (更新工程失败)"); } } - @RequestMapping( value = "/projects/{id}", method = RequestMethod.DELETE) - public Message deleteProject(HttpServletRequest request, @PathVariable("id") String id) { + /** + * Delete project + * @param request http request + * @param id project id + * @return + */ + @RequestMapping( value = "/projects/{id:\\d+}", method = RequestMethod.DELETE) + public Message deleteProject(HttpServletRequest request, @PathVariable("id") Long id) { String username = SecurityFilter.getLoginUsername(request); try { - projectService.deleteProject(request, id); + ExchangisProjectInfo projectInfo = projectService.getProjectById(id); + if (!hasAuthority(username, projectInfo)){ + return Message.error("You have no permission to delete (删除工程失败)"); + } + String domain = projectInfo.getDomain(); + if (StringUtils.isNotBlank(domain) && !ExchangisProject.Domain.STANDALONE.name() + .equalsIgnoreCase(domain)){ + return Message.error("Cannot delete the outer project (无法删除来自 " + domain + " 的外部项目)"); + } + projectService.deleteProject(id); return ExchangisProjectRestfulUtils.dealOk("删除工程成功"); - } catch (final Throwable t) { - LOGGER.error("failed to update project for user {}", username, t); - return Message.error("删除工程失败,原因是:" + t.getMessage()); + } catch (Exception t) { + LOG.error("Failed to delete project for user {}", username, t); + return Message.error("Failed to delete project (删除工程失败)"); } } - @RequestMapping( value = "/projects/dss/{dssName}", method = RequestMethod.DELETE) - public Message deleteProjectByDss(HttpServletRequest request, @PathVariable("dssName") String dssName) { - String username = SecurityFilter.getLoginUsername(request); - try { - LOGGER.info("deleteProjectByDss dssName {}", dssName); - projectService.deleteProjectByDss(request, dssName); - return ExchangisProjectRestfulUtils.dealOk("删除工程成功"); - } catch (final Throwable t) { - LOGGER.error("failed to update project for user {}", username, t); - return Message.error("删除工程失败,原因是:" + t.getMessage()); - } + /** + * TODO complete the authority strategy + * @param username username + * @param project project + * @return + */ + private boolean hasAuthority(String username, ExchangisProjectInfo project){ + return Objects.nonNull(project) && username.equals(project.getCreateUser()); } } diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectDssAppConnRestfulApi.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectDssAppConnRestfulApi.java index 529a35c1b..bd44bcde8 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectDssAppConnRestfulApi.java +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/restful/external/ExchangisProjectDssAppConnRestfulApi.java @@ -1,4 +1,157 @@ package com.webank.wedatasphere.exchangis.project.server.restful.external; +import com.webank.wedatasphere.exchangis.common.validator.groups.UpdateGroup; +import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; +import com.webank.wedatasphere.exchangis.project.server.service.ProjectService; +import com.webank.wedatasphere.exchangis.project.server.utils.ExchangisProjectRestfulUtils; +import com.webank.wedatasphere.exchangis.project.server.vo.ExchangisProjectAppVo; +import com.webank.wedatasphere.exchangis.project.server.vo.ExchangisProjectAppVo; +import com.webank.wedatasphere.exchangis.project.server.vo.ExchangisProjectInfo; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.math3.util.Pair; +import org.apache.linkis.common.utils.JsonUtils; +import org.apache.linkis.server.Message; +import org.apache.linkis.server.security.SecurityFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.validation.BindingResult; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; + +import javax.annotation.Resource; +import javax.servlet.http.HttpServletRequest; +import javax.validation.groups.Default; +import java.util.Objects; + +/** + * Restful class for dss project + */ +@RestController +@RequestMapping(value = "/dss/exchangis/main/appProject", produces = {"application/json;charset=utf-8"}) public class ExchangisProjectDssAppConnRestfulApi { + private static final Logger LOG = LoggerFactory.getLogger(ExchangisProjectDssAppConnRestfulApi.class); + + /** + * Project service + */ + @Resource + private ProjectService projectService; + + @RequestMapping(value = "", method = RequestMethod.POST) + public Message createProject(@Validated @RequestBody ExchangisProjectAppVo project, + BindingResult result, HttpServletRequest request){ + ExchangisProjectInfo projectVo = new ExchangisProjectInfo(project); + if (result.hasErrors()){ + return Message.error(result.getFieldErrors().get(0).getDefaultMessage()); + } + String username = SecurityFilter.getLoginUsername(request); + try { + LOG.info("CreateProject from DSS AppConn, vo: {}, userName: {}", JsonUtils.jackson().writeValueAsString(projectVo), username); + if (projectService.existsProject(null, projectVo.getName())){ + return Message.error("Have the same name project (存在同名工程)"); + } + long projectId = projectService.createProject(projectVo, username); + return ExchangisProjectRestfulUtils.dealOk("创建工程成功", + new Pair<>("projectName", projectVo.getName()), + new Pair<>("projectId", projectId)); + } catch (Exception t) { + LOG.error("Failed to create project for user {} from DSS", username, t); + return Message.error("Fail to create project from DSS(创建工程失败)"); + } + } + + + /** + * Update project + * @param request request + * @param projectVo project vo + * @return + */ + @RequestMapping( value = "/{id:\\d+}", method = RequestMethod.PUT) + public Message updateProject(@PathVariable("id") Long id, @Validated({UpdateGroup.class, Default.class}) @RequestBody ExchangisProjectInfo projectVo + , BindingResult result, HttpServletRequest request) { + if (result.hasErrors()){ + return Message.error(result.getFieldErrors().get(0).getDefaultMessage()); + } + String username = SecurityFilter.getLoginUsername(request); + try { + ExchangisProjectInfo projectStored = projectService.getProjectById(Long.valueOf(projectVo.getId())); + if (!hasAuthority(username, projectStored)){ + return Message.error("You have no permission to update (没有项目的更新权限)"); + } + String domain = projectStored.getDomain(); + if (StringUtils.isNotBlank(domain) && !ExchangisProject.Domain.STANDALONE.name() + .equalsIgnoreCase(domain)){ + return Message.error("Cannot update the outer project (无法更新来自 " + domain + " 的外部项目)"); + } + LOG.info("UpdateProject vo: {}, userName: {}", JsonUtils.jackson().writeValueAsString(projectVo), username); + projectService.updateProject(projectVo, username); + return ExchangisProjectRestfulUtils.dealOk("更新工程成功", + new Pair<>("projectName", projectVo.getName()), + new Pair<>("projectId", projectVo.getId())); + } catch (Exception t) { + LOG.error("Failed to update project for user {}", username, t); + return Message.error("Fail to update project (更新工程失败)"); + } + } + + /** + * Delete project + * @param request http request + * @param name project name + * @return + */ + @RequestMapping( value = "/{name}", method = RequestMethod.POST) + public Message deleteProject(HttpServletRequest request, @PathVariable("name") String name) { + String username = SecurityFilter.getLoginUsername(request); + try { + ExchangisProjectInfo projectInfo = projectService.selectByName(name); +// if (!hasAuthority(username, projectInfo)){ +// return Message.error("You have no permission to delete (删除工程失败)"); +// } +// String domain = projectInfo.getDomain(); +// if (StringUtils.isNotBlank(domain) && !ExchangisProject.Domain.STANDALONE.name() +// .equalsIgnoreCase(domain)){ +// return Message.error("Cannot delete the outer project (无法删除来自 " + domain + " 的外部项目)"); +// } + projectService.deleteProjectByName(name); + return ExchangisProjectRestfulUtils.dealOk("删除工程成功"); + } catch (Exception t) { + LOG.error("Failed to delete project for user {}", username, t); + return Message.error("Failed to delete project (删除工程失败)"); + } + + } + + + /** + * check project name + * @param request http request + * @param name project name + * @return + */ + @RequestMapping( value = "/check/{name}", method = RequestMethod.POST) + public Message getProjectByName(HttpServletRequest request, @PathVariable("name") String name) { + String username = SecurityFilter.getLoginUsername(request); + try { + ExchangisProjectInfo projectInfo = projectService.selectByName(name); + return ExchangisProjectRestfulUtils.dealOk("根据名字获取工程成功", + new Pair<>("projectInfo",projectInfo)); + } catch (Exception t) { + LOG.error("Failed to delete project for user {}", username, t); + return Message.error("Failed to delete project (根据名字获取工程失败)"); + } + + + } + + + /** + * @param username username + * @param project project + * @return + */ + private boolean hasAuthority(String username, ExchangisProjectInfo project){ + return Objects.nonNull(project) && username.equals(project.getCreateUser()); + } } diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/ExchangisProjectRelationService.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/ExchangisProjectRelationService.java deleted file mode 100644 index 89274b678..000000000 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/ExchangisProjectRelationService.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.webank.wedatasphere.exchangis.project.server.service; - -import com.webank.wedatasphere.exchangis.project.server.dto.ExchangisProjectRelationDTO; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProjectRelation; - -import java.util.Optional; - -public interface ExchangisProjectRelationService { - - void save(ExchangisProjectRelationDTO dto); - Optional getByNodeId(Long nodeId); - -} diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/ExchangisProjectService.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/ExchangisProjectService.java deleted file mode 100644 index 3905fa5a8..000000000 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/ExchangisProjectService.java +++ /dev/null @@ -1,42 +0,0 @@ -package com.webank.wedatasphere.exchangis.project.server.service; - - -import com.webank.wedatasphere.exchangis.project.server.dto.ExchangisProjectDTO; -import com.webank.wedatasphere.exchangis.project.server.dto.ExchangisProjectGetDTO; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; -import com.webank.wedatasphere.exchangis.project.server.exception.ExchangisProjectErrorException; -import com.webank.wedatasphere.exchangis.project.server.request.CreateProjectRequest; -import com.webank.wedatasphere.exchangis.project.server.request.ProjectQueryRequest; -import com.webank.wedatasphere.exchangis.project.server.request.UpdateProjectRequest; - -import javax.servlet.http.HttpServletRequest; -import java.util.List; - -public interface ExchangisProjectService { - - ExchangisProject createProject(String username, CreateProjectRequest createProjectRequest) throws ExchangisProjectErrorException; - - ExchangisProject updateProject(String username, UpdateProjectRequest updateProjectRequest) throws ExchangisProjectErrorException; - - List queryProjects(ProjectQueryRequest projectQueryRequest, int current, int size, String name); - - int count(ProjectQueryRequest projectQueryRequest, String name); - - void deleteProject(HttpServletRequest request, String id); - - void deleteProjectByDss(HttpServletRequest request, String dssName); - - ExchangisProjectGetDTO getProjectById(String projectId); - - ExchangisProjectGetDTO getProjectByDssName(String dssName); - -// ExchangisProject createProject(CreateStreamProjectRequest createStreamProjectRequest) throws ExchangisProjectErrorException; -// -// -// void updateProject(UpdateStreamProjectRequest updateStreamProjectRequest) throws ExchangisProjectErrorException; -// -// -// void deleteProject(DeleteStreamProjectRequest deleteStreamProjectRequest) throws ExchangisProjectErrorException; - - -} diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/ProjectService.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/ProjectService.java new file mode 100644 index 000000000..6ae561dd6 --- /dev/null +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/ProjectService.java @@ -0,0 +1,68 @@ +package com.webank.wedatasphere.exchangis.project.server.service; + + +import com.webank.wedatasphere.exchangis.common.pager.PageResult; +import com.webank.wedatasphere.exchangis.job.exception.ExchangisJobException; +import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; +import com.webank.wedatasphere.exchangis.project.server.vo.ExchangisProjectInfo; +import com.webank.wedatasphere.exchangis.project.server.vo.ProjectQueryVo; + +import javax.servlet.http.HttpServletRequest; + +/** + * Project query + */ +public interface ProjectService { + + /** + * Create project + * @param projectVo project vo + * @return new project id + */ + long createProject(ExchangisProjectInfo projectVo, String userName); + + /** + * Exists project + * @param projectId project id + * @param projectName project name + * @return boolean + */ + boolean existsProject(Long projectId, String projectName); + + /** + * Update project + * @param projectVo project vo + * @param userName userName + */ + void updateProject(ExchangisProjectInfo projectVo, String userName); + + /** + * Query the page result + * @param queryVo result vo + * @return page result + */ + PageResult queryProjects(ProjectQueryVo queryVo); + + /** + * Delete project + * @param projectId project id + */ + void deleteProject(Long projectId) throws ExchangisJobException; + + /** + * Delete project by name + * @param name + */ + void deleteProjectByName(String name) throws ExchangisJobException; + + /** + * Query the project detail + * @param projectId project id + * @return project vo + */ + ExchangisProjectInfo getProjectDetailById(Long projectId); + + ExchangisProjectInfo getProjectById(Long projectId); + + ExchangisProjectInfo selectByName(String name); +} diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ExchangisProjectRelationServiceImpl.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ExchangisProjectRelationServiceImpl.java deleted file mode 100644 index 3844e2989..000000000 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ExchangisProjectRelationServiceImpl.java +++ /dev/null @@ -1,50 +0,0 @@ -package com.webank.wedatasphere.exchangis.project.server.service.impl; - -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.webank.wedatasphere.exchangis.project.server.dao.ExchangisProjectRelationMapper; -import com.webank.wedatasphere.exchangis.project.server.dto.ExchangisProjectRelationDTO; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProjectRelation; -import com.webank.wedatasphere.exchangis.project.server.service.ExchangisProjectRelationService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -import java.util.Optional; - -@Service -public class ExchangisProjectRelationServiceImpl implements ExchangisProjectRelationService { - - private static final Logger LOGGER = LoggerFactory.getLogger(ExchangisProjectRelationServiceImpl.class); - - @Autowired - private ExchangisProjectRelationMapper projectRelationMapper; - - @Transactional - @Override - public void save(ExchangisProjectRelationDTO dto) { - Optional optional = this.getByNodeId(Long.parseLong(dto.getNodeId())); - ExchangisProjectRelation relation = optional.orElseGet(ExchangisProjectRelation::new); - relation.setProjectId(Long.parseLong(dto.getProjectId())); - relation.setNodeId(Long.parseLong(dto.getNodeId())); - relation.setProjectVersion(dto.getProjectVersion()); - relation.setFlowVersion(dto.getFlowVersion()); - relation.setResourceId(Long.parseLong(dto.getResourceId())); - relation.setVersion(dto.getVersion()); - if (optional.isPresent()) { - this.projectRelationMapper.updateById(relation); - } else { - this.projectRelationMapper.insert(relation); - } - } - - @Override - public Optional getByNodeId(Long nodeId) { - QueryWrapper query = new QueryWrapper<>(); - query.eq("node_id", nodeId); - ExchangisProjectRelation relation = this.projectRelationMapper.selectOne(query); - return Optional.ofNullable(relation); - } - -} diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ExchangisProjectServiceImpl.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ExchangisProjectServiceImpl.java deleted file mode 100644 index d6f37c823..000000000 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ExchangisProjectServiceImpl.java +++ /dev/null @@ -1,279 +0,0 @@ -package com.webank.wedatasphere.exchangis.project.server.service.impl; - -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import com.google.common.base.Strings; -import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobDsBind; -import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobEntity; -import com.webank.wedatasphere.exchangis.dao.mapper.ExchangisJobDsBindMapper; -import com.webank.wedatasphere.exchangis.dao.mapper.ExchangisJobInfoMapper; -import com.webank.wedatasphere.exchangis.project.server.dao.ExchangisProjectMapper; -import com.webank.wedatasphere.exchangis.project.server.dto.ExchangisProjectDTO; -import com.webank.wedatasphere.exchangis.project.server.dto.ExchangisProjectGetDTO; -import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; -import com.webank.wedatasphere.exchangis.project.server.exception.ExchangisProjectErrorException; -import com.webank.wedatasphere.exchangis.project.server.request.CreateProjectRequest; -import com.webank.wedatasphere.exchangis.project.server.request.ProjectQueryRequest; -import com.webank.wedatasphere.exchangis.project.server.request.UpdateProjectRequest; -import com.webank.wedatasphere.exchangis.project.server.service.ExchangisProjectService; -import net.sf.cglib.core.CollectionUtils; -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -import javax.servlet.http.HttpServletRequest; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; - -@Service -public class ExchangisProjectServiceImpl implements ExchangisProjectService { - - private static final Logger LOGGER = LoggerFactory.getLogger(ExchangisProjectServiceImpl.class); - - @Autowired - private ExchangisProjectMapper exchangisProjectMapper; - - @Autowired - private ExchangisJobInfoMapper exchangisJobInfoMapper; - - @Autowired - private ExchangisJobDsBindMapper exchangisJobDsBindMapper; - - @Transactional - @Override - public ExchangisProject createProject(String username, CreateProjectRequest createProjectRequest) throws ExchangisProjectErrorException { - LOGGER.info("user {} starts to create project {}", username, createProjectRequest.getProjectName()); - String workspaceName = createProjectRequest.getWorkspaceName(); - String projectName = createProjectRequest.getProjectName(); - String dssProjectName = createProjectRequest.getDssProjectName(); - String description = createProjectRequest.getDescription(); - String tags = createProjectRequest.getTags(); - String editUsers = createProjectRequest.getEditUsers(); - String viewUsers = createProjectRequest.getViewUsers(); - String execUsers = createProjectRequest.getExecUsers(); - - QueryWrapper query = new QueryWrapper<>(); - query.eq("name", projectName); - Integer count = this.exchangisProjectMapper.selectCount(query); - if (count > 0) { - throw new ExchangisProjectErrorException(30041, "exchangis.project.create.error.name.exists"); - } - - Date now = new Date(); - ExchangisProject entity = new ExchangisProject(); - entity.setCreateBy(username); - entity.setCreateTime(now); - Optional.ofNullable(createProjectRequest.getDssProjectId()).ifPresent(dssProjectId -> { - if (StringUtils.isNotBlank(dssProjectId)) { - entity.setDssProjectId(Long.parseLong(dssProjectId)); - } - }); - entity.setWorkspaceName(workspaceName); - entity.setName(StringUtils.isNotBlank(dssProjectName) ? dssProjectName : projectName); - entity.setDssName(dssProjectName); - entity.setTags(tags); - entity.setEditUsers(editUsers); - entity.setViewUsers(viewUsers); - entity.setExecUsers(execUsers); - entity.setWorkspaceName(workspaceName); - entity.setDescription(description); - entity.setDomain(StringUtils.isNotBlank(dssProjectName) ? ExchangisProject.Domain.WORKSPACE.name() : ExchangisProject.Domain.STANDALONE.name()); - - int insert = this.exchangisProjectMapper.insert(entity); - if (insert <= 0) { - throw new ExchangisProjectErrorException(30041, "exchangis.project.create.error"); - } - return entity; - } - - @Transactional - @Override - public ExchangisProject updateProject(String username, UpdateProjectRequest updateProjectRequest) throws ExchangisProjectErrorException { - ExchangisProject exchangisProject = null; - if (null != updateProjectRequest.getId()) { - exchangisProject = this.exchangisProjectMapper.selectById(updateProjectRequest.getId()); - } else if (StringUtils.isNotBlank(updateProjectRequest.getDssProjectName())) { - QueryWrapper wrapper = new QueryWrapper<>(); - wrapper.eq("dss_name", updateProjectRequest.getDssProjectName()); - exchangisProject = this.exchangisProjectMapper.selectOne(wrapper); - } - - if (null == exchangisProject) { - throw new ExchangisProjectErrorException(30041, "exchangis.project.update.error"); - } - - QueryWrapper query = new QueryWrapper<>(); - query.eq("name", updateProjectRequest.getProjectName()); - query.ne("id", exchangisProject.getId()); - Integer count = this.exchangisProjectMapper.selectCount(query); - if (count > 0) { - throw new ExchangisProjectErrorException(30041, "exchangis.project.update.error.name.exists"); - } - - if (!Strings.isNullOrEmpty(updateProjectRequest.getDescription())) { - exchangisProject.setDescription(updateProjectRequest.getDescription()); - } - - if (!Strings.isNullOrEmpty(updateProjectRequest.getProjectName())) { - if(ExchangisProject.Domain.STANDALONE.name().equals(exchangisProject.getDomain())) { - exchangisProject.setName(updateProjectRequest.getProjectName()); - } - } - - if (!Strings.isNullOrEmpty(updateProjectRequest.getWorkspaceName())) { - exchangisProject.setWorkspaceName(updateProjectRequest.getWorkspaceName()); - exchangisProject.setName(updateProjectRequest.getWorkspaceName()); - } - - if (!Strings.isNullOrEmpty(updateProjectRequest.getTags())) { - exchangisProject.setTags(updateProjectRequest.getTags()); - } - - if (!Strings.isNullOrEmpty(updateProjectRequest.getViewUsers())) { - exchangisProject.setViewUsers(updateProjectRequest.getViewUsers()); - } - - if (!Strings.isNullOrEmpty(updateProjectRequest.getEditUsers())) { - exchangisProject.setExecUsers(updateProjectRequest.getEditUsers()); - } - - if (!Strings.isNullOrEmpty(updateProjectRequest.getExecUsers())) { - exchangisProject.setExecUsers(updateProjectRequest.getExecUsers()); - } - - int ret = this.exchangisProjectMapper.updateById(exchangisProject); - if (ret <= 0) { - throw new ExchangisProjectErrorException(30041, "exchangis.project.update.error"); - } - return exchangisProject; - } - - @Override - public List queryProjects(ProjectQueryRequest projectQueryRequest, int current, int size, String name) { - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.eq("create_by", projectQueryRequest.getUsername()); - - if (!Strings.isNullOrEmpty(name)) { - queryWrapper.like("name", name); - } - - if (!StringUtils.isBlank(projectQueryRequest.getDomain())) { - queryWrapper.eq("domain", projectQueryRequest.getDomain()); - } - - if (current <= 0) { - current = 1; - } - if (size <= 0) { - size = 10; - } - /* Page page = new Page<>(1, 10); - IPage templateIPage = this.exchangisProjectMapper.selectPage(page, queryWrapper); - List exchangisProjects = new ArrayList<>(); - for(ExchangisProject item : templateIPage.getRecords()){ - exchangisProjects.add(item); - }*/ - List exchangisProjects = this.exchangisProjectMapper.selectList(queryWrapper); - - int start = (current - 1) * size; - int last = 0; - if(current * size > exchangisProjects.size()) { - last = exchangisProjects.size(); - } - else { - last = current * size; - } - - return exchangisProjects.stream().map(ExchangisProjectDTO::new).collect(Collectors.toList()).subList(start,last); - - } - - @Override - public int count(ProjectQueryRequest projectQueryRequest, String name) { - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.eq("create_by", projectQueryRequest.getUsername()); - - if (!Strings.isNullOrEmpty(name)) { - queryWrapper.like("name", name); - } - - if (!StringUtils.isBlank(projectQueryRequest.getDomain())) { - queryWrapper.eq("domain", projectQueryRequest.getDomain()); - } - - List exchangisProjects = this.exchangisProjectMapper.selectList(queryWrapper); - return exchangisProjects.size(); - } - - @Transactional - @Override - public void deleteProject(HttpServletRequest request, String id) { - - QueryWrapper jobquery = new QueryWrapper<>(); - jobquery.eq("project_id", id); - - List jobs = this.exchangisJobInfoMapper.selectList(jobquery); - - if (null != jobs && jobs.size() > 0) { - this.exchangisJobInfoMapper.delete(jobquery); - - QueryWrapper dsBindQuery = new QueryWrapper().in("job_id", - jobs.stream().map(ExchangisJobEntity::getId).toArray() - ); - this.exchangisJobDsBindMapper.delete(dsBindQuery); - } - - this.exchangisProjectMapper.deleteById(Long.parseLong(id)); - } - - @Transactional - @Override - public void deleteProjectByDss(HttpServletRequest request, String dssName) { - ExchangisProjectGetDTO dto = this.getProjectByDssName(dssName); - if (null != dto && null != dto.getId() && StringUtils.isNotBlank(dto.getId())) { - this.deleteProject(request, dto.getId()); - } - } - - @Override - public ExchangisProjectGetDTO getProjectById(String projectId) { - ExchangisProject project = this.exchangisProjectMapper.selectById(projectId); - return new ExchangisProjectGetDTO(project); - } - - @Override - public ExchangisProjectGetDTO getProjectByDssName(String dssName) { - QueryWrapper wrapper = new QueryWrapper<>(); - wrapper.eq("dss_name", dssName); - return new ExchangisProjectGetDTO(this.exchangisProjectMapper.selectOne(wrapper)); - } - -// @Override -// @Transactional(rollbackFor = Exception.class) -// public StreamisProject createProject(CreateStreamProjectRequest createStreamProjectRequest) throws StreamisProjectErrorException { -// LOGGER.info("user {} starts to create project {}", createStreamProjectRequest.createBy(), createStreamProjectRequest.projectName()); -// StreamisProject streamisProject = new StreamisProject(createStreamProjectRequest.projectName(), createStreamProjectRequest.description(), createStreamProjectRequest.createBy()); -// //streamisProjectMapper.insertProject(streamisProject); -// LOGGER.info("user {} ends to create project {} and id is {}", createStreamProjectRequest.createBy(), createStreamProjectRequest.projectName(), streamisProject.getId()); -// return streamisProject; -// } -// -// @Override -// public void updateProject(UpdateStreamProjectRequest updateStreamProjectRequest) throws StreamisProjectErrorException { -// LOGGER.info("User {} begins to update project {}", updateStreamProjectRequest.updateBy(), updateStreamProjectRequest.projectName()); -// -// } -// -// -// @Override -// public void deleteProject(DeleteStreamProjectRequest deleteStreamProjectRequest) throws StreamisProjectErrorException { -// -// } -} diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectServiceImpl.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectServiceImpl.java new file mode 100644 index 000000000..c078133e5 --- /dev/null +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/service/impl/ProjectServiceImpl.java @@ -0,0 +1,162 @@ +package com.webank.wedatasphere.exchangis.project.server.service.impl; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.github.pagehelper.PageHelper; +import com.github.pagehelper.PageInfo; +import com.webank.wedatasphere.exchangis.common.pager.PageResult; +import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobDsBind; +import com.webank.wedatasphere.exchangis.dao.mapper.ExchangisJobDsBindMapper; +import com.webank.wedatasphere.exchangis.job.api.ExchangisJobOpenService; +import com.webank.wedatasphere.exchangis.job.domain.ExchangisJobEntity; +import com.webank.wedatasphere.exchangis.job.exception.ExchangisJobException; +import com.webank.wedatasphere.exchangis.job.vo.ExchangisJobQueryVo; +import com.webank.wedatasphere.exchangis.project.server.mapper.ProjectMapper; +import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; +import com.webank.wedatasphere.exchangis.project.server.service.ProjectService; +import com.webank.wedatasphere.exchangis.project.server.vo.ExchangisProjectInfo; +import com.webank.wedatasphere.exchangis.project.server.vo.ProjectQueryVo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import javax.annotation.Resource; +import java.util.*; +import java.util.stream.Collectors; + +@Service +public class ProjectServiceImpl implements ProjectService { + + private static final Logger LOGGER = LoggerFactory.getLogger(ProjectServiceImpl.class); + + @Autowired + private ProjectMapper projectMapper; + + @Resource + private ExchangisJobOpenService jobServiceOpenApi; + + @Autowired + private ExchangisJobDsBindMapper jobDataSourceBindMapper; + + @Override + @Transactional(rollbackFor = Exception.class) + public long createProject(ExchangisProjectInfo projectVo, String userName) { + // Construct the entity + ExchangisProject project = new ExchangisProject(); + project.setName(projectVo.getName()); + project.setDescription(projectVo.getDescription()); + project.setDomain(Optional.ofNullable(projectVo.getDomain()).orElse(ExchangisProject.Domain.STANDALONE.name())); + project.setLabels(projectVo.getLabel()); + project.setSourceMap(projectVo.getSource()); + project.setViewUsers(projectVo.getViewUsers()); + project.setEditUsers(projectVo.getEditUsers()); + project.setExecUsers(projectVo.getExecUsers()); + project.setCreateUser(userName); + project.setCreateTime(Calendar.getInstance().getTime()); + this.projectMapper.insertOne(project); + return project.getId(); + } + + @Override + public boolean existsProject(Long projectId, String projectName) { + Integer count = this.projectMapper.existsOne(projectId, projectName); + return null != count && count > 0; + } + + @Override + @Transactional(rollbackFor = Exception.class) + public void updateProject(ExchangisProjectInfo projectVo, String userName) { + ExchangisProject updatedProject = new ExchangisProject(); + updatedProject.setId(Long.valueOf(projectVo.getId())); + updatedProject.setName(projectVo.getName()); + updatedProject.setDescription(projectVo.getDescription()); + updatedProject.setLabels(projectVo.getLabel()); + updatedProject.setViewUsers(projectVo.getViewUsers()); + updatedProject.setEditUsers(projectVo.getEditUsers()); + updatedProject.setExecUsers(projectVo.getExecUsers()); + updatedProject.setDescription(projectVo.getDescription()); + // Set the updated properties + updatedProject.setLastUpdateUser(userName); + updatedProject.setLastUpdateTime(Calendar.getInstance().getTime()); + this.projectMapper.updateOne(updatedProject); + } + + @Override + public PageResult queryProjects(ProjectQueryVo queryVo) { + PageHelper.startPage(queryVo.getPage(), queryVo.getPageSize()); + try{ + List projects = this.projectMapper.queryPageList(queryVo); + PageInfo pageInfo = new PageInfo<>(projects); + List infoList = projects + .stream().map(ExchangisProjectInfo::new).collect(Collectors.toList()); + PageResult pageResult = new PageResult<>(); + pageResult.setList(infoList); + pageResult.setTotal(pageInfo.getTotal()); + return pageResult; + }finally { + PageHelper.clearPage(); + } + } + + @Override + @Transactional(rollbackFor = Exception.class) + public void deleteProject(Long projectId) throws ExchangisJobException { + // First to delete the project to lock the record + this.projectMapper.deleteOne(projectId); + // Query the related job + ExchangisJobQueryVo queryVo = new ExchangisJobQueryVo(); + queryVo.setProjectId(projectId); + List jobEntities = this.jobServiceOpenApi.queryJobs(queryVo, false); + if (!jobEntities.isEmpty()){ + List ids = jobEntities.stream().map(ExchangisJobEntity::getId).collect(Collectors.toList()); + this.jobServiceOpenApi.deleteJobBatch(ids); + QueryWrapper dsBindQuery = new QueryWrapper().in("job_id", ids); + this.jobDataSourceBindMapper.delete(dsBindQuery); + } + } + + @Override + @Transactional(rollbackFor = Exception.class) + public void deleteProjectByName(String name) throws ExchangisJobException { + // First to delete the project to lock the record + ExchangisProject project = this.projectMapper.selectByName(name); + this.projectMapper.deleteByName(name); + } + @Override + public ExchangisProjectInfo getProjectDetailById(Long projectId) { + ExchangisProject project = this.projectMapper.getDetailById(projectId); + if (Objects.nonNull(project)){ + ExchangisProjectInfo projectVo = new ExchangisProjectInfo(project); + projectVo.setViewUsers(project.getViewUsers()); + projectVo.setEditUsers(project.getEditUsers()); + projectVo.setExecUsers(project.getExecUsers()); + projectVo.setSource(project.getSourceMap()); + return projectVo; + } + return null; + } + + /** + * Basic info query + * @param projectId project id + * @return project info + */ + @Override + public ExchangisProjectInfo getProjectById(Long projectId) { + ExchangisProject project = projectMapper.getBasicById(projectId); + if (Objects.nonNull(project)){ + return new ExchangisProjectInfo(project); + } + return null; + } + + @Override + public ExchangisProjectInfo selectByName(String name){ + ExchangisProject project = this.projectMapper.selectByName(name); + if (Objects.nonNull(project)){ + return new ExchangisProjectInfo(project); + } + return null; + } +} diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/utils/ExchangisProjectRestfulUtils.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/utils/ExchangisProjectRestfulUtils.java index 46580a21c..06a27f9c2 100644 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/utils/ExchangisProjectRestfulUtils.java +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/utils/ExchangisProjectRestfulUtils.java @@ -5,6 +5,9 @@ import java.util.Arrays; +/** + * Utils for restful + */ public class ExchangisProjectRestfulUtils { diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectAppVo.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectAppVo.java new file mode 100644 index 000000000..5316482e4 --- /dev/null +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectAppVo.java @@ -0,0 +1,196 @@ +package com.webank.wedatasphere.exchangis.project.server.vo; + +import com.fasterxml.jackson.annotation.JsonAlias; +import com.webank.wedatasphere.exchangis.common.validator.groups.UpdateGroup; +import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Size; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * @author tikazhang + * @Date 2022/3/26 15:36 + */ +public class ExchangisProjectAppVo { + /** + * ID + */ + @NotNull(message = "Project id cannot be null (工程ID不能为空)", groups = UpdateGroup.class) + private Long id; + + /** + * Project name + */ + @NotBlank(message = "Project name cannot be null (工程名不能为空)") + @Size(max = 64, message = "Length of project name should be less than 64 (工程名长度不超过64)") + @JsonAlias("projectName") + private String name; + + /** + * Description + */ + @Size(max = 200, message = "Length of desc should be less than 200 (描述长度不超过200)") + private String description; + + /** + * Request domain + */ + private String domain; + + /** + * Information from the dss + */ + private Map source = new HashMap<>(); + + /** + * User has the edit permission + */ + private String editUsers; + + /** + * User has the view permission + */ + private String viewUsers; + + /** + * User has the execute permission + */ + private String execUsers; + + /** + * labels + */ + @JsonAlias("tags") + private String label; + + /** + * Create user + */ + private String createUser; + + /** + * Create time + */ + private Date createTime; + + private String labels; + + public ExchangisProjectAppVo(){ + + } + + public ExchangisProjectAppVo(ExchangisProject project){ + this.setId(project.getId()); + this.setName(project.getName()); + this.setDescription(project.getDescription()); + this.setDomain(project.getDomain()); + this.setLabel(project.getLabels()); + this.setCreateUser(project.getCreateUser()); + this.setCreateTime(project.getCreateTime()); + } + public String getId() { + return id + ""; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getDomain() { + return domain; + } + + public void setDomain(String domain) { + this.domain = domain; + } + + public Map getSource() { + return source; + } + + public void setSource(Map source) { + this.source = source; + } + + public String getEditUsers() { + return editUsers; + } + + public void setEditUsers(String editUsers) { + this.editUsers = editUsers; + } + + public String getViewUsers() { + return viewUsers; + } + + public void setViewUsers(String viewUsers) { + this.viewUsers = viewUsers; + } + + public String getExecUsers() { + return execUsers; + } + + public void setExecUsers(String execUsers) { + this.execUsers = execUsers; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + public String getCreateUser() { + return createUser; + } + + public void setCreateUser(String createUser) { + this.createUser = createUser; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + @Deprecated + public String getTags(){ + return Objects.nonNull(label)? label : ""; + } + + public String getLabels() { + return labels; + } + + public void setLabels(String labels) { + this.labels = labels; + } +} diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectInfo.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectInfo.java new file mode 100644 index 000000000..56f3449b2 --- /dev/null +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectInfo.java @@ -0,0 +1,214 @@ +package com.webank.wedatasphere.exchangis.project.server.vo; + +import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.webank.wedatasphere.exchangis.common.validator.groups.UpdateGroup; +import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; + +import javax.validation.constraints.NotBlank; +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Size; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * View object + * TODO @JsonInclude(JsonInclude.Include.NON_EMPTY) + */ +//@JsonInclude(JsonInclude.Include.NON_EMPTY) +public class ExchangisProjectInfo { + /** + * ID + */ + @NotNull(message = "Project id cannot be null (工程ID不能为空)", groups = UpdateGroup.class) + private Long id; + + /** + * Project name + */ + @NotBlank(message = "Project name cannot be null (工程名不能为空)") + @Size(max = 64, message = "Length of project name should be less than 64 (工程名长度不超过64)") + @JsonAlias("projectName") + private String name; + + /** + * Description + */ + @Size(max = 200, message = "Length of desc should be less than 200 (描述长度不超过200)") + private String description; + + /** + * Request domain + */ + private String domain; + + /** + * Information from the dss + */ + private Map source = new HashMap<>(); + + /** + * User has the edit permission + */ + private String editUsers; + + /** + * User has the view permission + */ + private String viewUsers; + + /** + * User has the execute permission + */ + private String execUsers; + + /** + * labels + */ + @JsonAlias("tags") + private String label; + + /** + * Create user + */ + private String createUser; + + /** + * Create time + */ + private Date createTime; + + private Map labels; + + public ExchangisProjectInfo(){ + + } + + public ExchangisProjectInfo(ExchangisProject project){ + this.setId(project.getId()); + this.setName(project.getName()); + this.setDescription(project.getDescription()); + this.setDomain(project.getDomain()); + this.setLabel(project.getLabels()); + this.setCreateUser(project.getCreateUser()); + this.setCreateTime(project.getCreateTime()); + } + + public ExchangisProjectInfo(ExchangisProjectAppVo project){ + Map labels = new HashMap<>(); + labels.put("route", project.getLabels()); + this.setName(project.getName()); + this.setDescription(project.getDescription()); + this.setDomain(project.getDomain()); + this.setSource(project.getSource()); + this.setEditUsers(project.getEditUsers()); + this.setViewUsers(project.getViewUsers()); + this.setExecUsers(project.getExecUsers()); + this.setLabel(project.getLabel()); + this.setCreateUser(project.getCreateUser()); + this.setCreateTime(project.getCreateTime()); + this.setLabels(labels); + } + public String getId() { + return id + ""; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getDomain() { + return domain; + } + + public void setDomain(String domain) { + this.domain = domain; + } + + public Map getSource() { + return source; + } + + public void setSource(Map source) { + this.source = source; + } + + public String getEditUsers() { + return editUsers; + } + + public void setEditUsers(String editUsers) { + this.editUsers = editUsers; + } + + public String getViewUsers() { + return viewUsers; + } + + public void setViewUsers(String viewUsers) { + this.viewUsers = viewUsers; + } + + public String getExecUsers() { + return execUsers; + } + + public void setExecUsers(String execUsers) { + this.execUsers = execUsers; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + public String getCreateUser() { + return createUser; + } + + public void setCreateUser(String createUser) { + this.createUser = createUser; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + @Deprecated + public String getTags(){ + return Objects.nonNull(label)? label : ""; + } + + public Map getLabels() { + return labels; + } + + public void setLabels(Map labels) { + this.labels = labels; + } +} diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectVo.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectVo.java deleted file mode 100644 index ef074cfbf..000000000 --- a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ExchangisProjectVo.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.webank.wedatasphere.exchangis.project.server.vo; - -public class ExchangisProjectVo { -} diff --git a/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ProjectQueryVo.java b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ProjectQueryVo.java new file mode 100644 index 000000000..7f56cc0f5 --- /dev/null +++ b/exchangis-project/exchangis-project-server/src/main/java/com/webank/wedatasphere/exchangis/project/server/vo/ProjectQueryVo.java @@ -0,0 +1,34 @@ +package com.webank.wedatasphere.exchangis.project.server.vo; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.webank.wedatasphere.exchangis.project.server.domain.ProjectPageQuery; + +import java.util.Map; +import java.util.Optional; + + +/** + * Query vo + */ +@JsonInclude(JsonInclude.Include.NON_EMPTY) +public class ProjectQueryVo extends ProjectPageQuery { + + public ProjectQueryVo(){ + } + private Map labels; + + public Map getLabels() { + return labels; + } + + public void setLabels(Map labels) { + this.labels = labels; + } + + public ProjectQueryVo(String name, Integer current, Integer size){ + this.name = name; + this.current = Optional.ofNullable(current).orElse(1); + this.size = Optional.ofNullable(size).orElse(10); + } + +} diff --git a/pom.xml b/pom.xml index b9ef183e9..df99eee4c 100644 --- a/pom.xml +++ b/pom.xml @@ -38,8 +38,8 @@ 1.0.0-RC1 - 1.0.1 - 1.0.3 + 1.1.2 + 1.1.1 0.1.0-SNAPSHOT 2.11.8 1.8 diff --git a/web/src/app.js b/web/src/app.js index 91004db04..85a9cb12f 100644 --- a/web/src/app.js +++ b/web/src/app.js @@ -49,9 +49,7 @@ export const request = { message.error(error?.response?.data?.message || error?.data?.message || "系统异常"); }, }, - otherConfigs: { - timeout: 30000 - } + timeout: 30000 }; // 这里 自定义注册header diff --git a/web/src/common/constants.js b/web/src/common/constants.js index e329b6427..4f25eb654 100644 --- a/web/src/common/constants.js +++ b/web/src/common/constants.js @@ -1 +1,3 @@ -export const BASE_URL = "/api/rest_j/v1/exchangis/"; +//export const BASE_URL = "/api/rest_j/v1/exchangis/"; +//export const BASE_URL = "/api/rest_j/v1/dss/exchangis/" +export const BASE_URL = "/api/rest_j/v1/dss/exchangis/main/" diff --git a/web/src/common/service.js b/web/src/common/service.js index 76a1ba943..37c00d4de 100644 --- a/web/src/common/service.js +++ b/web/src/common/service.js @@ -1,41 +1,67 @@ // 服务端接口管理 import { request } from "@fesjs/fes"; import { BASE_URL } from "@/common/constants"; + +import { getEnvironment } from "@/common/utils"; + //////////////////////////////////////////////////////////////////// export const getProjectList = (name, current, size) => { - return request(`/projects?name=${name}¤t=${current}&size=${size}`, {}, { method: "POST" }); + return request(`/projects?name=${name}¤t=${current}&size=${size}`, { + labels: { + route: getEnvironment() + } + }, { method: "POST" }); }; export const createProject = (body) => { - return request("/createProject", body); + return request("/createProject", { + ...body, + labels: { + route: getEnvironment() + } + }); }; export const deleteProject = (id) => { - return request("/projects/" + id, null, { + return request("/projects/" + id, { + labels: { + route: getEnvironment() + } + }, { method: "DELETE", }); }; export const getProjectById = (id) => { - return request("/projects/" + id, null, { + return request("/projects/" + id + '?labels=' + getEnvironment(), null, { method: "GET", }); }; export const updateProject = (body) => { - return request("/updateProject", body, { + return request("/updateProject", { + ...body, + labels: { + route: getEnvironment() + } + }, { method: "PUT", }); }; export const getDataSourceList = (params) => { - return request("/datasources/query", { ...params }, { method: "POST" }); + return request("/datasources/query", { + ...params, + labels: { + route: getEnvironment() + } + }, { method: "POST" }); }; // 数据源管理 获取数据源 export const getDataSourceTypes = () => { return request( - `/datasources/type?t=_${new Date().getTime()}`, + `/datasources/type?labels=${getEnvironment()}&t=_${new Date().getTime()}`, {}, { method: "GET" } ); @@ -44,7 +70,7 @@ export const getDataSourceTypes = () => { // 数据源管理 获取动态参数 export const getKeyDefine = (dataSourceTypeId) => { return request( - `/datasources/types/${dataSourceTypeId}/keydefines?t=_${new Date().getTime()}`, + `/datasources/types/${dataSourceTypeId}/keydefines?labels=${getEnvironment()}&t=_${new Date().getTime()}`, {}, { method: "GET" } ); @@ -52,16 +78,21 @@ export const getKeyDefine = (dataSourceTypeId) => { // 查询数据源 export const getDataSource = (body) => { - return request("/datasources/query", body, { method: "POST" }); + return request("/datasources/query", { + ...body, + labels: { + route: getEnvironment() + } + }, { method: "POST" }); }; export const getDBs = (type, id) => { - return request(`/datasources/${type}/${id}/dbs`, {}, { method: "GET" }); + return request(`/datasources/${type}/${id}/dbs?labels=${getEnvironment()}`, {}, { method: "GET" }); }; export const getTables = (type, id, dbName) => { return request( - `/datasources/${type}/${id}/dbs/${dbName}/tables`, + `/datasources/${type}/${id}/dbs/${dbName}/tables?labels=${getEnvironment()}`, {}, { method: "GET" } ); @@ -78,51 +109,83 @@ export const getTables = (type, id, dbName) => { export const getFields = (params) => { return request( `/datasources/fieldsmapping`, - { ...params }, + { + ...params, + labels: { + route: getEnvironment() + } + }, { method: "POST" } ); }; export const createDataSource = (params) => { - return request("/datasources", { ...params }, { method: "POST" }); + return request("/datasources", { + ...params, + labels: { + route: getEnvironment() + } + }, { method: "POST" }); }; export const updateDataSource = (id, params) => { - return request("/datasources/" + id, { ...params }, { method: "PUT" }); + return request("/datasources/" + id, { + ...params, + labels: { + route: getEnvironment() + } + }, { method: "PUT" }); }; export const deleteDataSource = (id) => { - return request(`/datasources/${id}`, {}, { method: "DELETE" }); + return request(`/datasources/${id}`, { + labels: { + route: getEnvironment() + } + }, { method: "DELETE" }); }; export const getDataSourceVersionList = (id) => { - return request(`/datasources/${id}/versions`, {}, { method: "GET" }); + return request(`/datasources/${id}/versions?labels=${getEnvironment()}`, {}, { method: "GET" }); }; export const testDataSourceConnect = (type, id) => { - return request(`/datasources/${type}/${id}/connect`, {}, { method: "PUT" }); + return request(`/datasources/${type}/${id}/connect?_=${Math.random()}`, { + labels: { + route: getEnvironment() + } + }, { method: "PUT" }); +}; + +export const testDataSourceNotSavedConnect = (params) => { + return request(`/datasources/op/connect?_=${Math.random()}`, { + ...params, + labels: { + route: getEnvironment() + } + }, { method: "POST" }); }; export const getDataSourceById = (id, versionId) => { - return request(`/datasources/${id}`, {versionId}, { method: "GET" }); + return request(`/datasources/${id}?labels=${getEnvironment()}`, {versionId}, { method: "GET" }); }; export const getJobInfo = (id) => { - return request(`/job/${id}`, null, { + return request(`/job/${id}?labels=${getEnvironment()}`, null, { method: "GET", }); }; //获取任务列表 export const getJobList = (query) => { - return request(`/job?${query}`, null, { + return request(`/job?labels=${getEnvironment()}&${query}`, null, { method: "GET", }); }; //获取执行引擎列表 export const getEngineType = () => { - return request(`/job/engineType`, null, { + return request(`/job/engineType?labels=${getEnvironment()}`, null, { method: "GET", }); }; @@ -131,7 +194,12 @@ export const getEngineType = () => { export const createJob = (params) => { return request( `/job`, - { ...params }, + { + ...params, + labels: { + route: getEnvironment() + } + }, { method: "POST", } @@ -142,7 +210,12 @@ export const createJob = (params) => { export const copyJob = (id, params) => { return request( `/job/${id}/copy`, - { ...params }, + { + ...params, + labels: { + route: getEnvironment() + } + }, { method: "POST", } @@ -153,7 +226,12 @@ export const copyJob = (id, params) => { export const modifyJob = (id, params) => { return request( `/job/${id}`, - { ...params }, + { + ...params, + labels: { + route: getEnvironment() + } + }, { method: "PUT", } @@ -162,7 +240,11 @@ export const modifyJob = (id, params) => { //删除任务 export const deleteJob = (id) => { - return request(`/job/${id}`, null, { + return request(`/job/${id}`, { + labels: { + route: getEnvironment() + } + }, { method: "DELETE", }); }; @@ -171,7 +253,12 @@ export const deleteJob = (id) => { export const importJob = (id, params) => { return request( `/job/import`, - { ...params }, + { + ...params, + labels: { + route: getEnvironment() + } + }, { method: "POST", } @@ -180,45 +267,67 @@ export const importJob = (id, params) => { //执行任务 export const executeTask = (id) => { - return request(`/job/${id}/action/execute`, null, { + return request(`/job/${id}/action/execute`, { + labels: { + route: getEnvironment() + } + }, { method: "POST", }); }; export const getJobs = (id, jobType, name, current, size) => { - return request(`/job?projectId=${id}&jobType=${jobType}&name=${name}¤t=${current}&size=${size}`, null, { + return request(`/job?labels=${getEnvironment()}&projectId=${id}&jobType=${jobType}&name=${name}¤t=${current}&size=${size}`, null, { method: "GET", }); }; export const saveProject = (id, body) => { - return request(`/job/${id}/content`, body, { + return request(`/job/${id}/content`, { + ...body, + labels: { + route: getEnvironment() + } + }, { method: "PUT", }); }; // 保存/更新任务配置 export const updateTaskConfiguration = (id, body) => { - return request(`/job/${id}/config`, body, { + return request(`/job/${id}/config`, { + ...body, + labels: { + route: getEnvironment() + } + }, { method: "PUT", }); }; export const expireDataSource = (id) => { - return request(`/datasources/${id}/expire`, {}, { method: "PUT" }); + return request(`/datasources/${id}/expire`, { + labels: { + route: getEnvironment() + } + }, { method: "PUT" }); }; export const publishDataSource = (id, versionId) => { return request( `/datasources/${id}/${versionId}/publish`, - {}, + { + labels: { + route: getEnvironment() + } + }, { method: "PUT" } ); }; export const getSourceParams = (engineType, type, ds) => { return request( - `/datasources/${engineType}/${type}/params/ui?dir=${ds}`, + `/datasources/${engineType}/${type}/params/ui?labels=${getEnvironment()}&dir=${ds}`, {}, { method: "GET" } ); @@ -226,7 +335,7 @@ export const getSourceParams = (engineType, type, ds) => { export const getSettingsParams = (engineType) => { return request( - `/jobs/engine/${engineType}/settings/ui`, + `/jobs/engine/${engineType}/settings/ui?labels=${getEnvironment()}`, {}, { method: "GET" } ); @@ -241,26 +350,30 @@ export const getSettingsParams = (engineType) => { // 同步历史 export const getSyncHistory = (body) => { - return request("/tasks", body, { + return request("/tasks?labels=" + getEnvironment(), body, { method: "GET", }); }; // 新版同步历史-获取job列表 export const getSyncHistoryJobList = (body) => { - return request("/job/listJobs", body, { + return request("/job/listJobs?labels=" + getEnvironment(), body, { method: "GET", }); }; // 删除同步历史 export const delSyncHistory = (jobExecutionId) => { - return request(`/job/${jobExecutionId}/deleteJob`, {}, { + return request(`/job/${jobExecutionId}/deleteJob`, { + labels: { + route: getEnvironment() + } + }, { method: "POST", }); }; // 读取Task限速配置 export const getSpeedLimit = (params) => { return request( - `/job/${params.jobId}/speedlimit/${params.taskName}/params/ui`, + `/job/${params.jobId}/speedlimit/${params.taskName}/params/ui?labels=${getEnvironment()}`, {}, { method: "GET", @@ -269,7 +382,12 @@ export const getSpeedLimit = (params) => { }; // 保存Task限速配置 export const saveSpeedLimit = (params, body) => { - return request(`/job/${params.jobId}/speedlimit/${params.taskName}`, body, { + return request(`/job/${params.jobId}/speedlimit/${params.taskName}`, { + ...body, + labels: { + route: getEnvironment() + } + }, { method: "PUT", }); }; @@ -277,7 +395,7 @@ export const saveSpeedLimit = (params, body) => { // 获取运行日志 export const getLogs = (params) => { return request( - `/execution/tasks/${params.taskID}/logs`, + `/execution/tasks/${params.taskID}/logs?labels=${getEnvironment()}`, { fromLine: params.fromLine || 1, pageSize: params.pageSize || 10 @@ -293,66 +411,79 @@ export const getLogs = (params) => { // 任务状态 export const getTaskState = () => { - return request("/metrics/taskstate", {}, { method: "GET" }); + return request("/metrics/taskstate?labels=" + getEnvironment(), {}, { method: "GET" }); }; // 任务进度 export const getTaskProcess = () => { - return request("/metrics/taskprocess", {}, { method: "GET" }); + return request("/metrics/taskprocess?labels=" + getEnvironment(), {}, { method: "GET" }); }; // 流量监控 export const getDataSourceFlow = () => { - return request("/metrics/datasourceflow", {}, { method: "GET" }); + return request("/metrics/datasourceflow?labels=" + getEnvironment(), {}, { method: "GET" }); }; // 资源使用 export const getEngineriesSource = () => { - return request("/metrics/engineresource", {}, { method: "GET" }); + return request("/metrics/engineresource?labels=" + getEnvironment(), {}, { method: "GET" }); }; export const getEngineriesSourceCpu = () => { - return request("/metrics/engineresourcecpu", {}, { method: "GET" }); + return request("/metrics/engineresourcecpu?labels=" + getEnvironment(), {}, { method: "GET" }); }; export const getEngineriesSourceMem = () => { - return request("/metrics/engineresourcemem", {}, { method: "GET" }); + return request("/metrics/engineresourcemem?labels=" + getEnvironment(), {}, { method: "GET" }); }; /* 作业执行模块接口 */ export const executeJob = (id) => { - return request(`/job/${id}/execute`,undefined, { + return request(`/job/${id}/execute`,{ + labels: { + route: getEnvironment() + } + }, { method: "POST", }) } export const getJobStatus = (id) => { - return request(`/job/execution/${id}/status`, {}, { + return request(`/job/execution/${id}/status?labels=${getEnvironment()}`, {}, { method: "GET", }) } export const getJobTasks = (id) => { - return request(`/job/execution/${id}/taskList`, null, { + return request(`/job/execution/${id}/taskList?labels=${getEnvironment()}`, null, { method: "GET", }) } export const getProgress = (id) => { - return request(`/job/execution/${id}/progress?_=${Math.random()}`, null, { + return request(`/job/execution/${id}/progress?labels=${getEnvironment()}&_=${Math.random()}`, null, { method: "GET", }) } export const getMetrics = (taskId, jobExecutionId) => { - return request(`/task/execution/${taskId}/metrics`, {jobExecutionId}, { + return request(`/task/execution/${taskId}/metrics`, { + jobExecutionId, + labels: { + route: getEnvironment() + } + }, { method: "POST", }) } export const killJob = (id) => { - return request(`/job/execution/${id}/kill`, null, { + return request(`/job/execution/${id}/kill`, { + labels: { + route: getEnvironment() + } + }, { method: "POST", }) } @@ -360,7 +491,7 @@ export const killJob = (id) => { // 获取job运行日志 export const getJobExecLog = (params) => { return request( - `/job/execution/${params.id}/log?_=${Math.random()}`, + `/job/execution/${params.id}/log?labels=${getEnvironment()}&_=${Math.random()}`, { fromLine: params.fromLine || 0, pageSize: params.pageSize || 50, @@ -377,7 +508,7 @@ export const getJobExecLog = (params) => { // 获取task运行日志 export const getTaskExecLog = (params) => { return request( - `/task/execution/${params.taskId}/log?_=${Math.random()}`, + `/task/execution/${params.taskId}/log?labels=${getEnvironment()}&_=${Math.random()}`, { fromLine: params.fromLine || 0, pageSize: params.pageSize || 50, @@ -397,7 +528,7 @@ export const getPartitionInfo = (params) => { if (!params.source) return const url = params.source.split(BASE_URL)[1] return request( - `${url}?dataSourceId=${params.dataSourceId}&database=${params.database}&table=${params.table}&_=${Math.random()}`, + `${url}?labels=${getEnvironment()}&dataSourceId=${params.dataSourceId}&database=${params.database}&table=${params.table}&_=${Math.random()}`, {}, { method: "GET", diff --git a/web/src/common/utils.js b/web/src/common/utils.js index 790a7bf34..2e259f943 100644 --- a/web/src/common/utils.js +++ b/web/src/common/utils.js @@ -46,6 +46,10 @@ export const genFileURL = (fileId, fileHash) => export const dateFormat = (timestamp) => { return moment(timestamp).format("YYYY-MM-DD HH:mm"); +} + +export const dateFormatSeconds = (timestamp) => { + return moment(timestamp).format("YYYY-MM-DD HH:mm:ss"); }; /* @@ -76,14 +80,14 @@ export const moveUpDown = (targetSelector, wrapSelector, moveTopSelector, bounda let boxTop = e.clientY - restTop; let topTop = e.clientY - topRestTop; const contentHeight = document.body.offsetHeight - wrap.offsetTop - // 头部限制默认为 350px + // 头部限制 if (boxTop - wrap.offsetTop < boundaryTop) { boxTop = boundaryTop } if (topTop - wrap.offsetTop + top.offsetHeight < boundaryTop) { topTop = boundaryTop - top.offsetHeight } - // 底部限制默认为 350px + // 底部限制 if (contentHeight - boxTop < boundaryBottom) { boxTop = contentHeight - boundaryBottom } @@ -108,3 +112,7 @@ export const moveUpDown = (targetSelector, wrapSelector, moveTopSelector, bounda } } } + +export const getEnvironment = () => { + return localStorage.getItem('exchangis_environment') || '' +} \ No newline at end of file diff --git a/web/src/pages/dataSourceManage/components/datasourceForm/index.vue b/web/src/pages/dataSourceManage/components/datasourceForm/index.vue index b4d6ca55e..daa1485a0 100644 --- a/web/src/pages/dataSourceManage/components/datasourceForm/index.vue +++ b/web/src/pages/dataSourceManage/components/datasourceForm/index.vue @@ -1,7 +1,7 @@