From 63b482603fd247ddb2cb52599c255908c443f486 Mon Sep 17 00:00:00 2001 From: davidhua Date: Mon, 29 Apr 2024 20:11:38 +0800 Subject: [PATCH 01/60] Update the exchangis version to '1.1.3-webank'; Update the linkis version to '1.3.0-wds'; --- assembly-package/pom.xml | 5 +- exchangis-dao/pom.xml | 3 +- .../exchangis-datasource-core/pom.xml | 5 +- .../exchangis-datasource-linkis/pom.xml | 5 +- .../exchangis-datasource-loader/pom.xml | 5 +- .../exchangis-datasource-server/pom.xml | 7 +- .../exchangis-datasource-service/pom.xml | 9 +- .../service/ExchangisDataSourceService.java | 2 +- ...DataSourceInfoByIdAndVersionIdAction.scala | 2 +- .../exchangis-datasource-streamis/pom.xml | 5 +- .../pom.xml | 8 +- .../exchangis-datasource-ext-hive/pom.xml | 8 +- .../exchangis-datasource-ext-mongodb/pom.xml | 8 +- .../exchangis-datasource-ext-mysql/pom.xml | 8 +- .../exchangis-datasource-ext-oracle/pom.xml | 8 +- .../exchangis-datasource-ext-sftp/pom.xml | 8 +- exchangis-datasource/pom.xml | 3 +- .../engineconn-plugins/datax/pom.xml | 2 +- .../engineconn-plugins/sqoop/pom.xml | 2 +- .../engines/datax/datax-assembly/pom.xml | 2 +- .../engines/datax/datax-core/pom.xml | 2 +- .../datax/datax-elasticsearchwriter/pom.xml | 2 +- .../engines/datax/datax-ftpreader/pom.xml | 2 +- .../engines/datax/datax-ftpwriter/pom.xml | 2 +- .../engines/datax/datax-hdfsreader/pom.xml | 2 +- .../reader/hdfsreader/HdfsReaderUtil.java | 2 +- .../engines/datax/datax-hdfswriter/pom.xml | 2 +- .../engines/datax/datax-mysqlreader/pom.xml | 2 +- .../engines/datax/datax-mysqlwriter/pom.xml | 2 +- .../engines/datax/datax-oraclereader/pom.xml | 2 +- .../engines/datax/datax-oraclewriter/pom.xml | 2 +- .../datax/datax-textfilereader/pom.xml | 2 +- .../datax/datax-textfilewriter/pom.xml | 2 +- exchangis-engines/engines/datax/pom.xml | 2 +- .../exchangis-engine-common/pom.xml | 4 +- .../exchangis-engine-core/pom.xml | 6 +- .../exchangis-engine-server/pom.xml | 6 +- exchangis-engines/pom.xml | 5 +- exchangis-job/exchangis-job-builder/pom.xml | 6 +- exchangis-job/exchangis-job-common/pom.xml | 6 +- exchangis-job/exchangis-job-launcher/pom.xml | 6 +- exchangis-job/exchangis-job-metrics/pom.xml | 2 +- exchangis-job/exchangis-job-server/pom.xml | 10 +- .../impl/ExchangisJobDsBindServiceImpl.java | 2 +- exchangis-job/exchangis-job-service/pom.xml | 6 +- exchangis-job/pom.xml | 4 +- exchangis-plugins/exchangis-appconn/pom.xml | 2 +- exchangis-plugins/pom.xml | 4 +- .../exchangis-project-entity/pom.xml | 4 +- .../exchangis-project-provider/pom.xml | 4 +- .../exchangis-project-server/pom.xml | 10 +- exchangis-project/pom.xml | 2 +- exchangis-server/pom.xml | 12 +- .../queue/BinlogArrayLockFreeQueue.java | 465 ++++++++++++++++++ pom.xml | 8 +- 55 files changed, 591 insertions(+), 116 deletions(-) create mode 100644 exchangis-server/src/main/java/com/webank/wedatasphere/exchangis/queue/BinlogArrayLockFreeQueue.java diff --git a/assembly-package/pom.xml b/assembly-package/pom.xml index 45315a5fe..dc473f537 100644 --- a/assembly-package/pom.xml +++ b/assembly-package/pom.xml @@ -21,7 +21,8 @@ exchangis com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} + ../pom.xml 4.0.0 assembly-package @@ -62,7 +63,7 @@ false - wedatasphere-exchangis-${exchangis.version} + wedatasphere-exchangis-${revision} false false diff --git a/exchangis-dao/pom.xml b/exchangis-dao/pom.xml index 5776a5b52..620401fac 100644 --- a/exchangis-dao/pom.xml +++ b/exchangis-dao/pom.xml @@ -5,7 +5,8 @@ exchangis com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} + ../pom.xml 4.0.0 diff --git a/exchangis-datasource/exchangis-datasource-core/pom.xml b/exchangis-datasource/exchangis-datasource-core/pom.xml index 990c92d6f..33b673931 100644 --- a/exchangis-datasource/exchangis-datasource-core/pom.xml +++ b/exchangis-datasource/exchangis-datasource-core/pom.xml @@ -5,7 +5,8 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} + ../pom.xml 4.0.0 @@ -20,7 +21,7 @@ com.webank.wedatasphere.exchangis exchangis-dao - 1.1.2 + ${revision} diff --git a/exchangis-datasource/exchangis-datasource-linkis/pom.xml b/exchangis-datasource/exchangis-datasource-linkis/pom.xml index 2ab7eda3d..cf202c20e 100644 --- a/exchangis-datasource/exchangis-datasource-linkis/pom.xml +++ b/exchangis-datasource/exchangis-datasource-linkis/pom.xml @@ -5,7 +5,8 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} + ../pom.xml 4.0.0 @@ -20,7 +21,7 @@ com.webank.wedatasphere.exchangis exchangis-datasource-core - 1.1.2 + ${revision} diff --git a/exchangis-datasource/exchangis-datasource-loader/pom.xml b/exchangis-datasource/exchangis-datasource-loader/pom.xml index 30ebb07ad..0bfe8fde2 100644 --- a/exchangis-datasource/exchangis-datasource-loader/pom.xml +++ b/exchangis-datasource/exchangis-datasource-loader/pom.xml @@ -5,7 +5,8 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} + ../pom.xml 4.0.0 @@ -20,7 +21,7 @@ com.webank.wedatasphere.exchangis exchangis-datasource-core - 1.1.2 + ${revision} diff --git a/exchangis-datasource/exchangis-datasource-server/pom.xml b/exchangis-datasource/exchangis-datasource-server/pom.xml index c41abaa25..347abac89 100644 --- a/exchangis-datasource/exchangis-datasource-server/pom.xml +++ b/exchangis-datasource/exchangis-datasource-server/pom.xml @@ -5,7 +5,8 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} + ../pom.xml 4.0.0 @@ -20,13 +21,13 @@ com.webank.wedatasphere.exchangis exchangis-datasource-service - 1.1.2 + ${revision} com.webank.wedatasphere.exchangis exchangis-datasource-loader - 1.1.2 + ${revision} diff --git a/exchangis-datasource/exchangis-datasource-service/pom.xml b/exchangis-datasource/exchangis-datasource-service/pom.xml index e41e65f03..1736570e1 100644 --- a/exchangis-datasource/exchangis-datasource-service/pom.xml +++ b/exchangis-datasource/exchangis-datasource-service/pom.xml @@ -5,7 +5,8 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} + ../pom.xml 4.0.0 @@ -21,19 +22,19 @@ com.webank.wedatasphere.exchangis exchangis-datasource-linkis - 1.1.2 + ${revision} com.webank.wedatasphere.exchangis exchangis-job-common - 1.1.2 + ${revision} com.webank.wedatasphere.exchangis exchangis-engine-core - 1.1.2 + ${revision} compile diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java index 40fcced7b..7243c075b 100644 --- a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java @@ -401,7 +401,7 @@ public Message deleteDataSource(HttpServletRequest request, /*String type,*/ Lon QueryWrapper condition = new QueryWrapper<>(); condition.eq("source_ds_id", id).or().eq("sink_ds_id", id); - Long inUseCount = this.exchangisJobDsBindMapper.selectCount(condition); + Integer inUseCount = this.exchangisJobDsBindMapper.selectCount(condition); if (inUseCount > 0) { throw new ExchangisDataSourceException(ExchangisDataSourceExceptionCode.CLIENT_DATASOURCE_DELETE_ERROR.getCode(), "目前存在引用依赖"); } diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/scala/com/webank/wedatasphere/exchangis/datasource/GetDataSourceInfoByIdAndVersionIdAction.scala b/exchangis-datasource/exchangis-datasource-service/src/main/scala/com/webank/wedatasphere/exchangis/datasource/GetDataSourceInfoByIdAndVersionIdAction.scala index b4329f88e..ac84231cd 100644 --- a/exchangis-datasource/exchangis-datasource-service/src/main/scala/com/webank/wedatasphere/exchangis/datasource/GetDataSourceInfoByIdAndVersionIdAction.scala +++ b/exchangis-datasource/exchangis-datasource-service/src/main/scala/com/webank/wedatasphere/exchangis/datasource/GetDataSourceInfoByIdAndVersionIdAction.scala @@ -49,7 +49,7 @@ object GetDataSourceInfoByIdAndVersionIdAction { } def build(): GetDataSourceInfoByIdAndVersionIdAction = { - if(dataSourceId == null) throw new DataSourceClientBuilderException("dataSourceId is needed!") + if(dataSourceId == 0L) throw new DataSourceClientBuilderException("dataSourceId is needed!") if(versionId == null) throw new DataSourceClientBuilderException("versionId is needed!") if(system == null) throw new DataSourceClientBuilderException("system is needed!") if(user == null) throw new DataSourceClientBuilderException("user is needed!") diff --git a/exchangis-datasource/exchangis-datasource-streamis/pom.xml b/exchangis-datasource/exchangis-datasource-streamis/pom.xml index 906e781a2..3d86b9522 100644 --- a/exchangis-datasource/exchangis-datasource-streamis/pom.xml +++ b/exchangis-datasource/exchangis-datasource-streamis/pom.xml @@ -5,7 +5,8 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} + ../pom.xml 4.0.0 @@ -20,7 +21,7 @@ com.webank.wedatasphere.exchangis exchangis-datasource-core - 1.1.2 + ${revision} diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-elasticsearch/pom.xml b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-elasticsearch/pom.xml index a92fa201c..86827ee7a 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-elasticsearch/pom.xml +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-elasticsearch/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} ../../pom.xml 4.0.0 @@ -21,17 +21,17 @@ com.webank.wedatasphere.exchangis exchangis-dao - 1.1.2 + ${revision} com.webank.wedatasphere.exchangis exchangis-datasource-linkis - 1.1.2 + ${revision} com.webank.wedatasphere.exchangis exchangis-datasource-core - 1.1.2 + ${revision} diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-hive/pom.xml b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-hive/pom.xml index c1e258f85..55d04f112 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-hive/pom.xml +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-hive/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} ../../pom.xml 4.0.0 @@ -22,17 +22,17 @@ com.webank.wedatasphere.exchangis exchangis-dao - 1.1.2 + ${revision} com.webank.wedatasphere.exchangis exchangis-datasource-linkis - 1.1.2 + ${revision} com.webank.wedatasphere.exchangis exchangis-dao - 1.1.2 + ${revision} diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mongodb/pom.xml b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mongodb/pom.xml index a26312d6e..e75f75b15 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mongodb/pom.xml +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mongodb/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} ../../pom.xml 4.0.0 @@ -21,17 +21,17 @@ com.webank.wedatasphere.exchangis exchangis-dao - 1.1.2 + ${revision} com.webank.wedatasphere.exchangis exchangis-datasource-linkis - 1.1.2 + ${revision} com.webank.wedatasphere.exchangis exchangis-datasource-core - 1.1.2 + ${revision} diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mysql/pom.xml b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mysql/pom.xml index 71bb30072..ff446a170 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mysql/pom.xml +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mysql/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} ../../pom.xml 4.0.0 @@ -21,17 +21,17 @@ com.webank.wedatasphere.exchangis exchangis-dao - 1.1.2 + ${revision} com.webank.wedatasphere.exchangis exchangis-datasource-linkis - 1.1.2 + ${revision} com.webank.wedatasphere.exchangis exchangis-datasource-core - 1.1.2 + ${revision} diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-oracle/pom.xml b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-oracle/pom.xml index e3cce71da..c5c90309f 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-oracle/pom.xml +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-oracle/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} ../../pom.xml 4.0.0 @@ -21,17 +21,17 @@ com.webank.wedatasphere.exchangis exchangis-dao - 1.1.2 + ${revision} com.webank.wedatasphere.exchangis exchangis-datasource-linkis - 1.1.2 + ${revision} com.webank.wedatasphere.exchangis exchangis-datasource-core - 1.1.2 + ${revision} diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-sftp/pom.xml b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-sftp/pom.xml index 47af538ab..6ea72b3dc 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-sftp/pom.xml +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-sftp/pom.xml @@ -5,7 +5,7 @@ exchangis-datasource com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} ../../pom.xml 4.0.0 @@ -21,17 +21,17 @@ com.webank.wedatasphere.exchangis exchangis-dao - 1.1.2 + ${revision} com.webank.wedatasphere.exchangis exchangis-datasource-linkis - 1.1.2 + ${revision} com.webank.wedatasphere.exchangis exchangis-datasource-core - 1.1.2 + ${revision} diff --git a/exchangis-datasource/pom.xml b/exchangis-datasource/pom.xml index ad1bd9fae..5cb7fa7be 100644 --- a/exchangis-datasource/pom.xml +++ b/exchangis-datasource/pom.xml @@ -5,7 +5,8 @@ exchangis com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} + ../pom.xml 4.0.0 diff --git a/exchangis-engines/engineconn-plugins/datax/pom.xml b/exchangis-engines/engineconn-plugins/datax/pom.xml index dd10e755f..af4629d1b 100644 --- a/exchangis-engines/engineconn-plugins/datax/pom.xml +++ b/exchangis-engines/engineconn-plugins/datax/pom.xml @@ -5,7 +5,7 @@ exchangis-engines com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} ../../pom.xml 4.0.0 diff --git a/exchangis-engines/engineconn-plugins/sqoop/pom.xml b/exchangis-engines/engineconn-plugins/sqoop/pom.xml index ee8398bf7..d2125da7f 100644 --- a/exchangis-engines/engineconn-plugins/sqoop/pom.xml +++ b/exchangis-engines/engineconn-plugins/sqoop/pom.xml @@ -22,7 +22,7 @@ exchangis-engines com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} ../../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-assembly/pom.xml b/exchangis-engines/engines/datax/datax-assembly/pom.xml index 74b755512..7f9873a45 100644 --- a/exchangis-engines/engines/datax/datax-assembly/pom.xml +++ b/exchangis-engines/engines/datax/datax-assembly/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-core/pom.xml b/exchangis-engines/engines/datax/datax-core/pom.xml index 48be143d3..ccf8c79a5 100644 --- a/exchangis-engines/engines/datax/datax-core/pom.xml +++ b/exchangis-engines/engines/datax/datax-core/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-elasticsearchwriter/pom.xml b/exchangis-engines/engines/datax/datax-elasticsearchwriter/pom.xml index 357a9b855..11adfd18a 100644 --- a/exchangis-engines/engines/datax/datax-elasticsearchwriter/pom.xml +++ b/exchangis-engines/engines/datax/datax-elasticsearchwriter/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-ftpreader/pom.xml b/exchangis-engines/engines/datax/datax-ftpreader/pom.xml index 8520f0145..8912e7745 100644 --- a/exchangis-engines/engines/datax/datax-ftpreader/pom.xml +++ b/exchangis-engines/engines/datax/datax-ftpreader/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-ftpwriter/pom.xml b/exchangis-engines/engines/datax/datax-ftpwriter/pom.xml index ec0635c29..d738c01d9 100644 --- a/exchangis-engines/engines/datax/datax-ftpwriter/pom.xml +++ b/exchangis-engines/engines/datax/datax-ftpwriter/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-hdfsreader/pom.xml b/exchangis-engines/engines/datax/datax-hdfsreader/pom.xml index 81789b25b..853f56e87 100644 --- a/exchangis-engines/engines/datax/datax-hdfsreader/pom.xml +++ b/exchangis-engines/engines/datax/datax-hdfsreader/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-hdfsreader/src/main/java/com/alibaba/datax/plugin/reader/hdfsreader/HdfsReaderUtil.java b/exchangis-engines/engines/datax/datax-hdfsreader/src/main/java/com/alibaba/datax/plugin/reader/hdfsreader/HdfsReaderUtil.java index 7738c9ce8..38e094578 100644 --- a/exchangis-engines/engines/datax/datax-hdfsreader/src/main/java/com/alibaba/datax/plugin/reader/hdfsreader/HdfsReaderUtil.java +++ b/exchangis-engines/engines/datax/datax-hdfsreader/src/main/java/com/alibaba/datax/plugin/reader/hdfsreader/HdfsReaderUtil.java @@ -603,7 +603,7 @@ private Record transportOneRecord(List columnConfigs, List } Type type = Type.valueOf(columnType.toUpperCase()); // it's all ok if nullFormat is null - if (StringUtils.equals(columnValue, nullFormat) || StringUtils.isBlank(columnValue)) { + if (StringUtils.equals(columnValue, nullFormat) || StringUtils.isEmpty(columnValue)) { columnValue = null; } switch (type) { diff --git a/exchangis-engines/engines/datax/datax-hdfswriter/pom.xml b/exchangis-engines/engines/datax/datax-hdfswriter/pom.xml index 11c814aec..69e76c86c 100644 --- a/exchangis-engines/engines/datax/datax-hdfswriter/pom.xml +++ b/exchangis-engines/engines/datax/datax-hdfswriter/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-mysqlreader/pom.xml b/exchangis-engines/engines/datax/datax-mysqlreader/pom.xml index 12389b823..483c78022 100644 --- a/exchangis-engines/engines/datax/datax-mysqlreader/pom.xml +++ b/exchangis-engines/engines/datax/datax-mysqlreader/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-mysqlwriter/pom.xml b/exchangis-engines/engines/datax/datax-mysqlwriter/pom.xml index 7339dbc27..df5898ab7 100644 --- a/exchangis-engines/engines/datax/datax-mysqlwriter/pom.xml +++ b/exchangis-engines/engines/datax/datax-mysqlwriter/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-oraclereader/pom.xml b/exchangis-engines/engines/datax/datax-oraclereader/pom.xml index fd6f86f07..3e87bfad4 100644 --- a/exchangis-engines/engines/datax/datax-oraclereader/pom.xml +++ b/exchangis-engines/engines/datax/datax-oraclereader/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-oraclewriter/pom.xml b/exchangis-engines/engines/datax/datax-oraclewriter/pom.xml index 054951019..d4f3813ff 100644 --- a/exchangis-engines/engines/datax/datax-oraclewriter/pom.xml +++ b/exchangis-engines/engines/datax/datax-oraclewriter/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-textfilereader/pom.xml b/exchangis-engines/engines/datax/datax-textfilereader/pom.xml index 6e7c18313..e8c690ddf 100644 --- a/exchangis-engines/engines/datax/datax-textfilereader/pom.xml +++ b/exchangis-engines/engines/datax/datax-textfilereader/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/datax-textfilewriter/pom.xml b/exchangis-engines/engines/datax/datax-textfilewriter/pom.xml index 1d0c9f1f3..7ce1bf54b 100644 --- a/exchangis-engines/engines/datax/datax-textfilewriter/pom.xml +++ b/exchangis-engines/engines/datax/datax-textfilewriter/pom.xml @@ -5,7 +5,7 @@ exchangis-engine-datax com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} ../pom.xml 4.0.0 diff --git a/exchangis-engines/engines/datax/pom.xml b/exchangis-engines/engines/datax/pom.xml index 15d7f5879..011f31d6f 100644 --- a/exchangis-engines/engines/datax/pom.xml +++ b/exchangis-engines/engines/datax/pom.xml @@ -5,7 +5,7 @@ com.webank.wedatasphere.exchangis exchangis - 1.1.2 + ${revision} ../../../pom.xml diff --git a/exchangis-engines/exchangis-engine-common/pom.xml b/exchangis-engines/exchangis-engine-common/pom.xml index d6cf14efa..6a1efe5e4 100644 --- a/exchangis-engines/exchangis-engine-common/pom.xml +++ b/exchangis-engines/exchangis-engine-common/pom.xml @@ -5,7 +5,7 @@ exchangis-engines com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} 4.0.0 @@ -19,7 +19,7 @@ com.webank.wedatasphere.exchangis exchangis-dao - ${exchangis.version} + ${revision} org.apache.linkis diff --git a/exchangis-engines/exchangis-engine-core/pom.xml b/exchangis-engines/exchangis-engine-core/pom.xml index 5d040b341..ecc445c3f 100644 --- a/exchangis-engines/exchangis-engine-core/pom.xml +++ b/exchangis-engines/exchangis-engine-core/pom.xml @@ -5,7 +5,7 @@ exchangis-engines com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} 4.0.0 @@ -20,12 +20,12 @@ com.webank.wedatasphere.exchangis exchangis-engine-common - ${exchangis.version} + ${revision} com.webank.wedatasphere.exchangis exchangis-dao - ${exchangis.version} + ${revision} diff --git a/exchangis-engines/exchangis-engine-server/pom.xml b/exchangis-engines/exchangis-engine-server/pom.xml index 5873fe538..8d863e09b 100644 --- a/exchangis-engines/exchangis-engine-server/pom.xml +++ b/exchangis-engines/exchangis-engine-server/pom.xml @@ -5,7 +5,7 @@ exchangis-engines com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} 4.0.0 @@ -20,12 +20,12 @@ com.webank.wedatasphere.exchangis exchangis-dao - ${exchangis.version} + ${revision} com.webank.wedatasphere.exchangis exchangis-engine-core - ${exchangis.version} + ${revision} diff --git a/exchangis-engines/pom.xml b/exchangis-engines/pom.xml index 7dfac3c51..edec6ab46 100644 --- a/exchangis-engines/pom.xml +++ b/exchangis-engines/pom.xml @@ -5,13 +5,14 @@ exchangis com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} + ../pom.xml 4.0.0 exchangis-engines pom - 1.1.2 + ${revision} exchangis-engine-common diff --git a/exchangis-job/exchangis-job-builder/pom.xml b/exchangis-job/exchangis-job-builder/pom.xml index acfbb460a..e5d3591fb 100644 --- a/exchangis-job/exchangis-job-builder/pom.xml +++ b/exchangis-job/exchangis-job-builder/pom.xml @@ -5,7 +5,7 @@ exchangis-job com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} 4.0.0 @@ -15,7 +15,7 @@ com.webank.wedatasphere.exchangis exchangis-job-common - 1.1.2 + ${revision} com.google.code.gson @@ -25,7 +25,7 @@ com.webank.wedatasphere.exchangis exchangis-datasource-service - 1.1.2 + ${revision} compile diff --git a/exchangis-job/exchangis-job-common/pom.xml b/exchangis-job/exchangis-job-common/pom.xml index 1bae2fa82..e04e3e922 100644 --- a/exchangis-job/exchangis-job-common/pom.xml +++ b/exchangis-job/exchangis-job-common/pom.xml @@ -5,7 +5,7 @@ exchangis-job com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} 4.0.0 @@ -15,7 +15,7 @@ com.webank.wedatasphere.exchangis exchangis-dao - 1.1.2 + ${revision} org.apache.linkis @@ -25,7 +25,7 @@ com.webank.wedatasphere.exchangis exchangis-engine-common - 1.1.2 + ${revision} org.apache.linkis diff --git a/exchangis-job/exchangis-job-launcher/pom.xml b/exchangis-job/exchangis-job-launcher/pom.xml index 2375cdea3..19bf193f4 100644 --- a/exchangis-job/exchangis-job-launcher/pom.xml +++ b/exchangis-job/exchangis-job-launcher/pom.xml @@ -5,7 +5,7 @@ exchangis-job com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} 4.0.0 @@ -15,12 +15,12 @@ com.webank.wedatasphere.exchangis exchangis-job-common - 1.1.2 + ${revision} com.webank.wedatasphere.exchangis exchangis-job-builder - 1.1.2 + ${revision} org.apache.linkis diff --git a/exchangis-job/exchangis-job-metrics/pom.xml b/exchangis-job/exchangis-job-metrics/pom.xml index 9dcdd12fb..5cfd72dc9 100644 --- a/exchangis-job/exchangis-job-metrics/pom.xml +++ b/exchangis-job/exchangis-job-metrics/pom.xml @@ -5,7 +5,7 @@ exchangis-job com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} 4.0.0 diff --git a/exchangis-job/exchangis-job-server/pom.xml b/exchangis-job/exchangis-job-server/pom.xml index 60c64165e..df144b413 100644 --- a/exchangis-job/exchangis-job-server/pom.xml +++ b/exchangis-job/exchangis-job-server/pom.xml @@ -5,7 +5,7 @@ exchangis-job com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} 4.0.0 @@ -20,22 +20,22 @@ com.webank.wedatasphere.exchangis exchangis-project-provider - 1.1.2 + ${revision} com.webank.wedatasphere.exchangis exchangis-job-launcher - 1.1.2 + ${revision} com.webank.wedatasphere.exchangis exchangis-datasource-service - 1.1.2 + ${revision} com.webank.wedatasphere.exchangis exchangis-engine-core - ${exchangis.version} + ${revision} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ExchangisJobDsBindServiceImpl.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ExchangisJobDsBindServiceImpl.java index 567895e08..1987366f4 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ExchangisJobDsBindServiceImpl.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ExchangisJobDsBindServiceImpl.java @@ -32,7 +32,7 @@ public void updateJobDsBind(Long jobId, List dsBinds) { public boolean inUse(Long datasourceId) { QueryWrapper condition = new QueryWrapper<>(); condition.eq("source_ds_id", datasourceId).or().eq("sink_ds_id", datasourceId); - Long count = Optional.ofNullable(this.dsBindMapper.selectCount(condition)).orElse(0L); + int count = Optional.ofNullable(this.dsBindMapper.selectCount(condition)).orElse(0); return count > 0; } } diff --git a/exchangis-job/exchangis-job-service/pom.xml b/exchangis-job/exchangis-job-service/pom.xml index 44ffac3db..10c0afb92 100644 --- a/exchangis-job/exchangis-job-service/pom.xml +++ b/exchangis-job/exchangis-job-service/pom.xml @@ -5,7 +5,7 @@ exchangis com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} ../../pom.xml 4.0.0 @@ -21,12 +21,12 @@ com.webank.wedatasphere.exchangis exchangis-job-common - 1.1.2 + ${revision} com.webank.wedatasphere.exchangis exchangis-job-launcher - 1.1.2 + ${revision} diff --git a/exchangis-job/pom.xml b/exchangis-job/pom.xml index e28c390ac..c8a18cc7e 100644 --- a/exchangis-job/pom.xml +++ b/exchangis-job/pom.xml @@ -5,13 +5,13 @@ exchangis com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} 4.0.0 exchangis-job pom - 1.1.2 + ${revision} exchangis-job-common diff --git a/exchangis-plugins/exchangis-appconn/pom.xml b/exchangis-plugins/exchangis-appconn/pom.xml index 0c1212479..c962ac295 100644 --- a/exchangis-plugins/exchangis-appconn/pom.xml +++ b/exchangis-plugins/exchangis-appconn/pom.xml @@ -5,7 +5,7 @@ exchangis-plugins com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} ../pom.xml 4.0.0 diff --git a/exchangis-plugins/pom.xml b/exchangis-plugins/pom.xml index 8b290ca55..8325cf1ba 100644 --- a/exchangis-plugins/pom.xml +++ b/exchangis-plugins/pom.xml @@ -5,13 +5,13 @@ exchangis com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} 4.0.0 exchangis-plugins pom - 1.1.2 + ${revision} exchangis-appconn diff --git a/exchangis-project/exchangis-project-entity/pom.xml b/exchangis-project/exchangis-project-entity/pom.xml index 198497921..aa053c166 100644 --- a/exchangis-project/exchangis-project-entity/pom.xml +++ b/exchangis-project/exchangis-project-entity/pom.xml @@ -5,7 +5,7 @@ exchangis-project com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} 4.0.0 @@ -20,7 +20,7 @@ com.webank.wedatasphere.exchangis exchangis-dao - 1.1.2 + ${revision} org.apache.commons diff --git a/exchangis-project/exchangis-project-provider/pom.xml b/exchangis-project/exchangis-project-provider/pom.xml index b658e8b07..90a65a2e1 100644 --- a/exchangis-project/exchangis-project-provider/pom.xml +++ b/exchangis-project/exchangis-project-provider/pom.xml @@ -5,7 +5,7 @@ exchangis-project com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} 4.0.0 @@ -20,7 +20,7 @@ com.webank.wedatasphere.exchangis exchangis-project-entity - 1.1.2 + ${revision} diff --git a/exchangis-project/exchangis-project-server/pom.xml b/exchangis-project/exchangis-project-server/pom.xml index 0853557d1..ce392fde1 100644 --- a/exchangis-project/exchangis-project-server/pom.xml +++ b/exchangis-project/exchangis-project-server/pom.xml @@ -5,7 +5,7 @@ exchangis-project com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} 4.0.0 @@ -20,22 +20,22 @@ com.webank.wedatasphere.exchangis exchangis-project-provider - 1.1.2 + ${revision} com.webank.wedatasphere.exchangis exchangis-job-server - 1.1.2 + ${revision} com.webank.wedatasphere.exchangis exchangis-dao - 1.1.2 + ${revision} com.webank.wedatasphere.exchangis exchangis-job-common - 1.1.2 + ${revision} diff --git a/exchangis-project/pom.xml b/exchangis-project/pom.xml index 99a3c3a17..d71cd1525 100644 --- a/exchangis-project/pom.xml +++ b/exchangis-project/pom.xml @@ -5,7 +5,7 @@ exchangis com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} 4.0.0 diff --git a/exchangis-server/pom.xml b/exchangis-server/pom.xml index 2b16d972d..d0ece72bd 100644 --- a/exchangis-server/pom.xml +++ b/exchangis-server/pom.xml @@ -5,7 +5,7 @@ exchangis com.webank.wedatasphere.exchangis - 1.1.2 + ${revision} 4.0.0 @@ -20,12 +20,12 @@ com.webank.wedatasphere.exchangis exchangis-datasource-server - 1.1.2 + ${revision} com.webank.wedatasphere.exchangis exchangis-engine-server - ${exchangis.version} + ${revision} org.apache.linkis @@ -40,20 +40,20 @@ com.webank.wedatasphere.exchangis exchangis-job-server - 1.1.2 + ${revision} com.webank.wedatasphere.exchangis exchangis-project-server - 1.1.2 + ${revision} org.apache.linkis diff --git a/exchangis-server/src/main/java/com/webank/wedatasphere/exchangis/queue/BinlogArrayLockFreeQueue.java b/exchangis-server/src/main/java/com/webank/wedatasphere/exchangis/queue/BinlogArrayLockFreeQueue.java new file mode 100644 index 000000000..8a40a0c05 --- /dev/null +++ b/exchangis-server/src/main/java/com/webank/wedatasphere/exchangis/queue/BinlogArrayLockFreeQueue.java @@ -0,0 +1,465 @@ +package com.webank.wedatasphere.exchangis.queue; + +import sun.misc.Unsafe; + +import java.lang.reflect.Field; +import java.security.AccessController; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.LockSupport; +import java.util.concurrent.locks.ReentrantLock; + +public class BinlogArrayLockFreeQueue { + /** + * Allocation 64 MB buffer default + */ + private static final int DEFAULT_QUEUE_BUFFER = 1024 * 1024 * 64; + + private static final long DEFAULT_MEASURE_INTERVAL = 2 * 1000L; + + /** + * We should reduce the cpu usage + */ + private static final long DEFAULT_SPIN_TIMES = 10; + + private final Unsafe unsafe = UnsafeUtil.unsafe; + + final Object[] items; + + /** + * take index + */ + private volatile long takeIndex; + + /** + * put index + */ + private volatile long putIndex; + + /** + * max take index + */ + private volatile long maxTakeIndex; + + /** + * Memory bytes accumulated + */ + private volatile long memoryBytes; + + /** + * Wait take time + */ + private volatile long waitTake; + + /** + * Wait put time + */ + private volatile long waitPut; + + /** + * Flag to measure + */ + private volatile long measureFlag; + + /** + * Buffer size limit + */ + private long bufferSize; + /** + * Measure interval + */ + private long measureInterval; + + + private final ReentrantLock waitLock = new ReentrantLock(false); + + private final Condition notEmpty = waitLock.newCondition(); + + public BinlogArrayLockFreeQueue(int capacity, long bufferSize, long measureInterval){ + // Init the array size as ring buffer, left one chunk + if ((capacity & (capacity - 1)) != 0){ + throw new IllegalArgumentException("the value of capacity must equal to 2^N and greater than 1"); + } + items = new Object[capacity]; + if (bufferSize <= 0){ + bufferSize = Integer.MAX_VALUE; + } + this.bufferSize = bufferSize; + this.measureInterval = measureInterval; + } + + public BinlogArrayLockFreeQueue(int capacity){ + this(capacity, DEFAULT_QUEUE_BUFFER, DEFAULT_MEASURE_INTERVAL); + } + + public void put(T message) throws InterruptedException { + if (Objects.nonNull(message)){ + long curTakeIndex; + long curPutIndex; + long nextPutIndex; + long waitTime = 0; + long clock = 0; + try { + do { + int counter = -1; + do { + counter ++; + // Lock and wait the queue not full + if (counter > 0) { + LockSupport.parkNanos(1L); + } + curPutIndex = this.putIndex; + curTakeIndex = this.takeIndex; + nextPutIndex = curPutIndex + 1; + clock = System.nanoTime(); + } while(toIndex(nextPutIndex) == toIndex(curTakeIndex)); + if (counter > 0){ + waitTime += (System.nanoTime() - clock); + } + } while (!unsafe.compareAndSwapLong(this, Offsets.putIndexOffset, curPutIndex, nextPutIndex)); + // Accumulate the memory + accumulateMemory(1); + // Write the circle + this.items[toIndex(curPutIndex)] = message; +// if (waitTime > 0) { +// unsafe.getAndAddLong(this, Offsets.waitTakeOffset, waitTime); +// } + while (!unsafe.compareAndSwapLong(this, Offsets.maxTakeIndexOffset, curPutIndex, nextPutIndex)){ + // Notify the older producer to update the max take index + Thread.yield(); + } + + }finally { + // Notify the waiter + waitLock.lock(); + try { + notEmpty.signalAll(); + } finally { + waitLock.unlock(); + } + // Try to measure the queue indicator +// measureIndicator(); + } + } + } + + + @SuppressWarnings("unchecked") + public T take(long timeout, TimeUnit unit) throws InterruptedException { + long nanos = unit.toNanos(timeout); + long curMaxTakeIndex; + long curTakeIndex; + long nextTakeIndex; + T element; + int takePos; + int iterator = 0; + long waitTime = 0; + do { + curMaxTakeIndex = this.maxTakeIndex; + curTakeIndex = this.takeIndex; + long clock = System.nanoTime(); + while (toIndex(curTakeIndex) == toIndex(curMaxTakeIndex)) { + // Wrap as wait strategy + ++ iterator; + // Enable to iterator times + if (iterator > DEFAULT_SPIN_TIMES && iterator <= DEFAULT_SPIN_TIMES * 2){ + // Try to park to release cpu + LockSupport.parkNanos(1L); + } else if (iterator > DEFAULT_SPIN_TIMES * 2){ + waitLock.lockInterruptibly(); + curTakeIndex = this.takeIndex; + curMaxTakeIndex = this.maxTakeIndex; + try { + if (toIndex(curTakeIndex) == toIndex(curMaxTakeIndex)) { + if (nanos <= 0) { + return null; + } + nanos = notEmpty.awaitNanos(nanos); + iterator = 0; + } + } finally { + waitLock.unlock(); + } + } + curTakeIndex = this.takeIndex; + curMaxTakeIndex = this.maxTakeIndex; + } + if (iterator > 0){ + waitTime += (System.nanoTime() - clock); + } + nextTakeIndex = curTakeIndex + 1; + takePos = toIndex(curTakeIndex); + element = (T) this.items[takePos]; + } while(!unsafe.compareAndSwapLong(this, Offsets.takeIndexOffset, curTakeIndex, nextTakeIndex)); + // Empty the cache and release the memory + if (null != element) { + this.items[takePos] = null; +// unsafe.getAndAddInt(this, Offsets.memoryBytesOffset, -1); + } +// if (waitTime > 0){ +// unsafe.getAndAddLong(this, Offsets.waitPutOffset, waitTime); +// } + this.items[takePos] = null; + // Try to measure the queue indicator + measureIndicator(); + return element; + } + + + @SuppressWarnings("unchecked") + public int drainTo(List elements, int maxElements) { + long curMaxTakeIndex = this.maxTakeIndex; + long curTakeIndex = this.takeIndex; + long nextTakeIndex; + int takePos; + int count = 0; + int bytesCnt = 0; + // Break if queue is empty + while(toIndex(curTakeIndex) != toIndex(curMaxTakeIndex)) { + nextTakeIndex = curTakeIndex + 1; + takePos = toIndex(curTakeIndex); + if (unsafe.compareAndSwapLong(this, Offsets.takeIndexOffset, curTakeIndex, nextTakeIndex)){ + T element = (T) this.items[takePos]; + elements.add(element); + count ++; + // Empty the cache + this.items[takePos] = null; + bytesCnt = bytesCnt + 1; + if (count >= maxElements){ + break; + } + } + curTakeIndex = this.takeIndex; + curMaxTakeIndex = this.maxTakeIndex; + } + if (bytesCnt > 0) { + unsafe.getAndAddInt(this, Offsets.memoryBytesOffset, -bytesCnt); + } + measureIndicator(); + return count; + } + + + + + public void adjustBuffer(long bufferSize) { + // Just update buffer size limit + this.bufferSize = bufferSize; + } + + /** + * Accumulate memory bytes + * @param byteSize byte Size + */ + private void accumulateMemory(int byteSize){ + // Add memory count + unsafe.getAndAddInt(this, Offsets.memoryBytesOffset, byteSize); + while(memoryBytes >= this.bufferSize){ + // Optimize the park strategy + LockSupport.parkNanos(1L); + } + } + /** + * Convert the long sequence to index + * @param sequence sequenceId + * @return position + */ + private int toIndex(long sequence){ + return (int) (sequence & (items.length - 1)); + } + + /** + * Measure method + */ + private void measureIndicator(){ +// long clock = System.currentTimeMillis(); +// long measureTime = this.measureFlag; +// if (clock >= measureTime){ +// // Only use the wait take time to measure pressure +// long waitTime = this.waitTake; +// if (unsafe.compareAndSwapLong(this, Offsets.measureFlagOffset, +// measureTime, clock + this.measureInterval)){ +// // decrease the wait take time +// indicator.setBufferUsed(memoryBytes); +// indicator.setPressure((double)waitTime/ ((double)(clock - measureTime) * Math.pow(10, 6))); +// long time = unsafe.getAndAddLong(this, Offsets.waitTakeOffset, -waitTime); +// if (time < waitTime){ +// // Occur some error? init to zero +// this.waitTake = 0; +// } +// this.waitPut = 0; +// //Invoke the listener +// try { +// listeners.forEach(listener -> listener.onMeasure(indicator)); +// }catch(Exception e){ +// LOG.warn("Error occurred while measuring the queue indicator", e); +// // Not to throw exception +// } +// } +// } + } + private static class UnsafeUtil{ + + private static final Unsafe unsafe; + + static { + final PrivilegedExceptionAction action = () -> { + Field theUnsafe = Unsafe.class.getDeclaredField("theUnsafe"); + theUnsafe.setAccessible(true); + return (Unsafe) theUnsafe.get(null); + }; + try { + unsafe = AccessController.doPrivileged(action); + } catch (PrivilegedActionException e) { + // Throw error + throw new Error(e); + } + } + } + + /** + * Queue field offsets + */ + private static class Offsets{ + /** + * Take index field offset + */ + private static final long takeIndexOffset; + + /** + * Put index field offset + */ + private static final long putIndexOffset; + + /** + * Max take index field offset + */ + private static final long maxTakeIndexOffset; + + /** + * Memory bytes field offset + */ + private static final long memoryBytesOffset; + + /** + * Wait put field offset + */ + private static final long waitPutOffset; + + /** + * Wait take field offset + */ + private static final long waitTakeOffset; + + /** + * Measure flag field offset + */ + private static final long measureFlagOffset; + + static { + Unsafe unsafe = UnsafeUtil.unsafe; + try { + takeIndexOffset = unsafe.objectFieldOffset + (BinlogArrayLockFreeQueue.class.getDeclaredField("takeIndex")); + putIndexOffset = unsafe.objectFieldOffset + (BinlogArrayLockFreeQueue.class.getDeclaredField("putIndex")); + maxTakeIndexOffset = unsafe.objectFieldOffset + (BinlogArrayLockFreeQueue.class.getDeclaredField("maxTakeIndex")); + memoryBytesOffset = unsafe.objectFieldOffset + (BinlogArrayLockFreeQueue.class.getDeclaredField("memoryBytes")); + waitPutOffset = unsafe.objectFieldOffset + (BinlogArrayLockFreeQueue.class.getDeclaredField("waitPut")); + waitTakeOffset = unsafe.objectFieldOffset + (BinlogArrayLockFreeQueue.class.getDeclaredField("waitTake")); + measureFlagOffset = unsafe.objectFieldOffset + (BinlogArrayLockFreeQueue.class.getDeclaredField("measureFlag")); + }catch (Exception e){ + throw new Error(e); + } + } + } + + public static void main(String[] args) { + ArrayBlockingQueue queue1 = new ArrayBlockingQueue<>((int)Math.pow(2, 10)); + Executors.newSingleThreadExecutor().submit(() -> { + int count = 0; + while(true) { + String value = null; + // value = queue1.poll(1, TimeUnit.SECONDS); +// queue1.drainTo(new ArrayList<>()); + queue1.take(); + // if (Objects.nonNull(value)){ +// count ++; +// } else { +// System.out.println("blockingQueue(num)" + count); +// break; +// } + } + }); + for (int j = 0; j < 1; j++){ + final int finalJ = j; + new Thread(new Runnable() { + + public void run() { + long time = System.currentTimeMillis(); + for(int i = 0; i < 6000000; i ++){ + try { + long clock = System.currentTimeMillis(); + queue1.put("hello"); + if (System.currentTimeMillis() - clock >= 3){ +// System.out.println("spend1: " + (System.currentTimeMillis() - clock)); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + System.out.println("blockingQueue" + finalJ + ": " + (System.currentTimeMillis() - time)); + } + }).start(); + } + BinlogArrayLockFreeQueue queue = new BinlogArrayLockFreeQueue<>((int)Math.pow(2, 10)); + Executors.newSingleThreadExecutor().submit(() -> { + int count = 0; + while(true) { + // value = queue.take(1, TimeUnit.SECONDS); + int size = queue.drainTo(new ArrayList<>(), Integer.MAX_VALUE); + +// if (Objects.nonNull(value)){ +// count = count + 1; +// } else { +// System.out.println("lockFreeQueue(num)" + count); +// break; +// } + } + }); + for (int j = 0; j < 1; j++){ + final int finalJ = j; + new Thread(new Runnable() { + + public void run() { + long time = System.currentTimeMillis(); + for(int i = 0; i < 6000000; i ++){ + long clock = System.currentTimeMillis(); + try { + queue.put("hello"); + } catch (InterruptedException e) { + e.printStackTrace(); + } + if (System.currentTimeMillis() - clock >= 3){ +// System.out.println("spend2: " + i + ":" + (System.currentTimeMillis() - clock)); + } + } + System.out.println("lockFreeQueue" + finalJ + ": " + (System.currentTimeMillis() - time)); + } + }).start(); + } + } + +} diff --git a/pom.xml b/pom.xml index 85cd79e9d..a1442acba 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ com.webank.wedatasphere.exchangis exchangis - 1.1.2 + ${revision} pom exchangis @@ -37,10 +37,10 @@ - 1.1.2 + 1.1.3-webank 1.1.2 - 1.4.0 - 1.4.0 + 1.3.0-wds + 1.3.0-wds 0.1.0-SNAPSHOT 2.12.12 4.7.1 From a27879ce5042b6d7f68a1cedf66f5c076a1b15b0 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Tue, 30 Apr 2024 18:28:53 +0800 Subject: [PATCH 02/60] Minor fix for packaging --- .../exchangis/datasource/Utils/RSAUtil.java | 11 +++-- .../engines/datax/datax-core/pom.xml | 9 ++++ exchangis-plugins/exchangis-appconn/pom.xml | 43 ++++++++++++++++++- .../src/main/assembly/distribution.xml | 1 - pom.xml | 4 ++ 5 files changed, 59 insertions(+), 9 deletions(-) diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/Utils/RSAUtil.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/Utils/RSAUtil.java index d75996d99..b85130df5 100644 --- a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/Utils/RSAUtil.java +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/Utils/RSAUtil.java @@ -1,14 +1,13 @@ package com.webank.wedatasphere.exchangis.datasource.Utils; import org.apache.linkis.common.conf.CommonVars; -import sun.misc.BASE64Decoder; -import sun.misc.BASE64Encoder; import javax.crypto.Cipher; import java.io.IOException; import java.security.*; import java.security.spec.PKCS8EncodedKeySpec; import java.security.spec.X509EncodedKeySpec; +import java.util.Base64; /** * @author tikazhang @@ -78,13 +77,13 @@ public static byte[] privateDecrypt(byte[] content, PrivateKey privateKey) throw //字节数组转Base64编码 public static String byte2Base64(byte[] bytes){ - BASE64Encoder encoder = new BASE64Encoder(); - return encoder.encode(bytes); + Base64.Encoder encoder = Base64.getEncoder(); + return encoder.encodeToString(bytes); } //Base64编码转字节数组 public static byte[] base642Byte(String base64Key) throws IOException { - BASE64Decoder decoder = new BASE64Decoder(); - return decoder.decodeBuffer(base64Key); + Base64.Decoder decoder = Base64.getDecoder(); + return decoder.decode(base64Key); } } diff --git a/exchangis-engines/engines/datax/datax-core/pom.xml b/exchangis-engines/engines/datax/datax-core/pom.xml index ccf8c79a5..18e65a5f5 100644 --- a/exchangis-engines/engines/datax/datax-core/pom.xml +++ b/exchangis-engines/engines/datax/datax-core/pom.xml @@ -198,6 +198,15 @@ core + + org.apache.maven.plugins + maven-compiler-plugin + 2.5.1 + + ${jdk.compile.version} + ${jdk.compile.version} + + \ No newline at end of file diff --git a/exchangis-plugins/exchangis-appconn/pom.xml b/exchangis-plugins/exchangis-appconn/pom.xml index c962ac295..4bfcbdba3 100644 --- a/exchangis-plugins/exchangis-appconn/pom.xml +++ b/exchangis-plugins/exchangis-appconn/pom.xml @@ -25,8 +25,24 @@ provided - httpclient org.apache.httpcomponents + httpclient + + + org.springframework + spring-core + + + org.springframework.boot + spring-boot + + + org.springframework.boot + spring-boot-starter-cache + + + org.springframework.cloud + spring-cloud-starter-netflix-eureka-client true @@ -48,8 +64,8 @@ provided - linkis-common org.apache.linkis + linkis-common @@ -73,6 +89,29 @@ dss-common ${dss.version} provided + + + + org.springframework + spring-aop + + + org.springframework + spring-context-support + + + org.springframework + spring-webmvc + + + org.springframework + spring-jdbc + + + org.springframework + spring-tx + + org.reflections diff --git a/exchangis-server/src/main/assembly/distribution.xml b/exchangis-server/src/main/assembly/distribution.xml index 5f7ffc96c..5145471d5 100644 --- a/exchangis-server/src/main/assembly/distribution.xml +++ b/exchangis-server/src/main/assembly/distribution.xml @@ -22,7 +22,6 @@ exchangis-server tar.gz - false diff --git a/pom.xml b/pom.xml index a1442acba..e543e521f 100644 --- a/pom.xml +++ b/pom.xml @@ -223,6 +223,10 @@ ${jdk.compile.version} ${jdk.compile.version} + + + ${java.home}/lib/rt.jar;${java.home}/lib/jce.jar + From 9728157a8fa42281cc0b2c4ce2502d07924a58a6 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Mon, 6 May 2024 15:28:14 +0800 Subject: [PATCH 03/60] Minor fix of dml --- db/exchangis_dml.sql | 2 +- exchangis-plugins/exchangis-appconn/pom.xml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/db/exchangis_dml.sql b/db/exchangis_dml.sql index 2e6bee29e..bcd727c25 100644 --- a/db/exchangis_dml.sql +++ b/db/exchangis_dml.sql @@ -64,7 +64,7 @@ INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_directio -- engine_settings records INSERT INTO `exchangis_engine_settings` (id, engine_name, engine_desc, engine_settings_value, engine_direction, res_loader_class, res_uploader_class, modify_time) VALUES -(1, 'datax', 'datax sync engine', '{}', 'mysql->hive,hive->mysql,mysql->oracle,oracle->mysql,oracle->hive,hive->oracle,mongodb->hive,hive->mongodb,mysql->elasticsearch,oracle->elasticsearch,mongodb->elasticsearch,mysql->mongodb,mongodb->mysql,oracle->mongodb,mongodb->oracle', 'com.webank.wedatasphere.exchangis.engine.resource.loader.datax.DataxEngineResourceLoader', NULL, NULL, '2022-08-09 18:20:51.0'), +(1, 'datax', 'datax sync engine', '{}', 'mysql->hive,hive->mysql,mysql->oracle,oracle->mysql,oracle->hive,hive->oracle,mongodb->hive,hive->mongodb,mysql->elasticsearch,oracle->elasticsearch,mongodb->elasticsearch,mysql->mongodb,mongodb->mysql,oracle->mongodb,mongodb->oracle', 'com.webank.wedatasphere.exchangis.engine.resource.loader.datax.DataxEngineResourceLoader', NULL, NULL), (2, 'sqoop', 'hadoop tool', '{}', 'mysql->hive,hive->mysql', '', NULL, NULL); -- exchangis_job_transform_rule records diff --git a/exchangis-plugins/exchangis-appconn/pom.xml b/exchangis-plugins/exchangis-appconn/pom.xml index 4bfcbdba3..7ee765569 100644 --- a/exchangis-plugins/exchangis-appconn/pom.xml +++ b/exchangis-plugins/exchangis-appconn/pom.xml @@ -90,7 +90,6 @@ ${dss.version} provided - org.springframework spring-aop From 16fb0be1beea77742fc9c823541579f260fb1e05 Mon Sep 17 00:00:00 2001 From: davidhua Date: Mon, 6 May 2024 20:56:09 +0800 Subject: [PATCH 04/60] Add priority queue to consumer manager. --- .../scheduler/ExchangisSchedulerTask.java | 3 +- .../execution/scheduler/SchedulerThread.java | 4 +- .../TenancyParallelConsumerManager.java | 48 +++-- .../priority/PriorityOrderedQueue.java | 169 ++++++++++++++++++ .../scheduler/priority/PriorityRunnable.java | 16 ++ 5 files changed, 228 insertions(+), 12 deletions(-) create mode 100644 exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/priority/PriorityOrderedQueue.java create mode 100644 exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/priority/PriorityRunnable.java diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/ExchangisSchedulerTask.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/ExchangisSchedulerTask.java index 904906274..a0ec6eeb4 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/ExchangisSchedulerTask.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/ExchangisSchedulerTask.java @@ -1,11 +1,12 @@ package com.webank.wedatasphere.exchangis.job.server.execution.scheduler; +import com.webank.wedatasphere.exchangis.job.server.execution.scheduler.priority.PriorityRunnable; import org.apache.linkis.scheduler.queue.SchedulerEvent; /** * Exchangis scheduler task */ -public interface ExchangisSchedulerTask extends SchedulerEvent { +public interface ExchangisSchedulerTask extends PriorityRunnable, SchedulerEvent { /** * Tenancy diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/SchedulerThread.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/SchedulerThread.java index df9717029..b9da86213 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/SchedulerThread.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/SchedulerThread.java @@ -1,9 +1,11 @@ package com.webank.wedatasphere.exchangis.job.server.execution.scheduler; +import com.webank.wedatasphere.exchangis.job.server.execution.scheduler.priority.PriorityRunnable; + /** * Define the basic interface of thread in scheduler */ -public interface SchedulerThread extends Runnable{ +public interface SchedulerThread extends PriorityRunnable { /** * Start entrance */ diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/TenancyParallelConsumerManager.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/TenancyParallelConsumerManager.java index da52b214b..0e2bd5caf 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/TenancyParallelConsumerManager.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/TenancyParallelConsumerManager.java @@ -1,6 +1,8 @@ package com.webank.wedatasphere.exchangis.job.server.execution.scheduler; import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisSchedulerException; +import com.webank.wedatasphere.exchangis.job.server.execution.scheduler.priority.PriorityOrderedQueue; +import com.webank.wedatasphere.exchangis.job.server.execution.scheduler.priority.PriorityRunnable; import org.apache.commons.lang.StringUtils; import org.apache.linkis.common.utils.Utils; import org.apache.linkis.scheduler.listener.ConsumerListener; @@ -9,14 +11,10 @@ import org.apache.linkis.scheduler.queue.fifoqueue.FIFOUserConsumer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; - -import javax.annotation.PreDestroy; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; + +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantLock; /** @@ -53,7 +51,7 @@ public ExecutorService getOrCreateExecutorService() { try{ Group group = getSchedulerContext().getOrCreateGroupFactory().getOrCreateGroup(null); if (group instanceof FIFOGroup){ - defaultExecutorService = Utils.newCachedThreadPool(((FIFOGroup) group).getMaxRunningJobs() + + defaultExecutorService = newPriorityThreadPool(((FIFOGroup) group).getMaxRunningJobs() + this.initResidentThreads + 1, TenancyParallelGroupFactory.GROUP_NAME_PREFIX + TenancyParallelGroupFactory.DEFAULT_TENANCY + "-Executor-", true); tenancyExecutorServices.put(TenancyParallelGroupFactory.DEFAULT_TENANCY, defaultExecutorService); @@ -128,7 +126,7 @@ protected ExecutorService getOrCreateExecutorService(String groupName){ if (StringUtils.isNotBlank(tenancy)){ return tenancyExecutorServices.computeIfAbsent(tenancy, tenancyName -> { // Use the default value of max running jobs - return Utils.newCachedThreadPool(parallelGroupFactory.getDefaultMaxRunningJobs() + parallelGroupFactory.getParallelPerTenancy(), + return newPriorityThreadPool(parallelGroupFactory.getDefaultMaxRunningJobs() + parallelGroupFactory.getParallelPerTenancy(), TenancyParallelGroupFactory.GROUP_NAME_PREFIX + tenancy + "-Executor-", true); }); } @@ -151,4 +149,34 @@ public void setInitResidentThreads(int initResidentThreads) { public Map getTenancyExecutorServices() { return tenancyExecutorServices; } + + /** + * Create thread pool with priority for tenancy consumer + * @return + */ + private ExecutorService newPriorityThreadPool(int threadNum, String threadName, boolean isDaemon){ + ThreadPoolExecutor threadPool = new ThreadPoolExecutor( + threadNum, + threadNum, + 120L, + TimeUnit.SECONDS, + new PriorityBlockingQueue<>(10 * threadNum, (o1, o2) -> { + int left = o1 instanceof PriorityRunnable ? ((PriorityRunnable) o1).getPriority() : 0; + int right = o2 instanceof PriorityRunnable ? ((PriorityRunnable) o2).getPriority() : 0; + return right - left; + }), + new ThreadFactory() { + final AtomicInteger num = new AtomicInteger(0); + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(r); + t.setDaemon(isDaemon); + t.setName(threadName + num.incrementAndGet()); + return t; + } + }); + threadPool.allowCoreThreadTimeOut(true); + return threadPool; + } + } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/priority/PriorityOrderedQueue.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/priority/PriorityOrderedQueue.java new file mode 100644 index 000000000..4277d768a --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/priority/PriorityOrderedQueue.java @@ -0,0 +1,169 @@ +package com.webank.wedatasphere.exchangis.job.server.execution.scheduler.priority; + +import com.webank.wedatasphere.exchangis.job.utils.SnowFlake; + +import java.util.*; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.PriorityBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +/** + * Refer to 'PriorityBlockingQueue', + * Use snowflake to generate order number of elements + */ +public class PriorityOrderedQueue extends AbstractQueue + implements BlockingQueue, java.io.Serializable { + + /** + * Priority queue + */ + private final PriorityBlockingQueue priorityQueue; + + /** + /** + * Snowflake context + */ + private final SnowFlake snowFlake; + public PriorityOrderedQueue(int initialCapacity, + Comparator comparator){ + if (Objects.isNull(comparator)){ + this.priorityQueue = new PriorityBlockingQueue<>(initialCapacity, + (left, right) -> (int) (right.seq - left.seq)); + } else { + this.priorityQueue = new PriorityBlockingQueue<>(initialCapacity, + (left, right) -> { + int result = comparator.compare(left.element, right.element); + if (result == 0){ + return (int)(left.seq - right.seq); + } + return result; + }); + } + this.snowFlake = new SnowFlake(0, 0, System.currentTimeMillis()); + } + @Override + public Iterator iterator() { + return new Itr(priorityQueue.iterator()); + } + + @Override + public int size() { + return priorityQueue.size(); + } + + @Override + public void put(E e) throws InterruptedException { + offer(e); + } + + @Override + public boolean offer(E e, long timeout, TimeUnit unit) throws InterruptedException { + return offer(e); + } + + @Override + public E take() throws InterruptedException { + Ordered ordered = this.priorityQueue.take(); + return ordered.element; + } + + @Override + public E poll(long timeout, TimeUnit unit) throws InterruptedException { + Ordered ordered = this.priorityQueue.poll(timeout, unit); + if (null != ordered){ + return ordered.element; + } + return null; + } + + @Override + public int remainingCapacity() { + return this.priorityQueue.remainingCapacity(); + } + + @Override + public int drainTo(Collection c) { + return drainTo(c, Integer.MAX_VALUE); + } + + @Override + @SuppressWarnings("unchecked") + public int drainTo(Collection c, int maxElements) { + Collection collection = null; + if (null != c && c != this){ + collection = c.stream().map(e -> new Ordered((E) e)).collect(Collectors.toList()); + } + return this.priorityQueue.drainTo(collection); + } + + @Override + public boolean offer(E e) { + return this.priorityQueue.offer(new Ordered(e)); + } + + @Override + public E poll() { + Ordered ordered = this.priorityQueue.poll(); + if (null != ordered){ + return ordered.element; + } + return null; + } + + @Override + public E peek() { + Ordered ordered = this.priorityQueue.peek(); + if (null != ordered){ + return ordered.element; + } + return null; + } + + private class Ordered{ + /** + * Seq number + */ + private long seq; + + /** + * Queue element + */ + private E element; + + public Ordered(E element){ + this.seq = snowFlake.nextId(); + this.element = element; + } + } + + private class Itr implements Iterator { + private Iterator innerItr; + public Itr(Iterator iterator){ + innerItr = iterator; + } + + + @Override + public boolean hasNext() { + return innerItr.hasNext(); + } + + @Override + public E next() { + return innerItr.next().element; + } + + @Override + public void remove() { + innerItr.remove(); + } + + @Override + public void forEachRemaining(Consumer action) { + innerItr.forEachRemaining(eOrdered -> + action.accept(eOrdered.element)); + } + } +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/priority/PriorityRunnable.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/priority/PriorityRunnable.java new file mode 100644 index 000000000..4d8ed3608 --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/priority/PriorityRunnable.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.exchangis.job.server.execution.scheduler.priority; + +/** + * Runnable with priority + */ +public interface PriorityRunnable extends Runnable{ + + /** + * Default: 1 + * @return value + */ + default int getPriority(){ + return 1; + } + +} From ab7e1d1923d932c1cfc86c569f9324c091059c58 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Mon, 13 May 2024 22:05:01 +0800 Subject: [PATCH 05/60] Minor fix of deploy scripts and sql --- assembly-package/config/application-exchangis.yml | 2 +- assembly-package/sbin/install.sh | 4 ++-- assembly-package/sbin/launcher.sh | 5 ++--- .../datax/src/main/assembly/distribution.xml | 1 - .../datax/executor/DataxContainerOnceExecutor.scala | 4 ++++ .../main/java/com/alibaba/datax/core/job/JobContainer.java | 4 ++-- .../engine/resource/AbstractEngineResourceContainer.java | 1 + .../exchangis-appconn/src/main/assembly/distribution.xml | 1 + .../exchangis-appconn/src/main/resources/init.sql | 4 ++-- 9 files changed, 15 insertions(+), 11 deletions(-) diff --git a/assembly-package/config/application-exchangis.yml b/assembly-package/config/application-exchangis.yml index 86268b937..946ee2cb8 100644 --- a/assembly-package/config/application-exchangis.yml +++ b/assembly-package/config/application-exchangis.yml @@ -2,7 +2,7 @@ server: port: 9321 spring: application: - name: exchangis-server + name: dss-exchangis-main-server-dev eureka: client: serviceUrl: diff --git a/assembly-package/sbin/install.sh b/assembly-package/sbin/install.sh index a9f23aa66..2ce1569f7 100644 --- a/assembly-package/sbin/install.sh +++ b/assembly-package/sbin/install.sh @@ -30,7 +30,7 @@ PACKAGE_DIR="${DIR}/../packages" # Home Path EXCHNGIS_HOME_PATH="${DIR}/../" -CONF_FILE_PATH="bin/configure.sh" +CONF_FILE_PATH="sbin/configure.sh" FORCE_INSTALL=false SKIP_PACKAGE=false USER=`whoami` @@ -125,7 +125,7 @@ interact_echo(){ # Initalize database init_database(){ - BOOTSTRAP_PROP_FILE="${CONF_PATH}/dss-exchangis-server.properties" + BOOTSTRAP_PROP_FILE="${CONF_PATH}/dss-exchangis-main-server-dev.properties" if [ "x${SQL_SOURCE_PATH}" != "x" ] && [ -f "${SQL_SOURCE_PATH}" ]; then `mysql --version >/dev/null 2>&1` DATASOURCE_URL="jdbc:mysql:\/\/${MYSQL_HOST}:${MYSQL_PORT}\/${DATABASE}\?useSSL=false\&characterEncoding=UTF-8\&allowMultiQueries=true" diff --git a/assembly-package/sbin/launcher.sh b/assembly-package/sbin/launcher.sh index 50a79d279..4c9530eae 100644 --- a/assembly-package/sbin/launcher.sh +++ b/assembly-package/sbin/launcher.sh @@ -110,9 +110,9 @@ construct_java_command(){ # mkdir mkdir -p ${EXCHANGIS_LOG_PATH} mkdir -p ${EXCHANGIS_PID_PATH} - local classpath=${EXCHANGIS_CONF_PATH}":." + local classpath=${EXCHANGIS_CONF_PATH} local opts="" - classpath=${EXCHANGIS_LIB_PATH}/"exchangis-server/*:"${classpath} + classpath=${classpath}":"${EXCHANGIS_LIB_PATH}/"exchangis-server/*:." LOG INFO "classpath:"${classpath} if [[ "x${EXCHANGIS_JAVA_OPTS}" == "x" ]]; then # Use G1 garbage collector @@ -133,7 +133,6 @@ construct_java_command(){ opts=${opts}" -Dlogging.level.reactor.ipc.netty.channel.CloseableContextHandler=off" opts=${opts}" -Duser.dir=${USER_DIR}" opts=${opts}" -classpath "${classpath} - LOG INFO "opts:"${opts} if [[ "x${JAVA_HOME}" != "x" ]]; then EXEC_JAVA=${JAVA_HOME}"/bin/java "${opts}" "$2 else diff --git a/exchangis-engines/engineconn-plugins/datax/src/main/assembly/distribution.xml b/exchangis-engines/engineconn-plugins/datax/src/main/assembly/distribution.xml index 6c517ef5b..dee47b9d9 100644 --- a/exchangis-engines/engineconn-plugins/datax/src/main/assembly/distribution.xml +++ b/exchangis-engines/engineconn-plugins/datax/src/main/assembly/distribution.xml @@ -39,7 +39,6 @@ false false true - antlr:antlr:jar aopalliance:aopalliance:jar diff --git a/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/executor/DataxContainerOnceExecutor.scala b/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/executor/DataxContainerOnceExecutor.scala index 9cec013ac..434108649 100644 --- a/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/executor/DataxContainerOnceExecutor.scala +++ b/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/executor/DataxContainerOnceExecutor.scala @@ -234,6 +234,10 @@ abstract class DataxContainerOnceExecutor extends DataxOnceExecutor with Operabl private def setPluginConfig(self: Configuration): Unit = { val plugins: util.Map[String, Configuration] = dataxEngineConnContext .getPluginDefinitions.asScala.map(define => (define.getPluginName, define.getPluginConf)).toMap.asJava + info(s"content is ${dataxEngineConnContext.toString}") + dataxEngineConnContext.getPluginDefinitions.asScala.foreach { definition => + info(s"PluginName: ${definition.getPluginName}, pluginConf: ${definition.getPluginConf}, pluginPath: ${definition.getPluginPath}") + } val pluginsNeed: util.Map[String, Configuration] = new util.HashMap() Option(self.getString(CoreConstant.DATAX_JOB_CONTENT_READER_NAME)).foreach(readerPlugin => pluginsNeed.put(readerPlugin, plugins.get(readerPlugin))) Option(self.getString(CoreConstant.DATAX_JOB_CONTENT_WRITER_NAME)).foreach(writerPlugin => pluginsNeed.put(writerPlugin, plugins.get(writerPlugin))) diff --git a/exchangis-engines/engines/datax/datax-core/src/main/java/com/alibaba/datax/core/job/JobContainer.java b/exchangis-engines/engines/datax/datax-core/src/main/java/com/alibaba/datax/core/job/JobContainer.java index c71523ece..73d1aceba 100644 --- a/exchangis-engines/engines/datax/datax-core/src/main/java/com/alibaba/datax/core/job/JobContainer.java +++ b/exchangis-engines/engines/datax/datax-core/src/main/java/com/alibaba/datax/core/job/JobContainer.java @@ -37,8 +37,8 @@ import com.webank.wedatasphere.exchangis.datax.util.Json; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.FileFileFilter; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.Validate; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Validate; import org.apache.http.client.methods.HttpPost; import org.apache.http.entity.StringEntity; import org.slf4j.Logger; diff --git a/exchangis-engines/exchangis-engine-core/src/main/java/com/webank/wedatasphere/exchangis/engine/resource/AbstractEngineResourceContainer.java b/exchangis-engines/exchangis-engine-core/src/main/java/com/webank/wedatasphere/exchangis/engine/resource/AbstractEngineResourceContainer.java index b7994bac4..8b353c837 100644 --- a/exchangis-engines/exchangis-engine-core/src/main/java/com/webank/wedatasphere/exchangis/engine/resource/AbstractEngineResourceContainer.java +++ b/exchangis-engines/exchangis-engine-core/src/main/java/com/webank/wedatasphere/exchangis/engine/resource/AbstractEngineResourceContainer.java @@ -245,6 +245,7 @@ private void flushResources(ResourcePathNode pathNode) throws ExchangisEngineRes // Try tp upload the node engine resource try { U uploadedRes = this.engineResourceUploader.upload(nodeEngineRes, pathNode.getRemoteResource()); + LOG.info("uploadedRes is {}", uploadedRes.toString()); if (Objects.nonNull(uploadedRes)) { // Store the uploaded remoted resource information if (Objects.nonNull(pathNode.getRemoteResource())) { diff --git a/exchangis-plugins/exchangis-appconn/src/main/assembly/distribution.xml b/exchangis-plugins/exchangis-appconn/src/main/assembly/distribution.xml index 9174d8a49..8a2f5187d 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/assembly/distribution.xml +++ b/exchangis-plugins/exchangis-appconn/src/main/assembly/distribution.xml @@ -72,6 +72,7 @@ ${basedir}/src/main/resources + datax.icon sqoop.icon 0777 diff --git a/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql b/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql index 55105578c..4e7e3d3fd 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql +++ b/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql @@ -3,8 +3,8 @@ -- 删除exchangis关联的数据 -- delete from `dss_appconn_instance` where `appconn_id` in (select `id` from `dss_appconn` where `appconn_name` = 'exchangis'); delete from `dss_workspace_menu_appconn` where `appconn_id` in (select `id` from `dss_appconn` where `appconn_name` = 'exchangis'); - delete from `dss_appconn` where `appconn_name`='exchangis'; + INSERT INTO `dss_appconn` (`appconn_name`, `is_user_need_init`, `level`, `if_iframe`, `is_external`, `reference`, `class_name`, `appconn_class_path`, `resource`) VALUES ('exchangis', 0, 1, 1, 1, NULL, 'com.webank.wedatasphere.exchangis.dss.appconn.ExchangisAppConn', 'DSS_INSTALL_HOME_VAL/dss-appconns/exchangis', ''); select @dss_appconn_exchangis_id:=id from `dss_appconn` where `appconn_name` = "exchangis"; @@ -18,7 +18,7 @@ INSERT INTO `dss_workspace_menu_appconn` (`appconn_id`, `menu_id`, `title_en`, ` VALUES(@dss_appconn_exchangis_id,@exchangis_menuId,'Exchangis','Exchangis','Exchangis','' ,'exchangis, statement','数据交换,数据源','1','enter Exchangis','进入Exchangis','user manual','用户手册','http://APPCONN_INSTALL_IP:APPCONN_INSTALL_PORT/#/projectManage','shujukeshihua-logo',NULL,NULL,NULL,NULL,NULL,'shujukeshihua-icon'); --- Sqoop节点安装 +-- 卸载节点 select @old_dss_exchangis_sqoopId:=id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.sqoop'; select @old_dss_exchangis_dataxId:=id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.datax'; delete from `dss_workflow_node` where `node_type` like '%exchangis%'; From c0a84979b6a862874c22414c7986be249c751d73 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Tue, 14 May 2024 22:43:06 +0800 Subject: [PATCH 06/60] Add datasource type of starrocks --- db/1.1.3/exchangis_dml.sql | 3 ++ db/exchangis_dml.sql | 2 +- .../core/domain/DataSourceType.java | 4 +- .../pom.xml | 54 +++++++++++++++++++ .../ExchangisStarRocksDataSource.java | 49 +++++++++++++++++ exchangis-datasource/pom.xml | 1 + .../engine/domain/EngineResource.java | 14 +++++ 7 files changed, 125 insertions(+), 2 deletions(-) create mode 100644 db/1.1.3/exchangis_dml.sql create mode 100644 exchangis-datasource/extension-datasources/exchangis-datasource-ext-starrocks/pom.xml create mode 100644 exchangis-datasource/extension-datasources/exchangis-datasource-ext-starrocks/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/starrocks/ExchangisStarRocksDataSource.java diff --git a/db/1.1.3/exchangis_dml.sql b/db/1.1.3/exchangis_dml.sql new file mode 100644 index 000000000..e8ca76494 --- /dev/null +++ b/db/1.1.3/exchangis_dml.sql @@ -0,0 +1,3 @@ +UPDATE exchangis_engine_settings SET engine_direction= +'mysql->hive,hive->mysql,mysql->oracle,oracle->mysql,oracle->hive,hive->oracle,mongodb->hive,hive->mongodb,mysql->elasticsearch,oracle->elasticsearch,mongodb->elasticsearch,mysql->mongodb,mongodb->mysql,oracle->mongodb,mongodb->oracle,mysql->starrocks' +WHERE engine_name='datax'; \ No newline at end of file diff --git a/db/exchangis_dml.sql b/db/exchangis_dml.sql index bcd727c25..2376b9c53 100644 --- a/db/exchangis_dml.sql +++ b/db/exchangis_dml.sql @@ -64,7 +64,7 @@ INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_directio -- engine_settings records INSERT INTO `exchangis_engine_settings` (id, engine_name, engine_desc, engine_settings_value, engine_direction, res_loader_class, res_uploader_class, modify_time) VALUES -(1, 'datax', 'datax sync engine', '{}', 'mysql->hive,hive->mysql,mysql->oracle,oracle->mysql,oracle->hive,hive->oracle,mongodb->hive,hive->mongodb,mysql->elasticsearch,oracle->elasticsearch,mongodb->elasticsearch,mysql->mongodb,mongodb->mysql,oracle->mongodb,mongodb->oracle', 'com.webank.wedatasphere.exchangis.engine.resource.loader.datax.DataxEngineResourceLoader', NULL, NULL), +(1, 'datax', 'datax sync engine', '{}', 'mysql->hive,hive->mysql,mysql->oracle,oracle->mysql,oracle->hive,hive->oracle,mongodb->hive,hive->mongodb,mysql->elasticsearch,oracle->elasticsearch,mongodb->elasticsearch,mysql->mongodb,mongodb->mysql,oracle->mongodb,mongodb->oracle,mysql->starrocks', 'com.webank.wedatasphere.exchangis.engine.resource.loader.datax.DataxEngineResourceLoader', NULL, NULL), (2, 'sqoop', 'hadoop tool', '{}', 'mysql->hive,hive->mysql', '', NULL, NULL); -- exchangis_job_transform_rule records diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/domain/DataSourceType.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/domain/DataSourceType.java index d0193c4a2..24155f89d 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/domain/DataSourceType.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/domain/DataSourceType.java @@ -12,7 +12,9 @@ public enum DataSourceType { SFTP("SFTP"), - ORACLE("ORACLE"); + ORACLE("ORACLE"), + + STARROCKS("STARROCKS"); public String name; diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-starrocks/pom.xml b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-starrocks/pom.xml new file mode 100644 index 000000000..ba062fcc9 --- /dev/null +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-starrocks/pom.xml @@ -0,0 +1,54 @@ + + + + exchangis-datasource + com.webank.wedatasphere.exchangis + ${revision} + ../../pom.xml + + 4.0.0 + + exchangis-datasource-ext-starrocks + + + 8 + 8 + + + + + com.webank.wedatasphere.exchangis + exchangis-dao + ${revision} + + + com.webank.wedatasphere.exchangis + exchangis-datasource-linkis + ${revision} + + + com.webank.wedatasphere.exchangis + exchangis-datasource-core + ${revision} + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + \ No newline at end of file diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-starrocks/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/starrocks/ExchangisStarRocksDataSource.java b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-starrocks/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/starrocks/ExchangisStarRocksDataSource.java new file mode 100644 index 000000000..935aed652 --- /dev/null +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-starrocks/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/starrocks/ExchangisStarRocksDataSource.java @@ -0,0 +1,49 @@ +package com.webank.wedatasphere.exchangis.extension.datasource.starrocks; + +import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; +import com.webank.wedatasphere.exchangis.datasource.linkis.ExchangisBatchDataSource; +import com.webank.wedatasphere.exchangis.datasource.core.domain.DataSourceType; + +import java.util.List; + +/** + * @author jefftlin + * @date 2024/5/14 + */ +public class ExchangisStarRocksDataSource extends ExchangisBatchDataSource { + + @Override + public String name() { + return DataSourceType.STARROCKS.name; + } + + @Override + public String description() { + return "This is StarRocks DataSource"; + } + + @Override + public String option() { + return "StarRocks数据库"; + } + + @Override + public String classifier() { + return null; + } + + @Override + public String structClassifier() { + return null; + } + + @Override + public String icon() { + return null; + } + + @Override + public List getDataSourceParamConfigs() { + return super.getDataSourceParamConfigs(DataSourceType.STARROCKS.name); + } +} diff --git a/exchangis-datasource/pom.xml b/exchangis-datasource/pom.xml index 5cb7fa7be..817f59268 100644 --- a/exchangis-datasource/pom.xml +++ b/exchangis-datasource/pom.xml @@ -25,6 +25,7 @@ extension-datasources/exchangis-datasource-ext-elasticsearch extension-datasources/exchangis-datasource-ext-mongodb extension-datasources/exchangis-datasource-ext-oracle + extension-datasources/exchangis-datasource-ext-starrocks exchangis-datasource-server diff --git a/exchangis-engines/exchangis-engine-common/src/main/java/com/webank/wedatasphere/exchangis/engine/domain/EngineResource.java b/exchangis-engines/exchangis-engine-common/src/main/java/com/webank/wedatasphere/exchangis/engine/domain/EngineResource.java index ba74d2596..632614c91 100644 --- a/exchangis-engines/exchangis-engine-common/src/main/java/com/webank/wedatasphere/exchangis/engine/domain/EngineResource.java +++ b/exchangis-engines/exchangis-engine-common/src/main/java/com/webank/wedatasphere/exchangis/engine/domain/EngineResource.java @@ -124,4 +124,18 @@ public String getPath() { public void setPath(String path) { this.path = path; } + + @Override + public String toString() { + return "EngineResource{" + + "engineType='" + engineType + '\'' + + ", id='" + id + '\'' + + ", name='" + name + '\'' + + ", type='" + type + '\'' + + ", path='" + path + '\'' + + ", createTime=" + createTime + + ", modifyTime=" + modifyTime + + ", creator='" + creator + '\'' + + '}'; + } } From b75c828ca39861a0d6b9d9b7e114f5e79e0621d9 Mon Sep 17 00:00:00 2001 From: davidhua Date: Thu, 16 May 2024 14:46:25 +0800 Subject: [PATCH 07/60] Enable to fetch remote log. --- .../config/dss-exchangis-server.properties | 3 + .../exchangis/common/EnvironmentUtils.java | 65 ++++ .../exchangis/job/log/LogQuery.java | 14 + .../entity/LaunchedExchangisJobEntity.java | 6 +- .../linkis/LinkisExchangisTaskLauncher.java | 5 +- exchangis-job/exchangis-job-server/pom.xml | 6 + .../ExchangisJobExecuteAutoConfiguration.java | 4 +- .../transform/TransformExchangisJob.java | 22 +- .../tasks/MetricUpdateSchedulerTask.java | 9 + .../tasks/StatusUpdateSchedulerTask.java | 9 + .../job/server/log/rpc/FetchLogRequest.java | 31 ++ .../job/server/log/rpc/FetchLogResponse.java | 20 ++ .../job/server/log/rpc/SendLogRequest.java | 57 ++++ .../log/service/AbstractJobLogService.java | 159 ++++++++++ .../log/service/LocalSimpleJobLogService.java | 282 ------------------ .../server/log/service/RpcJobLogService.java | 280 +++++++++++++++++ .../render/transform/TransformRequestVo.java | 25 ++ .../job/server/log/cache/JobLogCache.scala | 4 +- pom.xml | 6 + 19 files changed, 714 insertions(+), 293 deletions(-) create mode 100644 exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/rpc/FetchLogRequest.java create mode 100644 exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/rpc/FetchLogResponse.java create mode 100644 exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/rpc/SendLogRequest.java create mode 100644 exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/AbstractJobLogService.java delete mode 100644 exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/LocalSimpleJobLogService.java create mode 100644 exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/RpcJobLogService.java diff --git a/assembly-package/config/dss-exchangis-server.properties b/assembly-package/config/dss-exchangis-server.properties index a8aa9a830..8f3b64bd3 100644 --- a/assembly-package/config/dss-exchangis-server.properties +++ b/assembly-package/config/dss-exchangis-server.properties @@ -26,6 +26,9 @@ wds.linkis.gateway.url=http://{LINKIS_IP}:{LINKIS_PORT}/ wds.linkis.log.clear=true wds.linkis.server.version=v1 +# server rpc +wds.linkis.ms.service.scan.package=com.webank.wedatasphere.exchangis + # datasource client wds.exchangis.datasource.client.serverurl=http://{LINKIS_IP}:{LINKIS_PORT}/ wds.exchangis.datasource.client.authtoken.key=EXCHANGIS-AUTH diff --git a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/EnvironmentUtils.java b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/EnvironmentUtils.java index eaf5d6c38..ac6e77198 100644 --- a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/EnvironmentUtils.java +++ b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/EnvironmentUtils.java @@ -1,6 +1,13 @@ package com.webank.wedatasphere.exchangis.common; +import org.apache.commons.lang3.StringUtils; +import org.apache.linkis.DataWorkCloudApplication; import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.conf.Configuration; +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.server.utils.LinkisMainHelper; +import org.springframework.context.ApplicationContext; + /** * Environment utils @@ -9,6 +16,8 @@ public class EnvironmentUtils { private static final CommonVars JVM_USER = CommonVars.apply("wds.exchangis.env.jvm.user", System.getProperty("user.name", "hadoop")); + private static final CommonVars SERVER_NAME = CommonVars.apply(LinkisMainHelper.SERVER_NAME_KEY(), "exchangis"); + /** * Jvm user * @return user name @@ -16,4 +25,60 @@ public class EnvironmentUtils { public static String getJvmUser(){ return JVM_USER.getValue(); } + + /** + * Server name + * @return name + */ + public static String getServerName(){ + return SERVER_NAME.getValue(); + } + + /** + * Get server address + * @return address + */ + public static String getServerAddress(){ + ApplicationContext context = DataWorkCloudApplication.getApplicationContext(); + String hostname; + if (Configuration.PREFER_IP_ADDRESS()) { + hostname = context + .getEnvironment().getProperty("spring.cloud.client.ip-address"); + } else { + hostname = context.getEnvironment().getProperty("eureka.instance.hostname", ""); + if (StringUtils.isBlank(hostname)) { + hostname = Utils.getComputerName(); + } + } + String serverPort = context.getEnvironment().getProperty("server.port"); + return hostname + (StringUtils.isNotBlank(serverPort) ? ":" + serverPort : ""); + } + /** + * Get server host name + * @return hostname + */ + public static String getServerHost(){ + ApplicationContext context = DataWorkCloudApplication.getApplicationContext(); + if (Configuration.PREFER_IP_ADDRESS()) { + return context + .getEnvironment().getProperty("spring.cloud.client.ip-address"); + } else { + String hostname = context.getEnvironment().getProperty("eureka.instance.hostname", ""); + if (StringUtils.isBlank(hostname)) { + return Utils.getComputerName(); + } + return hostname; + } + } + + /** + * Get server port + * @return port number + */ + public static Integer getServerPort(){ + String serverPort = DataWorkCloudApplication.getApplicationContext() + .getEnvironment().getProperty("server.port"); + return StringUtils.isNotBlank(serverPort) ? Integer.parseInt(serverPort) : null; + } + } diff --git a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/log/LogQuery.java b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/log/LogQuery.java index 0ab7780b9..de19116c9 100644 --- a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/log/LogQuery.java +++ b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/log/LogQuery.java @@ -21,6 +21,11 @@ public class LogQuery { private Integer lastRows; + /** + * Reverse the reader + */ + private boolean enableTail; + public LogQuery(){ } @@ -88,4 +93,13 @@ public List getOnlyKeywordsList(){ public void setOnlyKeywords(String onlyKeywords) { this.onlyKeywords = onlyKeywords; } + + + public boolean isEnableTail() { + return enableTail; + } + + public void setEnableTail(boolean enableTail) { + this.enableTail = enableTail; + } } diff --git a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/entity/LaunchedExchangisJobEntity.java b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/entity/LaunchedExchangisJobEntity.java index 59b991cd5..bfc9d2a8e 100644 --- a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/entity/LaunchedExchangisJobEntity.java +++ b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/entity/LaunchedExchangisJobEntity.java @@ -1,5 +1,6 @@ package com.webank.wedatasphere.exchangis.job.launcher.entity; +import com.webank.wedatasphere.exchangis.common.EnvironmentUtils; import com.webank.wedatasphere.exchangis.job.domain.ExchangisJobEntity; import com.webank.wedatasphere.exchangis.job.launcher.domain.LaunchableExchangisJob; @@ -8,6 +9,7 @@ import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; +import java.util.Optional; /** * Entity to persist the launched job @@ -58,8 +60,10 @@ public LaunchedExchangisJobEntity(LaunchableExchangisJob job){ this.lastUpdateTime = job.getLastUpdateTime(); this.jobExecutionId = job.getJobExecutionId(); SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd"); - this.logPath = this.executeUser + IOUtils.DIR_SEPARATOR_UNIX + + String logPath = this.executeUser + IOUtils.DIR_SEPARATOR_UNIX + simpleDateFormat.format(new Date()) + IOUtils.DIR_SEPARATOR_UNIX + this.jobExecutionId; + logPath = EnvironmentUtils.getServerAddress() + "@" + logPath; + this.logPath = logPath; } public String getJobExecutionId() { return jobExecutionId; diff --git a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisExchangisTaskLauncher.java b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisExchangisTaskLauncher.java index d5ef7e8e0..ca505987d 100644 --- a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisExchangisTaskLauncher.java +++ b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisExchangisTaskLauncher.java @@ -1,6 +1,6 @@ package com.webank.wedatasphere.exchangis.job.launcher.linkis; -import com.webank.wedatasphere.exchangis.common.linkis.ClientConfiguration; +import com.webank.wedatasphere.exchangis.common.linkis.client.ClientConfiguration; import com.webank.wedatasphere.exchangis.job.enums.EngineTypeEnum; import com.webank.wedatasphere.exchangis.job.launcher.AccessibleLauncherTask; import com.webank.wedatasphere.exchangis.job.launcher.exception.ExchangisTaskLaunchException; @@ -10,16 +10,13 @@ import com.webank.wedatasphere.exchangis.job.launcher.domain.LaunchedExchangisTask; import org.apache.commons.lang.StringUtils; import org.apache.linkis.common.conf.Configuration; -import org.apache.linkis.common.conf.Configuration$; import org.apache.linkis.common.exception.LinkisRetryException; import org.apache.linkis.common.utils.DefaultRetryHandler; import org.apache.linkis.common.utils.RetryHandler; -import org.apache.linkis.computation.client.LinkisJobClient; import org.apache.linkis.computation.client.LinkisJobClient$; import org.apache.linkis.httpclient.config.ClientConfig; import org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy; import org.apache.linkis.httpclient.dws.config.DWSClientConfig; -import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder; import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder$; import java.util.*; diff --git a/exchangis-job/exchangis-job-server/pom.xml b/exchangis-job/exchangis-job-server/pom.xml index df144b413..c4f680478 100644 --- a/exchangis-job/exchangis-job-server/pom.xml +++ b/exchangis-job/exchangis-job-server/pom.xml @@ -49,6 +49,12 @@ mysql-connector-java 5.1.49 + + + org.apache.linkis + linkis-rpc + ${linkis.version} + diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/ExchangisJobExecuteAutoConfiguration.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/ExchangisJobExecuteAutoConfiguration.java index 0c3391d94..3d2139c68 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/ExchangisJobExecuteAutoConfiguration.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/ExchangisJobExecuteAutoConfiguration.java @@ -20,7 +20,7 @@ import com.webank.wedatasphere.exchangis.job.server.execution.subscriber.TaskObserver; import com.webank.wedatasphere.exchangis.job.server.log.DefaultRpcJobLogger; import com.webank.wedatasphere.exchangis.job.server.log.JobLogService; -import com.webank.wedatasphere.exchangis.job.server.log.service.LocalSimpleJobLogService; +import com.webank.wedatasphere.exchangis.job.server.log.service.RpcJobLogService; import com.webank.wedatasphere.exchangis.job.server.utils.SpringContextHolder; import org.apache.linkis.scheduler.Scheduler; import org.apache.linkis.scheduler.executer.ExecutorManager; @@ -50,7 +50,7 @@ public JobLogListener logListener(){ @Bean @ConditionalOnMissingBean(JobLogService.class) public JobLogService jobLogService(){ - return new LocalSimpleJobLogService(); + return new RpcJobLogService(); } /** diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/TransformExchangisJob.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/TransformExchangisJob.java index f2fb0ce24..319f0c7c6 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/TransformExchangisJob.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/TransformExchangisJob.java @@ -168,7 +168,15 @@ private void convertContentToParams(ExchangisJobInfoContent content){ ExchangisJobParamsContent.ExchangisJobParamsItem::getConfigValue)); }); if(Objects.nonNull(paramSet)) { - this.sourceType = resolveDataSourceId(content.getDataSources().getSourceId(), paramSet); + String sourceId = content.getDataSources().getSourceId(); + if (StringUtils.isNotBlank(sourceId)){ + this.sourceType = resolveDataSourceId(content.getDataSources().getSourceId(), paramSet); + } else { + Optional.ofNullable(content.getDataSources().getSource().getType()).ifPresent(type -> { + this.sourceType = type.name; + }); + } + } } @@ -183,7 +191,14 @@ private void convertContentToParams(ExchangisJobInfoContent content){ ExchangisJobParamsContent.ExchangisJobParamsItem::getConfigValue)); }); if(Objects.nonNull(paramSet)) { - this.sinkType = resolveDataSourceId(content.getDataSources().getSinkId(), paramSet); + String sinkId = content.getDataSources().getSinkId(); + if (StringUtils.isNotBlank(sinkId)){ + this.sinkType = resolveDataSourceId(content.getDataSources().getSinkId(), paramSet); + } else { + Optional.ofNullable(content.getDataSources().getSink().getType()).ifPresent(type -> { + this.sinkType = type.name; + }); + } } } } @@ -230,6 +245,9 @@ private String resolveDataSourceId(String dataSourceId, JobParamSet paramSet){ return null; } + private String resolveDataSource(ExchangisJobDataSourcesContent.ExchangisJobDataSource dataSource){ + return null; + } /** * * @param items diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/MetricUpdateSchedulerTask.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/MetricUpdateSchedulerTask.java index 4fded220d..941d36461 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/MetricUpdateSchedulerTask.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/MetricUpdateSchedulerTask.java @@ -31,6 +31,15 @@ public MetricUpdateSchedulerTask(TaskManager taskManager) this.taskManager = taskManager; } + /** + * High priority to get schedule resource + * @return priority + */ + @Override + public int getPriority() { + return 2; + } + @Override protected void onPoll(LaunchedExchangisTask launchedExchangisTask) throws ExchangisSchedulerException, ExchangisSchedulerRetryException { LOG.trace("Metrics update task: [{}] in scheduler: [{}]", launchedExchangisTask.getTaskId(), getName()); diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/StatusUpdateSchedulerTask.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/StatusUpdateSchedulerTask.java index 3bfc64374..7551ec207 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/StatusUpdateSchedulerTask.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/StatusUpdateSchedulerTask.java @@ -26,6 +26,15 @@ public class StatusUpdateSchedulerTask extends AbstractLoadBalanceSchedulerTask< private TaskManager taskManager; + /** + * High priority to get schedule resource + * @return priority + */ + @Override + public int getPriority() { + return 2; + } + public StatusUpdateSchedulerTask(TaskManager taskManager){ this.taskManager = taskManager; } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/rpc/FetchLogRequest.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/rpc/FetchLogRequest.java new file mode 100644 index 000000000..18d9a5f8b --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/rpc/FetchLogRequest.java @@ -0,0 +1,31 @@ +package com.webank.wedatasphere.exchangis.job.server.log.rpc; + +import com.webank.wedatasphere.exchangis.job.log.LogQuery; +import org.apache.linkis.protocol.message.RequestProtocol; + +/** + * Fetch log request + */ +public class FetchLogRequest extends LogQuery implements RequestProtocol { + + /** + * Log path + */ + private String logPath; + + public FetchLogRequest(LogQuery logQuery, String logPath){ + super(logQuery.getFromLine(), logQuery.getPageSize(), + logQuery.getIgnoreKeywords(), logQuery.getOnlyKeywords(), + logQuery.getLastRows()); + setEnableTail(logQuery.isEnableTail()); + this.logPath = logPath; + } + + public String getLogPath() { + return logPath; + } + + public void setLogPath(String logPath) { + this.logPath = logPath; + } +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/rpc/FetchLogResponse.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/rpc/FetchLogResponse.java new file mode 100644 index 000000000..b9dd399b4 --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/rpc/FetchLogResponse.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.exchangis.job.server.log.rpc; + +import com.webank.wedatasphere.exchangis.job.log.LogResult; +import org.apache.linkis.protocol.message.RequestProtocol; + +import java.util.List; + +/** + * Extend log result + */ +public class FetchLogResponse extends LogResult implements RequestProtocol { + + public FetchLogResponse(LogResult logResult){ + super(logResult.getEndLine(), logResult.isEnd(), logResult.getLogs()); + } + + public FetchLogResponse(int endLine, boolean isEnd, List logs) { + super(endLine, isEnd, logs); + } +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/rpc/SendLogRequest.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/rpc/SendLogRequest.java new file mode 100644 index 000000000..8310dfef0 --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/rpc/SendLogRequest.java @@ -0,0 +1,57 @@ +package com.webank.wedatasphere.exchangis.job.server.log.rpc; + +import org.apache.linkis.protocol.message.RequestProtocol; + +import java.util.ArrayList; +import java.util.List; + +/** + * Send log request + */ +public class SendLogRequest implements RequestProtocol { + /** + * Exec id + */ + private String jobExecId; + + /** + * Is reached the end of log + */ + private boolean isEnd; + /** + * Log lines + */ + private List logLines = new ArrayList<>(); + + public SendLogRequest(String jobExecId, + boolean isEnd, + List logLines){ + this.jobExecId = jobExecId; + this.isEnd = isEnd; + this.logLines = logLines; + } + + public String getJobExecId() { + return jobExecId; + } + + public void setJobExecId(String jobExecId) { + this.jobExecId = jobExecId; + } + + public List getLogLines() { + return logLines; + } + + public void setLogLines(List logLines) { + this.logLines = logLines; + } + + public boolean isEnd() { + return isEnd; + } + + public void setEnd(boolean end) { + isEnd = end; + } +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/AbstractJobLogService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/AbstractJobLogService.java new file mode 100644 index 000000000..f75591a92 --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/AbstractJobLogService.java @@ -0,0 +1,159 @@ +package com.webank.wedatasphere.exchangis.job.server.log.service; + +import com.google.common.cache.*; +import com.webank.wedatasphere.exchangis.job.launcher.entity.LaunchedExchangisJobEntity; +import com.webank.wedatasphere.exchangis.job.log.LogQuery; +import com.webank.wedatasphere.exchangis.job.log.LogResult; +import com.webank.wedatasphere.exchangis.job.server.mapper.LaunchedJobDao; +import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; +import com.webank.wedatasphere.exchangis.job.server.execution.scheduler.AbstractExchangisSchedulerTask; +import com.webank.wedatasphere.exchangis.job.server.log.JobLogService; +import com.webank.wedatasphere.exchangis.job.server.log.cache.AbstractJobLogCache; +import com.webank.wedatasphere.exchangis.job.server.log.cache.JobLogCache; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.scheduler.Scheduler; +import org.apache.linkis.scheduler.queue.JobInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import javax.annotation.Resource; +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.webank.wedatasphere.exchangis.job.exception.ExchangisJobExceptionCode.LOG_OP_ERROR; + +/** + * Abstract Job log service + */ +public abstract class AbstractJobLogService implements JobLogService { + + private static final Logger LOG = LoggerFactory.getLogger(AbstractJobLogService.class); + + protected Cache> cacheHolder; + + private AbstractExchangisSchedulerTask cleaner; + + private volatile boolean cleanerOn; + + protected static class Constraints{ + public static final CommonVars LOG_LOCAL_PATH = CommonVars.apply("wds.exchangis.job.log.local.path", "/data/bdp/dss/exchangis/main/logs"); + + public static final CommonVars lOG_CACHE_SIZE = CommonVars.apply("wds.exchangis.job.log.cache.size", 15); + + public static final CommonVars LOG_CACHE_EXPIRE_TIME_IN_SECONDS = CommonVars.apply("wds.exchangis.job.log.cache.expire.time-in-seconds", 5); + + public static final CommonVars LOG_MULTILINE_PATTERN = CommonVars.apply("wds.exchangis.log.multiline.pattern", "^\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}\\.\\d{3}"); + } + + @Resource + protected Scheduler scheduler; + + @Resource + private LaunchedJobDao launchedJobDao; + @PostConstruct + public void init(){ + cleanerOn = true; + cacheHolder = CacheBuilder.newBuilder().maximumSize(Constraints.lOG_CACHE_SIZE.getValue()) + .expireAfterAccess(Constraints.LOG_CACHE_EXPIRE_TIME_IN_SECONDS.getValue(), TimeUnit.SECONDS) + .removalListener((RemovalListener>) removalNotification -> { + // Flush for expired + if (removalNotification.getCause() == RemovalCause.EXPIRED){ + removalNotification.getValue().flushCache(true); + } + }) + .build(); + cleaner = new AbstractExchangisSchedulerTask("Job-Log-Cache-Cleaner") { + @Override + public String getTenancy() { + return "log"; + } + + @Override + public String getName() { + return getId(); + } + + @Override + public JobInfo getJobInfo() { + return null; + } + + @Override + protected void schedule() { + while(cleanerOn){ + try { + Thread.sleep(Constraints.LOG_CACHE_EXPIRE_TIME_IN_SECONDS.getValue()); + //Just invoke the auto cleaner + cacheHolder.get("log", () -> null); + } catch (Exception e){ + //Ignore + } + } + } + }; + scheduler.submit(cleaner); + } + + @PreDestroy + public void destroy(){ + this.cleanerOn = false; + if (Objects.nonNull(this.cleaner.future())){ + this.cleaner.future().cancel(true); + } + } + + @Override + public LogResult logsFromPage( String jobExecId, LogQuery logQuery) { + LaunchedExchangisJobEntity launchedExchangisJob = launchedJobDao.searchLogPathInfo(jobExecId); + return logsFromPageAndPath(launchedExchangisJob.getLogPath(), logQuery); + } + + @Override + public void appendLog(String tenancy, String jobExecId, List logs) { + appendLog(jobExecId, logs); + } + + @Override + public void appendLog(String jobExecId, List logs) { + JobLogCache cache = getOrCreateLogCache(jobExecId); + logs.forEach(cache ::cacheLog); + } + + + @Override + public JobLogCache getOrCreateLogCache(String jobExecId){ + try { + return cacheHolder.get(jobExecId, () -> { + LaunchedExchangisJobEntity launchedExchangisJob = launchedJobDao.searchLogPathInfo(jobExecId); + if (Objects.nonNull(launchedExchangisJob)) { + + } + return null; + }); + } catch (ExecutionException e) { + throw new ExchangisJobServerException.Runtime(LOG_OP_ERROR.getCode(),"Fail to create the job log cache of [" + jobExecId +"]", e); + } + } + + /** + * Load job log cache + * @param launchedExchangisJob job + * @return log cache + */ + protected abstract AbstractJobLogCache loadJobLogCache(String jobExcId, LaunchedExchangisJobEntity launchedExchangisJob) + throws Exception; +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/LocalSimpleJobLogService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/LocalSimpleJobLogService.java deleted file mode 100644 index 1eea657ce..000000000 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/LocalSimpleJobLogService.java +++ /dev/null @@ -1,282 +0,0 @@ -package com.webank.wedatasphere.exchangis.job.server.log.service; - -import com.google.common.cache.*; -import com.webank.wedatasphere.exchangis.job.launcher.entity.LaunchedExchangisJobEntity; -import com.webank.wedatasphere.exchangis.job.log.LogQuery; -import com.webank.wedatasphere.exchangis.job.log.LogResult; -import com.webank.wedatasphere.exchangis.job.server.mapper.LaunchedJobDao; -import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; -import com.webank.wedatasphere.exchangis.job.server.execution.scheduler.AbstractExchangisSchedulerTask; -import com.webank.wedatasphere.exchangis.job.server.log.JobLogService; -import com.webank.wedatasphere.exchangis.job.server.log.cache.AbstractJobLogCache; -import com.webank.wedatasphere.exchangis.job.server.log.cache.JobLogCache; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.linkis.common.conf.CommonVars; -import org.apache.linkis.common.utils.Utils; -import org.apache.linkis.scheduler.Scheduler; -import org.apache.linkis.scheduler.queue.JobInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import javax.annotation.Resource; -import java.io.File; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.util.*; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import static com.webank.wedatasphere.exchangis.job.exception.ExchangisJobExceptionCode.LOG_OP_ERROR; - -/** - * Just store the log into the local - */ -public class LocalSimpleJobLogService implements JobLogService { - - private static final Logger LOG = LoggerFactory.getLogger(LocalSimpleJobLogService.class); - - private Cache> cacheHolder; - - private AbstractExchangisSchedulerTask cleaner; - - private volatile boolean cleanerOn; - - private static class Constraints{ - public static final CommonVars LOG_LOCAL_PATH = CommonVars.apply("wds.exchangis.job.log.local.path", "/data/bdp/dss/exchangis/main/logs"); - - public static final CommonVars lOG_CACHE_SIZE = CommonVars.apply("wds.exchangis.job.log.cache.size", 15); - - public static final CommonVars LOG_CACHE_EXPIRE_TIME_IN_SECONDS = CommonVars.apply("wds.exchangis.job.log.cache.expire.time-in-seconds", 5); - - public static final CommonVars LOG_MULTILINE_PATTERN = CommonVars.apply("wds.exchangis.log.multiline.pattern", "^\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}\\.\\d{3}"); - } - - @Resource - private Scheduler scheduler; - - @Resource - private LaunchedJobDao launchedJobDao; - @PostConstruct - public void init(){ - cleanerOn = true; - cacheHolder = CacheBuilder.newBuilder().maximumSize(Constraints.lOG_CACHE_SIZE.getValue()) - .expireAfterAccess(Constraints.LOG_CACHE_EXPIRE_TIME_IN_SECONDS.getValue(), TimeUnit.SECONDS) - .removalListener((RemovalListener>) removalNotification -> { - // Flush for expired - if (removalNotification.getCause() == RemovalCause.EXPIRED){ - removalNotification.getValue().flushCache(true); - } - }) - .build(); - cleaner = new AbstractExchangisSchedulerTask("Job-Log-Cache-Cleaner") { - @Override - public String getTenancy() { - return "log"; - } - - @Override - public String getName() { - return getId(); - } - - @Override - public JobInfo getJobInfo() { - return null; - } - - @Override - protected void schedule() { - while(cleanerOn){ - try { - Thread.sleep(Constraints.LOG_CACHE_EXPIRE_TIME_IN_SECONDS.getValue()); - //Just invoke the auto cleaner - cacheHolder.get("log", () -> null); - } catch (Exception e){ - //Ignore - } - } - } - }; - scheduler.submit(cleaner); - } - - @PreDestroy - public void destroy(){ - this.cleanerOn = false; - if (Objects.nonNull(this.cleaner.future())){ - this.cleaner.future().cancel(true); - } - } - - @Override - public LogResult logsFromPage( String jobExecId, LogQuery logQuery) { - LaunchedExchangisJobEntity launchedExchangisJob = launchedJobDao.searchLogPathInfo(jobExecId); - return logsFromPageAndPath(launchedExchangisJob.getLogPath(), logQuery); - } - - @Override - public LogResult logsFromPageAndPath(String logPath, LogQuery logQuery) { - String fullPath = Constraints.LOG_LOCAL_PATH.getValue() + IOUtils.DIR_SEPARATOR_UNIX + logPath; - LogResult result = new LogResult(0, false, Collections.emptyList()); - if (!new File(fullPath).exists()){ - return result; - } - if (logQuery.getLastRows() != null && logQuery.getLastRows() > 0){ - return getLastRows(fullPath, logQuery.getLastRows()); - } - RandomAccessFile logReader = null; - try { - logReader = new RandomAccessFile(fullPath, "rw"); - String patternValue = Constraints.LOG_MULTILINE_PATTERN.getValue(); - Pattern linePattern = StringUtils.isNotBlank(patternValue)? Pattern.compile(patternValue) : null; - int readLine = 0; - int lineNum = 0; - int skippedLine = 0; - int ignoreLine = 0; - int pageSize = logQuery.getPageSize(); - int fromLine = logQuery.getFromLine(); - List ignoreKeywords = logQuery.getIgnoreKeywordsList(); - List onlyKeywords = logQuery.getOnlyKeywordsList(); - boolean rowIgnore = false; - String line = logReader.readLine(); - List logs = new ArrayList<>(); - while (readLine < pageSize && line != null){ - lineNum += 1; - if (skippedLine < fromLine - 1){ - skippedLine += 1; - } else { - if (rowIgnore) { - if (Objects.nonNull(linePattern)){ - Matcher matcher = linePattern.matcher(line); - if (matcher.find()){ - ignoreLine = 0; - rowIgnore = !isIncludeLine(line, onlyKeywords, ignoreKeywords); - } else { - ignoreLine += 1; - // TODO limit the value of ignoreLine - } - }else{ - rowIgnore = !isIncludeLine(line, onlyKeywords, ignoreKeywords); - } - }else { - rowIgnore = !isIncludeLine(line, onlyKeywords, ignoreKeywords); - } - if (!rowIgnore) { - if (line.contains("password")) { - LOG.info("have error information"); - } - if (!line.contains("password")) { - logs.add(new String(line.getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8)); - } - readLine += 1; - } - } - line = logReader.readLine(); - } - result = new LogResult(lineNum, false, logs); - } catch (IOException e) { - throw new ExchangisJobServerException.Runtime(LOG_OP_ERROR.getCode(),"Unable to query the logs from path: [" + logPath + "]", e); - } finally { - if (Objects.nonNull(logReader)) { - try { - logReader.close(); - } catch (IOException e) { - //Ignore - } - } - } - return result; - } - - @Override - public void appendLog(String tenancy, String jobExecId, List logs) { - appendLog(jobExecId, logs); - } - - @Override - public void appendLog(String jobExecId, List logs) { - JobLogCache cache = getOrCreateLogCache(jobExecId); - logs.forEach(cache ::cacheLog); - } - - - private boolean isIncludeLine(String line, List onlyKeywordList, List ignoreKeywordList){ - boolean accept = ignoreKeywordList.isEmpty() || ignoreKeywordList.stream().noneMatch(line::contains); - if (accept){ - accept = onlyKeywordList.isEmpty() || onlyKeywordList.stream().anyMatch(line::contains); - } - return accept; - } - - /** - * Get last rows - * @param fullPath full path - * @param lastRows last rows - * @return - */ - private LogResult getLastRows(String fullPath, int lastRows){ - try { - List logs = Arrays.asList(Utils.exec(new String[]{"tail", "-n", lastRows + "", fullPath}, 5000L).split("\n")); - return new LogResult(0, true, logs); - }catch (Exception e){ - throw new ExchangisJobServerException.Runtime(LOG_OP_ERROR.getCode(), "Fail to get last rows from path: [" + fullPath + "]", e); - } - } - @Override - public JobLogCache getOrCreateLogCache(String jobExecId){ - try { - return cacheHolder.get(jobExecId, () -> { - LaunchedExchangisJobEntity launchedExchangisJob = launchedJobDao.searchLogPathInfo(jobExecId); - if (Objects.nonNull(launchedExchangisJob)) { - File logFile = new File(Constraints.LOG_LOCAL_PATH.getValue() + IOUtils.DIR_SEPARATOR_UNIX + - launchedExchangisJob.getLogPath()); - if (!logFile.exists()){ - // Write empty string to create new file - FileUtils.writeStringToFile(logFile, ""); - LOG.info("Create the new job log file: {}", logFile.getAbsolutePath()); - } - RandomAccessFile file = new RandomAccessFile(logFile, "rw"); - // Seek to the end of file - file.seek(file.length()); - return new AbstractJobLogCache(scheduler, 100, 2000) { - @Override - public synchronized void flushCache(boolean isEnd) { - // Store into local path - if (!cacheQueue().isEmpty()) { - try { - List logLines = new ArrayList<>(); - cacheQueue().drainTo(logLines); - for (Object line : logLines) { - file.write(String.valueOf(line).getBytes(Charset.defaultCharset())); - } - } catch (IOException ex) { - LOG.error("Fail to flush the log cache of [" + launchedExchangisJob.getJobExecutionId() + "]", ex); - } - } - if (isEnd) { - cacheHolder.invalidate(jobExecId); - try { - file.close(); - } catch (IOException e) { - //Ignore - } - } - } - }; - } - return null; - }); - } catch (ExecutionException e) { - throw new ExchangisJobServerException.Runtime(LOG_OP_ERROR.getCode(),"Fail to create the job log cache of [" + jobExecId +"]", e); - } - } - -} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/RpcJobLogService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/RpcJobLogService.java new file mode 100644 index 000000000..14b73d585 --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/RpcJobLogService.java @@ -0,0 +1,280 @@ +package com.webank.wedatasphere.exchangis.job.server.log.service; + +import com.webank.wedatasphere.exchangis.common.EnvironmentUtils; +import com.webank.wedatasphere.exchangis.job.launcher.entity.LaunchedExchangisJobEntity; +import com.webank.wedatasphere.exchangis.job.log.LogQuery; +import com.webank.wedatasphere.exchangis.job.log.LogResult; +import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; +import com.webank.wedatasphere.exchangis.job.server.log.cache.AbstractJobLogCache; +import com.webank.wedatasphere.exchangis.job.server.log.cache.JobLogCache; +import com.webank.wedatasphere.exchangis.job.server.log.rpc.FetchLogRequest; +import com.webank.wedatasphere.exchangis.job.server.log.rpc.FetchLogResponse; +import com.webank.wedatasphere.exchangis.job.server.log.rpc.SendLogRequest; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.input.ReversedLinesFileReader; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.common.ServiceInstance; +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.rpc.Sender; +import org.apache.linkis.rpc.message.annotation.Receiver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.function.Supplier; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.webank.wedatasphere.exchangis.job.exception.ExchangisJobExceptionCode.LOG_OP_ERROR; + +/** + * Rpc job log service + */ +public class RpcJobLogService extends AbstractJobLogService{ + + private static final Logger LOG = LoggerFactory.getLogger(RpcJobLogService.class); + + + @Receiver + public void appendLog(SendLogRequest sendLogRequest){ + String jobExecId = sendLogRequest.getJobExecId(); + List logLines = sendLogRequest.getLogLines(); + if (logLines.size() > 0) { + // Two level cache + JobLogCache cache = getOrCreateLogCache(jobExecId); + logLines.forEach(cache :: cacheLog); + if (sendLogRequest.isEnd()){ + cache.flushCache(true); + } + } else if (sendLogRequest.isEnd()){ + Optional.ofNullable(cacheHolder.getIfPresent(jobExecId)).ifPresent( cache -> { + cache.flushCache(true); + }); + } + } + + @Receiver + public FetchLogResponse logsFromPage(FetchLogRequest fetchLogRequest){ + return new FetchLogResponse( + logsFromPageAndPath(fetchLogRequest.getLogPath(), fetchLogRequest)); + } + @Override + protected AbstractJobLogCache loadJobLogCache(String jobExecId, + LaunchedExchangisJobEntity launchedExchangisJob) throws Exception{ + String logPath = launchedExchangisJob.getLogPath(); + int splitPos = logPath.indexOf("@"); + if (splitPos > 0){ + String logAddress = logPath.substring(0, splitPos); + if (!logAddress.equals(EnvironmentUtils.getServerAddress())){ + ServiceInstance instance = ServiceInstance.apply(EnvironmentUtils.getServerName(), logAddress); + return new AbstractJobLogCache(scheduler, 100, 2000) { + @Override + public void flushCache(boolean isEnd) { + // Send rpc + if (!cacheQueue().isEmpty()) { + try { + List logLines = new ArrayList<>(); + cacheQueue().drainTo(logLines); + Sender.getSender(instance).send(new SendLogRequest(jobExecId, isEnd, logLines)); + } catch (Exception ex) { + LOG.error("Fail to send the log cache of [" + launchedExchangisJob.getJobExecutionId() + + "] to remote rpc [" + logAddress + "]", ex); + } + } + if (isEnd) { + cacheHolder.invalidate(jobExecId); + } + } + }; + } + } + File logFile = new File(Constraints.LOG_LOCAL_PATH.getValue() + IOUtils.DIR_SEPARATOR_UNIX + + launchedExchangisJob.getLogPath()); + + if (!logFile.exists()){ + // Write empty string to create new file + FileUtils.writeStringToFile(logFile, ""); + LOG.info("Create the new job log file: {}", logFile.getAbsolutePath()); + } + RandomAccessFile file = new RandomAccessFile(logFile, "rw"); + // Seek to the end of file + file.seek(file.length()); + return new AbstractJobLogCache(scheduler, 100, 2000) { + @Override + public synchronized void flushCache(boolean isEnd) { + // Store into local path + if (!cacheQueue().isEmpty()) { + try { + List logLines = new ArrayList<>(); + cacheQueue().drainTo(logLines); + for (Object line : logLines) { + file.write(String.valueOf(line).getBytes(Charset.defaultCharset())); + } + } catch (IOException ex) { + LOG.error("Fail to flush the log cache of [" + launchedExchangisJob.getJobExecutionId() + "]", ex); + } + } + if (isEnd) { + cacheHolder.invalidate(jobExecId); + try { + file.close(); + } catch (IOException e) { + //Ignore + } + } + } + }; + } + + @Override + public LogResult logsFromPageAndPath(String logPath, LogQuery logQuery) { + int splitPos = logPath.indexOf("@"); + if (splitPos > 0) { + String logAddress = logPath.substring(0, splitPos); + if (!logAddress.equals(EnvironmentUtils.getServerAddress())) { + Object response; + try { + response = Sender.getSender(ServiceInstance.apply(EnvironmentUtils.getServerName(), logAddress)) + .ask(new FetchLogRequest(logQuery, logPath)); + } catch (Exception e){ + throw new ExchangisJobServerException.Runtime(LOG_OP_ERROR.getCode(), + "Remote exception in fetching log from: [" + logPath + "]", e); + } + if (response instanceof FetchLogResponse){ + return (LogResult) response; + } + throw new ExchangisJobServerException.Runtime(LOG_OP_ERROR.getCode(),"Unable to fetch log from: [" + logPath + + "], unknown request protocol: [" + response + "]", null); + } + } + String fullPath = Constraints.LOG_LOCAL_PATH.getValue() + IOUtils.DIR_SEPARATOR_UNIX + logPath; + LogResult result = new LogResult(0, false, Collections.emptyList()); + if (!new File(fullPath).exists()){ + return result; + } + if (logQuery.getLastRows() != null && logQuery.getLastRows() > 0){ + return getLastRows(fullPath, logQuery.getLastRows()); + } + RandomAccessFile logReader = null; + ReversedLinesFileReader reverseReader = null; + try { + String patternValue = Constraints.LOG_MULTILINE_PATTERN.getValue(); + Pattern linePattern = StringUtils.isNotBlank(patternValue)? Pattern.compile(patternValue) : null; + int readLine = 0; + int lineNum = 0; + int skippedLine = 0; + int ignoreLine = 0; + int pageSize = logQuery.getPageSize(); + int fromLine = logQuery.getFromLine(); + List ignoreKeywords = logQuery.getIgnoreKeywordsList(); + List onlyKeywords = logQuery.getOnlyKeywordsList(); + boolean rowIgnore = false; + Supplier lineSupplier = null; + if (logQuery.isEnableTail()){ + reverseReader = new ReversedLinesFileReader(new File(fullPath), Charset.defaultCharset()); + LOG.trace("Enable reverse read the log: {}, fromLine: {}, pageSize: {}", fullPath, fromLine, pageSize); + ReversedLinesFileReader finalReverseReader = reverseReader; + lineSupplier = () -> { + try { + return finalReverseReader.readLine(); + } catch (IOException e) { + throw new ExchangisJobServerException.Runtime(LOG_OP_ERROR.getCode(), e.getMessage(), e); + } + }; + } else { + logReader = new RandomAccessFile(fullPath, "rw"); + RandomAccessFile finalLogReader = logReader; + lineSupplier = () -> { + try { + return finalLogReader.readLine(); + } catch (IOException e) { + throw new ExchangisJobServerException.Runtime(LOG_OP_ERROR.getCode(), e.getMessage(), e); + } + }; + } + String line = lineSupplier.get(); + List logs = new ArrayList<>(); + while (readLine < pageSize && line != null){ + lineNum += 1; + if (skippedLine < fromLine - 1){ + skippedLine += 1; + } else { + if (rowIgnore) { + if (Objects.nonNull(linePattern)){ + Matcher matcher = linePattern.matcher(line); + if (matcher.find()){ + ignoreLine = 0; + rowIgnore = !isIncludeLine(line, onlyKeywords, ignoreKeywords); + } else { + ignoreLine += 1; + // TODO limit the value of ignoreLine + } + }else{ + rowIgnore = !isIncludeLine(line, onlyKeywords, ignoreKeywords); + } + }else { + rowIgnore = !isIncludeLine(line, onlyKeywords, ignoreKeywords); + } + if (!rowIgnore) { + if (line.contains("password")) { + LOG.info("have error information"); + } + if (!line.contains("password")) { + logs.add(new String(line.getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8)); + } + readLine += 1; + } + } + line = lineSupplier.get(); + } + result = new LogResult(lineNum, false, logs); + } catch (IOException e) { + throw new ExchangisJobServerException.Runtime(LOG_OP_ERROR.getCode(),"Unable to query the logs from path: [" + logPath + "]", e); + } finally { + if (Objects.nonNull(logReader)) { + try { + logReader.close(); + } catch (IOException e) { + //Ignore + } + } + if (Objects.nonNull(reverseReader)) { + try { + reverseReader.close(); + } catch (IOException e) { + //Ignore + } + } + } + return result; + } + + /** + * Get last rows + * @param fullPath full path + * @param lastRows last rows + * @return + */ + private LogResult getLastRows(String fullPath, int lastRows){ + try { + List logs = Arrays.asList(Utils.exec(new String[]{"tail", "-n", lastRows + "", fullPath}, 5000L).split("\n")); + return new LogResult(0, true, logs); + }catch (Exception e){ + throw new ExchangisJobServerException.Runtime(LOG_OP_ERROR.getCode(), "Fail to get last rows from path: [" + fullPath + "]", e); + } + } + + private boolean isIncludeLine(String line, List onlyKeywordList, List ignoreKeywordList){ + boolean accept = ignoreKeywordList.isEmpty() || ignoreKeywordList.stream().noneMatch(line::contains); + if (accept){ + accept = onlyKeywordList.isEmpty() || onlyKeywordList.stream().anyMatch(line::contains); + } + return accept; + } +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/render/transform/TransformRequestVo.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/render/transform/TransformRequestVo.java index 124f894eb..2bbc689f9 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/render/transform/TransformRequestVo.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/render/transform/TransformRequestVo.java @@ -32,6 +32,11 @@ public class TransformRequestVo { */ private String sourceTable; + /** + * Table (source) not exist + */ + private boolean srcTblNotExist = false; + /** * Sink type id */ @@ -54,6 +59,10 @@ public class TransformRequestVo { */ private String sinkTable; + /** + * Table (sink) not exist + */ + private boolean sinkTblNotExist = false; /** * Labels */ @@ -150,4 +159,20 @@ public String getOperator() { public void setOperator(String operator) { this.operator = operator; } + + public void setSrcTblNotExist(boolean srcTblNotExist) { + this.srcTblNotExist = srcTblNotExist; + } + + public void setSinkTblNotExist(boolean sinkTblNotExist) { + this.sinkTblNotExist = sinkTblNotExist; + } + + public boolean isSrcTblNotExist() { + return srcTblNotExist; + } + + public boolean isSinkTblNotExist() { + return sinkTblNotExist; + } } diff --git a/exchangis-job/exchangis-job-server/src/main/scala/com/webank/wedatasphere/exchangis/job/server/log/cache/JobLogCache.scala b/exchangis-job/exchangis-job-server/src/main/scala/com/webank/wedatasphere/exchangis/job/server/log/cache/JobLogCache.scala index 1986373ae..75721f8ee 100644 --- a/exchangis-job/exchangis-job-server/src/main/scala/com/webank/wedatasphere/exchangis/job/server/log/cache/JobLogCache.scala +++ b/exchangis-job/exchangis-job-server/src/main/scala/com/webank/wedatasphere/exchangis/job/server/log/cache/JobLogCache.scala @@ -35,7 +35,7 @@ abstract class AbstractJobLogCache[V](scheduler: Scheduler, maxSize: Int = 100, var lastFlush: Long = -1L - var cacheQueue: util.concurrent.ArrayBlockingQueue[Any] = new ArrayBlockingQueue[Any](maxSize) + var cacheQueue: util.concurrent.ArrayBlockingQueue[V] = new ArrayBlockingQueue[V](maxSize) var isShutdown: Boolean = false @@ -65,7 +65,7 @@ abstract class AbstractJobLogCache[V](scheduler: Scheduler, maxSize: Int = 100, override def cacheLog(log: V): Unit = { val element: Any = getCacheQueueElement(log) - if (!cacheQueue.offer(element)) { + if (!cacheQueue.offer(element.asInstanceOf[V])) { warn("The cache queue is full, should flush the cache immediately") flushCache(false) } else if (lastFlush + flushInterval < System.currentTimeMillis){ diff --git a/pom.xml b/pom.xml index e543e521f..a8c905e84 100644 --- a/pom.xml +++ b/pom.xml @@ -131,6 +131,12 @@ + + + org.apache.linkis + linkis-gateway-httpclient-support + ${linkis.version} + org.apache.linkis linkis-common From 3bdb4f60bf7ee0fd69a64ed9b61cdfb460633ffe Mon Sep 17 00:00:00 2001 From: davidhua Date: Fri, 17 May 2024 15:54:39 +0800 Subject: [PATCH 08/60] Add heartbeat feature to datax engine plugin. --- .../engineconn-plugins/datax/pom.xml | 10 +++++++ .../config/DataxSpringConfiguration.scala | 23 ++++++++++++++ .../executor/DataxContainerOnceExecutor.scala | 23 ++++++++++++++ .../service/DataxHeartbeatMsgManager.scala | 30 +++++++++++++++++++ 4 files changed, 86 insertions(+) create mode 100644 exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/config/DataxSpringConfiguration.scala create mode 100644 exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/service/DataxHeartbeatMsgManager.scala diff --git a/exchangis-engines/engineconn-plugins/datax/pom.xml b/exchangis-engines/engineconn-plugins/datax/pom.xml index af4629d1b..8fc8d4977 100644 --- a/exchangis-engines/engineconn-plugins/datax/pom.xml +++ b/exchangis-engines/engineconn-plugins/datax/pom.xml @@ -116,6 +116,16 @@ linkis-udf-client 1.4.0 + + org.apache.linkis + linkis-module + + + org.springframework.cloud + spring-cloud-commons + + + diff --git a/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/config/DataxSpringConfiguration.scala b/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/config/DataxSpringConfiguration.scala new file mode 100644 index 000000000..06609c615 --- /dev/null +++ b/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/config/DataxSpringConfiguration.scala @@ -0,0 +1,23 @@ +package org.apache.linkis.engineconnplugin.datax.config + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.engineconn.acessible.executor.info.NodeHeartbeatMsgManager +import org.apache.linkis.engineconnplugin.datax.service.DataxHeartbeatMsgManager +import org.springframework.context.annotation.{Bean, Configuration, Primary} + +/** + * Spring configuration for datax + */ +@Configuration +class DataxSpringConfiguration extends Logging { + + /** + * Override the heartbeat manager + * @return + */ + @Bean + @Primary + def nodeHeartbeatMsgManager(): NodeHeartbeatMsgManager = { + new DataxHeartbeatMsgManager() + } +} diff --git a/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/executor/DataxContainerOnceExecutor.scala b/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/executor/DataxContainerOnceExecutor.scala index 9cec013ac..22de5da0b 100644 --- a/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/executor/DataxContainerOnceExecutor.scala +++ b/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/executor/DataxContainerOnceExecutor.scala @@ -26,7 +26,9 @@ import com.alibaba.datax.core.util.container.{CoreConstant, LoadUtil} import com.alibaba.datax.core.util.{ConfigurationValidate, ExceptionTracker, FrameworkErrorCode, SecretUtil} import org.apache.commons.lang3.StringUtils import org.apache.linkis.common.utils.{ClassUtils, Utils} +import org.apache.linkis.engineconn.acessible.executor.service.{ExecutorHeartbeatService, ExecutorHeartbeatServiceHolder} import org.apache.linkis.engineconn.common.creation.EngineCreationContext +import org.apache.linkis.engineconn.executor.service.ManagerService import org.apache.linkis.engineconn.once.executor.{OnceExecutorExecutionContext, OperableOnceExecutor} import org.apache.linkis.engineconnplugin.datax.config.DataxConfiguration import org.apache.linkis.engineconnplugin.datax.exception.{DataxJobExecutionException, DataxPluginLoadException} @@ -86,6 +88,8 @@ abstract class DataxContainerOnceExecutor extends DataxOnceExecutor with Operabl } info(s"The executor: [${getId}] has been finished, now to stop DataxEngineConn.") closeDaemon() + // Try to heartbeat at last + tryToHeartbeat() if (!isFailed) { trySucceed() } @@ -103,6 +107,8 @@ abstract class DataxContainerOnceExecutor extends DataxOnceExecutor with Operabl if (!(future.isDone || future.isCancelled)) { trace(s"The executor: [$getId] has been still running") } + // Heartbeat action interval + tryToHeartbeat() } }, DataxConfiguration.STATUS_FETCH_INTERVAL.getValue.toLong, DataxConfiguration.STATUS_FETCH_INTERVAL.getValue.toLong, TimeUnit.MILLISECONDS) @@ -143,6 +149,10 @@ abstract class DataxContainerOnceExecutor extends DataxOnceExecutor with Operabl metrics } + def getMessage(key: String):util.Map[String, util.List[String]] = { + null + } + override def getDiagnosis: util.Map[String, Any] = { // Not support diagnosis new util.HashMap[String, Any]() @@ -156,6 +166,19 @@ abstract class DataxContainerOnceExecutor extends DataxOnceExecutor with Operabl // Option(this.container).foreach(_.shutdown()) super.tryFailed() } + + /** + * Try to send heartbeat message to ecm + */ + private def tryToHeartbeat(): Unit = { + logger.trace("heartbeat and record to linkis manager") + ExecutorHeartbeatServiceHolder.getDefaultHeartbeatService() match { + case heartbeatService: ExecutorHeartbeatService => + val heartbeatMsg = heartbeatService.generateHeartBeatMsg(this) + ManagerService.getManagerService.heartbeatReport(heartbeatMsg) + logger.trace(s"Succeed to report heartbeatMsg: [${heartbeatMsg}]") + } + } /** * Execute with job content * @param jobContent job content diff --git a/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/service/DataxHeartbeatMsgManager.scala b/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/service/DataxHeartbeatMsgManager.scala new file mode 100644 index 000000000..26b26392c --- /dev/null +++ b/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/service/DataxHeartbeatMsgManager.scala @@ -0,0 +1,30 @@ +package org.apache.linkis.engineconnplugin.datax.service + +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.engineconn.acessible.executor.info.NodeHeartbeatMsgManager +import org.apache.linkis.engineconn.executor.entity.Executor +import org.apache.linkis.engineconnplugin.datax.executor.DataxContainerOnceExecutor +import org.apache.linkis.server.BDPJettyServerHelper + +import scala.collection.JavaConverters.mapAsScalaMapConverter + +/** + * Datax heartbeat message (include: metric, error message) + */ +class DataxHeartbeatMsgManager extends NodeHeartbeatMsgManager with Logging{ + override def getHeartBeatMsg(executor: Executor): String = { + executor match { + case dataxExecutor: DataxContainerOnceExecutor => + val metric = dataxExecutor.getMetrics + Utils.tryCatch(BDPJettyServerHelper.gson.toJson(metric)) { case e: Exception => + val mV = metric.asScala + .map { case (k, v) => if (null == v) s"${k}->null" else s"${k}->${v.toString}" } + .mkString(",") + val errMsg = e.getMessage + logger.error(s"Convert metric to json failed because : ${errMsg}, metric values : {${mV}}") + "{\"errorMsg\":\"Convert metric to json failed because : " + errMsg + "\"}" + } + case _ => "{}" + } + } +} From acb44ffd1ded6e03889ecdb69ade2153be686505 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Fri, 17 May 2024 16:07:38 +0800 Subject: [PATCH 09/60] Minor fix of package import --- .../job/launcher/linkis/LinkisExchangisTaskLauncher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisExchangisTaskLauncher.java b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisExchangisTaskLauncher.java index ca505987d..b366b8dc4 100644 --- a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisExchangisTaskLauncher.java +++ b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisExchangisTaskLauncher.java @@ -1,6 +1,6 @@ package com.webank.wedatasphere.exchangis.job.launcher.linkis; -import com.webank.wedatasphere.exchangis.common.linkis.client.ClientConfiguration; +import com.webank.wedatasphere.exchangis.common.linkis.ClientConfiguration; import com.webank.wedatasphere.exchangis.job.enums.EngineTypeEnum; import com.webank.wedatasphere.exchangis.job.launcher.AccessibleLauncherTask; import com.webank.wedatasphere.exchangis.job.launcher.exception.ExchangisTaskLaunchException; From 5ee1f8ab5c96716a50643bbe626df5a73d8172cc Mon Sep 17 00:00:00 2001 From: davidhua Date: Mon, 20 May 2024 16:24:46 +0800 Subject: [PATCH 10/60] Enable to reverse read log. --- .../exchangis/job/domain/ExchangisJob.java | 1 + .../launcher/linkis/LinkisLauncherTask.java | 14 +- .../linkis/LaunchTaskLogOperator.scala | 28 ++++ .../server/execution/AbstractTaskManager.java | 158 ++++++++++++------ .../job/server/execution/TaskManager.java | 10 ++ .../tasks/StatusUpdateSchedulerTask.java | 9 +- .../scheduler/tasks/SubmitSchedulerTask.java | 1 + .../server/log/service/RpcJobLogService.java | 3 + .../ExchangisJobExecuteRestfulApi.java | 4 + .../ExchangisTaskExecuteRestfulApi.java | 4 + 10 files changed, 182 insertions(+), 50 deletions(-) create mode 100644 exchangis-job/exchangis-job-launcher/src/main/scala/com/webank/wedatasphere/exchangis/job/launcher/linkis/LaunchTaskLogOperator.scala diff --git a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisJob.java b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisJob.java index 3107b81ed..8afd21e49 100644 --- a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisJob.java +++ b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisJob.java @@ -41,6 +41,7 @@ public interface ExchangisJob extends ExchangisBase{ * @param jobLabels */ void setJobLabels(Map jobLabels); + /** * Create user * @return user name diff --git a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisLauncherTask.java b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisLauncherTask.java index 408fdc43b..c51a1293c 100644 --- a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisLauncherTask.java +++ b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisLauncherTask.java @@ -1,5 +1,6 @@ package com.webank.wedatasphere.exchangis.job.launcher.linkis; +import com.webank.wedatasphere.exchangis.datasource.core.utils.Json; import com.webank.wedatasphere.exchangis.job.launcher.exception.ExchangisTaskLaunchException; import com.webank.wedatasphere.exchangis.job.launcher.exception.ExchangisTaskNotExistException; import com.webank.wedatasphere.exchangis.job.launcher.AccessibleLauncherTask; @@ -9,7 +10,6 @@ import com.webank.wedatasphere.exchangis.job.launcher.domain.task.TaskProgressInfo; import com.webank.wedatasphere.exchangis.job.launcher.domain.task.TaskStatus; import org.apache.commons.lang.StringUtils; -import org.apache.linkis.common.utils.Utils; import org.apache.linkis.computation.client.LinkisJobBuilder; import org.apache.linkis.computation.client.LinkisJobClient; import org.apache.linkis.computation.client.once.SubmittableOnceJob; @@ -35,6 +35,8 @@ public class LinkisLauncherTask implements AccessibleLauncherTask { private static final Logger LOG = LoggerFactory.getLogger(LinkisLauncherTask.class); + private static final String METRIC_NAME = "ecMetrics"; + /** * Engine versions */ @@ -162,6 +164,13 @@ public Map getMetricsInfo() throws ExchangisTaskLaunchException // Init the error count this.reqError.set(0); return metrics; + }else { + // Try to get metric from job info + Map jobInfo = getJobInfo(false); + Object metric = jobInfo.get(METRIC_NAME); + if (Objects.nonNull(metric)){ + return Json.fromJson(String.valueOf(metric), Map.class); + } } }catch(Exception e){ dealException(e); @@ -222,13 +231,14 @@ public LogResult queryLogs(LogQuery query) throws ExchangisTaskLaunchException { // The logOperator is not thread safe, so create it each time if (Objects.nonNull(this.onceJob)){ try{ - EngineConnLogOperator logOperator = (EngineConnLogOperator) this.onceJob.getOperator(EngineConnLogOperator.OPERATOR_NAME()); + LaunchTaskLogOperator logOperator = (LaunchTaskLogOperator) this.onceJob.getOperator(LaunchTaskLogOperator.OPERATOR_NAME()); logOperator.setFromLine(query.getFromLine()); logOperator.setPageSize(query.getPageSize()); logOperator.setEngineConnType(this.engineConn); logOperator.setECMServiceInstance(this.onceJob.getECMServiceInstance(this.jobInfo)); logOperator.setIgnoreKeywords(query.getIgnoreKeywords()); logOperator.setOnlyKeywords(query.getOnlyKeywords()); + logOperator.setEnableTail(query.isEnableTail()); if (Objects.nonNull(query.getLastRows())){ logOperator.setLastRows(query.getLastRows()); } diff --git a/exchangis-job/exchangis-job-launcher/src/main/scala/com/webank/wedatasphere/exchangis/job/launcher/linkis/LaunchTaskLogOperator.scala b/exchangis-job/exchangis-job-launcher/src/main/scala/com/webank/wedatasphere/exchangis/job/launcher/linkis/LaunchTaskLogOperator.scala new file mode 100644 index 000000000..93e6fe159 --- /dev/null +++ b/exchangis-job/exchangis-job-launcher/src/main/scala/com/webank/wedatasphere/exchangis/job/launcher/linkis/LaunchTaskLogOperator.scala @@ -0,0 +1,28 @@ +package com.webank.wedatasphere.exchangis.job.launcher.linkis + +import org.apache.linkis.computation.client.once.action.EngineConnOperateAction +import org.apache.linkis.computation.client.operator.impl.EngineConnLogOperator + +/** + * Enable to reverse read log file + */ +class LaunchTaskLogOperator extends EngineConnLogOperator{ + + private var enableTail: Boolean = false + + def setEnableTail(enableTail: Boolean): Unit = { + this.enableTail = enableTail + } + + def isEnableTail: Boolean = { + this.enableTail + } + + protected override def addParameters(builder: EngineConnOperateAction.Builder): Unit = { + super.addParameters(builder) + builder.addParameter("enableTail", enableTail) + } +} +object LaunchTaskLogOperator { + val OPERATOR_NAME = "launchTaskLog" +} \ No newline at end of file diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/AbstractTaskManager.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/AbstractTaskManager.java index b3a05b8af..e31d3f087 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/AbstractTaskManager.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/AbstractTaskManager.java @@ -14,6 +14,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; /** * Launched task manager @@ -33,7 +35,7 @@ public abstract class AbstractTaskManager implements TaskManager Running task */ - private ConcurrentHashMap runningTasks = new ConcurrentHashMap<>(); + private ConcurrentHashMap runningTasks = new ConcurrentHashMap<>(); /** * job_execution_id => List(Running tasks) @@ -50,18 +52,22 @@ public List getJobExecutionIds(){ @Override public List getRunningTasks() { - return new ArrayList<>(runningTasks.values()); + return runningTasks.values().stream().map(ctx -> + ctx.task).collect(Collectors.toList()); } @Override public void cancelRunningTask(String taskId) { - LaunchedExchangisTask task = runningTasks.get(taskId); - if (Objects.nonNull(task)){ - onEvent(new TaskStatusUpdateEvent(task, TaskStatus.Cancelled)); - info(task, "Status of task: [name: {}, id: {}] change {} => {}", - task.getName(), task.getTaskId(), task.getStatus(), TaskStatus.Cancelled); - JobLogCacheUtils.flush(task.getJobExecutionId(), false); - runningTasks.remove(taskId); + TaskContext context = runningTasks.get(taskId); + if (Objects.nonNull(context)){ + LaunchedExchangisTask task = context.task; + context.access(() -> { + onEvent(new TaskStatusUpdateEvent(task, TaskStatus.Cancelled)); + info(task, "Status of task: [name: {}, id: {}] change {} => {}", + task.getName(), task.getTaskId(), task.getStatus(), TaskStatus.Cancelled); + JobLogCacheUtils.flush(task.getJobExecutionId(), false); + runningTasks.remove(taskId); + }); JobWrapper wrapper = jobWrappers.get(task.getJobExecutionId()); if (Objects.nonNull(wrapper)){ wrapper.removeTask(task); @@ -75,7 +81,7 @@ public void addRunningTask(LaunchedExchangisTask task) { task.setRunningTime(Calendar.getInstance().getTime()); onEvent(new TaskLaunchEvent(task)); info(task, "Status of task: [name: {}, id: {}] change to {}, info: [{}]", task.getName(), task.getTaskId(), task.getStatus(), ""); - if (Objects.isNull(runningTasks.putIfAbsent(task.getTaskId(), task))){ + if (Objects.isNull(runningTasks.putIfAbsent(task.getTaskId(), new TaskContext(task)))){ jobWrappers.compute(task.getJobExecutionId(), (jobExecutionId, jobWrapper) -> { if (Objects.nonNull(jobWrapper) && jobWrapper.addTask(task)){ return jobWrapper; @@ -95,12 +101,9 @@ public void removeRunningTask(String taskId) { @Override public boolean refreshRunningTaskMetrics(LaunchedExchangisTask task, Map metricsMap) { - task = runningTasks.get(task.getTaskId()); - if (Objects.nonNull(task)) { - onEvent(new TaskMetricsUpdateEvent(task, metricsMap)); - task.setMetrics(null); - task.setMetricsMap(metricsMap); - trace(task, "Metrics info of task: [{}]", Json.toJson(metricsMap, null)); + TaskContext context = runningTasks.get(task.getTaskId()); + if (Objects.nonNull(context)) { + refreshRunningTaskMetrics(context, metricsMap); return true; } return false; @@ -108,37 +111,53 @@ public boolean refreshRunningTaskMetrics(LaunchedExchangisTask task, Map metricsMap) { TaskStatus beforeStatus = task.getStatus(); - if (TaskStatus.isCompleted(status)){ - info(task, "Status of task: [name: {}, id: {}] change {} => {}", - task.getName(), task.getTaskId(), beforeStatus, status); - onEvent(new TaskStatusUpdateEvent(task, status)); - removeRunningTaskInner(task.getTaskId(), false); - return true; - } else { - task = runningTasks.get(task.getTaskId()); - if (Objects.nonNull(task) ) { - onEvent(new TaskStatusUpdateEvent(task, status)); - if (isTransition(task, status)) { - info(task, "Status of task: [name: {}, id: {}] change {} => {}", - task.getName(), task.getTaskId(), beforeStatus, status); + TaskContext context = runningTasks.get(task.getTaskId()); + if (Objects.nonNull(context)){ + task = context.task; + LaunchedExchangisTask finalTask = task; + context.access( () -> { + if (Objects.nonNull(metricsMap)){ + refreshRunningTaskMetrics(context, metricsMap); } - task.setStatus(status); - return true; - } - return false; + if (TaskStatus.isCompleted(status)){ + info(finalTask, "Status of task: [name: {}, id: {}] change {} => {}", + finalTask.getName(), finalTask.getTaskId(), beforeStatus, status); + onEvent(new TaskStatusUpdateEvent(finalTask, status)); + removeRunningTaskInner(finalTask.getTaskId(), false); + } else { + onEvent(new TaskStatusUpdateEvent(finalTask, status)); + if (isTransition(finalTask, status)) { + info(finalTask, "Status of task: [name: {}, id: {}] change {} => {}", + finalTask.getName(), finalTask.getTaskId(), beforeStatus, status); + } + } + finalTask.setStatus(status); + }); + return true; } + return false; } @Override public boolean refreshRunningTaskProgress(LaunchedExchangisTask task, TaskProgressInfo progressInfo) { - task = runningTasks.get(task.getTaskId()); - if (Objects.nonNull(task)){ - onEvent(new TaskProgressUpdateEvent(task, progressInfo)); - if (task.getProgress() != progressInfo.getProgress()){ - info(task, "Progress of task: [{}] change {} => {}", task.getTaskId(), task.getProgress(), progressInfo.getProgress()); - } - task.setProgress(progressInfo.getProgress()); + TaskContext context = runningTasks.get(task.getTaskId()); + if (Objects.nonNull(context)){ + task = context.task; + LaunchedExchangisTask finalTask = task; + context.access(() -> { + onEvent(new TaskProgressUpdateEvent(finalTask, progressInfo)); + if (finalTask.getProgress() != progressInfo.getProgress()){ + info(finalTask, "Progress of task: [{}] change {} => {}", + finalTask.getTaskId(), finalTask.getProgress(), progressInfo.getProgress()); + } + finalTask.setProgress(progressInfo.getProgress()); + }); return true; } return false; @@ -146,7 +165,8 @@ public boolean refreshRunningTaskProgress(LaunchedExchangisTask task, TaskProgre @Override public LaunchedExchangisTask getRunningTask(String taskId) { - return runningTasks.get(taskId); + TaskContext context = runningTasks.get(taskId); + return context != null ? context.task : null; } public TaskExecutionListener getExecutionListener() { @@ -157,18 +177,38 @@ public void setExecutionListener(TaskExecutionListener executionListener) { this.executionListener = executionListener; } + /** + * Refresh running task metrics + * @param context context + * @param metricsMap metric map + */ + private void refreshRunningTaskMetrics(TaskContext context, Map metricsMap){ + LaunchedExchangisTask finalTask = context.task; + context.access(() -> { + if (!TaskStatus.isCompleted(finalTask.getStatus())) { + onEvent(new TaskMetricsUpdateEvent(finalTask, metricsMap)); + finalTask.setMetrics(null); + finalTask.setMetricsMap(metricsMap); + trace(finalTask, "Metrics info of task: [{}]", Json.toJson(metricsMap, null)); + } + }); + } + /** * Remove inner * @param taskId task id * @param updateStatus if update status */ private void removeRunningTaskInner(String taskId, boolean updateStatus){ - LaunchedExchangisTask task = runningTasks.get(taskId); - if (Objects.nonNull(task)){ - if (updateStatus) { - onEvent(new TaskStatusUpdateEvent(task, task.getStatus())); - } - runningTasks.remove(taskId); + TaskContext context = runningTasks.get(taskId); + if (Objects.nonNull(context)){ + LaunchedExchangisTask task = context.task; + context.access(() -> { + if (updateStatus) { + onEvent(new TaskStatusUpdateEvent(task, task.getStatus())); + } + runningTasks.remove(taskId); + }); JobWrapper wrapper = jobWrappers.get(task.getJobExecutionId()); if (Objects.nonNull(wrapper)){ wrapper.removeTask(task); @@ -199,6 +239,30 @@ public JobLogEvent getJobLogEvent(JobLogEvent.Level level, LaunchedExchangisTask return new JobLogEvent(level, task.getExecuteUser(), task.getJobExecutionId(), message, args); } + private static class TaskContext{ + /** + * Access lock + */ + private final ReentrantLock accessLock = new ReentrantLock(); + + private final LaunchedExchangisTask task; + + public TaskContext(LaunchedExchangisTask task){ + this.task = task; + } + /** + * Access the process + * @param exec exec process + */ + private void access(Runnable exec){ + accessLock.lock(); + try{ + exec.run(); + }finally { + accessLock.unlock(); + } + } + } private class JobWrapper{ /** diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/TaskManager.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/TaskManager.java index 0ffecb43a..80d4bf9de 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/TaskManager.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/TaskManager.java @@ -41,6 +41,7 @@ public interface TaskManager extends JobServerLogging metricsMap); + /** * Refresh running task status * @param task @@ -49,6 +50,15 @@ public interface TaskManager extends JobServerLogging metricsMap); + /** * Refresh progress * @param task diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/StatusUpdateSchedulerTask.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/StatusUpdateSchedulerTask.java index 7551ec207..4772fdc22 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/StatusUpdateSchedulerTask.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/StatusUpdateSchedulerTask.java @@ -1,5 +1,6 @@ package com.webank.wedatasphere.exchangis.job.server.execution.scheduler.tasks; +import com.webank.wedatasphere.exchangis.job.launcher.domain.task.TaskStatus; import com.webank.wedatasphere.exchangis.job.launcher.exception.ExchangisTaskLaunchException; import com.webank.wedatasphere.exchangis.job.launcher.AccessibleLauncherTask; import com.webank.wedatasphere.exchangis.job.launcher.domain.LaunchedExchangisTask; @@ -47,7 +48,13 @@ protected void onPoll(LaunchedExchangisTask launchedExchangisTask) throws Exchan if (Objects.nonNull(progressInfo)){ this.taskManager.refreshRunningTaskProgress(launchedExchangisTask, progressInfo); } - this.taskManager.refreshRunningTaskStatus(launchedExchangisTask, launcherTask.getLocalStatus()); + TaskStatus status = launcherTask.getLocalStatus(); + if (TaskStatus.isCompleted(status)){ + this.taskManager.refreshRunningTaskStatusAndMetrics(launchedExchangisTask, + status, launcherTask.getMetricsInfo()); + } else { + this.taskManager.refreshRunningTaskStatus(launchedExchangisTask, status); + } } catch (ExchangisTaskLaunchException e){ throw new ExchangisSchedulerException("Fail to update status(progress) for task: [" + launchedExchangisTask.getTaskId() + "]", e); } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/SubmitSchedulerTask.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/SubmitSchedulerTask.java index 4a3df1e06..89c9deee7 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/SubmitSchedulerTask.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/SubmitSchedulerTask.java @@ -72,6 +72,7 @@ public SubmitSchedulerTask(LaunchableExchangisTask task, Callable submi // Ignore } } + // Set max retry } @Override protected void schedule() throws ExchangisSchedulerException, ExchangisSchedulerRetryException { diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/RpcJobLogService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/RpcJobLogService.java index 14b73d585..46853ceb7 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/RpcJobLogService.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/RpcJobLogService.java @@ -233,6 +233,9 @@ public LogResult logsFromPageAndPath(String logPath, LogQuery logQuery) { } line = lineSupplier.get(); } + if (logQuery.isEnableTail()){ + Collections.reverse(logs); + } result = new LogResult(lineNum, false, logs); } catch (IOException e) { throw new ExchangisJobServerException.Runtime(LOG_OP_ERROR.getCode(),"Unable to query the logs from path: [" + logPath + "]", e); diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java index 5ed4cfb4a..77ffb45ae 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java @@ -157,11 +157,15 @@ public Message getJobExecutionLogs(@PathVariable(value = "jobExecutionId") Strin @RequestParam(value = "pageSize", required = false) Integer pageSize, @RequestParam(value = "ignoreKeywords", required = false) String ignoreKeywords, @RequestParam(value = "onlyKeywords", required = false) String onlyKeywords, + @RequestParam(value = "enableTail", required = false) Boolean enableTail, @RequestParam(value = "lastRows", required = false) Integer lastRows, HttpServletRequest request) { Message result = Message.ok("Submitted succeed(提交成功)！"); LogQuery logQuery = new LogQuery(fromLine, pageSize, ignoreKeywords, onlyKeywords, lastRows); + if (null != enableTail) { + logQuery.setEnableTail(enableTail); + } String loginUser = UserUtils.getLoginUser(request); try { if(!JobAuthorityUtils.hasJobExecuteSituationAuthority(loginUser, jobExecutionId, OperationType.JOB_QUERY)) { diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisTaskExecuteRestfulApi.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisTaskExecuteRestfulApi.java index 003c24c1e..aa2a45e1b 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisTaskExecuteRestfulApi.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisTaskExecuteRestfulApi.java @@ -77,10 +77,14 @@ public Message getTaskExecutionLogs(@PathVariable(value = "taskId") String taskI @RequestParam(value = "pageSize", required = false) Integer pageSize, @RequestParam(value = "ignoreKeywords", required = false) String ignoreKeywords, @RequestParam(value = "onlyKeywords", required = false) String onlyKeywords, + @RequestParam(value = "enableTail", required = false) Boolean enableTail, @RequestParam(value = "lastRows", required = false) Integer lastRows, HttpServletRequest request) { Message result = Message.ok("Submitted succeed(提交成功)！"); LogQuery logQuery = new LogQuery(fromLine, pageSize, ignoreKeywords, onlyKeywords, lastRows); + if (null != enableTail) { + logQuery.setEnableTail(enableTail); + } String userName = UserUtils.getLoginUser(request); try { if (!JobAuthorityUtils.hasJobExecuteSituationAuthority(userName, jobExecutionId, OperationType.JOB_QUERY)) { From 8dbe48290ff704043b0afc5e8e466c56b2c316d1 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Mon, 20 May 2024 18:17:32 +0800 Subject: [PATCH 11/60] Update version of linkis to 1.4.0-wds --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index a8c905e84..6ffb9c271 100644 --- a/pom.xml +++ b/pom.xml @@ -39,10 +39,10 @@ 1.1.3-webank 1.1.2 - 1.3.0-wds - 1.3.0-wds + 1.4.0-wds + 1.4.0-wds 0.1.0-SNAPSHOT - 2.12.12 + 2.11.12 4.7.1 1.8 3.3.3 From 38bc89ed50d157174c0bb7e5cdefe0113f379b80 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Mon, 20 May 2024 18:07:16 +0800 Subject: [PATCH 12/60] Update version of linkis to 1.4.0-wds --- .../builder/transform/TransformExchangisJob.java | 11 ++--------- pom.xml | 2 +- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/TransformExchangisJob.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/TransformExchangisJob.java index 319f0c7c6..a9798b845 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/TransformExchangisJob.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/TransformExchangisJob.java @@ -172,9 +172,7 @@ private void convertContentToParams(ExchangisJobInfoContent content){ if (StringUtils.isNotBlank(sourceId)){ this.sourceType = resolveDataSourceId(content.getDataSources().getSourceId(), paramSet); } else { - Optional.ofNullable(content.getDataSources().getSource().getType()).ifPresent(type -> { - this.sourceType = type.name; - }); + } } @@ -195,9 +193,7 @@ private void convertContentToParams(ExchangisJobInfoContent content){ if (StringUtils.isNotBlank(sinkId)){ this.sinkType = resolveDataSourceId(content.getDataSources().getSinkId(), paramSet); } else { - Optional.ofNullable(content.getDataSources().getSink().getType()).ifPresent(type -> { - this.sinkType = type.name; - }); + } } } @@ -245,9 +241,6 @@ private String resolveDataSourceId(String dataSourceId, JobParamSet paramSet){ return null; } - private String resolveDataSource(ExchangisJobDataSourcesContent.ExchangisJobDataSource dataSource){ - return null; - } /** * * @param items diff --git a/pom.xml b/pom.xml index 6ffb9c271..461a3cce1 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ com.webank.wedatasphere.exchangis exchangis - ${revision} + 1.1.3-webank pom exchangis From 5aee9433a503048fba8d585c8a1be88aa5b56c6a Mon Sep 17 00:00:00 2001 From: davidhua Date: Mon, 20 May 2024 17:52:28 +0800 Subject: [PATCH 13/60] Adjust the class structure of data source; Custom the http client; --- .../config/dss-exchangis-server.properties | 5 +- exchangis-dao/pom.xml | 4 + .../common/http/HttpClientConfiguration.java | 25 +++ .../common/linkis/ClientConfiguration.java | 26 --- .../linkis/client/ClientConfiguration.java | 54 ++++++ .../dao/domain/ExchangisJobDsBind.java | 26 +++ .../linkis/client/ExchangisHttpClient.scala | 39 ++++ .../client/config/ExchangisClientConfig.scala | 36 ++++ .../config/ExchangisClientConfigBuilder.scala | 59 ++++++ ...AbstractExchangisDataSourceDefinition.java | 85 +++++++++ .../datasource/core/ExchangisDataSource.java | 51 +----- .../ExchangisDataSourceConfiguration.java | 31 ++-- .../core/ExchangisDataSourceDefinition.java | 75 ++++++++ ...xt.java => DefaultExchangisDsContext.java} | 26 +-- .../context/ExchangisDataSourceContext.java | 67 +++++-- .../core/domain/DataSourceType.java | 22 --- .../core/domain/ExchangisDataSourceType.java | 30 +++ ...java => ExchangisDataSourceDefLoader.java} | 8 +- .../vo/ExchangisJobDataSourcesContent.java | 123 ++++++++++++- .../core/vo/ExchangisJobInfoContent.java | 18 ++ .../core/vo/ExchangisJobParamsContent.java | 12 ++ .../core/vo/ExchangisJobTransformsItem.java | 32 ++++ .../linkis/ExchangisBatchDataSource.java | 52 +----- .../ExchangisDataSourceConfiguration.java | 19 -- .../linkis/ExchangisLinkisRemoteClient.scala | 94 +++++----- .../ExchangisDataSourceLoaderFactory.java | 16 +- .../LocalExchangisDataSourceLoader.java | 18 +- .../datasource/loader/utils/ExtDsUtils.java | 12 +- .../server/configuration/ServerConfig.java | 8 +- .../datasource/domain/ExchangisDsProject.java | 8 + .../datasource/dto/DataSourceDTO.java | 169 ----------------- .../dto/ExchangisDataSourceDTO.java | 171 +++++++++++++---- .../dto/ExchangisDataSourceDefDTO.java | 62 +++++++ .../service/AbstractDataSourceService.java | 10 +- .../service/ExchangisDataSourceService.java | 50 ++--- .../exchangis-datasource-streamis/pom.xml | 6 +- .../streamis/ExchangisStreamisDataSource.java | 34 +--- .../ExchangisStreamisRemoteClient.scala | 172 ------------------ .../mysql/ExchangisESDataSource.java | 22 +-- .../hive/ExchangisHiveDataSource.java | 22 +-- .../mysql/ExchangisMongoDbDataSource.java | 22 +-- .../mysql/ExchangisMySQLDataSource.java | 22 +-- .../oracle/ExchangisOracleDataSource.java | 15 +- .../sftp/ExchangisSftpDataSource.java | 16 +- exchangis-datasource/pom.xml | 2 +- .../linkis/LinkisExchangisTaskLauncher.java | 54 +++--- .../linkis/client/ExchangisLaunchClient.scala | 54 ++++++ 47 files changed, 1164 insertions(+), 820 deletions(-) create mode 100644 exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/http/HttpClientConfiguration.java delete mode 100644 exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/linkis/ClientConfiguration.java create mode 100644 exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/linkis/client/ClientConfiguration.java create mode 100644 exchangis-dao/src/main/scala/com/webank/wedatasphere/exchangis/common/linkis/client/ExchangisHttpClient.scala create mode 100644 exchangis-dao/src/main/scala/com/webank/wedatasphere/exchangis/common/linkis/client/config/ExchangisClientConfig.scala create mode 100644 exchangis-dao/src/main/scala/com/webank/wedatasphere/exchangis/common/linkis/client/config/ExchangisClientConfigBuilder.scala create mode 100644 exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/AbstractExchangisDataSourceDefinition.java create mode 100644 exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ExchangisDataSourceDefinition.java rename exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/context/{DefaultExchangisDataSourceContext.java => DefaultExchangisDsContext.java} (56%) delete mode 100644 exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/domain/DataSourceType.java create mode 100644 exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/domain/ExchangisDataSourceType.java rename exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/loader/{ExchangisDataSourceLoader.java => ExchangisDataSourceDefLoader.java} (81%) delete mode 100644 exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/ExchangisDataSourceConfiguration.java create mode 100644 exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/domain/ExchangisDsProject.java delete mode 100644 exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/DataSourceDTO.java create mode 100644 exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/ExchangisDataSourceDefDTO.java delete mode 100644 exchangis-datasource/exchangis-datasource-streamis/src/main/scala/com/webank/wedatasphere/exchangis/datasource/streamis/ExchangisStreamisRemoteClient.scala create mode 100644 exchangis-job/exchangis-job-launcher/src/main/scala/com/webank/wedatasphere/exchangis/job/launcher/linkis/client/ExchangisLaunchClient.scala diff --git a/assembly-package/config/dss-exchangis-server.properties b/assembly-package/config/dss-exchangis-server.properties index 8f3b64bd3..70ebaca62 100644 --- a/assembly-package/config/dss-exchangis-server.properties +++ b/assembly-package/config/dss-exchangis-server.properties @@ -30,9 +30,8 @@ wds.linkis.server.version=v1 wds.linkis.ms.service.scan.package=com.webank.wedatasphere.exchangis # datasource client -wds.exchangis.datasource.client.serverurl=http://{LINKIS_IP}:{LINKIS_PORT}/ -wds.exchangis.datasource.client.authtoken.key=EXCHANGIS-AUTH -wds.exchangis.datasource.client.authtoken.value=EXCHANGIS-AUTH +wds.exchangis.datasource.client.server-url=http://{LINKIS_IP}:{LINKIS_PORT}/ +wds.exchangis.datasource.client.token.value=EXCHANGIS-AUTH wds.exchangis.datasource.client.dws.version=v1 # launcher client diff --git a/exchangis-dao/pom.xml b/exchangis-dao/pom.xml index 620401fac..4eac9752a 100644 --- a/exchangis-dao/pom.xml +++ b/exchangis-dao/pom.xml @@ -27,6 +27,10 @@ org.apache.linkis linkis-module + + org.apache.linkis + linkis-gateway-httpclient-support + org.hibernate hibernate-validator diff --git a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/http/HttpClientConfiguration.java b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/http/HttpClientConfiguration.java new file mode 100644 index 000000000..548b99650 --- /dev/null +++ b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/http/HttpClientConfiguration.java @@ -0,0 +1,25 @@ +package com.webank.wedatasphere.exchangis.common.http; + +import org.apache.linkis.common.conf.CommonVars; + +/** + * Define the http configuration + */ +public class HttpClientConfiguration { + + /** + * Connect timeout + */ + public static final CommonVars CONNECTION_TIMEOUT = CommonVars.apply("wds.exchangis.http.client.connection.timeout", 30000L); + + /** + * Max connection size + */ + public static final CommonVars MAX_CONNECTION_SIZE = CommonVars.apply("wds.exchangis.http.client.connection.max-size", 100); + + /** + * Read timeout + */ + public static final CommonVars READ_TIMEOUT = CommonVars.apply("wds.exchangis.http.client.read-timeout", 90000L); + +} diff --git a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/linkis/ClientConfiguration.java b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/linkis/ClientConfiguration.java deleted file mode 100644 index 3725610fd..000000000 --- a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/linkis/ClientConfiguration.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.webank.wedatasphere.exchangis.common.linkis; - -import org.apache.linkis.common.conf.CommonVars; -import org.apache.linkis.common.conf.Configuration; - -/** - * Configuration for linkis client - */ -public class ClientConfiguration { - - /** - * Linkis server url - */ - public static final CommonVars LINKIS_SERVER_URL = CommonVars.apply("wds.exchangis.client.linkis.server-url", Configuration.getGateWayURL()); - - /** - * Linkis token value - */ - public static final CommonVars LINKIS_TOKEN_VALUE = CommonVars.apply("wds.exchangis.client.linkis.token.value", "EXCHANGIS-TOKEN"); - - /** - * Linkis client max connections - */ - public static final CommonVars LINKIS_DEFAULT_MAX_CONNECTIONS = CommonVars.apply("wds.exchangis.client.linkis.max-connections.default", 70); - -} diff --git a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/linkis/client/ClientConfiguration.java b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/linkis/client/ClientConfiguration.java new file mode 100644 index 000000000..bb65867a4 --- /dev/null +++ b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/linkis/client/ClientConfiguration.java @@ -0,0 +1,54 @@ +package com.webank.wedatasphere.exchangis.common.linkis.client; + +import com.webank.wedatasphere.exchangis.common.http.HttpClientConfiguration; +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.conf.Configuration; + +/** + * Configuration for linkis client + */ +public class ClientConfiguration { + + /** + * Linkis server url + */ + public static final CommonVars LINKIS_SERVER_URL = CommonVars.apply("wds.exchangis.client.linkis.server-url", Configuration.getGateWayURL()); + + /** + * Linkis token value + */ + public static final CommonVars LINKIS_TOKEN_VALUE = CommonVars.apply("wds.exchangis.client.linkis.token.value", "EXCHANGIS-TOKEN"); + + /** + * Linkis client max connections + */ + public static final CommonVars LINKIS_DEFAULT_MAX_CONNECTIONS = CommonVars.apply("wds.exchangis.client.linkis.max-connections.default", + HttpClientConfiguration.MAX_CONNECTION_SIZE.getValue()); + + + /** + * Linkis discovery enable + */ + public static final CommonVars LINKIS_DISCOVERY_ENABLED = CommonVars.apply("wds.exchangis.client.linkis.discovery.enabled", true); + + /** + * Linkis discovery frequency + */ + public static final CommonVars LINKIS_DISCOVERY_FREQUENCY_PERIOD = CommonVars.apply("wds.exchangis.client.linkis.discovery.frequency-period", 1L); + + /** + * Linkis client load balance + */ + public static final CommonVars LINKIS_LOAD_BALANCER_ENABLED = CommonVars.apply("wds.exchangis.client.linkis.load-balancer.enabled", true); + + + /** + * Linkis client retry + */ + public static final CommonVars LINKIS_RETRY_ENABLED = CommonVars.apply("wds.exchangis.client.linkis.retry.enabled", false); + + /** + * DWS version + */ + public static final CommonVars LINKIS_DWS_VERSION = CommonVars.apply("wds.exchangis.client.linkis.dws.version", Configuration.LINKIS_WEB_VERSION().getValue()); +} diff --git a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/dao/domain/ExchangisJobDsBind.java b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/dao/domain/ExchangisJobDsBind.java index e21695e6d..8b99351ca 100644 --- a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/dao/domain/ExchangisJobDsBind.java +++ b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/dao/domain/ExchangisJobDsBind.java @@ -18,8 +18,18 @@ public class ExchangisJobDsBind { private Long sourceDsId; + /** + * Source data source name + */ + private String sourceDsName; + private Long sinkDsId; + /** + * Sink data source name + */ + private String sinkDsName; + public Long getId() { return id; } @@ -59,4 +69,20 @@ public Long getSinkDsId() { public void setSinkDsId(Long sinkDsId) { this.sinkDsId = sinkDsId; } + + public String getSourceDsName() { + return sourceDsName; + } + + public void setSourceDsName(String sourceDsName) { + this.sourceDsName = sourceDsName; + } + + public String getSinkDsName() { + return sinkDsName; + } + + public void setSinkDsName(String sinkDsName) { + this.sinkDsName = sinkDsName; + } } diff --git a/exchangis-dao/src/main/scala/com/webank/wedatasphere/exchangis/common/linkis/client/ExchangisHttpClient.scala b/exchangis-dao/src/main/scala/com/webank/wedatasphere/exchangis/common/linkis/client/ExchangisHttpClient.scala new file mode 100644 index 000000000..45efe146d --- /dev/null +++ b/exchangis-dao/src/main/scala/com/webank/wedatasphere/exchangis/common/linkis/client/ExchangisHttpClient.scala @@ -0,0 +1,39 @@ +package com.webank.wedatasphere.exchangis.common.linkis.client + +import com.webank.wedatasphere.exchangis.common.linkis.client.config.{ExchangisClientConfig} +import org.apache.http.client.config.RequestConfig +import org.apache.http.impl.client.{CloseableHttpClient, HttpClients} +import org.apache.linkis.httpclient.dws.DWSHttpClient + +import java.util.concurrent.TimeUnit + +/** + * Enhanced http client config + */ +class ExchangisHttpClient(clientConfig: ExchangisClientConfig, clientName: String) + extends DWSHttpClient(clientConfig, clientName){ + /** + * Build http client + */ + override protected val httpClient: CloseableHttpClient = { + val defaultRequestConfig = RequestConfig.custom() + .setConnectTimeout(clientConfig.getConnectTimeout.toInt) + .setConnectionRequestTimeout(clientConfig.getConnReqTimeout.toInt) + .setSocketTimeout(clientConfig.getReadTimeout.toInt) + .build() + val clientBuilder = HttpClients.custom() + clientBuilder.setDefaultRequestConfig(defaultRequestConfig).useSystemProperties() + .setMaxConnPerRoute(clientConfig.getMaxConnection / 2).setMaxConnTotal(clientConfig.getMaxConnection) + val maxIdleTime = clientConfig.getMaxIdleTime + if (maxIdleTime > 0){ + // Evict idle connections + clientBuilder.evictExpiredConnections(); + clientBuilder.evictIdleConnections(maxIdleTime, TimeUnit.MILLISECONDS) + } + clientBuilder.build() + } + + def getHttpClient: CloseableHttpClient = { + httpClient + } +} diff --git a/exchangis-dao/src/main/scala/com/webank/wedatasphere/exchangis/common/linkis/client/config/ExchangisClientConfig.scala b/exchangis-dao/src/main/scala/com/webank/wedatasphere/exchangis/common/linkis/client/config/ExchangisClientConfig.scala new file mode 100644 index 000000000..ab9767ab9 --- /dev/null +++ b/exchangis-dao/src/main/scala/com/webank/wedatasphere/exchangis/common/linkis/client/config/ExchangisClientConfig.scala @@ -0,0 +1,36 @@ +package com.webank.wedatasphere.exchangis.common.linkis.client.config + +import org.apache.linkis.httpclient.config.ClientConfig +import org.apache.linkis.httpclient.dws.config.DWSClientConfig + +/** + * Enhanced dws client config + */ +class ExchangisClientConfig private[config]( + clientConfig: ClientConfig, + maxIdleTime: Long, + connReqTimeout: Long + ) extends DWSClientConfig(clientConfig) { + + /** + * Max idle time + * @return + */ + def getMaxIdleTime: Long = { + maxIdleTime + } + + /** + * Connection request timeout + * @return + */ + def getConnReqTimeout: Long = { + connReqTimeout + } +} + +object ExchangisClientConfig{ + def newBuilder: ExchangisClientConfigBuilder = { + new ExchangisClientConfigBuilder() + } +} \ No newline at end of file diff --git a/exchangis-dao/src/main/scala/com/webank/wedatasphere/exchangis/common/linkis/client/config/ExchangisClientConfigBuilder.scala b/exchangis-dao/src/main/scala/com/webank/wedatasphere/exchangis/common/linkis/client/config/ExchangisClientConfigBuilder.scala new file mode 100644 index 000000000..47df001f4 --- /dev/null +++ b/exchangis-dao/src/main/scala/com/webank/wedatasphere/exchangis/common/linkis/client/config/ExchangisClientConfigBuilder.scala @@ -0,0 +1,59 @@ +package com.webank.wedatasphere.exchangis.common.linkis.client.config + +import com.webank.wedatasphere.exchangis.common.http.HttpClientConfiguration +import com.webank.wedatasphere.exchangis.common.linkis.client.ClientConfiguration +import org.apache.linkis.httpclient.config.{ClientConfig, ClientConfigBuilder} +import org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy + +import java.util.concurrent.TimeUnit + +/** + * Enhanced dws client config builder + */ +class ExchangisClientConfigBuilder extends ClientConfigBuilder{ + + private var maxIdleTime: Long = _ + + private var connReqTimeout: Long = _ + + private var dwsVersion: String = _ + + // Load from vars default + // Http common + maxConnection = HttpClientConfiguration.MAX_CONNECTION_SIZE.getValue + connectTimeout = HttpClientConfiguration.CONNECTION_TIMEOUT.getValue + readTimeout = HttpClientConfiguration.READ_TIMEOUT.getValue + // Linkis client, use token auth default + dwsVersion = ClientConfiguration.LINKIS_DWS_VERSION.getValue + serverUrl = ClientConfiguration.LINKIS_SERVER_URL.getValue + discoveryEnabled = ClientConfiguration.LINKIS_DISCOVERY_ENABLED.getValue + discoveryFrequency(ClientConfiguration.LINKIS_DISCOVERY_FREQUENCY_PERIOD.getValue, TimeUnit.MINUTES) + loadbalancerEnabled = ClientConfiguration.LINKIS_LOAD_BALANCER_ENABLED.getValue + retryEnabled = ClientConfiguration.LINKIS_RETRY_ENABLED.getValue + authenticationStrategy = new TokenAuthenticationStrategy() + authTokenKey = TokenAuthenticationStrategy.TOKEN_KEY + authTokenValue = ClientConfiguration.LINKIS_TOKEN_VALUE.getValue + + def maxIdleTime(maxIdleTime: Long): this.type = { + this.maxIdleTime = maxIdleTime + this + } + + def connReqTimeout(connReqTimeout: Long): this.type = { + this.connReqTimeout = connReqTimeout + this + } + + def setDWSVersion(dwsVersion: String): this.type = { + this.dwsVersion = dwsVersion + this + } + + override def build(): ExchangisClientConfig = { + val clientConfig = new ExchangisClientConfig(super.build(), maxIdleTime, connReqTimeout) + clientConfig.setDWSVersion(dwsVersion) + clientConfig + } + + +} diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/AbstractExchangisDataSourceDefinition.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/AbstractExchangisDataSourceDefinition.java new file mode 100644 index 000000000..c86fa6ef6 --- /dev/null +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/AbstractExchangisDataSourceDefinition.java @@ -0,0 +1,85 @@ +package com.webank.wedatasphere.exchangis.datasource.core; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; +import com.webank.wedatasphere.exchangis.dao.hook.MapperHook; +import com.webank.wedatasphere.exchangis.dao.mapper.ExchangisJobParamConfigMapper; +import com.webank.wedatasphere.exchangis.datasource.core.domain.ExchangisDataSourceType; +import org.apache.commons.lang3.StringUtils; +import org.apache.linkis.datasourcemanager.common.domain.DataSourceType; + +import java.util.List; + +public abstract class AbstractExchangisDataSourceDefinition implements ExchangisDataSourceDefinition { + + /** + * Mapper hook from common module? + */ + protected MapperHook mapperHook; + /** + * Type id + */ + protected String id; + @Override + public String name() { + return type().name; + } + + + @Override + public String classifier() { + return type().classifier; + } + + + @Override + public void setMapperHook(MapperHook mapperHook) { + this.mapperHook = mapperHook; + } + + @Override + public List getDataSourceTypes(String user) { + return ExchangisDataSourceDefinition.super.getDataSourceTypes(user); + } + + @Override + public String id() { + if (null == id || id.equalsIgnoreCase("")) { + List types = getDataSourceTypes("hdfs"); + for (DataSourceType type : types) { + if (type.getName().equalsIgnoreCase(name())) { + this.id = type.getId(); + } + } + } + return this.id; + } + + @Override + public List getDataSourceParamConfigs() { + return getDataSourceParamConfigs(type().name); + } + + protected List getDataSourceParamConfigs(String type) { + return getDataSourceParamConfigs(type, null); + } + + + protected List getDataSourceParamConfigs(String type, String dir) { + ExchangisJobParamConfigMapper exchangisJobParamConfigMapper = this.mapperHook.getExchangisJobParamConfigMapper(); + QueryWrapper queryWrapper = new QueryWrapper<>(); + if (StringUtils.isNotBlank(dir)) { + queryWrapper.eq("config_direction", dir); + } + queryWrapper.eq("type", type); + queryWrapper.eq("is_hidden", 0); + queryWrapper.eq("status", 1); + return exchangisJobParamConfigMapper.selectList(queryWrapper); + } + + /** + * Data source type + * @return type + */ + protected abstract ExchangisDataSourceType type(); +} diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ExchangisDataSource.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ExchangisDataSource.java index 13ecdfdac..2eea07dd7 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ExchangisDataSource.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ExchangisDataSource.java @@ -1,52 +1,19 @@ package com.webank.wedatasphere.exchangis.datasource.core; -import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; -import com.webank.wedatasphere.exchangis.dao.hook.MapperHook; -import org.apache.linkis.datasource.client.impl.LinkisDataSourceRemoteClient; -import org.apache.linkis.datasource.client.impl.LinkisMetaDataRemoteClient; -import org.apache.linkis.datasource.client.request.GetAllDataSourceTypesAction; -import org.apache.linkis.datasource.client.response.GetAllDataSourceTypesResult; -import org.apache.linkis.datasourcemanager.common.domain.DataSourceType; - -import java.util.Collections; -import java.util.List; -import java.util.Objects; - +/** + * Data source basic inf + */ public interface ExchangisDataSource { - String id(); - - String name(); - - String description(); - - String option(); - - String classifier(); -// String type(); - - String structClassifier(); - -// String category(); - - String icon(); - - List getDataSourceParamConfigs(); + /** + * Id + * @return id + */ + Long getId(); - LinkisDataSourceRemoteClient getDataSourceRemoteClient(); + void setId(Long id); - LinkisMetaDataRemoteClient getMetaDataRemoteClient(); - void setMapperHook(MapperHook mapperHook); - default List getDataSourceTypes(String user) { - GetAllDataSourceTypesResult result = getDataSourceRemoteClient().getAllDataSourceTypes(GetAllDataSourceTypesAction.builder() - .setUser(user) - .build() - ); - List allDataSourceType = result.getAllDataSourceType(); - if (Objects.isNull(allDataSourceType)) allDataSourceType = Collections.emptyList(); - return allDataSourceType; - } } diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ExchangisDataSourceConfiguration.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ExchangisDataSourceConfiguration.java index 77e94b12a..782c26977 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ExchangisDataSourceConfiguration.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ExchangisDataSourceConfiguration.java @@ -1,19 +1,28 @@ package com.webank.wedatasphere.exchangis.datasource.core; +import com.webank.wedatasphere.exchangis.common.linkis.client.ClientConfiguration; import org.apache.linkis.common.conf.CommonVars; +/** + * Exchangis data source config + */ public class ExchangisDataSourceConfiguration { - public static final CommonVars SERVER_URL = CommonVars.apply("wds.exchangis.datasource.client.serverurl", ""); - public static final CommonVars CONNECTION_TIMEOUT = CommonVars.apply("wds.exchangis.datasource.client.connection.timeout", 30000L); - public static final CommonVars DISCOVERY_ENABLED = CommonVars.apply("wds.exchangis.datasource.client.discovery.enabled", true); - public static final CommonVars DISCOVERY_FREQUENCY_PERIOD = CommonVars.apply("wds.exchangis.datasource.client.discoveryfrequency.period", 1L); - public static final CommonVars LOAD_BALANCER_ENABLED = CommonVars.apply("wds.exchangis.datasource.client.loadbalancer.enabled", true); - public static final CommonVars MAX_CONNECTION_SIZE = CommonVars.apply("wds.exchangis.datasource.client.maxconnection.size", 5); - public static final CommonVars RETRY_ENABLED = CommonVars.apply("wds.exchangis.datasource.client.retryenabled", false); - public static final CommonVars READ_TIMEOUT = CommonVars.apply("wds.exchangis.datasource.client.readtimeout", 30000L); + /** + * Server url + */ + public static final CommonVars SERVER_URL = CommonVars.apply("wds.exchangis.datasource.client.server-url", + ClientConfiguration.LINKIS_SERVER_URL.getValue()); - public static final CommonVars AUTHTOKEN_KEY = CommonVars.apply("wds.exchangis.datasource.client.authtoken.key", ""); - public static final CommonVars AUTHTOKEN_VALUE = CommonVars.apply("wds.exchangis.datasource.client.authtoken.value", ""); - public static final CommonVars DWS_VERSION = CommonVars.apply("wds.exchangis.datasource.client.dws.version", ""); + /** + * Token value + */ + public static final CommonVars AUTH_TOKEN_VALUE = CommonVars.apply("wds.exchangis.datasource.client.token.value", + ClientConfiguration.LINKIS_TOKEN_VALUE.getValue()); + + /** + * Dws version + */ + public static final CommonVars DWS_VERSION = CommonVars.apply("wds.exchangis.datasource.client.dws.version", + ClientConfiguration.LINKIS_DWS_VERSION.getValue()); } diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ExchangisDataSourceDefinition.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ExchangisDataSourceDefinition.java new file mode 100644 index 000000000..8c6e440c2 --- /dev/null +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ExchangisDataSourceDefinition.java @@ -0,0 +1,75 @@ +package com.webank.wedatasphere.exchangis.datasource.core; + +import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; +import com.webank.wedatasphere.exchangis.dao.hook.MapperHook; +import org.apache.linkis.datasource.client.impl.LinkisDataSourceRemoteClient; +import org.apache.linkis.datasource.client.impl.LinkisMetaDataRemoteClient; +import org.apache.linkis.datasource.client.request.GetAllDataSourceTypesAction; +import org.apache.linkis.datasource.client.response.GetAllDataSourceTypesResult; +import org.apache.linkis.datasourcemanager.common.domain.DataSourceType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * Ds type definition + */ +public interface ExchangisDataSourceDefinition { + + /** + * Type id + * @return + */ + String id(); + + /** + * Type name + * @return name + */ + String name(); + + /** + * Description + * @return desc + */ + String description(); + + String option(); + + String classifier(); + + String structClassifier(); + + String icon(); + + /** + * Parameter config in + * @return + */ + default List getDataSourceParamConfigs(){ + return new ArrayList<>(); + }; + + default LinkisDataSourceRemoteClient getDataSourceRemoteClient(){ + throw new IllegalArgumentException("unsupported to get data source remote client"); + } + + default LinkisMetaDataRemoteClient getMetaDataRemoteClient(){ + throw new IllegalArgumentException("unsupported to get metadata remote client"); + } + + void setMapperHook(MapperHook mapperHook); + + default List getDataSourceTypes(String user) { + GetAllDataSourceTypesResult result = getDataSourceRemoteClient().getAllDataSourceTypes(GetAllDataSourceTypesAction.builder() + .setUser(user) + .build() + ); + + List allDataSourceType = result.getAllDataSourceType(); + if (Objects.isNull(allDataSourceType)) allDataSourceType = Collections.emptyList(); + return allDataSourceType; + } +} diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/context/DefaultExchangisDataSourceContext.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/context/DefaultExchangisDsContext.java similarity index 56% rename from exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/context/DefaultExchangisDataSourceContext.java rename to exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/context/DefaultExchangisDsContext.java index db43c030f..c59ffe36f 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/context/DefaultExchangisDataSourceContext.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/context/DefaultExchangisDsContext.java @@ -2,8 +2,8 @@ import com.google.common.base.Strings; -import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSource; -import com.webank.wedatasphere.exchangis.datasource.core.loader.ExchangisDataSourceLoader; +import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSourceDefinition; +import com.webank.wedatasphere.exchangis.datasource.core.loader.ExchangisDataSourceDefLoader; import java.util.Collection; import java.util.Map; @@ -11,34 +11,34 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -public class DefaultExchangisDataSourceContext implements ExchangisDataSourceContext { +public class DefaultExchangisDsContext implements ExchangisDataSourceContext { - private final Map dataSources = new ConcurrentHashMap<>(24); + private final Map dataSources = new ConcurrentHashMap<>(24); @Override - public boolean registerDataSourceLoader(ExchangisDataSourceLoader loader) { + public boolean registerLoader(ExchangisDataSourceDefLoader loader) { return false; } @Override - public void addExchangisDataSource(ExchangisDataSource dataSource) { + public void addExchangisDsDefinition(ExchangisDataSourceDefinition dataSource) { Objects.requireNonNull(dataSource, "dataSource required"); String name = dataSource.name(); dataSources.put(name, dataSource); } @Override - public ExchangisDataSource removeExchangisDataSource(String type) { + public ExchangisDataSourceDefinition removeExchangisDsDefinition(String type) { return null; } @Override - public ExchangisDataSource updateExchangisDataSource(ExchangisDataSource dataSource) { + public ExchangisDataSourceDefinition updateExchangisDsDefinition(ExchangisDataSourceDefinition dataSource) { return null; } @Override - public ExchangisDataSource getExchangisDataSource(String type) { + public ExchangisDataSourceDefinition getExchangisDsDefinition(String type) { if (Strings.isNullOrEmpty(type)) { return null; } @@ -46,12 +46,12 @@ public ExchangisDataSource getExchangisDataSource(String type) { } @Override - public ExchangisDataSource getExchangisDataSource(Long dataSourceTypeId) { + public ExchangisDataSourceDefinition getExchangisDsDefinition(Long dataSourceTypeId) { if (Objects.isNull(dataSourceTypeId)) { return null; } - Collection values = this.dataSources.values(); - for (ExchangisDataSource ds : values) { + Collection values = this.dataSources.values(); + for (ExchangisDataSourceDefinition ds : values) { if (ds.id().equalsIgnoreCase(dataSourceTypeId+"")) { return ds; } @@ -61,7 +61,7 @@ public ExchangisDataSource getExchangisDataSource(Long dataSourceTypeId) { } @Override - public Collection all() { + public Collection all() { return this.dataSources.values(); } diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/context/ExchangisDataSourceContext.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/context/ExchangisDataSourceContext.java index f15e614a0..c86cf9870 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/context/ExchangisDataSourceContext.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/context/ExchangisDataSourceContext.java @@ -1,27 +1,62 @@ package com.webank.wedatasphere.exchangis.datasource.core.context; -import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSource; -import com.webank.wedatasphere.exchangis.datasource.core.loader.ExchangisDataSourceLoader; +import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSourceDefinition; +import com.webank.wedatasphere.exchangis.datasource.core.loader.ExchangisDataSourceDefLoader; import java.util.Collection; import java.util.Set; +/** + * Data source context + */ public interface ExchangisDataSourceContext { - boolean registerDataSourceLoader(ExchangisDataSourceLoader loader); - - void addExchangisDataSource(ExchangisDataSource dataSource); - - ExchangisDataSource removeExchangisDataSource(String type); - - ExchangisDataSource updateExchangisDataSource(ExchangisDataSource dataSource); - - ExchangisDataSource getExchangisDataSource(String type); - - ExchangisDataSource getExchangisDataSource(Long dataSourceTypeId); - - Collection all(); - + boolean registerLoader(ExchangisDataSourceDefLoader loader); + + /** + * Add ds definition + * @param dataSource ds + */ + void addExchangisDsDefinition(ExchangisDataSourceDefinition dataSource); + + /** + * Remove definition + * @param type type + * @return definition + */ + ExchangisDataSourceDefinition removeExchangisDsDefinition(String type); + + /** + * Update definition + * @param dataSource ds + * @return definition + */ + ExchangisDataSourceDefinition updateExchangisDsDefinition(ExchangisDataSourceDefinition dataSource); + + /** + * Get ds definition + * @param type type + * @return definition + */ + ExchangisDataSourceDefinition getExchangisDsDefinition(String type); + + /** + * Get ds definition + * @param dataSourceTypeId type id + * @return definition + */ + ExchangisDataSourceDefinition getExchangisDsDefinition(Long dataSourceTypeId); + + /** + * All definition + * @return definitions + */ + Collection all(); + + /** + * Type names + * @return set + */ Set keys(); } diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/domain/DataSourceType.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/domain/DataSourceType.java deleted file mode 100644 index d0193c4a2..000000000 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/domain/DataSourceType.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.webank.wedatasphere.exchangis.datasource.core.domain; - -public enum DataSourceType { - - ELASTICSEARCH("ELASTICSEARCH"), - - HIVE("HIVE"), - - MONGODB("MONGODB"), - - MYSQL("MYSQL"), - - SFTP("SFTP"), - - ORACLE("ORACLE"); - - public String name; - - DataSourceType(String name) { - this.name = name; - } -} diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/domain/ExchangisDataSourceType.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/domain/ExchangisDataSourceType.java new file mode 100644 index 000000000..445f3f00b --- /dev/null +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/domain/ExchangisDataSourceType.java @@ -0,0 +1,30 @@ +package com.webank.wedatasphere.exchangis.datasource.core.domain; + +public enum ExchangisDataSourceType { + + ELASTICSEARCH("ELASTICSEARCH", "分布式全文索引"), + + HIVE("HIVE", "大数据存储"), + + MONGODB("MONGODB", "非关系型数据库"), + + MYSQL("MYSQL", "关系型数据库"), + + SFTP("SFTP", "sftp连接"), + + ORACLE("ORACLE", "关系型数据库"); + + /** + * Type name + */ + public String name; + + /** + * Classifier + */ + public String classifier; + ExchangisDataSourceType(String name, String classifier) { + this.name = name; + this.classifier = classifier; + } +} diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/loader/ExchangisDataSourceLoader.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/loader/ExchangisDataSourceDefLoader.java similarity index 81% rename from exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/loader/ExchangisDataSourceLoader.java rename to exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/loader/ExchangisDataSourceDefLoader.java index f68f696e8..a1fdc8a34 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/loader/ExchangisDataSourceLoader.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/loader/ExchangisDataSourceDefLoader.java @@ -2,13 +2,13 @@ import com.webank.wedatasphere.exchangis.dao.hook.MapperHook; -import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSource; +import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSourceDefinition; import com.webank.wedatasphere.exchangis.datasource.core.context.ExchangisDataSourceContext; import org.apache.linkis.common.conf.CommonVars; import java.util.Objects; -public interface ExchangisDataSourceLoader { +public interface ExchangisDataSourceDefLoader { String EXCHANGIS_DIR_NAME = Objects.isNull(CommonVars.apply("wds.exchangis.datasource.extension.dir").getValue()) ? "exchangis-extds" : CommonVars.apply("wds.exchangis.datasource.extension.dir").getValue().toString(); @@ -26,8 +26,8 @@ public interface ExchangisDataSourceLoader { void init(MapperHook mapperHook) throws Exception; - ExchangisDataSource load(String dataSourceType); + ExchangisDataSourceDefinition load(String dataSourceType); - ExchangisDataSource get(String dataSourceType, boolean reload); + ExchangisDataSourceDefinition get(String dataSourceType, boolean reload); } diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobDataSourcesContent.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobDataSourcesContent.java index fc1618318..36af7634f 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobDataSourcesContent.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobDataSourcesContent.java @@ -1,17 +1,38 @@ package com.webank.wedatasphere.exchangis.datasource.core.vo; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.webank.wedatasphere.exchangis.datasource.core.domain.ExchangisDataSourceType; +/** + * string: HIVE.ID.DB.TABLE + * json: { + * "type": "HIVE", + * "id": 467, + * "name": "HIVE-DEMO", + * "database": "default", + * "table": "demo-test" + * } + */ public class ExchangisJobDataSourcesContent { - // 唯一标识符 - // HIVE.ID.DB.TABLE @JsonProperty("source_id") private String sourceId; + /** + * Source ds + */ + private ExchangisJobDataSource source = new ExchangisJobDataSource(); + + @JsonProperty("sink_id") private String sinkId; + /** + * Sink ds + */ + private ExchangisJobDataSource sink = new ExchangisJobDataSource(); + public String getSourceId() { return sourceId; } @@ -27,4 +48,102 @@ public String getSinkId() { public void setSinkId(String sinkId) { this.sinkId = sinkId; } + + public void setSource(ExchangisJobDataSource source) { + this.source = source; + } + + public ExchangisJobDataSource getSource() { + return source; + } + + public void setSink(ExchangisJobDataSource sink) { + this.sink = sink; + } + + public ExchangisJobDataSource getSink() { + return sink; + } + + @JsonInclude(JsonInclude.Include.NON_EMPTY) + public static class ExchangisJobDataSource { + + /** + * Data source type + */ + private ExchangisDataSourceType type; + + /** + * Data source id + */ + private String id; + + /** + * Data source name + */ + private String name; + + /** + * Database field + */ + private String database; + + /** + * Table field + */ + private String table; + + /** + * Uri field + */ + private String uri; + + public void setType(ExchangisDataSourceType type) { + this.type = type; + } + + public ExchangisDataSourceType getType() { + return type; + } + + public void setId(String id) { + this.id = id; + } + + public String getId() { + return id; + } + + public void setName(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public void setDatabase(String database) { + this.database = database; + } + + public String getDatabase() { + return database; + } + + public void setTable(String table) { + this.table = table; + } + + public String getTable() { + return table; + } + + public void setUri(String uri) { + this.uri = uri; + } + + public String getUri() { + return uri; + } + } } diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobInfoContent.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobInfoContent.java index 25ef70e09..eae71c5a6 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobInfoContent.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobInfoContent.java @@ -7,17 +7,35 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class ExchangisJobInfoContent { + /** + * Engine name + */ private String engine; + /** + * Sub job name + */ private String subJobName; + /** + * Data source content + */ private ExchangisJobDataSourcesContent dataSources; + /** + * Extra params + */ private ExchangisJobParamsContent params; + /** + * Transform define + */ // private List transforms; private ExchangisJobTransformsContent transforms; + /** + * Settings + */ private List settings; public String getEngine() { diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobParamsContent.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobParamsContent.java index ef8dbcb57..64d3a6106 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobParamsContent.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobParamsContent.java @@ -1,12 +1,20 @@ package com.webank.wedatasphere.exchangis.datasource.core.vo; +import com.fasterxml.jackson.annotation.JsonAlias; import com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; public class ExchangisJobParamsContent { + /** + * Source params + */ private List sources; + + /** + * Sink params + */ private List sinks; public List getSources() { @@ -26,13 +34,17 @@ public void setSinks(List sinks) { } public static class ExchangisJobParamsItem { + @JsonProperty("config_key") + @JsonAlias({"key", "k"}) private String configKey; @JsonProperty("config_name") + @JsonAlias({"name", "n"}) private String configName; @JsonProperty("config_value") + @JsonAlias({"value", "v"}) private Object configValue; private Integer sort; diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobTransformsItem.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobTransformsItem.java index d0ae8ede0..10ddad5a6 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobTransformsItem.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobTransformsItem.java @@ -2,24 +2,56 @@ import java.util.List; +import com.fasterxml.jackson.annotation.JsonAlias; import com.fasterxml.jackson.annotation.JsonProperty; public class ExchangisJobTransformsItem { + /** + * Source field name + */ @JsonProperty("source_field_name") + @JsonAlias({"srcFieldName"}) private String sourceFieldName; + + /** + * Source field type + */ @JsonProperty("source_field_type") + @JsonAlias({"srcFieldType"}) private String sourceFieldType; + + /** + * Sink field name + */ @JsonProperty("sink_field_name") + @JsonAlias({"sinkFieldName"}) private String sinkFieldName; + + /** + * Sink field type + */ @JsonProperty("sink_field_type") + @JsonAlias({"sinkFieldType"}) private String sinkFieldType; + + /** + * Delete enable + */ @JsonProperty("deleteEnable") private boolean deleteEnable; + /** + * Source field index + */ @JsonProperty("source_field_index") + @JsonAlias({"srcFieldIdx"}) private Integer sourceFieldIndex; + /** + * Sink field index + */ @JsonProperty("sink_field_index") + @JsonAlias({"sinkFi"}) private Integer sinkFieldIndex; @JsonProperty("source_field_editable") diff --git a/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/ExchangisBatchDataSource.java b/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/ExchangisBatchDataSource.java index 38058e58c..277eee047 100644 --- a/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/ExchangisBatchDataSource.java +++ b/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/ExchangisBatchDataSource.java @@ -1,44 +1,14 @@ package com.webank.wedatasphere.exchangis.datasource.linkis; -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; -import com.webank.wedatasphere.exchangis.dao.hook.MapperHook; -import com.webank.wedatasphere.exchangis.dao.mapper.ExchangisJobParamConfigMapper; -import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSource; +import com.webank.wedatasphere.exchangis.datasource.core.AbstractExchangisDataSourceDefinition; import org.apache.linkis.datasource.client.impl.LinkisDataSourceRemoteClient; import org.apache.linkis.datasource.client.impl.LinkisMetaDataRemoteClient; -import org.apache.linkis.datasourcemanager.common.domain.DataSourceType; -import java.util.List; +/** + * Batch data source + */ +public abstract class ExchangisBatchDataSource extends AbstractExchangisDataSourceDefinition { -public abstract class ExchangisBatchDataSource implements ExchangisDataSource { - - protected MapperHook mapperHook; - protected String id; - - @Override - public void setMapperHook(MapperHook mapperHook) { - this.mapperHook = mapperHook; - } - - protected List getDataSourceParamConfigs(String type) { - ExchangisJobParamConfigMapper exchangisJobParamConfigMapper = this.mapperHook.getExchangisJobParamConfigMapper(); - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.eq("type", type); - queryWrapper.eq("is_hidden", 0); - queryWrapper.eq("status", 1); - return exchangisJobParamConfigMapper.selectList(queryWrapper); - } - - protected List getDataSourceParamConfigs(String type, String dir) { - ExchangisJobParamConfigMapper exchangisJobParamConfigMapper = this.mapperHook.getExchangisJobParamConfigMapper(); - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.eq("config_direction", dir); - queryWrapper.eq("type", type); - queryWrapper.eq("is_hidden", 0); - queryWrapper.eq("status", 1); - return exchangisJobParamConfigMapper.selectList(queryWrapper); - } @Override public LinkisDataSourceRemoteClient getDataSourceRemoteClient() { @@ -50,16 +20,4 @@ public LinkisMetaDataRemoteClient getMetaDataRemoteClient() { return ExchangisLinkisRemoteClient.getLinkisMetadataRemoteClient(); } - @Override - public String id() { - if (null == id || id.equalsIgnoreCase("")) { - List types = getDataSourceTypes("hdfs"); - for (DataSourceType type : types) { - if (type.getName().equalsIgnoreCase(name())) { - this.id = type.getId(); - } - } - } - return this.id; - } } diff --git a/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/ExchangisDataSourceConfiguration.java b/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/ExchangisDataSourceConfiguration.java deleted file mode 100644 index ad2e1deab..000000000 --- a/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/ExchangisDataSourceConfiguration.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.webank.wedatasphere.exchangis.datasource.linkis; - - -import org.apache.linkis.common.conf.CommonVars; - -public class ExchangisDataSourceConfiguration { - public static final CommonVars SERVER_URL = CommonVars.apply("wds.exchangis.datasource.client.serverurl", ""); - public static final CommonVars CONNECTION_TIMEOUT = CommonVars.apply("wds.exchangis.datasource.client.connection.timeout", 30000L); - public static final CommonVars DISCOVERY_ENABLED = CommonVars.apply("wds.exchangis.datasource.client.discovery.enabled", false); - public static final CommonVars DISCOVERY_FREQUENCY_PERIOD = CommonVars.apply("wds.exchangis.datasource.client.discoveryfrequency.period", 1L); - public static final CommonVars LOAD_BALANCER_ENABLED = CommonVars.apply("wds.exchangis.datasource.client.loadbalancer.enabled", true); - public static final CommonVars MAX_CONNECTION_SIZE = CommonVars.apply("wds.exchangis.datasource.client.maxconnection.size", 5); - public static final CommonVars RETRY_ENABLED = CommonVars.apply("wds.exchangis.datasource.client.retryenabled", false); - public static final CommonVars READ_TIMEOUT = CommonVars.apply("wds.exchangis.datasource.client.readtimeout", 30000L); - - public static final CommonVars AUTHTOKEN_KEY = CommonVars.apply("wds.exchangis.datasource.client.authtoken.key", ""); - public static final CommonVars AUTHTOKEN_VALUE = CommonVars.apply("wds.exchangis.datasource.client.authtoken.value", ""); - public static final CommonVars DWS_VERSION = CommonVars.apply("wds.exchangis.datasource.client.dws.version", ""); -} diff --git a/exchangis-datasource/exchangis-datasource-linkis/src/main/scala/com/webank/wedatasphere/exchangis/datasource/linkis/ExchangisLinkisRemoteClient.scala b/exchangis-datasource/exchangis-datasource-linkis/src/main/scala/com/webank/wedatasphere/exchangis/datasource/linkis/ExchangisLinkisRemoteClient.scala index a8126a04c..c3d4d242b 100644 --- a/exchangis-datasource/exchangis-datasource-linkis/src/main/scala/com/webank/wedatasphere/exchangis/datasource/linkis/ExchangisLinkisRemoteClient.scala +++ b/exchangis-datasource/exchangis-datasource-linkis/src/main/scala/com/webank/wedatasphere/exchangis/datasource/linkis/ExchangisLinkisRemoteClient.scala @@ -1,62 +1,36 @@ package com.webank.wedatasphere.exchangis.datasource.linkis -import com.webank.wedatasphere.exchangis.common.linkis.ClientConfiguration +import com.webank.wedatasphere.exchangis.common.linkis.client.{ClientConfiguration, ExchangisHttpClient} +import com.webank.wedatasphere.exchangis.common.linkis.client.config.ExchangisClientConfig +import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSourceConfiguration +import com.webank.wedatasphere.exchangis.datasource.core.exception.{ExchangisDataSourceException, ExchangisDataSourceExceptionCode} +import org.apache.commons.lang3.StringUtils +import org.apache.linkis.datasource.client.config.DatasourceClientConfig.DATA_SOURCE_SERVICE_CLIENT_NAME import org.apache.linkis.datasource.client.impl.{LinkisDataSourceRemoteClient, LinkisMetaDataRemoteClient} import org.apache.linkis.datasource.client.request._ import org.apache.linkis.datasource.client.response._ import org.apache.linkis.datasourcemanager.common.domain.{DataSource, DataSourceType} -import org.apache.linkis.httpclient.dws.authentication.{StaticAuthenticationStrategy, TokenAuthenticationStrategy} -import org.apache.linkis.httpclient.dws.config.{DWSClientConfig, DWSClientConfigBuilder} +import org.apache.linkis.httpclient.dws.DWSHttpClient -import java.lang -import java.util.concurrent.TimeUnit object ExchangisLinkisRemoteClient { + //Linkis Datasource Client Config - val serverUrl: String = ClientConfiguration.LINKIS_SERVER_URL.getValue - val authTokenValue: String = ClientConfiguration.LINKIS_TOKEN_VALUE.getValue - val connectionTimeout: lang.Long = ExchangisDataSourceConfiguration.CONNECTION_TIMEOUT.getValue - val discoveryEnabled: lang.Boolean = ExchangisDataSourceConfiguration.DISCOVERY_ENABLED.getValue - val discoveryFrequencyPeriod: lang.Long = ExchangisDataSourceConfiguration.DISCOVERY_FREQUENCY_PERIOD.getValue - val loadbalancerEnabled: lang.Boolean = ExchangisDataSourceConfiguration.LOAD_BALANCER_ENABLED.getValue - val maxConnectionSize: Integer = ExchangisDataSourceConfiguration.MAX_CONNECTION_SIZE.getValue - val retryEnabled: lang.Boolean = ExchangisDataSourceConfiguration.RETRY_ENABLED.getValue - val readTimeout: lang.Long = ExchangisDataSourceConfiguration.READ_TIMEOUT.getValue - val dwsVersion: String = ExchangisDataSourceConfiguration.DWS_VERSION.getValue - - - // val clientConfig = DWSClientConfigBuilder.newBuilder() - // .addServerUrl(serverUrl) - // .connectionTimeout(connectionTimeout) - // .discoveryEnabled(discoveryEnabled) - // .discoveryFrequency(1,TimeUnit.MINUTES) - // .loadbalancerEnabled(loadbalancerEnabled) - // .maxConnectionSize(maxConnectionSize) - // .retryEnabled(retryEnabled) - // .readTimeout(readTimeout) - // .setAuthenticationStrategy(new StaticAuthenticationStrategy()) - // .setAuthTokenKey(authTokenKey) - // .setAuthTokenValue(authTokenValue) - // .setDWSVersion(dwsVersion) - // .build() - - val clientConfig: DWSClientConfig = DWSClientConfigBuilder.newBuilder() - .addServerUrl(serverUrl) - .connectionTimeout(connectionTimeout) - .discoveryEnabled(discoveryEnabled) - .discoveryFrequency(discoveryFrequencyPeriod, TimeUnit.MINUTES) - .loadbalancerEnabled(loadbalancerEnabled) - .maxConnectionSize(maxConnectionSize) - .retryEnabled(retryEnabled) - .readTimeout(readTimeout) - .setAuthenticationStrategy(new TokenAuthenticationStrategy()) - .setAuthTokenValue(authTokenValue) - .setDWSVersion(dwsVersion) + val clientConfig: ExchangisClientConfig = ExchangisClientConfig.newBuilder + .addServerUrl(ExchangisDataSourceConfiguration.SERVER_URL.getValue) + .setAuthTokenValue(ExchangisDataSourceConfiguration.AUTH_TOKEN_VALUE.getValue) + .setDWSVersion(ExchangisDataSourceConfiguration.DWS_VERSION.getValue) .build() - val dataSourceClient = new LinkisDataSourceRemoteClient(clientConfig) + /** + * Data source client + */ + val dataSourceClient = new ExchangisDataSourceClient(clientConfig, null) - val metaDataClient = new LinkisMetaDataRemoteClient(clientConfig) + /** + * Meta data client + */ + val metaDataClient = new ExchangisMetadataClient(clientConfig) def getLinkisDataSourceRemoteClient: LinkisDataSourceRemoteClient = { dataSourceClient @@ -83,10 +57,6 @@ object ExchangisLinkisRemoteClient { ) } -// def createDataSource() = { -// dataSourceClient.execute().asInstanceOf[] -// } - /** * get datasourceConnect information * @@ -172,3 +142,27 @@ object ExchangisLinkisRemoteClient { } + +/** + * Exchangis data source client + * @param clientConfig client config + * @param clientName client name + */ +class ExchangisDataSourceClient(clientConfig: ExchangisClientConfig, clientName: String) extends LinkisDataSourceRemoteClient(clientConfig, clientName){ + + protected override val dwsHttpClient: DWSHttpClient = { + val client = if (StringUtils.isEmpty(clientName)) DATA_SOURCE_SERVICE_CLIENT_NAME.getValue else clientName + Option(clientConfig) match { + case Some(config) => new ExchangisHttpClient(config, client) + case _ => throw new ExchangisDataSourceException(ExchangisDataSourceExceptionCode.PARAMETER_INVALID.getCode, "Linkis client config cannot be null") + } + } +} + +/** + * Exchangis meta data client + * @param clientConfig client config + */ +class ExchangisMetadataClient(clientConfig: ExchangisClientConfig) extends LinkisMetaDataRemoteClient(clientConfig){ + protected override val dwsHttpClient: DWSHttpClient = new ExchangisHttpClient(clientConfig, "MetaData-Client") +} diff --git a/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/loader/ExchangisDataSourceLoaderFactory.java b/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/loader/ExchangisDataSourceLoaderFactory.java index 58d6c29dc..979ca9a86 100644 --- a/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/loader/ExchangisDataSourceLoaderFactory.java +++ b/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/loader/ExchangisDataSourceLoaderFactory.java @@ -1,6 +1,6 @@ package com.webank.wedatasphere.exchangis.datasource.loader.loader; -import com.webank.wedatasphere.exchangis.datasource.core.loader.ExchangisDataSourceLoader; +import com.webank.wedatasphere.exchangis.datasource.core.loader.ExchangisDataSourceDefLoader; import org.apache.commons.lang.ClassUtils; import org.apache.commons.lang.StringUtils; import org.apache.linkis.common.conf.CommonVars; @@ -11,13 +11,13 @@ public class ExchangisDataSourceLoaderFactory { private static final Logger logger = LoggerFactory.getLogger(ExchangisDataSourceLoaderFactory.class); - private static Class clazz = LocalExchangisDataSourceLoader.class; - private static ExchangisDataSourceLoader exchangisDataSourceLoader = null; + private static Class clazz = LocalExchangisDataSourceLoader.class; + private static ExchangisDataSourceDefLoader exchangisDataSourceDefLoader = null; - public static ExchangisDataSourceLoader getLoader(){ - if (exchangisDataSourceLoader == null){ + public static ExchangisDataSourceDefLoader getLoader(){ + if (exchangisDataSourceDefLoader == null){ synchronized (ExchangisDataSourceLoaderFactory.class){ - if (exchangisDataSourceLoader == null){ + if (exchangisDataSourceDefLoader == null){ // 可以通过配置自行加载对应的类 CommonVars apply = CommonVars.apply("exchangis.extds.loader.classname", ""); String className = apply.getValue(); @@ -29,7 +29,7 @@ public static ExchangisDataSourceLoader getLoader(){ } } try { - exchangisDataSourceLoader = clazz.newInstance(); + exchangisDataSourceDefLoader = clazz.newInstance(); } catch (Exception e) { logger.error(String.format("Can not initialize ExchangisDataSourceLoader class %s.", clazz.getSimpleName()), e); } @@ -37,7 +37,7 @@ public static ExchangisDataSourceLoader getLoader(){ } } } - return exchangisDataSourceLoader; + return exchangisDataSourceDefLoader; } } diff --git a/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/loader/LocalExchangisDataSourceLoader.java b/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/loader/LocalExchangisDataSourceLoader.java index 50f6f59e3..18d843f09 100644 --- a/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/loader/LocalExchangisDataSourceLoader.java +++ b/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/loader/LocalExchangisDataSourceLoader.java @@ -1,9 +1,9 @@ package com.webank.wedatasphere.exchangis.datasource.loader.loader; import com.webank.wedatasphere.exchangis.dao.hook.MapperHook; -import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSource; +import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSourceDefinition; import com.webank.wedatasphere.exchangis.datasource.core.context.ExchangisDataSourceContext; -import com.webank.wedatasphere.exchangis.datasource.core.loader.ExchangisDataSourceLoader; +import com.webank.wedatasphere.exchangis.datasource.core.loader.ExchangisDataSourceDefLoader; import com.webank.wedatasphere.exchangis.datasource.loader.clazzloader.ExchangisDataSourceClassLoader; import com.webank.wedatasphere.exchangis.datasource.loader.utils.ExceptionHelper; import com.webank.wedatasphere.exchangis.datasource.loader.utils.ExtDsUtils; @@ -17,7 +17,7 @@ import java.util.List; import java.util.Objects; -public class LocalExchangisDataSourceLoader implements ExchangisDataSourceLoader { +public class LocalExchangisDataSourceLoader implements ExchangisDataSourceDefLoader { private static final Logger LOGGER = LoggerFactory.getLogger(LocalExchangisDataSourceLoader.class); @@ -61,24 +61,24 @@ public void init(MapperHook mapperHook) throws Exception { if (clazz == null) { Thread.currentThread().setContextClassLoader(currentClassLoader); } else { - ExchangisDataSource exchangisDataSource = (ExchangisDataSource) clazz.newInstance(); - exchangisDataSource.setMapperHook(mapperHook); + ExchangisDataSourceDefinition dsType = (ExchangisDataSourceDefinition) clazz.newInstance(); + dsType.setMapperHook(mapperHook); Thread.currentThread().setContextClassLoader(currentClassLoader); - LOGGER.info("ExchangisDataSource is {}", exchangisDataSource.getClass().toString()); + LOGGER.info("ExchangisDataSource is {}", dsType.getClass().toString()); - context.addExchangisDataSource(exchangisDataSource); + context.addExchangisDsDefinition(dsType); } } } @Override - public ExchangisDataSource load(String dataSourceType) { + public ExchangisDataSourceDefinition load(String dataSourceType) { return null; } @Override - public ExchangisDataSource get(String dataSourceType, boolean reload) { + public ExchangisDataSourceDefinition get(String dataSourceType, boolean reload) { return null; } } diff --git a/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/utils/ExtDsUtils.java b/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/utils/ExtDsUtils.java index 69db1da10..59b40ea66 100644 --- a/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/utils/ExtDsUtils.java +++ b/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/utils/ExtDsUtils.java @@ -18,8 +18,8 @@ package com.webank.wedatasphere.exchangis.datasource.loader.utils; -import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSource; -import com.webank.wedatasphere.exchangis.datasource.core.loader.ExchangisDataSourceLoader; +import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSourceDefinition; +import com.webank.wedatasphere.exchangis.datasource.core.loader.ExchangisDataSourceDefLoader; import com.webank.wedatasphere.exchangis.datasource.loader.exception.NoSuchExchangisExtDataSourceException; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; @@ -40,7 +40,7 @@ public class ExtDsUtils { private static final Logger logger = LoggerFactory.getLogger(ExtDsUtils.class); - private static Class PARENT_CLASS = ExchangisDataSource.class; + private static Class PARENT_CLASS = ExchangisDataSourceDefinition.class; public static String getExchangisExtDataSourceClassName(String libPath, @@ -99,7 +99,7 @@ public static List getJarsOfPath(String path) { if (file.listFiles() != null) { for (File f : file.listFiles()) { // exchangis-xxxxx.jar - if (!f.isDirectory() && f.getName().endsWith(ExchangisDataSourceLoader.JAR_SUF_NAME) && f.getName().startsWith("exchangis")) { + if (!f.isDirectory() && f.getName().endsWith(ExchangisDataSourceDefLoader.JAR_SUF_NAME) && f.getName().startsWith("exchangis")) { jars.add(f.getPath()); } } @@ -115,11 +115,11 @@ public static List getJarsUrlsOfPath(String path) { List jars = new ArrayList<>(); if (file.listFiles() != null) { for (File f : file.listFiles()) { - if (!f.isDirectory() && f.getName().endsWith(ExchangisDataSourceLoader.JAR_SUF_NAME)) { + if (!f.isDirectory() && f.getName().endsWith(ExchangisDataSourceDefLoader.JAR_SUF_NAME)) { try { jars.add(f.toURI().toURL()); } catch (MalformedURLException e) { - logger.warn("url {} cannot be added", ExchangisDataSourceLoader.FILE_SCHEMA + f.getPath()); + logger.warn("url {} cannot be added", ExchangisDataSourceDefLoader.FILE_SCHEMA + f.getPath()); } } } diff --git a/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/configuration/ServerConfig.java b/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/configuration/ServerConfig.java index 85a71ef80..5f9809e85 100644 --- a/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/configuration/ServerConfig.java +++ b/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/configuration/ServerConfig.java @@ -2,9 +2,9 @@ import com.webank.wedatasphere.exchangis.dao.hook.MapperHook; -import com.webank.wedatasphere.exchangis.datasource.core.context.DefaultExchangisDataSourceContext; +import com.webank.wedatasphere.exchangis.datasource.core.context.DefaultExchangisDsContext; import com.webank.wedatasphere.exchangis.datasource.core.context.ExchangisDataSourceContext; -import com.webank.wedatasphere.exchangis.datasource.core.loader.ExchangisDataSourceLoader; +import com.webank.wedatasphere.exchangis.datasource.core.loader.ExchangisDataSourceDefLoader; import com.webank.wedatasphere.exchangis.datasource.loader.loader.ExchangisDataSourceLoaderFactory; import org.apache.linkis.common.exception.ErrorException; import org.springframework.context.annotation.Bean; @@ -15,8 +15,8 @@ public class ServerConfig { @Bean public ExchangisDataSourceContext context(MapperHook mapperHook) throws Exception { - DefaultExchangisDataSourceContext context = new DefaultExchangisDataSourceContext(); - ExchangisDataSourceLoader loader = ExchangisDataSourceLoaderFactory.getLoader(); + DefaultExchangisDsContext context = new DefaultExchangisDsContext(); + ExchangisDataSourceDefLoader loader = ExchangisDataSourceLoaderFactory.getLoader(); loader.setContext(context); try { loader.init(mapperHook); diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/domain/ExchangisDsProject.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/domain/ExchangisDsProject.java new file mode 100644 index 000000000..322424cb9 --- /dev/null +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/domain/ExchangisDsProject.java @@ -0,0 +1,8 @@ +package com.webank.wedatasphere.exchangis.datasource.domain; + +/** + * The relation between data source and project + */ +public class ExchangisDsProject { +// private String +} diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/DataSourceDTO.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/DataSourceDTO.java deleted file mode 100644 index 2ee15e9ed..000000000 --- a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/DataSourceDTO.java +++ /dev/null @@ -1,169 +0,0 @@ -package com.webank.wedatasphere.exchangis.datasource.dto; - -import java.util.Date; - -public class DataSourceDTO { - - private Long id; - private String name; - private String type; - private Long dataSourceTypeId; - private String createIdentify; - private String createSystem; - private String desc; - private String createUser; - private String labels; - private String label; - private Long versionId; - private String modifyUser; - private Date modifyTime; - private boolean expire; - private boolean writeAble; - private boolean readAble; - private String authDbs; - private String authTbls; - - public boolean isExpire() { - return expire; - } - - public void setExpire(boolean expire) { - this.expire = expire; - } - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public String getCreateIdentify() { - return createIdentify; - } - - public void setCreateIdentify(String createIdentify) { - this.createIdentify = createIdentify; - } - - public Long getDataSourceTypeId() { - return dataSourceTypeId; - } - - public void setDataSourceTypeId(Long dataSourceTypeId) { - this.dataSourceTypeId = dataSourceTypeId; - } - - public String getDesc() { - return desc; - } - - public void setDesc(String desc) { - this.desc = desc; - } - - public String getCreateUser() { - return createUser; - } - - public void setCreateUser(String createUser) { - this.createUser = createUser; - } - - public String getLabels() { - return labels; - } - - public void setLabels(String labels) { - this.labels = labels; - } - - public String getLabel() { - return label; - } - - public void setLabel(String label) { - this.label = label; - } - - public Long getVersionId() { - return versionId; - } - - public void setVersionId(Long versionId) { - this.versionId = versionId; - } - - public String getModifyUser() { - return modifyUser; - } - - public void setModifyUser(String modifyUser) { - this.modifyUser = modifyUser; - } - - public Date getModifyTime() { - return modifyTime; - } - - public void setModifyTime(Date modifyTime) { - this.modifyTime = modifyTime; - } - - public String getCreateSystem() { - return createSystem; - } - - public void setCreateSystem(String createSystem) { - this.createSystem = createSystem; - } - - public boolean isWriteAble() { - return writeAble; - } - - public void setWriteAble(boolean writeAble) { - this.writeAble = writeAble; - } - - public boolean isReadAble() { - return readAble; - } - - public void setReadAble(boolean readAble) { - this.readAble = readAble; - } - - public String getAuthDbs() { - return authDbs; - } - - public void setAuthDbs(String authDbs) { - this.authDbs = authDbs; - } - - public String getAuthTbls() { - return authTbls; - } - - public void setAuthTbls(String authTbls) { - this.authTbls = authTbls; - } -} diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/ExchangisDataSourceDTO.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/ExchangisDataSourceDTO.java index b6a2fff3a..4f2c06fe3 100644 --- a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/ExchangisDataSourceDTO.java +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/ExchangisDataSourceDTO.java @@ -1,62 +1,169 @@ package com.webank.wedatasphere.exchangis.datasource.dto; +import java.util.Date; + public class ExchangisDataSourceDTO { - private final String id; - private final String classifier; - private final String name; - private String option; - private String description; - private String icon; - private String struct_classifier; - - public ExchangisDataSourceDTO(String id, String classifier, String name, String struct_classifier) { - this.id = id; - this.classifier = classifier; - this.name = name; - this.struct_classifier = struct_classifier; + + private Long id; + private String name; + private String type; + private Long dataSourceTypeId; + private String createIdentify; + private String createSystem; + private String desc; + private String createUser; + private String labels; + private String label; + private Long versionId; + private String modifyUser; + private Date modifyTime; + private boolean expire; + private boolean writeAble; + private boolean readAble; + private String authDbs; + private String authTbls; + + public boolean isExpire() { + return expire; + } + + public void setExpire(boolean expire) { + this.expire = expire; } - public String getId() { + public Long getId() { return id; } - public String getClassifier() { - return classifier; + public void setId(Long id) { + this.id = id; } public String getName() { return name; } - public String getOption() { - return option; + public void setName(String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getCreateIdentify() { + return createIdentify; + } + + public void setCreateIdentify(String createIdentify) { + this.createIdentify = createIdentify; + } + + public Long getDataSourceTypeId() { + return dataSourceTypeId; + } + + public void setDataSourceTypeId(Long dataSourceTypeId) { + this.dataSourceTypeId = dataSourceTypeId; + } + + public String getDesc() { + return desc; + } + + public void setDesc(String desc) { + this.desc = desc; + } + + public String getCreateUser() { + return createUser; + } + + public void setCreateUser(String createUser) { + this.createUser = createUser; + } + + public String getLabels() { + return labels; + } + + public void setLabels(String labels) { + this.labels = labels; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + public Long getVersionId() { + return versionId; + } + + public void setVersionId(Long versionId) { + this.versionId = versionId; + } + + public String getModifyUser() { + return modifyUser; + } + + public void setModifyUser(String modifyUser) { + this.modifyUser = modifyUser; + } + + public Date getModifyTime() { + return modifyTime; + } + + public void setModifyTime(Date modifyTime) { + this.modifyTime = modifyTime; + } + + public String getCreateSystem() { + return createSystem; + } + + public void setCreateSystem(String createSystem) { + this.createSystem = createSystem; + } + + public boolean isWriteAble() { + return writeAble; } - public void setOption(String option) { - this.option = option; + public void setWriteAble(boolean writeAble) { + this.writeAble = writeAble; } - public void setDescription(String description) { - this.description = description; + public boolean isReadAble() { + return readAble; } - public void setIcon(String icon) { - this.icon = icon; + public void setReadAble(boolean readAble) { + this.readAble = readAble; } - public String getDescription() { - return description; + public String getAuthDbs() { + return authDbs; } - public String getIcon() { - return icon; + public void setAuthDbs(String authDbs) { + this.authDbs = authDbs; } - public String getStruct_classifier() { - return struct_classifier; + public String getAuthTbls() { + return authTbls; } - public void setStruct_classifier(String struct_classifier) { - this.struct_classifier = struct_classifier; + public void setAuthTbls(String authTbls) { + this.authTbls = authTbls; } } diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/ExchangisDataSourceDefDTO.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/ExchangisDataSourceDefDTO.java new file mode 100644 index 000000000..c71f60fd9 --- /dev/null +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/ExchangisDataSourceDefDTO.java @@ -0,0 +1,62 @@ +package com.webank.wedatasphere.exchangis.datasource.dto; + +public class ExchangisDataSourceDefDTO { + private final String id; + private final String classifier; + private final String name; + private String option; + private String description; + private String icon; + private String struct_classifier; + + public ExchangisDataSourceDefDTO(String id, String classifier, String name, String struct_classifier) { + this.id = id; + this.classifier = classifier; + this.name = name; + this.struct_classifier = struct_classifier; + } + + public String getId() { + return id; + } + + public String getClassifier() { + return classifier; + } + + public String getName() { + return name; + } + + public String getOption() { + return option; + } + + public void setOption(String option) { + this.option = option; + } + + public void setDescription(String description) { + this.description = description; + } + + public void setIcon(String icon) { + this.icon = icon; + } + + public String getDescription() { + return description; + } + + public String getIcon() { + return icon; + } + + public String getStruct_classifier() { + return struct_classifier; + } + + public void setStruct_classifier(String struct_classifier) { + this.struct_classifier = struct_classifier; + } +} diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/AbstractDataSourceService.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/AbstractDataSourceService.java index 8688554be..f4ebaab69 100644 --- a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/AbstractDataSourceService.java +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/AbstractDataSourceService.java @@ -8,7 +8,7 @@ import com.webank.wedatasphere.exchangis.common.UserUtils; import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; import com.webank.wedatasphere.exchangis.dao.mapper.ExchangisJobParamConfigMapper; -import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSource; +import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSourceDefinition; import com.webank.wedatasphere.exchangis.datasource.core.context.ExchangisDataSourceContext; import com.webank.wedatasphere.exchangis.datasource.core.ui.*; import com.webank.wedatasphere.exchangis.datasource.core.ui.viewer.DefaultDataSourceUIViewer; @@ -84,7 +84,7 @@ private ExchangisDataSourceIdsUI buildDataSourceIdsUI(HttpServletRequest request source.setType(split[0]); source.setId(split[1]); Optional.ofNullable(loginUser).ifPresent(u -> { - Optional.ofNullable(this.context.getExchangisDataSource(split[0])).ifPresent(o -> { + Optional.ofNullable(this.context.getExchangisDsDefinition(split[0])).ifPresent(o -> { LinkisDataSourceRemoteClient dsClient = o.getDataSourceRemoteClient(); GetInfoByDataSourceIdAction action = GetInfoByDataSourceIdAction.builder() .setDataSourceId(Long.parseLong(split[1])) @@ -112,7 +112,7 @@ private ExchangisDataSourceIdsUI buildDataSourceIdsUI(HttpServletRequest request sink.setType(split[0]); sink.setId(split[1]); Optional.ofNullable(loginUser).ifPresent(u -> { - Optional.ofNullable(this.context.getExchangisDataSource(split[0])).ifPresent(o -> { + Optional.ofNullable(this.context.getExchangisDsDefinition(split[0])).ifPresent(o -> { LinkisDataSourceRemoteClient dsClient = o.getDataSourceRemoteClient(); GetInfoByDataSourceIdAction action = GetInfoByDataSourceIdAction.builder() .setDataSourceId(Long.parseLong(split[1])) @@ -146,7 +146,7 @@ protected ExchangisDataSourceParamsUI buildDataSourceParamsUI(ExchangisJobInfoCo ExchangisDataSourceIdUI source = dataSourceIdsUI.getSource(); if (null != source) { String type = source.getType(); - ExchangisDataSource exchangisSourceDataSource = this.context.getExchangisDataSource(type); + ExchangisDataSourceDefinition exchangisSourceDataSource = this.context.getExchangisDsDefinition(type); if (null != exchangisSourceDataSource) { sourceParamConfigs = exchangisSourceDataSource.getDataSourceParamConfigs().stream().filter( i -> i.getConfigDirection().equals(content.getEngine() + "-SOURCE") || "SOURCE".equalsIgnoreCase(i.getConfigDirection())).collect(Collectors.toList()); @@ -156,7 +156,7 @@ protected ExchangisDataSourceParamsUI buildDataSourceParamsUI(ExchangisJobInfoCo ExchangisDataSourceIdUI sink = dataSourceIdsUI.getSink(); if (null != sink) { String type = sink.getType(); - ExchangisDataSource exchangisSinkDataSource = this.context.getExchangisDataSource(type); + ExchangisDataSourceDefinition exchangisSinkDataSource = this.context.getExchangisDsDefinition(type); if (null != exchangisSinkDataSource) { sinkParamConfigs = exchangisSinkDataSource.getDataSourceParamConfigs().stream().filter(i -> i.getConfigDirection().equals(content.getEngine() + "-SINK") || "SINK".equalsIgnoreCase(i.getConfigDirection())).collect(Collectors.toList()); diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java index 7243c075b..727db8f49 100644 --- a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java @@ -11,7 +11,7 @@ import com.webank.wedatasphere.exchangis.dao.mapper.ExchangisJobParamConfigMapper; import com.webank.wedatasphere.exchangis.datasource.GetDataSourceInfoByIdAndVersionIdAction; import com.webank.wedatasphere.exchangis.datasource.Utils.RSAUtil; -import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSource; +import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSourceDefinition; import com.webank.wedatasphere.exchangis.datasource.core.context.ExchangisDataSourceContext; import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisDataSourceException; import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisDataSourceExceptionCode; @@ -112,7 +112,7 @@ public List getJobDataSourceUIs(HttpServletRequest @Override public List> getDataSourceParamsUI(String dsType, String engineAndDirection) { - ExchangisDataSource exchangisDataSource = this.context.getExchangisDataSource(dsType); + ExchangisDataSourceDefinition exchangisDataSource = this.context.getExchangisDsDefinition(dsType); List paramConfigs = exchangisDataSource.getDataSourceParamConfigs(); List filteredConfigs = new ArrayList<>(); String[] engineDirect = engineAndDirection.split("-"); @@ -138,8 +138,8 @@ public List> getJobEngineSettingsUI(String engineType) { * 做比较，筛选出可以给前端展示的数据源类型 */ public Message listDataSources(HttpServletRequest request, String engineType, String direct, String sourceType) throws Exception { - Collection all = this.context.all(); - List dtos = new ArrayList<>(); + Collection all = this.context.all(); + List dtos = new ArrayList<>(); List settingsList = this.settingsDao.getSettings(); List engineSettings = new ArrayList<>(); @@ -207,9 +207,9 @@ public Message listDataSources(HttpServletRequest request, String engineType, St for (DataSourceType type : allDataSourceType) { LOGGER.info("Current datasource Type is :{}", type.getName()); - for (ExchangisDataSource item : all) { + for (ExchangisDataSourceDefinition item : all) { if (item.name().equalsIgnoreCase(type.getName())) { - ExchangisDataSourceDTO dto = new ExchangisDataSourceDTO( + ExchangisDataSourceDefDTO dto = new ExchangisDataSourceDefDTO( type.getId(), type.getClassifier(), // item.classifier(), @@ -258,12 +258,12 @@ public Message create(HttpServletRequest request, /*String type, */DataSourceCre String user = UserUtils.getLoginUser(request); LOGGER.info("createDatasource userName:" + user); - ExchangisDataSource exchangisDataSource = context.getExchangisDataSource(vo.getDataSourceTypeId()); - if (Objects.isNull(exchangisDataSource)) { + ExchangisDataSourceDefinition dsType = context.getExchangisDsDefinition(vo.getDataSourceTypeId()); + if (Objects.isNull(dsType)) { throw new ExchangisDataSourceException(CONTEXT_GET_DATASOURCE_NULL.getCode(), "exchangis context get datasource null"); } - LinkisDataSourceRemoteClient client = exchangisDataSource.getDataSourceRemoteClient(); + LinkisDataSourceRemoteClient client = dsType.getDataSourceRemoteClient(); LOGGER.info("create datasource json as follows"); Set> entries = json.entrySet(); for (Map.Entry entry : entries) { @@ -341,12 +341,12 @@ public Message updateDataSource(HttpServletRequest request,/* String type,*/ Lon LOGGER.info("updateDataSource userName:" + user); LOGGER.info("DataSourceTypeId:" + vo.getDataSourceTypeId()); - ExchangisDataSource exchangisDataSource = context.getExchangisDataSource(vo.getDataSourceTypeId()); - if (Objects.isNull(exchangisDataSource)) { + ExchangisDataSourceDefinition dsType = context.getExchangisDsDefinition(vo.getDataSourceTypeId()); + if (Objects.isNull(dsType)) { throw new ExchangisDataSourceException(30401, "exchangis.datasource.null"); } - LinkisDataSourceRemoteClient client = exchangisDataSource.getDataSourceRemoteClient(); + LinkisDataSourceRemoteClient client = dsType.getDataSourceRemoteClient(); // UpdateDataSourceResult updateDataSourceResult; String responseBody; try { @@ -441,8 +441,8 @@ public Message deleteDataSource(HttpServletRequest request, /*String type,*/ Lon } public Message queryDataSourceDBs(HttpServletRequest request, String type, Long id) throws Exception { - ExchangisDataSource exchangisDataSource = context.getExchangisDataSource(type); - LinkisMetaDataRemoteClient metaDataRemoteClient = exchangisDataSource.getMetaDataRemoteClient(); + ExchangisDataSourceDefinition definition = context.getExchangisDsDefinition(type); + LinkisMetaDataRemoteClient metaDataRemoteClient = definition.getMetaDataRemoteClient(); String userName = UserUtils.getLoginUser(request); LOGGER.info("queryDataSourceDBs userName:" + userName); @@ -471,10 +471,10 @@ public Message queryDataSourceDBTables(HttpServletRequest request, String type, String user = UserUtils.getLoginUser(request); LOGGER.info("queryDataSourceDBTables userName:" + user); - ExchangisDataSource exchangisDataSource = context.getExchangisDataSource(type); + ExchangisDataSourceDefinition definition = context.getExchangisDsDefinition(type); MetadataGetTablesResult tables; try { - LinkisMetaDataRemoteClient metaDataRemoteClient = exchangisDataSource.getMetaDataRemoteClient(); + LinkisMetaDataRemoteClient metaDataRemoteClient = definition.getMetaDataRemoteClient(); tables = metaDataRemoteClient.getTables(MetadataGetTablesAction.builder() .setSystem(type) .setDataSourceId(id) @@ -586,8 +586,8 @@ public Message getJobDataSourceSettingsUI(Long jobId, String jobName) throws Exc } public Message queryDataSourceDBTableFields(HttpServletRequest request, String type, Long id, String dbName, String tableName) throws Exception { - ExchangisDataSource exchangisDataSource = context.getExchangisDataSource(type); - LinkisMetaDataRemoteClient metaDataRemoteClient = exchangisDataSource.getMetaDataRemoteClient(); + ExchangisDataSourceDefinition definition = context.getExchangisDsDefinition(type); + LinkisMetaDataRemoteClient metaDataRemoteClient = definition.getMetaDataRemoteClient(); String user = UserUtils.getLoginUser(request); LOGGER.info("queryDataSourceDBTableFields userName:" + user); @@ -671,10 +671,10 @@ public Message queryDataSources(HttpServletRequest request, DataSourceQueryVO vo List allDataSource = result.getAllDataSource(); - List originDataSources = new ArrayList<>(); - List dataSources = new ArrayList<>(); + List originDataSources = new ArrayList<>(); + List dataSources = new ArrayList<>(); allDataSource.forEach(ds -> { - DataSourceDTO item = new DataSourceDTO(); + ExchangisDataSourceDTO item = new ExchangisDataSourceDTO(); item.setId(ds.getId()); item.setCreateIdentify(ds.getCreateIdentify()); item.setName(ds.getDataSourceName()); @@ -701,13 +701,13 @@ public Message queryDataSources(HttpServletRequest request, DataSourceQueryVO vo LOGGER.info("originDatasource is: {}", originDataSources); if (direct!=null) { if ("source".equals(direct)) { - for (DataSourceDTO originDataSource : originDataSources) { + for (ExchangisDataSourceDTO originDataSource : originDataSources) { if (originDataSource.isReadAble()) { dataSources.add(originDataSource); } } } else if ("sink".equals(direct)) { - for (DataSourceDTO originDataSource : originDataSources) { + for (ExchangisDataSourceDTO originDataSource : originDataSources) { if (originDataSource.isReadAble()) { dataSources.add(originDataSource); } @@ -758,10 +758,10 @@ public Message listAllDataSources(HttpServletRequest request, String typeName, L } catch (Exception e) { throw new ExchangisDataSourceException(ExchangisDataSourceExceptionCode.CLIENT_QUERY_DATASOURCE_ERROR.getCode(), e.getMessage()); } - List dataSources = new ArrayList<>(); + List dataSources = new ArrayList<>(); if (!Objects.isNull(allDataSource)) { allDataSource.forEach(ds -> { - DataSourceDTO item = new DataSourceDTO(); + ExchangisDataSourceDTO item = new ExchangisDataSourceDTO(); item.setId(ds.getId()); item.setCreateIdentify(ds.getCreateIdentify()); item.setName(ds.getDataSourceName()); diff --git a/exchangis-datasource/exchangis-datasource-streamis/pom.xml b/exchangis-datasource/exchangis-datasource-streamis/pom.xml index 3d86b9522..b27dfb010 100644 --- a/exchangis-datasource/exchangis-datasource-streamis/pom.xml +++ b/exchangis-datasource/exchangis-datasource-streamis/pom.xml @@ -23,7 +23,11 @@ exchangis-datasource-core ${revision} - + + com.webank.wedatasphere.exchangis + exchangis-datasource-linkis + ${revision} + diff --git a/exchangis-datasource/exchangis-datasource-streamis/src/main/java/com/webank/wedatasphere/exchangis/datasource/streamis/ExchangisStreamisDataSource.java b/exchangis-datasource/exchangis-datasource-streamis/src/main/java/com/webank/wedatasphere/exchangis/datasource/streamis/ExchangisStreamisDataSource.java index a47a040ca..71b614f5a 100644 --- a/exchangis-datasource/exchangis-datasource-streamis/src/main/java/com/webank/wedatasphere/exchangis/datasource/streamis/ExchangisStreamisDataSource.java +++ b/exchangis-datasource/exchangis-datasource-streamis/src/main/java/com/webank/wedatasphere/exchangis/datasource/streamis/ExchangisStreamisDataSource.java @@ -1,40 +1,22 @@ package com.webank.wedatasphere.exchangis.datasource.streamis; -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; -import com.webank.wedatasphere.exchangis.dao.hook.MapperHook; -import com.webank.wedatasphere.exchangis.dao.mapper.ExchangisJobParamConfigMapper; -import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSource; +import com.webank.wedatasphere.exchangis.datasource.core.AbstractExchangisDataSourceDefinition; +import com.webank.wedatasphere.exchangis.datasource.linkis.ExchangisLinkisRemoteClient; import org.apache.linkis.datasource.client.impl.LinkisDataSourceRemoteClient; import org.apache.linkis.datasource.client.impl.LinkisMetaDataRemoteClient; -import java.util.List; - -public abstract class ExchangisStreamisDataSource implements ExchangisDataSource { - - protected MapperHook mapperHook; - - @Override - public void setMapperHook(MapperHook mapperHook) { - this.mapperHook = mapperHook; - } - - protected List getDataSourceParamConfigs(String type) { - ExchangisJobParamConfigMapper exchangisJobParamConfigMapper = this.mapperHook.getExchangisJobParamConfigMapper(); - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.eq("type", type); - queryWrapper.eq("is_hidden", 0); - queryWrapper.eq("status", 1); - return exchangisJobParamConfigMapper.selectList(queryWrapper); - } +/** + * Exchangis streamis data source + */ +public abstract class ExchangisStreamisDataSource extends AbstractExchangisDataSourceDefinition { @Override public LinkisDataSourceRemoteClient getDataSourceRemoteClient() { - return ExchangisStreamisRemoteClient.getStreamisDataSourceRemoteClient(); + return ExchangisLinkisRemoteClient.getLinkisDataSourceRemoteClient(); } @Override public LinkisMetaDataRemoteClient getMetaDataRemoteClient() { - return ExchangisStreamisRemoteClient.getStreamisMetadataRemoteClient(); + return ExchangisLinkisRemoteClient.getLinkisMetadataRemoteClient(); } } diff --git a/exchangis-datasource/exchangis-datasource-streamis/src/main/scala/com/webank/wedatasphere/exchangis/datasource/streamis/ExchangisStreamisRemoteClient.scala b/exchangis-datasource/exchangis-datasource-streamis/src/main/scala/com/webank/wedatasphere/exchangis/datasource/streamis/ExchangisStreamisRemoteClient.scala deleted file mode 100644 index 0085608bb..000000000 --- a/exchangis-datasource/exchangis-datasource-streamis/src/main/scala/com/webank/wedatasphere/exchangis/datasource/streamis/ExchangisStreamisRemoteClient.scala +++ /dev/null @@ -1,172 +0,0 @@ -package com.webank.wedatasphere.exchangis.datasource.streamis - -import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSourceConfiguration -import java.lang -import java.util.concurrent.TimeUnit - -import org.apache.linkis.datasource.client.impl.{LinkisDataSourceRemoteClient, LinkisMetaDataRemoteClient} -import org.apache.linkis.datasource.client.request.{GetAllDataSourceTypesAction, GetConnectParamsByDataSourceIdAction, MetadataGetColumnsAction, MetadataGetDatabasesAction, MetadataGetTablesAction, QueryDataSourceAction} -import org.apache.linkis.datasource.client.response.{GetConnectParamsByDataSourceIdResult, MetadataGetColumnsResult, MetadataGetDatabasesResult, MetadataGetTablesResult, QueryDataSourceResult} -import org.apache.linkis.datasourcemanager.common.domain.{DataSource, DataSourceType} -import org.apache.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy -import org.apache.linkis.httpclient.dws.config.{DWSClientConfig, DWSClientConfigBuilder} - -object ExchangisStreamisRemoteClient { - //Linkis Datasource Client Config - val serverUrl: String = ExchangisDataSourceConfiguration.SERVER_URL.getValue - val connectionTimeout: lang.Long = ExchangisDataSourceConfiguration.CONNECTION_TIMEOUT.getValue - val discoveryEnabled: lang.Boolean = ExchangisDataSourceConfiguration.DISCOVERY_ENABLED.getValue - val discoveryFrequencyPeriod: lang.Long = ExchangisDataSourceConfiguration.DISCOVERY_FREQUENCY_PERIOD.getValue - val loadbalancerEnabled: lang.Boolean = ExchangisDataSourceConfiguration.LOAD_BALANCER_ENABLED.getValue - val maxConnectionSize: Integer = ExchangisDataSourceConfiguration.MAX_CONNECTION_SIZE.getValue - val retryEnabled: lang.Boolean = ExchangisDataSourceConfiguration.RETRY_ENABLED.getValue - val readTimeout: lang.Long = ExchangisDataSourceConfiguration.READ_TIMEOUT.getValue - val authTokenKey: String = ExchangisDataSourceConfiguration.AUTHTOKEN_KEY.getValue - val authTokenValue: String = ExchangisDataSourceConfiguration.AUTHTOKEN_VALUE.getValue - val dwsVersion: String = ExchangisDataSourceConfiguration.DWS_VERSION.getValue - - - // val clientConfig = DWSClientConfigBuilder.newBuilder() - // .addServerUrl(serverUrl) - // .connectionTimeout(connectionTimeout) - // .discoveryEnabled(discoveryEnabled) - // .discoveryFrequency(1,TimeUnit.MINUTES) - // .loadbalancerEnabled(loadbalancerEnabled) - // .maxConnectionSize(maxConnectionSize) - // .retryEnabled(retryEnabled) - // .readTimeout(readTimeout) - // .setAuthenticationStrategy(new StaticAuthenticationStrategy()) - // .setAuthTokenKey(authTokenKey) - // .setAuthTokenValue(authTokenValue) - // .setDWSVersion(dwsVersion) - // .build() - - val clientConfig: DWSClientConfig = DWSClientConfigBuilder.newBuilder() - .addServerUrl(serverUrl) - .connectionTimeout(connectionTimeout) - .discoveryEnabled(discoveryEnabled) - .discoveryFrequency(discoveryFrequencyPeriod, TimeUnit.MINUTES) - .loadbalancerEnabled(loadbalancerEnabled) - .maxConnectionSize(maxConnectionSize) - .retryEnabled(retryEnabled) - .readTimeout(readTimeout) - .setAuthenticationStrategy(new StaticAuthenticationStrategy()) - .setAuthTokenKey(authTokenKey) - .setAuthTokenValue(authTokenValue) - .setDWSVersion(dwsVersion) - .build() - - val dataSourceClient = new LinkisDataSourceRemoteClient(clientConfig) - - val metaDataClient = new LinkisMetaDataRemoteClient(clientConfig) - - def getStreamisDataSourceRemoteClient: LinkisDataSourceRemoteClient = { - dataSourceClient - } - - def getStreamisMetadataRemoteClient: LinkisMetaDataRemoteClient = { - metaDataClient - } - - def close(): Unit = { - dataSourceClient.close() - metaDataClient.close() - } - - def queryDataSource(linkisDatasourceName: String): QueryDataSourceResult = { - dataSourceClient.queryDataSource(QueryDataSourceAction.builder() - .setSystem("") - .setName(linkisDatasourceName) - .setTypeId(1) - .setIdentifies("") - .setCurrentPage(1) - .setUser("hadoop") - .setPageSize(1).build() - ) - } - - /** - * get datasourceConnect information - * - * @param dataSourceId id - * @param system dssSystem - * @param user username - * @return - */ - def queryConnectParams(dataSourceId: Long, system: String, user: String): GetConnectParamsByDataSourceIdResult = { - dataSourceClient.getConnectParams(GetConnectParamsByDataSourceIdAction.builder() - .setDataSourceId(dataSourceId) - .setSystem(system) - .setUser(user) - .build() - ) - } - - /** - * get all DataSourceTypes - * - * @param user user - * @return - */ - def queryDataSourceTypes(user: String): java.util.List[DataSourceType] = { - dataSourceClient.getAllDataSourceTypes(GetAllDataSourceTypesAction.builder() - .setUser(user) - .build() - ).getAllDataSourceType - } - - - def queryClusterByDataSourceType(system: String, name: String, typeId: Long, user: String): java.util.List[DataSource] = { - dataSourceClient.queryDataSource(QueryDataSourceAction.builder() - .setSystem(system) - .setName(name) - .setTypeId(typeId) - .setIdentifies("") - .setCurrentPage(1) - .setPageSize(10) - .setUser(user) - .build() - ).getAllDataSource - } - - - /** - * get DataBases list - * - * @param system - * @param dataSourceId - * @param user - * @return list - */ - def queryDataBasesByCuster(system: String, dataSourceId: Long, user: String): MetadataGetDatabasesResult = { - metaDataClient.getDatabases(MetadataGetDatabasesAction.builder() - .setSystem(system) - .setDataSourceId(dataSourceId) - .setUser(user) - .build() - ) - } - - def queryTablesByDataBase(system: String, dataSourceId: Long, dataBase: String, user: String): MetadataGetTablesResult = { - metaDataClient.getTables(MetadataGetTablesAction.builder() - .setSystem(system) - .setDataSourceId(dataSourceId) - .setDatabase(dataBase) - .setUser(user) - .build() - ) - } - - def queryColumnsByTable(system: String, dataSourceId: Long, dataBase: String, table: String, user: String): MetadataGetColumnsResult = { - metaDataClient.getColumns(MetadataGetColumnsAction.builder() - .setSystem(system) - .setDataSourceId(dataSourceId) - .setDatabase(dataBase) - .setTable(table) - .setUser(user) - .build() - ) - } - - -} diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-elasticsearch/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/mysql/ExchangisESDataSource.java b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-elasticsearch/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/mysql/ExchangisESDataSource.java index 607af79de..a049d07ce 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-elasticsearch/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/mysql/ExchangisESDataSource.java +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-elasticsearch/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/mysql/ExchangisESDataSource.java @@ -1,23 +1,17 @@ package com.webank.wedatasphere.exchangis.extension.datasource.mysql; -import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; -import com.webank.wedatasphere.exchangis.datasource.core.domain.Classifier; -import com.webank.wedatasphere.exchangis.datasource.core.domain.DataSourceType; +import com.webank.wedatasphere.exchangis.datasource.core.domain.ExchangisDataSourceType; import com.webank.wedatasphere.exchangis.datasource.core.domain.StructClassifier; import com.webank.wedatasphere.exchangis.datasource.linkis.ExchangisBatchDataSource; -import java.util.List; - +/** + * Note: ES data source + */ public class ExchangisESDataSource extends ExchangisBatchDataSource { @Override - public String name() { - return DataSourceType.ELASTICSEARCH.name; - } - - @Override - public String classifier() { - return Classifier.ELASTICSEARCH.name; + protected ExchangisDataSourceType type() { + return ExchangisDataSourceType.ELASTICSEARCH; } @Override @@ -40,8 +34,4 @@ public String icon() { return "icon-es"; } - @Override - public List getDataSourceParamConfigs() { - return super.getDataSourceParamConfigs(DataSourceType.ELASTICSEARCH.name); - } } \ No newline at end of file diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-hive/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/hive/ExchangisHiveDataSource.java b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-hive/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/hive/ExchangisHiveDataSource.java index e9ce77738..c59cfe277 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-hive/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/hive/ExchangisHiveDataSource.java +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-hive/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/hive/ExchangisHiveDataSource.java @@ -1,23 +1,17 @@ package com.webank.wedatasphere.exchangis.extension.datasource.hive; -import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; -import com.webank.wedatasphere.exchangis.datasource.core.domain.Classifier; -import com.webank.wedatasphere.exchangis.datasource.core.domain.DataSourceType; +import com.webank.wedatasphere.exchangis.datasource.core.domain.ExchangisDataSourceType; import com.webank.wedatasphere.exchangis.datasource.core.domain.StructClassifier; import com.webank.wedatasphere.exchangis.datasource.linkis.ExchangisBatchDataSource; -import java.util.List; - +/** + * Note: Hive data source + */ public class ExchangisHiveDataSource extends ExchangisBatchDataSource { @Override - public String name() { - return DataSourceType.HIVE.name; - } - - @Override - public String classifier() { - return Classifier.HIVE.name; + protected ExchangisDataSourceType type() { + return ExchangisDataSourceType.HIVE; } @Override @@ -40,8 +34,4 @@ public String icon() { return "icon-hive"; } - @Override - public List getDataSourceParamConfigs() { - return super.getDataSourceParamConfigs(DataSourceType.HIVE.name); - } } diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mongodb/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/mysql/ExchangisMongoDbDataSource.java b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mongodb/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/mysql/ExchangisMongoDbDataSource.java index 269efce87..0fec6da36 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mongodb/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/mysql/ExchangisMongoDbDataSource.java +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mongodb/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/mysql/ExchangisMongoDbDataSource.java @@ -1,23 +1,17 @@ package com.webank.wedatasphere.exchangis.extension.datasource.mysql; -import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; -import com.webank.wedatasphere.exchangis.datasource.core.domain.Classifier; -import com.webank.wedatasphere.exchangis.datasource.core.domain.DataSourceType; +import com.webank.wedatasphere.exchangis.datasource.core.domain.ExchangisDataSourceType; import com.webank.wedatasphere.exchangis.datasource.core.domain.StructClassifier; import com.webank.wedatasphere.exchangis.datasource.linkis.ExchangisBatchDataSource; -import java.util.List; - +/** + * Note: MongoDB data source + */ public class ExchangisMongoDbDataSource extends ExchangisBatchDataSource { @Override - public String name() { - return DataSourceType.MONGODB.name; - } - - @Override - public String classifier() { - return Classifier.MONGODB.name; + protected ExchangisDataSourceType type() { + return ExchangisDataSourceType.MONGODB; } @Override @@ -40,8 +34,4 @@ public String icon() { return "icon-mongodb"; } - @Override - public List getDataSourceParamConfigs() { - return super.getDataSourceParamConfigs(DataSourceType.MONGODB.name); - } } \ No newline at end of file diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mysql/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/mysql/ExchangisMySQLDataSource.java b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mysql/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/mysql/ExchangisMySQLDataSource.java index e05e790da..d818a1f65 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mysql/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/mysql/ExchangisMySQLDataSource.java +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mysql/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/mysql/ExchangisMySQLDataSource.java @@ -1,23 +1,18 @@ package com.webank.wedatasphere.exchangis.extension.datasource.mysql; -import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; -import com.webank.wedatasphere.exchangis.datasource.core.domain.Classifier; -import com.webank.wedatasphere.exchangis.datasource.core.domain.DataSourceType; +import com.webank.wedatasphere.exchangis.datasource.core.domain.ExchangisDataSourceType; import com.webank.wedatasphere.exchangis.datasource.core.domain.StructClassifier; import com.webank.wedatasphere.exchangis.datasource.linkis.ExchangisBatchDataSource; -import java.util.List; +/** + * Note: MYSQL data source + */ public class ExchangisMySQLDataSource extends ExchangisBatchDataSource { @Override - public String name() { - return DataSourceType.MYSQL.name; - } - - @Override - public String classifier() { - return Classifier.MYSQL.name; + protected ExchangisDataSourceType type() { + return ExchangisDataSourceType.MYSQL; } @Override @@ -40,8 +35,5 @@ public String icon() { return "icon-mysql"; } - @Override - public List getDataSourceParamConfigs() { - return super.getDataSourceParamConfigs(DataSourceType.MYSQL.name); - } + } \ No newline at end of file diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-oracle/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/oracle/ExchangisOracleDataSource.java b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-oracle/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/oracle/ExchangisOracleDataSource.java index 8d3ca3adb..38d0c1a8b 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-oracle/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/oracle/ExchangisOracleDataSource.java +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-oracle/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/oracle/ExchangisOracleDataSource.java @@ -2,7 +2,7 @@ import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; import com.webank.wedatasphere.exchangis.datasource.core.domain.Classifier; -import com.webank.wedatasphere.exchangis.datasource.core.domain.DataSourceType; +import com.webank.wedatasphere.exchangis.datasource.core.domain.ExchangisDataSourceType; import com.webank.wedatasphere.exchangis.datasource.core.domain.StructClassifier; import com.webank.wedatasphere.exchangis.datasource.linkis.ExchangisBatchDataSource; @@ -13,9 +13,10 @@ * @create 2022-09-14 **/ public class ExchangisOracleDataSource extends ExchangisBatchDataSource { + @Override - public String name() { - return DataSourceType.ORACLE.name; + protected ExchangisDataSourceType type() { + return ExchangisDataSourceType.ORACLE; } @Override @@ -28,10 +29,6 @@ public String option() { return "Oracle数据库"; } - @Override - public String classifier() { - return Classifier.ORACLE.name; - } @Override public String structClassifier() { @@ -43,9 +40,5 @@ public String icon() { return "icon-oracle"; } - @Override - public List getDataSourceParamConfigs() { - return super.getDataSourceParamConfigs(DataSourceType.ORACLE.name); - } } diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-sftp/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/sftp/ExchangisSftpDataSource.java b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-sftp/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/sftp/ExchangisSftpDataSource.java index c8256dddb..b9ecdfbb4 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-sftp/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/sftp/ExchangisSftpDataSource.java +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-sftp/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/sftp/ExchangisSftpDataSource.java @@ -2,7 +2,7 @@ import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; import com.webank.wedatasphere.exchangis.datasource.core.domain.Classifier; -import com.webank.wedatasphere.exchangis.datasource.core.domain.DataSourceType; +import com.webank.wedatasphere.exchangis.datasource.core.domain.ExchangisDataSourceType; import com.webank.wedatasphere.exchangis.datasource.core.domain.StructClassifier; import com.webank.wedatasphere.exchangis.datasource.linkis.ExchangisBatchDataSource; @@ -11,13 +11,8 @@ public class ExchangisSftpDataSource extends ExchangisBatchDataSource { @Override - public String name() { - return DataSourceType.SFTP.name; - } - - @Override - public String classifier() { - return Classifier.SFTP.name; + protected ExchangisDataSourceType type() { + return ExchangisDataSourceType.SFTP; } @Override @@ -27,7 +22,7 @@ public String structClassifier() { @Override public String description() { - return "This is Sftp"; + return "This is sftp"; } @Override @@ -42,6 +37,7 @@ public String icon() { @Override public List getDataSourceParamConfigs() { - return super.getDataSourceParamConfigs(DataSourceType.SFTP.name); + return super.getDataSourceParamConfigs(ExchangisDataSourceType.SFTP.name); } + } diff --git a/exchangis-datasource/pom.xml b/exchangis-datasource/pom.xml index 5cb7fa7be..0eef68263 100644 --- a/exchangis-datasource/pom.xml +++ b/exchangis-datasource/pom.xml @@ -16,8 +16,8 @@ exchangis-datasource-core exchangis-datasource-loader - exchangis-datasource-streamis exchangis-datasource-linkis + exchangis-datasource-streamis exchangis-datasource-service extension-datasources/exchangis-datasource-ext-mysql extension-datasources/exchangis-datasource-ext-hive diff --git a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisExchangisTaskLauncher.java b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisExchangisTaskLauncher.java index b366b8dc4..730ff21b7 100644 --- a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisExchangisTaskLauncher.java +++ b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisExchangisTaskLauncher.java @@ -1,6 +1,7 @@ package com.webank.wedatasphere.exchangis.job.launcher.linkis; -import com.webank.wedatasphere.exchangis.common.linkis.ClientConfiguration; +import com.webank.wedatasphere.exchangis.common.linkis.client.config.ExchangisClientConfig; +import com.webank.wedatasphere.exchangis.common.linkis.client.config.ExchangisClientConfigBuilder; import com.webank.wedatasphere.exchangis.job.enums.EngineTypeEnum; import com.webank.wedatasphere.exchangis.job.launcher.AccessibleLauncherTask; import com.webank.wedatasphere.exchangis.job.launcher.exception.ExchangisTaskLaunchException; @@ -8,19 +9,16 @@ import com.webank.wedatasphere.exchangis.job.launcher.ExchangisTaskLauncher; import com.webank.wedatasphere.exchangis.job.launcher.domain.LaunchableExchangisTask; import com.webank.wedatasphere.exchangis.job.launcher.domain.LaunchedExchangisTask; +import com.webank.wedatasphere.exchangis.job.launcher.linkis.client.ExchangisLaunchClient; import org.apache.commons.lang.StringUtils; -import org.apache.linkis.common.conf.Configuration; import org.apache.linkis.common.exception.LinkisRetryException; import org.apache.linkis.common.utils.DefaultRetryHandler; import org.apache.linkis.common.utils.RetryHandler; import org.apache.linkis.computation.client.LinkisJobClient$; -import org.apache.linkis.httpclient.config.ClientConfig; -import org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy; -import org.apache.linkis.httpclient.dws.config.DWSClientConfig; -import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder$; +import org.apache.linkis.computation.client.once.simple.SimpleOnceJobBuilder$; +import java.lang.reflect.Field; import java.util.*; -import java.util.concurrent.TimeUnit; /** * Linkis task launcher @@ -44,23 +42,31 @@ public void init(ExchangisTaskLaunchManager jobLaunchManager) { this.engineVersions.put(EngineTypeEnum.DATAX.name().toLowerCase(), "3.0.0"); RetryHandler retryHandler = new DefaultRetryHandler(){}; retryHandler.addRetryException(LinkisRetryException.class); - ClientConfig clientConfig = DWSClientConfigBuilder$.MODULE$ - .newBuilder() - .setDWSVersion(Configuration.LINKIS_WEB_VERSION().getValue()) - .addServerUrl(ClientConfiguration.LINKIS_SERVER_URL.getValue()) - .connectionTimeout(45000) - .discoveryEnabled(false) - .discoveryFrequency(1, TimeUnit.MINUTES) - .loadbalancerEnabled(false) - .maxConnectionSize(ClientConfiguration.LINKIS_DEFAULT_MAX_CONNECTIONS.getValue()) + ExchangisClientConfigBuilder builder = (ExchangisClientConfigBuilder) ExchangisClientConfig.newBuilder().discoveryEnabled(false) .retryEnabled(true) - .setRetryHandler(retryHandler) - .readTimeout(90000) // We think 90s is enough, if SocketTimeoutException is throw, just set a new clientConfig to modify it. - .setAuthenticationStrategy(new TokenAuthenticationStrategy()) - .setAuthTokenKey(TokenAuthenticationStrategy.TOKEN_KEY()) - .setAuthTokenValue(ClientConfiguration.LINKIS_TOKEN_VALUE.getValue()) - .build(); - LinkisJobClient$.MODULE$.config().setDefaultClientConfig((DWSClientConfig) clientConfig); + .setRetryHandler(retryHandler); + ExchangisClientConfig clientConfig = builder.build(); + // Try to set the static method + Class clz = SimpleOnceJobBuilder$.MODULE$.getClass(); + Field field; + boolean setField = false; + try { + field = clz.getDeclaredField(SimpleOnceJobBuilder$.class.getName().replace(".", "$") + "$linkisManagerClient"); + field.setAccessible(true); + try { + ExchangisLaunchClient client = new ExchangisLaunchClient(clientConfig); + field.set("", client); + Runtime.getRuntime().addShutdownHook(new Thread(client::close)); + setField = true; + } catch (IllegalAccessException e) { + // Ignore + } + } catch (NoSuchFieldException e) { + // Ignore + } + if (!setField){ + LinkisJobClient$.MODULE$.config().setDefaultClientConfig(clientConfig); + } } @Override @@ -104,4 +110,6 @@ private Map convertJobInfoToStore(Map jobInfo){ }); return storeInfo; } + } + diff --git a/exchangis-job/exchangis-job-launcher/src/main/scala/com/webank/wedatasphere/exchangis/job/launcher/linkis/client/ExchangisLaunchClient.scala b/exchangis-job/exchangis-job-launcher/src/main/scala/com/webank/wedatasphere/exchangis/job/launcher/linkis/client/ExchangisLaunchClient.scala new file mode 100644 index 000000000..309a05228 --- /dev/null +++ b/exchangis-job/exchangis-job-launcher/src/main/scala/com/webank/wedatasphere/exchangis/job/launcher/linkis/client/ExchangisLaunchClient.scala @@ -0,0 +1,54 @@ +package com.webank.wedatasphere.exchangis.job.launcher.linkis.client + +import com.webank.wedatasphere.exchangis.common.linkis.client.ExchangisHttpClient +import com.webank.wedatasphere.exchangis.common.linkis.client.config.ExchangisClientConfig +import org.apache.linkis.common.utils.Utils +import org.apache.linkis.computation.client.once.LinkisManagerClient +import org.apache.linkis.computation.client.once.action.{AskEngineConnAction, CreateEngineConnAction, EngineConnOperateAction, GetEngineConnAction, KillEngineConnAction, LinkisManagerAction, ListEngineConnAction} +import org.apache.linkis.computation.client.once.result.{AskEngineConnResult, CreateEngineConnResult, EngineConnOperateResult, GetEngineConnResult, KillEngineConnResult, LinkisManagerResult, ListEngineConnResult} +import org.apache.linkis.httpclient.request.Action + +/** + * Exchangis launch client + */ +class ExchangisLaunchClient(clientConfig: ExchangisClientConfig) extends LinkisManagerClient{ + private val dwsHttpClient = new ExchangisHttpClient(clientConfig, "Linkis-Job-Execution-Thread") + + protected def execute[T <: LinkisManagerResult](linkisManagerAction: LinkisManagerAction): T = + linkisManagerAction match { + case action: Action => dwsHttpClient.execute(action).asInstanceOf[T] + } + + override def createEngineConn( + createEngineConnAction: CreateEngineConnAction + ): CreateEngineConnResult = execute(createEngineConnAction) + + override def getEngineConn(getEngineConnAction: GetEngineConnAction): GetEngineConnResult = + execute(getEngineConnAction) + + override def killEngineConn(killEngineConnAction: KillEngineConnAction): KillEngineConnResult = + execute(killEngineConnAction) + + override def executeEngineConnOperation( + engineConnOperateAction: EngineConnOperateAction + ): EngineConnOperateResult = { + Utils.tryCatch { + val rs = execute[EngineConnOperateResult](engineConnOperateAction) + rs + } { case e: Exception => + val rs = new EngineConnOperateResult + rs.setIsError(true) + rs.setErrorMsg(e.getMessage) + rs + } + } + + override def close(): Unit = dwsHttpClient.close() + + override def askEngineConn(askEngineConnAction: AskEngineConnAction): AskEngineConnResult = + execute(askEngineConnAction) + + override def listEngineConn(listEngineConnAction: ListEngineConnAction): ListEngineConnResult = { + execute(listEngineConnAction) + } +} From 9da1ebb66b9f17e4f91b08461410a63cc5d8b1b0 Mon Sep 17 00:00:00 2001 From: davidhua Date: Mon, 20 May 2024 19:34:17 +0800 Subject: [PATCH 14/60] Fix some compile problem. --- exchangis-job/exchangis-job-launcher/pom.xml | 17 +++++++++++++++++ exchangis-project/pom.xml | 2 +- pom.xml | 10 +++++----- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/exchangis-job/exchangis-job-launcher/pom.xml b/exchangis-job/exchangis-job-launcher/pom.xml index 19bf193f4..22ca5cc78 100644 --- a/exchangis-job/exchangis-job-launcher/pom.xml +++ b/exchangis-job/exchangis-job-launcher/pom.xml @@ -29,4 +29,21 @@ + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + \ No newline at end of file diff --git a/exchangis-project/pom.xml b/exchangis-project/pom.xml index d71cd1525..773af5277 100644 --- a/exchangis-project/pom.xml +++ b/exchangis-project/pom.xml @@ -12,9 +12,9 @@ exchangis-project pom + exchangis-project-entity exchangis-project-server exchangis-project-provider - exchangis-project-entity diff --git a/pom.xml b/pom.xml index 461a3cce1..523a18983 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ com.webank.wedatasphere.exchangis exchangis - 1.1.3-webank + ${revision} pom exchangis @@ -229,10 +229,10 @@ ${jdk.compile.version} ${jdk.compile.version} - - - ${java.home}/lib/rt.jar;${java.home}/lib/jce.jar - + + + + From 368191731512610db5aa08287e994b5617b62404 Mon Sep 17 00:00:00 2001 From: davidhua Date: Mon, 6 May 2024 20:56:09 +0800 Subject: [PATCH 15/60] Add priority queue to consumer manager. --- .../scheduler/ExchangisSchedulerTask.java | 3 +- .../execution/scheduler/SchedulerThread.java | 4 +- .../TenancyParallelConsumerManager.java | 48 +++-- .../priority/PriorityOrderedQueue.java | 169 ++++++++++++++++++ .../scheduler/priority/PriorityRunnable.java | 16 ++ 5 files changed, 228 insertions(+), 12 deletions(-) create mode 100644 exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/priority/PriorityOrderedQueue.java create mode 100644 exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/priority/PriorityRunnable.java diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/ExchangisSchedulerTask.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/ExchangisSchedulerTask.java index 904906274..a0ec6eeb4 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/ExchangisSchedulerTask.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/ExchangisSchedulerTask.java @@ -1,11 +1,12 @@ package com.webank.wedatasphere.exchangis.job.server.execution.scheduler; +import com.webank.wedatasphere.exchangis.job.server.execution.scheduler.priority.PriorityRunnable; import org.apache.linkis.scheduler.queue.SchedulerEvent; /** * Exchangis scheduler task */ -public interface ExchangisSchedulerTask extends SchedulerEvent { +public interface ExchangisSchedulerTask extends PriorityRunnable, SchedulerEvent { /** * Tenancy diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/SchedulerThread.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/SchedulerThread.java index df9717029..b9da86213 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/SchedulerThread.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/SchedulerThread.java @@ -1,9 +1,11 @@ package com.webank.wedatasphere.exchangis.job.server.execution.scheduler; +import com.webank.wedatasphere.exchangis.job.server.execution.scheduler.priority.PriorityRunnable; + /** * Define the basic interface of thread in scheduler */ -public interface SchedulerThread extends Runnable{ +public interface SchedulerThread extends PriorityRunnable { /** * Start entrance */ diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/TenancyParallelConsumerManager.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/TenancyParallelConsumerManager.java index da52b214b..0e2bd5caf 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/TenancyParallelConsumerManager.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/TenancyParallelConsumerManager.java @@ -1,6 +1,8 @@ package com.webank.wedatasphere.exchangis.job.server.execution.scheduler; import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisSchedulerException; +import com.webank.wedatasphere.exchangis.job.server.execution.scheduler.priority.PriorityOrderedQueue; +import com.webank.wedatasphere.exchangis.job.server.execution.scheduler.priority.PriorityRunnable; import org.apache.commons.lang.StringUtils; import org.apache.linkis.common.utils.Utils; import org.apache.linkis.scheduler.listener.ConsumerListener; @@ -9,14 +11,10 @@ import org.apache.linkis.scheduler.queue.fifoqueue.FIFOUserConsumer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; - -import javax.annotation.PreDestroy; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; + +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.ReentrantLock; /** @@ -53,7 +51,7 @@ public ExecutorService getOrCreateExecutorService() { try{ Group group = getSchedulerContext().getOrCreateGroupFactory().getOrCreateGroup(null); if (group instanceof FIFOGroup){ - defaultExecutorService = Utils.newCachedThreadPool(((FIFOGroup) group).getMaxRunningJobs() + + defaultExecutorService = newPriorityThreadPool(((FIFOGroup) group).getMaxRunningJobs() + this.initResidentThreads + 1, TenancyParallelGroupFactory.GROUP_NAME_PREFIX + TenancyParallelGroupFactory.DEFAULT_TENANCY + "-Executor-", true); tenancyExecutorServices.put(TenancyParallelGroupFactory.DEFAULT_TENANCY, defaultExecutorService); @@ -128,7 +126,7 @@ protected ExecutorService getOrCreateExecutorService(String groupName){ if (StringUtils.isNotBlank(tenancy)){ return tenancyExecutorServices.computeIfAbsent(tenancy, tenancyName -> { // Use the default value of max running jobs - return Utils.newCachedThreadPool(parallelGroupFactory.getDefaultMaxRunningJobs() + parallelGroupFactory.getParallelPerTenancy(), + return newPriorityThreadPool(parallelGroupFactory.getDefaultMaxRunningJobs() + parallelGroupFactory.getParallelPerTenancy(), TenancyParallelGroupFactory.GROUP_NAME_PREFIX + tenancy + "-Executor-", true); }); } @@ -151,4 +149,34 @@ public void setInitResidentThreads(int initResidentThreads) { public Map getTenancyExecutorServices() { return tenancyExecutorServices; } + + /** + * Create thread pool with priority for tenancy consumer + * @return + */ + private ExecutorService newPriorityThreadPool(int threadNum, String threadName, boolean isDaemon){ + ThreadPoolExecutor threadPool = new ThreadPoolExecutor( + threadNum, + threadNum, + 120L, + TimeUnit.SECONDS, + new PriorityBlockingQueue<>(10 * threadNum, (o1, o2) -> { + int left = o1 instanceof PriorityRunnable ? ((PriorityRunnable) o1).getPriority() : 0; + int right = o2 instanceof PriorityRunnable ? ((PriorityRunnable) o2).getPriority() : 0; + return right - left; + }), + new ThreadFactory() { + final AtomicInteger num = new AtomicInteger(0); + @Override + public Thread newThread(Runnable r) { + Thread t = new Thread(r); + t.setDaemon(isDaemon); + t.setName(threadName + num.incrementAndGet()); + return t; + } + }); + threadPool.allowCoreThreadTimeOut(true); + return threadPool; + } + } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/priority/PriorityOrderedQueue.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/priority/PriorityOrderedQueue.java new file mode 100644 index 000000000..4277d768a --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/priority/PriorityOrderedQueue.java @@ -0,0 +1,169 @@ +package com.webank.wedatasphere.exchangis.job.server.execution.scheduler.priority; + +import com.webank.wedatasphere.exchangis.job.utils.SnowFlake; + +import java.util.*; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.PriorityBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +/** + * Refer to 'PriorityBlockingQueue', + * Use snowflake to generate order number of elements + */ +public class PriorityOrderedQueue extends AbstractQueue + implements BlockingQueue, java.io.Serializable { + + /** + * Priority queue + */ + private final PriorityBlockingQueue priorityQueue; + + /** + /** + * Snowflake context + */ + private final SnowFlake snowFlake; + public PriorityOrderedQueue(int initialCapacity, + Comparator comparator){ + if (Objects.isNull(comparator)){ + this.priorityQueue = new PriorityBlockingQueue<>(initialCapacity, + (left, right) -> (int) (right.seq - left.seq)); + } else { + this.priorityQueue = new PriorityBlockingQueue<>(initialCapacity, + (left, right) -> { + int result = comparator.compare(left.element, right.element); + if (result == 0){ + return (int)(left.seq - right.seq); + } + return result; + }); + } + this.snowFlake = new SnowFlake(0, 0, System.currentTimeMillis()); + } + @Override + public Iterator iterator() { + return new Itr(priorityQueue.iterator()); + } + + @Override + public int size() { + return priorityQueue.size(); + } + + @Override + public void put(E e) throws InterruptedException { + offer(e); + } + + @Override + public boolean offer(E e, long timeout, TimeUnit unit) throws InterruptedException { + return offer(e); + } + + @Override + public E take() throws InterruptedException { + Ordered ordered = this.priorityQueue.take(); + return ordered.element; + } + + @Override + public E poll(long timeout, TimeUnit unit) throws InterruptedException { + Ordered ordered = this.priorityQueue.poll(timeout, unit); + if (null != ordered){ + return ordered.element; + } + return null; + } + + @Override + public int remainingCapacity() { + return this.priorityQueue.remainingCapacity(); + } + + @Override + public int drainTo(Collection c) { + return drainTo(c, Integer.MAX_VALUE); + } + + @Override + @SuppressWarnings("unchecked") + public int drainTo(Collection c, int maxElements) { + Collection collection = null; + if (null != c && c != this){ + collection = c.stream().map(e -> new Ordered((E) e)).collect(Collectors.toList()); + } + return this.priorityQueue.drainTo(collection); + } + + @Override + public boolean offer(E e) { + return this.priorityQueue.offer(new Ordered(e)); + } + + @Override + public E poll() { + Ordered ordered = this.priorityQueue.poll(); + if (null != ordered){ + return ordered.element; + } + return null; + } + + @Override + public E peek() { + Ordered ordered = this.priorityQueue.peek(); + if (null != ordered){ + return ordered.element; + } + return null; + } + + private class Ordered{ + /** + * Seq number + */ + private long seq; + + /** + * Queue element + */ + private E element; + + public Ordered(E element){ + this.seq = snowFlake.nextId(); + this.element = element; + } + } + + private class Itr implements Iterator { + private Iterator innerItr; + public Itr(Iterator iterator){ + innerItr = iterator; + } + + + @Override + public boolean hasNext() { + return innerItr.hasNext(); + } + + @Override + public E next() { + return innerItr.next().element; + } + + @Override + public void remove() { + innerItr.remove(); + } + + @Override + public void forEachRemaining(Consumer action) { + innerItr.forEachRemaining(eOrdered -> + action.accept(eOrdered.element)); + } + } +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/priority/PriorityRunnable.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/priority/PriorityRunnable.java new file mode 100644 index 000000000..4d8ed3608 --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/priority/PriorityRunnable.java @@ -0,0 +1,16 @@ +package com.webank.wedatasphere.exchangis.job.server.execution.scheduler.priority; + +/** + * Runnable with priority + */ +public interface PriorityRunnable extends Runnable{ + + /** + * Default: 1 + * @return value + */ + default int getPriority(){ + return 1; + } + +} From d113b36dcb7935f3f6baf14ec0118d1fccd73d6e Mon Sep 17 00:00:00 2001 From: davidhua Date: Thu, 16 May 2024 14:46:25 +0800 Subject: [PATCH 16/60] Enable to fetch remote log. --- .../config/dss-exchangis-server.properties | 3 + .../exchangis/common/EnvironmentUtils.java | 65 ++++ .../exchangis/job/log/LogQuery.java | 14 + .../entity/LaunchedExchangisJobEntity.java | 6 +- .../linkis/LinkisExchangisTaskLauncher.java | 5 +- exchangis-job/exchangis-job-server/pom.xml | 6 + .../ExchangisJobExecuteAutoConfiguration.java | 4 +- .../transform/TransformExchangisJob.java | 22 +- .../tasks/MetricUpdateSchedulerTask.java | 9 + .../tasks/StatusUpdateSchedulerTask.java | 9 + .../job/server/log/rpc/FetchLogRequest.java | 31 ++ .../job/server/log/rpc/FetchLogResponse.java | 20 ++ .../job/server/log/rpc/SendLogRequest.java | 57 ++++ .../log/service/AbstractJobLogService.java | 159 ++++++++++ .../log/service/LocalSimpleJobLogService.java | 282 ------------------ .../server/log/service/RpcJobLogService.java | 280 +++++++++++++++++ .../render/transform/TransformRequestVo.java | 25 ++ .../job/server/log/cache/JobLogCache.scala | 4 +- pom.xml | 6 + 19 files changed, 714 insertions(+), 293 deletions(-) create mode 100644 exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/rpc/FetchLogRequest.java create mode 100644 exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/rpc/FetchLogResponse.java create mode 100644 exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/rpc/SendLogRequest.java create mode 100644 exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/AbstractJobLogService.java delete mode 100644 exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/LocalSimpleJobLogService.java create mode 100644 exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/RpcJobLogService.java diff --git a/assembly-package/config/dss-exchangis-server.properties b/assembly-package/config/dss-exchangis-server.properties index a8aa9a830..8f3b64bd3 100644 --- a/assembly-package/config/dss-exchangis-server.properties +++ b/assembly-package/config/dss-exchangis-server.properties @@ -26,6 +26,9 @@ wds.linkis.gateway.url=http://{LINKIS_IP}:{LINKIS_PORT}/ wds.linkis.log.clear=true wds.linkis.server.version=v1 +# server rpc +wds.linkis.ms.service.scan.package=com.webank.wedatasphere.exchangis + # datasource client wds.exchangis.datasource.client.serverurl=http://{LINKIS_IP}:{LINKIS_PORT}/ wds.exchangis.datasource.client.authtoken.key=EXCHANGIS-AUTH diff --git a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/EnvironmentUtils.java b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/EnvironmentUtils.java index eaf5d6c38..ac6e77198 100644 --- a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/EnvironmentUtils.java +++ b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/EnvironmentUtils.java @@ -1,6 +1,13 @@ package com.webank.wedatasphere.exchangis.common; +import org.apache.commons.lang3.StringUtils; +import org.apache.linkis.DataWorkCloudApplication; import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.conf.Configuration; +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.server.utils.LinkisMainHelper; +import org.springframework.context.ApplicationContext; + /** * Environment utils @@ -9,6 +16,8 @@ public class EnvironmentUtils { private static final CommonVars JVM_USER = CommonVars.apply("wds.exchangis.env.jvm.user", System.getProperty("user.name", "hadoop")); + private static final CommonVars SERVER_NAME = CommonVars.apply(LinkisMainHelper.SERVER_NAME_KEY(), "exchangis"); + /** * Jvm user * @return user name @@ -16,4 +25,60 @@ public class EnvironmentUtils { public static String getJvmUser(){ return JVM_USER.getValue(); } + + /** + * Server name + * @return name + */ + public static String getServerName(){ + return SERVER_NAME.getValue(); + } + + /** + * Get server address + * @return address + */ + public static String getServerAddress(){ + ApplicationContext context = DataWorkCloudApplication.getApplicationContext(); + String hostname; + if (Configuration.PREFER_IP_ADDRESS()) { + hostname = context + .getEnvironment().getProperty("spring.cloud.client.ip-address"); + } else { + hostname = context.getEnvironment().getProperty("eureka.instance.hostname", ""); + if (StringUtils.isBlank(hostname)) { + hostname = Utils.getComputerName(); + } + } + String serverPort = context.getEnvironment().getProperty("server.port"); + return hostname + (StringUtils.isNotBlank(serverPort) ? ":" + serverPort : ""); + } + /** + * Get server host name + * @return hostname + */ + public static String getServerHost(){ + ApplicationContext context = DataWorkCloudApplication.getApplicationContext(); + if (Configuration.PREFER_IP_ADDRESS()) { + return context + .getEnvironment().getProperty("spring.cloud.client.ip-address"); + } else { + String hostname = context.getEnvironment().getProperty("eureka.instance.hostname", ""); + if (StringUtils.isBlank(hostname)) { + return Utils.getComputerName(); + } + return hostname; + } + } + + /** + * Get server port + * @return port number + */ + public static Integer getServerPort(){ + String serverPort = DataWorkCloudApplication.getApplicationContext() + .getEnvironment().getProperty("server.port"); + return StringUtils.isNotBlank(serverPort) ? Integer.parseInt(serverPort) : null; + } + } diff --git a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/log/LogQuery.java b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/log/LogQuery.java index 0ab7780b9..de19116c9 100644 --- a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/log/LogQuery.java +++ b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/log/LogQuery.java @@ -21,6 +21,11 @@ public class LogQuery { private Integer lastRows; + /** + * Reverse the reader + */ + private boolean enableTail; + public LogQuery(){ } @@ -88,4 +93,13 @@ public List getOnlyKeywordsList(){ public void setOnlyKeywords(String onlyKeywords) { this.onlyKeywords = onlyKeywords; } + + + public boolean isEnableTail() { + return enableTail; + } + + public void setEnableTail(boolean enableTail) { + this.enableTail = enableTail; + } } diff --git a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/entity/LaunchedExchangisJobEntity.java b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/entity/LaunchedExchangisJobEntity.java index 59b991cd5..bfc9d2a8e 100644 --- a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/entity/LaunchedExchangisJobEntity.java +++ b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/entity/LaunchedExchangisJobEntity.java @@ -1,5 +1,6 @@ package com.webank.wedatasphere.exchangis.job.launcher.entity; +import com.webank.wedatasphere.exchangis.common.EnvironmentUtils; import com.webank.wedatasphere.exchangis.job.domain.ExchangisJobEntity; import com.webank.wedatasphere.exchangis.job.launcher.domain.LaunchableExchangisJob; @@ -8,6 +9,7 @@ import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; +import java.util.Optional; /** * Entity to persist the launched job @@ -58,8 +60,10 @@ public LaunchedExchangisJobEntity(LaunchableExchangisJob job){ this.lastUpdateTime = job.getLastUpdateTime(); this.jobExecutionId = job.getJobExecutionId(); SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd"); - this.logPath = this.executeUser + IOUtils.DIR_SEPARATOR_UNIX + + String logPath = this.executeUser + IOUtils.DIR_SEPARATOR_UNIX + simpleDateFormat.format(new Date()) + IOUtils.DIR_SEPARATOR_UNIX + this.jobExecutionId; + logPath = EnvironmentUtils.getServerAddress() + "@" + logPath; + this.logPath = logPath; } public String getJobExecutionId() { return jobExecutionId; diff --git a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisExchangisTaskLauncher.java b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisExchangisTaskLauncher.java index d5ef7e8e0..ca505987d 100644 --- a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisExchangisTaskLauncher.java +++ b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisExchangisTaskLauncher.java @@ -1,6 +1,6 @@ package com.webank.wedatasphere.exchangis.job.launcher.linkis; -import com.webank.wedatasphere.exchangis.common.linkis.ClientConfiguration; +import com.webank.wedatasphere.exchangis.common.linkis.client.ClientConfiguration; import com.webank.wedatasphere.exchangis.job.enums.EngineTypeEnum; import com.webank.wedatasphere.exchangis.job.launcher.AccessibleLauncherTask; import com.webank.wedatasphere.exchangis.job.launcher.exception.ExchangisTaskLaunchException; @@ -10,16 +10,13 @@ import com.webank.wedatasphere.exchangis.job.launcher.domain.LaunchedExchangisTask; import org.apache.commons.lang.StringUtils; import org.apache.linkis.common.conf.Configuration; -import org.apache.linkis.common.conf.Configuration$; import org.apache.linkis.common.exception.LinkisRetryException; import org.apache.linkis.common.utils.DefaultRetryHandler; import org.apache.linkis.common.utils.RetryHandler; -import org.apache.linkis.computation.client.LinkisJobClient; import org.apache.linkis.computation.client.LinkisJobClient$; import org.apache.linkis.httpclient.config.ClientConfig; import org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy; import org.apache.linkis.httpclient.dws.config.DWSClientConfig; -import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder; import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder$; import java.util.*; diff --git a/exchangis-job/exchangis-job-server/pom.xml b/exchangis-job/exchangis-job-server/pom.xml index df144b413..c4f680478 100644 --- a/exchangis-job/exchangis-job-server/pom.xml +++ b/exchangis-job/exchangis-job-server/pom.xml @@ -49,6 +49,12 @@ mysql-connector-java 5.1.49 + + + org.apache.linkis + linkis-rpc + ${linkis.version} + diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/ExchangisJobExecuteAutoConfiguration.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/ExchangisJobExecuteAutoConfiguration.java index 0c3391d94..3d2139c68 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/ExchangisJobExecuteAutoConfiguration.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/ExchangisJobExecuteAutoConfiguration.java @@ -20,7 +20,7 @@ import com.webank.wedatasphere.exchangis.job.server.execution.subscriber.TaskObserver; import com.webank.wedatasphere.exchangis.job.server.log.DefaultRpcJobLogger; import com.webank.wedatasphere.exchangis.job.server.log.JobLogService; -import com.webank.wedatasphere.exchangis.job.server.log.service.LocalSimpleJobLogService; +import com.webank.wedatasphere.exchangis.job.server.log.service.RpcJobLogService; import com.webank.wedatasphere.exchangis.job.server.utils.SpringContextHolder; import org.apache.linkis.scheduler.Scheduler; import org.apache.linkis.scheduler.executer.ExecutorManager; @@ -50,7 +50,7 @@ public JobLogListener logListener(){ @Bean @ConditionalOnMissingBean(JobLogService.class) public JobLogService jobLogService(){ - return new LocalSimpleJobLogService(); + return new RpcJobLogService(); } /** diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/TransformExchangisJob.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/TransformExchangisJob.java index f2fb0ce24..319f0c7c6 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/TransformExchangisJob.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/TransformExchangisJob.java @@ -168,7 +168,15 @@ private void convertContentToParams(ExchangisJobInfoContent content){ ExchangisJobParamsContent.ExchangisJobParamsItem::getConfigValue)); }); if(Objects.nonNull(paramSet)) { - this.sourceType = resolveDataSourceId(content.getDataSources().getSourceId(), paramSet); + String sourceId = content.getDataSources().getSourceId(); + if (StringUtils.isNotBlank(sourceId)){ + this.sourceType = resolveDataSourceId(content.getDataSources().getSourceId(), paramSet); + } else { + Optional.ofNullable(content.getDataSources().getSource().getType()).ifPresent(type -> { + this.sourceType = type.name; + }); + } + } } @@ -183,7 +191,14 @@ private void convertContentToParams(ExchangisJobInfoContent content){ ExchangisJobParamsContent.ExchangisJobParamsItem::getConfigValue)); }); if(Objects.nonNull(paramSet)) { - this.sinkType = resolveDataSourceId(content.getDataSources().getSinkId(), paramSet); + String sinkId = content.getDataSources().getSinkId(); + if (StringUtils.isNotBlank(sinkId)){ + this.sinkType = resolveDataSourceId(content.getDataSources().getSinkId(), paramSet); + } else { + Optional.ofNullable(content.getDataSources().getSink().getType()).ifPresent(type -> { + this.sinkType = type.name; + }); + } } } } @@ -230,6 +245,9 @@ private String resolveDataSourceId(String dataSourceId, JobParamSet paramSet){ return null; } + private String resolveDataSource(ExchangisJobDataSourcesContent.ExchangisJobDataSource dataSource){ + return null; + } /** * * @param items diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/MetricUpdateSchedulerTask.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/MetricUpdateSchedulerTask.java index 4fded220d..941d36461 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/MetricUpdateSchedulerTask.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/MetricUpdateSchedulerTask.java @@ -31,6 +31,15 @@ public MetricUpdateSchedulerTask(TaskManager taskManager) this.taskManager = taskManager; } + /** + * High priority to get schedule resource + * @return priority + */ + @Override + public int getPriority() { + return 2; + } + @Override protected void onPoll(LaunchedExchangisTask launchedExchangisTask) throws ExchangisSchedulerException, ExchangisSchedulerRetryException { LOG.trace("Metrics update task: [{}] in scheduler: [{}]", launchedExchangisTask.getTaskId(), getName()); diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/StatusUpdateSchedulerTask.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/StatusUpdateSchedulerTask.java index 3bfc64374..7551ec207 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/StatusUpdateSchedulerTask.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/StatusUpdateSchedulerTask.java @@ -26,6 +26,15 @@ public class StatusUpdateSchedulerTask extends AbstractLoadBalanceSchedulerTask< private TaskManager taskManager; + /** + * High priority to get schedule resource + * @return priority + */ + @Override + public int getPriority() { + return 2; + } + public StatusUpdateSchedulerTask(TaskManager taskManager){ this.taskManager = taskManager; } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/rpc/FetchLogRequest.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/rpc/FetchLogRequest.java new file mode 100644 index 000000000..18d9a5f8b --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/rpc/FetchLogRequest.java @@ -0,0 +1,31 @@ +package com.webank.wedatasphere.exchangis.job.server.log.rpc; + +import com.webank.wedatasphere.exchangis.job.log.LogQuery; +import org.apache.linkis.protocol.message.RequestProtocol; + +/** + * Fetch log request + */ +public class FetchLogRequest extends LogQuery implements RequestProtocol { + + /** + * Log path + */ + private String logPath; + + public FetchLogRequest(LogQuery logQuery, String logPath){ + super(logQuery.getFromLine(), logQuery.getPageSize(), + logQuery.getIgnoreKeywords(), logQuery.getOnlyKeywords(), + logQuery.getLastRows()); + setEnableTail(logQuery.isEnableTail()); + this.logPath = logPath; + } + + public String getLogPath() { + return logPath; + } + + public void setLogPath(String logPath) { + this.logPath = logPath; + } +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/rpc/FetchLogResponse.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/rpc/FetchLogResponse.java new file mode 100644 index 000000000..b9dd399b4 --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/rpc/FetchLogResponse.java @@ -0,0 +1,20 @@ +package com.webank.wedatasphere.exchangis.job.server.log.rpc; + +import com.webank.wedatasphere.exchangis.job.log.LogResult; +import org.apache.linkis.protocol.message.RequestProtocol; + +import java.util.List; + +/** + * Extend log result + */ +public class FetchLogResponse extends LogResult implements RequestProtocol { + + public FetchLogResponse(LogResult logResult){ + super(logResult.getEndLine(), logResult.isEnd(), logResult.getLogs()); + } + + public FetchLogResponse(int endLine, boolean isEnd, List logs) { + super(endLine, isEnd, logs); + } +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/rpc/SendLogRequest.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/rpc/SendLogRequest.java new file mode 100644 index 000000000..8310dfef0 --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/rpc/SendLogRequest.java @@ -0,0 +1,57 @@ +package com.webank.wedatasphere.exchangis.job.server.log.rpc; + +import org.apache.linkis.protocol.message.RequestProtocol; + +import java.util.ArrayList; +import java.util.List; + +/** + * Send log request + */ +public class SendLogRequest implements RequestProtocol { + /** + * Exec id + */ + private String jobExecId; + + /** + * Is reached the end of log + */ + private boolean isEnd; + /** + * Log lines + */ + private List logLines = new ArrayList<>(); + + public SendLogRequest(String jobExecId, + boolean isEnd, + List logLines){ + this.jobExecId = jobExecId; + this.isEnd = isEnd; + this.logLines = logLines; + } + + public String getJobExecId() { + return jobExecId; + } + + public void setJobExecId(String jobExecId) { + this.jobExecId = jobExecId; + } + + public List getLogLines() { + return logLines; + } + + public void setLogLines(List logLines) { + this.logLines = logLines; + } + + public boolean isEnd() { + return isEnd; + } + + public void setEnd(boolean end) { + isEnd = end; + } +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/AbstractJobLogService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/AbstractJobLogService.java new file mode 100644 index 000000000..f75591a92 --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/AbstractJobLogService.java @@ -0,0 +1,159 @@ +package com.webank.wedatasphere.exchangis.job.server.log.service; + +import com.google.common.cache.*; +import com.webank.wedatasphere.exchangis.job.launcher.entity.LaunchedExchangisJobEntity; +import com.webank.wedatasphere.exchangis.job.log.LogQuery; +import com.webank.wedatasphere.exchangis.job.log.LogResult; +import com.webank.wedatasphere.exchangis.job.server.mapper.LaunchedJobDao; +import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; +import com.webank.wedatasphere.exchangis.job.server.execution.scheduler.AbstractExchangisSchedulerTask; +import com.webank.wedatasphere.exchangis.job.server.log.JobLogService; +import com.webank.wedatasphere.exchangis.job.server.log.cache.AbstractJobLogCache; +import com.webank.wedatasphere.exchangis.job.server.log.cache.JobLogCache; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.scheduler.Scheduler; +import org.apache.linkis.scheduler.queue.JobInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import javax.annotation.Resource; +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.webank.wedatasphere.exchangis.job.exception.ExchangisJobExceptionCode.LOG_OP_ERROR; + +/** + * Abstract Job log service + */ +public abstract class AbstractJobLogService implements JobLogService { + + private static final Logger LOG = LoggerFactory.getLogger(AbstractJobLogService.class); + + protected Cache> cacheHolder; + + private AbstractExchangisSchedulerTask cleaner; + + private volatile boolean cleanerOn; + + protected static class Constraints{ + public static final CommonVars LOG_LOCAL_PATH = CommonVars.apply("wds.exchangis.job.log.local.path", "/data/bdp/dss/exchangis/main/logs"); + + public static final CommonVars lOG_CACHE_SIZE = CommonVars.apply("wds.exchangis.job.log.cache.size", 15); + + public static final CommonVars LOG_CACHE_EXPIRE_TIME_IN_SECONDS = CommonVars.apply("wds.exchangis.job.log.cache.expire.time-in-seconds", 5); + + public static final CommonVars LOG_MULTILINE_PATTERN = CommonVars.apply("wds.exchangis.log.multiline.pattern", "^\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}\\.\\d{3}"); + } + + @Resource + protected Scheduler scheduler; + + @Resource + private LaunchedJobDao launchedJobDao; + @PostConstruct + public void init(){ + cleanerOn = true; + cacheHolder = CacheBuilder.newBuilder().maximumSize(Constraints.lOG_CACHE_SIZE.getValue()) + .expireAfterAccess(Constraints.LOG_CACHE_EXPIRE_TIME_IN_SECONDS.getValue(), TimeUnit.SECONDS) + .removalListener((RemovalListener>) removalNotification -> { + // Flush for expired + if (removalNotification.getCause() == RemovalCause.EXPIRED){ + removalNotification.getValue().flushCache(true); + } + }) + .build(); + cleaner = new AbstractExchangisSchedulerTask("Job-Log-Cache-Cleaner") { + @Override + public String getTenancy() { + return "log"; + } + + @Override + public String getName() { + return getId(); + } + + @Override + public JobInfo getJobInfo() { + return null; + } + + @Override + protected void schedule() { + while(cleanerOn){ + try { + Thread.sleep(Constraints.LOG_CACHE_EXPIRE_TIME_IN_SECONDS.getValue()); + //Just invoke the auto cleaner + cacheHolder.get("log", () -> null); + } catch (Exception e){ + //Ignore + } + } + } + }; + scheduler.submit(cleaner); + } + + @PreDestroy + public void destroy(){ + this.cleanerOn = false; + if (Objects.nonNull(this.cleaner.future())){ + this.cleaner.future().cancel(true); + } + } + + @Override + public LogResult logsFromPage( String jobExecId, LogQuery logQuery) { + LaunchedExchangisJobEntity launchedExchangisJob = launchedJobDao.searchLogPathInfo(jobExecId); + return logsFromPageAndPath(launchedExchangisJob.getLogPath(), logQuery); + } + + @Override + public void appendLog(String tenancy, String jobExecId, List logs) { + appendLog(jobExecId, logs); + } + + @Override + public void appendLog(String jobExecId, List logs) { + JobLogCache cache = getOrCreateLogCache(jobExecId); + logs.forEach(cache ::cacheLog); + } + + + @Override + public JobLogCache getOrCreateLogCache(String jobExecId){ + try { + return cacheHolder.get(jobExecId, () -> { + LaunchedExchangisJobEntity launchedExchangisJob = launchedJobDao.searchLogPathInfo(jobExecId); + if (Objects.nonNull(launchedExchangisJob)) { + + } + return null; + }); + } catch (ExecutionException e) { + throw new ExchangisJobServerException.Runtime(LOG_OP_ERROR.getCode(),"Fail to create the job log cache of [" + jobExecId +"]", e); + } + } + + /** + * Load job log cache + * @param launchedExchangisJob job + * @return log cache + */ + protected abstract AbstractJobLogCache loadJobLogCache(String jobExcId, LaunchedExchangisJobEntity launchedExchangisJob) + throws Exception; +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/LocalSimpleJobLogService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/LocalSimpleJobLogService.java deleted file mode 100644 index 1eea657ce..000000000 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/LocalSimpleJobLogService.java +++ /dev/null @@ -1,282 +0,0 @@ -package com.webank.wedatasphere.exchangis.job.server.log.service; - -import com.google.common.cache.*; -import com.webank.wedatasphere.exchangis.job.launcher.entity.LaunchedExchangisJobEntity; -import com.webank.wedatasphere.exchangis.job.log.LogQuery; -import com.webank.wedatasphere.exchangis.job.log.LogResult; -import com.webank.wedatasphere.exchangis.job.server.mapper.LaunchedJobDao; -import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; -import com.webank.wedatasphere.exchangis.job.server.execution.scheduler.AbstractExchangisSchedulerTask; -import com.webank.wedatasphere.exchangis.job.server.log.JobLogService; -import com.webank.wedatasphere.exchangis.job.server.log.cache.AbstractJobLogCache; -import com.webank.wedatasphere.exchangis.job.server.log.cache.JobLogCache; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.linkis.common.conf.CommonVars; -import org.apache.linkis.common.utils.Utils; -import org.apache.linkis.scheduler.Scheduler; -import org.apache.linkis.scheduler.queue.JobInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import javax.annotation.Resource; -import java.io.File; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.util.*; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import static com.webank.wedatasphere.exchangis.job.exception.ExchangisJobExceptionCode.LOG_OP_ERROR; - -/** - * Just store the log into the local - */ -public class LocalSimpleJobLogService implements JobLogService { - - private static final Logger LOG = LoggerFactory.getLogger(LocalSimpleJobLogService.class); - - private Cache> cacheHolder; - - private AbstractExchangisSchedulerTask cleaner; - - private volatile boolean cleanerOn; - - private static class Constraints{ - public static final CommonVars LOG_LOCAL_PATH = CommonVars.apply("wds.exchangis.job.log.local.path", "/data/bdp/dss/exchangis/main/logs"); - - public static final CommonVars lOG_CACHE_SIZE = CommonVars.apply("wds.exchangis.job.log.cache.size", 15); - - public static final CommonVars LOG_CACHE_EXPIRE_TIME_IN_SECONDS = CommonVars.apply("wds.exchangis.job.log.cache.expire.time-in-seconds", 5); - - public static final CommonVars LOG_MULTILINE_PATTERN = CommonVars.apply("wds.exchangis.log.multiline.pattern", "^\\d{4}-\\d{2}-\\d{2}\\s+\\d{2}:\\d{2}:\\d{2}\\.\\d{3}"); - } - - @Resource - private Scheduler scheduler; - - @Resource - private LaunchedJobDao launchedJobDao; - @PostConstruct - public void init(){ - cleanerOn = true; - cacheHolder = CacheBuilder.newBuilder().maximumSize(Constraints.lOG_CACHE_SIZE.getValue()) - .expireAfterAccess(Constraints.LOG_CACHE_EXPIRE_TIME_IN_SECONDS.getValue(), TimeUnit.SECONDS) - .removalListener((RemovalListener>) removalNotification -> { - // Flush for expired - if (removalNotification.getCause() == RemovalCause.EXPIRED){ - removalNotification.getValue().flushCache(true); - } - }) - .build(); - cleaner = new AbstractExchangisSchedulerTask("Job-Log-Cache-Cleaner") { - @Override - public String getTenancy() { - return "log"; - } - - @Override - public String getName() { - return getId(); - } - - @Override - public JobInfo getJobInfo() { - return null; - } - - @Override - protected void schedule() { - while(cleanerOn){ - try { - Thread.sleep(Constraints.LOG_CACHE_EXPIRE_TIME_IN_SECONDS.getValue()); - //Just invoke the auto cleaner - cacheHolder.get("log", () -> null); - } catch (Exception e){ - //Ignore - } - } - } - }; - scheduler.submit(cleaner); - } - - @PreDestroy - public void destroy(){ - this.cleanerOn = false; - if (Objects.nonNull(this.cleaner.future())){ - this.cleaner.future().cancel(true); - } - } - - @Override - public LogResult logsFromPage( String jobExecId, LogQuery logQuery) { - LaunchedExchangisJobEntity launchedExchangisJob = launchedJobDao.searchLogPathInfo(jobExecId); - return logsFromPageAndPath(launchedExchangisJob.getLogPath(), logQuery); - } - - @Override - public LogResult logsFromPageAndPath(String logPath, LogQuery logQuery) { - String fullPath = Constraints.LOG_LOCAL_PATH.getValue() + IOUtils.DIR_SEPARATOR_UNIX + logPath; - LogResult result = new LogResult(0, false, Collections.emptyList()); - if (!new File(fullPath).exists()){ - return result; - } - if (logQuery.getLastRows() != null && logQuery.getLastRows() > 0){ - return getLastRows(fullPath, logQuery.getLastRows()); - } - RandomAccessFile logReader = null; - try { - logReader = new RandomAccessFile(fullPath, "rw"); - String patternValue = Constraints.LOG_MULTILINE_PATTERN.getValue(); - Pattern linePattern = StringUtils.isNotBlank(patternValue)? Pattern.compile(patternValue) : null; - int readLine = 0; - int lineNum = 0; - int skippedLine = 0; - int ignoreLine = 0; - int pageSize = logQuery.getPageSize(); - int fromLine = logQuery.getFromLine(); - List ignoreKeywords = logQuery.getIgnoreKeywordsList(); - List onlyKeywords = logQuery.getOnlyKeywordsList(); - boolean rowIgnore = false; - String line = logReader.readLine(); - List logs = new ArrayList<>(); - while (readLine < pageSize && line != null){ - lineNum += 1; - if (skippedLine < fromLine - 1){ - skippedLine += 1; - } else { - if (rowIgnore) { - if (Objects.nonNull(linePattern)){ - Matcher matcher = linePattern.matcher(line); - if (matcher.find()){ - ignoreLine = 0; - rowIgnore = !isIncludeLine(line, onlyKeywords, ignoreKeywords); - } else { - ignoreLine += 1; - // TODO limit the value of ignoreLine - } - }else{ - rowIgnore = !isIncludeLine(line, onlyKeywords, ignoreKeywords); - } - }else { - rowIgnore = !isIncludeLine(line, onlyKeywords, ignoreKeywords); - } - if (!rowIgnore) { - if (line.contains("password")) { - LOG.info("have error information"); - } - if (!line.contains("password")) { - logs.add(new String(line.getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8)); - } - readLine += 1; - } - } - line = logReader.readLine(); - } - result = new LogResult(lineNum, false, logs); - } catch (IOException e) { - throw new ExchangisJobServerException.Runtime(LOG_OP_ERROR.getCode(),"Unable to query the logs from path: [" + logPath + "]", e); - } finally { - if (Objects.nonNull(logReader)) { - try { - logReader.close(); - } catch (IOException e) { - //Ignore - } - } - } - return result; - } - - @Override - public void appendLog(String tenancy, String jobExecId, List logs) { - appendLog(jobExecId, logs); - } - - @Override - public void appendLog(String jobExecId, List logs) { - JobLogCache cache = getOrCreateLogCache(jobExecId); - logs.forEach(cache ::cacheLog); - } - - - private boolean isIncludeLine(String line, List onlyKeywordList, List ignoreKeywordList){ - boolean accept = ignoreKeywordList.isEmpty() || ignoreKeywordList.stream().noneMatch(line::contains); - if (accept){ - accept = onlyKeywordList.isEmpty() || onlyKeywordList.stream().anyMatch(line::contains); - } - return accept; - } - - /** - * Get last rows - * @param fullPath full path - * @param lastRows last rows - * @return - */ - private LogResult getLastRows(String fullPath, int lastRows){ - try { - List logs = Arrays.asList(Utils.exec(new String[]{"tail", "-n", lastRows + "", fullPath}, 5000L).split("\n")); - return new LogResult(0, true, logs); - }catch (Exception e){ - throw new ExchangisJobServerException.Runtime(LOG_OP_ERROR.getCode(), "Fail to get last rows from path: [" + fullPath + "]", e); - } - } - @Override - public JobLogCache getOrCreateLogCache(String jobExecId){ - try { - return cacheHolder.get(jobExecId, () -> { - LaunchedExchangisJobEntity launchedExchangisJob = launchedJobDao.searchLogPathInfo(jobExecId); - if (Objects.nonNull(launchedExchangisJob)) { - File logFile = new File(Constraints.LOG_LOCAL_PATH.getValue() + IOUtils.DIR_SEPARATOR_UNIX + - launchedExchangisJob.getLogPath()); - if (!logFile.exists()){ - // Write empty string to create new file - FileUtils.writeStringToFile(logFile, ""); - LOG.info("Create the new job log file: {}", logFile.getAbsolutePath()); - } - RandomAccessFile file = new RandomAccessFile(logFile, "rw"); - // Seek to the end of file - file.seek(file.length()); - return new AbstractJobLogCache(scheduler, 100, 2000) { - @Override - public synchronized void flushCache(boolean isEnd) { - // Store into local path - if (!cacheQueue().isEmpty()) { - try { - List logLines = new ArrayList<>(); - cacheQueue().drainTo(logLines); - for (Object line : logLines) { - file.write(String.valueOf(line).getBytes(Charset.defaultCharset())); - } - } catch (IOException ex) { - LOG.error("Fail to flush the log cache of [" + launchedExchangisJob.getJobExecutionId() + "]", ex); - } - } - if (isEnd) { - cacheHolder.invalidate(jobExecId); - try { - file.close(); - } catch (IOException e) { - //Ignore - } - } - } - }; - } - return null; - }); - } catch (ExecutionException e) { - throw new ExchangisJobServerException.Runtime(LOG_OP_ERROR.getCode(),"Fail to create the job log cache of [" + jobExecId +"]", e); - } - } - -} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/RpcJobLogService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/RpcJobLogService.java new file mode 100644 index 000000000..14b73d585 --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/RpcJobLogService.java @@ -0,0 +1,280 @@ +package com.webank.wedatasphere.exchangis.job.server.log.service; + +import com.webank.wedatasphere.exchangis.common.EnvironmentUtils; +import com.webank.wedatasphere.exchangis.job.launcher.entity.LaunchedExchangisJobEntity; +import com.webank.wedatasphere.exchangis.job.log.LogQuery; +import com.webank.wedatasphere.exchangis.job.log.LogResult; +import com.webank.wedatasphere.exchangis.job.server.exception.ExchangisJobServerException; +import com.webank.wedatasphere.exchangis.job.server.log.cache.AbstractJobLogCache; +import com.webank.wedatasphere.exchangis.job.server.log.cache.JobLogCache; +import com.webank.wedatasphere.exchangis.job.server.log.rpc.FetchLogRequest; +import com.webank.wedatasphere.exchangis.job.server.log.rpc.FetchLogResponse; +import com.webank.wedatasphere.exchangis.job.server.log.rpc.SendLogRequest; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.input.ReversedLinesFileReader; +import org.apache.commons.lang.StringUtils; +import org.apache.linkis.common.ServiceInstance; +import org.apache.linkis.common.utils.Utils; +import org.apache.linkis.rpc.Sender; +import org.apache.linkis.rpc.message.annotation.Receiver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.function.Supplier; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.webank.wedatasphere.exchangis.job.exception.ExchangisJobExceptionCode.LOG_OP_ERROR; + +/** + * Rpc job log service + */ +public class RpcJobLogService extends AbstractJobLogService{ + + private static final Logger LOG = LoggerFactory.getLogger(RpcJobLogService.class); + + + @Receiver + public void appendLog(SendLogRequest sendLogRequest){ + String jobExecId = sendLogRequest.getJobExecId(); + List logLines = sendLogRequest.getLogLines(); + if (logLines.size() > 0) { + // Two level cache + JobLogCache cache = getOrCreateLogCache(jobExecId); + logLines.forEach(cache :: cacheLog); + if (sendLogRequest.isEnd()){ + cache.flushCache(true); + } + } else if (sendLogRequest.isEnd()){ + Optional.ofNullable(cacheHolder.getIfPresent(jobExecId)).ifPresent( cache -> { + cache.flushCache(true); + }); + } + } + + @Receiver + public FetchLogResponse logsFromPage(FetchLogRequest fetchLogRequest){ + return new FetchLogResponse( + logsFromPageAndPath(fetchLogRequest.getLogPath(), fetchLogRequest)); + } + @Override + protected AbstractJobLogCache loadJobLogCache(String jobExecId, + LaunchedExchangisJobEntity launchedExchangisJob) throws Exception{ + String logPath = launchedExchangisJob.getLogPath(); + int splitPos = logPath.indexOf("@"); + if (splitPos > 0){ + String logAddress = logPath.substring(0, splitPos); + if (!logAddress.equals(EnvironmentUtils.getServerAddress())){ + ServiceInstance instance = ServiceInstance.apply(EnvironmentUtils.getServerName(), logAddress); + return new AbstractJobLogCache(scheduler, 100, 2000) { + @Override + public void flushCache(boolean isEnd) { + // Send rpc + if (!cacheQueue().isEmpty()) { + try { + List logLines = new ArrayList<>(); + cacheQueue().drainTo(logLines); + Sender.getSender(instance).send(new SendLogRequest(jobExecId, isEnd, logLines)); + } catch (Exception ex) { + LOG.error("Fail to send the log cache of [" + launchedExchangisJob.getJobExecutionId() + + "] to remote rpc [" + logAddress + "]", ex); + } + } + if (isEnd) { + cacheHolder.invalidate(jobExecId); + } + } + }; + } + } + File logFile = new File(Constraints.LOG_LOCAL_PATH.getValue() + IOUtils.DIR_SEPARATOR_UNIX + + launchedExchangisJob.getLogPath()); + + if (!logFile.exists()){ + // Write empty string to create new file + FileUtils.writeStringToFile(logFile, ""); + LOG.info("Create the new job log file: {}", logFile.getAbsolutePath()); + } + RandomAccessFile file = new RandomAccessFile(logFile, "rw"); + // Seek to the end of file + file.seek(file.length()); + return new AbstractJobLogCache(scheduler, 100, 2000) { + @Override + public synchronized void flushCache(boolean isEnd) { + // Store into local path + if (!cacheQueue().isEmpty()) { + try { + List logLines = new ArrayList<>(); + cacheQueue().drainTo(logLines); + for (Object line : logLines) { + file.write(String.valueOf(line).getBytes(Charset.defaultCharset())); + } + } catch (IOException ex) { + LOG.error("Fail to flush the log cache of [" + launchedExchangisJob.getJobExecutionId() + "]", ex); + } + } + if (isEnd) { + cacheHolder.invalidate(jobExecId); + try { + file.close(); + } catch (IOException e) { + //Ignore + } + } + } + }; + } + + @Override + public LogResult logsFromPageAndPath(String logPath, LogQuery logQuery) { + int splitPos = logPath.indexOf("@"); + if (splitPos > 0) { + String logAddress = logPath.substring(0, splitPos); + if (!logAddress.equals(EnvironmentUtils.getServerAddress())) { + Object response; + try { + response = Sender.getSender(ServiceInstance.apply(EnvironmentUtils.getServerName(), logAddress)) + .ask(new FetchLogRequest(logQuery, logPath)); + } catch (Exception e){ + throw new ExchangisJobServerException.Runtime(LOG_OP_ERROR.getCode(), + "Remote exception in fetching log from: [" + logPath + "]", e); + } + if (response instanceof FetchLogResponse){ + return (LogResult) response; + } + throw new ExchangisJobServerException.Runtime(LOG_OP_ERROR.getCode(),"Unable to fetch log from: [" + logPath + + "], unknown request protocol: [" + response + "]", null); + } + } + String fullPath = Constraints.LOG_LOCAL_PATH.getValue() + IOUtils.DIR_SEPARATOR_UNIX + logPath; + LogResult result = new LogResult(0, false, Collections.emptyList()); + if (!new File(fullPath).exists()){ + return result; + } + if (logQuery.getLastRows() != null && logQuery.getLastRows() > 0){ + return getLastRows(fullPath, logQuery.getLastRows()); + } + RandomAccessFile logReader = null; + ReversedLinesFileReader reverseReader = null; + try { + String patternValue = Constraints.LOG_MULTILINE_PATTERN.getValue(); + Pattern linePattern = StringUtils.isNotBlank(patternValue)? Pattern.compile(patternValue) : null; + int readLine = 0; + int lineNum = 0; + int skippedLine = 0; + int ignoreLine = 0; + int pageSize = logQuery.getPageSize(); + int fromLine = logQuery.getFromLine(); + List ignoreKeywords = logQuery.getIgnoreKeywordsList(); + List onlyKeywords = logQuery.getOnlyKeywordsList(); + boolean rowIgnore = false; + Supplier lineSupplier = null; + if (logQuery.isEnableTail()){ + reverseReader = new ReversedLinesFileReader(new File(fullPath), Charset.defaultCharset()); + LOG.trace("Enable reverse read the log: {}, fromLine: {}, pageSize: {}", fullPath, fromLine, pageSize); + ReversedLinesFileReader finalReverseReader = reverseReader; + lineSupplier = () -> { + try { + return finalReverseReader.readLine(); + } catch (IOException e) { + throw new ExchangisJobServerException.Runtime(LOG_OP_ERROR.getCode(), e.getMessage(), e); + } + }; + } else { + logReader = new RandomAccessFile(fullPath, "rw"); + RandomAccessFile finalLogReader = logReader; + lineSupplier = () -> { + try { + return finalLogReader.readLine(); + } catch (IOException e) { + throw new ExchangisJobServerException.Runtime(LOG_OP_ERROR.getCode(), e.getMessage(), e); + } + }; + } + String line = lineSupplier.get(); + List logs = new ArrayList<>(); + while (readLine < pageSize && line != null){ + lineNum += 1; + if (skippedLine < fromLine - 1){ + skippedLine += 1; + } else { + if (rowIgnore) { + if (Objects.nonNull(linePattern)){ + Matcher matcher = linePattern.matcher(line); + if (matcher.find()){ + ignoreLine = 0; + rowIgnore = !isIncludeLine(line, onlyKeywords, ignoreKeywords); + } else { + ignoreLine += 1; + // TODO limit the value of ignoreLine + } + }else{ + rowIgnore = !isIncludeLine(line, onlyKeywords, ignoreKeywords); + } + }else { + rowIgnore = !isIncludeLine(line, onlyKeywords, ignoreKeywords); + } + if (!rowIgnore) { + if (line.contains("password")) { + LOG.info("have error information"); + } + if (!line.contains("password")) { + logs.add(new String(line.getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8)); + } + readLine += 1; + } + } + line = lineSupplier.get(); + } + result = new LogResult(lineNum, false, logs); + } catch (IOException e) { + throw new ExchangisJobServerException.Runtime(LOG_OP_ERROR.getCode(),"Unable to query the logs from path: [" + logPath + "]", e); + } finally { + if (Objects.nonNull(logReader)) { + try { + logReader.close(); + } catch (IOException e) { + //Ignore + } + } + if (Objects.nonNull(reverseReader)) { + try { + reverseReader.close(); + } catch (IOException e) { + //Ignore + } + } + } + return result; + } + + /** + * Get last rows + * @param fullPath full path + * @param lastRows last rows + * @return + */ + private LogResult getLastRows(String fullPath, int lastRows){ + try { + List logs = Arrays.asList(Utils.exec(new String[]{"tail", "-n", lastRows + "", fullPath}, 5000L).split("\n")); + return new LogResult(0, true, logs); + }catch (Exception e){ + throw new ExchangisJobServerException.Runtime(LOG_OP_ERROR.getCode(), "Fail to get last rows from path: [" + fullPath + "]", e); + } + } + + private boolean isIncludeLine(String line, List onlyKeywordList, List ignoreKeywordList){ + boolean accept = ignoreKeywordList.isEmpty() || ignoreKeywordList.stream().noneMatch(line::contains); + if (accept){ + accept = onlyKeywordList.isEmpty() || onlyKeywordList.stream().anyMatch(line::contains); + } + return accept; + } +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/render/transform/TransformRequestVo.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/render/transform/TransformRequestVo.java index 124f894eb..2bbc689f9 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/render/transform/TransformRequestVo.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/render/transform/TransformRequestVo.java @@ -32,6 +32,11 @@ public class TransformRequestVo { */ private String sourceTable; + /** + * Table (source) not exist + */ + private boolean srcTblNotExist = false; + /** * Sink type id */ @@ -54,6 +59,10 @@ public class TransformRequestVo { */ private String sinkTable; + /** + * Table (sink) not exist + */ + private boolean sinkTblNotExist = false; /** * Labels */ @@ -150,4 +159,20 @@ public String getOperator() { public void setOperator(String operator) { this.operator = operator; } + + public void setSrcTblNotExist(boolean srcTblNotExist) { + this.srcTblNotExist = srcTblNotExist; + } + + public void setSinkTblNotExist(boolean sinkTblNotExist) { + this.sinkTblNotExist = sinkTblNotExist; + } + + public boolean isSrcTblNotExist() { + return srcTblNotExist; + } + + public boolean isSinkTblNotExist() { + return sinkTblNotExist; + } } diff --git a/exchangis-job/exchangis-job-server/src/main/scala/com/webank/wedatasphere/exchangis/job/server/log/cache/JobLogCache.scala b/exchangis-job/exchangis-job-server/src/main/scala/com/webank/wedatasphere/exchangis/job/server/log/cache/JobLogCache.scala index 1986373ae..75721f8ee 100644 --- a/exchangis-job/exchangis-job-server/src/main/scala/com/webank/wedatasphere/exchangis/job/server/log/cache/JobLogCache.scala +++ b/exchangis-job/exchangis-job-server/src/main/scala/com/webank/wedatasphere/exchangis/job/server/log/cache/JobLogCache.scala @@ -35,7 +35,7 @@ abstract class AbstractJobLogCache[V](scheduler: Scheduler, maxSize: Int = 100, var lastFlush: Long = -1L - var cacheQueue: util.concurrent.ArrayBlockingQueue[Any] = new ArrayBlockingQueue[Any](maxSize) + var cacheQueue: util.concurrent.ArrayBlockingQueue[V] = new ArrayBlockingQueue[V](maxSize) var isShutdown: Boolean = false @@ -65,7 +65,7 @@ abstract class AbstractJobLogCache[V](scheduler: Scheduler, maxSize: Int = 100, override def cacheLog(log: V): Unit = { val element: Any = getCacheQueueElement(log) - if (!cacheQueue.offer(element)) { + if (!cacheQueue.offer(element.asInstanceOf[V])) { warn("The cache queue is full, should flush the cache immediately") flushCache(false) } else if (lastFlush + flushInterval < System.currentTimeMillis){ diff --git a/pom.xml b/pom.xml index e543e521f..a8c905e84 100644 --- a/pom.xml +++ b/pom.xml @@ -131,6 +131,12 @@ + + + org.apache.linkis + linkis-gateway-httpclient-support + ${linkis.version} + org.apache.linkis linkis-common From 505b01df5ddf18b0164f9e159d4762f7c93dca40 Mon Sep 17 00:00:00 2001 From: davidhua Date: Fri, 17 May 2024 15:54:39 +0800 Subject: [PATCH 17/60] Add heartbeat feature to datax engine plugin. --- .../engineconn-plugins/datax/pom.xml | 10 +++++++ .../config/DataxSpringConfiguration.scala | 23 ++++++++++++++ .../executor/DataxContainerOnceExecutor.scala | 23 ++++++++++++++ .../service/DataxHeartbeatMsgManager.scala | 30 +++++++++++++++++++ 4 files changed, 86 insertions(+) create mode 100644 exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/config/DataxSpringConfiguration.scala create mode 100644 exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/service/DataxHeartbeatMsgManager.scala diff --git a/exchangis-engines/engineconn-plugins/datax/pom.xml b/exchangis-engines/engineconn-plugins/datax/pom.xml index af4629d1b..8fc8d4977 100644 --- a/exchangis-engines/engineconn-plugins/datax/pom.xml +++ b/exchangis-engines/engineconn-plugins/datax/pom.xml @@ -116,6 +116,16 @@ linkis-udf-client 1.4.0 + + org.apache.linkis + linkis-module + + + org.springframework.cloud + spring-cloud-commons + + + diff --git a/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/config/DataxSpringConfiguration.scala b/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/config/DataxSpringConfiguration.scala new file mode 100644 index 000000000..06609c615 --- /dev/null +++ b/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/config/DataxSpringConfiguration.scala @@ -0,0 +1,23 @@ +package org.apache.linkis.engineconnplugin.datax.config + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.engineconn.acessible.executor.info.NodeHeartbeatMsgManager +import org.apache.linkis.engineconnplugin.datax.service.DataxHeartbeatMsgManager +import org.springframework.context.annotation.{Bean, Configuration, Primary} + +/** + * Spring configuration for datax + */ +@Configuration +class DataxSpringConfiguration extends Logging { + + /** + * Override the heartbeat manager + * @return + */ + @Bean + @Primary + def nodeHeartbeatMsgManager(): NodeHeartbeatMsgManager = { + new DataxHeartbeatMsgManager() + } +} diff --git a/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/executor/DataxContainerOnceExecutor.scala b/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/executor/DataxContainerOnceExecutor.scala index 434108649..b3f7a671a 100644 --- a/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/executor/DataxContainerOnceExecutor.scala +++ b/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/executor/DataxContainerOnceExecutor.scala @@ -26,7 +26,9 @@ import com.alibaba.datax.core.util.container.{CoreConstant, LoadUtil} import com.alibaba.datax.core.util.{ConfigurationValidate, ExceptionTracker, FrameworkErrorCode, SecretUtil} import org.apache.commons.lang3.StringUtils import org.apache.linkis.common.utils.{ClassUtils, Utils} +import org.apache.linkis.engineconn.acessible.executor.service.{ExecutorHeartbeatService, ExecutorHeartbeatServiceHolder} import org.apache.linkis.engineconn.common.creation.EngineCreationContext +import org.apache.linkis.engineconn.executor.service.ManagerService import org.apache.linkis.engineconn.once.executor.{OnceExecutorExecutionContext, OperableOnceExecutor} import org.apache.linkis.engineconnplugin.datax.config.DataxConfiguration import org.apache.linkis.engineconnplugin.datax.exception.{DataxJobExecutionException, DataxPluginLoadException} @@ -86,6 +88,8 @@ abstract class DataxContainerOnceExecutor extends DataxOnceExecutor with Operabl } info(s"The executor: [${getId}] has been finished, now to stop DataxEngineConn.") closeDaemon() + // Try to heartbeat at last + tryToHeartbeat() if (!isFailed) { trySucceed() } @@ -103,6 +107,8 @@ abstract class DataxContainerOnceExecutor extends DataxOnceExecutor with Operabl if (!(future.isDone || future.isCancelled)) { trace(s"The executor: [$getId] has been still running") } + // Heartbeat action interval + tryToHeartbeat() } }, DataxConfiguration.STATUS_FETCH_INTERVAL.getValue.toLong, DataxConfiguration.STATUS_FETCH_INTERVAL.getValue.toLong, TimeUnit.MILLISECONDS) @@ -143,6 +149,10 @@ abstract class DataxContainerOnceExecutor extends DataxOnceExecutor with Operabl metrics } + def getMessage(key: String):util.Map[String, util.List[String]] = { + null + } + override def getDiagnosis: util.Map[String, Any] = { // Not support diagnosis new util.HashMap[String, Any]() @@ -156,6 +166,19 @@ abstract class DataxContainerOnceExecutor extends DataxOnceExecutor with Operabl // Option(this.container).foreach(_.shutdown()) super.tryFailed() } + + /** + * Try to send heartbeat message to ecm + */ + private def tryToHeartbeat(): Unit = { + logger.trace("heartbeat and record to linkis manager") + ExecutorHeartbeatServiceHolder.getDefaultHeartbeatService() match { + case heartbeatService: ExecutorHeartbeatService => + val heartbeatMsg = heartbeatService.generateHeartBeatMsg(this) + ManagerService.getManagerService.heartbeatReport(heartbeatMsg) + logger.trace(s"Succeed to report heartbeatMsg: [${heartbeatMsg}]") + } + } /** * Execute with job content * @param jobContent job content diff --git a/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/service/DataxHeartbeatMsgManager.scala b/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/service/DataxHeartbeatMsgManager.scala new file mode 100644 index 000000000..26b26392c --- /dev/null +++ b/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/service/DataxHeartbeatMsgManager.scala @@ -0,0 +1,30 @@ +package org.apache.linkis.engineconnplugin.datax.service + +import org.apache.linkis.common.utils.{Logging, Utils} +import org.apache.linkis.engineconn.acessible.executor.info.NodeHeartbeatMsgManager +import org.apache.linkis.engineconn.executor.entity.Executor +import org.apache.linkis.engineconnplugin.datax.executor.DataxContainerOnceExecutor +import org.apache.linkis.server.BDPJettyServerHelper + +import scala.collection.JavaConverters.mapAsScalaMapConverter + +/** + * Datax heartbeat message (include: metric, error message) + */ +class DataxHeartbeatMsgManager extends NodeHeartbeatMsgManager with Logging{ + override def getHeartBeatMsg(executor: Executor): String = { + executor match { + case dataxExecutor: DataxContainerOnceExecutor => + val metric = dataxExecutor.getMetrics + Utils.tryCatch(BDPJettyServerHelper.gson.toJson(metric)) { case e: Exception => + val mV = metric.asScala + .map { case (k, v) => if (null == v) s"${k}->null" else s"${k}->${v.toString}" } + .mkString(",") + val errMsg = e.getMessage + logger.error(s"Convert metric to json failed because : ${errMsg}, metric values : {${mV}}") + "{\"errorMsg\":\"Convert metric to json failed because : " + errMsg + "\"}" + } + case _ => "{}" + } + } +} From cf01324bc2838504f82f5856049fbb661e3a9c54 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Fri, 17 May 2024 16:07:38 +0800 Subject: [PATCH 18/60] Minor fix of package import --- .../job/launcher/linkis/LinkisExchangisTaskLauncher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisExchangisTaskLauncher.java b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisExchangisTaskLauncher.java index ca505987d..b366b8dc4 100644 --- a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisExchangisTaskLauncher.java +++ b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisExchangisTaskLauncher.java @@ -1,6 +1,6 @@ package com.webank.wedatasphere.exchangis.job.launcher.linkis; -import com.webank.wedatasphere.exchangis.common.linkis.client.ClientConfiguration; +import com.webank.wedatasphere.exchangis.common.linkis.ClientConfiguration; import com.webank.wedatasphere.exchangis.job.enums.EngineTypeEnum; import com.webank.wedatasphere.exchangis.job.launcher.AccessibleLauncherTask; import com.webank.wedatasphere.exchangis.job.launcher.exception.ExchangisTaskLaunchException; From 6903e6b3e650dbc4d74fa38c32951321e90fe6fa Mon Sep 17 00:00:00 2001 From: davidhua Date: Mon, 20 May 2024 16:24:46 +0800 Subject: [PATCH 19/60] Enable to reverse read log. --- .../exchangis/job/domain/ExchangisJob.java | 1 + .../launcher/linkis/LinkisLauncherTask.java | 14 +- .../linkis/LaunchTaskLogOperator.scala | 28 ++++ .../server/execution/AbstractTaskManager.java | 158 ++++++++++++------ .../job/server/execution/TaskManager.java | 10 ++ .../tasks/StatusUpdateSchedulerTask.java | 9 +- .../scheduler/tasks/SubmitSchedulerTask.java | 1 + .../server/log/service/RpcJobLogService.java | 3 + .../ExchangisJobExecuteRestfulApi.java | 4 + .../ExchangisTaskExecuteRestfulApi.java | 4 + 10 files changed, 182 insertions(+), 50 deletions(-) create mode 100644 exchangis-job/exchangis-job-launcher/src/main/scala/com/webank/wedatasphere/exchangis/job/launcher/linkis/LaunchTaskLogOperator.scala diff --git a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisJob.java b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisJob.java index 3107b81ed..8afd21e49 100644 --- a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisJob.java +++ b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/ExchangisJob.java @@ -41,6 +41,7 @@ public interface ExchangisJob extends ExchangisBase{ * @param jobLabels */ void setJobLabels(Map jobLabels); + /** * Create user * @return user name diff --git a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisLauncherTask.java b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisLauncherTask.java index 408fdc43b..c51a1293c 100644 --- a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisLauncherTask.java +++ b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisLauncherTask.java @@ -1,5 +1,6 @@ package com.webank.wedatasphere.exchangis.job.launcher.linkis; +import com.webank.wedatasphere.exchangis.datasource.core.utils.Json; import com.webank.wedatasphere.exchangis.job.launcher.exception.ExchangisTaskLaunchException; import com.webank.wedatasphere.exchangis.job.launcher.exception.ExchangisTaskNotExistException; import com.webank.wedatasphere.exchangis.job.launcher.AccessibleLauncherTask; @@ -9,7 +10,6 @@ import com.webank.wedatasphere.exchangis.job.launcher.domain.task.TaskProgressInfo; import com.webank.wedatasphere.exchangis.job.launcher.domain.task.TaskStatus; import org.apache.commons.lang.StringUtils; -import org.apache.linkis.common.utils.Utils; import org.apache.linkis.computation.client.LinkisJobBuilder; import org.apache.linkis.computation.client.LinkisJobClient; import org.apache.linkis.computation.client.once.SubmittableOnceJob; @@ -35,6 +35,8 @@ public class LinkisLauncherTask implements AccessibleLauncherTask { private static final Logger LOG = LoggerFactory.getLogger(LinkisLauncherTask.class); + private static final String METRIC_NAME = "ecMetrics"; + /** * Engine versions */ @@ -162,6 +164,13 @@ public Map getMetricsInfo() throws ExchangisTaskLaunchException // Init the error count this.reqError.set(0); return metrics; + }else { + // Try to get metric from job info + Map jobInfo = getJobInfo(false); + Object metric = jobInfo.get(METRIC_NAME); + if (Objects.nonNull(metric)){ + return Json.fromJson(String.valueOf(metric), Map.class); + } } }catch(Exception e){ dealException(e); @@ -222,13 +231,14 @@ public LogResult queryLogs(LogQuery query) throws ExchangisTaskLaunchException { // The logOperator is not thread safe, so create it each time if (Objects.nonNull(this.onceJob)){ try{ - EngineConnLogOperator logOperator = (EngineConnLogOperator) this.onceJob.getOperator(EngineConnLogOperator.OPERATOR_NAME()); + LaunchTaskLogOperator logOperator = (LaunchTaskLogOperator) this.onceJob.getOperator(LaunchTaskLogOperator.OPERATOR_NAME()); logOperator.setFromLine(query.getFromLine()); logOperator.setPageSize(query.getPageSize()); logOperator.setEngineConnType(this.engineConn); logOperator.setECMServiceInstance(this.onceJob.getECMServiceInstance(this.jobInfo)); logOperator.setIgnoreKeywords(query.getIgnoreKeywords()); logOperator.setOnlyKeywords(query.getOnlyKeywords()); + logOperator.setEnableTail(query.isEnableTail()); if (Objects.nonNull(query.getLastRows())){ logOperator.setLastRows(query.getLastRows()); } diff --git a/exchangis-job/exchangis-job-launcher/src/main/scala/com/webank/wedatasphere/exchangis/job/launcher/linkis/LaunchTaskLogOperator.scala b/exchangis-job/exchangis-job-launcher/src/main/scala/com/webank/wedatasphere/exchangis/job/launcher/linkis/LaunchTaskLogOperator.scala new file mode 100644 index 000000000..93e6fe159 --- /dev/null +++ b/exchangis-job/exchangis-job-launcher/src/main/scala/com/webank/wedatasphere/exchangis/job/launcher/linkis/LaunchTaskLogOperator.scala @@ -0,0 +1,28 @@ +package com.webank.wedatasphere.exchangis.job.launcher.linkis + +import org.apache.linkis.computation.client.once.action.EngineConnOperateAction +import org.apache.linkis.computation.client.operator.impl.EngineConnLogOperator + +/** + * Enable to reverse read log file + */ +class LaunchTaskLogOperator extends EngineConnLogOperator{ + + private var enableTail: Boolean = false + + def setEnableTail(enableTail: Boolean): Unit = { + this.enableTail = enableTail + } + + def isEnableTail: Boolean = { + this.enableTail + } + + protected override def addParameters(builder: EngineConnOperateAction.Builder): Unit = { + super.addParameters(builder) + builder.addParameter("enableTail", enableTail) + } +} +object LaunchTaskLogOperator { + val OPERATOR_NAME = "launchTaskLog" +} \ No newline at end of file diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/AbstractTaskManager.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/AbstractTaskManager.java index b3a05b8af..e31d3f087 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/AbstractTaskManager.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/AbstractTaskManager.java @@ -14,6 +14,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; /** * Launched task manager @@ -33,7 +35,7 @@ public abstract class AbstractTaskManager implements TaskManager Running task */ - private ConcurrentHashMap runningTasks = new ConcurrentHashMap<>(); + private ConcurrentHashMap runningTasks = new ConcurrentHashMap<>(); /** * job_execution_id => List(Running tasks) @@ -50,18 +52,22 @@ public List getJobExecutionIds(){ @Override public List getRunningTasks() { - return new ArrayList<>(runningTasks.values()); + return runningTasks.values().stream().map(ctx -> + ctx.task).collect(Collectors.toList()); } @Override public void cancelRunningTask(String taskId) { - LaunchedExchangisTask task = runningTasks.get(taskId); - if (Objects.nonNull(task)){ - onEvent(new TaskStatusUpdateEvent(task, TaskStatus.Cancelled)); - info(task, "Status of task: [name: {}, id: {}] change {} => {}", - task.getName(), task.getTaskId(), task.getStatus(), TaskStatus.Cancelled); - JobLogCacheUtils.flush(task.getJobExecutionId(), false); - runningTasks.remove(taskId); + TaskContext context = runningTasks.get(taskId); + if (Objects.nonNull(context)){ + LaunchedExchangisTask task = context.task; + context.access(() -> { + onEvent(new TaskStatusUpdateEvent(task, TaskStatus.Cancelled)); + info(task, "Status of task: [name: {}, id: {}] change {} => {}", + task.getName(), task.getTaskId(), task.getStatus(), TaskStatus.Cancelled); + JobLogCacheUtils.flush(task.getJobExecutionId(), false); + runningTasks.remove(taskId); + }); JobWrapper wrapper = jobWrappers.get(task.getJobExecutionId()); if (Objects.nonNull(wrapper)){ wrapper.removeTask(task); @@ -75,7 +81,7 @@ public void addRunningTask(LaunchedExchangisTask task) { task.setRunningTime(Calendar.getInstance().getTime()); onEvent(new TaskLaunchEvent(task)); info(task, "Status of task: [name: {}, id: {}] change to {}, info: [{}]", task.getName(), task.getTaskId(), task.getStatus(), ""); - if (Objects.isNull(runningTasks.putIfAbsent(task.getTaskId(), task))){ + if (Objects.isNull(runningTasks.putIfAbsent(task.getTaskId(), new TaskContext(task)))){ jobWrappers.compute(task.getJobExecutionId(), (jobExecutionId, jobWrapper) -> { if (Objects.nonNull(jobWrapper) && jobWrapper.addTask(task)){ return jobWrapper; @@ -95,12 +101,9 @@ public void removeRunningTask(String taskId) { @Override public boolean refreshRunningTaskMetrics(LaunchedExchangisTask task, Map metricsMap) { - task = runningTasks.get(task.getTaskId()); - if (Objects.nonNull(task)) { - onEvent(new TaskMetricsUpdateEvent(task, metricsMap)); - task.setMetrics(null); - task.setMetricsMap(metricsMap); - trace(task, "Metrics info of task: [{}]", Json.toJson(metricsMap, null)); + TaskContext context = runningTasks.get(task.getTaskId()); + if (Objects.nonNull(context)) { + refreshRunningTaskMetrics(context, metricsMap); return true; } return false; @@ -108,37 +111,53 @@ public boolean refreshRunningTaskMetrics(LaunchedExchangisTask task, Map metricsMap) { TaskStatus beforeStatus = task.getStatus(); - if (TaskStatus.isCompleted(status)){ - info(task, "Status of task: [name: {}, id: {}] change {} => {}", - task.getName(), task.getTaskId(), beforeStatus, status); - onEvent(new TaskStatusUpdateEvent(task, status)); - removeRunningTaskInner(task.getTaskId(), false); - return true; - } else { - task = runningTasks.get(task.getTaskId()); - if (Objects.nonNull(task) ) { - onEvent(new TaskStatusUpdateEvent(task, status)); - if (isTransition(task, status)) { - info(task, "Status of task: [name: {}, id: {}] change {} => {}", - task.getName(), task.getTaskId(), beforeStatus, status); + TaskContext context = runningTasks.get(task.getTaskId()); + if (Objects.nonNull(context)){ + task = context.task; + LaunchedExchangisTask finalTask = task; + context.access( () -> { + if (Objects.nonNull(metricsMap)){ + refreshRunningTaskMetrics(context, metricsMap); } - task.setStatus(status); - return true; - } - return false; + if (TaskStatus.isCompleted(status)){ + info(finalTask, "Status of task: [name: {}, id: {}] change {} => {}", + finalTask.getName(), finalTask.getTaskId(), beforeStatus, status); + onEvent(new TaskStatusUpdateEvent(finalTask, status)); + removeRunningTaskInner(finalTask.getTaskId(), false); + } else { + onEvent(new TaskStatusUpdateEvent(finalTask, status)); + if (isTransition(finalTask, status)) { + info(finalTask, "Status of task: [name: {}, id: {}] change {} => {}", + finalTask.getName(), finalTask.getTaskId(), beforeStatus, status); + } + } + finalTask.setStatus(status); + }); + return true; } + return false; } @Override public boolean refreshRunningTaskProgress(LaunchedExchangisTask task, TaskProgressInfo progressInfo) { - task = runningTasks.get(task.getTaskId()); - if (Objects.nonNull(task)){ - onEvent(new TaskProgressUpdateEvent(task, progressInfo)); - if (task.getProgress() != progressInfo.getProgress()){ - info(task, "Progress of task: [{}] change {} => {}", task.getTaskId(), task.getProgress(), progressInfo.getProgress()); - } - task.setProgress(progressInfo.getProgress()); + TaskContext context = runningTasks.get(task.getTaskId()); + if (Objects.nonNull(context)){ + task = context.task; + LaunchedExchangisTask finalTask = task; + context.access(() -> { + onEvent(new TaskProgressUpdateEvent(finalTask, progressInfo)); + if (finalTask.getProgress() != progressInfo.getProgress()){ + info(finalTask, "Progress of task: [{}] change {} => {}", + finalTask.getTaskId(), finalTask.getProgress(), progressInfo.getProgress()); + } + finalTask.setProgress(progressInfo.getProgress()); + }); return true; } return false; @@ -146,7 +165,8 @@ public boolean refreshRunningTaskProgress(LaunchedExchangisTask task, TaskProgre @Override public LaunchedExchangisTask getRunningTask(String taskId) { - return runningTasks.get(taskId); + TaskContext context = runningTasks.get(taskId); + return context != null ? context.task : null; } public TaskExecutionListener getExecutionListener() { @@ -157,18 +177,38 @@ public void setExecutionListener(TaskExecutionListener executionListener) { this.executionListener = executionListener; } + /** + * Refresh running task metrics + * @param context context + * @param metricsMap metric map + */ + private void refreshRunningTaskMetrics(TaskContext context, Map metricsMap){ + LaunchedExchangisTask finalTask = context.task; + context.access(() -> { + if (!TaskStatus.isCompleted(finalTask.getStatus())) { + onEvent(new TaskMetricsUpdateEvent(finalTask, metricsMap)); + finalTask.setMetrics(null); + finalTask.setMetricsMap(metricsMap); + trace(finalTask, "Metrics info of task: [{}]", Json.toJson(metricsMap, null)); + } + }); + } + /** * Remove inner * @param taskId task id * @param updateStatus if update status */ private void removeRunningTaskInner(String taskId, boolean updateStatus){ - LaunchedExchangisTask task = runningTasks.get(taskId); - if (Objects.nonNull(task)){ - if (updateStatus) { - onEvent(new TaskStatusUpdateEvent(task, task.getStatus())); - } - runningTasks.remove(taskId); + TaskContext context = runningTasks.get(taskId); + if (Objects.nonNull(context)){ + LaunchedExchangisTask task = context.task; + context.access(() -> { + if (updateStatus) { + onEvent(new TaskStatusUpdateEvent(task, task.getStatus())); + } + runningTasks.remove(taskId); + }); JobWrapper wrapper = jobWrappers.get(task.getJobExecutionId()); if (Objects.nonNull(wrapper)){ wrapper.removeTask(task); @@ -199,6 +239,30 @@ public JobLogEvent getJobLogEvent(JobLogEvent.Level level, LaunchedExchangisTask return new JobLogEvent(level, task.getExecuteUser(), task.getJobExecutionId(), message, args); } + private static class TaskContext{ + /** + * Access lock + */ + private final ReentrantLock accessLock = new ReentrantLock(); + + private final LaunchedExchangisTask task; + + public TaskContext(LaunchedExchangisTask task){ + this.task = task; + } + /** + * Access the process + * @param exec exec process + */ + private void access(Runnable exec){ + accessLock.lock(); + try{ + exec.run(); + }finally { + accessLock.unlock(); + } + } + } private class JobWrapper{ /** diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/TaskManager.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/TaskManager.java index 0ffecb43a..80d4bf9de 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/TaskManager.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/TaskManager.java @@ -41,6 +41,7 @@ public interface TaskManager extends JobServerLogging metricsMap); + /** * Refresh running task status * @param task @@ -49,6 +50,15 @@ public interface TaskManager extends JobServerLogging metricsMap); + /** * Refresh progress * @param task diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/StatusUpdateSchedulerTask.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/StatusUpdateSchedulerTask.java index 7551ec207..4772fdc22 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/StatusUpdateSchedulerTask.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/StatusUpdateSchedulerTask.java @@ -1,5 +1,6 @@ package com.webank.wedatasphere.exchangis.job.server.execution.scheduler.tasks; +import com.webank.wedatasphere.exchangis.job.launcher.domain.task.TaskStatus; import com.webank.wedatasphere.exchangis.job.launcher.exception.ExchangisTaskLaunchException; import com.webank.wedatasphere.exchangis.job.launcher.AccessibleLauncherTask; import com.webank.wedatasphere.exchangis.job.launcher.domain.LaunchedExchangisTask; @@ -47,7 +48,13 @@ protected void onPoll(LaunchedExchangisTask launchedExchangisTask) throws Exchan if (Objects.nonNull(progressInfo)){ this.taskManager.refreshRunningTaskProgress(launchedExchangisTask, progressInfo); } - this.taskManager.refreshRunningTaskStatus(launchedExchangisTask, launcherTask.getLocalStatus()); + TaskStatus status = launcherTask.getLocalStatus(); + if (TaskStatus.isCompleted(status)){ + this.taskManager.refreshRunningTaskStatusAndMetrics(launchedExchangisTask, + status, launcherTask.getMetricsInfo()); + } else { + this.taskManager.refreshRunningTaskStatus(launchedExchangisTask, status); + } } catch (ExchangisTaskLaunchException e){ throw new ExchangisSchedulerException("Fail to update status(progress) for task: [" + launchedExchangisTask.getTaskId() + "]", e); } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/SubmitSchedulerTask.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/SubmitSchedulerTask.java index 4a3df1e06..89c9deee7 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/SubmitSchedulerTask.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/execution/scheduler/tasks/SubmitSchedulerTask.java @@ -72,6 +72,7 @@ public SubmitSchedulerTask(LaunchableExchangisTask task, Callable submi // Ignore } } + // Set max retry } @Override protected void schedule() throws ExchangisSchedulerException, ExchangisSchedulerRetryException { diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/RpcJobLogService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/RpcJobLogService.java index 14b73d585..46853ceb7 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/RpcJobLogService.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/RpcJobLogService.java @@ -233,6 +233,9 @@ public LogResult logsFromPageAndPath(String logPath, LogQuery logQuery) { } line = lineSupplier.get(); } + if (logQuery.isEnableTail()){ + Collections.reverse(logs); + } result = new LogResult(lineNum, false, logs); } catch (IOException e) { throw new ExchangisJobServerException.Runtime(LOG_OP_ERROR.getCode(),"Unable to query the logs from path: [" + logPath + "]", e); diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java index 5ed4cfb4a..77ffb45ae 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisJobExecuteRestfulApi.java @@ -157,11 +157,15 @@ public Message getJobExecutionLogs(@PathVariable(value = "jobExecutionId") Strin @RequestParam(value = "pageSize", required = false) Integer pageSize, @RequestParam(value = "ignoreKeywords", required = false) String ignoreKeywords, @RequestParam(value = "onlyKeywords", required = false) String onlyKeywords, + @RequestParam(value = "enableTail", required = false) Boolean enableTail, @RequestParam(value = "lastRows", required = false) Integer lastRows, HttpServletRequest request) { Message result = Message.ok("Submitted succeed(提交成功)！"); LogQuery logQuery = new LogQuery(fromLine, pageSize, ignoreKeywords, onlyKeywords, lastRows); + if (null != enableTail) { + logQuery.setEnableTail(enableTail); + } String loginUser = UserUtils.getLoginUser(request); try { if(!JobAuthorityUtils.hasJobExecuteSituationAuthority(loginUser, jobExecutionId, OperationType.JOB_QUERY)) { diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisTaskExecuteRestfulApi.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisTaskExecuteRestfulApi.java index 003c24c1e..aa2a45e1b 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisTaskExecuteRestfulApi.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/execute/ExchangisTaskExecuteRestfulApi.java @@ -77,10 +77,14 @@ public Message getTaskExecutionLogs(@PathVariable(value = "taskId") String taskI @RequestParam(value = "pageSize", required = false) Integer pageSize, @RequestParam(value = "ignoreKeywords", required = false) String ignoreKeywords, @RequestParam(value = "onlyKeywords", required = false) String onlyKeywords, + @RequestParam(value = "enableTail", required = false) Boolean enableTail, @RequestParam(value = "lastRows", required = false) Integer lastRows, HttpServletRequest request) { Message result = Message.ok("Submitted succeed(提交成功)！"); LogQuery logQuery = new LogQuery(fromLine, pageSize, ignoreKeywords, onlyKeywords, lastRows); + if (null != enableTail) { + logQuery.setEnableTail(enableTail); + } String userName = UserUtils.getLoginUser(request); try { if (!JobAuthorityUtils.hasJobExecuteSituationAuthority(userName, jobExecutionId, OperationType.JOB_QUERY)) { From fce40720e71f646caa6516357ab5f8ed8efacacc Mon Sep 17 00:00:00 2001 From: jefftlin Date: Mon, 20 May 2024 18:17:32 +0800 Subject: [PATCH 20/60] Update version of linkis to 1.4.0-wds --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index a8c905e84..6ffb9c271 100644 --- a/pom.xml +++ b/pom.xml @@ -39,10 +39,10 @@ 1.1.3-webank 1.1.2 - 1.3.0-wds - 1.3.0-wds + 1.4.0-wds + 1.4.0-wds 0.1.0-SNAPSHOT - 2.12.12 + 2.11.12 4.7.1 1.8 3.3.3 From 8722d1d086aa0aa3cdc676df717ba046055a3f17 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Mon, 20 May 2024 18:07:16 +0800 Subject: [PATCH 21/60] Update version of linkis to 1.4.0-wds --- .../builder/transform/TransformExchangisJob.java | 11 ++--------- pom.xml | 2 +- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/TransformExchangisJob.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/TransformExchangisJob.java index 319f0c7c6..a9798b845 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/TransformExchangisJob.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/TransformExchangisJob.java @@ -172,9 +172,7 @@ private void convertContentToParams(ExchangisJobInfoContent content){ if (StringUtils.isNotBlank(sourceId)){ this.sourceType = resolveDataSourceId(content.getDataSources().getSourceId(), paramSet); } else { - Optional.ofNullable(content.getDataSources().getSource().getType()).ifPresent(type -> { - this.sourceType = type.name; - }); + } } @@ -195,9 +193,7 @@ private void convertContentToParams(ExchangisJobInfoContent content){ if (StringUtils.isNotBlank(sinkId)){ this.sinkType = resolveDataSourceId(content.getDataSources().getSinkId(), paramSet); } else { - Optional.ofNullable(content.getDataSources().getSink().getType()).ifPresent(type -> { - this.sinkType = type.name; - }); + } } } @@ -245,9 +241,6 @@ private String resolveDataSourceId(String dataSourceId, JobParamSet paramSet){ return null; } - private String resolveDataSource(ExchangisJobDataSourcesContent.ExchangisJobDataSource dataSource){ - return null; - } /** * * @param items diff --git a/pom.xml b/pom.xml index 6ffb9c271..461a3cce1 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ com.webank.wedatasphere.exchangis exchangis - ${revision} + 1.1.3-webank pom exchangis From 15b562192e9a4c1d0201fa1ab59900428d3881b0 Mon Sep 17 00:00:00 2001 From: davidhua Date: Mon, 20 May 2024 17:52:28 +0800 Subject: [PATCH 22/60] Adjust the class structure of data source; Custom the http client; --- .../config/dss-exchangis-server.properties | 5 +- exchangis-dao/pom.xml | 4 + .../common/http/HttpClientConfiguration.java | 25 +++ .../common/linkis/ClientConfiguration.java | 26 --- .../linkis/client/ClientConfiguration.java | 54 ++++++ .../dao/domain/ExchangisJobDsBind.java | 26 +++ .../linkis/client/ExchangisHttpClient.scala | 39 ++++ .../client/config/ExchangisClientConfig.scala | 36 ++++ .../config/ExchangisClientConfigBuilder.scala | 59 ++++++ ...AbstractExchangisDataSourceDefinition.java | 85 +++++++++ .../datasource/core/ExchangisDataSource.java | 51 +----- .../ExchangisDataSourceConfiguration.java | 31 ++-- .../core/ExchangisDataSourceDefinition.java | 75 ++++++++ ...xt.java => DefaultExchangisDsContext.java} | 26 +-- .../context/ExchangisDataSourceContext.java | 67 +++++-- .../core/domain/DataSourceType.java | 24 --- .../core/domain/ExchangisDataSourceType.java | 30 +++ ...java => ExchangisDataSourceDefLoader.java} | 8 +- .../vo/ExchangisJobDataSourcesContent.java | 123 ++++++++++++- .../core/vo/ExchangisJobInfoContent.java | 18 ++ .../core/vo/ExchangisJobParamsContent.java | 12 ++ .../core/vo/ExchangisJobTransformsItem.java | 32 ++++ .../linkis/ExchangisBatchDataSource.java | 52 +----- .../ExchangisDataSourceConfiguration.java | 19 -- .../linkis/ExchangisLinkisRemoteClient.scala | 94 +++++----- .../ExchangisDataSourceLoaderFactory.java | 16 +- .../LocalExchangisDataSourceLoader.java | 18 +- .../datasource/loader/utils/ExtDsUtils.java | 12 +- .../server/configuration/ServerConfig.java | 8 +- .../datasource/domain/ExchangisDsProject.java | 8 + .../datasource/dto/DataSourceDTO.java | 169 ----------------- .../dto/ExchangisDataSourceDTO.java | 171 +++++++++++++---- .../dto/ExchangisDataSourceDefDTO.java | 62 +++++++ .../service/AbstractDataSourceService.java | 10 +- .../service/ExchangisDataSourceService.java | 50 ++--- .../exchangis-datasource-streamis/pom.xml | 6 +- .../streamis/ExchangisStreamisDataSource.java | 34 +--- .../ExchangisStreamisRemoteClient.scala | 172 ------------------ .../mysql/ExchangisESDataSource.java | 22 +-- .../hive/ExchangisHiveDataSource.java | 22 +-- .../mysql/ExchangisMongoDbDataSource.java | 22 +-- .../mysql/ExchangisMySQLDataSource.java | 22 +-- .../oracle/ExchangisOracleDataSource.java | 15 +- .../sftp/ExchangisSftpDataSource.java | 16 +- exchangis-datasource/pom.xml | 2 +- .../linkis/LinkisExchangisTaskLauncher.java | 54 +++--- .../linkis/client/ExchangisLaunchClient.scala | 54 ++++++ 47 files changed, 1164 insertions(+), 822 deletions(-) create mode 100644 exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/http/HttpClientConfiguration.java delete mode 100644 exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/linkis/ClientConfiguration.java create mode 100644 exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/linkis/client/ClientConfiguration.java create mode 100644 exchangis-dao/src/main/scala/com/webank/wedatasphere/exchangis/common/linkis/client/ExchangisHttpClient.scala create mode 100644 exchangis-dao/src/main/scala/com/webank/wedatasphere/exchangis/common/linkis/client/config/ExchangisClientConfig.scala create mode 100644 exchangis-dao/src/main/scala/com/webank/wedatasphere/exchangis/common/linkis/client/config/ExchangisClientConfigBuilder.scala create mode 100644 exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/AbstractExchangisDataSourceDefinition.java create mode 100644 exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ExchangisDataSourceDefinition.java rename exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/context/{DefaultExchangisDataSourceContext.java => DefaultExchangisDsContext.java} (56%) delete mode 100644 exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/domain/DataSourceType.java create mode 100644 exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/domain/ExchangisDataSourceType.java rename exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/loader/{ExchangisDataSourceLoader.java => ExchangisDataSourceDefLoader.java} (81%) delete mode 100644 exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/ExchangisDataSourceConfiguration.java create mode 100644 exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/domain/ExchangisDsProject.java delete mode 100644 exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/DataSourceDTO.java create mode 100644 exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/ExchangisDataSourceDefDTO.java delete mode 100644 exchangis-datasource/exchangis-datasource-streamis/src/main/scala/com/webank/wedatasphere/exchangis/datasource/streamis/ExchangisStreamisRemoteClient.scala create mode 100644 exchangis-job/exchangis-job-launcher/src/main/scala/com/webank/wedatasphere/exchangis/job/launcher/linkis/client/ExchangisLaunchClient.scala diff --git a/assembly-package/config/dss-exchangis-server.properties b/assembly-package/config/dss-exchangis-server.properties index 8f3b64bd3..70ebaca62 100644 --- a/assembly-package/config/dss-exchangis-server.properties +++ b/assembly-package/config/dss-exchangis-server.properties @@ -30,9 +30,8 @@ wds.linkis.server.version=v1 wds.linkis.ms.service.scan.package=com.webank.wedatasphere.exchangis # datasource client -wds.exchangis.datasource.client.serverurl=http://{LINKIS_IP}:{LINKIS_PORT}/ -wds.exchangis.datasource.client.authtoken.key=EXCHANGIS-AUTH -wds.exchangis.datasource.client.authtoken.value=EXCHANGIS-AUTH +wds.exchangis.datasource.client.server-url=http://{LINKIS_IP}:{LINKIS_PORT}/ +wds.exchangis.datasource.client.token.value=EXCHANGIS-AUTH wds.exchangis.datasource.client.dws.version=v1 # launcher client diff --git a/exchangis-dao/pom.xml b/exchangis-dao/pom.xml index 620401fac..4eac9752a 100644 --- a/exchangis-dao/pom.xml +++ b/exchangis-dao/pom.xml @@ -27,6 +27,10 @@ org.apache.linkis linkis-module + + org.apache.linkis + linkis-gateway-httpclient-support + org.hibernate hibernate-validator diff --git a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/http/HttpClientConfiguration.java b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/http/HttpClientConfiguration.java new file mode 100644 index 000000000..548b99650 --- /dev/null +++ b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/http/HttpClientConfiguration.java @@ -0,0 +1,25 @@ +package com.webank.wedatasphere.exchangis.common.http; + +import org.apache.linkis.common.conf.CommonVars; + +/** + * Define the http configuration + */ +public class HttpClientConfiguration { + + /** + * Connect timeout + */ + public static final CommonVars CONNECTION_TIMEOUT = CommonVars.apply("wds.exchangis.http.client.connection.timeout", 30000L); + + /** + * Max connection size + */ + public static final CommonVars MAX_CONNECTION_SIZE = CommonVars.apply("wds.exchangis.http.client.connection.max-size", 100); + + /** + * Read timeout + */ + public static final CommonVars READ_TIMEOUT = CommonVars.apply("wds.exchangis.http.client.read-timeout", 90000L); + +} diff --git a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/linkis/ClientConfiguration.java b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/linkis/ClientConfiguration.java deleted file mode 100644 index 3725610fd..000000000 --- a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/linkis/ClientConfiguration.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.webank.wedatasphere.exchangis.common.linkis; - -import org.apache.linkis.common.conf.CommonVars; -import org.apache.linkis.common.conf.Configuration; - -/** - * Configuration for linkis client - */ -public class ClientConfiguration { - - /** - * Linkis server url - */ - public static final CommonVars LINKIS_SERVER_URL = CommonVars.apply("wds.exchangis.client.linkis.server-url", Configuration.getGateWayURL()); - - /** - * Linkis token value - */ - public static final CommonVars LINKIS_TOKEN_VALUE = CommonVars.apply("wds.exchangis.client.linkis.token.value", "EXCHANGIS-TOKEN"); - - /** - * Linkis client max connections - */ - public static final CommonVars LINKIS_DEFAULT_MAX_CONNECTIONS = CommonVars.apply("wds.exchangis.client.linkis.max-connections.default", 70); - -} diff --git a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/linkis/client/ClientConfiguration.java b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/linkis/client/ClientConfiguration.java new file mode 100644 index 000000000..bb65867a4 --- /dev/null +++ b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/common/linkis/client/ClientConfiguration.java @@ -0,0 +1,54 @@ +package com.webank.wedatasphere.exchangis.common.linkis.client; + +import com.webank.wedatasphere.exchangis.common.http.HttpClientConfiguration; +import org.apache.linkis.common.conf.CommonVars; +import org.apache.linkis.common.conf.Configuration; + +/** + * Configuration for linkis client + */ +public class ClientConfiguration { + + /** + * Linkis server url + */ + public static final CommonVars LINKIS_SERVER_URL = CommonVars.apply("wds.exchangis.client.linkis.server-url", Configuration.getGateWayURL()); + + /** + * Linkis token value + */ + public static final CommonVars LINKIS_TOKEN_VALUE = CommonVars.apply("wds.exchangis.client.linkis.token.value", "EXCHANGIS-TOKEN"); + + /** + * Linkis client max connections + */ + public static final CommonVars LINKIS_DEFAULT_MAX_CONNECTIONS = CommonVars.apply("wds.exchangis.client.linkis.max-connections.default", + HttpClientConfiguration.MAX_CONNECTION_SIZE.getValue()); + + + /** + * Linkis discovery enable + */ + public static final CommonVars LINKIS_DISCOVERY_ENABLED = CommonVars.apply("wds.exchangis.client.linkis.discovery.enabled", true); + + /** + * Linkis discovery frequency + */ + public static final CommonVars LINKIS_DISCOVERY_FREQUENCY_PERIOD = CommonVars.apply("wds.exchangis.client.linkis.discovery.frequency-period", 1L); + + /** + * Linkis client load balance + */ + public static final CommonVars LINKIS_LOAD_BALANCER_ENABLED = CommonVars.apply("wds.exchangis.client.linkis.load-balancer.enabled", true); + + + /** + * Linkis client retry + */ + public static final CommonVars LINKIS_RETRY_ENABLED = CommonVars.apply("wds.exchangis.client.linkis.retry.enabled", false); + + /** + * DWS version + */ + public static final CommonVars LINKIS_DWS_VERSION = CommonVars.apply("wds.exchangis.client.linkis.dws.version", Configuration.LINKIS_WEB_VERSION().getValue()); +} diff --git a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/dao/domain/ExchangisJobDsBind.java b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/dao/domain/ExchangisJobDsBind.java index e21695e6d..8b99351ca 100644 --- a/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/dao/domain/ExchangisJobDsBind.java +++ b/exchangis-dao/src/main/java/com/webank/wedatasphere/exchangis/dao/domain/ExchangisJobDsBind.java @@ -18,8 +18,18 @@ public class ExchangisJobDsBind { private Long sourceDsId; + /** + * Source data source name + */ + private String sourceDsName; + private Long sinkDsId; + /** + * Sink data source name + */ + private String sinkDsName; + public Long getId() { return id; } @@ -59,4 +69,20 @@ public Long getSinkDsId() { public void setSinkDsId(Long sinkDsId) { this.sinkDsId = sinkDsId; } + + public String getSourceDsName() { + return sourceDsName; + } + + public void setSourceDsName(String sourceDsName) { + this.sourceDsName = sourceDsName; + } + + public String getSinkDsName() { + return sinkDsName; + } + + public void setSinkDsName(String sinkDsName) { + this.sinkDsName = sinkDsName; + } } diff --git a/exchangis-dao/src/main/scala/com/webank/wedatasphere/exchangis/common/linkis/client/ExchangisHttpClient.scala b/exchangis-dao/src/main/scala/com/webank/wedatasphere/exchangis/common/linkis/client/ExchangisHttpClient.scala new file mode 100644 index 000000000..45efe146d --- /dev/null +++ b/exchangis-dao/src/main/scala/com/webank/wedatasphere/exchangis/common/linkis/client/ExchangisHttpClient.scala @@ -0,0 +1,39 @@ +package com.webank.wedatasphere.exchangis.common.linkis.client + +import com.webank.wedatasphere.exchangis.common.linkis.client.config.{ExchangisClientConfig} +import org.apache.http.client.config.RequestConfig +import org.apache.http.impl.client.{CloseableHttpClient, HttpClients} +import org.apache.linkis.httpclient.dws.DWSHttpClient + +import java.util.concurrent.TimeUnit + +/** + * Enhanced http client config + */ +class ExchangisHttpClient(clientConfig: ExchangisClientConfig, clientName: String) + extends DWSHttpClient(clientConfig, clientName){ + /** + * Build http client + */ + override protected val httpClient: CloseableHttpClient = { + val defaultRequestConfig = RequestConfig.custom() + .setConnectTimeout(clientConfig.getConnectTimeout.toInt) + .setConnectionRequestTimeout(clientConfig.getConnReqTimeout.toInt) + .setSocketTimeout(clientConfig.getReadTimeout.toInt) + .build() + val clientBuilder = HttpClients.custom() + clientBuilder.setDefaultRequestConfig(defaultRequestConfig).useSystemProperties() + .setMaxConnPerRoute(clientConfig.getMaxConnection / 2).setMaxConnTotal(clientConfig.getMaxConnection) + val maxIdleTime = clientConfig.getMaxIdleTime + if (maxIdleTime > 0){ + // Evict idle connections + clientBuilder.evictExpiredConnections(); + clientBuilder.evictIdleConnections(maxIdleTime, TimeUnit.MILLISECONDS) + } + clientBuilder.build() + } + + def getHttpClient: CloseableHttpClient = { + httpClient + } +} diff --git a/exchangis-dao/src/main/scala/com/webank/wedatasphere/exchangis/common/linkis/client/config/ExchangisClientConfig.scala b/exchangis-dao/src/main/scala/com/webank/wedatasphere/exchangis/common/linkis/client/config/ExchangisClientConfig.scala new file mode 100644 index 000000000..ab9767ab9 --- /dev/null +++ b/exchangis-dao/src/main/scala/com/webank/wedatasphere/exchangis/common/linkis/client/config/ExchangisClientConfig.scala @@ -0,0 +1,36 @@ +package com.webank.wedatasphere.exchangis.common.linkis.client.config + +import org.apache.linkis.httpclient.config.ClientConfig +import org.apache.linkis.httpclient.dws.config.DWSClientConfig + +/** + * Enhanced dws client config + */ +class ExchangisClientConfig private[config]( + clientConfig: ClientConfig, + maxIdleTime: Long, + connReqTimeout: Long + ) extends DWSClientConfig(clientConfig) { + + /** + * Max idle time + * @return + */ + def getMaxIdleTime: Long = { + maxIdleTime + } + + /** + * Connection request timeout + * @return + */ + def getConnReqTimeout: Long = { + connReqTimeout + } +} + +object ExchangisClientConfig{ + def newBuilder: ExchangisClientConfigBuilder = { + new ExchangisClientConfigBuilder() + } +} \ No newline at end of file diff --git a/exchangis-dao/src/main/scala/com/webank/wedatasphere/exchangis/common/linkis/client/config/ExchangisClientConfigBuilder.scala b/exchangis-dao/src/main/scala/com/webank/wedatasphere/exchangis/common/linkis/client/config/ExchangisClientConfigBuilder.scala new file mode 100644 index 000000000..47df001f4 --- /dev/null +++ b/exchangis-dao/src/main/scala/com/webank/wedatasphere/exchangis/common/linkis/client/config/ExchangisClientConfigBuilder.scala @@ -0,0 +1,59 @@ +package com.webank.wedatasphere.exchangis.common.linkis.client.config + +import com.webank.wedatasphere.exchangis.common.http.HttpClientConfiguration +import com.webank.wedatasphere.exchangis.common.linkis.client.ClientConfiguration +import org.apache.linkis.httpclient.config.{ClientConfig, ClientConfigBuilder} +import org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy + +import java.util.concurrent.TimeUnit + +/** + * Enhanced dws client config builder + */ +class ExchangisClientConfigBuilder extends ClientConfigBuilder{ + + private var maxIdleTime: Long = _ + + private var connReqTimeout: Long = _ + + private var dwsVersion: String = _ + + // Load from vars default + // Http common + maxConnection = HttpClientConfiguration.MAX_CONNECTION_SIZE.getValue + connectTimeout = HttpClientConfiguration.CONNECTION_TIMEOUT.getValue + readTimeout = HttpClientConfiguration.READ_TIMEOUT.getValue + // Linkis client, use token auth default + dwsVersion = ClientConfiguration.LINKIS_DWS_VERSION.getValue + serverUrl = ClientConfiguration.LINKIS_SERVER_URL.getValue + discoveryEnabled = ClientConfiguration.LINKIS_DISCOVERY_ENABLED.getValue + discoveryFrequency(ClientConfiguration.LINKIS_DISCOVERY_FREQUENCY_PERIOD.getValue, TimeUnit.MINUTES) + loadbalancerEnabled = ClientConfiguration.LINKIS_LOAD_BALANCER_ENABLED.getValue + retryEnabled = ClientConfiguration.LINKIS_RETRY_ENABLED.getValue + authenticationStrategy = new TokenAuthenticationStrategy() + authTokenKey = TokenAuthenticationStrategy.TOKEN_KEY + authTokenValue = ClientConfiguration.LINKIS_TOKEN_VALUE.getValue + + def maxIdleTime(maxIdleTime: Long): this.type = { + this.maxIdleTime = maxIdleTime + this + } + + def connReqTimeout(connReqTimeout: Long): this.type = { + this.connReqTimeout = connReqTimeout + this + } + + def setDWSVersion(dwsVersion: String): this.type = { + this.dwsVersion = dwsVersion + this + } + + override def build(): ExchangisClientConfig = { + val clientConfig = new ExchangisClientConfig(super.build(), maxIdleTime, connReqTimeout) + clientConfig.setDWSVersion(dwsVersion) + clientConfig + } + + +} diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/AbstractExchangisDataSourceDefinition.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/AbstractExchangisDataSourceDefinition.java new file mode 100644 index 000000000..c86fa6ef6 --- /dev/null +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/AbstractExchangisDataSourceDefinition.java @@ -0,0 +1,85 @@ +package com.webank.wedatasphere.exchangis.datasource.core; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; +import com.webank.wedatasphere.exchangis.dao.hook.MapperHook; +import com.webank.wedatasphere.exchangis.dao.mapper.ExchangisJobParamConfigMapper; +import com.webank.wedatasphere.exchangis.datasource.core.domain.ExchangisDataSourceType; +import org.apache.commons.lang3.StringUtils; +import org.apache.linkis.datasourcemanager.common.domain.DataSourceType; + +import java.util.List; + +public abstract class AbstractExchangisDataSourceDefinition implements ExchangisDataSourceDefinition { + + /** + * Mapper hook from common module? + */ + protected MapperHook mapperHook; + /** + * Type id + */ + protected String id; + @Override + public String name() { + return type().name; + } + + + @Override + public String classifier() { + return type().classifier; + } + + + @Override + public void setMapperHook(MapperHook mapperHook) { + this.mapperHook = mapperHook; + } + + @Override + public List getDataSourceTypes(String user) { + return ExchangisDataSourceDefinition.super.getDataSourceTypes(user); + } + + @Override + public String id() { + if (null == id || id.equalsIgnoreCase("")) { + List types = getDataSourceTypes("hdfs"); + for (DataSourceType type : types) { + if (type.getName().equalsIgnoreCase(name())) { + this.id = type.getId(); + } + } + } + return this.id; + } + + @Override + public List getDataSourceParamConfigs() { + return getDataSourceParamConfigs(type().name); + } + + protected List getDataSourceParamConfigs(String type) { + return getDataSourceParamConfigs(type, null); + } + + + protected List getDataSourceParamConfigs(String type, String dir) { + ExchangisJobParamConfigMapper exchangisJobParamConfigMapper = this.mapperHook.getExchangisJobParamConfigMapper(); + QueryWrapper queryWrapper = new QueryWrapper<>(); + if (StringUtils.isNotBlank(dir)) { + queryWrapper.eq("config_direction", dir); + } + queryWrapper.eq("type", type); + queryWrapper.eq("is_hidden", 0); + queryWrapper.eq("status", 1); + return exchangisJobParamConfigMapper.selectList(queryWrapper); + } + + /** + * Data source type + * @return type + */ + protected abstract ExchangisDataSourceType type(); +} diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ExchangisDataSource.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ExchangisDataSource.java index 13ecdfdac..2eea07dd7 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ExchangisDataSource.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ExchangisDataSource.java @@ -1,52 +1,19 @@ package com.webank.wedatasphere.exchangis.datasource.core; -import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; -import com.webank.wedatasphere.exchangis.dao.hook.MapperHook; -import org.apache.linkis.datasource.client.impl.LinkisDataSourceRemoteClient; -import org.apache.linkis.datasource.client.impl.LinkisMetaDataRemoteClient; -import org.apache.linkis.datasource.client.request.GetAllDataSourceTypesAction; -import org.apache.linkis.datasource.client.response.GetAllDataSourceTypesResult; -import org.apache.linkis.datasourcemanager.common.domain.DataSourceType; - -import java.util.Collections; -import java.util.List; -import java.util.Objects; - +/** + * Data source basic inf + */ public interface ExchangisDataSource { - String id(); - - String name(); - - String description(); - - String option(); - - String classifier(); -// String type(); - - String structClassifier(); - -// String category(); - - String icon(); - - List getDataSourceParamConfigs(); + /** + * Id + * @return id + */ + Long getId(); - LinkisDataSourceRemoteClient getDataSourceRemoteClient(); + void setId(Long id); - LinkisMetaDataRemoteClient getMetaDataRemoteClient(); - void setMapperHook(MapperHook mapperHook); - default List getDataSourceTypes(String user) { - GetAllDataSourceTypesResult result = getDataSourceRemoteClient().getAllDataSourceTypes(GetAllDataSourceTypesAction.builder() - .setUser(user) - .build() - ); - List allDataSourceType = result.getAllDataSourceType(); - if (Objects.isNull(allDataSourceType)) allDataSourceType = Collections.emptyList(); - return allDataSourceType; - } } diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ExchangisDataSourceConfiguration.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ExchangisDataSourceConfiguration.java index 77e94b12a..782c26977 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ExchangisDataSourceConfiguration.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ExchangisDataSourceConfiguration.java @@ -1,19 +1,28 @@ package com.webank.wedatasphere.exchangis.datasource.core; +import com.webank.wedatasphere.exchangis.common.linkis.client.ClientConfiguration; import org.apache.linkis.common.conf.CommonVars; +/** + * Exchangis data source config + */ public class ExchangisDataSourceConfiguration { - public static final CommonVars SERVER_URL = CommonVars.apply("wds.exchangis.datasource.client.serverurl", ""); - public static final CommonVars CONNECTION_TIMEOUT = CommonVars.apply("wds.exchangis.datasource.client.connection.timeout", 30000L); - public static final CommonVars DISCOVERY_ENABLED = CommonVars.apply("wds.exchangis.datasource.client.discovery.enabled", true); - public static final CommonVars DISCOVERY_FREQUENCY_PERIOD = CommonVars.apply("wds.exchangis.datasource.client.discoveryfrequency.period", 1L); - public static final CommonVars LOAD_BALANCER_ENABLED = CommonVars.apply("wds.exchangis.datasource.client.loadbalancer.enabled", true); - public static final CommonVars MAX_CONNECTION_SIZE = CommonVars.apply("wds.exchangis.datasource.client.maxconnection.size", 5); - public static final CommonVars RETRY_ENABLED = CommonVars.apply("wds.exchangis.datasource.client.retryenabled", false); - public static final CommonVars READ_TIMEOUT = CommonVars.apply("wds.exchangis.datasource.client.readtimeout", 30000L); + /** + * Server url + */ + public static final CommonVars SERVER_URL = CommonVars.apply("wds.exchangis.datasource.client.server-url", + ClientConfiguration.LINKIS_SERVER_URL.getValue()); - public static final CommonVars AUTHTOKEN_KEY = CommonVars.apply("wds.exchangis.datasource.client.authtoken.key", ""); - public static final CommonVars AUTHTOKEN_VALUE = CommonVars.apply("wds.exchangis.datasource.client.authtoken.value", ""); - public static final CommonVars DWS_VERSION = CommonVars.apply("wds.exchangis.datasource.client.dws.version", ""); + /** + * Token value + */ + public static final CommonVars AUTH_TOKEN_VALUE = CommonVars.apply("wds.exchangis.datasource.client.token.value", + ClientConfiguration.LINKIS_TOKEN_VALUE.getValue()); + + /** + * Dws version + */ + public static final CommonVars DWS_VERSION = CommonVars.apply("wds.exchangis.datasource.client.dws.version", + ClientConfiguration.LINKIS_DWS_VERSION.getValue()); } diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ExchangisDataSourceDefinition.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ExchangisDataSourceDefinition.java new file mode 100644 index 000000000..8c6e440c2 --- /dev/null +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/ExchangisDataSourceDefinition.java @@ -0,0 +1,75 @@ +package com.webank.wedatasphere.exchangis.datasource.core; + +import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; +import com.webank.wedatasphere.exchangis.dao.hook.MapperHook; +import org.apache.linkis.datasource.client.impl.LinkisDataSourceRemoteClient; +import org.apache.linkis.datasource.client.impl.LinkisMetaDataRemoteClient; +import org.apache.linkis.datasource.client.request.GetAllDataSourceTypesAction; +import org.apache.linkis.datasource.client.response.GetAllDataSourceTypesResult; +import org.apache.linkis.datasourcemanager.common.domain.DataSourceType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * Ds type definition + */ +public interface ExchangisDataSourceDefinition { + + /** + * Type id + * @return + */ + String id(); + + /** + * Type name + * @return name + */ + String name(); + + /** + * Description + * @return desc + */ + String description(); + + String option(); + + String classifier(); + + String structClassifier(); + + String icon(); + + /** + * Parameter config in + * @return + */ + default List getDataSourceParamConfigs(){ + return new ArrayList<>(); + }; + + default LinkisDataSourceRemoteClient getDataSourceRemoteClient(){ + throw new IllegalArgumentException("unsupported to get data source remote client"); + } + + default LinkisMetaDataRemoteClient getMetaDataRemoteClient(){ + throw new IllegalArgumentException("unsupported to get metadata remote client"); + } + + void setMapperHook(MapperHook mapperHook); + + default List getDataSourceTypes(String user) { + GetAllDataSourceTypesResult result = getDataSourceRemoteClient().getAllDataSourceTypes(GetAllDataSourceTypesAction.builder() + .setUser(user) + .build() + ); + + List allDataSourceType = result.getAllDataSourceType(); + if (Objects.isNull(allDataSourceType)) allDataSourceType = Collections.emptyList(); + return allDataSourceType; + } +} diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/context/DefaultExchangisDataSourceContext.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/context/DefaultExchangisDsContext.java similarity index 56% rename from exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/context/DefaultExchangisDataSourceContext.java rename to exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/context/DefaultExchangisDsContext.java index db43c030f..c59ffe36f 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/context/DefaultExchangisDataSourceContext.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/context/DefaultExchangisDsContext.java @@ -2,8 +2,8 @@ import com.google.common.base.Strings; -import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSource; -import com.webank.wedatasphere.exchangis.datasource.core.loader.ExchangisDataSourceLoader; +import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSourceDefinition; +import com.webank.wedatasphere.exchangis.datasource.core.loader.ExchangisDataSourceDefLoader; import java.util.Collection; import java.util.Map; @@ -11,34 +11,34 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -public class DefaultExchangisDataSourceContext implements ExchangisDataSourceContext { +public class DefaultExchangisDsContext implements ExchangisDataSourceContext { - private final Map dataSources = new ConcurrentHashMap<>(24); + private final Map dataSources = new ConcurrentHashMap<>(24); @Override - public boolean registerDataSourceLoader(ExchangisDataSourceLoader loader) { + public boolean registerLoader(ExchangisDataSourceDefLoader loader) { return false; } @Override - public void addExchangisDataSource(ExchangisDataSource dataSource) { + public void addExchangisDsDefinition(ExchangisDataSourceDefinition dataSource) { Objects.requireNonNull(dataSource, "dataSource required"); String name = dataSource.name(); dataSources.put(name, dataSource); } @Override - public ExchangisDataSource removeExchangisDataSource(String type) { + public ExchangisDataSourceDefinition removeExchangisDsDefinition(String type) { return null; } @Override - public ExchangisDataSource updateExchangisDataSource(ExchangisDataSource dataSource) { + public ExchangisDataSourceDefinition updateExchangisDsDefinition(ExchangisDataSourceDefinition dataSource) { return null; } @Override - public ExchangisDataSource getExchangisDataSource(String type) { + public ExchangisDataSourceDefinition getExchangisDsDefinition(String type) { if (Strings.isNullOrEmpty(type)) { return null; } @@ -46,12 +46,12 @@ public ExchangisDataSource getExchangisDataSource(String type) { } @Override - public ExchangisDataSource getExchangisDataSource(Long dataSourceTypeId) { + public ExchangisDataSourceDefinition getExchangisDsDefinition(Long dataSourceTypeId) { if (Objects.isNull(dataSourceTypeId)) { return null; } - Collection values = this.dataSources.values(); - for (ExchangisDataSource ds : values) { + Collection values = this.dataSources.values(); + for (ExchangisDataSourceDefinition ds : values) { if (ds.id().equalsIgnoreCase(dataSourceTypeId+"")) { return ds; } @@ -61,7 +61,7 @@ public ExchangisDataSource getExchangisDataSource(Long dataSourceTypeId) { } @Override - public Collection all() { + public Collection all() { return this.dataSources.values(); } diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/context/ExchangisDataSourceContext.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/context/ExchangisDataSourceContext.java index f15e614a0..c86cf9870 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/context/ExchangisDataSourceContext.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/context/ExchangisDataSourceContext.java @@ -1,27 +1,62 @@ package com.webank.wedatasphere.exchangis.datasource.core.context; -import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSource; -import com.webank.wedatasphere.exchangis.datasource.core.loader.ExchangisDataSourceLoader; +import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSourceDefinition; +import com.webank.wedatasphere.exchangis.datasource.core.loader.ExchangisDataSourceDefLoader; import java.util.Collection; import java.util.Set; +/** + * Data source context + */ public interface ExchangisDataSourceContext { - boolean registerDataSourceLoader(ExchangisDataSourceLoader loader); - - void addExchangisDataSource(ExchangisDataSource dataSource); - - ExchangisDataSource removeExchangisDataSource(String type); - - ExchangisDataSource updateExchangisDataSource(ExchangisDataSource dataSource); - - ExchangisDataSource getExchangisDataSource(String type); - - ExchangisDataSource getExchangisDataSource(Long dataSourceTypeId); - - Collection all(); - + boolean registerLoader(ExchangisDataSourceDefLoader loader); + + /** + * Add ds definition + * @param dataSource ds + */ + void addExchangisDsDefinition(ExchangisDataSourceDefinition dataSource); + + /** + * Remove definition + * @param type type + * @return definition + */ + ExchangisDataSourceDefinition removeExchangisDsDefinition(String type); + + /** + * Update definition + * @param dataSource ds + * @return definition + */ + ExchangisDataSourceDefinition updateExchangisDsDefinition(ExchangisDataSourceDefinition dataSource); + + /** + * Get ds definition + * @param type type + * @return definition + */ + ExchangisDataSourceDefinition getExchangisDsDefinition(String type); + + /** + * Get ds definition + * @param dataSourceTypeId type id + * @return definition + */ + ExchangisDataSourceDefinition getExchangisDsDefinition(Long dataSourceTypeId); + + /** + * All definition + * @return definitions + */ + Collection all(); + + /** + * Type names + * @return set + */ Set keys(); } diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/domain/DataSourceType.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/domain/DataSourceType.java deleted file mode 100644 index 24155f89d..000000000 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/domain/DataSourceType.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.webank.wedatasphere.exchangis.datasource.core.domain; - -public enum DataSourceType { - - ELASTICSEARCH("ELASTICSEARCH"), - - HIVE("HIVE"), - - MONGODB("MONGODB"), - - MYSQL("MYSQL"), - - SFTP("SFTP"), - - ORACLE("ORACLE"), - - STARROCKS("STARROCKS"); - - public String name; - - DataSourceType(String name) { - this.name = name; - } -} diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/domain/ExchangisDataSourceType.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/domain/ExchangisDataSourceType.java new file mode 100644 index 000000000..445f3f00b --- /dev/null +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/domain/ExchangisDataSourceType.java @@ -0,0 +1,30 @@ +package com.webank.wedatasphere.exchangis.datasource.core.domain; + +public enum ExchangisDataSourceType { + + ELASTICSEARCH("ELASTICSEARCH", "分布式全文索引"), + + HIVE("HIVE", "大数据存储"), + + MONGODB("MONGODB", "非关系型数据库"), + + MYSQL("MYSQL", "关系型数据库"), + + SFTP("SFTP", "sftp连接"), + + ORACLE("ORACLE", "关系型数据库"); + + /** + * Type name + */ + public String name; + + /** + * Classifier + */ + public String classifier; + ExchangisDataSourceType(String name, String classifier) { + this.name = name; + this.classifier = classifier; + } +} diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/loader/ExchangisDataSourceLoader.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/loader/ExchangisDataSourceDefLoader.java similarity index 81% rename from exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/loader/ExchangisDataSourceLoader.java rename to exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/loader/ExchangisDataSourceDefLoader.java index f68f696e8..a1fdc8a34 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/loader/ExchangisDataSourceLoader.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/loader/ExchangisDataSourceDefLoader.java @@ -2,13 +2,13 @@ import com.webank.wedatasphere.exchangis.dao.hook.MapperHook; -import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSource; +import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSourceDefinition; import com.webank.wedatasphere.exchangis.datasource.core.context.ExchangisDataSourceContext; import org.apache.linkis.common.conf.CommonVars; import java.util.Objects; -public interface ExchangisDataSourceLoader { +public interface ExchangisDataSourceDefLoader { String EXCHANGIS_DIR_NAME = Objects.isNull(CommonVars.apply("wds.exchangis.datasource.extension.dir").getValue()) ? "exchangis-extds" : CommonVars.apply("wds.exchangis.datasource.extension.dir").getValue().toString(); @@ -26,8 +26,8 @@ public interface ExchangisDataSourceLoader { void init(MapperHook mapperHook) throws Exception; - ExchangisDataSource load(String dataSourceType); + ExchangisDataSourceDefinition load(String dataSourceType); - ExchangisDataSource get(String dataSourceType, boolean reload); + ExchangisDataSourceDefinition get(String dataSourceType, boolean reload); } diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobDataSourcesContent.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobDataSourcesContent.java index fc1618318..36af7634f 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobDataSourcesContent.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobDataSourcesContent.java @@ -1,17 +1,38 @@ package com.webank.wedatasphere.exchangis.datasource.core.vo; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.webank.wedatasphere.exchangis.datasource.core.domain.ExchangisDataSourceType; +/** + * string: HIVE.ID.DB.TABLE + * json: { + * "type": "HIVE", + * "id": 467, + * "name": "HIVE-DEMO", + * "database": "default", + * "table": "demo-test" + * } + */ public class ExchangisJobDataSourcesContent { - // 唯一标识符 - // HIVE.ID.DB.TABLE @JsonProperty("source_id") private String sourceId; + /** + * Source ds + */ + private ExchangisJobDataSource source = new ExchangisJobDataSource(); + + @JsonProperty("sink_id") private String sinkId; + /** + * Sink ds + */ + private ExchangisJobDataSource sink = new ExchangisJobDataSource(); + public String getSourceId() { return sourceId; } @@ -27,4 +48,102 @@ public String getSinkId() { public void setSinkId(String sinkId) { this.sinkId = sinkId; } + + public void setSource(ExchangisJobDataSource source) { + this.source = source; + } + + public ExchangisJobDataSource getSource() { + return source; + } + + public void setSink(ExchangisJobDataSource sink) { + this.sink = sink; + } + + public ExchangisJobDataSource getSink() { + return sink; + } + + @JsonInclude(JsonInclude.Include.NON_EMPTY) + public static class ExchangisJobDataSource { + + /** + * Data source type + */ + private ExchangisDataSourceType type; + + /** + * Data source id + */ + private String id; + + /** + * Data source name + */ + private String name; + + /** + * Database field + */ + private String database; + + /** + * Table field + */ + private String table; + + /** + * Uri field + */ + private String uri; + + public void setType(ExchangisDataSourceType type) { + this.type = type; + } + + public ExchangisDataSourceType getType() { + return type; + } + + public void setId(String id) { + this.id = id; + } + + public String getId() { + return id; + } + + public void setName(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public void setDatabase(String database) { + this.database = database; + } + + public String getDatabase() { + return database; + } + + public void setTable(String table) { + this.table = table; + } + + public String getTable() { + return table; + } + + public void setUri(String uri) { + this.uri = uri; + } + + public String getUri() { + return uri; + } + } } diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobInfoContent.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobInfoContent.java index 25ef70e09..eae71c5a6 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobInfoContent.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobInfoContent.java @@ -7,17 +7,35 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class ExchangisJobInfoContent { + /** + * Engine name + */ private String engine; + /** + * Sub job name + */ private String subJobName; + /** + * Data source content + */ private ExchangisJobDataSourcesContent dataSources; + /** + * Extra params + */ private ExchangisJobParamsContent params; + /** + * Transform define + */ // private List transforms; private ExchangisJobTransformsContent transforms; + /** + * Settings + */ private List settings; public String getEngine() { diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobParamsContent.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobParamsContent.java index ef8dbcb57..64d3a6106 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobParamsContent.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobParamsContent.java @@ -1,12 +1,20 @@ package com.webank.wedatasphere.exchangis.datasource.core.vo; +import com.fasterxml.jackson.annotation.JsonAlias; import com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; public class ExchangisJobParamsContent { + /** + * Source params + */ private List sources; + + /** + * Sink params + */ private List sinks; public List getSources() { @@ -26,13 +34,17 @@ public void setSinks(List sinks) { } public static class ExchangisJobParamsItem { + @JsonProperty("config_key") + @JsonAlias({"key", "k"}) private String configKey; @JsonProperty("config_name") + @JsonAlias({"name", "n"}) private String configName; @JsonProperty("config_value") + @JsonAlias({"value", "v"}) private Object configValue; private Integer sort; diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobTransformsItem.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobTransformsItem.java index d0ae8ede0..10ddad5a6 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobTransformsItem.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobTransformsItem.java @@ -2,24 +2,56 @@ import java.util.List; +import com.fasterxml.jackson.annotation.JsonAlias; import com.fasterxml.jackson.annotation.JsonProperty; public class ExchangisJobTransformsItem { + /** + * Source field name + */ @JsonProperty("source_field_name") + @JsonAlias({"srcFieldName"}) private String sourceFieldName; + + /** + * Source field type + */ @JsonProperty("source_field_type") + @JsonAlias({"srcFieldType"}) private String sourceFieldType; + + /** + * Sink field name + */ @JsonProperty("sink_field_name") + @JsonAlias({"sinkFieldName"}) private String sinkFieldName; + + /** + * Sink field type + */ @JsonProperty("sink_field_type") + @JsonAlias({"sinkFieldType"}) private String sinkFieldType; + + /** + * Delete enable + */ @JsonProperty("deleteEnable") private boolean deleteEnable; + /** + * Source field index + */ @JsonProperty("source_field_index") + @JsonAlias({"srcFieldIdx"}) private Integer sourceFieldIndex; + /** + * Sink field index + */ @JsonProperty("sink_field_index") + @JsonAlias({"sinkFi"}) private Integer sinkFieldIndex; @JsonProperty("source_field_editable") diff --git a/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/ExchangisBatchDataSource.java b/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/ExchangisBatchDataSource.java index 38058e58c..277eee047 100644 --- a/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/ExchangisBatchDataSource.java +++ b/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/ExchangisBatchDataSource.java @@ -1,44 +1,14 @@ package com.webank.wedatasphere.exchangis.datasource.linkis; -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; -import com.webank.wedatasphere.exchangis.dao.hook.MapperHook; -import com.webank.wedatasphere.exchangis.dao.mapper.ExchangisJobParamConfigMapper; -import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSource; +import com.webank.wedatasphere.exchangis.datasource.core.AbstractExchangisDataSourceDefinition; import org.apache.linkis.datasource.client.impl.LinkisDataSourceRemoteClient; import org.apache.linkis.datasource.client.impl.LinkisMetaDataRemoteClient; -import org.apache.linkis.datasourcemanager.common.domain.DataSourceType; -import java.util.List; +/** + * Batch data source + */ +public abstract class ExchangisBatchDataSource extends AbstractExchangisDataSourceDefinition { -public abstract class ExchangisBatchDataSource implements ExchangisDataSource { - - protected MapperHook mapperHook; - protected String id; - - @Override - public void setMapperHook(MapperHook mapperHook) { - this.mapperHook = mapperHook; - } - - protected List getDataSourceParamConfigs(String type) { - ExchangisJobParamConfigMapper exchangisJobParamConfigMapper = this.mapperHook.getExchangisJobParamConfigMapper(); - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.eq("type", type); - queryWrapper.eq("is_hidden", 0); - queryWrapper.eq("status", 1); - return exchangisJobParamConfigMapper.selectList(queryWrapper); - } - - protected List getDataSourceParamConfigs(String type, String dir) { - ExchangisJobParamConfigMapper exchangisJobParamConfigMapper = this.mapperHook.getExchangisJobParamConfigMapper(); - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.eq("config_direction", dir); - queryWrapper.eq("type", type); - queryWrapper.eq("is_hidden", 0); - queryWrapper.eq("status", 1); - return exchangisJobParamConfigMapper.selectList(queryWrapper); - } @Override public LinkisDataSourceRemoteClient getDataSourceRemoteClient() { @@ -50,16 +20,4 @@ public LinkisMetaDataRemoteClient getMetaDataRemoteClient() { return ExchangisLinkisRemoteClient.getLinkisMetadataRemoteClient(); } - @Override - public String id() { - if (null == id || id.equalsIgnoreCase("")) { - List types = getDataSourceTypes("hdfs"); - for (DataSourceType type : types) { - if (type.getName().equalsIgnoreCase(name())) { - this.id = type.getId(); - } - } - } - return this.id; - } } diff --git a/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/ExchangisDataSourceConfiguration.java b/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/ExchangisDataSourceConfiguration.java deleted file mode 100644 index ad2e1deab..000000000 --- a/exchangis-datasource/exchangis-datasource-linkis/src/main/java/com/webank/wedatasphere/exchangis/datasource/linkis/ExchangisDataSourceConfiguration.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.webank.wedatasphere.exchangis.datasource.linkis; - - -import org.apache.linkis.common.conf.CommonVars; - -public class ExchangisDataSourceConfiguration { - public static final CommonVars SERVER_URL = CommonVars.apply("wds.exchangis.datasource.client.serverurl", ""); - public static final CommonVars CONNECTION_TIMEOUT = CommonVars.apply("wds.exchangis.datasource.client.connection.timeout", 30000L); - public static final CommonVars DISCOVERY_ENABLED = CommonVars.apply("wds.exchangis.datasource.client.discovery.enabled", false); - public static final CommonVars DISCOVERY_FREQUENCY_PERIOD = CommonVars.apply("wds.exchangis.datasource.client.discoveryfrequency.period", 1L); - public static final CommonVars LOAD_BALANCER_ENABLED = CommonVars.apply("wds.exchangis.datasource.client.loadbalancer.enabled", true); - public static final CommonVars MAX_CONNECTION_SIZE = CommonVars.apply("wds.exchangis.datasource.client.maxconnection.size", 5); - public static final CommonVars RETRY_ENABLED = CommonVars.apply("wds.exchangis.datasource.client.retryenabled", false); - public static final CommonVars READ_TIMEOUT = CommonVars.apply("wds.exchangis.datasource.client.readtimeout", 30000L); - - public static final CommonVars AUTHTOKEN_KEY = CommonVars.apply("wds.exchangis.datasource.client.authtoken.key", ""); - public static final CommonVars AUTHTOKEN_VALUE = CommonVars.apply("wds.exchangis.datasource.client.authtoken.value", ""); - public static final CommonVars DWS_VERSION = CommonVars.apply("wds.exchangis.datasource.client.dws.version", ""); -} diff --git a/exchangis-datasource/exchangis-datasource-linkis/src/main/scala/com/webank/wedatasphere/exchangis/datasource/linkis/ExchangisLinkisRemoteClient.scala b/exchangis-datasource/exchangis-datasource-linkis/src/main/scala/com/webank/wedatasphere/exchangis/datasource/linkis/ExchangisLinkisRemoteClient.scala index a8126a04c..c3d4d242b 100644 --- a/exchangis-datasource/exchangis-datasource-linkis/src/main/scala/com/webank/wedatasphere/exchangis/datasource/linkis/ExchangisLinkisRemoteClient.scala +++ b/exchangis-datasource/exchangis-datasource-linkis/src/main/scala/com/webank/wedatasphere/exchangis/datasource/linkis/ExchangisLinkisRemoteClient.scala @@ -1,62 +1,36 @@ package com.webank.wedatasphere.exchangis.datasource.linkis -import com.webank.wedatasphere.exchangis.common.linkis.ClientConfiguration +import com.webank.wedatasphere.exchangis.common.linkis.client.{ClientConfiguration, ExchangisHttpClient} +import com.webank.wedatasphere.exchangis.common.linkis.client.config.ExchangisClientConfig +import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSourceConfiguration +import com.webank.wedatasphere.exchangis.datasource.core.exception.{ExchangisDataSourceException, ExchangisDataSourceExceptionCode} +import org.apache.commons.lang3.StringUtils +import org.apache.linkis.datasource.client.config.DatasourceClientConfig.DATA_SOURCE_SERVICE_CLIENT_NAME import org.apache.linkis.datasource.client.impl.{LinkisDataSourceRemoteClient, LinkisMetaDataRemoteClient} import org.apache.linkis.datasource.client.request._ import org.apache.linkis.datasource.client.response._ import org.apache.linkis.datasourcemanager.common.domain.{DataSource, DataSourceType} -import org.apache.linkis.httpclient.dws.authentication.{StaticAuthenticationStrategy, TokenAuthenticationStrategy} -import org.apache.linkis.httpclient.dws.config.{DWSClientConfig, DWSClientConfigBuilder} +import org.apache.linkis.httpclient.dws.DWSHttpClient -import java.lang -import java.util.concurrent.TimeUnit object ExchangisLinkisRemoteClient { + //Linkis Datasource Client Config - val serverUrl: String = ClientConfiguration.LINKIS_SERVER_URL.getValue - val authTokenValue: String = ClientConfiguration.LINKIS_TOKEN_VALUE.getValue - val connectionTimeout: lang.Long = ExchangisDataSourceConfiguration.CONNECTION_TIMEOUT.getValue - val discoveryEnabled: lang.Boolean = ExchangisDataSourceConfiguration.DISCOVERY_ENABLED.getValue - val discoveryFrequencyPeriod: lang.Long = ExchangisDataSourceConfiguration.DISCOVERY_FREQUENCY_PERIOD.getValue - val loadbalancerEnabled: lang.Boolean = ExchangisDataSourceConfiguration.LOAD_BALANCER_ENABLED.getValue - val maxConnectionSize: Integer = ExchangisDataSourceConfiguration.MAX_CONNECTION_SIZE.getValue - val retryEnabled: lang.Boolean = ExchangisDataSourceConfiguration.RETRY_ENABLED.getValue - val readTimeout: lang.Long = ExchangisDataSourceConfiguration.READ_TIMEOUT.getValue - val dwsVersion: String = ExchangisDataSourceConfiguration.DWS_VERSION.getValue - - - // val clientConfig = DWSClientConfigBuilder.newBuilder() - // .addServerUrl(serverUrl) - // .connectionTimeout(connectionTimeout) - // .discoveryEnabled(discoveryEnabled) - // .discoveryFrequency(1,TimeUnit.MINUTES) - // .loadbalancerEnabled(loadbalancerEnabled) - // .maxConnectionSize(maxConnectionSize) - // .retryEnabled(retryEnabled) - // .readTimeout(readTimeout) - // .setAuthenticationStrategy(new StaticAuthenticationStrategy()) - // .setAuthTokenKey(authTokenKey) - // .setAuthTokenValue(authTokenValue) - // .setDWSVersion(dwsVersion) - // .build() - - val clientConfig: DWSClientConfig = DWSClientConfigBuilder.newBuilder() - .addServerUrl(serverUrl) - .connectionTimeout(connectionTimeout) - .discoveryEnabled(discoveryEnabled) - .discoveryFrequency(discoveryFrequencyPeriod, TimeUnit.MINUTES) - .loadbalancerEnabled(loadbalancerEnabled) - .maxConnectionSize(maxConnectionSize) - .retryEnabled(retryEnabled) - .readTimeout(readTimeout) - .setAuthenticationStrategy(new TokenAuthenticationStrategy()) - .setAuthTokenValue(authTokenValue) - .setDWSVersion(dwsVersion) + val clientConfig: ExchangisClientConfig = ExchangisClientConfig.newBuilder + .addServerUrl(ExchangisDataSourceConfiguration.SERVER_URL.getValue) + .setAuthTokenValue(ExchangisDataSourceConfiguration.AUTH_TOKEN_VALUE.getValue) + .setDWSVersion(ExchangisDataSourceConfiguration.DWS_VERSION.getValue) .build() - val dataSourceClient = new LinkisDataSourceRemoteClient(clientConfig) + /** + * Data source client + */ + val dataSourceClient = new ExchangisDataSourceClient(clientConfig, null) - val metaDataClient = new LinkisMetaDataRemoteClient(clientConfig) + /** + * Meta data client + */ + val metaDataClient = new ExchangisMetadataClient(clientConfig) def getLinkisDataSourceRemoteClient: LinkisDataSourceRemoteClient = { dataSourceClient @@ -83,10 +57,6 @@ object ExchangisLinkisRemoteClient { ) } -// def createDataSource() = { -// dataSourceClient.execute().asInstanceOf[] -// } - /** * get datasourceConnect information * @@ -172,3 +142,27 @@ object ExchangisLinkisRemoteClient { } + +/** + * Exchangis data source client + * @param clientConfig client config + * @param clientName client name + */ +class ExchangisDataSourceClient(clientConfig: ExchangisClientConfig, clientName: String) extends LinkisDataSourceRemoteClient(clientConfig, clientName){ + + protected override val dwsHttpClient: DWSHttpClient = { + val client = if (StringUtils.isEmpty(clientName)) DATA_SOURCE_SERVICE_CLIENT_NAME.getValue else clientName + Option(clientConfig) match { + case Some(config) => new ExchangisHttpClient(config, client) + case _ => throw new ExchangisDataSourceException(ExchangisDataSourceExceptionCode.PARAMETER_INVALID.getCode, "Linkis client config cannot be null") + } + } +} + +/** + * Exchangis meta data client + * @param clientConfig client config + */ +class ExchangisMetadataClient(clientConfig: ExchangisClientConfig) extends LinkisMetaDataRemoteClient(clientConfig){ + protected override val dwsHttpClient: DWSHttpClient = new ExchangisHttpClient(clientConfig, "MetaData-Client") +} diff --git a/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/loader/ExchangisDataSourceLoaderFactory.java b/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/loader/ExchangisDataSourceLoaderFactory.java index 58d6c29dc..979ca9a86 100644 --- a/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/loader/ExchangisDataSourceLoaderFactory.java +++ b/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/loader/ExchangisDataSourceLoaderFactory.java @@ -1,6 +1,6 @@ package com.webank.wedatasphere.exchangis.datasource.loader.loader; -import com.webank.wedatasphere.exchangis.datasource.core.loader.ExchangisDataSourceLoader; +import com.webank.wedatasphere.exchangis.datasource.core.loader.ExchangisDataSourceDefLoader; import org.apache.commons.lang.ClassUtils; import org.apache.commons.lang.StringUtils; import org.apache.linkis.common.conf.CommonVars; @@ -11,13 +11,13 @@ public class ExchangisDataSourceLoaderFactory { private static final Logger logger = LoggerFactory.getLogger(ExchangisDataSourceLoaderFactory.class); - private static Class clazz = LocalExchangisDataSourceLoader.class; - private static ExchangisDataSourceLoader exchangisDataSourceLoader = null; + private static Class clazz = LocalExchangisDataSourceLoader.class; + private static ExchangisDataSourceDefLoader exchangisDataSourceDefLoader = null; - public static ExchangisDataSourceLoader getLoader(){ - if (exchangisDataSourceLoader == null){ + public static ExchangisDataSourceDefLoader getLoader(){ + if (exchangisDataSourceDefLoader == null){ synchronized (ExchangisDataSourceLoaderFactory.class){ - if (exchangisDataSourceLoader == null){ + if (exchangisDataSourceDefLoader == null){ // 可以通过配置自行加载对应的类 CommonVars apply = CommonVars.apply("exchangis.extds.loader.classname", ""); String className = apply.getValue(); @@ -29,7 +29,7 @@ public static ExchangisDataSourceLoader getLoader(){ } } try { - exchangisDataSourceLoader = clazz.newInstance(); + exchangisDataSourceDefLoader = clazz.newInstance(); } catch (Exception e) { logger.error(String.format("Can not initialize ExchangisDataSourceLoader class %s.", clazz.getSimpleName()), e); } @@ -37,7 +37,7 @@ public static ExchangisDataSourceLoader getLoader(){ } } } - return exchangisDataSourceLoader; + return exchangisDataSourceDefLoader; } } diff --git a/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/loader/LocalExchangisDataSourceLoader.java b/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/loader/LocalExchangisDataSourceLoader.java index 50f6f59e3..18d843f09 100644 --- a/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/loader/LocalExchangisDataSourceLoader.java +++ b/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/loader/LocalExchangisDataSourceLoader.java @@ -1,9 +1,9 @@ package com.webank.wedatasphere.exchangis.datasource.loader.loader; import com.webank.wedatasphere.exchangis.dao.hook.MapperHook; -import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSource; +import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSourceDefinition; import com.webank.wedatasphere.exchangis.datasource.core.context.ExchangisDataSourceContext; -import com.webank.wedatasphere.exchangis.datasource.core.loader.ExchangisDataSourceLoader; +import com.webank.wedatasphere.exchangis.datasource.core.loader.ExchangisDataSourceDefLoader; import com.webank.wedatasphere.exchangis.datasource.loader.clazzloader.ExchangisDataSourceClassLoader; import com.webank.wedatasphere.exchangis.datasource.loader.utils.ExceptionHelper; import com.webank.wedatasphere.exchangis.datasource.loader.utils.ExtDsUtils; @@ -17,7 +17,7 @@ import java.util.List; import java.util.Objects; -public class LocalExchangisDataSourceLoader implements ExchangisDataSourceLoader { +public class LocalExchangisDataSourceLoader implements ExchangisDataSourceDefLoader { private static final Logger LOGGER = LoggerFactory.getLogger(LocalExchangisDataSourceLoader.class); @@ -61,24 +61,24 @@ public void init(MapperHook mapperHook) throws Exception { if (clazz == null) { Thread.currentThread().setContextClassLoader(currentClassLoader); } else { - ExchangisDataSource exchangisDataSource = (ExchangisDataSource) clazz.newInstance(); - exchangisDataSource.setMapperHook(mapperHook); + ExchangisDataSourceDefinition dsType = (ExchangisDataSourceDefinition) clazz.newInstance(); + dsType.setMapperHook(mapperHook); Thread.currentThread().setContextClassLoader(currentClassLoader); - LOGGER.info("ExchangisDataSource is {}", exchangisDataSource.getClass().toString()); + LOGGER.info("ExchangisDataSource is {}", dsType.getClass().toString()); - context.addExchangisDataSource(exchangisDataSource); + context.addExchangisDsDefinition(dsType); } } } @Override - public ExchangisDataSource load(String dataSourceType) { + public ExchangisDataSourceDefinition load(String dataSourceType) { return null; } @Override - public ExchangisDataSource get(String dataSourceType, boolean reload) { + public ExchangisDataSourceDefinition get(String dataSourceType, boolean reload) { return null; } } diff --git a/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/utils/ExtDsUtils.java b/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/utils/ExtDsUtils.java index 69db1da10..59b40ea66 100644 --- a/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/utils/ExtDsUtils.java +++ b/exchangis-datasource/exchangis-datasource-loader/src/main/java/com/webank/wedatasphere/exchangis/datasource/loader/utils/ExtDsUtils.java @@ -18,8 +18,8 @@ package com.webank.wedatasphere.exchangis.datasource.loader.utils; -import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSource; -import com.webank.wedatasphere.exchangis.datasource.core.loader.ExchangisDataSourceLoader; +import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSourceDefinition; +import com.webank.wedatasphere.exchangis.datasource.core.loader.ExchangisDataSourceDefLoader; import com.webank.wedatasphere.exchangis.datasource.loader.exception.NoSuchExchangisExtDataSourceException; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; @@ -40,7 +40,7 @@ public class ExtDsUtils { private static final Logger logger = LoggerFactory.getLogger(ExtDsUtils.class); - private static Class PARENT_CLASS = ExchangisDataSource.class; + private static Class PARENT_CLASS = ExchangisDataSourceDefinition.class; public static String getExchangisExtDataSourceClassName(String libPath, @@ -99,7 +99,7 @@ public static List getJarsOfPath(String path) { if (file.listFiles() != null) { for (File f : file.listFiles()) { // exchangis-xxxxx.jar - if (!f.isDirectory() && f.getName().endsWith(ExchangisDataSourceLoader.JAR_SUF_NAME) && f.getName().startsWith("exchangis")) { + if (!f.isDirectory() && f.getName().endsWith(ExchangisDataSourceDefLoader.JAR_SUF_NAME) && f.getName().startsWith("exchangis")) { jars.add(f.getPath()); } } @@ -115,11 +115,11 @@ public static List getJarsUrlsOfPath(String path) { List jars = new ArrayList<>(); if (file.listFiles() != null) { for (File f : file.listFiles()) { - if (!f.isDirectory() && f.getName().endsWith(ExchangisDataSourceLoader.JAR_SUF_NAME)) { + if (!f.isDirectory() && f.getName().endsWith(ExchangisDataSourceDefLoader.JAR_SUF_NAME)) { try { jars.add(f.toURI().toURL()); } catch (MalformedURLException e) { - logger.warn("url {} cannot be added", ExchangisDataSourceLoader.FILE_SCHEMA + f.getPath()); + logger.warn("url {} cannot be added", ExchangisDataSourceDefLoader.FILE_SCHEMA + f.getPath()); } } } diff --git a/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/configuration/ServerConfig.java b/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/configuration/ServerConfig.java index 85a71ef80..5f9809e85 100644 --- a/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/configuration/ServerConfig.java +++ b/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/configuration/ServerConfig.java @@ -2,9 +2,9 @@ import com.webank.wedatasphere.exchangis.dao.hook.MapperHook; -import com.webank.wedatasphere.exchangis.datasource.core.context.DefaultExchangisDataSourceContext; +import com.webank.wedatasphere.exchangis.datasource.core.context.DefaultExchangisDsContext; import com.webank.wedatasphere.exchangis.datasource.core.context.ExchangisDataSourceContext; -import com.webank.wedatasphere.exchangis.datasource.core.loader.ExchangisDataSourceLoader; +import com.webank.wedatasphere.exchangis.datasource.core.loader.ExchangisDataSourceDefLoader; import com.webank.wedatasphere.exchangis.datasource.loader.loader.ExchangisDataSourceLoaderFactory; import org.apache.linkis.common.exception.ErrorException; import org.springframework.context.annotation.Bean; @@ -15,8 +15,8 @@ public class ServerConfig { @Bean public ExchangisDataSourceContext context(MapperHook mapperHook) throws Exception { - DefaultExchangisDataSourceContext context = new DefaultExchangisDataSourceContext(); - ExchangisDataSourceLoader loader = ExchangisDataSourceLoaderFactory.getLoader(); + DefaultExchangisDsContext context = new DefaultExchangisDsContext(); + ExchangisDataSourceDefLoader loader = ExchangisDataSourceLoaderFactory.getLoader(); loader.setContext(context); try { loader.init(mapperHook); diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/domain/ExchangisDsProject.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/domain/ExchangisDsProject.java new file mode 100644 index 000000000..322424cb9 --- /dev/null +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/domain/ExchangisDsProject.java @@ -0,0 +1,8 @@ +package com.webank.wedatasphere.exchangis.datasource.domain; + +/** + * The relation between data source and project + */ +public class ExchangisDsProject { +// private String +} diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/DataSourceDTO.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/DataSourceDTO.java deleted file mode 100644 index 2ee15e9ed..000000000 --- a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/DataSourceDTO.java +++ /dev/null @@ -1,169 +0,0 @@ -package com.webank.wedatasphere.exchangis.datasource.dto; - -import java.util.Date; - -public class DataSourceDTO { - - private Long id; - private String name; - private String type; - private Long dataSourceTypeId; - private String createIdentify; - private String createSystem; - private String desc; - private String createUser; - private String labels; - private String label; - private Long versionId; - private String modifyUser; - private Date modifyTime; - private boolean expire; - private boolean writeAble; - private boolean readAble; - private String authDbs; - private String authTbls; - - public boolean isExpire() { - return expire; - } - - public void setExpire(boolean expire) { - this.expire = expire; - } - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public String getCreateIdentify() { - return createIdentify; - } - - public void setCreateIdentify(String createIdentify) { - this.createIdentify = createIdentify; - } - - public Long getDataSourceTypeId() { - return dataSourceTypeId; - } - - public void setDataSourceTypeId(Long dataSourceTypeId) { - this.dataSourceTypeId = dataSourceTypeId; - } - - public String getDesc() { - return desc; - } - - public void setDesc(String desc) { - this.desc = desc; - } - - public String getCreateUser() { - return createUser; - } - - public void setCreateUser(String createUser) { - this.createUser = createUser; - } - - public String getLabels() { - return labels; - } - - public void setLabels(String labels) { - this.labels = labels; - } - - public String getLabel() { - return label; - } - - public void setLabel(String label) { - this.label = label; - } - - public Long getVersionId() { - return versionId; - } - - public void setVersionId(Long versionId) { - this.versionId = versionId; - } - - public String getModifyUser() { - return modifyUser; - } - - public void setModifyUser(String modifyUser) { - this.modifyUser = modifyUser; - } - - public Date getModifyTime() { - return modifyTime; - } - - public void setModifyTime(Date modifyTime) { - this.modifyTime = modifyTime; - } - - public String getCreateSystem() { - return createSystem; - } - - public void setCreateSystem(String createSystem) { - this.createSystem = createSystem; - } - - public boolean isWriteAble() { - return writeAble; - } - - public void setWriteAble(boolean writeAble) { - this.writeAble = writeAble; - } - - public boolean isReadAble() { - return readAble; - } - - public void setReadAble(boolean readAble) { - this.readAble = readAble; - } - - public String getAuthDbs() { - return authDbs; - } - - public void setAuthDbs(String authDbs) { - this.authDbs = authDbs; - } - - public String getAuthTbls() { - return authTbls; - } - - public void setAuthTbls(String authTbls) { - this.authTbls = authTbls; - } -} diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/ExchangisDataSourceDTO.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/ExchangisDataSourceDTO.java index b6a2fff3a..4f2c06fe3 100644 --- a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/ExchangisDataSourceDTO.java +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/ExchangisDataSourceDTO.java @@ -1,62 +1,169 @@ package com.webank.wedatasphere.exchangis.datasource.dto; +import java.util.Date; + public class ExchangisDataSourceDTO { - private final String id; - private final String classifier; - private final String name; - private String option; - private String description; - private String icon; - private String struct_classifier; - - public ExchangisDataSourceDTO(String id, String classifier, String name, String struct_classifier) { - this.id = id; - this.classifier = classifier; - this.name = name; - this.struct_classifier = struct_classifier; + + private Long id; + private String name; + private String type; + private Long dataSourceTypeId; + private String createIdentify; + private String createSystem; + private String desc; + private String createUser; + private String labels; + private String label; + private Long versionId; + private String modifyUser; + private Date modifyTime; + private boolean expire; + private boolean writeAble; + private boolean readAble; + private String authDbs; + private String authTbls; + + public boolean isExpire() { + return expire; + } + + public void setExpire(boolean expire) { + this.expire = expire; } - public String getId() { + public Long getId() { return id; } - public String getClassifier() { - return classifier; + public void setId(Long id) { + this.id = id; } public String getName() { return name; } - public String getOption() { - return option; + public void setName(String name) { + this.name = name; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getCreateIdentify() { + return createIdentify; + } + + public void setCreateIdentify(String createIdentify) { + this.createIdentify = createIdentify; + } + + public Long getDataSourceTypeId() { + return dataSourceTypeId; + } + + public void setDataSourceTypeId(Long dataSourceTypeId) { + this.dataSourceTypeId = dataSourceTypeId; + } + + public String getDesc() { + return desc; + } + + public void setDesc(String desc) { + this.desc = desc; + } + + public String getCreateUser() { + return createUser; + } + + public void setCreateUser(String createUser) { + this.createUser = createUser; + } + + public String getLabels() { + return labels; + } + + public void setLabels(String labels) { + this.labels = labels; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + public Long getVersionId() { + return versionId; + } + + public void setVersionId(Long versionId) { + this.versionId = versionId; + } + + public String getModifyUser() { + return modifyUser; + } + + public void setModifyUser(String modifyUser) { + this.modifyUser = modifyUser; + } + + public Date getModifyTime() { + return modifyTime; + } + + public void setModifyTime(Date modifyTime) { + this.modifyTime = modifyTime; + } + + public String getCreateSystem() { + return createSystem; + } + + public void setCreateSystem(String createSystem) { + this.createSystem = createSystem; + } + + public boolean isWriteAble() { + return writeAble; } - public void setOption(String option) { - this.option = option; + public void setWriteAble(boolean writeAble) { + this.writeAble = writeAble; } - public void setDescription(String description) { - this.description = description; + public boolean isReadAble() { + return readAble; } - public void setIcon(String icon) { - this.icon = icon; + public void setReadAble(boolean readAble) { + this.readAble = readAble; } - public String getDescription() { - return description; + public String getAuthDbs() { + return authDbs; } - public String getIcon() { - return icon; + public void setAuthDbs(String authDbs) { + this.authDbs = authDbs; } - public String getStruct_classifier() { - return struct_classifier; + public String getAuthTbls() { + return authTbls; } - public void setStruct_classifier(String struct_classifier) { - this.struct_classifier = struct_classifier; + public void setAuthTbls(String authTbls) { + this.authTbls = authTbls; } } diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/ExchangisDataSourceDefDTO.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/ExchangisDataSourceDefDTO.java new file mode 100644 index 000000000..c71f60fd9 --- /dev/null +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/dto/ExchangisDataSourceDefDTO.java @@ -0,0 +1,62 @@ +package com.webank.wedatasphere.exchangis.datasource.dto; + +public class ExchangisDataSourceDefDTO { + private final String id; + private final String classifier; + private final String name; + private String option; + private String description; + private String icon; + private String struct_classifier; + + public ExchangisDataSourceDefDTO(String id, String classifier, String name, String struct_classifier) { + this.id = id; + this.classifier = classifier; + this.name = name; + this.struct_classifier = struct_classifier; + } + + public String getId() { + return id; + } + + public String getClassifier() { + return classifier; + } + + public String getName() { + return name; + } + + public String getOption() { + return option; + } + + public void setOption(String option) { + this.option = option; + } + + public void setDescription(String description) { + this.description = description; + } + + public void setIcon(String icon) { + this.icon = icon; + } + + public String getDescription() { + return description; + } + + public String getIcon() { + return icon; + } + + public String getStruct_classifier() { + return struct_classifier; + } + + public void setStruct_classifier(String struct_classifier) { + this.struct_classifier = struct_classifier; + } +} diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/AbstractDataSourceService.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/AbstractDataSourceService.java index 8688554be..f4ebaab69 100644 --- a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/AbstractDataSourceService.java +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/AbstractDataSourceService.java @@ -8,7 +8,7 @@ import com.webank.wedatasphere.exchangis.common.UserUtils; import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; import com.webank.wedatasphere.exchangis.dao.mapper.ExchangisJobParamConfigMapper; -import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSource; +import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSourceDefinition; import com.webank.wedatasphere.exchangis.datasource.core.context.ExchangisDataSourceContext; import com.webank.wedatasphere.exchangis.datasource.core.ui.*; import com.webank.wedatasphere.exchangis.datasource.core.ui.viewer.DefaultDataSourceUIViewer; @@ -84,7 +84,7 @@ private ExchangisDataSourceIdsUI buildDataSourceIdsUI(HttpServletRequest request source.setType(split[0]); source.setId(split[1]); Optional.ofNullable(loginUser).ifPresent(u -> { - Optional.ofNullable(this.context.getExchangisDataSource(split[0])).ifPresent(o -> { + Optional.ofNullable(this.context.getExchangisDsDefinition(split[0])).ifPresent(o -> { LinkisDataSourceRemoteClient dsClient = o.getDataSourceRemoteClient(); GetInfoByDataSourceIdAction action = GetInfoByDataSourceIdAction.builder() .setDataSourceId(Long.parseLong(split[1])) @@ -112,7 +112,7 @@ private ExchangisDataSourceIdsUI buildDataSourceIdsUI(HttpServletRequest request sink.setType(split[0]); sink.setId(split[1]); Optional.ofNullable(loginUser).ifPresent(u -> { - Optional.ofNullable(this.context.getExchangisDataSource(split[0])).ifPresent(o -> { + Optional.ofNullable(this.context.getExchangisDsDefinition(split[0])).ifPresent(o -> { LinkisDataSourceRemoteClient dsClient = o.getDataSourceRemoteClient(); GetInfoByDataSourceIdAction action = GetInfoByDataSourceIdAction.builder() .setDataSourceId(Long.parseLong(split[1])) @@ -146,7 +146,7 @@ protected ExchangisDataSourceParamsUI buildDataSourceParamsUI(ExchangisJobInfoCo ExchangisDataSourceIdUI source = dataSourceIdsUI.getSource(); if (null != source) { String type = source.getType(); - ExchangisDataSource exchangisSourceDataSource = this.context.getExchangisDataSource(type); + ExchangisDataSourceDefinition exchangisSourceDataSource = this.context.getExchangisDsDefinition(type); if (null != exchangisSourceDataSource) { sourceParamConfigs = exchangisSourceDataSource.getDataSourceParamConfigs().stream().filter( i -> i.getConfigDirection().equals(content.getEngine() + "-SOURCE") || "SOURCE".equalsIgnoreCase(i.getConfigDirection())).collect(Collectors.toList()); @@ -156,7 +156,7 @@ protected ExchangisDataSourceParamsUI buildDataSourceParamsUI(ExchangisJobInfoCo ExchangisDataSourceIdUI sink = dataSourceIdsUI.getSink(); if (null != sink) { String type = sink.getType(); - ExchangisDataSource exchangisSinkDataSource = this.context.getExchangisDataSource(type); + ExchangisDataSourceDefinition exchangisSinkDataSource = this.context.getExchangisDsDefinition(type); if (null != exchangisSinkDataSource) { sinkParamConfigs = exchangisSinkDataSource.getDataSourceParamConfigs().stream().filter(i -> i.getConfigDirection().equals(content.getEngine() + "-SINK") || "SINK".equalsIgnoreCase(i.getConfigDirection())).collect(Collectors.toList()); diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java index 7243c075b..727db8f49 100644 --- a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/ExchangisDataSourceService.java @@ -11,7 +11,7 @@ import com.webank.wedatasphere.exchangis.dao.mapper.ExchangisJobParamConfigMapper; import com.webank.wedatasphere.exchangis.datasource.GetDataSourceInfoByIdAndVersionIdAction; import com.webank.wedatasphere.exchangis.datasource.Utils.RSAUtil; -import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSource; +import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSourceDefinition; import com.webank.wedatasphere.exchangis.datasource.core.context.ExchangisDataSourceContext; import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisDataSourceException; import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisDataSourceExceptionCode; @@ -112,7 +112,7 @@ public List getJobDataSourceUIs(HttpServletRequest @Override public List> getDataSourceParamsUI(String dsType, String engineAndDirection) { - ExchangisDataSource exchangisDataSource = this.context.getExchangisDataSource(dsType); + ExchangisDataSourceDefinition exchangisDataSource = this.context.getExchangisDsDefinition(dsType); List paramConfigs = exchangisDataSource.getDataSourceParamConfigs(); List filteredConfigs = new ArrayList<>(); String[] engineDirect = engineAndDirection.split("-"); @@ -138,8 +138,8 @@ public List> getJobEngineSettingsUI(String engineType) { * 做比较，筛选出可以给前端展示的数据源类型 */ public Message listDataSources(HttpServletRequest request, String engineType, String direct, String sourceType) throws Exception { - Collection all = this.context.all(); - List dtos = new ArrayList<>(); + Collection all = this.context.all(); + List dtos = new ArrayList<>(); List settingsList = this.settingsDao.getSettings(); List engineSettings = new ArrayList<>(); @@ -207,9 +207,9 @@ public Message listDataSources(HttpServletRequest request, String engineType, St for (DataSourceType type : allDataSourceType) { LOGGER.info("Current datasource Type is :{}", type.getName()); - for (ExchangisDataSource item : all) { + for (ExchangisDataSourceDefinition item : all) { if (item.name().equalsIgnoreCase(type.getName())) { - ExchangisDataSourceDTO dto = new ExchangisDataSourceDTO( + ExchangisDataSourceDefDTO dto = new ExchangisDataSourceDefDTO( type.getId(), type.getClassifier(), // item.classifier(), @@ -258,12 +258,12 @@ public Message create(HttpServletRequest request, /*String type, */DataSourceCre String user = UserUtils.getLoginUser(request); LOGGER.info("createDatasource userName:" + user); - ExchangisDataSource exchangisDataSource = context.getExchangisDataSource(vo.getDataSourceTypeId()); - if (Objects.isNull(exchangisDataSource)) { + ExchangisDataSourceDefinition dsType = context.getExchangisDsDefinition(vo.getDataSourceTypeId()); + if (Objects.isNull(dsType)) { throw new ExchangisDataSourceException(CONTEXT_GET_DATASOURCE_NULL.getCode(), "exchangis context get datasource null"); } - LinkisDataSourceRemoteClient client = exchangisDataSource.getDataSourceRemoteClient(); + LinkisDataSourceRemoteClient client = dsType.getDataSourceRemoteClient(); LOGGER.info("create datasource json as follows"); Set> entries = json.entrySet(); for (Map.Entry entry : entries) { @@ -341,12 +341,12 @@ public Message updateDataSource(HttpServletRequest request,/* String type,*/ Lon LOGGER.info("updateDataSource userName:" + user); LOGGER.info("DataSourceTypeId:" + vo.getDataSourceTypeId()); - ExchangisDataSource exchangisDataSource = context.getExchangisDataSource(vo.getDataSourceTypeId()); - if (Objects.isNull(exchangisDataSource)) { + ExchangisDataSourceDefinition dsType = context.getExchangisDsDefinition(vo.getDataSourceTypeId()); + if (Objects.isNull(dsType)) { throw new ExchangisDataSourceException(30401, "exchangis.datasource.null"); } - LinkisDataSourceRemoteClient client = exchangisDataSource.getDataSourceRemoteClient(); + LinkisDataSourceRemoteClient client = dsType.getDataSourceRemoteClient(); // UpdateDataSourceResult updateDataSourceResult; String responseBody; try { @@ -441,8 +441,8 @@ public Message deleteDataSource(HttpServletRequest request, /*String type,*/ Lon } public Message queryDataSourceDBs(HttpServletRequest request, String type, Long id) throws Exception { - ExchangisDataSource exchangisDataSource = context.getExchangisDataSource(type); - LinkisMetaDataRemoteClient metaDataRemoteClient = exchangisDataSource.getMetaDataRemoteClient(); + ExchangisDataSourceDefinition definition = context.getExchangisDsDefinition(type); + LinkisMetaDataRemoteClient metaDataRemoteClient = definition.getMetaDataRemoteClient(); String userName = UserUtils.getLoginUser(request); LOGGER.info("queryDataSourceDBs userName:" + userName); @@ -471,10 +471,10 @@ public Message queryDataSourceDBTables(HttpServletRequest request, String type, String user = UserUtils.getLoginUser(request); LOGGER.info("queryDataSourceDBTables userName:" + user); - ExchangisDataSource exchangisDataSource = context.getExchangisDataSource(type); + ExchangisDataSourceDefinition definition = context.getExchangisDsDefinition(type); MetadataGetTablesResult tables; try { - LinkisMetaDataRemoteClient metaDataRemoteClient = exchangisDataSource.getMetaDataRemoteClient(); + LinkisMetaDataRemoteClient metaDataRemoteClient = definition.getMetaDataRemoteClient(); tables = metaDataRemoteClient.getTables(MetadataGetTablesAction.builder() .setSystem(type) .setDataSourceId(id) @@ -586,8 +586,8 @@ public Message getJobDataSourceSettingsUI(Long jobId, String jobName) throws Exc } public Message queryDataSourceDBTableFields(HttpServletRequest request, String type, Long id, String dbName, String tableName) throws Exception { - ExchangisDataSource exchangisDataSource = context.getExchangisDataSource(type); - LinkisMetaDataRemoteClient metaDataRemoteClient = exchangisDataSource.getMetaDataRemoteClient(); + ExchangisDataSourceDefinition definition = context.getExchangisDsDefinition(type); + LinkisMetaDataRemoteClient metaDataRemoteClient = definition.getMetaDataRemoteClient(); String user = UserUtils.getLoginUser(request); LOGGER.info("queryDataSourceDBTableFields userName:" + user); @@ -671,10 +671,10 @@ public Message queryDataSources(HttpServletRequest request, DataSourceQueryVO vo List allDataSource = result.getAllDataSource(); - List originDataSources = new ArrayList<>(); - List dataSources = new ArrayList<>(); + List originDataSources = new ArrayList<>(); + List dataSources = new ArrayList<>(); allDataSource.forEach(ds -> { - DataSourceDTO item = new DataSourceDTO(); + ExchangisDataSourceDTO item = new ExchangisDataSourceDTO(); item.setId(ds.getId()); item.setCreateIdentify(ds.getCreateIdentify()); item.setName(ds.getDataSourceName()); @@ -701,13 +701,13 @@ public Message queryDataSources(HttpServletRequest request, DataSourceQueryVO vo LOGGER.info("originDatasource is: {}", originDataSources); if (direct!=null) { if ("source".equals(direct)) { - for (DataSourceDTO originDataSource : originDataSources) { + for (ExchangisDataSourceDTO originDataSource : originDataSources) { if (originDataSource.isReadAble()) { dataSources.add(originDataSource); } } } else if ("sink".equals(direct)) { - for (DataSourceDTO originDataSource : originDataSources) { + for (ExchangisDataSourceDTO originDataSource : originDataSources) { if (originDataSource.isReadAble()) { dataSources.add(originDataSource); } @@ -758,10 +758,10 @@ public Message listAllDataSources(HttpServletRequest request, String typeName, L } catch (Exception e) { throw new ExchangisDataSourceException(ExchangisDataSourceExceptionCode.CLIENT_QUERY_DATASOURCE_ERROR.getCode(), e.getMessage()); } - List dataSources = new ArrayList<>(); + List dataSources = new ArrayList<>(); if (!Objects.isNull(allDataSource)) { allDataSource.forEach(ds -> { - DataSourceDTO item = new DataSourceDTO(); + ExchangisDataSourceDTO item = new ExchangisDataSourceDTO(); item.setId(ds.getId()); item.setCreateIdentify(ds.getCreateIdentify()); item.setName(ds.getDataSourceName()); diff --git a/exchangis-datasource/exchangis-datasource-streamis/pom.xml b/exchangis-datasource/exchangis-datasource-streamis/pom.xml index 3d86b9522..b27dfb010 100644 --- a/exchangis-datasource/exchangis-datasource-streamis/pom.xml +++ b/exchangis-datasource/exchangis-datasource-streamis/pom.xml @@ -23,7 +23,11 @@ exchangis-datasource-core ${revision} - + + com.webank.wedatasphere.exchangis + exchangis-datasource-linkis + ${revision} + diff --git a/exchangis-datasource/exchangis-datasource-streamis/src/main/java/com/webank/wedatasphere/exchangis/datasource/streamis/ExchangisStreamisDataSource.java b/exchangis-datasource/exchangis-datasource-streamis/src/main/java/com/webank/wedatasphere/exchangis/datasource/streamis/ExchangisStreamisDataSource.java index a47a040ca..71b614f5a 100644 --- a/exchangis-datasource/exchangis-datasource-streamis/src/main/java/com/webank/wedatasphere/exchangis/datasource/streamis/ExchangisStreamisDataSource.java +++ b/exchangis-datasource/exchangis-datasource-streamis/src/main/java/com/webank/wedatasphere/exchangis/datasource/streamis/ExchangisStreamisDataSource.java @@ -1,40 +1,22 @@ package com.webank.wedatasphere.exchangis.datasource.streamis; -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; -import com.webank.wedatasphere.exchangis.dao.hook.MapperHook; -import com.webank.wedatasphere.exchangis.dao.mapper.ExchangisJobParamConfigMapper; -import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSource; +import com.webank.wedatasphere.exchangis.datasource.core.AbstractExchangisDataSourceDefinition; +import com.webank.wedatasphere.exchangis.datasource.linkis.ExchangisLinkisRemoteClient; import org.apache.linkis.datasource.client.impl.LinkisDataSourceRemoteClient; import org.apache.linkis.datasource.client.impl.LinkisMetaDataRemoteClient; -import java.util.List; - -public abstract class ExchangisStreamisDataSource implements ExchangisDataSource { - - protected MapperHook mapperHook; - - @Override - public void setMapperHook(MapperHook mapperHook) { - this.mapperHook = mapperHook; - } - - protected List getDataSourceParamConfigs(String type) { - ExchangisJobParamConfigMapper exchangisJobParamConfigMapper = this.mapperHook.getExchangisJobParamConfigMapper(); - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.eq("type", type); - queryWrapper.eq("is_hidden", 0); - queryWrapper.eq("status", 1); - return exchangisJobParamConfigMapper.selectList(queryWrapper); - } +/** + * Exchangis streamis data source + */ +public abstract class ExchangisStreamisDataSource extends AbstractExchangisDataSourceDefinition { @Override public LinkisDataSourceRemoteClient getDataSourceRemoteClient() { - return ExchangisStreamisRemoteClient.getStreamisDataSourceRemoteClient(); + return ExchangisLinkisRemoteClient.getLinkisDataSourceRemoteClient(); } @Override public LinkisMetaDataRemoteClient getMetaDataRemoteClient() { - return ExchangisStreamisRemoteClient.getStreamisMetadataRemoteClient(); + return ExchangisLinkisRemoteClient.getLinkisMetadataRemoteClient(); } } diff --git a/exchangis-datasource/exchangis-datasource-streamis/src/main/scala/com/webank/wedatasphere/exchangis/datasource/streamis/ExchangisStreamisRemoteClient.scala b/exchangis-datasource/exchangis-datasource-streamis/src/main/scala/com/webank/wedatasphere/exchangis/datasource/streamis/ExchangisStreamisRemoteClient.scala deleted file mode 100644 index 0085608bb..000000000 --- a/exchangis-datasource/exchangis-datasource-streamis/src/main/scala/com/webank/wedatasphere/exchangis/datasource/streamis/ExchangisStreamisRemoteClient.scala +++ /dev/null @@ -1,172 +0,0 @@ -package com.webank.wedatasphere.exchangis.datasource.streamis - -import com.webank.wedatasphere.exchangis.datasource.core.ExchangisDataSourceConfiguration -import java.lang -import java.util.concurrent.TimeUnit - -import org.apache.linkis.datasource.client.impl.{LinkisDataSourceRemoteClient, LinkisMetaDataRemoteClient} -import org.apache.linkis.datasource.client.request.{GetAllDataSourceTypesAction, GetConnectParamsByDataSourceIdAction, MetadataGetColumnsAction, MetadataGetDatabasesAction, MetadataGetTablesAction, QueryDataSourceAction} -import org.apache.linkis.datasource.client.response.{GetConnectParamsByDataSourceIdResult, MetadataGetColumnsResult, MetadataGetDatabasesResult, MetadataGetTablesResult, QueryDataSourceResult} -import org.apache.linkis.datasourcemanager.common.domain.{DataSource, DataSourceType} -import org.apache.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy -import org.apache.linkis.httpclient.dws.config.{DWSClientConfig, DWSClientConfigBuilder} - -object ExchangisStreamisRemoteClient { - //Linkis Datasource Client Config - val serverUrl: String = ExchangisDataSourceConfiguration.SERVER_URL.getValue - val connectionTimeout: lang.Long = ExchangisDataSourceConfiguration.CONNECTION_TIMEOUT.getValue - val discoveryEnabled: lang.Boolean = ExchangisDataSourceConfiguration.DISCOVERY_ENABLED.getValue - val discoveryFrequencyPeriod: lang.Long = ExchangisDataSourceConfiguration.DISCOVERY_FREQUENCY_PERIOD.getValue - val loadbalancerEnabled: lang.Boolean = ExchangisDataSourceConfiguration.LOAD_BALANCER_ENABLED.getValue - val maxConnectionSize: Integer = ExchangisDataSourceConfiguration.MAX_CONNECTION_SIZE.getValue - val retryEnabled: lang.Boolean = ExchangisDataSourceConfiguration.RETRY_ENABLED.getValue - val readTimeout: lang.Long = ExchangisDataSourceConfiguration.READ_TIMEOUT.getValue - val authTokenKey: String = ExchangisDataSourceConfiguration.AUTHTOKEN_KEY.getValue - val authTokenValue: String = ExchangisDataSourceConfiguration.AUTHTOKEN_VALUE.getValue - val dwsVersion: String = ExchangisDataSourceConfiguration.DWS_VERSION.getValue - - - // val clientConfig = DWSClientConfigBuilder.newBuilder() - // .addServerUrl(serverUrl) - // .connectionTimeout(connectionTimeout) - // .discoveryEnabled(discoveryEnabled) - // .discoveryFrequency(1,TimeUnit.MINUTES) - // .loadbalancerEnabled(loadbalancerEnabled) - // .maxConnectionSize(maxConnectionSize) - // .retryEnabled(retryEnabled) - // .readTimeout(readTimeout) - // .setAuthenticationStrategy(new StaticAuthenticationStrategy()) - // .setAuthTokenKey(authTokenKey) - // .setAuthTokenValue(authTokenValue) - // .setDWSVersion(dwsVersion) - // .build() - - val clientConfig: DWSClientConfig = DWSClientConfigBuilder.newBuilder() - .addServerUrl(serverUrl) - .connectionTimeout(connectionTimeout) - .discoveryEnabled(discoveryEnabled) - .discoveryFrequency(discoveryFrequencyPeriod, TimeUnit.MINUTES) - .loadbalancerEnabled(loadbalancerEnabled) - .maxConnectionSize(maxConnectionSize) - .retryEnabled(retryEnabled) - .readTimeout(readTimeout) - .setAuthenticationStrategy(new StaticAuthenticationStrategy()) - .setAuthTokenKey(authTokenKey) - .setAuthTokenValue(authTokenValue) - .setDWSVersion(dwsVersion) - .build() - - val dataSourceClient = new LinkisDataSourceRemoteClient(clientConfig) - - val metaDataClient = new LinkisMetaDataRemoteClient(clientConfig) - - def getStreamisDataSourceRemoteClient: LinkisDataSourceRemoteClient = { - dataSourceClient - } - - def getStreamisMetadataRemoteClient: LinkisMetaDataRemoteClient = { - metaDataClient - } - - def close(): Unit = { - dataSourceClient.close() - metaDataClient.close() - } - - def queryDataSource(linkisDatasourceName: String): QueryDataSourceResult = { - dataSourceClient.queryDataSource(QueryDataSourceAction.builder() - .setSystem("") - .setName(linkisDatasourceName) - .setTypeId(1) - .setIdentifies("") - .setCurrentPage(1) - .setUser("hadoop") - .setPageSize(1).build() - ) - } - - /** - * get datasourceConnect information - * - * @param dataSourceId id - * @param system dssSystem - * @param user username - * @return - */ - def queryConnectParams(dataSourceId: Long, system: String, user: String): GetConnectParamsByDataSourceIdResult = { - dataSourceClient.getConnectParams(GetConnectParamsByDataSourceIdAction.builder() - .setDataSourceId(dataSourceId) - .setSystem(system) - .setUser(user) - .build() - ) - } - - /** - * get all DataSourceTypes - * - * @param user user - * @return - */ - def queryDataSourceTypes(user: String): java.util.List[DataSourceType] = { - dataSourceClient.getAllDataSourceTypes(GetAllDataSourceTypesAction.builder() - .setUser(user) - .build() - ).getAllDataSourceType - } - - - def queryClusterByDataSourceType(system: String, name: String, typeId: Long, user: String): java.util.List[DataSource] = { - dataSourceClient.queryDataSource(QueryDataSourceAction.builder() - .setSystem(system) - .setName(name) - .setTypeId(typeId) - .setIdentifies("") - .setCurrentPage(1) - .setPageSize(10) - .setUser(user) - .build() - ).getAllDataSource - } - - - /** - * get DataBases list - * - * @param system - * @param dataSourceId - * @param user - * @return list - */ - def queryDataBasesByCuster(system: String, dataSourceId: Long, user: String): MetadataGetDatabasesResult = { - metaDataClient.getDatabases(MetadataGetDatabasesAction.builder() - .setSystem(system) - .setDataSourceId(dataSourceId) - .setUser(user) - .build() - ) - } - - def queryTablesByDataBase(system: String, dataSourceId: Long, dataBase: String, user: String): MetadataGetTablesResult = { - metaDataClient.getTables(MetadataGetTablesAction.builder() - .setSystem(system) - .setDataSourceId(dataSourceId) - .setDatabase(dataBase) - .setUser(user) - .build() - ) - } - - def queryColumnsByTable(system: String, dataSourceId: Long, dataBase: String, table: String, user: String): MetadataGetColumnsResult = { - metaDataClient.getColumns(MetadataGetColumnsAction.builder() - .setSystem(system) - .setDataSourceId(dataSourceId) - .setDatabase(dataBase) - .setTable(table) - .setUser(user) - .build() - ) - } - - -} diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-elasticsearch/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/mysql/ExchangisESDataSource.java b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-elasticsearch/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/mysql/ExchangisESDataSource.java index 607af79de..a049d07ce 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-elasticsearch/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/mysql/ExchangisESDataSource.java +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-elasticsearch/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/mysql/ExchangisESDataSource.java @@ -1,23 +1,17 @@ package com.webank.wedatasphere.exchangis.extension.datasource.mysql; -import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; -import com.webank.wedatasphere.exchangis.datasource.core.domain.Classifier; -import com.webank.wedatasphere.exchangis.datasource.core.domain.DataSourceType; +import com.webank.wedatasphere.exchangis.datasource.core.domain.ExchangisDataSourceType; import com.webank.wedatasphere.exchangis.datasource.core.domain.StructClassifier; import com.webank.wedatasphere.exchangis.datasource.linkis.ExchangisBatchDataSource; -import java.util.List; - +/** + * Note: ES data source + */ public class ExchangisESDataSource extends ExchangisBatchDataSource { @Override - public String name() { - return DataSourceType.ELASTICSEARCH.name; - } - - @Override - public String classifier() { - return Classifier.ELASTICSEARCH.name; + protected ExchangisDataSourceType type() { + return ExchangisDataSourceType.ELASTICSEARCH; } @Override @@ -40,8 +34,4 @@ public String icon() { return "icon-es"; } - @Override - public List getDataSourceParamConfigs() { - return super.getDataSourceParamConfigs(DataSourceType.ELASTICSEARCH.name); - } } \ No newline at end of file diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-hive/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/hive/ExchangisHiveDataSource.java b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-hive/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/hive/ExchangisHiveDataSource.java index e9ce77738..c59cfe277 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-hive/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/hive/ExchangisHiveDataSource.java +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-hive/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/hive/ExchangisHiveDataSource.java @@ -1,23 +1,17 @@ package com.webank.wedatasphere.exchangis.extension.datasource.hive; -import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; -import com.webank.wedatasphere.exchangis.datasource.core.domain.Classifier; -import com.webank.wedatasphere.exchangis.datasource.core.domain.DataSourceType; +import com.webank.wedatasphere.exchangis.datasource.core.domain.ExchangisDataSourceType; import com.webank.wedatasphere.exchangis.datasource.core.domain.StructClassifier; import com.webank.wedatasphere.exchangis.datasource.linkis.ExchangisBatchDataSource; -import java.util.List; - +/** + * Note: Hive data source + */ public class ExchangisHiveDataSource extends ExchangisBatchDataSource { @Override - public String name() { - return DataSourceType.HIVE.name; - } - - @Override - public String classifier() { - return Classifier.HIVE.name; + protected ExchangisDataSourceType type() { + return ExchangisDataSourceType.HIVE; } @Override @@ -40,8 +34,4 @@ public String icon() { return "icon-hive"; } - @Override - public List getDataSourceParamConfigs() { - return super.getDataSourceParamConfigs(DataSourceType.HIVE.name); - } } diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mongodb/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/mysql/ExchangisMongoDbDataSource.java b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mongodb/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/mysql/ExchangisMongoDbDataSource.java index 269efce87..0fec6da36 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mongodb/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/mysql/ExchangisMongoDbDataSource.java +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mongodb/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/mysql/ExchangisMongoDbDataSource.java @@ -1,23 +1,17 @@ package com.webank.wedatasphere.exchangis.extension.datasource.mysql; -import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; -import com.webank.wedatasphere.exchangis.datasource.core.domain.Classifier; -import com.webank.wedatasphere.exchangis.datasource.core.domain.DataSourceType; +import com.webank.wedatasphere.exchangis.datasource.core.domain.ExchangisDataSourceType; import com.webank.wedatasphere.exchangis.datasource.core.domain.StructClassifier; import com.webank.wedatasphere.exchangis.datasource.linkis.ExchangisBatchDataSource; -import java.util.List; - +/** + * Note: MongoDB data source + */ public class ExchangisMongoDbDataSource extends ExchangisBatchDataSource { @Override - public String name() { - return DataSourceType.MONGODB.name; - } - - @Override - public String classifier() { - return Classifier.MONGODB.name; + protected ExchangisDataSourceType type() { + return ExchangisDataSourceType.MONGODB; } @Override @@ -40,8 +34,4 @@ public String icon() { return "icon-mongodb"; } - @Override - public List getDataSourceParamConfigs() { - return super.getDataSourceParamConfigs(DataSourceType.MONGODB.name); - } } \ No newline at end of file diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mysql/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/mysql/ExchangisMySQLDataSource.java b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mysql/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/mysql/ExchangisMySQLDataSource.java index e05e790da..d818a1f65 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mysql/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/mysql/ExchangisMySQLDataSource.java +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mysql/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/mysql/ExchangisMySQLDataSource.java @@ -1,23 +1,18 @@ package com.webank.wedatasphere.exchangis.extension.datasource.mysql; -import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; -import com.webank.wedatasphere.exchangis.datasource.core.domain.Classifier; -import com.webank.wedatasphere.exchangis.datasource.core.domain.DataSourceType; +import com.webank.wedatasphere.exchangis.datasource.core.domain.ExchangisDataSourceType; import com.webank.wedatasphere.exchangis.datasource.core.domain.StructClassifier; import com.webank.wedatasphere.exchangis.datasource.linkis.ExchangisBatchDataSource; -import java.util.List; +/** + * Note: MYSQL data source + */ public class ExchangisMySQLDataSource extends ExchangisBatchDataSource { @Override - public String name() { - return DataSourceType.MYSQL.name; - } - - @Override - public String classifier() { - return Classifier.MYSQL.name; + protected ExchangisDataSourceType type() { + return ExchangisDataSourceType.MYSQL; } @Override @@ -40,8 +35,5 @@ public String icon() { return "icon-mysql"; } - @Override - public List getDataSourceParamConfigs() { - return super.getDataSourceParamConfigs(DataSourceType.MYSQL.name); - } + } \ No newline at end of file diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-oracle/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/oracle/ExchangisOracleDataSource.java b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-oracle/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/oracle/ExchangisOracleDataSource.java index 8d3ca3adb..38d0c1a8b 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-oracle/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/oracle/ExchangisOracleDataSource.java +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-oracle/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/oracle/ExchangisOracleDataSource.java @@ -2,7 +2,7 @@ import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; import com.webank.wedatasphere.exchangis.datasource.core.domain.Classifier; -import com.webank.wedatasphere.exchangis.datasource.core.domain.DataSourceType; +import com.webank.wedatasphere.exchangis.datasource.core.domain.ExchangisDataSourceType; import com.webank.wedatasphere.exchangis.datasource.core.domain.StructClassifier; import com.webank.wedatasphere.exchangis.datasource.linkis.ExchangisBatchDataSource; @@ -13,9 +13,10 @@ * @create 2022-09-14 **/ public class ExchangisOracleDataSource extends ExchangisBatchDataSource { + @Override - public String name() { - return DataSourceType.ORACLE.name; + protected ExchangisDataSourceType type() { + return ExchangisDataSourceType.ORACLE; } @Override @@ -28,10 +29,6 @@ public String option() { return "Oracle数据库"; } - @Override - public String classifier() { - return Classifier.ORACLE.name; - } @Override public String structClassifier() { @@ -43,9 +40,5 @@ public String icon() { return "icon-oracle"; } - @Override - public List getDataSourceParamConfigs() { - return super.getDataSourceParamConfigs(DataSourceType.ORACLE.name); - } } diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-sftp/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/sftp/ExchangisSftpDataSource.java b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-sftp/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/sftp/ExchangisSftpDataSource.java index c8256dddb..b9ecdfbb4 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-sftp/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/sftp/ExchangisSftpDataSource.java +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-sftp/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/sftp/ExchangisSftpDataSource.java @@ -2,7 +2,7 @@ import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; import com.webank.wedatasphere.exchangis.datasource.core.domain.Classifier; -import com.webank.wedatasphere.exchangis.datasource.core.domain.DataSourceType; +import com.webank.wedatasphere.exchangis.datasource.core.domain.ExchangisDataSourceType; import com.webank.wedatasphere.exchangis.datasource.core.domain.StructClassifier; import com.webank.wedatasphere.exchangis.datasource.linkis.ExchangisBatchDataSource; @@ -11,13 +11,8 @@ public class ExchangisSftpDataSource extends ExchangisBatchDataSource { @Override - public String name() { - return DataSourceType.SFTP.name; - } - - @Override - public String classifier() { - return Classifier.SFTP.name; + protected ExchangisDataSourceType type() { + return ExchangisDataSourceType.SFTP; } @Override @@ -27,7 +22,7 @@ public String structClassifier() { @Override public String description() { - return "This is Sftp"; + return "This is sftp"; } @Override @@ -42,6 +37,7 @@ public String icon() { @Override public List getDataSourceParamConfigs() { - return super.getDataSourceParamConfigs(DataSourceType.SFTP.name); + return super.getDataSourceParamConfigs(ExchangisDataSourceType.SFTP.name); } + } diff --git a/exchangis-datasource/pom.xml b/exchangis-datasource/pom.xml index 817f59268..a0a40021c 100644 --- a/exchangis-datasource/pom.xml +++ b/exchangis-datasource/pom.xml @@ -16,8 +16,8 @@ exchangis-datasource-core exchangis-datasource-loader - exchangis-datasource-streamis exchangis-datasource-linkis + exchangis-datasource-streamis exchangis-datasource-service extension-datasources/exchangis-datasource-ext-mysql extension-datasources/exchangis-datasource-ext-hive diff --git a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisExchangisTaskLauncher.java b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisExchangisTaskLauncher.java index b366b8dc4..730ff21b7 100644 --- a/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisExchangisTaskLauncher.java +++ b/exchangis-job/exchangis-job-launcher/src/main/java/com/webank/wedatasphere/exchangis/job/launcher/linkis/LinkisExchangisTaskLauncher.java @@ -1,6 +1,7 @@ package com.webank.wedatasphere.exchangis.job.launcher.linkis; -import com.webank.wedatasphere.exchangis.common.linkis.ClientConfiguration; +import com.webank.wedatasphere.exchangis.common.linkis.client.config.ExchangisClientConfig; +import com.webank.wedatasphere.exchangis.common.linkis.client.config.ExchangisClientConfigBuilder; import com.webank.wedatasphere.exchangis.job.enums.EngineTypeEnum; import com.webank.wedatasphere.exchangis.job.launcher.AccessibleLauncherTask; import com.webank.wedatasphere.exchangis.job.launcher.exception.ExchangisTaskLaunchException; @@ -8,19 +9,16 @@ import com.webank.wedatasphere.exchangis.job.launcher.ExchangisTaskLauncher; import com.webank.wedatasphere.exchangis.job.launcher.domain.LaunchableExchangisTask; import com.webank.wedatasphere.exchangis.job.launcher.domain.LaunchedExchangisTask; +import com.webank.wedatasphere.exchangis.job.launcher.linkis.client.ExchangisLaunchClient; import org.apache.commons.lang.StringUtils; -import org.apache.linkis.common.conf.Configuration; import org.apache.linkis.common.exception.LinkisRetryException; import org.apache.linkis.common.utils.DefaultRetryHandler; import org.apache.linkis.common.utils.RetryHandler; import org.apache.linkis.computation.client.LinkisJobClient$; -import org.apache.linkis.httpclient.config.ClientConfig; -import org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy; -import org.apache.linkis.httpclient.dws.config.DWSClientConfig; -import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder$; +import org.apache.linkis.computation.client.once.simple.SimpleOnceJobBuilder$; +import java.lang.reflect.Field; import java.util.*; -import java.util.concurrent.TimeUnit; /** * Linkis task launcher @@ -44,23 +42,31 @@ public void init(ExchangisTaskLaunchManager jobLaunchManager) { this.engineVersions.put(EngineTypeEnum.DATAX.name().toLowerCase(), "3.0.0"); RetryHandler retryHandler = new DefaultRetryHandler(){}; retryHandler.addRetryException(LinkisRetryException.class); - ClientConfig clientConfig = DWSClientConfigBuilder$.MODULE$ - .newBuilder() - .setDWSVersion(Configuration.LINKIS_WEB_VERSION().getValue()) - .addServerUrl(ClientConfiguration.LINKIS_SERVER_URL.getValue()) - .connectionTimeout(45000) - .discoveryEnabled(false) - .discoveryFrequency(1, TimeUnit.MINUTES) - .loadbalancerEnabled(false) - .maxConnectionSize(ClientConfiguration.LINKIS_DEFAULT_MAX_CONNECTIONS.getValue()) + ExchangisClientConfigBuilder builder = (ExchangisClientConfigBuilder) ExchangisClientConfig.newBuilder().discoveryEnabled(false) .retryEnabled(true) - .setRetryHandler(retryHandler) - .readTimeout(90000) // We think 90s is enough, if SocketTimeoutException is throw, just set a new clientConfig to modify it. - .setAuthenticationStrategy(new TokenAuthenticationStrategy()) - .setAuthTokenKey(TokenAuthenticationStrategy.TOKEN_KEY()) - .setAuthTokenValue(ClientConfiguration.LINKIS_TOKEN_VALUE.getValue()) - .build(); - LinkisJobClient$.MODULE$.config().setDefaultClientConfig((DWSClientConfig) clientConfig); + .setRetryHandler(retryHandler); + ExchangisClientConfig clientConfig = builder.build(); + // Try to set the static method + Class clz = SimpleOnceJobBuilder$.MODULE$.getClass(); + Field field; + boolean setField = false; + try { + field = clz.getDeclaredField(SimpleOnceJobBuilder$.class.getName().replace(".", "$") + "$linkisManagerClient"); + field.setAccessible(true); + try { + ExchangisLaunchClient client = new ExchangisLaunchClient(clientConfig); + field.set("", client); + Runtime.getRuntime().addShutdownHook(new Thread(client::close)); + setField = true; + } catch (IllegalAccessException e) { + // Ignore + } + } catch (NoSuchFieldException e) { + // Ignore + } + if (!setField){ + LinkisJobClient$.MODULE$.config().setDefaultClientConfig(clientConfig); + } } @Override @@ -104,4 +110,6 @@ private Map convertJobInfoToStore(Map jobInfo){ }); return storeInfo; } + } + diff --git a/exchangis-job/exchangis-job-launcher/src/main/scala/com/webank/wedatasphere/exchangis/job/launcher/linkis/client/ExchangisLaunchClient.scala b/exchangis-job/exchangis-job-launcher/src/main/scala/com/webank/wedatasphere/exchangis/job/launcher/linkis/client/ExchangisLaunchClient.scala new file mode 100644 index 000000000..309a05228 --- /dev/null +++ b/exchangis-job/exchangis-job-launcher/src/main/scala/com/webank/wedatasphere/exchangis/job/launcher/linkis/client/ExchangisLaunchClient.scala @@ -0,0 +1,54 @@ +package com.webank.wedatasphere.exchangis.job.launcher.linkis.client + +import com.webank.wedatasphere.exchangis.common.linkis.client.ExchangisHttpClient +import com.webank.wedatasphere.exchangis.common.linkis.client.config.ExchangisClientConfig +import org.apache.linkis.common.utils.Utils +import org.apache.linkis.computation.client.once.LinkisManagerClient +import org.apache.linkis.computation.client.once.action.{AskEngineConnAction, CreateEngineConnAction, EngineConnOperateAction, GetEngineConnAction, KillEngineConnAction, LinkisManagerAction, ListEngineConnAction} +import org.apache.linkis.computation.client.once.result.{AskEngineConnResult, CreateEngineConnResult, EngineConnOperateResult, GetEngineConnResult, KillEngineConnResult, LinkisManagerResult, ListEngineConnResult} +import org.apache.linkis.httpclient.request.Action + +/** + * Exchangis launch client + */ +class ExchangisLaunchClient(clientConfig: ExchangisClientConfig) extends LinkisManagerClient{ + private val dwsHttpClient = new ExchangisHttpClient(clientConfig, "Linkis-Job-Execution-Thread") + + protected def execute[T <: LinkisManagerResult](linkisManagerAction: LinkisManagerAction): T = + linkisManagerAction match { + case action: Action => dwsHttpClient.execute(action).asInstanceOf[T] + } + + override def createEngineConn( + createEngineConnAction: CreateEngineConnAction + ): CreateEngineConnResult = execute(createEngineConnAction) + + override def getEngineConn(getEngineConnAction: GetEngineConnAction): GetEngineConnResult = + execute(getEngineConnAction) + + override def killEngineConn(killEngineConnAction: KillEngineConnAction): KillEngineConnResult = + execute(killEngineConnAction) + + override def executeEngineConnOperation( + engineConnOperateAction: EngineConnOperateAction + ): EngineConnOperateResult = { + Utils.tryCatch { + val rs = execute[EngineConnOperateResult](engineConnOperateAction) + rs + } { case e: Exception => + val rs = new EngineConnOperateResult + rs.setIsError(true) + rs.setErrorMsg(e.getMessage) + rs + } + } + + override def close(): Unit = dwsHttpClient.close() + + override def askEngineConn(askEngineConnAction: AskEngineConnAction): AskEngineConnResult = + execute(askEngineConnAction) + + override def listEngineConn(listEngineConnAction: ListEngineConnAction): ListEngineConnResult = { + execute(listEngineConnAction) + } +} From 60a035d85873af9a6676abf353640e806d68ab0d Mon Sep 17 00:00:00 2001 From: davidhua Date: Mon, 20 May 2024 19:34:17 +0800 Subject: [PATCH 23/60] Fix some compile problem. --- exchangis-job/exchangis-job-launcher/pom.xml | 17 +++++++++++++++++ exchangis-project/pom.xml | 2 +- pom.xml | 10 +++++----- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/exchangis-job/exchangis-job-launcher/pom.xml b/exchangis-job/exchangis-job-launcher/pom.xml index 19bf193f4..22ca5cc78 100644 --- a/exchangis-job/exchangis-job-launcher/pom.xml +++ b/exchangis-job/exchangis-job-launcher/pom.xml @@ -29,4 +29,21 @@ + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + \ No newline at end of file diff --git a/exchangis-project/pom.xml b/exchangis-project/pom.xml index d71cd1525..773af5277 100644 --- a/exchangis-project/pom.xml +++ b/exchangis-project/pom.xml @@ -12,9 +12,9 @@ exchangis-project pom + exchangis-project-entity exchangis-project-server exchangis-project-provider - exchangis-project-entity diff --git a/pom.xml b/pom.xml index 461a3cce1..523a18983 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ com.webank.wedatasphere.exchangis exchangis - 1.1.3-webank + ${revision} pom exchangis @@ -229,10 +229,10 @@ ${jdk.compile.version} ${jdk.compile.version} - - - ${java.home}/lib/rt.jar;${java.home}/lib/jce.jar - + + + + From 48b1c34508e44228c8630dc659180eddf3ab2773 Mon Sep 17 00:00:00 2001 From: davidhua Date: Mon, 20 May 2024 20:00:08 +0800 Subject: [PATCH 24/60] Fix some conflict problem. --- .../core/domain/ExchangisDataSourceType.java | 3 ++- .../ExchangisStarRocksDataSource.java | 18 +++++++----------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/domain/ExchangisDataSourceType.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/domain/ExchangisDataSourceType.java index 445f3f00b..de359ca4f 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/domain/ExchangisDataSourceType.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/domain/ExchangisDataSourceType.java @@ -12,8 +12,9 @@ public enum ExchangisDataSourceType { SFTP("SFTP", "sftp连接"), - ORACLE("ORACLE", "关系型数据库"); + ORACLE("ORACLE", "关系型数据库"), + STARROCKS("STARROCKS", "大数据存储"); /** * Type name */ diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-starrocks/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/starrocks/ExchangisStarRocksDataSource.java b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-starrocks/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/starrocks/ExchangisStarRocksDataSource.java index 935aed652..894b45439 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-starrocks/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/starrocks/ExchangisStarRocksDataSource.java +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-starrocks/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/starrocks/ExchangisStarRocksDataSource.java @@ -1,8 +1,8 @@ package com.webank.wedatasphere.exchangis.extension.datasource.starrocks; import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; +import com.webank.wedatasphere.exchangis.datasource.core.domain.ExchangisDataSourceType; import com.webank.wedatasphere.exchangis.datasource.linkis.ExchangisBatchDataSource; -import com.webank.wedatasphere.exchangis.datasource.core.domain.DataSourceType; import java.util.List; @@ -13,10 +13,9 @@ public class ExchangisStarRocksDataSource extends ExchangisBatchDataSource { @Override - public String name() { - return DataSourceType.STARROCKS.name; + protected ExchangisDataSourceType type() { + return ExchangisDataSourceType.STARROCKS; } - @Override public String description() { return "This is StarRocks DataSource"; @@ -27,11 +26,6 @@ public String option() { return "StarRocks数据库"; } - @Override - public String classifier() { - return null; - } - @Override public String structClassifier() { return null; @@ -39,11 +33,13 @@ public String structClassifier() { @Override public String icon() { - return null; + return "icon-starrocks"; } @Override public List getDataSourceParamConfigs() { - return super.getDataSourceParamConfigs(DataSourceType.STARROCKS.name); + return super.getDataSourceParamConfigs(ExchangisDataSourceType.STARROCKS.name); } + + } From 8d0201fb84cb58415679d14a28fcf93a54ad7f6f Mon Sep 17 00:00:00 2001 From: davidhua Date: Mon, 20 May 2024 23:31:25 +0800 Subject: [PATCH 25/60] Enable to autofill columns (1/2); --- .../ExchangisDataSourceRenderRestfulApi.java | 7 ++- .../service/DataSourceRenderService.java | 3 +- .../impl/DefaultDataSourceRenderService.java | 10 ++-- ...AbstractLoggingSubExchangisJobHandler.java | 52 +++++++++++++++++++ 4 files changed, 66 insertions(+), 6 deletions(-) diff --git a/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/restful/api/ExchangisDataSourceRenderRestfulApi.java b/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/restful/api/ExchangisDataSourceRenderRestfulApi.java index 2e4bdd724..d83538d14 100644 --- a/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/restful/api/ExchangisDataSourceRenderRestfulApi.java +++ b/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/restful/api/ExchangisDataSourceRenderRestfulApi.java @@ -12,6 +12,7 @@ import javax.servlet.http.HttpServletRequest; import java.util.Locale; import java.util.Objects; +import java.util.Optional; /** * Expose the ui interface to front-end rendering @@ -29,6 +30,7 @@ public class ExchangisDataSourceRenderRestfulApi { public Message partition(@PathVariable("elementType") String type, @RequestParam("dataSourceId") Long dataSourceId, @RequestParam("database") String database, + @RequestParam(value = "tableNotExist", required = false) Boolean tableNotExist, @RequestParam("table") String table, HttpServletRequest request){ String userName = UserUtils.getLoginUser(request); ElementUI.Type uiType; @@ -39,8 +41,11 @@ public Message partition(@PathVariable("elementType") String type, } Message result = Message.ok(); try{ - ElementUI elementUI = renderService.getPartitionAndRender(userName, dataSourceId, database, table, uiType); + boolean notExist = Optional.ofNullable(tableNotExist).orElse(false); + ElementUI elementUI = renderService.getPartitionAndRender(userName, dataSourceId, + database, table, uiType, notExist); result.data("type", uiType.name()); + result.data("customize", notExist); if (Objects.nonNull(elementUI)){ result.data("render", elementUI.getValue()); } diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/DataSourceRenderService.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/DataSourceRenderService.java index 8e237fa48..5df843cdc 100644 --- a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/DataSourceRenderService.java +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/DataSourceRenderService.java @@ -11,6 +11,7 @@ public interface DataSourceRenderService { * @return element ui */ ElementUI getPartitionAndRender(String userName, - Long dataSourceId, String database, String table, ElementUI.Type uiType) throws ExchangisDataSourceException; + Long dataSourceId, String database, + String table, ElementUI.Type uiType, boolean tableNotExist) throws ExchangisDataSourceException; } diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/impl/DefaultDataSourceRenderService.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/impl/DefaultDataSourceRenderService.java index ff870752b..6452c8071 100644 --- a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/impl/DefaultDataSourceRenderService.java +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/service/impl/DefaultDataSourceRenderService.java @@ -36,11 +36,13 @@ public class DefaultDataSourceRenderService implements DataSourceRenderService { @Override public ElementUI getPartitionAndRender(String userName, - Long dataSourceId, String database, String table, ElementUI.Type uiType) throws ExchangisDataSourceException { - List partitionKeys = metadataInfoService.getPartitionKeys(userName, dataSourceId, database, table); + Long dataSourceId, String database, String table, ElementUI.Type uiType, boolean tableNotExist) throws ExchangisDataSourceException { Map renderParams = new LinkedHashMap<>(); - List placeHolder = Arrays.asList(DEFAULT_PLACEHOLDER); - partitionKeys.forEach(partition -> renderParams.putIfAbsent(partition, placeHolder)); + if (!tableNotExist) { + List partitionKeys = metadataInfoService.getPartitionKeys(userName, dataSourceId, database, table); + List placeHolder = Arrays.asList(DEFAULT_PLACEHOLDER); + partitionKeys.forEach(partition -> renderParams.putIfAbsent(partition, placeHolder)); + } return elementUIFactory.createElement(uiType.name(), renderParams, Map.class); } } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/AbstractLoggingSubExchangisJobHandler.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/AbstractLoggingSubExchangisJobHandler.java index 1d190cbaa..8bf4106ae 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/AbstractLoggingSubExchangisJobHandler.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/AbstractLoggingSubExchangisJobHandler.java @@ -7,8 +7,10 @@ import com.webank.wedatasphere.exchangis.job.domain.params.JobParamSet; import com.webank.wedatasphere.exchangis.job.exception.ExchangisJobException; import com.webank.wedatasphere.exchangis.job.server.builder.SpringExchangisJobBuilderContext; +import org.apache.commons.lang.StringUtils; import org.apache.linkis.common.exception.ErrorException; +import java.util.List; import java.util.Objects; import java.util.Optional; @@ -23,6 +25,7 @@ public abstract class AbstractLoggingSubExchangisJobHandler implements SubExchan public final void handleSource(SubExchangisJob subExchangisJob, ExchangisJobBuilderContext ctx) throws ErrorException { wrapFuncWithContext(ctx, () -> { try { + handleSrcColumns(subExchangisJob.getSourceColumns()); handleJobSource(subExchangisJob, ctx); }catch (ErrorException e){ throw new ExchangisJobException.Runtime(-1, "Exception in handling job source parameters", e); @@ -34,6 +37,7 @@ public final void handleSource(SubExchangisJob subExchangisJob, ExchangisJobBuil public final void handleSink(SubExchangisJob subExchangisJob, ExchangisJobBuilderContext ctx) throws ErrorException { wrapFuncWithContext(ctx, () -> { try { + handleSinkColumns(subExchangisJob.getSinkColumns()); handleJobSink(subExchangisJob, ctx); } catch (ErrorException e) { throw new ExchangisJobException.Runtime(-1, "Exception in handling job sink parameters", e); @@ -68,6 +72,42 @@ private void wrapFuncWithContext(ExchangisJobBuilderContext context, Runnable ru } } + /** + * If auto fill column + * @return bool + */ + protected boolean fillColumn(){ + return false; + } + + /** + * Handle source columns + * @param columns columns + */ + protected void handleSrcColumns(List columns) { + if (fillColumn()){ + boolean complete = columns.stream().noneMatch(column -> StringUtils.isBlank(column.getType()) || null == column.getIndex()); + if (!complete){ + doFillColumns(columns); + } + } + } + + /** + * Handle sink columns + * @param columns columns + */ + protected void handleSinkColumns(List columns){ + if (fillColumn()){ + boolean complete = columns.stream().noneMatch(column -> StringUtils.isBlank(column.getType())); + if (!complete){ + doFillColumns(columns); + } + } + } + + + /** * handle job source params * @param subExchangisJob sub exchangis job @@ -82,6 +122,14 @@ private void wrapFuncWithContext(ExchangisJobBuilderContext context, Runnable ru */ public abstract void handleJobSink(SubExchangisJob subExchangisJob, ExchangisJobBuilderContext ctx) throws ErrorException; + /** + * Do fill column + * @param columns columns + */ + protected void doFillColumns(List columns){ + + } + /** * Warn message * @param message message @@ -113,4 +161,8 @@ public static T getBean(Class clazz){ protected static SpringExchangisJobBuilderContext getJobBuilderContext(){ return springContext.get(); } + + private void completeColumns(){ + + } } From 518800e92da745338e4f199c9f00426be770facf Mon Sep 17 00:00:00 2001 From: jefftlin Date: Fri, 17 May 2024 16:01:04 +0800 Subject: [PATCH 26/60] Enhance start scripts and add job param config for starrocks --- assembly-package/sbin/daemon.sh | 2 +- assembly-package/sbin/env.properties | 2 ++ db/1.1.3/exchangis_dml.sql | 8 +++-- db/exchangis_dml.sql | 44 +++++++++++++++++----------- 4 files changed, 36 insertions(+), 20 deletions(-) diff --git a/assembly-package/sbin/daemon.sh b/assembly-package/sbin/daemon.sh index a21ccfab6..c7ee8f1e1 100644 --- a/assembly-package/sbin/daemon.sh +++ b/assembly-package/sbin/daemon.sh @@ -15,6 +15,7 @@ # limitations under the License. # +load_env_definitions ${ENV_FILE} if [[ "x"${EXCHANGIS_HOME} != "x" ]]; then source ${EXCHANGIS_HOME}/sbin/launcher.sh source ${EXCHANGIS_HOME}/sbin/common.sh @@ -48,7 +49,6 @@ restart(){ COMMAND=$1 case $COMMAND in start|stop|restart) - load_env_definitions ${ENV_FILE} if [[ ! -z $2 ]]; then SERVICE_NAME=${MODULE_DEFAULT_PREFIX}$2${MODULE_DEFAULT_SUFFIX} MAIN_CLASS=${MODULE_MAIN_CLASS[${SERVICE_NAME}]} diff --git a/assembly-package/sbin/env.properties b/assembly-package/sbin/env.properties index f849b4fa9..c6e528ab4 100644 --- a/assembly-package/sbin/env.properties +++ b/assembly-package/sbin/env.properties @@ -2,3 +2,5 @@ EXCHANGIS_CONF_PATH=/appcom/config/exchangis-config/background EXCHANGIS_LOG_PATH=/appcom/logs/exchangis/background MODULE_DEFAULT_PREFIX="dss-exchangis-main-" MODULE_DEFAULT_SUFFIX="-dev" +DEBUG_MODE=false +DEBUG_PORT=8321 \ No newline at end of file diff --git a/db/1.1.3/exchangis_dml.sql b/db/1.1.3/exchangis_dml.sql index e8ca76494..b1d3457da 100644 --- a/db/1.1.3/exchangis_dml.sql +++ b/db/1.1.3/exchangis_dml.sql @@ -1,3 +1,7 @@ -UPDATE exchangis_engine_settings SET engine_direction= -'mysql->hive,hive->mysql,mysql->oracle,oracle->mysql,oracle->hive,hive->oracle,mongodb->hive,hive->mongodb,mysql->elasticsearch,oracle->elasticsearch,mongodb->elasticsearch,mysql->mongodb,mongodb->mysql,oracle->mongodb,mongodb->oracle,mysql->starrocks' +INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status,ref_id) VALUES +,('writeMode','写入方式','DATAX-SINK','STARROCKS','OPTION','writeMode','写入方式','',1,'OPTION','["insert"]','insert','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) +,('batchSize','批量字节数大小','DATAX-SINK','STARROCKS','INPUT','maxBatchSize','批量字节数大小','',0,'NUMBER','','','REGEX','^[1-9]\\d*$','批量大小输入错误',0,0,'',1,'',2,'',1,NULL); + +UPDATE exchangis_engine_settings +SET engine_direction='mysql->hive,hive->mysql,mysql->oracle,oracle->mysql,oracle->hive,hive->oracle,mongodb->hive,hive->mongodb,mysql->elasticsearch,oracle->elasticsearch,mongodb->elasticsearch,mysql->mongodb,mongodb->mysql,oracle->mongodb,mongodb->oracle,mysql->starrocks' WHERE engine_name='datax'; \ No newline at end of file diff --git a/db/exchangis_dml.sql b/db/exchangis_dml.sql index 2376b9c53..70abb7ef1 100644 --- a/db/exchangis_dml.sql +++ b/db/exchangis_dml.sql @@ -38,29 +38,38 @@ INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_directio ,('setting.max.memory','作业最大使用内存','','DATAX','INPUT','setting.max.memory','作业最大使用内存','Mb',1,'NUMBER','','1024','REGEX','^[1-9]\\d*$','作业最大使用内存输入错误',0,0,'',1,'',4,'',1,NULL) ,('setting.errorLimit.record','最多错误记录数','','DATAX','INPUT','setting.errorlimit.record','最多错误记录数','条',0,'NUMBER','','','REGEX','^[0-9]\\d*$','最多错误记录数输入错误',0,0,'',1,'',5,'',1,NULL) ,('setting.max.parallelism','作业最大并行数','','SQOOP','INPUT','setting.max.parallelism','作业最大并行数','个',1,'NUMBER','','1','REGEX','^[1-9]\\d*$','作业最大并行数输入错误',0,0,'',1,'',1,'',1,NULL) -,('setting.max.memory','作业最大内存','','SQOOP','INPUT','setting.max.memory','作业最大内存','Mb',1,'NUMBER','','1024','REGEX','^[1-9]\\d*$','作业最大内存输入错误',0,0,'',1,'',2,'',1,NULL) -,('where','WHERE条件','SOURCE','MYSQL','INPUT','where','WHERE条件','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,500}$','WHERE条件输入过长',0,0,'',1,'',2,'',1,NULL) -,('writeMode','写入方式','SQOOP-SINK','HIVE','OPTION','writeMode','写入方式(OVERWRITE只对TEXT类型表生效)','',1,'OPTION','["OVERWRITE","APPEND"]','OVERWRITE','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) -,('partition','分区信息','SINK','HIVE','MAP','partition','分区信息(文本)','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','分区信息过长',0,0,'/api/rest_j/v1/dss/exchangis/main/datasources/render/partition/element/map',1,'',2,'',1,NULL) -; +,('setting.max.memory','作业最大内存','','SQOOP','INPUT','setting.max.memory','作业最大内存','Mb',1,'NUMBER','','1024','REGEX','^[1-9]\\d*$','作业最大内存输入错误',0,0,'',1,'',2,'',1,NULL); + INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status,ref_id) VALUES -('partition','分区信息','SOURCE','HIVE','MAP','partition','分区信息(文本)','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','分区信息过长',0,0,'/api/rest_j/v1/dss/exchangis/main/datasources/render/partition/element/map',1,'',2,'',1,NULL) +('where','WHERE条件','SOURCE','MYSQL','INPUT','where','WHERE条件','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,500}$','WHERE条件输入过长',0,0,'',1,'',2,'',1,NULL); ,('writeMode','写入方式','SQOOP-SINK','MYSQL','OPTION','writeMode','写入方式','',1,'OPTION','["INSERT","UPDATE"]','INSERT','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) -,('batchSize','批量大小','DATAX-SINK','ELASTICSEARCH','INPUT','batchSize','批量大小','',0,'NUMBER','','','REGEX','^[1-9]\\d*$','批量大小输入错误',0,0,'',1,'',1,'',1,NULL) -,('query','query条件','DATAX-SOURCE','MONGODB','INPUT','query','query条件','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,500}$','query条件输入过长',0,0,'',1,'',2,'',1,NULL) -,('writeMode','写入方式','DATAX-SINK','MONGODB','OPTION','writeMode','写入方式','',1,'OPTION','["insert","replace"]','insert','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) -,('batchSize','批量大小','DATAX-SINK','MONGODB','INPUT','batchSize','批量大小','',0,'NUMBER','','','REGEX','^[1-9]\\d*$','批量大小输入错误',0,0,'',1,'',2,'',1,NULL) +,('writeMode','写入方式','DATAX-SINK','MYSQL','OPTION','writeMode','写入方式','',1,'OPTION','["INSERT","UPDATE"]','INSERT','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL); + +INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status,ref_id) VALUES +('writeMode','写入方式','SQOOP-SINK','HIVE','OPTION','writeMode','写入方式(OVERWRITE只对TEXT类型表生效)','',1,'OPTION','["OVERWRITE","APPEND"]','OVERWRITE','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) +,('partition','分区信息','SINK','HIVE','MAP','partition','分区信息(文本)','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','分区信息过长',0,0,'/api/rest_j/v1/dss/exchangis/main/datasources/render/partition/element/map',1,'',2,'',1,NULL) +,('partition','分区信息','SOURCE','HIVE','MAP','partition','分区信息(文本)','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','分区信息过长',0,0,'/api/rest_j/v1/dss/exchangis/main/datasources/render/partition/element/map',1,'',2,'',1,NULL); ,('transferMode','传输方式','DATAX-SOURCE','HIVE','OPTION','transferMode','传输方式','',1,'OPTION','["二进制","记录"]','二进制','','','该传输方式不可用',0,0,'',1,'',1,'',1,NULL) ,('nullFormat','空值字符','DATAX-SOURCE','HIVE','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,49) -,('writeMode','写入方式','DATAX-SINK','MYSQL','OPTION','writeMode','写入方式','',1,'OPTION','["INSERT","UPDATE"]','INSERT','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) ,('writeMode','写入方式','DATAX-SINK','HIVE','OPTION','writeMode','写入方式(OVERWRITE只对TEXT类型表生效)','',1,'OPTION','["append","truncate"]','append','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) -; +,('nullFormat','空值字符','DATAX-SINK','HIVE','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,49); + INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status,ref_id) VALUES -('nullFormat','空值字符','DATAX-SINK','HIVE','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,49) -,('nullFormat','空值字符','DATAX-SINK','ELASTICSEARCH','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,49) -,('where','WHERE条件','SOURCE','ORACLE','INPUT','where','WHERE条件',NULL,0,'VARCHAR',NULL,NULL,'REGEX','^[\\s\\S]{0,500}$','WHERE条件输入过长',0,0,NULL,1,'',2,NULL,1,NULL) -,('writeMode','写入方式','DATAX-SINK','ORACLE','OPTION','writeMode','写入方式',NULL,1,'OPTION','["INSERT","UPDATE"]','INSERT',NULL,NULL,'写入方式输入错误',0,0,NULL,1,NULL,1,NULL,1,NULL) -; +('batchSize','批量大小','DATAX-SINK','ELASTICSEARCH','INPUT','batchSize','批量大小','',0,'NUMBER','','','REGEX','^[1-9]\\d*$','批量大小输入错误',0,0,'',1,'',1,'',1,NULL) +,('nullFormat','空值字符','DATAX-SINK','ELASTICSEARCH','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,49); + +INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status,ref_id) VALUES +('where','WHERE条件','SOURCE','ORACLE','INPUT','where','WHERE条件',NULL,0,'VARCHAR',NULL,NULL,'REGEX','^[\\s\\S]{0,500}$','WHERE条件输入过长',0,0,NULL,1,'',2,NULL,1,NULL) +,('writeMode','写入方式','DATAX-SINK','ORACLE','OPTION','writeMode','写入方式',NULL,1,'OPTION','["INSERT","UPDATE"]','INSERT',NULL,NULL,'写入方式输入错误',0,0,NULL,1,NULL,1,NULL,1,NULL); + +INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status,ref_id) VALUES +('query','query条件','DATAX-SOURCE','MONGODB','INPUT','query','query条件','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,500}$','query条件输入过长',0,0,'',1,'',2,'',1,NULL) +,('writeMode','写入方式','DATAX-SINK','MONGODB','OPTION','writeMode','写入方式','',1,'OPTION','["insert","replace"]','insert','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) +,('batchSize','批量大小','DATAX-SINK','MONGODB','INPUT','batchSize','批量大小','',0,'NUMBER','','','REGEX','^[1-9]\\d*$','批量大小输入错误',0,0,'',1,'',2,'',1,NULL); + +INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status,ref_id) VALUES +('writeMode','写入方式','DATAX-SINK','STARROCKS','OPTION','writeMode','写入方式','',1,'OPTION','["insert"]','insert','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) +,('batchSize','批量字节数大小','DATAX-SINK','STARROCKS','INPUT','maxBatchSize','批量字节数大小','',0,'NUMBER','','','REGEX','^[1-9]\\d*$','批量大小输入错误',0,0,'',1,'',2,'',1,NULL); -- engine_settings records INSERT INTO `exchangis_engine_settings` (id, engine_name, engine_desc, engine_settings_value, engine_direction, res_loader_class, res_uploader_class, modify_time) VALUES @@ -74,5 +83,6 @@ INSERT INTO `exchangis_job_transform_rule` (rule_name,rule_type,rule_source,data ,('hive_sink_not_access','MAPPING','{"fieldEditEnable": false, "fieldDeleteEnable": false, "fieldAddEnable": false}','HIVE',NULL,'SINK') ,('mongo_field_match','MAPPING','{"fieldMatchStrategyName": "CAMEL_CASE_MATCH"}','MONGODB',NULL,'SINK') ,('mysql_field_source_match','MAPPING','{"fieldMatchStrategyName": "CAMEL_CASE_MATCH","fieldEditEnable": true, "fieldDeleteEnable": true, "fieldAddEnable": false}','MYSQL',NULL,'SOURCE') +,('mysql_field_source_match','MAPPING','{"fieldMatchStrategyName": "CAMEL_CASE_MATCH","fieldEditEnable": true, "fieldDeleteEnable": true, "fieldAddEnable": true}','STARROCKS',NULL,'SINK') ; From 241264a67ee9b4c00b2c19b5c26f0c7a7a114f10 Mon Sep 17 00:00:00 2001 From: davidhua Date: Tue, 21 May 2024 14:04:40 +0800 Subject: [PATCH 27/60] Enable to autofill columns (2/2); --- .../exchangis/job/domain/SubExchangisJob.java | 50 ++++++++ .../job/utils/ColumnDefineUtils.java | 40 +++++++ ...AbstractLoggingSubExchangisJobHandler.java | 42 ++----- .../AutoColumnSubExchangisJobHandler.java | 112 ++++++++++++++++++ 4 files changed, 210 insertions(+), 34 deletions(-) create mode 100644 exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/utils/ColumnDefineUtils.java create mode 100644 exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/AutoColumnSubExchangisJobHandler.java diff --git a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/SubExchangisJob.java b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/SubExchangisJob.java index 1679e38ff..40f1e8576 100644 --- a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/SubExchangisJob.java +++ b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/SubExchangisJob.java @@ -144,6 +144,11 @@ public ColumnDefine(){ } + public ColumnDefine(String name, String type){ + this.name = name; + this.type = type; + } + public ColumnDefine(String name, String type, Integer index){ this.name = name; this.type = type; @@ -174,6 +179,51 @@ public void setIndex(Integer index) { } } + /** + * Column define with precision and scale + */ + public static class DecimalColumnDefine extends ColumnDefine{ + + private static final int DEFAULT_PRECISION = 38; + + private static final int DEFAULT_SCALE = 18; + + /** + * Precision + */ + private int precision = DEFAULT_PRECISION; + + /** + * Scale + */ + private int scale = DEFAULT_SCALE; + + public DecimalColumnDefine(){ + + } + + public DecimalColumnDefine(String name, String type, Integer index, int precision, int scale){ + super(name, type, index); + this.precision = precision; + this.scale = scale; + } + + public int getPrecision() { + return precision; + } + + public void setPrecision(int precision) { + this.precision = precision; + } + + public int getScale() { + return scale; + } + + public void setScale(int scale) { + this.scale = scale; + } + } /** * Column function */ diff --git a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/utils/ColumnDefineUtils.java b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/utils/ColumnDefineUtils.java new file mode 100644 index 000000000..6c440f320 --- /dev/null +++ b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/utils/ColumnDefineUtils.java @@ -0,0 +1,40 @@ +package com.webank.wedatasphere.exchangis.job.utils; + +import com.webank.wedatasphere.exchangis.job.domain.SubExchangisJob; +import org.apache.commons.lang3.StringUtils; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Utils to data column + */ +public class ColumnDefineUtils { + /** + * Pattern of decimal column + */ + public static final Pattern DECIMAL_PATTERN = Pattern.compile("^decimal[(（](\\d+)[^,]*?,[^,]*?(\\d+)[)）]$"); + + /** + * Get data column + * @param name column name + * @param type column type + * @param index index + * @return data column + */ + public static SubExchangisJob.ColumnDefine getColumn(String name, String type, Integer index){ + if (StringUtils.isNotBlank(type)) { + Matcher decimalMatch = DECIMAL_PATTERN.matcher(type.toLowerCase()); + if (decimalMatch.matches()) { + int precision = Integer.parseInt(decimalMatch.group(1)); + int scale = Integer.parseInt(decimalMatch.group(2)); + return new SubExchangisJob.DecimalColumnDefine(name, type, index, precision, scale); + } + } + return new SubExchangisJob.ColumnDefine(name, type, index); + } + + public static SubExchangisJob.ColumnDefine getColumn(String name, String type){ + return getColumn(name, type, null); + } +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/AbstractLoggingSubExchangisJobHandler.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/AbstractLoggingSubExchangisJobHandler.java index 8bf4106ae..6b27d74e5 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/AbstractLoggingSubExchangisJobHandler.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/AbstractLoggingSubExchangisJobHandler.java @@ -7,7 +7,6 @@ import com.webank.wedatasphere.exchangis.job.domain.params.JobParamSet; import com.webank.wedatasphere.exchangis.job.exception.ExchangisJobException; import com.webank.wedatasphere.exchangis.job.server.builder.SpringExchangisJobBuilderContext; -import org.apache.commons.lang.StringUtils; import org.apache.linkis.common.exception.ErrorException; import java.util.List; @@ -25,7 +24,7 @@ public abstract class AbstractLoggingSubExchangisJobHandler implements SubExchan public final void handleSource(SubExchangisJob subExchangisJob, ExchangisJobBuilderContext ctx) throws ErrorException { wrapFuncWithContext(ctx, () -> { try { - handleSrcColumns(subExchangisJob.getSourceColumns()); + handleSrcColumns(subExchangisJob, ctx, subExchangisJob.getSourceColumns()); handleJobSource(subExchangisJob, ctx); }catch (ErrorException e){ throw new ExchangisJobException.Runtime(-1, "Exception in handling job source parameters", e); @@ -37,7 +36,7 @@ public final void handleSource(SubExchangisJob subExchangisJob, ExchangisJobBuil public final void handleSink(SubExchangisJob subExchangisJob, ExchangisJobBuilderContext ctx) throws ErrorException { wrapFuncWithContext(ctx, () -> { try { - handleSinkColumns(subExchangisJob.getSinkColumns()); + handleSinkColumns(subExchangisJob, ctx, subExchangisJob.getSinkColumns()); handleJobSink(subExchangisJob, ctx); } catch (ErrorException e) { throw new ExchangisJobException.Runtime(-1, "Exception in handling job sink parameters", e); @@ -72,38 +71,23 @@ private void wrapFuncWithContext(ExchangisJobBuilderContext context, Runnable ru } } - /** - * If auto fill column - * @return bool - */ - protected boolean fillColumn(){ - return false; - } /** * Handle source columns * @param columns columns */ - protected void handleSrcColumns(List columns) { - if (fillColumn()){ - boolean complete = columns.stream().noneMatch(column -> StringUtils.isBlank(column.getType()) || null == column.getIndex()); - if (!complete){ - doFillColumns(columns); - } - } + protected void handleSrcColumns(SubExchangisJob subExchangisJob, ExchangisJobBuilderContext ctx, + List columns) { + // Empty } /** * Handle sink columns * @param columns columns */ - protected void handleSinkColumns(List columns){ - if (fillColumn()){ - boolean complete = columns.stream().noneMatch(column -> StringUtils.isBlank(column.getType())); - if (!complete){ - doFillColumns(columns); - } - } + protected void handleSinkColumns(SubExchangisJob subExchangisJob, ExchangisJobBuilderContext ctx, + List columns){ + // Empty } @@ -122,13 +106,6 @@ protected void handleSinkColumns(List columns){ */ public abstract void handleJobSink(SubExchangisJob subExchangisJob, ExchangisJobBuilderContext ctx) throws ErrorException; - /** - * Do fill column - * @param columns columns - */ - protected void doFillColumns(List columns){ - - } /** * Warn message @@ -162,7 +139,4 @@ protected static SpringExchangisJobBuilderContext getJobBuilderContext(){ return springContext.get(); } - private void completeColumns(){ - - } } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/AutoColumnSubExchangisJobHandler.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/AutoColumnSubExchangisJobHandler.java new file mode 100644 index 000000000..449ed1b2b --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/AutoColumnSubExchangisJobHandler.java @@ -0,0 +1,112 @@ +package com.webank.wedatasphere.exchangis.job.server.builder.transform.handlers; + +import com.webank.wedatasphere.exchangis.datasource.core.domain.MetaColumn; +import com.webank.wedatasphere.exchangis.job.builder.ExchangisJobBuilderContext; +import com.webank.wedatasphere.exchangis.job.domain.SubExchangisJob; +import com.webank.wedatasphere.exchangis.job.domain.params.JobParamSet; +import com.webank.wedatasphere.exchangis.job.exception.ExchangisJobException; +import com.webank.wedatasphere.exchangis.job.utils.ColumnDefineUtils; +import org.apache.commons.lang.StringUtils; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * Provide method to autofill columns + */ +public abstract class AutoColumnSubExchangisJobHandler extends AbstractLoggingSubExchangisJobHandler{ + /** + * Auto type name + */ + private static final String AUTO_TYPE = "[Auto]"; + + /** + * Handle source columns + * @param columns columns + */ + protected void handleSrcColumns(SubExchangisJob subExchangisJob, ExchangisJobBuilderContext ctx, + List columns) { + if (autoColumn()){ + boolean complete = columns.stream().noneMatch(column -> StringUtils.isBlank(column.getType()) || column.getType().equals(AUTO_TYPE) || null == column.getIndex()); + if (!complete){ + JobParamSet paramSet = subExchangisJob.getRealmParams(SubExchangisJob.REALM_JOB_CONTENT_SOURCE); + doFillColumns(paramSet, columns); + } + } + } + + /** + * Handle sink columns + * @param columns columns + */ + protected void handleSinkColumns(SubExchangisJob subExchangisJob, ExchangisJobBuilderContext ctx, + List columns){ + if (autoColumn()){ + boolean complete = columns.stream().noneMatch(column -> StringUtils.isBlank(column.getType()) || column.getType().equals(AUTO_TYPE)); + if (!complete){ + JobParamSet paramSet = subExchangisJob.getRealmParams(SubExchangisJob.REALM_JOB_CONTENT_SINK); + doFillColumns(paramSet, columns); + } + } + } + + /** + * If auto fill column + * @return bool + */ + protected boolean autoColumn(){ + return true; + } + + /** + * Do fill column + * @param columns columns + */ + protected void doFillColumns(JobParamSet paramSet, List columns){ + List metaColumns = getMetaColumns(paramSet); + if (Objects.nonNull(metaColumns) && !metaColumns.isEmpty()){ + if (columns.size() <= 0){ + for(MetaColumn metaColumn : metaColumns){ + SubExchangisJob.ColumnDefine columnDefine = ColumnDefineUtils + .getColumn(metaColumn.getName(), metaColumn.getType()); + columnDefine.setIndex(metaColumn.getIndex()); + columns.add(columnDefine); + } + } else { + completeColumns(columns, metaColumns); + } + } + } + + /** + * Get columns for metadata server + * @param paramSet param set + * @return + */ + protected List getMetaColumns(JobParamSet paramSet){ + return Collections.emptyList(); + } + /** + * + * @param columns columns + * @param metaColumns meta columns + */ + protected final void completeColumns(List columns, List metaColumns){ + Map metaColumnMap = metaColumns.stream().collect(Collectors.toMap( + MetaColumn::getName, metaColumn -> metaColumn, (left, right) -> left + )); + for (int i = 0; i < columns.size(); i ++){ + SubExchangisJob.ColumnDefine column = columns.get(i); + String name = column.getName(); + MetaColumn metaColumn = metaColumnMap.get(name); + if (Objects.isNull(metaColumn)){ + throw new ExchangisJobException.Runtime(-1, "Unable to find match column: [" + name + "] (表中找不到对应的字段)", null); + } + columns.set(i, ColumnDefineUtils.getColumn(name, metaColumn.getType(), metaColumn.getIndex())); + } + } + +} From 977013b29ad0c8b6f30165c334abcc4d8af930f0 Mon Sep 17 00:00:00 2001 From: davidhua Date: Tue, 21 May 2024 14:42:40 +0800 Subject: [PATCH 28/60] Fix the problem in rpc job log service. --- .../exchangis/job/server/log/service/RpcJobLogService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/RpcJobLogService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/RpcJobLogService.java index 46853ceb7..72e0f5267 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/RpcJobLogService.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/RpcJobLogService.java @@ -93,9 +93,10 @@ public void flushCache(boolean isEnd) { } }; } + logPath = logPath.substring(splitPos + 1); } File logFile = new File(Constraints.LOG_LOCAL_PATH.getValue() + IOUtils.DIR_SEPARATOR_UNIX + - launchedExchangisJob.getLogPath()); + logPath); if (!logFile.exists()){ // Write empty string to create new file From f51ce259fa3c656ff3b3b11cc938a752b5b95b1d Mon Sep 17 00:00:00 2001 From: davidhua Date: Tue, 21 May 2024 16:02:56 +0800 Subject: [PATCH 29/60] Add 'ES', 'Hive', 'Mongo', 'MySQL', 'Oracle' auto column handler. --- .../AutoColumnSubExchangisJobHandler.java | 56 +++++++++++++++---- .../column/EsAutoColumnJobHandler.java | 21 +++++++ .../column/HiveAutoColumnJobHandler.java | 26 +++++++++ .../column/MongoAutoColumnJobHandler.java | 22 ++++++++ .../column/MySQLAutoColumnJobHandler.java | 22 ++++++++ .../column/OracleAutoColumnJobHandler.java | 21 +++++++ .../mapping/FieldMappingTransformer.java | 44 ++++++++------- 7 files changed, 181 insertions(+), 31 deletions(-) rename exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/{ => column}/AutoColumnSubExchangisJobHandler.java (68%) create mode 100644 exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/column/EsAutoColumnJobHandler.java create mode 100644 exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/column/HiveAutoColumnJobHandler.java create mode 100644 exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/column/MongoAutoColumnJobHandler.java create mode 100644 exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/column/MySQLAutoColumnJobHandler.java create mode 100644 exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/column/OracleAutoColumnJobHandler.java diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/AutoColumnSubExchangisJobHandler.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/column/AutoColumnSubExchangisJobHandler.java similarity index 68% rename from exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/AutoColumnSubExchangisJobHandler.java rename to exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/column/AutoColumnSubExchangisJobHandler.java index 449ed1b2b..acee252a2 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/AutoColumnSubExchangisJobHandler.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/column/AutoColumnSubExchangisJobHandler.java @@ -1,12 +1,20 @@ -package com.webank.wedatasphere.exchangis.job.server.builder.transform.handlers; +package com.webank.wedatasphere.exchangis.job.server.builder.transform.handlers.column; import com.webank.wedatasphere.exchangis.datasource.core.domain.MetaColumn; +import com.webank.wedatasphere.exchangis.datasource.core.exception.ExchangisDataSourceException; +import com.webank.wedatasphere.exchangis.datasource.core.service.MetadataInfoService; import com.webank.wedatasphere.exchangis.job.builder.ExchangisJobBuilderContext; import com.webank.wedatasphere.exchangis.job.domain.SubExchangisJob; +import com.webank.wedatasphere.exchangis.job.domain.params.JobParam; +import com.webank.wedatasphere.exchangis.job.domain.params.JobParamDefine; import com.webank.wedatasphere.exchangis.job.domain.params.JobParamSet; +import com.webank.wedatasphere.exchangis.job.domain.params.JobParams; import com.webank.wedatasphere.exchangis.job.exception.ExchangisJobException; +import com.webank.wedatasphere.exchangis.job.server.builder.JobParamConstraints; +import com.webank.wedatasphere.exchangis.job.server.builder.transform.handlers.AbstractLoggingSubExchangisJobHandler; import com.webank.wedatasphere.exchangis.job.utils.ColumnDefineUtils; import org.apache.commons.lang.StringUtils; +import org.apache.linkis.common.exception.ErrorException; import java.util.Collections; import java.util.List; @@ -17,12 +25,32 @@ /** * Provide method to autofill columns */ -public abstract class AutoColumnSubExchangisJobHandler extends AbstractLoggingSubExchangisJobHandler{ +public abstract class AutoColumnSubExchangisJobHandler extends AbstractLoggingSubExchangisJobHandler { /** * Auto type name */ private static final String AUTO_TYPE = "[Auto]"; + /** + * Database + */ + private static final JobParamDefine DATABASE = JobParams.define(JobParamConstraints.DATABASE); + + /** + * Table + */ + private static final JobParamDefine TABLE = JobParams.define(JobParamConstraints.TABLE); + + @Override + public void handleJobSource(SubExchangisJob subExchangisJob, ExchangisJobBuilderContext ctx) throws ErrorException { + // Ignore + } + + @Override + public void handleJobSink(SubExchangisJob subExchangisJob, ExchangisJobBuilderContext ctx) throws ErrorException { + // Ignore + } + /** * Handle source columns * @param columns columns @@ -53,13 +81,6 @@ protected void handleSinkColumns(SubExchangisJob subExchangisJob, ExchangisJobBu } } - /** - * If auto fill column - * @return bool - */ - protected boolean autoColumn(){ - return true; - } /** * Do fill column @@ -84,10 +105,18 @@ protected void doFillColumns(JobParamSet paramSet, List getMetaColumns(JobParamSet paramSet){ - return Collections.emptyList(); + String database = DATABASE.getValue(paramSet); + String table = TABLE.getValue(paramSet); + JobParam dataSourceId = paramSet.get(JobParamConstraints.DATA_SOURCE_ID); + try { + return Objects.requireNonNull(getBean(MetadataInfoService.class)).getColumns(getJobBuilderContext().getOriginalJob().getCreateUser(), + Long.valueOf(dataSourceId.getValue()), database, table); + } catch (ExchangisDataSourceException e) { + throw new ExchangisJobException.Runtime(e.getErrCode(), e.getMessage(), e.getCause()); + } } /** * @@ -109,4 +138,9 @@ protected final void completeColumns(List columns, } } + /** + * If auto fill column + * @return bool + */ + protected abstract boolean autoColumn(); } diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/column/EsAutoColumnJobHandler.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/column/EsAutoColumnJobHandler.java new file mode 100644 index 000000000..dafab1ead --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/column/EsAutoColumnJobHandler.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.exchangis.job.server.builder.transform.handlers.column; + +/** + * ES auto column handler + */ +public class EsAutoColumnJobHandler extends AutoColumnSubExchangisJobHandler{ + @Override + public String dataSourceType() { + return "elasticsearch"; + } + + @Override + public int order() { + return 0; + } + + @Override + protected boolean autoColumn() { + return true; + } +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/column/HiveAutoColumnJobHandler.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/column/HiveAutoColumnJobHandler.java new file mode 100644 index 000000000..2901fb07d --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/column/HiveAutoColumnJobHandler.java @@ -0,0 +1,26 @@ +package com.webank.wedatasphere.exchangis.job.server.builder.transform.handlers.column; + + + +/** + * Hive auto column handler + */ +public class HiveAutoColumnJobHandler extends AutoColumnSubExchangisJobHandler { + + + @Override + public String dataSourceType() { + return "hive"; + } + + @Override + public int order() { + return 0; + } + + @Override + protected boolean autoColumn() { + return true; + } + +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/column/MongoAutoColumnJobHandler.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/column/MongoAutoColumnJobHandler.java new file mode 100644 index 000000000..c3abba033 --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/column/MongoAutoColumnJobHandler.java @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.exchangis.job.server.builder.transform.handlers.column; + +/** + * Mongo auto column handler + */ +public class MongoAutoColumnJobHandler extends AutoColumnSubExchangisJobHandler{ + + @Override + public String dataSourceType() { + return "mongodb"; + } + + @Override + public int order() { + return 0; + } + + @Override + protected boolean autoColumn() { + return true; + } +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/column/MySQLAutoColumnJobHandler.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/column/MySQLAutoColumnJobHandler.java new file mode 100644 index 000000000..20ce2de16 --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/column/MySQLAutoColumnJobHandler.java @@ -0,0 +1,22 @@ +package com.webank.wedatasphere.exchangis.job.server.builder.transform.handlers.column; + +/** + * Mysql auto column handler + */ +public class MySQLAutoColumnJobHandler extends AutoColumnSubExchangisJobHandler { + @Override + protected boolean autoColumn() { + return false; + } + + @Override + public String dataSourceType() { + return "mysql"; + } + + + @Override + public int order() { + return 0; + } +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/column/OracleAutoColumnJobHandler.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/column/OracleAutoColumnJobHandler.java new file mode 100644 index 000000000..22866c48e --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/column/OracleAutoColumnJobHandler.java @@ -0,0 +1,21 @@ +package com.webank.wedatasphere.exchangis.job.server.builder.transform.handlers.column; + +/** + * Oracle auto column handler + */ +public class OracleAutoColumnJobHandler extends AutoColumnSubExchangisJobHandler{ + @Override + public String dataSourceType() { + return "oracle"; + } + + @Override + public int order() { + return 0; + } + + @Override + protected boolean autoColumn() { + return true; + } +} diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/render/transform/field/mapping/FieldMappingTransformer.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/render/transform/field/mapping/FieldMappingTransformer.java index 6cbbc64a9..19454a4f2 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/render/transform/field/mapping/FieldMappingTransformer.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/render/transform/field/mapping/FieldMappingTransformer.java @@ -84,31 +84,35 @@ private FieldMappingSettings getFieldMappingSettings(FieldMappingRule rule, Tran settings.setTransformEnable(rule.isFieldTransformEnable()); // Get raw meta columns List sourceColumns = new ArrayList<>(); - try { - List metaColumns = getOrLoadMetadataInfoService(). - getColumns(requestVo.getOperator(), requestVo.getSourceDataSourceId(), - requestVo.getSourceDataBase(), requestVo.getSourceTable()); - boolean editable = rule.getFieldEditEnableRuleItem().getOrDefault(TransformRule.Direction.SOURCE.name(), true); - for (int i = 0; i < metaColumns.size(); i++) { - MetaColumn metaColumn = metaColumns.get(i); - sourceColumns.add(new FieldColumnWrapper(metaColumn.getName(), metaColumn.getType(), i, editable)); + if (!requestVo.isSrcTblNotExist()) { + try { + List metaColumns = getOrLoadMetadataInfoService(). + getColumns(requestVo.getOperator(), requestVo.getSourceDataSourceId(), + requestVo.getSourceDataBase(), requestVo.getSourceTable()); + boolean editable = rule.getFieldEditEnableRuleItem().getOrDefault(TransformRule.Direction.SOURCE.name(), true); + for (int i = 0; i < metaColumns.size(); i++) { + MetaColumn metaColumn = metaColumns.get(i); + sourceColumns.add(new FieldColumnWrapper(metaColumn.getName(), metaColumn.getType(), i, editable)); + } + } catch (ExchangisDataSourceException e) { + throw new ExchangisJobException.Runtime(ExchangisJobExceptionCode.RENDER_TRANSFORM_ERROR.getCode(), "Fail to get source meta columns in generating field mapping settings", e); } - } catch (ExchangisDataSourceException e) { - throw new ExchangisJobException.Runtime(ExchangisJobExceptionCode.RENDER_TRANSFORM_ERROR.getCode(), "Fail to get source meta columns in generating field mapping settings", e); } settings.setSourceFields(sourceColumns); List sinkColumns = new ArrayList<>(); - try { - List metaColumns = getOrLoadMetadataInfoService(). - getColumns(requestVo.getOperator(), requestVo.getSinkDataSourceId(), - requestVo.getSinkDataBase(), requestVo.getSinkTable()); - boolean editable = rule.getFieldEditEnableRuleItem().getOrDefault(TransformRule.Direction.SINK.name(), true); - for (int i = 0; i < metaColumns.size(); i++) { - MetaColumn metaColumn = metaColumns.get(i); - sinkColumns.add(new FieldColumnWrapper(metaColumn.getName(), metaColumn.getType(), i, editable)); + if (!requestVo.isSinkTblNotExist()) { + try { + List metaColumns = getOrLoadMetadataInfoService(). + getColumns(requestVo.getOperator(), requestVo.getSinkDataSourceId(), + requestVo.getSinkDataBase(), requestVo.getSinkTable()); + boolean editable = rule.getFieldEditEnableRuleItem().getOrDefault(TransformRule.Direction.SINK.name(), true); + for (int i = 0; i < metaColumns.size(); i++) { + MetaColumn metaColumn = metaColumns.get(i); + sinkColumns.add(new FieldColumnWrapper(metaColumn.getName(), metaColumn.getType(), i, editable)); + } + } catch (ExchangisDataSourceException e) { + throw new ExchangisJobException.Runtime(ExchangisJobExceptionCode.RENDER_TRANSFORM_ERROR.getCode(), "Fail to get sink meta columns in generating field mapping settings", e); } - } catch (ExchangisDataSourceException e) { - throw new ExchangisJobException.Runtime(ExchangisJobExceptionCode.RENDER_TRANSFORM_ERROR.getCode(), "Fail to get sink meta columns in generating field mapping settings", e); } settings.setSinkFields(sinkColumns); FieldMatchStrategy matchStrategy = rule.getFieldMatchStrategy(); From 9ef2965de406afa09d67b4a754a1dbbd52907331 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Tue, 21 May 2024 17:42:26 +0800 Subject: [PATCH 30/60] Rename BOOL to BOOLEAN in Column.java --- .../java/com/alibaba/datax/common/element/BoolColumn.java | 4 ++-- .../main/java/com/alibaba/datax/common/element/Column.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/exchangis-engines/engines/datax/datax-core/src/main/java/com/alibaba/datax/common/element/BoolColumn.java b/exchangis-engines/engines/datax/datax-core/src/main/java/com/alibaba/datax/common/element/BoolColumn.java index ab4ba98fe..ee5fca453 100644 --- a/exchangis-engines/engines/datax/datax-core/src/main/java/com/alibaba/datax/common/element/BoolColumn.java +++ b/exchangis-engines/engines/datax/datax-core/src/main/java/com/alibaba/datax/common/element/BoolColumn.java @@ -13,7 +13,7 @@ public class BoolColumn extends Column { public BoolColumn(Boolean bool) { - super(bool, Column.Type.BOOL, 1); + super(bool, Column.Type.BOOLEAN, 1); } public BoolColumn(final String data) { @@ -30,7 +30,7 @@ public BoolColumn(final String data) { } public BoolColumn() { - super(null, Column.Type.BOOL, 1); + super(null, Column.Type.BOOLEAN, 1); } @Override diff --git a/exchangis-engines/engines/datax/datax-core/src/main/java/com/alibaba/datax/common/element/Column.java b/exchangis-engines/engines/datax/datax-core/src/main/java/com/alibaba/datax/common/element/Column.java index c5a121bd3..829fd520c 100644 --- a/exchangis-engines/engines/datax/datax-core/src/main/java/com/alibaba/datax/common/element/Column.java +++ b/exchangis-engines/engines/datax/datax-core/src/main/java/com/alibaba/datax/common/element/Column.java @@ -71,6 +71,6 @@ public String toString() { } public enum Type { - BAD, NULL, INT, LONG, DOUBLE, STRING, BOOL, DATE, BYTES + BAD, NULL, INT, LONG, DOUBLE, STRING, BOOLEAN, DATE, BYTES } } From fba043d1d218908d15ef91aef2b1ba00f5befb20 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Tue, 21 May 2024 17:42:49 +0800 Subject: [PATCH 31/60] Enable to edit fields --- db/exchangis_dml.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/db/exchangis_dml.sql b/db/exchangis_dml.sql index 70abb7ef1..0c90fa450 100644 --- a/db/exchangis_dml.sql +++ b/db/exchangis_dml.sql @@ -79,10 +79,10 @@ INSERT INTO `exchangis_engine_settings` (id, engine_name, engine_desc, engine_se -- exchangis_job_transform_rule records INSERT INTO `exchangis_job_transform_rule` (rule_name,rule_type,rule_source,data_source_type,engine_type,direction) VALUES ('es_with_post_processor','DEF','{"types": ["MAPPING", "PROCESSOR"]}','ELASTICSEARCH',NULL,'SINK') -,('es_fields_not_editable','MAPPING','{"fieldEditEnable": false, "fieldDeleteEnable": false}','ELASTICSEARCH',NULL,'SINK') -,('hive_sink_not_access','MAPPING','{"fieldEditEnable": false, "fieldDeleteEnable": false, "fieldAddEnable": false}','HIVE',NULL,'SINK') +,('es_fields_not_editable','MAPPING','{"fieldEditEnable": true, "fieldDeleteEnable": true}','ELASTICSEARCH',NULL,'SINK') +,('hive_sink_not_access','MAPPING','{"fieldEditEnable": true, "fieldDeleteEnable": true, "fieldAddEnable": true}','HIVE',NULL,'SINK') ,('mongo_field_match','MAPPING','{"fieldMatchStrategyName": "CAMEL_CASE_MATCH"}','MONGODB',NULL,'SINK') -,('mysql_field_source_match','MAPPING','{"fieldMatchStrategyName": "CAMEL_CASE_MATCH","fieldEditEnable": true, "fieldDeleteEnable": true, "fieldAddEnable": false}','MYSQL',NULL,'SOURCE') -,('mysql_field_source_match','MAPPING','{"fieldMatchStrategyName": "CAMEL_CASE_MATCH","fieldEditEnable": true, "fieldDeleteEnable": true, "fieldAddEnable": true}','STARROCKS',NULL,'SINK') +,('mysql_field_source_match','MAPPING','{"fieldMatchStrategyName": "CAMEL_CASE_MATCH","fieldEditEnable": true, "fieldDeleteEnable": true, "fieldAddEnable": true}','MYSQL',NULL,'SOURCE') +,('starrocks_field_source_match','MAPPING','{"fieldMatchStrategyName": "CAMEL_CASE_MATCH","fieldEditEnable": true, "fieldDeleteEnable": true, "fieldAddEnable": true}','STARROCKS',NULL,'SINK') ; From 2dd2ae03018615c27fc2b627f98ccf54ec40784e Mon Sep 17 00:00:00 2001 From: jefftlin Date: Tue, 21 May 2024 21:31:51 +0800 Subject: [PATCH 32/60] Minor fix for packaging --- .../restful/api/ExchangisDataSourceRestfulApi.java | 2 +- .../launcher/linkis/LinkisExchangisTaskLauncher.java | 2 +- .../job/server/builder/JobParamConstraints.java | 4 ++++ .../handlers/MySQLDataxSubExchangisJobHandler.java | 4 ++-- exchangis-server/pom.xml | 10 ---------- pom.xml | 11 ----------- 6 files changed, 8 insertions(+), 25 deletions(-) diff --git a/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/restful/api/ExchangisDataSourceRestfulApi.java b/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/restful/api/ExchangisDataSourceRestfulApi.java index 474a94439..dbb0e2526 100644 --- a/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/restful/api/ExchangisDataSourceRestfulApi.java +++ b/exchangis-datasource/exchangis-datasource-server/src/main/java/com/webank/wedatasphere/exchangis/datasource/server/restful/api/ExchangisDataSourceRestfulApi.java @@ -162,7 +162,7 @@ public Message create(/*@PathParam("type") String type, */@Valid @RequestBody Da Message message = new Message(); String loginUser = UserUtils.getLoginUser(request); String oringinUser = SecurityFilter.getLoginUsername(request); - LOG.info("dataSourceName: " + dataSourceCreateVO.getDataSourceName() + "dataSourceDesc: " + dataSourceCreateVO.getDataSourceDesc() + "label: " + dataSourceCreateVO.getLabels()); + LOG.info("dataSourceName: " + dataSourceCreateVO.getDataSourceName() + ", dataSourceDesc: " + dataSourceCreateVO.getDataSourceDesc() + ", label: " + dataSourceCreateVO.getLabels()); if(bindingResult.hasErrors()){ List fieldErrors = bindingResult.getFieldErrors(); for(int i=0;i[] sourceMappings(){ } public JobParamDefine[] sinkMappings(){ - return new JobParamDefine[]{USERNAME, PASSWORD, SINK_DATABASE, SINK_TABLE, - SINK_HOST, SINK_PORT, SINK_PARAMS_MAP}; + return new JobParamDefine[]{SINK_HOST, SINK_PORT, USERNAME, PASSWORD, + SINK_DATABASE, SINK_TABLE, SINK_PARAMS_MAP}; } } diff --git a/exchangis-server/pom.xml b/exchangis-server/pom.xml index d0ece72bd..3a40a6b10 100644 --- a/exchangis-server/pom.xml +++ b/exchangis-server/pom.xml @@ -26,16 +26,6 @@ com.webank.wedatasphere.exchangis exchangis-engine-server ${revision} - - - org.apache.linkis - linkis-module - - - spring-jdbc - org.springframework - - com.webank.wedatasphere.exchangis diff --git a/pom.xml b/pom.xml index 523a18983..d8beb752f 100644 --- a/pom.xml +++ b/pom.xml @@ -109,12 +109,6 @@ org.apache.linkis linkis-mybatis ${linkis.version} - - - org.springframework - spring-orm - - org.apache.linkis @@ -177,11 +171,6 @@ com.thoughtworks.xstream ${xstream.version} - - org.springframework - spring-orm - 5.2.15.RELEASE - From 253d87a4013dbf3189d2bf6626c718e6cd34347c Mon Sep 17 00:00:00 2001 From: jefftlin Date: Wed, 22 May 2024 11:07:10 +0800 Subject: [PATCH 33/60] Fix of job log --- db/1.1.3/exchangis_dml.sql | 2 +- db/exchangis_dml.sql | 2 +- .../vo/ExchangisJobDataSourcesContent.java | 28 +++++++++---------- .../ExchangisStarRocksDataSource.java | 3 +- .../linkis/LaunchTaskLogOperator.scala | 3 ++ .../log/service/AbstractJobLogService.java | 2 +- .../server/log/service/RpcJobLogService.java | 1 + 7 files changed, 23 insertions(+), 18 deletions(-) diff --git a/db/1.1.3/exchangis_dml.sql b/db/1.1.3/exchangis_dml.sql index b1d3457da..8d8530575 100644 --- a/db/1.1.3/exchangis_dml.sql +++ b/db/1.1.3/exchangis_dml.sql @@ -3,5 +3,5 @@ INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_directio ,('batchSize','批量字节数大小','DATAX-SINK','STARROCKS','INPUT','maxBatchSize','批量字节数大小','',0,'NUMBER','','','REGEX','^[1-9]\\d*$','批量大小输入错误',0,0,'',1,'',2,'',1,NULL); UPDATE exchangis_engine_settings -SET engine_direction='mysql->hive,hive->mysql,mysql->oracle,oracle->mysql,oracle->hive,hive->oracle,mongodb->hive,hive->mongodb,mysql->elasticsearch,oracle->elasticsearch,mongodb->elasticsearch,mysql->mongodb,mongodb->mysql,oracle->mongodb,mongodb->oracle,mysql->starrocks' +SET engine_direction='mysql->hive,hive->mysql,mysql->oracle,oracle->mysql,oracle->hive,hive->oracle,mongodb->hive,hive->mongodb,mysql->elasticsearch,oracle->elasticsearch,mongodb->elasticsearch,mysql->mongodb,mongodb->mysql,oracle->mongodb,mongodb->oracle,hive->starrocks' WHERE engine_name='datax'; \ No newline at end of file diff --git a/db/exchangis_dml.sql b/db/exchangis_dml.sql index 0c90fa450..6c2c7fe8f 100644 --- a/db/exchangis_dml.sql +++ b/db/exchangis_dml.sql @@ -73,7 +73,7 @@ INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_directio -- engine_settings records INSERT INTO `exchangis_engine_settings` (id, engine_name, engine_desc, engine_settings_value, engine_direction, res_loader_class, res_uploader_class, modify_time) VALUES -(1, 'datax', 'datax sync engine', '{}', 'mysql->hive,hive->mysql,mysql->oracle,oracle->mysql,oracle->hive,hive->oracle,mongodb->hive,hive->mongodb,mysql->elasticsearch,oracle->elasticsearch,mongodb->elasticsearch,mysql->mongodb,mongodb->mysql,oracle->mongodb,mongodb->oracle,mysql->starrocks', 'com.webank.wedatasphere.exchangis.engine.resource.loader.datax.DataxEngineResourceLoader', NULL, NULL), +(1, 'datax', 'datax sync engine', '{}', 'mysql->hive,hive->mysql,mysql->oracle,oracle->mysql,oracle->hive,hive->oracle,mongodb->hive,hive->mongodb,mysql->elasticsearch,oracle->elasticsearch,mongodb->elasticsearch,mysql->mongodb,mongodb->mysql,oracle->mongodb,mongodb->oracle,hive->starrocks', 'com.webank.wedatasphere.exchangis.engine.resource.loader.datax.DataxEngineResourceLoader', NULL, NULL), (2, 'sqoop', 'hadoop tool', '{}', 'mysql->hive,hive->mysql', '', NULL, NULL); -- exchangis_job_transform_rule records diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobDataSourcesContent.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobDataSourcesContent.java index 36af7634f..368cce9ea 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobDataSourcesContent.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobDataSourcesContent.java @@ -22,7 +22,7 @@ public class ExchangisJobDataSourcesContent { /** * Source ds */ - private ExchangisJobDataSource source = new ExchangisJobDataSource(); +// private ExchangisJobDataSource source = new ExchangisJobDataSource(); @JsonProperty("sink_id") @@ -31,7 +31,7 @@ public class ExchangisJobDataSourcesContent { /** * Sink ds */ - private ExchangisJobDataSource sink = new ExchangisJobDataSource(); +// private ExchangisJobDataSource sink = new ExchangisJobDataSource(); public String getSourceId() { return sourceId; @@ -49,21 +49,21 @@ public void setSinkId(String sinkId) { this.sinkId = sinkId; } - public void setSource(ExchangisJobDataSource source) { - this.source = source; - } +// public void setSource(ExchangisJobDataSource source) { +// this.source = source; +// } - public ExchangisJobDataSource getSource() { - return source; - } +// public ExchangisJobDataSource getSource() { +// return source; +// } - public void setSink(ExchangisJobDataSource sink) { - this.sink = sink; - } +// public void setSink(ExchangisJobDataSource sink) { +// this.sink = sink; +// } - public ExchangisJobDataSource getSink() { - return sink; - } +// public ExchangisJobDataSource getSink() { +// return sink; +// } @JsonInclude(JsonInclude.Include.NON_EMPTY) public static class ExchangisJobDataSource { diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-starrocks/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/starrocks/ExchangisStarRocksDataSource.java b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-starrocks/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/starrocks/ExchangisStarRocksDataSource.java index 894b45439..f0732ba53 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-starrocks/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/starrocks/ExchangisStarRocksDataSource.java +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-starrocks/src/main/java/com/webank/wedatasphere/exchangis/extension/datasource/starrocks/ExchangisStarRocksDataSource.java @@ -2,6 +2,7 @@ import com.webank.wedatasphere.exchangis.dao.domain.ExchangisJobParamConfig; import com.webank.wedatasphere.exchangis.datasource.core.domain.ExchangisDataSourceType; +import com.webank.wedatasphere.exchangis.datasource.core.domain.StructClassifier; import com.webank.wedatasphere.exchangis.datasource.linkis.ExchangisBatchDataSource; import java.util.List; @@ -28,7 +29,7 @@ public String option() { @Override public String structClassifier() { - return null; + return StructClassifier.STRUCTURED.name; } @Override diff --git a/exchangis-job/exchangis-job-launcher/src/main/scala/com/webank/wedatasphere/exchangis/job/launcher/linkis/LaunchTaskLogOperator.scala b/exchangis-job/exchangis-job-launcher/src/main/scala/com/webank/wedatasphere/exchangis/job/launcher/linkis/LaunchTaskLogOperator.scala index 93e6fe159..c31911609 100644 --- a/exchangis-job/exchangis-job-launcher/src/main/scala/com/webank/wedatasphere/exchangis/job/launcher/linkis/LaunchTaskLogOperator.scala +++ b/exchangis-job/exchangis-job-launcher/src/main/scala/com/webank/wedatasphere/exchangis/job/launcher/linkis/LaunchTaskLogOperator.scala @@ -20,8 +20,11 @@ class LaunchTaskLogOperator extends EngineConnLogOperator{ protected override def addParameters(builder: EngineConnOperateAction.Builder): Unit = { super.addParameters(builder) + builder.operatorName(EngineConnLogOperator.OPERATOR_NAME) builder.addParameter("enableTail", enableTail) } + + override def getName: String = LaunchTaskLogOperator.OPERATOR_NAME } object LaunchTaskLogOperator { val OPERATOR_NAME = "launchTaskLog" diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/AbstractJobLogService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/AbstractJobLogService.java index f75591a92..cb47f18c4 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/AbstractJobLogService.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/AbstractJobLogService.java @@ -140,7 +140,7 @@ public JobLogCache getOrCreateLogCache(String jobExecId){ return cacheHolder.get(jobExecId, () -> { LaunchedExchangisJobEntity launchedExchangisJob = launchedJobDao.searchLogPathInfo(jobExecId); if (Objects.nonNull(launchedExchangisJob)) { - + return loadJobLogCache(jobExecId, launchedExchangisJob); } return null; }); diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/RpcJobLogService.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/RpcJobLogService.java index 72e0f5267..e99005fa1 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/RpcJobLogService.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/log/service/RpcJobLogService.java @@ -153,6 +153,7 @@ public LogResult logsFromPageAndPath(String logPath, LogQuery logQuery) { throw new ExchangisJobServerException.Runtime(LOG_OP_ERROR.getCode(),"Unable to fetch log from: [" + logPath + "], unknown request protocol: [" + response + "]", null); } + logPath = logPath.substring(splitPos + 1); } String fullPath = Constraints.LOG_LOCAL_PATH.getValue() + IOUtils.DIR_SEPARATOR_UNIX + logPath; LogResult result = new LogResult(0, false, Collections.emptyList()); From d6bd6116a95f31ac6349b5c4b197c640bb053a92 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Wed, 22 May 2024 11:10:32 +0800 Subject: [PATCH 34/60] Add job handler for starrocks datasource --- exchangis-dao/pom.xml | 4 - .../engines/datax/datax-core/pom.xml | 2 +- .../exchangis/job/domain/SubExchangisJob.java | 4 +- .../StarRocksDataxSubExchangisJobHandler.java | 107 ++++++++++++++++++ exchangis-server/pom.xml | 6 +- 5 files changed, 113 insertions(+), 10 deletions(-) create mode 100644 exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/StarRocksDataxSubExchangisJobHandler.java diff --git a/exchangis-dao/pom.xml b/exchangis-dao/pom.xml index 4eac9752a..ca406e7fa 100644 --- a/exchangis-dao/pom.xml +++ b/exchangis-dao/pom.xml @@ -36,10 +36,6 @@ hibernate-validator ${hibernate.validator} - - org.springframework - spring-orm - diff --git a/exchangis-engines/engines/datax/datax-core/pom.xml b/exchangis-engines/engines/datax/datax-core/pom.xml index 18e65a5f5..d7d17f09d 100644 --- a/exchangis-engines/engines/datax/datax-core/pom.xml +++ b/exchangis-engines/engines/datax/datax-core/pom.xml @@ -147,7 +147,7 @@ org.apache.hadoop hadoop-common - ${hadoop.version} + 3.3.4 org.apache.commons diff --git a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/SubExchangisJob.java b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/SubExchangisJob.java index 40f1e8576..9facdd5a0 100644 --- a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/SubExchangisJob.java +++ b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/SubExchangisJob.java @@ -24,10 +24,10 @@ public class SubExchangisJob extends GenericExchangisJob { public static final String REALM_JOB_DATA_SOURCE = "job.realm.data-source"; - public static final String REALM_JOB_CONTENT_SINK = "job.realm.content.sink"; - public static final String REALM_JOB_CONTENT_SOURCE = "job.realm.content.source"; + public static final String REALM_JOB_CONTENT_SINK = "job.realm.content.sink"; + // public static final String REALM_JOB_COLUMN_MAPPING = "job.realm.column-mappings"; /** diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/StarRocksDataxSubExchangisJobHandler.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/StarRocksDataxSubExchangisJobHandler.java new file mode 100644 index 000000000..93e10e0f3 --- /dev/null +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/handlers/StarRocksDataxSubExchangisJobHandler.java @@ -0,0 +1,107 @@ +package com.webank.wedatasphere.exchangis.job.server.builder.transform.handlers; + +import com.webank.wedatasphere.exchangis.datasource.core.utils.Json; +import com.webank.wedatasphere.exchangis.job.builder.ExchangisJobBuilderContext; +import com.webank.wedatasphere.exchangis.job.domain.SubExchangisJob; +import com.webank.wedatasphere.exchangis.job.domain.params.JobParam; +import com.webank.wedatasphere.exchangis.job.domain.params.JobParamDefine; +import com.webank.wedatasphere.exchangis.job.domain.params.JobParamSet; +import com.webank.wedatasphere.exchangis.job.domain.params.JobParams; +import com.webank.wedatasphere.exchangis.job.server.builder.JobParamConstraints; +import org.apache.commons.lang3.StringUtils; +import org.apache.linkis.common.exception.ErrorException; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * StarRocks in datax + */ +public class StarRocksDataxSubExchangisJobHandler extends AuthEnabledSubExchangisJobHandler { + + /** + * Host + */ + private static final JobParamDefine SINK_HOST = JobParams.define("connection[0].host", JobParamConstraints.HOST); + + /** + * TCP_Port + */ + private static final JobParamDefine SINK_PORT = JobParams.define("connection[0].port", JobParamConstraints.PORT); + + /** + * HTTP_Port + */ + private static final JobParamDefine SINK_LOAD_URL = JobParams.define("loadUrl[0]", paramSet -> { + JobParam host = paramSet.get("connection[0].host"); + JobParam httpPort = paramSet.get(JobParamConstraints.HTTP_PORT); + if (Objects.nonNull(host) && StringUtils.isNotBlank(host.getValue()) && + Objects.nonNull(httpPort) && StringUtils.isNotBlank(httpPort.getValue())) { + return host.getValue() + ":" + httpPort.getValue(); + } + return null; + }); + + /** + * Database + */ + private static final JobParamDefine SINK_DATABASE = JobParams.define("database", JobParamConstraints.DATABASE); + + /** + * Table + */ + private static final JobParamDefine SINK_TABLE = JobParams.define("table", JobParamConstraints.TABLE); + + /** + * Connect params + */ + private static final JobParamDefine> SINK_PARAMS_MAP = JobParams.define("connection[0].connParams", JobParamConstraints.CONNECT_PARAMS, + connectParams -> Json.fromJson(connectParams, Map.class), String.class); + + /** + * SQL column + */ + private static final JobParamDefine> SQL_COLUMN = JobParams.define("column", job -> { + List columns = job.getSinkColumns().stream().map(SubExchangisJob.ColumnDefine::getName).collect(Collectors.toList()); + if (columns.isEmpty()){ + columns.add("*"); + } + return columns; + }, SubExchangisJob.class); + + @Override + public void handleJobSource(SubExchangisJob subExchangisJob, ExchangisJobBuilderContext ctx) throws ErrorException { + } + + @Override + public void handleJobSink(SubExchangisJob subExchangisJob, ExchangisJobBuilderContext ctx) throws ErrorException { + JobParamSet paramSet = subExchangisJob.getRealmParams(SubExchangisJob.REALM_JOB_CONTENT_SINK); + if (Objects.nonNull(paramSet)){ + JobParamDefine[] jobParamDefines = sinkMappings(); + Arrays.asList(jobParamDefines).forEach( + define -> paramSet.addNonNull(define.get(paramSet)) + ); + } + } + + @Override + public String dataSourceType() { + return "starrocks"; + } + + @Override + public boolean acceptEngine(String engineType) { + return "datax".equalsIgnoreCase(engineType); + } + + private JobParamDefine[] sourceMappings(){ + return null; + } + + public JobParamDefine[] sinkMappings(){ + return new JobParamDefine[]{USERNAME, PASSWORD, SINK_HOST, SINK_PORT, SINK_LOAD_URL, SINK_DATABASE, SINK_TABLE, SINK_PARAMS_MAP}; + } +} diff --git a/exchangis-server/pom.xml b/exchangis-server/pom.xml index 3a40a6b10..bc3734e5c 100644 --- a/exchangis-server/pom.xml +++ b/exchangis-server/pom.xml @@ -46,10 +46,10 @@ ${revision} - org.apache.linkis - linkis-module + com.webank.wedatasphere.exchangis + exchangis-dao + ${revision} - com.fasterxml classmate From 178587ff41fe3691f27e873c36e0e9aae2a3c4fb Mon Sep 17 00:00:00 2001 From: davidhua Date: Wed, 22 May 2024 15:16:12 +0800 Subject: [PATCH 35/60] Adjust ExchangisJobTransformsContent. --- .../vo/ExchangisJobTransformsContent.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobTransformsContent.java b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobTransformsContent.java index f588f5786..7b7b61e85 100644 --- a/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobTransformsContent.java +++ b/exchangis-datasource/exchangis-datasource-core/src/main/java/com/webank/wedatasphere/exchangis/datasource/core/vo/ExchangisJobTransformsContent.java @@ -12,6 +12,16 @@ public class ExchangisJobTransformsContent { @JsonProperty("code_id") private String codeId; + /** + * Table (source) not exist + */ + private boolean srcTblNotExist = false; + + /** + * Table (sink) not exist + */ + private boolean sinkTblNotExist = false; + private List mapping; public boolean isAddEnable() { @@ -53,4 +63,20 @@ public String getCodeId() { public void setCodeId(String codeId) { this.codeId = codeId; } + + public boolean isSrcTblNotExist() { + return srcTblNotExist; + } + + public void setSrcTblNotExist(boolean srcTblNotExist) { + this.srcTblNotExist = srcTblNotExist; + } + + public boolean isSinkTblNotExist() { + return sinkTblNotExist; + } + + public void setSinkTblNotExist(boolean sinkTblNotExist) { + this.sinkTblNotExist = sinkTblNotExist; + } } From f5a3736c026544e36dd4e5fc0875e56e8ca61e7e Mon Sep 17 00:00:00 2001 From: davidhua Date: Wed, 22 May 2024 20:21:39 +0800 Subject: [PATCH 36/60] Use 'revision' to manage project version. --- .gitignore | 4 +- .../exchangis-datasource-core/pom.xml | 6 +-- .../exchangis-datasource-linkis/pom.xml | 6 +-- .../exchangis-datasource-loader/pom.xml | 6 +-- .../exchangis-datasource-server/pom.xml | 8 ++-- .../exchangis-datasource-service/pom.xml | 10 ++--- .../exchangis-datasource-streamis/pom.xml | 8 ++-- .../pom.xml | 10 ++--- .../exchangis-datasource-ext-hive/pom.xml | 10 ++--- .../exchangis-datasource-ext-mongodb/pom.xml | 10 ++--- .../exchangis-datasource-ext-mysql/pom.xml | 10 ++--- .../exchangis-datasource-ext-oracle/pom.xml | 10 ++--- .../exchangis-datasource-ext-sftp/pom.xml | 10 ++--- .../pom.xml | 10 ++--- .../engineconn-plugins/datax/pom.xml | 4 +- .../engineconn-plugins/sqoop/pom.xml | 4 +- .../exchangis-engine-common/pom.xml | 5 ++- .../exchangis-engine-core/pom.xml | 7 ++-- .../exchangis-engine-server/pom.xml | 7 ++-- exchangis-job/exchangis-job-builder/pom.xml | 7 ++-- exchangis-job/exchangis-job-common/pom.xml | 7 ++-- exchangis-job/exchangis-job-launcher/pom.xml | 7 ++-- exchangis-job/exchangis-job-metrics/pom.xml | 3 +- exchangis-job/exchangis-job-server/pom.xml | 16 +++++--- exchangis-job/pom.xml | 8 +--- exchangis-plugins/exchangis-appconn/pom.xml | 4 +- exchangis-plugins/pom.xml | 1 + .../exchangis-project-entity/pom.xml | 5 ++- .../exchangis-project-provider/pom.xml | 5 ++- .../exchangis-project-server/pom.xml | 11 +++--- exchangis-project/pom.xml | 1 + exchangis-server/pom.xml | 11 +++--- pom.xml | 38 ++++++++++++++++++- 33 files changed, 159 insertions(+), 110 deletions(-) diff --git a/.gitignore b/.gitignore index 46dec8ece..a2857cb35 100644 --- a/.gitignore +++ b/.gitignore @@ -35,4 +35,6 @@ package-lock.json web/dist -workspace/ \ No newline at end of file +workspace/ + +.flattened-pom.xml \ No newline at end of file diff --git a/exchangis-datasource/exchangis-datasource-core/pom.xml b/exchangis-datasource/exchangis-datasource-core/pom.xml index 33b673931..a3f253324 100644 --- a/exchangis-datasource/exchangis-datasource-core/pom.xml +++ b/exchangis-datasource/exchangis-datasource-core/pom.xml @@ -3,10 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - exchangis-datasource + exchangis com.webank.wedatasphere.exchangis ${revision} - ../pom.xml + ../../pom.xml 4.0.0 @@ -21,7 +21,7 @@ com.webank.wedatasphere.exchangis exchangis-dao - ${revision} + ${project.version} diff --git a/exchangis-datasource/exchangis-datasource-linkis/pom.xml b/exchangis-datasource/exchangis-datasource-linkis/pom.xml index cf202c20e..d2a6553dd 100644 --- a/exchangis-datasource/exchangis-datasource-linkis/pom.xml +++ b/exchangis-datasource/exchangis-datasource-linkis/pom.xml @@ -3,10 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - exchangis-datasource + exchangis com.webank.wedatasphere.exchangis ${revision} - ../pom.xml + ../../pom.xml 4.0.0 @@ -21,7 +21,7 @@ com.webank.wedatasphere.exchangis exchangis-datasource-core - ${revision} + ${project.version} diff --git a/exchangis-datasource/exchangis-datasource-loader/pom.xml b/exchangis-datasource/exchangis-datasource-loader/pom.xml index 0bfe8fde2..97af1ee06 100644 --- a/exchangis-datasource/exchangis-datasource-loader/pom.xml +++ b/exchangis-datasource/exchangis-datasource-loader/pom.xml @@ -3,10 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - exchangis-datasource + exchangis com.webank.wedatasphere.exchangis ${revision} - ../pom.xml + ../../pom.xml 4.0.0 @@ -21,7 +21,7 @@ com.webank.wedatasphere.exchangis exchangis-datasource-core - ${revision} + ${project.version} diff --git a/exchangis-datasource/exchangis-datasource-server/pom.xml b/exchangis-datasource/exchangis-datasource-server/pom.xml index 347abac89..e2b04ecad 100644 --- a/exchangis-datasource/exchangis-datasource-server/pom.xml +++ b/exchangis-datasource/exchangis-datasource-server/pom.xml @@ -3,10 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - exchangis-datasource + exchangis com.webank.wedatasphere.exchangis ${revision} - ../pom.xml + ../../pom.xml 4.0.0 @@ -21,13 +21,13 @@ com.webank.wedatasphere.exchangis exchangis-datasource-service - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-datasource-loader - ${revision} + ${project.version} diff --git a/exchangis-datasource/exchangis-datasource-service/pom.xml b/exchangis-datasource/exchangis-datasource-service/pom.xml index 1736570e1..02f408fe5 100644 --- a/exchangis-datasource/exchangis-datasource-service/pom.xml +++ b/exchangis-datasource/exchangis-datasource-service/pom.xml @@ -3,10 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - exchangis-datasource + exchangis com.webank.wedatasphere.exchangis ${revision} - ../pom.xml + ../../pom.xml 4.0.0 @@ -22,19 +22,19 @@ com.webank.wedatasphere.exchangis exchangis-datasource-linkis - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-job-common - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-engine-core - ${revision} + ${project.version} compile diff --git a/exchangis-datasource/exchangis-datasource-streamis/pom.xml b/exchangis-datasource/exchangis-datasource-streamis/pom.xml index b27dfb010..bf63d0db6 100644 --- a/exchangis-datasource/exchangis-datasource-streamis/pom.xml +++ b/exchangis-datasource/exchangis-datasource-streamis/pom.xml @@ -3,10 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - exchangis-datasource + exchangis com.webank.wedatasphere.exchangis ${revision} - ../pom.xml + ../../pom.xml 4.0.0 @@ -21,12 +21,12 @@ com.webank.wedatasphere.exchangis exchangis-datasource-core - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-datasource-linkis - ${revision} + ${project.version} diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-elasticsearch/pom.xml b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-elasticsearch/pom.xml index 86827ee7a..0ea794a53 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-elasticsearch/pom.xml +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-elasticsearch/pom.xml @@ -3,10 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - exchangis-datasource + exchangis com.webank.wedatasphere.exchangis ${revision} - ../../pom.xml + ../../../pom.xml 4.0.0 @@ -21,17 +21,17 @@ com.webank.wedatasphere.exchangis exchangis-dao - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-datasource-linkis - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-datasource-core - ${revision} + ${project.version} diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-hive/pom.xml b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-hive/pom.xml index 55d04f112..3f4323ab1 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-hive/pom.xml +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-hive/pom.xml @@ -3,10 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - exchangis-datasource + exchangis com.webank.wedatasphere.exchangis ${revision} - ../../pom.xml + ../../../pom.xml 4.0.0 @@ -22,17 +22,17 @@ com.webank.wedatasphere.exchangis exchangis-dao - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-datasource-linkis - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-dao - ${revision} + ${project.version} diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mongodb/pom.xml b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mongodb/pom.xml index e75f75b15..7bbd306c7 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mongodb/pom.xml +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mongodb/pom.xml @@ -3,10 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - exchangis-datasource + exchangis com.webank.wedatasphere.exchangis ${revision} - ../../pom.xml + ../../../pom.xml 4.0.0 @@ -21,17 +21,17 @@ com.webank.wedatasphere.exchangis exchangis-dao - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-datasource-linkis - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-datasource-core - ${revision} + ${project.version} diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mysql/pom.xml b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mysql/pom.xml index ff446a170..6b8bc388f 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mysql/pom.xml +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-mysql/pom.xml @@ -3,10 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - exchangis-datasource + exchangis com.webank.wedatasphere.exchangis ${revision} - ../../pom.xml + ../../../pom.xml 4.0.0 @@ -21,17 +21,17 @@ com.webank.wedatasphere.exchangis exchangis-dao - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-datasource-linkis - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-datasource-core - ${revision} + ${project.version} diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-oracle/pom.xml b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-oracle/pom.xml index c5c90309f..8741dfdd7 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-oracle/pom.xml +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-oracle/pom.xml @@ -3,10 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - exchangis-datasource + exchangis com.webank.wedatasphere.exchangis ${revision} - ../../pom.xml + ../../../pom.xml 4.0.0 @@ -21,17 +21,17 @@ com.webank.wedatasphere.exchangis exchangis-dao - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-datasource-linkis - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-datasource-core - ${revision} + ${project.version} diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-sftp/pom.xml b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-sftp/pom.xml index 6ea72b3dc..f8b375288 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-sftp/pom.xml +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-sftp/pom.xml @@ -3,10 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - exchangis-datasource + exchangis com.webank.wedatasphere.exchangis ${revision} - ../../pom.xml + ../../../pom.xml 4.0.0 @@ -21,17 +21,17 @@ com.webank.wedatasphere.exchangis exchangis-dao - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-datasource-linkis - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-datasource-core - ${revision} + ${project.version} diff --git a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-starrocks/pom.xml b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-starrocks/pom.xml index ba062fcc9..d04f4d53f 100644 --- a/exchangis-datasource/extension-datasources/exchangis-datasource-ext-starrocks/pom.xml +++ b/exchangis-datasource/extension-datasources/exchangis-datasource-ext-starrocks/pom.xml @@ -3,10 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - exchangis-datasource + exchangis com.webank.wedatasphere.exchangis ${revision} - ../../pom.xml + ../../../pom.xml 4.0.0 @@ -21,17 +21,17 @@ com.webank.wedatasphere.exchangis exchangis-dao - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-datasource-linkis - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-datasource-core - ${revision} + ${project.version} diff --git a/exchangis-engines/engineconn-plugins/datax/pom.xml b/exchangis-engines/engineconn-plugins/datax/pom.xml index 8fc8d4977..b7eab874b 100644 --- a/exchangis-engines/engineconn-plugins/datax/pom.xml +++ b/exchangis-engines/engineconn-plugins/datax/pom.xml @@ -3,10 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - exchangis-engines + exchangis com.webank.wedatasphere.exchangis ${revision} - ../../pom.xml + ../../../pom.xml 4.0.0 diff --git a/exchangis-engines/engineconn-plugins/sqoop/pom.xml b/exchangis-engines/engineconn-plugins/sqoop/pom.xml index d2125da7f..bd6afb7d2 100644 --- a/exchangis-engines/engineconn-plugins/sqoop/pom.xml +++ b/exchangis-engines/engineconn-plugins/sqoop/pom.xml @@ -20,10 +20,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - exchangis-engines + exchangis com.webank.wedatasphere.exchangis ${revision} - ../../pom.xml + ../../../pom.xml 4.0.0 diff --git a/exchangis-engines/exchangis-engine-common/pom.xml b/exchangis-engines/exchangis-engine-common/pom.xml index 6a1efe5e4..16cf86da2 100644 --- a/exchangis-engines/exchangis-engine-common/pom.xml +++ b/exchangis-engines/exchangis-engine-common/pom.xml @@ -3,9 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - exchangis-engines + exchangis com.webank.wedatasphere.exchangis ${revision} + ../../pom.xml 4.0.0 @@ -19,7 +20,7 @@ com.webank.wedatasphere.exchangis exchangis-dao - ${revision} + ${project.version} org.apache.linkis diff --git a/exchangis-engines/exchangis-engine-core/pom.xml b/exchangis-engines/exchangis-engine-core/pom.xml index ecc445c3f..379c1dbfd 100644 --- a/exchangis-engines/exchangis-engine-core/pom.xml +++ b/exchangis-engines/exchangis-engine-core/pom.xml @@ -3,9 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - exchangis-engines + exchangis com.webank.wedatasphere.exchangis ${revision} + ../../pom.xml 4.0.0 @@ -20,12 +21,12 @@ com.webank.wedatasphere.exchangis exchangis-engine-common - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-dao - ${revision} + ${project.version} diff --git a/exchangis-engines/exchangis-engine-server/pom.xml b/exchangis-engines/exchangis-engine-server/pom.xml index 8d863e09b..5671e01bb 100644 --- a/exchangis-engines/exchangis-engine-server/pom.xml +++ b/exchangis-engines/exchangis-engine-server/pom.xml @@ -3,9 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - exchangis-engines + exchangis com.webank.wedatasphere.exchangis ${revision} + ../../pom.xml 4.0.0 @@ -20,12 +21,12 @@ com.webank.wedatasphere.exchangis exchangis-dao - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-engine-core - ${revision} + ${project.version} diff --git a/exchangis-job/exchangis-job-builder/pom.xml b/exchangis-job/exchangis-job-builder/pom.xml index e5d3591fb..e2eec71ec 100644 --- a/exchangis-job/exchangis-job-builder/pom.xml +++ b/exchangis-job/exchangis-job-builder/pom.xml @@ -3,9 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - exchangis-job + exchangis com.webank.wedatasphere.exchangis ${revision} + ../../pom.xml 4.0.0 @@ -15,7 +16,7 @@ com.webank.wedatasphere.exchangis exchangis-job-common - ${revision} + ${project.version} com.google.code.gson @@ -25,7 +26,7 @@ com.webank.wedatasphere.exchangis exchangis-datasource-service - ${revision} + ${project.version} compile diff --git a/exchangis-job/exchangis-job-common/pom.xml b/exchangis-job/exchangis-job-common/pom.xml index e04e3e922..aba4edd1a 100644 --- a/exchangis-job/exchangis-job-common/pom.xml +++ b/exchangis-job/exchangis-job-common/pom.xml @@ -3,9 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - exchangis-job + exchangis com.webank.wedatasphere.exchangis ${revision} + ../../pom.xml 4.0.0 @@ -15,7 +16,7 @@ com.webank.wedatasphere.exchangis exchangis-dao - ${revision} + ${project.version} org.apache.linkis @@ -25,7 +26,7 @@ com.webank.wedatasphere.exchangis exchangis-engine-common - ${revision} + ${project.version} org.apache.linkis diff --git a/exchangis-job/exchangis-job-launcher/pom.xml b/exchangis-job/exchangis-job-launcher/pom.xml index 22ca5cc78..b06b04516 100644 --- a/exchangis-job/exchangis-job-launcher/pom.xml +++ b/exchangis-job/exchangis-job-launcher/pom.xml @@ -3,9 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - exchangis-job + exchangis com.webank.wedatasphere.exchangis ${revision} + ../../pom.xml 4.0.0 @@ -15,12 +16,12 @@ com.webank.wedatasphere.exchangis exchangis-job-common - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-job-builder - ${revision} + ${project.version} org.apache.linkis diff --git a/exchangis-job/exchangis-job-metrics/pom.xml b/exchangis-job/exchangis-job-metrics/pom.xml index 5cfd72dc9..7da6f5992 100644 --- a/exchangis-job/exchangis-job-metrics/pom.xml +++ b/exchangis-job/exchangis-job-metrics/pom.xml @@ -3,9 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - exchangis-job + exchangis com.webank.wedatasphere.exchangis ${revision} + ../../pom.xml 4.0.0 diff --git a/exchangis-job/exchangis-job-server/pom.xml b/exchangis-job/exchangis-job-server/pom.xml index c4f680478..9eef4558a 100644 --- a/exchangis-job/exchangis-job-server/pom.xml +++ b/exchangis-job/exchangis-job-server/pom.xml @@ -3,9 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - exchangis-job + exchangis com.webank.wedatasphere.exchangis ${revision} + ../../pom.xml 4.0.0 @@ -20,22 +21,22 @@ com.webank.wedatasphere.exchangis exchangis-project-provider - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-job-launcher - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-datasource-service - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-engine-core - ${revision} + ${project.version} @@ -55,6 +56,11 @@ linkis-rpc ${linkis.version} + + org.modelmapper + modelmapper + 2.4.3 + diff --git a/exchangis-job/pom.xml b/exchangis-job/pom.xml index c8a18cc7e..2ccba4766 100644 --- a/exchangis-job/pom.xml +++ b/exchangis-job/pom.xml @@ -6,6 +6,7 @@ exchangis com.webank.wedatasphere.exchangis ${revision} + ../pom.xml 4.0.0 @@ -26,11 +27,4 @@ 8 - - - org.modelmapper - modelmapper - 2.4.3 - - \ No newline at end of file diff --git a/exchangis-plugins/exchangis-appconn/pom.xml b/exchangis-plugins/exchangis-appconn/pom.xml index 7ee765569..b7bffe95c 100644 --- a/exchangis-plugins/exchangis-appconn/pom.xml +++ b/exchangis-plugins/exchangis-appconn/pom.xml @@ -3,10 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - exchangis-plugins + exchangis com.webank.wedatasphere.exchangis ${revision} - ../pom.xml + ../../pom.xml 4.0.0 diff --git a/exchangis-plugins/pom.xml b/exchangis-plugins/pom.xml index 8325cf1ba..2a93dfb19 100644 --- a/exchangis-plugins/pom.xml +++ b/exchangis-plugins/pom.xml @@ -6,6 +6,7 @@ exchangis com.webank.wedatasphere.exchangis ${revision} + ../pom.xml 4.0.0 diff --git a/exchangis-project/exchangis-project-entity/pom.xml b/exchangis-project/exchangis-project-entity/pom.xml index aa053c166..44bba43b6 100644 --- a/exchangis-project/exchangis-project-entity/pom.xml +++ b/exchangis-project/exchangis-project-entity/pom.xml @@ -3,9 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - exchangis-project + exchangis com.webank.wedatasphere.exchangis ${revision} + ../../pom.xml 4.0.0 @@ -20,7 +21,7 @@ com.webank.wedatasphere.exchangis exchangis-dao - ${revision} + ${project.version} org.apache.commons diff --git a/exchangis-project/exchangis-project-provider/pom.xml b/exchangis-project/exchangis-project-provider/pom.xml index 90a65a2e1..d87a4d29a 100644 --- a/exchangis-project/exchangis-project-provider/pom.xml +++ b/exchangis-project/exchangis-project-provider/pom.xml @@ -3,9 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - exchangis-project + exchangis com.webank.wedatasphere.exchangis ${revision} + ../../pom.xml 4.0.0 @@ -20,7 +21,7 @@ com.webank.wedatasphere.exchangis exchangis-project-entity - ${revision} + ${project.version} diff --git a/exchangis-project/exchangis-project-server/pom.xml b/exchangis-project/exchangis-project-server/pom.xml index ce392fde1..d19247918 100644 --- a/exchangis-project/exchangis-project-server/pom.xml +++ b/exchangis-project/exchangis-project-server/pom.xml @@ -3,9 +3,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - exchangis-project + exchangis com.webank.wedatasphere.exchangis ${revision} + ../../pom.xml 4.0.0 @@ -20,22 +21,22 @@ com.webank.wedatasphere.exchangis exchangis-project-provider - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-job-server - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-dao - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-job-common - ${revision} + ${project.version} diff --git a/exchangis-project/pom.xml b/exchangis-project/pom.xml index 773af5277..f4decd1d6 100644 --- a/exchangis-project/pom.xml +++ b/exchangis-project/pom.xml @@ -6,6 +6,7 @@ exchangis com.webank.wedatasphere.exchangis ${revision} + ../pom.xml 4.0.0 diff --git a/exchangis-server/pom.xml b/exchangis-server/pom.xml index bc3734e5c..e8d5a5619 100644 --- a/exchangis-server/pom.xml +++ b/exchangis-server/pom.xml @@ -6,6 +6,7 @@ exchangis com.webank.wedatasphere.exchangis ${revision} + ../pom.xml 4.0.0 @@ -20,17 +21,17 @@ com.webank.wedatasphere.exchangis exchangis-datasource-server - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-engine-server - ${revision} + ${project.version} com.webank.wedatasphere.exchangis exchangis-job-server - ${revision} + ${project.version} + + org.jasypt + jasypt + ${jasypt.version} + diff --git a/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/config/DataxCoreConfiguration.scala b/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/config/DataxCoreConfiguration.scala index 01907434f..29aa81749 100644 --- a/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/config/DataxCoreConfiguration.scala +++ b/exchangis-engines/engineconn-plugins/datax/src/main/scala/org/apache/linkis/engineconnplugin/datax/config/DataxCoreConfiguration.scala @@ -91,7 +91,7 @@ object DataxCoreConfiguration { /** * Stream channel class */ - val CORE_TRANSPORT_STREAM_CHANNEL_CLASS: CommonVars[String] = CommonVars(CoreConstant.DATAX_CORE_TRANSPORT_STREAM_CHANNEL_CLASS, "com.webank.wedatasphere.exchangis.datax.core.transport.channel.memory.MemoryStreamChannel") + val CORE_TRANSPORT_STREAM_CHANNEL_CLASS: CommonVars[String] = CommonVars(CoreConstant.DATAX_CORE_TRANSPORT_STREAM_CHANNEL_CLASS, "com.alibaba.datax.core.transport.channel.memory.MemoryStreamChannel") /** * Block size of stream channel diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ExchangisJobDsBindServiceImpl.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ExchangisJobDsBindServiceImpl.java index 1987366f4..c3b99702d 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ExchangisJobDsBindServiceImpl.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/service/impl/ExchangisJobDsBindServiceImpl.java @@ -32,7 +32,7 @@ public void updateJobDsBind(Long jobId, List dsBinds) { public boolean inUse(Long datasourceId) { QueryWrapper condition = new QueryWrapper<>(); condition.eq("source_ds_id", datasourceId).or().eq("sink_ds_id", datasourceId); - int count = Optional.ofNullable(this.dsBindMapper.selectCount(condition)).orElse(0); + Integer count = Optional.ofNullable(this.dsBindMapper.selectCount(condition)).orElse(0); return count > 0; } } diff --git a/pom.xml b/pom.xml index cb0be09cd..f51255da1 100644 --- a/pom.xml +++ b/pom.xml @@ -39,7 +39,7 @@ 1.1.3-webank 1.1.2 - 1.4.0 + 1.5.0-wds 1.4.0 0.1.0-SNAPSHOT 1.3.0 @@ -69,6 +69,7 @@ 1.9.5 1.4.19 0.1.0-SNAPSHOT + 1.9.3 @@ -221,10 +222,10 @@ ${jdk.compile.version} ${jdk.compile.version} - - - - + + + + @@ -236,6 +237,17 @@ net.alchim31.maven scala-maven-plugin ${scala-maven-plugin.version} + + + + + + + + + + + eclipse-add-source @@ -265,10 +277,6 @@ - - ${scala.version} - incremental - org.apache.maven.plugins @@ -302,12 +310,12 @@ - - - org.codehaus.mojo - flatten-maven-plugin - - + + + + + + From 72e40014721af1a67a820e5ed9bda3d811c592a8 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Wed, 5 Jun 2024 14:09:16 +0800 Subject: [PATCH 54/60] Update linkis datasource version to 1.5.0-wds --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f51255da1..d09b94500 100644 --- a/pom.xml +++ b/pom.xml @@ -40,7 +40,7 @@ 1.1.3-webank 1.1.2 1.5.0-wds - 1.4.0 + 1.5.0-wds 0.1.0-SNAPSHOT 1.3.0 3.0.0 From df204b69c81c678cae6fec6953317d26373ed640 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Wed, 5 Jun 2024 17:20:36 +0800 Subject: [PATCH 55/60] Update linkis version to 1.5.0-wds-SNAPSHOT --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index d09b94500..eb0ff49c3 100644 --- a/pom.xml +++ b/pom.xml @@ -39,8 +39,8 @@ 1.1.3-webank 1.1.2 - 1.5.0-wds - 1.5.0-wds + 1.5.0-wds-SNAPSHOT + 1.5.0-wds-SNAPSHOT 0.1.0-SNAPSHOT 1.3.0 3.0.0 From 915e3f5d81af66bb1ac0c5f74c4f487d34ed002b Mon Sep 17 00:00:00 2001 From: jefftlin Date: Thu, 6 Jun 2024 14:39:27 +0800 Subject: [PATCH 56/60] Declare version of some maven pluigns --- exchangis-job/exchangis-job-server/pom.xml | 4 +++- exchangis-plugins/exchangis-appconn/pom.xml | 7 ++----- pom.xml | 17 +++++++++-------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/exchangis-job/exchangis-job-server/pom.xml b/exchangis-job/exchangis-job-server/pom.xml index 9eef4558a..0c83c65d4 100644 --- a/exchangis-job/exchangis-job-server/pom.xml +++ b/exchangis-job/exchangis-job-server/pom.xml @@ -68,15 +68,17 @@ org.apache.maven.plugins maven-deploy-plugin + ${maven-deploy-plugin.version} - net.alchim31.maven scala-maven-plugin + ${scala-maven-plugin.version} org.apache.maven.plugins maven-jar-plugin + ${maven-jar-plugin.version} diff --git a/exchangis-plugins/exchangis-appconn/pom.xml b/exchangis-plugins/exchangis-appconn/pom.xml index b7bffe95c..aa84a4261 100644 --- a/exchangis-plugins/exchangis-appconn/pom.xml +++ b/exchangis-plugins/exchangis-appconn/pom.xml @@ -124,15 +124,12 @@ org.apache.maven.plugins maven-deploy-plugin - - - - net.alchim31.maven - scala-maven-plugin + ${maven-deploy-plugin.version} org.apache.maven.plugins maven-jar-plugin + ${maven-jar-plugin.version} org.apache.maven.plugins diff --git a/pom.xml b/pom.xml index eb0ff49c3..12e0cd4f7 100644 --- a/pom.xml +++ b/pom.xml @@ -38,7 +38,7 @@ 1.1.3-webank - 1.1.2 + 1.5.0-SNAPSHOT 1.5.0-wds-SNAPSHOT 1.5.0-wds-SNAPSHOT 0.1.0-SNAPSHOT @@ -49,6 +49,7 @@ 4.7.1 1.8 3.3.3 + 2.6 2.8.5 2.11.3 1.9.13 @@ -281,7 +282,7 @@ org.apache.maven.plugins maven-jar-plugin - 2.6 + ${maven-jar-plugin.version} org.codehaus.mojo @@ -310,12 +311,12 @@ - - - - - - + + + org.codehaus.mojo + flatten-maven-plugin + + From 0b6c2d61205c56f2910cc9285a32a917bf616052 Mon Sep 17 00:00:00 2001 From: davidhua Date: Fri, 7 Jun 2024 14:11:25 +0800 Subject: [PATCH 57/60] Add rawType in column definition. --- .../exchangis/job/domain/SubExchangisJob.java | 14 ++++++++++++++ .../engine/DataxExchangisEngineJobBuilder.java | 6 ++++-- .../builder/engine/DataxMappingContext.java | 18 ++++++++++++++++++ .../mappings/HiveDataxParamsMapping.java | 4 ++++ 4 files changed, 40 insertions(+), 2 deletions(-) diff --git a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/SubExchangisJob.java b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/SubExchangisJob.java index 9facdd5a0..7812212ac 100644 --- a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/SubExchangisJob.java +++ b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/domain/SubExchangisJob.java @@ -135,6 +135,12 @@ public static class ColumnDefine{ * Column type */ private String type; + + /** + * Raw column type + */ + private String rawType; + /** * Column index */ @@ -177,6 +183,14 @@ public Integer getIndex() { public void setIndex(Integer index) { this.index = index; } + + public String getRawType() { + return rawType; + } + + public void setRawType(String rawType) { + this.rawType = rawType; + } } /** diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/engine/DataxExchangisEngineJobBuilder.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/engine/DataxExchangisEngineJobBuilder.java index a40eac517..df0ec7131 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/engine/DataxExchangisEngineJobBuilder.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/engine/DataxExchangisEngineJobBuilder.java @@ -48,10 +48,12 @@ public class DataxExchangisEngineJobBuilder extends AbstractResourceEngineJobBui private static final JobParamDefine COLUMN_MAPPINGS = JobParams.define("column.mappings", job -> { DataxMappingContext mappingContext = new DataxMappingContext(); job.getSourceColumns().forEach(columnDefine -> mappingContext.getSourceColumns().add( - new DataxMappingContext.Column(columnDefine.getName(), columnDefine.getType(), columnDefine.getIndex() + "") + new DataxMappingContext.Column(columnDefine.getName(), columnDefine.getType(), + columnDefine.getRawType(), columnDefine.getIndex() + "") )); job.getSinkColumns().forEach(columnDefine -> mappingContext.getSinkColumns().add( - new DataxMappingContext.Column(columnDefine.getName(), columnDefine.getType(), columnDefine.getIndex() + "") + new DataxMappingContext.Column(columnDefine.getName(), columnDefine.getType(), + columnDefine.getRawType(), columnDefine.getIndex() + "") )); job.getColumnFunctions().forEach(function -> { DataxMappingContext.Transformer.Parameter parameter = new DataxMappingContext.Transformer.Parameter(); diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/engine/DataxMappingContext.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/engine/DataxMappingContext.java index 5bc868162..56e90c863 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/engine/DataxMappingContext.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/engine/DataxMappingContext.java @@ -61,6 +61,11 @@ public static class Column{ */ private String type; + /** + * Raw column type + */ + private String rawType; + /** * Index name */ @@ -71,8 +76,13 @@ public Column(){ } public Column(String name, String type, String index){ + this(name, type, null, index); + } + + public Column(String name, String type, String rawType, String index){ this.name = name; this.type = type; + this.rawType = rawType; this.index = index; } public String getName() { @@ -98,6 +108,14 @@ public String getIndex() { public void setIndex(String index) { this.index = index; } + + public String getRawType() { + return rawType; + } + + public void setRawType(String rawType) { + this.rawType = rawType; + } } /** diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/mappings/HiveDataxParamsMapping.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/mappings/HiveDataxParamsMapping.java index 8851632d6..ca3e36cff 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/mappings/HiveDataxParamsMapping.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/builder/transform/mappings/HiveDataxParamsMapping.java @@ -321,6 +321,10 @@ protected Consumer srcColumnMappingFunc() { Type t = FIELD_MAP.get(type.toUpperCase().replaceAll("[(<（][\\s\\S]+", "")); if (null != t){ columnDefine.setType(t.toString()); + if (t == Type.OBJECT){ + // Set the raw column type + columnDefine.setRawType(type); + } } else { columnDefine.setType(Type.STRING.toString()); } From fad80319ad26537402f170b6a6cedec52f0cc39a Mon Sep 17 00:00:00 2001 From: jefftlin Date: Tue, 11 Jun 2024 20:13:47 +0800 Subject: [PATCH 58/60] Upgrade sql and update ExchangisJobDssAppConnRestfulApi --- db/1.1.3/exchangis_ddl.sql | 1 + db/exchangis_ddl.sql | 4 +-- db/exchangis_dml.sql | 6 ++--- .../exchangis/job/vo/ExchangisJobVo.java | 1 - .../ExchangisJobDssAppConnRestfulApi.java | 25 ++++++++++--------- .../src/main/resources/init.sql | 24 +++++++++--------- 6 files changed, 31 insertions(+), 30 deletions(-) create mode 100644 db/1.1.3/exchangis_ddl.sql diff --git a/db/1.1.3/exchangis_ddl.sql b/db/1.1.3/exchangis_ddl.sql new file mode 100644 index 000000000..a503dba4e --- /dev/null +++ b/db/1.1.3/exchangis_ddl.sql @@ -0,0 +1 @@ +ALTER TABLE exchangis_launchable_task CHANGE linkis_job_content linkis_job_content mediumtext NULL; \ No newline at end of file diff --git a/db/exchangis_ddl.sql b/db/exchangis_ddl.sql index c04796624..28ce7ac9d 100644 --- a/db/exchangis_ddl.sql +++ b/db/exchangis_ddl.sql @@ -92,7 +92,7 @@ CREATE TABLE `exchangis_project_user` ( `priv` int(20) DEFAULT NULL, `last_update_time` datetime DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), - UNIQUE KEY `exchangis_project_user_un` (`project_id`) + UNIQUE KEY `exchangis_project_user_un` (`project_id`,`priv_user`,`priv`) ) ENGINE=InnoDB AUTO_INCREMENT=844 DEFAULT CHARSET=utf8 COLLATE=utf8_bin ROW_FORMAT=COMPACT; -- exchangis_launchable_task definition @@ -106,7 +106,7 @@ CREATE TABLE `exchangis_launchable_task` ( `engine_type` varchar(45) DEFAULT '', `execute_user` varchar(50) DEFAULT '', `linkis_job_name` varchar(100) NOT NULL, - `linkis_job_content` text NOT NULL, + `linkis_job_content` mediumtext NOT NULL, `linkis_params` text DEFAULT NULL, `linkis_source` varchar(64) DEFAULT NULL, `labels` varchar(64) DEFAULT NULL, diff --git a/db/exchangis_dml.sql b/db/exchangis_dml.sql index e4ed15df6..967381fa0 100644 --- a/db/exchangis_dml.sql +++ b/db/exchangis_dml.sql @@ -54,8 +54,8 @@ INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_directio ('writeMode','写入方式','SQOOP-SINK','HIVE','OPTION','writeMode','写入方式(OVERWRITE只对TEXT类型表生效)','',1,'OPTION','["OVERWRITE","APPEND"]','OVERWRITE','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) ,('partition','分区信息','SINK','HIVE','MAP','partition','分区信息(文本)','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','分区信息过长',0,0,'/api/rest_j/v1/dss/exchangis/main/datasources/render/partition/element/map',1,'',2,'',1,NULL) ,('partition','分区信息','SOURCE','HIVE','MAP','partition','分区信息(文本)','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','分区信息过长',0,0,'/api/rest_j/v1/dss/exchangis/main/datasources/render/partition/element/map',1,'',2,'',1,NULL); -,('transferMode','传输方式','DATAX-SOURCE','HIVE','OPTION','transferMode','传输方式','',1,'OPTION','["二进制","记录"]','二进制','','','该传输方式不可用',0,0,'',1,'',1,'',1,NULL) -,('nullFormat','空值字符','DATAX-SOURCE','HIVE','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,49) +,('transferMode','传输方式','DATAX-SOURCE','HIVE','OPTION','transferMode','传输方式','',1,'OPTION','["记录"]','二进制','','','该传输方式不可用',0,0,'',1,'',1,'',1,NULL) +,('nullFormat','空值字符','DATAX-SOURCE','HIVE','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,48) ,('writeMode','写入方式','DATAX-SINK','HIVE','OPTION','writeMode','写入方式(OVERWRITE只对TEXT类型表生效)','',1,'OPTION','["append","truncate"]','append','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) ,('nullFormat','空值字符','DATAX-SINK','HIVE','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,49); @@ -73,7 +73,7 @@ INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_directio ,('batchSize','批量大小','DATAX-SINK','MONGODB','INPUT','batchSize','批量大小','',0,'NUMBER','','','REGEX','^[1-9]\\d*$','批量大小输入错误',0,0,'',1,'',2,'',1,NULL); INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status,ref_id) VALUES -('writeMode','写入方式','DATAX-SINK','STARROCKS','OPTION','writeMode','写入方式','',1,'OPTION','["insert"]','insert','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) +('writeMode','写入方式','DATAX-SINK','STARROCKS','OPTION','writeMode','写入方式','',1,'OPTION','["upsert"]','upsert','','','写入方式输入错误',0,0,'',1,'',1,'',1,NULL) ,('batchSize','批量字节数大小','DATAX-SINK','STARROCKS','INPUT','maxBatchSize','批量字节数大小','',0,'NUMBER','','','REGEX','^[1-9]\\d*$','批量大小输入错误',0,0,'',1,'',2,'',1,NULL); -- engine_settings records diff --git a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/vo/ExchangisJobVo.java b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/vo/ExchangisJobVo.java index 6bc16b717..bfbf1d957 100644 --- a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/vo/ExchangisJobVo.java +++ b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/vo/ExchangisJobVo.java @@ -24,7 +24,6 @@ public class ExchangisJobVo { /** * Project id */ - @NotNull(groups = InsertGroup.class, message = "Project id cannot be null (工程ID不能为空)") private Long projectId; /** diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/external/ExchangisJobDssAppConnRestfulApi.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/external/ExchangisJobDssAppConnRestfulApi.java index 9b91a5814..72ae0dc09 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/external/ExchangisJobDssAppConnRestfulApi.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/external/ExchangisJobDssAppConnRestfulApi.java @@ -82,10 +82,10 @@ public Message createJob( } catch (Exception e){ String message = "Fail to create dss job: " + exchangisJobVo.getJobName() +" (创建DSS任务失败)"; LOG.error(message, e); - response = Message.error(message); + return Message.error(message); } assert id != null; - AuditLogUtils.printLog(oringinUser, loginUser, TargetTypeEnum.JOB, id.toString(), "Job name is: " + exchangisJobVo.getJobName(), OperateTypeEnum.CREATE, request); + AuditLogUtils.printLog(oringinUser, loginUser, TargetTypeEnum.JOB, String.valueOf(id), "Job name is: " + exchangisJobVo.getJobName(), OperateTypeEnum.CREATE, request); return response; } @@ -115,9 +115,9 @@ public Message deleteJob(@PathVariable("id") Long id, HttpServletRequest request } catch (Exception e){ String message = "Fail to delete dss job [ id: " + id + "] (删除DSS任务失败)"; LOG.error(message, e); - response = Message.error(message); + return Message.error(message); } - AuditLogUtils.printLog(oringinUser, loginUser, TargetTypeEnum.JOB, id.toString(), "Job", OperateTypeEnum.DELETE, request); + AuditLogUtils.printLog(oringinUser, loginUser, TargetTypeEnum.JOB, String.valueOf(id), "Job", OperateTypeEnum.DELETE, request); return response; } @@ -155,9 +155,9 @@ public Message updateJob(@PathVariable("id") Long id, } catch (Exception e){ String message = "Fail to update dss job: " + exchangisJobVo.getJobName() +" (更新DSS任务失败)"; LOG.error(message, e); - response = Message.error(message); + return Message.error(message); } - AuditLogUtils.printLog(oringinUser, loginUser, TargetTypeEnum.JOB, id.toString(), "Job name is: " + exchangisJobVo.getJobName(), OperateTypeEnum.UPDATE, request); + AuditLogUtils.printLog(oringinUser, loginUser, TargetTypeEnum.JOB, String.valueOf(id), "Job name is: " + exchangisJobVo.getJobName(), OperateTypeEnum.UPDATE, request); return response; } @@ -178,7 +178,7 @@ public Message executeJob(@PathVariable("id") Long id, HttpServletRequest reques String submitUser = params.get("submitUser").toString(); String oringinUser = SecurityFilter.getLoginUsername(request); String loginUser = UserUtils.getLoginUser(request); - Message result = Message.ok(); + Message response = Message.ok(); ExchangisJobInfo jobInfo = null; LOG.info("wds execute user: {}", loginUser); try { @@ -203,7 +203,7 @@ public Message executeJob(@PathVariable("id") Long id, HttpServletRequest reques // Send to execute service String jobExecutionId = executeService.executeJob(jobInfo, StringUtils.isNotBlank(jobInfo.getExecuteUser()) ? jobInfo.getExecuteUser() : loginUser); - result.data("jobExecutionId", jobExecutionId); + response.data("jobExecutionId", jobExecutionId); LOG.info("Prepare to get job status"); /*while (true) { @@ -223,15 +223,16 @@ public Message executeJob(@PathVariable("id") Long id, HttpServletRequest reques String message; if (Objects.nonNull(jobInfo)) { message = "Error occur while executing job: [id: " + jobInfo.getId() + " name: " + jobInfo.getName() + "]"; - result = Message.error(message + "(执行任务出错), reason: " + e.getMessage()); + response = Message.error(message + "(执行任务出错), reason: " + e.getMessage()); } else { message = "Error to get the job detail (获取任务信息出错)"; - result = Message.error(message); + response = Message.error(message); } LOG.error(message, e); + return response; } assert jobInfo != null; - AuditLogUtils.printLog(oringinUser, loginUser, TargetTypeEnum.JOB, id.toString(), "Execute task is: " + jobInfo.getName(), OperateTypeEnum.EXECUTE, request); - return result; + AuditLogUtils.printLog(oringinUser, loginUser, TargetTypeEnum.JOB, String.valueOf(id), "Execute task is: " + jobInfo.getName(), OperateTypeEnum.EXECUTE, request); + return response; } } diff --git a/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql b/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql index f49c56da8..0b98053c0 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql +++ b/exchangis-plugins/exchangis-appconn/src/main/resources/init.sql @@ -26,17 +26,17 @@ insert into `dss_workflow_node` (`name`, `appconn_name`, `node_type`, `jump_type values('datax','exchangis','linkis.appconn.exchangis.datax',1,'1','1','0','1','icons/datax.icon'); -- 节点组表dss_workflow_node_to_group -INSERT INTO `dss_workflow_node_to_group`(`node_id`,`group_id`) values ((select id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.sqoop'), (select id from `dss_workflow_node_group` where `name` = '数据交换')); -INSERT INTO `dss_workflow_node_to_group`(`node_id`,`group_id`) values ((select id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.datax'), (select id from `dss_workflow_node_group` where `name` = '数据交换')); +INSERT INTO `dss_workflow_node_to_group`(`node_id`,`group_id`) values ((select id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.sqoop' limit 1), (select id from `dss_workflow_node_group` where `name` = '数据交换')); +INSERT INTO `dss_workflow_node_to_group`(`node_id`,`group_id`) values ((select id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.datax' limit 1), (select id from `dss_workflow_node_group` where `name` = '数据交换')); -- 节点UI表dss_workflow_node_to_ui -INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values ((select id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.sqoop'), 1); -INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values ((select id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.sqoop'), 2); -INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values ((select id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.sqoop'), 3); -INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values ((select id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.sqoop'), 4); -INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values ((select id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.sqoop'), 5); -INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values ((select id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.datax'), 1); -INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values ((select id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.datax'), 2); -INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values ((select id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.datax'), 3); -INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values ((select id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.datax'), 4); -INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values ((select id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.datax'), 5); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values ((select id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.sqoop' limit 1), 1); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values ((select id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.sqoop' limit 1), 2); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values ((select id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.sqoop' limit 1), 3); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values ((select id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.sqoop' limit 1), 4); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values ((select id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.sqoop' limit 1), 5); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values ((select id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.datax' limit 1), 1); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values ((select id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.datax' limit 1), 2); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values ((select id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.datax' limit 1), 3); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values ((select id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.datax' limit 1), 4); +INSERT INTO `dss_workflow_node_to_ui`(`workflow_node_id`,`ui_id`) values ((select id from `dss_workflow_node` where `node_type` = 'linkis.appconn.exchangis.datax' limit 1), 5); From b7efd218d930a1660b8836d17467b6c40f9dd61f Mon Sep 17 00:00:00 2001 From: jefftlin Date: Wed, 12 Jun 2024 11:05:24 +0800 Subject: [PATCH 59/60] Remove sensitive ip --- .../exchangis/job/server/builder/JobBuilderTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exchangis-job/exchangis-job-server/src/main/test/com/webank/wedatasphere/exchangis/job/server/builder/JobBuilderTest.java b/exchangis-job/exchangis-job-server/src/main/test/com/webank/wedatasphere/exchangis/job/server/builder/JobBuilderTest.java index e2472d34e..5a30a1231 100644 --- a/exchangis-job/exchangis-job-server/src/main/test/com/webank/wedatasphere/exchangis/job/server/builder/JobBuilderTest.java +++ b/exchangis-job/exchangis-job-server/src/main/test/com/webank/wedatasphere/exchangis/job/server/builder/JobBuilderTest.java @@ -57,7 +57,7 @@ public static void main(String[] args) throws ExchangisJobException, JsonProcess " \"preSql\": [], \n" + " \"connection\": [\n" + " {\n" + - " \"jdbcUrl\":\"jdbc:mysql://172.24.2.61:3306/test\", \n" + + " \"jdbcUrl\":\"jdbc:mysql://127.0.0.1:3306/test\", \n" + " \"table\": [\"testtab\"]\n" + " }\n" + " ]\n" + From 4c14bdf64a985ee873ac5272bfcdf45050a89300 Mon Sep 17 00:00:00 2001 From: jefftlin Date: Thu, 21 Mar 2024 15:57:28 +0800 Subject: [PATCH 60/60] Update picture of communication --- README-ZH.md | 2 +- README.md | 2 +- images/en_US/ch1/code.png | Bin 0 -> 30826 bytes images/zh_CN/ch1/code.png | Bin 0 -> 30826 bytes 4 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 images/en_US/ch1/code.png create mode 100644 images/zh_CN/ch1/code.png diff --git a/README-ZH.md b/README-ZH.md index 2264648a1..ffdc9911f 100644 --- a/README-ZH.md +++ b/README-ZH.md @@ -63,7 +63,7 @@ Exchangis 抽象了一套统一的数据源和同步作业定义插件，允许 如果您想得到最快的响应，请给我们提 issue，或者扫码进群： -![communication](images/zh_CN/ch1/communication.png) +![communication](images/zh_CN/ch1/code.png) ## License diff --git a/README.md b/README.md index 889504b5a..5e211e77a 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ With the help of [Linkis](https://github.com/apache/incubator-linkis) computing If you want to get the fastest response, please mention issue to us, or scan the code into the group ： -![communication](images/en_US/ch1/communication.png) +![communication](images/en_US/ch1/code.png) ## License diff --git a/images/en_US/ch1/code.png b/images/en_US/ch1/code.png new file mode 100644 index 0000000000000000000000000000000000000000..cec5ef68ce12f04462fc6ff85d2d2175c6691e87 GIT binary patch literal 30826 zcmcG#XE>Yh8$WE;tWmp0C^c#oUll8;RaLa5MnjEQtrgU&5TdPFp=N8OrKPRC_o@*@ zsZt>+QlmyFf|%h+f6s9|@17U`7ylRck>g5oUFUV3*L|PY=kqzwJMpfi=|$GdtTZ$< z7tL-N-KU|U6`@{o%onKN7#J3uQ9o#d@0;GFsT>tqqc#}43@i+2Xljsbr>>0DHVgEY zLof{uNAJIj79H@(orWgt+|0+FQo5VWeh&=zc#Vty8Q|J2A zkYmre{lCF;y};d+7V^W$O$zqhu7b1k;cR~Jh{~Vb`L{l(4l+Dm6*7<_1b*%S^&Dw_ zwa}NyQ`%yg;-{RAV)PpD6El>V`yS+S$3dmU7w~c#6n%P#$P9#wR(F8#+%*6f{(PY( zhrS{HH?H0-Dt;#R{;NiQ61YtD+FcQ=hKs(Dx4;|kA63ro0_}vHF#g}=dw;E z=G9}hD^o=ICiaHSatptogi~Jp)R=Fh%Zz#zvo6W`ro3jqtQtG+hcuWa@jD-6OYHYptU zsKY#N+TS2woNuYr59D|zLJTiZB!ZJAxB|9AJ`Q< z`7Py(rNM?iToFxR#!RgQwC@bO*c>}Q9aDBb?RSQm!2d>!MHXHQA@?1Rc!7dVX5LHP zXrworY(4KLN~z%=Z2BdMCzn>As{;iV6V795#x`!b2OT@*+OoWc!qPxHd zVK~2U8ee*B8iA`Vxa>HQJvd4``ZF%4fL zaI+1vNwp+H?s(Zb>9Np~tM+b8Z_})&b5W-57+-Lh-e2~j^;7{UNxcdEW-KpOJ~nq> zp;=k)!Hm+THa&!HrypaEy%idZ3E!;A-p~hE7 zNZCV@>OHj8@60i$TDVtXSDvpdYIg60@`!xy*lhkPl)PFBE#KkXJHp0OBV&)yCSdt7 zCiDqZ(fnce5%6AJQ1<7lKph(>@7oZT)big)g4Wlk=A>IVQg*61>j-za@;4cdr{AU= zFFqGqpVtyRq2Gkuh7Zag)%roMp_T{bvHuFym^>#}j=GJ2^|FR9f<1)vu`pV5myU*i9?-E6^}+Ovpp zIaAx>CC7s1Y#B<)!UINxS-D3vQf6v-j*!TvwpqV>g}}Xj6j8lvHM!IKUQX!r?aG>0 zJbed);m!$=?)}zppvH|MIi8?|j;`~A^D{T1J-7C1xQAc;bi4{;u?>2f&8_Fr)9{(0 z({#021rXX!_(k~}rstHSkypTAJ(Z(M6UrTe6Ld^@)Yfc;!FuGN|WZ=M(9_AP&q7Y%lrlETj+R z%Q>77*1ivAz8sS89r)G470?_F<`%)2th0S^dj7oeNo1PYVqba`gpk7}$#{cHQ<}j; zt!GE$(H+~gh8fd)_zY_XuY?Y73{D)te1`)X3EAQi!c0=c($BmZAbejZ@M$McC88Cr+NR?Gz8kiX;CaGU0UMGNzg*0;*MF#$4 zc3|%YcXL3ci9|RvNDW8|Jsg4w}(!KH7~= z_+1z*6EQWG8#{FC<`1zj82+OU{u$yfXm0kXrdq{&CWFYIk1Gt_J6BIpkr-omUzD56H2^Ccd5;-w%oR&wn^$N^A}*P&J&6r5AA@ zw+8A9#rra&q?t5C04xJVjb4B{mKF|gWCa-}@(O7~ytyY=JErI6fF$xhQ%jm2N6>$Q zKntJ_!jS3skw?_^R-dU|wJLnR`l9T1e54M*QcI$}L}@QpimwNN9W`mycfJQ15s|Y-^_B&Nz z1k|9P2t{T)2n@182D96THjDv_U1AjoEr7e|mFMmx2FjIV#*Doug6i+#!7s)erLtjv<1XsO{P|VFAd_nDn;|j@_a*oH zvGYNn1MtxovSua%0`5#Y@%{0(@QrnVT82Q&UDqe+(i&7=Mh$LwAn0#Z=Mq4m$w8`# z=`iDkFVCO9lFxoCr{QMOp{8@rX1bOyx&^{+s&oH`}1n4^=8cSL!-*nH@`& zK()^QoBio)<)haVl>Y3R<;!>rW{!;9p{;OQ$_1SBi$ifu+dHdW8h{NivWR?Yq5%HT z8+4(Z*H1F2oUkg8vLadB^*8uY*3?>dJKG zHQ=dQtEM`?HA8&#gv>4B?~o`&_Aao{t4!B7gN^FCCx$2IwteLz8lE|JE13Gj-~GpEQhW)A%StVuhV;QINaNajzD z(II+(?QjLP>vFl1m2Yo7lfGjM7wt1W{S@_I$v#6MUF}Dd^sWu-`G|^gwI`**EZ>EA z`E0uiS`;nX3+GfV^7Q{&RMZvMrZk?FqP>tI8Mcvatq6~Wk%M^4U`{R+T~cQuMP zZyp<8g5Y;If=5*$Z+ch~0;DQH{z({E3eWSQFN4fvZd6E`PG9Px_KQOVXS6KGDNQcA z%$pM>mt1wdvF=*qLumP8$R@YBZ~cZtlRxLbapei1gO;%(KeO}t6}>ua*f%+X3@gLg z9SR+e%Q0d(s}zECmR33mL`<3qpHFw~t38xfm6xGKRC5UG;fxu!E6%>qeHYWa_x`#{ zC)O!!)(x&uL@=5kbgDM%3EUtk0O_gGLW~wtrUzj}57}f$)%)=zv`K*f=If6Stt&&= z6n0zwCLy9HL$Jp90jthPL_nYipA54+edE9z#bt}9cFbvu;i*x>W>pTU!6Lh|aj&NA z1P3mC4~SY!XVLz1alug2cpxtxRGC!4pFe1S!MSGKtEy7D)Qvx%?|ZGOo`&nz6oThD zgL=)rsDimYQBe3zK|t1vX(nQ-T(ox_2xBpmjF|9JPL>HAa+?rn8nlP==Tm``Cz4S6 z6BQyoW{YoYAPiFnm*9E+Y)WO^Beb*1R}o75S|I@(@5=5_D4Z;V&}#}TW@qiRW2vyA z8{|%ARsvHTyhy$c9bmR-=z8CzX=+dH^N%odqwE6YuS^=FuIV2E%}$J^e0AZYK{f1q z`Q+*v-&589ZHY_S=Iq06)6*m#>v?J~^`n`A8pqH_L3hA;ydr3hlZ}8MbD5^s);&~w z3U+n&a($>VQ#DfPe@m|ZN%mJHNY;dvAw}XmSbS>hLPWq+m}905oj?>voqQeUD96v) z^n8Z&f$HbecUnV=bbS@-assd2xB>)UJO>b zZZ@7)j-h=FW!yjbRnXx|EpQ2bSPEvE>3ItIiT~}p@ou+tmTg856M>><`z#zEoI`4# zDP`j=y;(*xS&C30Fc(ot%uo4L3o9;aXUaF6Uk&FU?{5xJtr3J82ZjM1YEM)on1&}n|0Gq zbtI|by_w^@h~TZ~6DHKe(|8{$zG?cbv$1K7<>txweEBT@$#u3-1t42xuO40|q8`=I zmV#%Y23Pf(tQq_s!+XkVz?6S-6cu412vj`RJ#96Y==!3E#{VJITe%b4*QQzjEfP3Q zHh;8d`okY(F@2M)#J6?aH~&77pk+@+Z=YN@fx;w$-ss{!ttDq1jy3*~W{?@+5md(g zaRMIq3;t*r{Ys=xpU2hE@YPZC&+)`CLy5-697EtPuKF=dV3PRW{MaR73L3fppKj@xlg`Pg& z$y5m|BFtX?piGYU`y-s6l*JfNMhuI5ly-YU`-j@*iO;5`mK$ z2At!Ak4B4~J$XF6DGMkb9%tIlKn=2&T7#+Mdh-y7ojbpVoWGHtAc=g5D~=Yy{4;EI zOJ}xgMR%)37JW#IhW^~)X&~skEr9lk;eL$!v$-dAqSPnGqetSXk)9IgS5xKNG7vh8 z=hCBv%LefJ=7<| z-6);1ZE@*VBavSOCn5LAt5(%zulJdhG|ycyD~EXE8raXuvwEw+Vp#s^T-2NGZpg8L z=CJ!lFIUfaVXhmJm?F8qXhgMFzwlVwSG4zXSqkb?fB(?8VLHHK5^>56mHYKGP?DZm zHO}naOesABf@bT#A7OOQ0o>M%@zeI`>FO#AgZqXX5Vp6gQjhBS-t_dXJoB_}t4Gq@ zorCWaEax;{lGT22n%DG5z3cLobH>LFgLv+U!vagAH$;#4#GFoYBw8v zU-xAPdhoawq$m=YuxmVlPZ{sxlR=g1D191l{8A&5$WK%A05PEvt^LY6(1dNDVeJHR zjxUgAO2|m_ae2@qVZvI+BIlh)zGaV3-K?{EjVwqDbX3jL&#Uj58fyef|9)j|vHo+f z*HDbUVe{Hct(c(9bkGl_-Ks{$&k*?MC*T~4*U>K7YG3VnZHW1Y4aV7>M=-$i}zg!dU0 zjRH|33n=V~ea;IfZb)>q$Qx-I)m)Ck&wf~$W^?GFIi1etX#cvT3*u+g6e`;AC~gx zC&-a)4Rmu}B5I2K^Lg;J!rUU4evWD8=3A013?doqBgdt&7Gh&n*MR%;-$M1lh+fvX zL?M}#j=v#QYJa!WA@69a8~lJK&|1*lc;z_SE=l?Y_fmHG_vL4;f)KfdL+;3FERucy zAiA3S4a+ERB;-AE=3+4Wr!trRBdCv?B!b??M$ALd@j?J=&ilqvLM+hRkWq$6{{du% zmI}Md*nHthk1NJ$Qd>@*X}J2EJ)S5DqG2?BFC+kC~h%Y^%H;r%9CKAF@?EymP3nT1vD)!aD5KTT| zhu_c@$wT%}L)uwU-^F;^8|~mzZreF5mcGT(a6N5A-+!9L1X-H&_E-4$)>Ez@(kf6= z3U@yHoMZk>MR%mtnp zsgw+H)>Jd!JX|qoS#64ZoPesNxq@c4oMMWxs_{EbqCzn{y}T<4C}CsSBv{D+#MTxt zrBZi(xRe{-o773So-9WmP-WGgB;U0!Z2d!nADObZ?8h-O8JS)z{Kr-$>P(^2*Epo; z1+M`W?-w@unodXaiZrT3ykyq*3}!F>CP6;_p7i*JM3a22U_(NHH3fqc#e}>Aj(O3yhwY}rj^M+AokqWMYwN2$q>Ifr<3~H#_9?(Fn(PRkrGBdSS7PC)d9LIv*Asqlu&SOA_E3k#J2l~z_ z%VRpbK^~rE{Znn=kghBH1;-VdHA?Gkt3|`nKO${=0p_wFXV%lsh6JsEuvUV1}~^8Kx}N zKJT`7ma^!je~hk=nM9HYbI;FRIBxXsiQP}mg1p&}-5uBCI_OYQoq>S&cLY5HO1+}u z9I^bt&5^U`>^0^Ox@X76=$F=d8~;Vp@xf=;OyJkHwAHCL=-@wzk}(;`Zc|z&@7c8Pnf6 z@a4|XYg9RdVX*7t#_XHe${9o$6Z~yYS6l^T=dx6_lBN)Q{#Al{9Y~)Gui-5n@3BpP zHEchw`3I-37kS{TnOBhuD!L!QjNN+1$e+INkumamxbvFQS!I$+zv;WNyh|cYQ^k~L zr#Ur^oR8IxnTp|#akv#PONHLhjjEMQy?cYW7B(LPt%w!C?8^w@lTawCqmw^Fa$ShSIQ&Tl(eTbg{U1jp~lH~AA3 zUDy9IbQOiXA|zXvl`B zm@KjJ@X`v=|KSa(f1Foim5!@%q`D`0Kp54Kx6fwTPtBK*4qbgH2@UMB&)Bc^N!akj z+uYldzW@Hy7zV_ZAnW)&xO)t_Hw3}-KMUgft`t&Z-nPT>m~=_lZ4yClO4_TFHXF<) zR?@W-g}fllH?9J|vk)YI+Pg=2AEBo}gIa)=&f9u4``?0bpHHr~q|Rr4tfJn*EU!@x zs3n_x^}2PJE5Ll=+hV=91hwMn8M37wSV)GvCW*+~G-O<_V=UI)@E!|Ya7w32XP_D@ zp$m0?1x9@L)5+66r1z;Bym;U=PG-=(f{1AaM14?%Fh%Sos(>{%^%_^@^_dqe5_i1& zd8f{9+Oa!gP4eA0=!a8{{Qy%%SJY7NkBps$_jPgm5_LYP>K>Kg?qH&{ANC0G4L1+^ z=Nd!0mP<5{9Ar$Y&|T^Lp7OO>wo56UG<&hcsR=P!55V;vzRNwJkcC>!pa*xcXWu!W zn*d(c`@NC)@J-ge5YM~Mq@C0#Um%E|M!k)4xoJ{UJIdRW*!Z!`H$g86{70?695H?% zwPHtnd%VY7 z+{SvAQ`sEXpkXAAe!U0xrHNqAGGuq>p+QZ?%V+~TXhQhZg^}YQrnutS-0HldGZ#*y zbd}B+qJy*RAGO86*$`}jglhUEeHpW*AK-^4Wo)yeJ#O;%&daC)e)ni9sYXCXZ*D*{ zQlgw>I(qWr4;MpaOBdLCgzD&4A$LGVBVH$$?-8a;{e1{N!s$3jneOWhiZ}h_%0v0) zkx|tjW6YPhzyx(Wa0eAh+beo8tM`0~>Hmn7x|DcxM;Bo*@G3 zRaTmlaqKOKxu&gY;0v2N&IUpsUvb%X$Wd-=<2M1{1)iuum_IcNCvauZl{zWm(M!D+Nst3GzBcu?AMevn^mc3 z5|35Mu0noedGsT0O-jc?+>}1&4GEolzRd=jVnRM}*VA>H%_*MXBr5oLuc7ao7qDUK z?1K|HEwWS(6F2*~9~Q0MgeRrI==O2XNiBp%l^$WE@=KNzVo zb2?7j_fvR`sc`!$$+&Re>at8Z`WNMdi^%cbPrH!Fu*q>`C;3m&zHUGiG@?q*<6_`N z^FpmcBF)ZC7sqj+Mn30sW&?cZ;kVDt4;axzUl#V!TDQb7_BS&K2J;>Xi8T#BV|?Is zQJU_fDdNR96`&ucFl?m?jqg5pBkZ}*DumZEYWf2~|594#CCFAn^;uz3$n+Ip6yVF$ z8Tno$*IpF-G5hL#?p8lR28?+{!EY-@W`+U8FZ~(O(jPq(D$!)F<~g~0f<3TN<7@gt zn1st*Za_ygs;hz1lquZ$cvthZ#o6nYw!=l_Ioef8O2wO*b;-Ect>Zhw<9hGz*2A8M z=!yINz)pvCt9OU^PQ;f7K+M49o~PvqLG+knvx>e^!tSHp%2#6zoT|Gm#+y68L4+Eh zPq4xoV8H_}0dlX2Nmcydd}pw=Dkw$QSVTBR6FZFN2jBH(h`O#Z1n_+(`I%zK>(jMl z^Xm(OKKJ=ATG_IWlsAfl>ITBB7Y!k=4 zf>sFjDp0y8g#!6;{r7ZLlcj9h5p4@7v)(*!PcD^c!-^BD{hA2s)sjw!Y#1OHuj%*{ z_p>k2?+VDeea%z8@ofgzMU5~O*MOOjE0?|J?mP911vJy)Tp79S{-X&&rxN_>7rj90 zs^F**2AShhq>kAB(KKCD!~EA9HD-BKTyu;{dE){j6Lxinh9FtXExQicKkovH3Owp9 z$Oci^Lb{*Mu)?BeapDj3@v-Nk|D0#TZpyrWF;mYC-8NSW~|`ncNyQg;HbQANQu=VZl5 zNP2@sTi&IZl)U;T)~U^T`%y$4h$?>l=;8;H2d?&Il)$`{8CYY21H(=y*`=e?Abz zpTF+;h&`NL_L#T<7Q2}RjThRydoZ;Q{!`P@lA~b zN>%fuw0Ir&d!7aV|CXL?KVa(i;Rk^vZsy%~+2F2ofR37YRvF_EcuVL4_*46~G5U#8 zoJTlQJo&JiTPAAk0(ypHS=D8%=~Ios$+Z>}0BkoGw9R351E{Fwy|KF>jJzbpe6lx@ zqrTY3+i8Pa(GH1SU|3+wYtQu(YSI=xzTH03actr3_UA+|Eyrw8+qv%wL^nFm^G3;D z`st^6Q~E=ZdAnr>}{oCK9zF^L>8I40-};{2#536}`n&ElsoX`Pn? z0-DYmn^~SulvTg*M)rWoCMtC6-BqMet&r2+$X{~d2ANSzHU#CwKK{soY|wzf@{=F% zsD#8dXTypfclAL1JE;;M;D&F0Iy3ngp+*e)UxxGeEPl(CT@qT~!T+~tfBS@kTGmj* z=-31*a)~^g$(hJb{;z7tI_N*v28NV1+r{!XBl>Fh1ov7q(iPM69W^9=-ONq^CSJrL zhuTvVjUp>=)MKxBT&Xi4N{TrxCq-n3AJ)Qb+ObM@zTeg|#7F3nUy zzYMxI>K&W-j-)z$M=;x&q12OYvJ|4kX?7uV>O=GIm%^4=B4HZ-?>&@>YMsqmuFlJ2 z7s`!oWx8XfJj+3j9y_}k^+Iq`OvMs)qn_6&nDzWPoEeKI7g@sUwYJ)Khp*;1)*co> z*14Hpz}P>lWjato($4B$yEY_2J1Lt1&WF7bGV&*M@9(7~=++Dm2v03GJ^nHFHb(Ew zX7k%J)w3cAeZdjs(R?AG|P??7bhj)_+&SitG&WGU#u*@E(!Ti~O^-unsd(3_r% z7wc4hNr9_TOK}ik77^kqO7BcF_mYV02K<6e{(G7)U&1)G;^=*S z8udwiEc9=d6abIe(%OkP>sLPbEAoi?X1=ZH)VyoBrq%$Kk*d~>`eT2k6?5TLYi7TU z5%+mJ2C|4?Q?lPcCx(M*&h%k6;o$Q!Xe6N@xOiy|eRT653{^|YHpm!fRCim3j*~Smx1^uf`7g|g_ z*?x#^UUFyR1T#Qc)ld?fUWmm&%Cw2I6q?-|upQTD^OT0n_1Ugt{ehS6!#l>j7s*W- z1o3j^s5NtelfImM1NR~qo;=^v<6DlDZkuOn*W<-KgwUG)tjv5-(`k>QqucM{H-nfhY>v5_G%oSwvS#Q<1m`vTTjr( z>K$s;#?`AQHb%~pHB@Qrd67ev(Mu5PIOqciv<5)UuR>51m4u?6v(QT-&~ak{MVR^p zXR?QLqy4z};GeN+`$K~#j%Kw0yN`W^p+fO0)g2!H<%r){{6Z}xtNX6S>2ZParQZ#} zHrvi8Mwz*#8!xgL(S*q*HRdh;U!qUxxNF7eF$KRM`Cfp&EGwgW?xAe*n=WjWg0D!stLe&j2sh0 zo>18MS_B!H!1c>2gSZ#gfiDTot<}9+ZI89Y-xUrPanwt>virh*i5u+n{v5D(Zxp9U zZC6B-Y42UWrn5I2*ew=v_&PshEcmZ7z46Oh#@34`!{t1OF9^+?Vu#&KN&_)N@Ab=B z*N>SuUfoj>52$Tc0)O<>oG5AxA2FfXler)GZM~O&knMQ=8ouQ>Ea8SXn&N>xwGJHc z)=ZrKagQ#hm;JBwpm=d5M-g@34_j!$#v^LpEA2_r41JOLn$;CCsUbmx ztx1M1i0nEiEEE^4)E!@^r__xJ4I2C6L1krvah(y=kj=so=FYCFve7M_Sf@1?uP}Xm zs(8%uzI4$EwIGPG#+1a#&Edu(2QZFpO)|DnVdDrL<&t1o#+t63^xHi*3M(7+`8B9!M}p{5 zrUh3Y%$0~4W{Q0D0BA>@FM+XHle_X;dwDn3J(QF5Y#Y?5>z-aSZ@v7>Z|Uv`BCep0 zaRqSJ3vX@~))um(jP%%Dkb)3DcrCDy5_3CpMUOM3ItBhZ znL7YcIF*Fdrn}z5!8%EzTbId48hmTDSER3pai(e*bhF09T#r2g;Rg{HFY;ddHakRx z>!Ms-gqpH~5&jQ;#~>qUySYitd$G+lU!i`k9?igV{8#Im5ki*nQLGMI{(7@0L%aaz z8zA;khoxCQNYCduPaq`t=Hoy#ZFPs zvb2_;8phwI-2>yS$InITUfpGSCd9%K8@&A^aWbyl;#OY0x)D@gVMR6PMs4G7w;+Jt zy;Kpc_IGGz40sG~t{J+t>V^t_QQ{gV%g311Ey=4rUh%vYkuGD&kZn0_nJ;y6Xu7N3 z=4n}r*NbqoXSX$$2L6hYaH}L-pK?&Z8q0jlXykOD@IPh?3T8O=9%UcS5u5HDp^}1s z>2Uoi=?@k-96w^hPwH>aLtpnMj)!|EzM+(|@cRkPQ=&gY36JxVYZq-_K@0$%RH1jp z*5D5e5WtD3gR)4hHr}sEMD(70JvSD8EINwvU6E?ZPPSB_s>7+}(aH6UUOh7}2lKZU zHw~ctyPcBY&(o8R%#2QfT?=au?qCNVt7y({e-4^-t{*SRW0Wzr?V`fqI)bmE`{=<= zqY`lMJzhIb=f+Nig$2m{Q9_@}^!`e?dsr%&Pvp6=ibH~Bla*-1m$|hET>Yj{_U*ub z+PXjAzs}?d&r_1(UGyL#- z(ds>;l8ku6_@h#+>9>zBJRh(X@ zHwr%ITtAZ@psfg!)Vk`MZ?`uczX8yEDOIQ=ZK)2{m*o@bV(k_#G_U zZJ{C*AxWtA#X3Sci)FqA3rE>owmH&NeXWg^q z&SH-+tQg|vl!(r=pmXTKbiWFs_r@cP{)XzmWW;F2z$S+@{q%E!2x#zHv4Ys=+{2Bn z*1s?Pr^0+v3};)HOJ)nSo+9sP%sGNi60NcqXCY&;^su&8o{YbNtoNE*c=DWPVO^o; zJGE`Xi?e8s5YGDrgHO-3!&3rE3VGvn+YWLtdXau%72xxyWKK~~Yj}zzfFsN}bZcwM z&yxbBHRm1tXadwh*k;x+Lil)^rnxXX$y*gXX6T`~dE7S!~~( zw~anvgx^={SQxI|f?}C$?=ywRxLurlLu6Q)1svN)3RrKr#=I zgT~rHLbpuLe$u_oSJw;>N{vwVdT>_{M}M{yyH&iJv-M{+xDGz+uKV4>+zsv$*7Rhp z5%#y1IDJ;z7P%ME?{oGg@8@jscqGX2(ROob=3-TpVGHT1vZ6-UUw9TN9WyKpUo!jH z{@3CoY`b@vWOq6iw2z;2N}X%2%0CH1U2{I4Mat!bCKrW8!{8BwIKZPv`%}nlt2xKv zt9BxaRqflgUpaN!Plk(^f9su6o~_cBttx<6*f@9C<1>s0MF+POO*F`y zEd&pr?{*-s0cIZzpHzZum5BAL9avF?s$Uk0pQf;2#<7f0!_@c65>#D03xol zHqvu(eDm8%K{AIDGjG0jXifXBsFlB!^PRN*nH)h{$nQ4OKcI8Ef%wH8iuN#6445iJ zY=3bNxS$>4AAwA2-59Q12=2wVbBWJIFX!vgcTYW_ zZAa_=O~?7TZp`zqw1Vi?kp>vCuk^6psig_!2-L|huX{@L9jzZD8LmjAZp>Me9V4A) zLNI@+kwNs#Y3Ky#sQa;~}I~nc@pH$8YIPFBqa-L2Blmr|f4fn$TL@X1}DH9m_vrWjbM?o29 z)P@!HPlcvgD;=cG<=2+RpxON+J<9Pe1`8W2#^^;IE)H{^j?`g{QMd=4oAD^yv#ofF zcf|KF-ICJOdY1kt#UP(@B1M z+s{eXV-c=LrSp_&(mvT;9d7 zvQ?v1LLN7T84?XY#PbMu)4XLwipA0puX zqrk2zc{rsTZu@Abfm2EO@NK}gE=mKZoO1Xy1Y-@~6I#Xtc6gHcFr}^E;k||^d!^>5 z&Jo}ySj2jrv*TGmr%s2?8TPEcy$bwCXB#i(@aV9$OED)@3ue}I<*350vrK1CZcORm z@OVqsF6>Nl47JM7+_k@;ksYvVd-I%90Z`zKUa}rmb3T1ZB85fQ!}hz5;22cF!5Qh@ z7R=_zNF%3f%WO=6Gm=!5Rdx6WaBZIS$t$z!EESyzKQgpZ?m9zPJz^eREZoUk#{D&0 z-m27fJR1trsfDdT;O}84$kY2rb26uHEeprGrFOv6T`fU*pjSUIO#~V6m_9)K*YzRo z;h$jJTOo7iyXvpC{?MPi4qjsSaP<2w)7_qMLPVUo&Ue0|)?6q6EvMeHi-+XHKyI2t zg1j-Jli8>)nqRKD-Dl0uzVX9S-YbEloj>e1^)Jo?OLcCb&nzu5VkfGs+!?)s&^wOJ z+54Q);{ve2mXq1ePp~1;zc!uZXy<#WqU&PkT0OtV*Bh!}9!+0ooaatARS$8NzVbSGf%PRI3*N=PT?XuWFz$EjO#0TF$iSv<MW^;qgZ+OM?Ak~z&;36oFhSZ0DRI4rj)N18`Wu{qmCuN_q>U_;NMi16^+2eMBe ztQ9`zUpdd;=*vevbiOz4k=qeZ%68*(7Xn&j(aK?j%+lU?ad1X7pdd?N9qUXKZ&{bua z4Ac?R>}{gTauL#TLRTrfpFFSoJ7@SjeBuPdHg_0Hs%?V3bF7PmM_?EO3Z#IKQ`%~v zr!TJ14MGN34P43x_0|JN`AiEAYDdA|mVFPpHhYc(lG|P(qlx-^NWpdIlyCnlRQ-2O zO;B}a*Z-*-jM7r{w}pr}jub@PqKI|8qFXP-u3i#M5X$s=M0bl~tQxvJGqu;gZ2u@V z>ZRXIGU&=RM~Jg)(o!(Bl(ijSu=Nns#gi8aI}^xef^VKbAa80Nt$5{HZ?&uzDG49s z=$(J`%JI@&- z_Pkh*FC9#oR50TH*4DosJ1?MDG1M^ML# zcZAzP0y$Skuii85{|IFqHt9QCYA_dPoRc_$UdSuSrkE-CLT2hf6%2|w5eQq;%SZR*& znIpd^LC%lDh9wKP>y{6k&t8(MP0x*vKF%McZ*6v);+1D_2k8*0!;fs`ui(iX%B1RY4Rc>g|E*d#Vf7E-;c*HjX%B@PD zgqZtBN>=JI(fuh3PK)kG0!pqNdns&>Q6%z{8rV3t{r#Qd%5Ppa@@pA>$8Pzv(SuP_x+pqBu_=(`V+{RQ7DSHX3#9(@-Y(Lj;ZBY zmTU`CJo8vRF8=5^@uJX5(SQ9W+^K1-WAD#j9ZBfV!noz(JaT57DKaaHi8E`~Df-cD zR!b%RQRmjJqYcG)jQR;@RvvQ4%}5T=wU$jvkkKpFA<1>wfS~;rR75*oMi!C7K6PKe zKI~i7dbD&9Q@P1L|7&J;A`6dgXlb6?oc}R1j}NqKA~c7dCk7cdIXP|(#t(E}rk&am z;yCK#Zyr{W(KSg^U=j)*Rm^*;;`LF}Lc22P*E2Ddg!60Crz0Lt-@Cjc*(Gh?Xn!Wi zGBmK7k|pN2^BwP)&>_=2M3)`sjeKJ}F9EY~q$H6PIwMUay3$`C^C&s9r|<^16Zx^^ zqNr9)%c7`UWGU%^^L}@~quYOnH)kUPKusdQ4M%514?@enLQJ)GfDg$#ua{$$()~4U zQnxy4MD)H7VPY*Wkws-q+h#_XI@%5!p-nF$TbDeO2RAfB-F6D{g%85QCjN#4s+68? zh7LCXP4c-uJ2{#91EJo5e_v9X6xQdy^F^)}7v7Ejt2st|zG~FcXnnDA_uk5xr6NZt zmNSGYUYm)d@XGn*tF-N_UwfTUsiEYnE6tulxJ1BZH91PzMdoi#dRw~~fnWRnGF_#U z`nqDTe_i;c4Ak}h)Xr4l%SDi;7qzu0)XIby$ z9eYpkUH(Guj#cp!L9)`LI!0*~?Bed1RFy&Y>mIA#u$LEr?gek7s>SfY_cg*N+{Ot} z*U4%kX)|@Rx0tLcWLxqc z-k+QEuk88ysyg_Eoe)Ri>H>zydD7}WjeTX-6uIAk&4i#iL+QRJ_K(MTSlRi9*^a}7^HuhVkJ2^j?wUG&D{YARnY%~meP3n-x5d`R}c^dbrlng`P#Qr~R-Jg+_P zA#o$DwO7lCO0o$!dsp+P1eTvC#S=w?u%OHDwe$Vq?KzasgKpOc+&{h*ui%+D%4dGnP|Ol273 znUabN2>I}{qhwS0PTdjoV0^3g$zT8c@XWP(_ZnEs$BgGuP8V(2lRa+9Fc*NH7LOLR zn=e6_h703PFtLA0KO@Om0@Nhga@NWt$GM%~W0?n0q>`WqrAj&rP)daIGmoQUyY`mT zKd7^ii23EVVYWx)b${bY7RxZoOaB=X9!`E86jCY*TaC1~JCiBg@mXFtaK_DuBU@Qw z2DeL&Z@OrixH?puM~^CwhMoMz-OOzTVG+H~bB=w;=FoOb!Qb}7KOrK;gHz3Ect;t| zg6|^q=fCO4cS?G~wIy0Cb&bAI^L2_p_&->^3m}=G>OJ7#jQL#zNhCO9Sx)MB#X3=A zxx9-@_@4gpv4R5I*d>w42U$#SS~(10TWG>fppW)!o-`Wm$gFg z{6{XH?HqG_II`jK2{4o5V-9Tw&wu2}Z~GN%&}Du^eRb%k6Mrb*V40-M@Qal~I4LXM zpR`kD+y1nriHDov&*6Bbi^fF2YPJ?OY&e*0;YZ@v9ZOCAS)vj^n+oA~yq}iC4u^$@ ztoc{Bn=eI zk$Bu5E(u9bn;pFwei%#MUUV{_{fh;5;0_n= zLb5y&QwmkW-D}=PN^uvShp=acJ>#_vF%}mj;P{Q+sa|##?44&+Se5Ukwi52@gpD zjA=DCwM|+GiaoWreHzu$lJQPhAZVhWpr2C>JH9;^iU|1xUjKhud+%tr-}wLA-ZhJ; zJw9|;q19N4R*f#STBEj@6%~|HVzx$+D6LU@bW&SX%@_%yl&TR$tWu-ImKeXce!utm zopb+rpZhN-N4#CvmGOR!=kxK(x6_fKI8hv$R`ut{R8Tg-2R@CBI;+7x=cabIj_(|G z7KIfx(w6Q*2L-9=n4|vrEXmz0@jUa~gYj~u^5gGi8{Qb|pN%6gbo#DsK!wg<%UsK_ z`M1L9YoVmIw6)_gnBwt@6AVZduoz`~e!Gm-(LYxfo)An>LAS16lMybdGcPU2JwSfZ zRyi`%#w1QKOH5e(ae*_>LLeF@E*NIoWNEKmhdc-OLibp&P%pmLz`sIBjP_9SOYT@y z+Xi+T8&er~RPQ5BeYGw8f@IG^>^MAf?R>*$DaIm7odD^ns=4&(%a`Z1cdRLqrK0p2 zltl{|<^~76zx4{uo$;=to;%ApJMrVNd1qg03;l!)-ZGgF+ijToC^CM33Hv^rQttJn z;MzpdPv?eF*!5xN^||^-0$)OQCk^}?v{BUc`OE2f;eDJv_35@Y&4({r%)%A~(_c*m zbx*xgfG*F^r`myz|59fYMtG051;UF$4|$Fr@4rsBwQAmn>D21{83`-59l1oPLx=x; za{S>Sygb-_KTScWT4zQ$tSWLV9JIdYPoNOyo`R*7%;VLI zvg<$6FPl<-vCrq~Ccsv29waFUs3rXOSnMIhy0IixTY|7LIz77a8<}7&`JlM5-g;c{ zS1)f`DfDzD2IcV@$K5eCoE_mD*Az6dVRIf5K_2i;VIl}g(&hRk~d$A0x#FWPVA15TVZ+w^sy9=|eI{3$bXGjTd=>frYe8iG$0x;a}J=PTcgqO35c z{`yrQ4kv1G(Agk-(Vr#RU3#hCW#<2c^qxahZ)XsUR$R9W@*BaQaGK@XDohKNP9n}k zVQa6J;tE}?NX>iz(I=C<=y9m?Lg26V-oK+VP&MX*mPxCPre!Bw$a`}lc8anb_moZ( zm4S4$_rGpFAhv`$5pgs6a7kZTMRWBiBOYj%?qtq_QO%A7^p0{+|J3Ud+jZQAfB*gu zAn-eWxW5ex)(<`^QC3c$Ko;AEZBB3WE*!fC4^|(A?Ei#?!-M;42|dx0JMHr=O^fA1 z&Rf(chu-^>MzFGjuJHYw4PrR;&BmSsEqX%<6MA<)N%-jg{@y8?(A4T3q)C|kj|rx) zf+n}FE&5*5!H_Eg_|>F?xE$Zk&aEW4vvj=B;KhIxXmhb`dUxP=Nv12c8QTR0%gIHg zm{Bz54{DX8X8?x1givVsduh?#X6HgtzHOc^{6C$#w^8jn`lkH$g7!z}N77mj9DL4={PF|x7()3uVbtQr#>klf9PJ4<{XSvwKP0^68z0( z%DRkHTfXvVW&LJ!>Bp~=k5MyuIsNuHk|p-4q!i{}8hr<2r4xdF1T>K4Loc-5O;dRxr@i25XeDh_N_|Vr$sb8f4-DqnewWrF{ikMEPH; zt#yr(Z44<|x^*7Qg!V_Rp@$;Vh7vI1HBM?#E5WB<7DzTTzOEgYnXiQ3t#M4geV122 z!%K8f#S#_T{q0%4E>Nnb8~r`H$?f)RK`E9j2AhmW?poEZk@=wd_Mz@E;kUI$Uyn__ zZhAXj6+X8&;74A>b+&v8U!m&GzxzwtHZpx>YUd9hYdSX5M9z8CJMA_<_gGyzQ{NoI zH2$;J|L~}2{!LSC&!v#BUOBo)mp*4PaN;z>ZXj1MA7~<&f~cZ?z#D5(kSl#S zJhV&JhzSPf_x6*O^EnWU{mIq#`-RQG!kG)Dl|YmA0>ba!C{JZ?ZobdJd!Z`771}h| zKX(7+SX7~FKR}xKmBz{KSbz3Q5m}Ml0BO^e9m~by^7h)yIaP%9_0FHKBrR-sGP*8# zX{S83b-0g&_(-aeG_Lzh(TziK6o8kpD+#IDvvZax2^yMi(4Ei?qqyR-b|>zk0}GLR zptai7Klu4~em^F&h`6#OMse%U-zSfB8;J2`5|S+9t3s(YVe7nwgA*?s)L5+yDASj8 zw&^u=LTAi0FX|!bXu0K*AOD6Q(^O_+RU5QcZRm&cm6({!F8}7Vk*qr>U`Z6K{?BOE zU1E#aj4S0)B86`bLC;=$<+L1T7@F5J%8I7ggg@liS@vbDL^R@|^FD$&Yi8>TLQ1Pv z7uO>|S(p5k2_t92((_pclB}K7Pl;GPjhl9Oj}AE2e7aH$LLSXMvHMEMHsL~xuRZ(C zG?hb|V(-cQdnWwhZ=}NMxXx}Ltvtx&=ev8#Jmt`e{3VjkKrK5)w(4O><2#qf`IL{0)gb&#`BS%O^_zpr?Y1XfqAD31t#?Q zN?x0|pf?|0Re;RDB4?AqrwXR$Z&>JZ3Q}#yB|`c}1Z9@O@Ng^}dI5J2S1GOvm~r-^ zzUn4#J2m*<$8{h0;Hs+07n6U+4*iF$6UY5Y>krK!!mOW+|E2$^-S5-BUal6~@cQ~l zo~2*&K2`2TBuG0QWjAAvfzmuu|`Gj$d46-qs(uA>A&431sPpl z!7>@W7)3`#_luRwK(Oz{#}pv1M%{WV6|KS>y*wEd7_k$E6Y57@D zK=gD$qT5}&s-foWHyC`gf5GT0H3$$iobM7=6AsuH{+xQ;%Z&+``_-m=>IZ10FHcV{ z+6-G!@#FOq*ik0v%O|qSqJWqq?BUk_zq1j_X0p!5r#McMq8L47y+j$7q#@sdl$d<> zK@@6?mD#9}Rb+s*mE-x(Q&B&1pq|m`D#Fg~H#{iE&p~(oZr%7W%aD*`9>tj1s4>a7 zayB0^QrYI|lu1@h@Qf+t8PCzSM>;PA#1uQ)nBK!=8r#NNn*s3xd%cMZ){&UoeZA_g zVfePB8{?b*E-mbRr8}KXWLm(ogr?p>+Ip|&T!;gS=0S1Yi+hlm<-Q!Dhmk}o3xQ1! zXw9ys6zzq2%?$vC|C38eERd~m-Fn(}mbm!Ve(F46mhQWD^0dBxg}465z>|J4U>k~z z!+q6dKcU5-TKiAnHP$acrT=~k?g3I!MM#qr`(rMC9Ts7QU{mHjWlm&;rJxv#!FqpJ zjegd_CYT`i`+R=C2!#OMGr@?Wb5^a&n9h!Gi8+`+29jJ(cGVNFSxe7LsNo?GVR^^JwzHWx4cy$A0_Rw(M zDSpmmRe1Ex{Y2kFb+BpW|6FXvP0JIpYptW!L44*XJ#+zK>Du8>w370}rzz6@XS+lvsgTIr23bfEi#Nf0B z#&fg$m|8AOsI8b7)@xoLSMT@^Itxj>G+RpE(!J|Se)5vOa9(_}vD-K^2&I@Iu;=th zF*>+Co9_4eJzgV{9JVe~klW;({EI})MY%+QPI80YeI#3Ue}lY~P1#Gi)}vC!Edhtq z!(7xq*ZAaf!rYkTW9E~@^CwJ0X9N-s$NAR>Y#pO0V~Hi2>MGu(h*cF%UUWVh3f5*B)~|6!BPH@|=k+Ag8-0z!y#VWv()Y^I z`T(T`0U0u3d}3Gb+t{Go3sg4p0BQO(2LL04)dufdOf^gJpMMF>YpLTfX*Yqw}7@ z%RXmH${wBmvs~VC^cZR|UB=8hB?#1*YCD0Gc>c5sLVk=qpM!{a#&63=WQvVQEO@w@ z|FRW+u*}H7%gU>`$&)B3hLE~gia7WcxlF=eh6xw0l3H>Y2y71PDtMlXp4mI^R30f| zTV0BE<#RCitR-+(G3;ZpS{MuIb;B;J-6Su-Ji6bdiE9pBOUMKF%tOI8aF(7S5nefM96|l)gO2|MMaCffpimrrhq~_Vi znIhDqv|f#SGMT7$vRr9>;ye%XRp@gbCJ24;>=3-B+{G#4~WRi(m-`^^XoQp5q;XNCb_Or<_dq*7)XKqDUNUeG}fRx^3i>Y z_YDbF5Du7#?lkUIi9j2d?baUHmcwe?)E0M57>`6m32eM0RP{7PHx!`nEma~Y{x+4% zPn@TkE!;_%tf=xba~Q*mx+ay1KZ_pCl$|p>SftA*fjP&tBxx#;?)37L=eH%Q3SG6&GRu58!>V=eV>gP^-gD1N?wi&6Hywd+^|yB zxggHQyHJ#r!zSrX5q5~j%tbrTH1-*N0+mb4@Dqa=J*0PZWl?uQqy0S@=E$+wA?*FS z*tiVXrN!{m*`!H%SdkXoO~e@c_|^3eJ)`%7YU`kAq_nl? zO^&jXzFc=zF2CPKfD`+$THM3$>%6{$g+bVGn=IeFnxqp;Xge!urs15W11ex2K0-Ce5mc#KDyV3}?=FOb7TOu^&+xWWX?av83@1`i|CyLOa!>pG5 ztWNN@a^reiXKYjBBiu6U0g)dt#2g0ohfJobEsvj=2Re5X4oqId&2~B8n688_HIW7O;N{{h^8ueV=_34~EFNBwS49N?(W2>71^4(v! zRB;aHE1I{t$_c&78r~&*EOXk7r2p7zeqH!1Uk*{FC6v-6*0CTfx!>hH=-xP&a`Ubm zDx!nV>52<)vS~Q=M}kuM8L5j0)!PK5DvdB z&@jh|9eW~U@F3Sr6P3Dk#i42&qS}jL4(*{@*Sote#keM`d($&fF35 zoe4GU87@9%GqGi^`0w4NRd}xo)^|llkY?;prz$U&8moj<8g_zt2|NFQ-k^R9#%{f4 zM|+&lSLMKXe{pS8w^1D)Kwpq^-e8UZH1A|l$jwnPTq z`Qx;ez2ThGzJ>Z0P`L5n&tgdj>RX4e9$BOOaHg1@F{8)4kPfzRDVg>g2^(yZ^J;D= za-;8+8os%-vmx#~=v!qk{R6~YIp^4XFQyUoL@c#Xs$nm;Qh#zacVbShM{oY-g-YwN z(UpkzAb+uXk4o5(WMG1#8%*Ij!eUu1HIkj^mb(5R1jiqxOD6m&=-D^m{FjsS%mU25 zzJ(tuOhWZS3SG;Y`EibX*P{B;uE=`Q(`)Qy35glcAaeqjHszQAgM+@4HFgr^tbNHQ z?l57_pB3wy>2qrg19rVtxz;hS6=ukJ3ynJeE zpzzu|M%uMaAG=1@8TH<)%`lm$s|GkDI7UmEBghrCJlxL+0OhVUDM)vC%tZC&N;M%j zfsotIliYdQs$WA_UT9g*yIk;CNL+Z_cf5@^ME}75K&d}%7EEzj29&|xgyuomBDki> zwdUYNe+k|!;u-q5`g2=c?ynT7vM-R~@FL%B<=t&r``KSr?EJ(Vupg&yo6cpet<9HqAGNE6`9Mv`e`>T50* z;fg1`Cv>JHD!*^l04X;@G01B*5ftmG_R)T1<1tp<=yuW$t`RvQmZZiHx0>!hkqcCveDS@ZINL$5*4~@MdXK*2^=RWk05O(5G z>FcO2QiRy<7)5Nz2tF8V)lzQP7ksNB-<14dB|7h;9`&2X4&iV#-OBKRhFaQt@4m4$ zElW)-mkr59TP_!XL0A$>o@?a}VO9l=s0Y9&6ei%4X^RIrXnO%Q7*yQhhw^G~@c+G1B9Lguk%zE_O~8(SO)|%NOO_h7Weip7mUkXnDerXqB=1Oj z{g@qngc2i83y&8tj2MvySWvMYyfpxSEkL`92OV3wJltG%gY-^`X18#jkmj_!m(-o! zI|;lA09RkGp1uGyEdll5l9Q`9Y?)_uR+eVIdId4mX4v2eH0tb%y$>p<0?PalB3M9L zidzS&h@h==QPmvqyueSoGc3CA>;Mpc)90(^pRvr6@hp)Wznm_0+yjqc{;3njl3cAo82QNL?SpM1~T z$U`73p&}KziNuU;dcUN)G*skE99$HqGAj*r6hGRp`ynmRcBO2bYyc?(?}h2bPoWqb z^WO0`goU+d4WT~Jbkarv0E(^T4TpBb~)y9F@Q0oINanR%`1;x$FbFe zv9ujVa+JCqX0|K5q=3zzZ}M{QC5h-4+y)X?wJAa}6vLNY+ONdm44Pe0;aL>zTIxCZ z$5qfhdB_N=L*8NRJx?~KUdG~^@T#vaDPjAeS`1ote9?UF9wL*>%O(#HK1mAs`kCAG zB%}K|+Fzr)!2ywBpNTNbTL(f(Gv)&1Crl+uh(SW4wKm+i%87BPh4Yp4CW?#Vnn-p* zJp!X&2Qxa~(lo6HPQG0ppq<99_+;L+M543&nM+et=t2&#w`;;MEUKy+Yuwgw1iqKTKVpPH^~{e*>;vyP(MQgL3RG0I+qqmD%@4`Inja91+$ zYHtN6l1mw=18g;qf5jD-?T<%yfQ!3hamdhmxtA{G77v5D_)mp?|u37kv4kuHw`qn zaGS#;j`M-+-pebk+_%jiT5Y40F_ zSRb9HijS9%F*tWxT?;QZW&B<_J8&a>HfZ=n)q>84nsBZeQ>?{+?+oH`DbIH(e<{c| z>IUzX?8nO+K5R?u)4o&U@0lg$tGcc9otTLhm!+2V53KJzv!V3OoK|>CLmkTuUi~de z3uvb6yCamWg2E)C;!qfu=ZXE)jMtu(%p2%stWjA?QR&$Xa-ZOD_PBb1@$yU@)>-q= zR}pn){f&$O0XlmRaSgebpAgYG2fh-KT4*AFEOiF;0eG7Y!eD37hnF(5AMZTJww^x5 z(`l!;I6FI~()W$VK4@J6{Sl3e&$|8K<#yx|pZP4KjilTX=M!`E{vBlU_NkqMo>O zn9tdKnM$Lvw~*P%ye(ZNpsW*%5M2548It>QPmI=Rp%^l^qrT{@D|+DNBj_Eewd z%*ZLRmN$tgH-NZ7VvfD`o|~gA`c=85*sBC~Jk-_Z$uX(9*shhX^GYEqrP}2C$D^d@ zs0zuFUoQoDoR`j{p`_z3h*M)i=j-3RU#`Xrq#SA)?i4iZ?@ausUULxrJGo>?%f0pd z1;Z2ETvFc7%D7p_j*O>hM$u#hNOa{a!<@R!VN6MDIEBAY^LbXFi2CfQB!YZpAWqtv z{SCYE*jXJGF~Zp8_|f-V+aDa%Nocux_Tobpd6@-|^kweWd&l@f6RR2hV@Mu2nn4e- zA_?~HjQ{LPQ85`jV=WnXDqlP$xwvNa4Zhz>Z682=J#_QfUtrCsQRo$+dMelYvFsEG z0WjfkPsu&&9y2$T*{v^slUNJDeG$vA-kRP)#JxXqL*TL!{pl7t_w&D#)oC3fv~0#& zam|y$2k#n0j?soyokhgaG>iKd9^n6YK{W=j-ksOoT5(zjq8#4w&G9WY0qF>TNDzPN!C)X}q(}2x( zxaMuXND9c%&z5b5JB2oW9BGw+YAL`QyVEN#%z#JZ$Yum`W-?1t?lb+>J22V#!*;u0 z=V%g^04ys4xZ&JeE4)-EyOdG&wM5|*!FE^2wW?S&3J6RT`UH%2r%wo7k4qgn!?2w} z$y?a}=ml_KCd7Vd@ue8Rat!d%aOVY7mKiCt6rrFWSGamn2j7*!h1@iR2+T499_jd0 z9dAL*om~#{;ny}qHd_4f@jNe+@ADtuD$QEo%}2WseJ+vca|j;xyU)cZ)*)Z!@%e`{ zOUnntpDx`Yd)@za5cqD`(<&Nv`X9R(zRrl)kj7{;OE;@ z@7x%q+xA*DClBldYn!wTNfEzrc22cDZ9*XZ)r!yK48JnP^?d>MzbQ6-TYL|q#!`zi z&I+?@6$BuLu$cujwxGt~?P%NW%M*{OG*2|V zMi}FcvyN^mO#=Tydvv~}yGAN}=_lWNw7m8Gk$nw}z9YtixN zsQk5;)+mVE!8;%*WvP?>N3#<13b8hM5aT_J4|I}T3^{=zh_sxbbN_9^9VbE{*{7wZ zIc3#W+KsYOB{F};!tp1x1W%-Pq2Lc|B)$@xK40uQ>gJ6Q-id5Ba@(HHpkx8@>zR|4 zJr>IL{r|RvQN;2%ul$Hwqz$X*IPRGZIA4m&hNl|u-=aHR1Cg$#qHLLK=}K+L4`E(G zC}W+G$lK#Zs7H58I{bmv{SoQ?m>A7|;xln))9}XZ01hmf4xHQH3AM7dQMQ$`@&YipabO^BD4(y*Fdzt4n%2!KTljJOx!)h^EChina%v?bcx^ z6CWTN3@z(|d9!l^4u}y{>cC~s{ZXNOm59x>fy-0GY1cdU5=j~|UQ^T$N|UTb%9U<+ zukppARDkS7Isl|Ssmn;w{X}&aukaCj$3_>P!JSw02VzodN#J^;(rEfTM+;gb2~?32 z&n;AG%%4!cn4%32y+tNFnv^2)WE~yht)ngdsK3>0e)y-3{|WTe^@`N&*{EL136i^9 z3}tNSvFS==hPRD=e}d=8Tbe3!^}E|wDObOg&LHb`EVb~bE?y;3azQED0qe@J>kFWZ zn)xi(@QwMStgD99yK;K3D6wC(_JfI}Y`qe}k0jXLa zYQ})2(0F$G8DW+4iu_4=$ET@soGIrUU*&Epw~mii*bk%-8E1S#IR~CiO-*$ z!?QI|pDB@s+^O9{Ul~5o&qET^H?L=vri?hLBxLRL^T7>hcaXoXTA=5GhLy8lR9iqyc(GtT z?IbQ18~%IYpdf_{@e53SW;~_E-`HkfK{_Sv*cie2!gGg5Hfe+W&Bc{~Xh; zGNFgZ`P;Wcr(Nb7U4Ny_N$f^0!@G)$O)FLKKiU6)cnqeds4*RfK!kv5w<9y*y2AzN}ey48a`AA z@X#L6)^<@nm3_2zJBll_KUo79|BxQ($N%MwXQk_p2B}$R8-SnVy78tLU(BNvq7P)0 zr!`ZsJZYl(Bv#+MWsG2VwU7aaice&^M;b%^2pO+q{AkH|O6g=*?NcuL)tL}fHK^_5 zC?%@>u0)k7-Km*C-t4E_bH0s5!@UR~vfKaiu#iXoT$nbc?k;ksIAF?%z?CL`mD-k* zjye$I-vp1I2M$gZ`bpLPRrz5;eeEZ5h6CT6ME&gT7t0j9&MWiEDzfU?@klTzC31p0 ziZQOJIQRI$%Y6p*ti86Wr=Prm2?lQ8-(B_hDn%ZoCjkP-#+^FM&eVPL!#FKL00R>d ze!EBlY|g88Oa_Gn^h(X|=jYfKor1Mj+DYhF{$a%de4Gbi%2$x}6p`qWlvBCIY-u zfl7o$hM-q1fEgsqdenG3+P!LE2OS(XzqU31(k6$_14l%!V^ROq_wDe8Poged@J~_5 zjz@wLJ8y1jlF$kAcg(~w*cg`SA;(FRyKHkPB3jz(4tRuJ8&DsJLwSE(x)AvIf)8K& zg&cHQf!CqtIUQCWId{IbOu!|_WBbWOl)!ApKHni+`=>;O@&`*WU6%fa?*tH#RUm$3 zF3!Evle{gVpm!5FGx$0TYd@7AW=ImF*7aIm5n2Wpha-Y}or$bVV>u=+(7WlcIl4{l zupKHUdT=pso@r2rtxM?)h~rC zsD}D^a{#fZLOzG60Ngwu11QV7^}|6-QR5?*m)yi?b-e_cgl*8Ek!5fdPo=Hgb!9Q) zXCbjGCS$AwUf$j2_@TB@fd3bk9rvo_FlD**UPy;mZ)5d}MekTGr%CA=)X~obxNpz$ z;->r9av%}QnYv4ETMnn`*iCKZjE3>&d!2P-w)k*YR1?{|5V%E~+XwKda&sWxu3<~8 zL~_9&)#57*U>Bq<56&=G$~;2+d4Kt324RR96*hvkmA1h}Z?hF@ue;T~ljYkpT~e`bxv93-X`Q+>rV9+m@^Asa{ZQb~Zu^Ao#+ z!%bd#*!P?sC=wM%zBWL=+q8xfc2&c+wjveeSZ*O!I$UykwEEzG#Iv9~O#*<4r81g* zb;4wT-kj+@cFG9mKmiMNKqE@ZzAlwMB`)qde(_wiJgwduK)VS`Ts|L83{@nIdP&p9 zXOz^8l=O4<{Ld`o4VhxKYJoW;i68hi?jP7Z#9CB45)RahcHsXLPW45lR`6s3)vXv`E;rm**#G=R zyET#@sGeQs>{CSppwV&6{|0BACT#$j9;jO3!GKPHR>`xat5(9}j=K&J`hj{@I|MKv zBVhj6kv6MZyHly(4PkTp{17e@%diOQkJE8;JUz8Csm{Q9Hky1+L2=-@)L zSxM&qcY|@WGnz5W#hyH*kRRMGdlO&W`~^rIpp1OpDS2n z*toYJ;-4N3oxM&{fHZ&WO9*N_XIpn}gn3HC-pz*#H#FteP4ebbwQtNcL7*a24hs;q z#G8f9w}`~d#7yta4VC$W>s(=Bj8=6#KwRE17Z<1U?u;pwv9})*Cba|eI|=qqw#AF{ zoL(v^w;xKt{xZZ-cEgO-4BS)WT2yKtg_v-MsYOmzWS)n@$sm!fP4Jz~Y=B%O643EO+0H#lGD0}30$zkQe(_H0(u{}P!sHE{uE zhKzn``%_)V4Jl|<3xO}1W~pjg8UR%K(W*5m>ZQ{GS;-mz<;e+#k zpR~|E^3O&n`9D_zCsHm7*E|20M5B3P0(s zW$mJY^G8$v!RS_^FpO{SuY;BW7gp zB$e_+XzVNHjG81W%*QFT-NQ)EtHa1Ag?8TyE0NMs?X`)c`c}p)JNt?X0%GKxA@2!` zIjsosza;&Y&sRg4u`m|v5R{9KNbFOwSQ#$?dN{FTm^J)TN_MMZufhl8*2?wES>70L z1i)sdIE!N7V{cA|gPfccz~W|l?tQo%pGgL+=MyhF9@TQbzZ&A4*;ILPfB8PnY-m-f z2<5)9FZ~xZw0hlw75~BrSXqbv;jO z-3Bv|HCa;CL}>Zpqk}Nk!E1g|z~}Nohx>s`LU;hbKRQ13=j&yLRFAkOq4**=+w7dV zS<=TR&aL&J*)!NX>Lh(G>PpfTuUmF+f5Mrvy?!U7D#J@6c%F6tSWNGLA%-=zT@k~FR)(Y~ zI?4KkL8G)TI%$u6wvX!1Ndm+b*Tk!E#Hb7t;#~A+wi#Q3mcBRV@Q#ZcCSLT=4*yiA zAg`g*R?6)~jV3c+&c~?PAhYyJ$D6P(xT?2yaAD>UBMxrW*6jr@U>PJy83e!QEabs0CEJMFu%J4Q4<2CzOG()E>Zb$OrP zzA|rnXeA8(DbaiI2rOz&Y62uewO=HrIih(cD{ub2fd+&|tuR#T`_Y}3Jv&zW$?soa z4@obh1lEG&nUQ)Hu9D8GG^Y0^w~omn-b5{t!tfU$WQCsX3j#KV%VPoqWvDZeBRLXV zUIg{peJ#7N+zm-1ss9Czfa#!wBfvWyn~xFs4o0WOUH=&?y)$D(SF4L&{lPovc6;2P z*`O4%24;vACjzaJ&1{|jjB;Xgci){&>_(p!D|pHwme&UD!2J=({vybIf^BCV{)L|Kbv2u45W1vP^%MxZJe}?IDaB{Y+1S)R+{rC9#UVF}4 zyrcr?1?F9GqYo9q;xeTrQl#q7X-(i>{6+LlUBmBRDltBP0T}ylvEVn~V19k}ItDU#xuRAmQ_gP={!)>{v&n{lgyMC8{ z*yTSB>%sU^o}uWPX}i(ubo)#~|8xx;S|K|nYcarkWV5afZSRx9Hu z!B!OHpEKjTXRKn$bSr25RtPFD2NcE=n~Bh&9qH?R#j2DXscd)RJ|r?u6JsG!&TtUF zTIssYzv>}*JX35Z2xlUw1>_XSL5D7xuo2%KigNf}ocz9MB*gj;ZzX`x{@(2-!>jn^ z9FDJ5DS(Pk4b$p+`fdc@iX=Hwos}ksdtk>1sbW!!}miW z0#zp5i?G^+$|!;H7maJEwT2HD)>$bOhcnldBI2e30IBlQBBF-sSm-}et-BThJ>rEM zuxY)tke-VFMELI2A>!z8Y3{?R`+(ESjxvc`>`AAxaMt8I&t`)YXDc*a!= z2kOGN26W_!KiIv&KQ&6fLB8b4`?RLYsMLWSxXp4KC}co^=(0UxfY7_fR_f1%GsF2N zDz%({7~=OL+&cVnH&8BeXa4^8q^6PxbY6ib8WZUsD7_N+g;4fuy@r3)6QO_141SL% zCqV}us6kmHEf@fmBoIJY0jzYkzC;mqHv|XptPC)m2*e)1V=UMIUk~f(*Js|Zw14-5 ztG@dTs4jmggCEbFQewrw5`_0)TOBALGJ!76%7x1;*h@p0lP8q%D%iNh6wKpmM4aji zQZMOym;7|I<93!X2y<^2Yd3>VA68n!AmltA8 z1}!wl<7oOMKywHc{p@95C)v75w2JoE53T400zkUYxLk(uQtKjU`s?^O@9s0!A{o*b z6h`Pt9~Ewu9F`(tn*AuDcIZ{!|7T+9c0j)h{8V2pg1yYfO(9+>Xv`li&q1VD6yu|yA52CbSBqf7?`PxTJ&o7%A2KZ b95bGV*re8remR*cOn2SDYh8$WE;tWmp0C^c#oUll8;RaLa5MnjEQtrgU&5TdPFp=N8OrKPRC_o@*@ zsZt>+QlmyFf|%h+f6s9|@17U`7ylRck>g5oUFUV3*L|PY=kqzwJMpfi=|$GdtTZ$< z7tL-N-KU|U6`@{o%onKN7#J3uQ9o#d@0;GFsT>tqqc#}43@i+2Xljsbr>>0DHVgEY zLof{uNAJIj79H@(orWgt+|0+FQo5VWeh&=zc#Vty8Q|J2A zkYmre{lCF;y};d+7V^W$O$zqhu7b1k;cR~Jh{~Vb`L{l(4l+Dm6*7<_1b*%S^&Dw_ zwa}NyQ`%yg;-{RAV)PpD6El>V`yS+S$3dmU7w~c#6n%P#$P9#wR(F8#+%*6f{(PY( zhrS{HH?H0-Dt;#R{;NiQ61YtD+FcQ=hKs(Dx4;|kA63ro0_}vHF#g}=dw;E z=G9}hD^o=ICiaHSatptogi~Jp)R=Fh%Zz#zvo6W`ro3jqtQtG+hcuWa@jD-6OYHYptU zsKY#N+TS2woNuYr59D|zLJTiZB!ZJAxB|9AJ`Q< z`7Py(rNM?iToFxR#!RgQwC@bO*c>}Q9aDBb?RSQm!2d>!MHXHQA@?1Rc!7dVX5LHP zXrworY(4KLN~z%=Z2BdMCzn>As{;iV6V795#x`!b2OT@*+OoWc!qPxHd zVK~2U8ee*B8iA`Vxa>HQJvd4``ZF%4fL zaI+1vNwp+H?s(Zb>9Np~tM+b8Z_})&b5W-57+-Lh-e2~j^;7{UNxcdEW-KpOJ~nq> zp;=k)!Hm+THa&!HrypaEy%idZ3E!;A-p~hE7 zNZCV@>OHj8@60i$TDVtXSDvpdYIg60@`!xy*lhkPl)PFBE#KkXJHp0OBV&)yCSdt7 zCiDqZ(fnce5%6AJQ1<7lKph(>@7oZT)big)g4Wlk=A>IVQg*61>j-za@;4cdr{AU= zFFqGqpVtyRq2Gkuh7Zag)%roMp_T{bvHuFym^>#}j=GJ2^|FR9f<1)vu`pV5myU*i9?-E6^}+Ovpp zIaAx>CC7s1Y#B<)!UINxS-D3vQf6v-j*!TvwpqV>g}}Xj6j8lvHM!IKUQX!r?aG>0 zJbed);m!$=?)}zppvH|MIi8?|j;`~A^D{T1J-7C1xQAc;bi4{;u?>2f&8_Fr)9{(0 z({#021rXX!_(k~}rstHSkypTAJ(Z(M6UrTe6Ld^@)Yfc;!FuGN|WZ=M(9_AP&q7Y%lrlETj+R z%Q>77*1ivAz8sS89r)G470?_F<`%)2th0S^dj7oeNo1PYVqba`gpk7}$#{cHQ<}j; zt!GE$(H+~gh8fd)_zY_XuY?Y73{D)te1`)X3EAQi!c0=c($BmZAbejZ@M$McC88Cr+NR?Gz8kiX;CaGU0UMGNzg*0;*MF#$4 zc3|%YcXL3ci9|RvNDW8|Jsg4w}(!KH7~= z_+1z*6EQWG8#{FC<`1zj82+OU{u$yfXm0kXrdq{&CWFYIk1Gt_J6BIpkr-omUzD56H2^Ccd5;-w%oR&wn^$N^A}*P&J&6r5AA@ zw+8A9#rra&q?t5C04xJVjb4B{mKF|gWCa-}@(O7~ytyY=JErI6fF$xhQ%jm2N6>$Q zKntJ_!jS3skw?_^R-dU|wJLnR`l9T1e54M*QcI$}L}@QpimwNN9W`mycfJQ15s|Y-^_B&Nz z1k|9P2t{T)2n@182D96THjDv_U1AjoEr7e|mFMmx2FjIV#*Doug6i+#!7s)erLtjv<1XsO{P|VFAd_nDn;|j@_a*oH zvGYNn1MtxovSua%0`5#Y@%{0(@QrnVT82Q&UDqe+(i&7=Mh$LwAn0#Z=Mq4m$w8`# z=`iDkFVCO9lFxoCr{QMOp{8@rX1bOyx&^{+s&oH`}1n4^=8cSL!-*nH@`& zK()^QoBio)<)haVl>Y3R<;!>rW{!;9p{;OQ$_1SBi$ifu+dHdW8h{NivWR?Yq5%HT z8+4(Z*H1F2oUkg8vLadB^*8uY*3?>dJKG zHQ=dQtEM`?HA8&#gv>4B?~o`&_Aao{t4!B7gN^FCCx$2IwteLz8lE|JE13Gj-~GpEQhW)A%StVuhV;QINaNajzD z(II+(?QjLP>vFl1m2Yo7lfGjM7wt1W{S@_I$v#6MUF}Dd^sWu-`G|^gwI`**EZ>EA z`E0uiS`;nX3+GfV^7Q{&RMZvMrZk?FqP>tI8Mcvatq6~Wk%M^4U`{R+T~cQuMP zZyp<8g5Y;If=5*$Z+ch~0;DQH{z({E3eWSQFN4fvZd6E`PG9Px_KQOVXS6KGDNQcA z%$pM>mt1wdvF=*qLumP8$R@YBZ~cZtlRxLbapei1gO;%(KeO}t6}>ua*f%+X3@gLg z9SR+e%Q0d(s}zECmR33mL`<3qpHFw~t38xfm6xGKRC5UG;fxu!E6%>qeHYWa_x`#{ zC)O!!)(x&uL@=5kbgDM%3EUtk0O_gGLW~wtrUzj}57}f$)%)=zv`K*f=If6Stt&&= z6n0zwCLy9HL$Jp90jthPL_nYipA54+edE9z#bt}9cFbvu;i*x>W>pTU!6Lh|aj&NA z1P3mC4~SY!XVLz1alug2cpxtxRGC!4pFe1S!MSGKtEy7D)Qvx%?|ZGOo`&nz6oThD zgL=)rsDimYQBe3zK|t1vX(nQ-T(ox_2xBpmjF|9JPL>HAa+?rn8nlP==Tm``Cz4S6 z6BQyoW{YoYAPiFnm*9E+Y)WO^Beb*1R}o75S|I@(@5=5_D4Z;V&}#}TW@qiRW2vyA z8{|%ARsvHTyhy$c9bmR-=z8CzX=+dH^N%odqwE6YuS^=FuIV2E%}$J^e0AZYK{f1q z`Q+*v-&589ZHY_S=Iq06)6*m#>v?J~^`n`A8pqH_L3hA;ydr3hlZ}8MbD5^s);&~w z3U+n&a($>VQ#DfPe@m|ZN%mJHNY;dvAw}XmSbS>hLPWq+m}905oj?>voqQeUD96v) z^n8Z&f$HbecUnV=bbS@-assd2xB>)UJO>b zZZ@7)j-h=FW!yjbRnXx|EpQ2bSPEvE>3ItIiT~}p@ou+tmTg856M>><`z#zEoI`4# zDP`j=y;(*xS&C30Fc(ot%uo4L3o9;aXUaF6Uk&FU?{5xJtr3J82ZjM1YEM)on1&}n|0Gq zbtI|by_w^@h~TZ~6DHKe(|8{$zG?cbv$1K7<>txweEBT@$#u3-1t42xuO40|q8`=I zmV#%Y23Pf(tQq_s!+XkVz?6S-6cu412vj`RJ#96Y==!3E#{VJITe%b4*QQzjEfP3Q zHh;8d`okY(F@2M)#J6?aH~&77pk+@+Z=YN@fx;w$-ss{!ttDq1jy3*~W{?@+5md(g zaRMIq3;t*r{Ys=xpU2hE@YPZC&+)`CLy5-697EtPuKF=dV3PRW{MaR73L3fppKj@xlg`Pg& z$y5m|BFtX?piGYU`y-s6l*JfNMhuI5ly-YU`-j@*iO;5`mK$ z2At!Ak4B4~J$XF6DGMkb9%tIlKn=2&T7#+Mdh-y7ojbpVoWGHtAc=g5D~=Yy{4;EI zOJ}xgMR%)37JW#IhW^~)X&~skEr9lk;eL$!v$-dAqSPnGqetSXk)9IgS5xKNG7vh8 z=hCBv%LefJ=7<| z-6);1ZE@*VBavSOCn5LAt5(%zulJdhG|ycyD~EXE8raXuvwEw+Vp#s^T-2NGZpg8L z=CJ!lFIUfaVXhmJm?F8qXhgMFzwlVwSG4zXSqkb?fB(?8VLHHK5^>56mHYKGP?DZm zHO}naOesABf@bT#A7OOQ0o>M%@zeI`>FO#AgZqXX5Vp6gQjhBS-t_dXJoB_}t4Gq@ zorCWaEax;{lGT22n%DG5z3cLobH>LFgLv+U!vagAH$;#4#GFoYBw8v zU-xAPdhoawq$m=YuxmVlPZ{sxlR=g1D191l{8A&5$WK%A05PEvt^LY6(1dNDVeJHR zjxUgAO2|m_ae2@qVZvI+BIlh)zGaV3-K?{EjVwqDbX3jL&#Uj58fyef|9)j|vHo+f z*HDbUVe{Hct(c(9bkGl_-Ks{$&k*?MC*T~4*U>K7YG3VnZHW1Y4aV7>M=-$i}zg!dU0 zjRH|33n=V~ea;IfZb)>q$Qx-I)m)Ck&wf~$W^?GFIi1etX#cvT3*u+g6e`;AC~gx zC&-a)4Rmu}B5I2K^Lg;J!rUU4evWD8=3A013?doqBgdt&7Gh&n*MR%;-$M1lh+fvX zL?M}#j=v#QYJa!WA@69a8~lJK&|1*lc;z_SE=l?Y_fmHG_vL4;f)KfdL+;3FERucy zAiA3S4a+ERB;-AE=3+4Wr!trRBdCv?B!b??M$ALd@j?J=&ilqvLM+hRkWq$6{{du% zmI}Md*nHthk1NJ$Qd>@*X}J2EJ)S5DqG2?BFC+kC~h%Y^%H;r%9CKAF@?EymP3nT1vD)!aD5KTT| zhu_c@$wT%}L)uwU-^F;^8|~mzZreF5mcGT(a6N5A-+!9L1X-H&_E-4$)>Ez@(kf6= z3U@yHoMZk>MR%mtnp zsgw+H)>Jd!JX|qoS#64ZoPesNxq@c4oMMWxs_{EbqCzn{y}T<4C}CsSBv{D+#MTxt zrBZi(xRe{-o773So-9WmP-WGgB;U0!Z2d!nADObZ?8h-O8JS)z{Kr-$>P(^2*Epo; z1+M`W?-w@unodXaiZrT3ykyq*3}!F>CP6;_p7i*JM3a22U_(NHH3fqc#e}>Aj(O3yhwY}rj^M+AokqWMYwN2$q>Ifr<3~H#_9?(Fn(PRkrGBdSS7PC)d9LIv*Asqlu&SOA_E3k#J2l~z_ z%VRpbK^~rE{Znn=kghBH1;-VdHA?Gkt3|`nKO${=0p_wFXV%lsh6JsEuvUV1}~^8Kx}N zKJT`7ma^!je~hk=nM9HYbI;FRIBxXsiQP}mg1p&}-5uBCI_OYQoq>S&cLY5HO1+}u z9I^bt&5^U`>^0^Ox@X76=$F=d8~;Vp@xf=;OyJkHwAHCL=-@wzk}(;`Zc|z&@7c8Pnf6 z@a4|XYg9RdVX*7t#_XHe${9o$6Z~yYS6l^T=dx6_lBN)Q{#Al{9Y~)Gui-5n@3BpP zHEchw`3I-37kS{TnOBhuD!L!QjNN+1$e+INkumamxbvFQS!I$+zv;WNyh|cYQ^k~L zr#Ur^oR8IxnTp|#akv#PONHLhjjEMQy?cYW7B(LPt%w!C?8^w@lTawCqmw^Fa$ShSIQ&Tl(eTbg{U1jp~lH~AA3 zUDy9IbQOiXA|zXvl`B zm@KjJ@X`v=|KSa(f1Foim5!@%q`D`0Kp54Kx6fwTPtBK*4qbgH2@UMB&)Bc^N!akj z+uYldzW@Hy7zV_ZAnW)&xO)t_Hw3}-KMUgft`t&Z-nPT>m~=_lZ4yClO4_TFHXF<) zR?@W-g}fllH?9J|vk)YI+Pg=2AEBo}gIa)=&f9u4``?0bpHHr~q|Rr4tfJn*EU!@x zs3n_x^}2PJE5Ll=+hV=91hwMn8M37wSV)GvCW*+~G-O<_V=UI)@E!|Ya7w32XP_D@ zp$m0?1x9@L)5+66r1z;Bym;U=PG-=(f{1AaM14?%Fh%Sos(>{%^%_^@^_dqe5_i1& zd8f{9+Oa!gP4eA0=!a8{{Qy%%SJY7NkBps$_jPgm5_LYP>K>Kg?qH&{ANC0G4L1+^ z=Nd!0mP<5{9Ar$Y&|T^Lp7OO>wo56UG<&hcsR=P!55V;vzRNwJkcC>!pa*xcXWu!W zn*d(c`@NC)@J-ge5YM~Mq@C0#Um%E|M!k)4xoJ{UJIdRW*!Z!`H$g86{70?695H?% zwPHtnd%VY7 z+{SvAQ`sEXpkXAAe!U0xrHNqAGGuq>p+QZ?%V+~TXhQhZg^}YQrnutS-0HldGZ#*y zbd}B+qJy*RAGO86*$`}jglhUEeHpW*AK-^4Wo)yeJ#O;%&daC)e)ni9sYXCXZ*D*{ zQlgw>I(qWr4;MpaOBdLCgzD&4A$LGVBVH$$?-8a;{e1{N!s$3jneOWhiZ}h_%0v0) zkx|tjW6YPhzyx(Wa0eAh+beo8tM`0~>Hmn7x|DcxM;Bo*@G3 zRaTmlaqKOKxu&gY;0v2N&IUpsUvb%X$Wd-=<2M1{1)iuum_IcNCvauZl{zWm(M!D+Nst3GzBcu?AMevn^mc3 z5|35Mu0noedGsT0O-jc?+>}1&4GEolzRd=jVnRM}*VA>H%_*MXBr5oLuc7ao7qDUK z?1K|HEwWS(6F2*~9~Q0MgeRrI==O2XNiBp%l^$WE@=KNzVo zb2?7j_fvR`sc`!$$+&Re>at8Z`WNMdi^%cbPrH!Fu*q>`C;3m&zHUGiG@?q*<6_`N z^FpmcBF)ZC7sqj+Mn30sW&?cZ;kVDt4;axzUl#V!TDQb7_BS&K2J;>Xi8T#BV|?Is zQJU_fDdNR96`&ucFl?m?jqg5pBkZ}*DumZEYWf2~|594#CCFAn^;uz3$n+Ip6yVF$ z8Tno$*IpF-G5hL#?p8lR28?+{!EY-@W`+U8FZ~(O(jPq(D$!)F<~g~0f<3TN<7@gt zn1st*Za_ygs;hz1lquZ$cvthZ#o6nYw!=l_Ioef8O2wO*b;-Ect>Zhw<9hGz*2A8M z=!yINz)pvCt9OU^PQ;f7K+M49o~PvqLG+knvx>e^!tSHp%2#6zoT|Gm#+y68L4+Eh zPq4xoV8H_}0dlX2Nmcydd}pw=Dkw$QSVTBR6FZFN2jBH(h`O#Z1n_+(`I%zK>(jMl z^Xm(OKKJ=ATG_IWlsAfl>ITBB7Y!k=4 zf>sFjDp0y8g#!6;{r7ZLlcj9h5p4@7v)(*!PcD^c!-^BD{hA2s)sjw!Y#1OHuj%*{ z_p>k2?+VDeea%z8@ofgzMU5~O*MOOjE0?|J?mP911vJy)Tp79S{-X&&rxN_>7rj90 zs^F**2AShhq>kAB(KKCD!~EA9HD-BKTyu;{dE){j6Lxinh9FtXExQicKkovH3Owp9 z$Oci^Lb{*Mu)?BeapDj3@v-Nk|D0#TZpyrWF;mYC-8NSW~|`ncNyQg;HbQANQu=VZl5 zNP2@sTi&IZl)U;T)~U^T`%y$4h$?>l=;8;H2d?&Il)$`{8CYY21H(=y*`=e?Abz zpTF+;h&`NL_L#T<7Q2}RjThRydoZ;Q{!`P@lA~b zN>%fuw0Ir&d!7aV|CXL?KVa(i;Rk^vZsy%~+2F2ofR37YRvF_EcuVL4_*46~G5U#8 zoJTlQJo&JiTPAAk0(ypHS=D8%=~Ios$+Z>}0BkoGw9R351E{Fwy|KF>jJzbpe6lx@ zqrTY3+i8Pa(GH1SU|3+wYtQu(YSI=xzTH03actr3_UA+|Eyrw8+qv%wL^nFm^G3;D z`st^6Q~E=ZdAnr>}{oCK9zF^L>8I40-};{2#536}`n&ElsoX`Pn? z0-DYmn^~SulvTg*M)rWoCMtC6-BqMet&r2+$X{~d2ANSzHU#CwKK{soY|wzf@{=F% zsD#8dXTypfclAL1JE;;M;D&F0Iy3ngp+*e)UxxGeEPl(CT@qT~!T+~tfBS@kTGmj* z=-31*a)~^g$(hJb{;z7tI_N*v28NV1+r{!XBl>Fh1ov7q(iPM69W^9=-ONq^CSJrL zhuTvVjUp>=)MKxBT&Xi4N{TrxCq-n3AJ)Qb+ObM@zTeg|#7F3nUy zzYMxI>K&W-j-)z$M=;x&q12OYvJ|4kX?7uV>O=GIm%^4=B4HZ-?>&@>YMsqmuFlJ2 z7s`!oWx8XfJj+3j9y_}k^+Iq`OvMs)qn_6&nDzWPoEeKI7g@sUwYJ)Khp*;1)*co> z*14Hpz}P>lWjato($4B$yEY_2J1Lt1&WF7bGV&*M@9(7~=++Dm2v03GJ^nHFHb(Ew zX7k%J)w3cAeZdjs(R?AG|P??7bhj)_+&SitG&WGU#u*@E(!Ti~O^-unsd(3_r% z7wc4hNr9_TOK}ik77^kqO7BcF_mYV02K<6e{(G7)U&1)G;^=*S z8udwiEc9=d6abIe(%OkP>sLPbEAoi?X1=ZH)VyoBrq%$Kk*d~>`eT2k6?5TLYi7TU z5%+mJ2C|4?Q?lPcCx(M*&h%k6;o$Q!Xe6N@xOiy|eRT653{^|YHpm!fRCim3j*~Smx1^uf`7g|g_ z*?x#^UUFyR1T#Qc)ld?fUWmm&%Cw2I6q?-|upQTD^OT0n_1Ugt{ehS6!#l>j7s*W- z1o3j^s5NtelfImM1NR~qo;=^v<6DlDZkuOn*W<-KgwUG)tjv5-(`k>QqucM{H-nfhY>v5_G%oSwvS#Q<1m`vTTjr( z>K$s;#?`AQHb%~pHB@Qrd67ev(Mu5PIOqciv<5)UuR>51m4u?6v(QT-&~ak{MVR^p zXR?QLqy4z};GeN+`$K~#j%Kw0yN`W^p+fO0)g2!H<%r){{6Z}xtNX6S>2ZParQZ#} zHrvi8Mwz*#8!xgL(S*q*HRdh;U!qUxxNF7eF$KRM`Cfp&EGwgW?xAe*n=WjWg0D!stLe&j2sh0 zo>18MS_B!H!1c>2gSZ#gfiDTot<}9+ZI89Y-xUrPanwt>virh*i5u+n{v5D(Zxp9U zZC6B-Y42UWrn5I2*ew=v_&PshEcmZ7z46Oh#@34`!{t1OF9^+?Vu#&KN&_)N@Ab=B z*N>SuUfoj>52$Tc0)O<>oG5AxA2FfXler)GZM~O&knMQ=8ouQ>Ea8SXn&N>xwGJHc z)=ZrKagQ#hm;JBwpm=d5M-g@34_j!$#v^LpEA2_r41JOLn$;CCsUbmx ztx1M1i0nEiEEE^4)E!@^r__xJ4I2C6L1krvah(y=kj=so=FYCFve7M_Sf@1?uP}Xm zs(8%uzI4$EwIGPG#+1a#&Edu(2QZFpO)|DnVdDrL<&t1o#+t63^xHi*3M(7+`8B9!M}p{5 zrUh3Y%$0~4W{Q0D0BA>@FM+XHle_X;dwDn3J(QF5Y#Y?5>z-aSZ@v7>Z|Uv`BCep0 zaRqSJ3vX@~))um(jP%%Dkb)3DcrCDy5_3CpMUOM3ItBhZ znL7YcIF*Fdrn}z5!8%EzTbId48hmTDSER3pai(e*bhF09T#r2g;Rg{HFY;ddHakRx z>!Ms-gqpH~5&jQ;#~>qUySYitd$G+lU!i`k9?igV{8#Im5ki*nQLGMI{(7@0L%aaz z8zA;khoxCQNYCduPaq`t=Hoy#ZFPs zvb2_;8phwI-2>yS$InITUfpGSCd9%K8@&A^aWbyl;#OY0x)D@gVMR6PMs4G7w;+Jt zy;Kpc_IGGz40sG~t{J+t>V^t_QQ{gV%g311Ey=4rUh%vYkuGD&kZn0_nJ;y6Xu7N3 z=4n}r*NbqoXSX$$2L6hYaH}L-pK?&Z8q0jlXykOD@IPh?3T8O=9%UcS5u5HDp^}1s z>2Uoi=?@k-96w^hPwH>aLtpnMj)!|EzM+(|@cRkPQ=&gY36JxVYZq-_K@0$%RH1jp z*5D5e5WtD3gR)4hHr}sEMD(70JvSD8EINwvU6E?ZPPSB_s>7+}(aH6UUOh7}2lKZU zHw~ctyPcBY&(o8R%#2QfT?=au?qCNVt7y({e-4^-t{*SRW0Wzr?V`fqI)bmE`{=<= zqY`lMJzhIb=f+Nig$2m{Q9_@}^!`e?dsr%&Pvp6=ibH~Bla*-1m$|hET>Yj{_U*ub z+PXjAzs}?d&r_1(UGyL#- z(ds>;l8ku6_@h#+>9>zBJRh(X@ zHwr%ITtAZ@psfg!)Vk`MZ?`uczX8yEDOIQ=ZK)2{m*o@bV(k_#G_U zZJ{C*AxWtA#X3Sci)FqA3rE>owmH&NeXWg^q z&SH-+tQg|vl!(r=pmXTKbiWFs_r@cP{)XzmWW;F2z$S+@{q%E!2x#zHv4Ys=+{2Bn z*1s?Pr^0+v3};)HOJ)nSo+9sP%sGNi60NcqXCY&;^su&8o{YbNtoNE*c=DWPVO^o; zJGE`Xi?e8s5YGDrgHO-3!&3rE3VGvn+YWLtdXau%72xxyWKK~~Yj}zzfFsN}bZcwM z&yxbBHRm1tXadwh*k;x+Lil)^rnxXX$y*gXX6T`~dE7S!~~( zw~anvgx^={SQxI|f?}C$?=ywRxLurlLu6Q)1svN)3RrKr#=I zgT~rHLbpuLe$u_oSJw;>N{vwVdT>_{M}M{yyH&iJv-M{+xDGz+uKV4>+zsv$*7Rhp z5%#y1IDJ;z7P%ME?{oGg@8@jscqGX2(ROob=3-TpVGHT1vZ6-UUw9TN9WyKpUo!jH z{@3CoY`b@vWOq6iw2z;2N}X%2%0CH1U2{I4Mat!bCKrW8!{8BwIKZPv`%}nlt2xKv zt9BxaRqflgUpaN!Plk(^f9su6o~_cBttx<6*f@9C<1>s0MF+POO*F`y zEd&pr?{*-s0cIZzpHzZum5BAL9avF?s$Uk0pQf;2#<7f0!_@c65>#D03xol zHqvu(eDm8%K{AIDGjG0jXifXBsFlB!^PRN*nH)h{$nQ4OKcI8Ef%wH8iuN#6445iJ zY=3bNxS$>4AAwA2-59Q12=2wVbBWJIFX!vgcTYW_ zZAa_=O~?7TZp`zqw1Vi?kp>vCuk^6psig_!2-L|huX{@L9jzZD8LmjAZp>Me9V4A) zLNI@+kwNs#Y3Ky#sQa;~}I~nc@pH$8YIPFBqa-L2Blmr|f4fn$TL@X1}DH9m_vrWjbM?o29 z)P@!HPlcvgD;=cG<=2+RpxON+J<9Pe1`8W2#^^;IE)H{^j?`g{QMd=4oAD^yv#ofF zcf|KF-ICJOdY1kt#UP(@B1M z+s{eXV-c=LrSp_&(mvT;9d7 zvQ?v1LLN7T84?XY#PbMu)4XLwipA0puX zqrk2zc{rsTZu@Abfm2EO@NK}gE=mKZoO1Xy1Y-@~6I#Xtc6gHcFr}^E;k||^d!^>5 z&Jo}ySj2jrv*TGmr%s2?8TPEcy$bwCXB#i(@aV9$OED)@3ue}I<*350vrK1CZcORm z@OVqsF6>Nl47JM7+_k@;ksYvVd-I%90Z`zKUa}rmb3T1ZB85fQ!}hz5;22cF!5Qh@ z7R=_zNF%3f%WO=6Gm=!5Rdx6WaBZIS$t$z!EESyzKQgpZ?m9zPJz^eREZoUk#{D&0 z-m27fJR1trsfDdT;O}84$kY2rb26uHEeprGrFOv6T`fU*pjSUIO#~V6m_9)K*YzRo z;h$jJTOo7iyXvpC{?MPi4qjsSaP<2w)7_qMLPVUo&Ue0|)?6q6EvMeHi-+XHKyI2t zg1j-Jli8>)nqRKD-Dl0uzVX9S-YbEloj>e1^)Jo?OLcCb&nzu5VkfGs+!?)s&^wOJ z+54Q);{ve2mXq1ePp~1;zc!uZXy<#WqU&PkT0OtV*Bh!}9!+0ooaatARS$8NzVbSGf%PRI3*N=PT?XuWFz$EjO#0TF$iSv<MW^;qgZ+OM?Ak~z&;36oFhSZ0DRI4rj)N18`Wu{qmCuN_q>U_;NMi16^+2eMBe ztQ9`zUpdd;=*vevbiOz4k=qeZ%68*(7Xn&j(aK?j%+lU?ad1X7pdd?N9qUXKZ&{bua z4Ac?R>}{gTauL#TLRTrfpFFSoJ7@SjeBuPdHg_0Hs%?V3bF7PmM_?EO3Z#IKQ`%~v zr!TJ14MGN34P43x_0|JN`AiEAYDdA|mVFPpHhYc(lG|P(qlx-^NWpdIlyCnlRQ-2O zO;B}a*Z-*-jM7r{w}pr}jub@PqKI|8qFXP-u3i#M5X$s=M0bl~tQxvJGqu;gZ2u@V z>ZRXIGU&=RM~Jg)(o!(Bl(ijSu=Nns#gi8aI}^xef^VKbAa80Nt$5{HZ?&uzDG49s z=$(J`%JI@&- z_Pkh*FC9#oR50TH*4DosJ1?MDG1M^ML# zcZAzP0y$Skuii85{|IFqHt9QCYA_dPoRc_$UdSuSrkE-CLT2hf6%2|w5eQq;%SZR*& znIpd^LC%lDh9wKP>y{6k&t8(MP0x*vKF%McZ*6v);+1D_2k8*0!;fs`ui(iX%B1RY4Rc>g|E*d#Vf7E-;c*HjX%B@PD zgqZtBN>=JI(fuh3PK)kG0!pqNdns&>Q6%z{8rV3t{r#Qd%5Ppa@@pA>$8Pzv(SuP_x+pqBu_=(`V+{RQ7DSHX3#9(@-Y(Lj;ZBY zmTU`CJo8vRF8=5^@uJX5(SQ9W+^K1-WAD#j9ZBfV!noz(JaT57DKaaHi8E`~Df-cD zR!b%RQRmjJqYcG)jQR;@RvvQ4%}5T=wU$jvkkKpFA<1>wfS~;rR75*oMi!C7K6PKe zKI~i7dbD&9Q@P1L|7&J;A`6dgXlb6?oc}R1j}NqKA~c7dCk7cdIXP|(#t(E}rk&am z;yCK#Zyr{W(KSg^U=j)*Rm^*;;`LF}Lc22P*E2Ddg!60Crz0Lt-@Cjc*(Gh?Xn!Wi zGBmK7k|pN2^BwP)&>_=2M3)`sjeKJ}F9EY~q$H6PIwMUay3$`C^C&s9r|<^16Zx^^ zqNr9)%c7`UWGU%^^L}@~quYOnH)kUPKusdQ4M%514?@enLQJ)GfDg$#ua{$$()~4U zQnxy4MD)H7VPY*Wkws-q+h#_XI@%5!p-nF$TbDeO2RAfB-F6D{g%85QCjN#4s+68? zh7LCXP4c-uJ2{#91EJo5e_v9X6xQdy^F^)}7v7Ejt2st|zG~FcXnnDA_uk5xr6NZt zmNSGYUYm)d@XGn*tF-N_UwfTUsiEYnE6tulxJ1BZH91PzMdoi#dRw~~fnWRnGF_#U z`nqDTe_i;c4Ak}h)Xr4l%SDi;7qzu0)XIby$ z9eYpkUH(Guj#cp!L9)`LI!0*~?Bed1RFy&Y>mIA#u$LEr?gek7s>SfY_cg*N+{Ot} z*U4%kX)|@Rx0tLcWLxqc z-k+QEuk88ysyg_Eoe)Ri>H>zydD7}WjeTX-6uIAk&4i#iL+QRJ_K(MTSlRi9*^a}7^HuhVkJ2^j?wUG&D{YARnY%~meP3n-x5d`R}c^dbrlng`P#Qr~R-Jg+_P zA#o$DwO7lCO0o$!dsp+P1eTvC#S=w?u%OHDwe$Vq?KzasgKpOc+&{h*ui%+D%4dGnP|Ol273 znUabN2>I}{qhwS0PTdjoV0^3g$zT8c@XWP(_ZnEs$BgGuP8V(2lRa+9Fc*NH7LOLR zn=e6_h703PFtLA0KO@Om0@Nhga@NWt$GM%~W0?n0q>`WqrAj&rP)daIGmoQUyY`mT zKd7^ii23EVVYWx)b${bY7RxZoOaB=X9!`E86jCY*TaC1~JCiBg@mXFtaK_DuBU@Qw z2DeL&Z@OrixH?puM~^CwhMoMz-OOzTVG+H~bB=w;=FoOb!Qb}7KOrK;gHz3Ect;t| zg6|^q=fCO4cS?G~wIy0Cb&bAI^L2_p_&->^3m}=G>OJ7#jQL#zNhCO9Sx)MB#X3=A zxx9-@_@4gpv4R5I*d>w42U$#SS~(10TWG>fppW)!o-`Wm$gFg z{6{XH?HqG_II`jK2{4o5V-9Tw&wu2}Z~GN%&}Du^eRb%k6Mrb*V40-M@Qal~I4LXM zpR`kD+y1nriHDov&*6Bbi^fF2YPJ?OY&e*0;YZ@v9ZOCAS)vj^n+oA~yq}iC4u^$@ ztoc{Bn=eI zk$Bu5E(u9bn;pFwei%#MUUV{_{fh;5;0_n= zLb5y&QwmkW-D}=PN^uvShp=acJ>#_vF%}mj;P{Q+sa|##?44&+Se5Ukwi52@gpD zjA=DCwM|+GiaoWreHzu$lJQPhAZVhWpr2C>JH9;^iU|1xUjKhud+%tr-}wLA-ZhJ; zJw9|;q19N4R*f#STBEj@6%~|HVzx$+D6LU@bW&SX%@_%yl&TR$tWu-ImKeXce!utm zopb+rpZhN-N4#CvmGOR!=kxK(x6_fKI8hv$R`ut{R8Tg-2R@CBI;+7x=cabIj_(|G z7KIfx(w6Q*2L-9=n4|vrEXmz0@jUa~gYj~u^5gGi8{Qb|pN%6gbo#DsK!wg<%UsK_ z`M1L9YoVmIw6)_gnBwt@6AVZduoz`~e!Gm-(LYxfo)An>LAS16lMybdGcPU2JwSfZ zRyi`%#w1QKOH5e(ae*_>LLeF@E*NIoWNEKmhdc-OLibp&P%pmLz`sIBjP_9SOYT@y z+Xi+T8&er~RPQ5BeYGw8f@IG^>^MAf?R>*$DaIm7odD^ns=4&(%a`Z1cdRLqrK0p2 zltl{|<^~76zx4{uo$;=to;%ApJMrVNd1qg03;l!)-ZGgF+ijToC^CM33Hv^rQttJn z;MzpdPv?eF*!5xN^||^-0$)OQCk^}?v{BUc`OE2f;eDJv_35@Y&4({r%)%A~(_c*m zbx*xgfG*F^r`myz|59fYMtG051;UF$4|$Fr@4rsBwQAmn>D21{83`-59l1oPLx=x; za{S>Sygb-_KTScWT4zQ$tSWLV9JIdYPoNOyo`R*7%;VLI zvg<$6FPl<-vCrq~Ccsv29waFUs3rXOSnMIhy0IixTY|7LIz77a8<}7&`JlM5-g;c{ zS1)f`DfDzD2IcV@$K5eCoE_mD*Az6dVRIf5K_2i;VIl}g(&hRk~d$A0x#FWPVA15TVZ+w^sy9=|eI{3$bXGjTd=>frYe8iG$0x;a}J=PTcgqO35c z{`yrQ4kv1G(Agk-(Vr#RU3#hCW#<2c^qxahZ)XsUR$R9W@*BaQaGK@XDohKNP9n}k zVQa6J;tE}?NX>iz(I=C<=y9m?Lg26V-oK+VP&MX*mPxCPre!Bw$a`}lc8anb_moZ( zm4S4$_rGpFAhv`$5pgs6a7kZTMRWBiBOYj%?qtq_QO%A7^p0{+|J3Ud+jZQAfB*gu zAn-eWxW5ex)(<`^QC3c$Ko;AEZBB3WE*!fC4^|(A?Ei#?!-M;42|dx0JMHr=O^fA1 z&Rf(chu-^>MzFGjuJHYw4PrR;&BmSsEqX%<6MA<)N%-jg{@y8?(A4T3q)C|kj|rx) zf+n}FE&5*5!H_Eg_|>F?xE$Zk&aEW4vvj=B;KhIxXmhb`dUxP=Nv12c8QTR0%gIHg zm{Bz54{DX8X8?x1givVsduh?#X6HgtzHOc^{6C$#w^8jn`lkH$g7!z}N77mj9DL4={PF|x7()3uVbtQr#>klf9PJ4<{XSvwKP0^68z0( z%DRkHTfXvVW&LJ!>Bp~=k5MyuIsNuHk|p-4q!i{}8hr<2r4xdF1T>K4Loc-5O;dRxr@i25XeDh_N_|Vr$sb8f4-DqnewWrF{ikMEPH; zt#yr(Z44<|x^*7Qg!V_Rp@$;Vh7vI1HBM?#E5WB<7DzTTzOEgYnXiQ3t#M4geV122 z!%K8f#S#_T{q0%4E>Nnb8~r`H$?f)RK`E9j2AhmW?poEZk@=wd_Mz@E;kUI$Uyn__ zZhAXj6+X8&;74A>b+&v8U!m&GzxzwtHZpx>YUd9hYdSX5M9z8CJMA_<_gGyzQ{NoI zH2$;J|L~}2{!LSC&!v#BUOBo)mp*4PaN;z>ZXj1MA7~<&f~cZ?z#D5(kSl#S zJhV&JhzSPf_x6*O^EnWU{mIq#`-RQG!kG)Dl|YmA0>ba!C{JZ?ZobdJd!Z`771}h| zKX(7+SX7~FKR}xKmBz{KSbz3Q5m}Ml0BO^e9m~by^7h)yIaP%9_0FHKBrR-sGP*8# zX{S83b-0g&_(-aeG_Lzh(TziK6o8kpD+#IDvvZax2^yMi(4Ei?qqyR-b|>zk0}GLR zptai7Klu4~em^F&h`6#OMse%U-zSfB8;J2`5|S+9t3s(YVe7nwgA*?s)L5+yDASj8 zw&^u=LTAi0FX|!bXu0K*AOD6Q(^O_+RU5QcZRm&cm6({!F8}7Vk*qr>U`Z6K{?BOE zU1E#aj4S0)B86`bLC;=$<+L1T7@F5J%8I7ggg@liS@vbDL^R@|^FD$&Yi8>TLQ1Pv z7uO>|S(p5k2_t92((_pclB}K7Pl;GPjhl9Oj}AE2e7aH$LLSXMvHMEMHsL~xuRZ(C zG?hb|V(-cQdnWwhZ=}NMxXx}Ltvtx&=ev8#Jmt`e{3VjkKrK5)w(4O><2#qf`IL{0)gb&#`BS%O^_zpr?Y1XfqAD31t#?Q zN?x0|pf?|0Re;RDB4?AqrwXR$Z&>JZ3Q}#yB|`c}1Z9@O@Ng^}dI5J2S1GOvm~r-^ zzUn4#J2m*<$8{h0;Hs+07n6U+4*iF$6UY5Y>krK!!mOW+|E2$^-S5-BUal6~@cQ~l zo~2*&K2`2TBuG0QWjAAvfzmuu|`Gj$d46-qs(uA>A&431sPpl z!7>@W7)3`#_luRwK(Oz{#}pv1M%{WV6|KS>y*wEd7_k$E6Y57@D zK=gD$qT5}&s-foWHyC`gf5GT0H3$$iobM7=6AsuH{+xQ;%Z&+``_-m=>IZ10FHcV{ z+6-G!@#FOq*ik0v%O|qSqJWqq?BUk_zq1j_X0p!5r#McMq8L47y+j$7q#@sdl$d<> zK@@6?mD#9}Rb+s*mE-x(Q&B&1pq|m`D#Fg~H#{iE&p~(oZr%7W%aD*`9>tj1s4>a7 zayB0^QrYI|lu1@h@Qf+t8PCzSM>;PA#1uQ)nBK!=8r#NNn*s3xd%cMZ){&UoeZA_g zVfePB8{?b*E-mbRr8}KXWLm(ogr?p>+Ip|&T!;gS=0S1Yi+hlm<-Q!Dhmk}o3xQ1! zXw9ys6zzq2%?$vC|C38eERd~m-Fn(}mbm!Ve(F46mhQWD^0dBxg}465z>|J4U>k~z z!+q6dKcU5-TKiAnHP$acrT=~k?g3I!MM#qr`(rMC9Ts7QU{mHjWlm&;rJxv#!FqpJ zjegd_CYT`i`+R=C2!#OMGr@?Wb5^a&n9h!Gi8+`+29jJ(cGVNFSxe7LsNo?GVR^^JwzHWx4cy$A0_Rw(M zDSpmmRe1Ex{Y2kFb+BpW|6FXvP0JIpYptW!L44*XJ#+zK>Du8>w370}rzz6@XS+lvsgTIr23bfEi#Nf0B z#&fg$m|8AOsI8b7)@xoLSMT@^Itxj>G+RpE(!J|Se)5vOa9(_}vD-K^2&I@Iu;=th zF*>+Co9_4eJzgV{9JVe~klW;({EI})MY%+QPI80YeI#3Ue}lY~P1#Gi)}vC!Edhtq z!(7xq*ZAaf!rYkTW9E~@^CwJ0X9N-s$NAR>Y#pO0V~Hi2>MGu(h*cF%UUWVh3f5*B)~|6!BPH@|=k+Ag8-0z!y#VWv()Y^I z`T(T`0U0u3d}3Gb+t{Go3sg4p0BQO(2LL04)dufdOf^gJpMMF>YpLTfX*Yqw}7@ z%RXmH${wBmvs~VC^cZR|UB=8hB?#1*YCD0Gc>c5sLVk=qpM!{a#&63=WQvVQEO@w@ z|FRW+u*}H7%gU>`$&)B3hLE~gia7WcxlF=eh6xw0l3H>Y2y71PDtMlXp4mI^R30f| zTV0BE<#RCitR-+(G3;ZpS{MuIb;B;J-6Su-Ji6bdiE9pBOUMKF%tOI8aF(7S5nefM96|l)gO2|MMaCffpimrrhq~_Vi znIhDqv|f#SGMT7$vRr9>;ye%XRp@gbCJ24;>=3-B+{G#4~WRi(m-`^^XoQp5q;XNCb_Or<_dq*7)XKqDUNUeG}fRx^3i>Y z_YDbF5Du7#?lkUIi9j2d?baUHmcwe?)E0M57>`6m32eM0RP{7PHx!`nEma~Y{x+4% zPn@TkE!;_%tf=xba~Q*mx+ay1KZ_pCl$|p>SftA*fjP&tBxx#;?)37L=eH%Q3SG6&GRu58!>V=eV>gP^-gD1N?wi&6Hywd+^|yB zxggHQyHJ#r!zSrX5q5~j%tbrTH1-*N0+mb4@Dqa=J*0PZWl?uQqy0S@=E$+wA?*FS z*tiVXrN!{m*`!H%SdkXoO~e@c_|^3eJ)`%7YU`kAq_nl? zO^&jXzFc=zF2CPKfD`+$THM3$>%6{$g+bVGn=IeFnxqp;Xge!urs15W11ex2K0-Ce5mc#KDyV3}?=FOb7TOu^&+xWWX?av83@1`i|CyLOa!>pG5 ztWNN@a^reiXKYjBBiu6U0g)dt#2g0ohfJobEsvj=2Re5X4oqId&2~B8n688_HIW7O;N{{h^8ueV=_34~EFNBwS49N?(W2>71^4(v! zRB;aHE1I{t$_c&78r~&*EOXk7r2p7zeqH!1Uk*{FC6v-6*0CTfx!>hH=-xP&a`Ubm zDx!nV>52<)vS~Q=M}kuM8L5j0)!PK5DvdB z&@jh|9eW~U@F3Sr6P3Dk#i42&qS}jL4(*{@*Sote#keM`d($&fF35 zoe4GU87@9%GqGi^`0w4NRd}xo)^|llkY?;prz$U&8moj<8g_zt2|NFQ-k^R9#%{f4 zM|+&lSLMKXe{pS8w^1D)Kwpq^-e8UZH1A|l$jwnPTq z`Qx;ez2ThGzJ>Z0P`L5n&tgdj>RX4e9$BOOaHg1@F{8)4kPfzRDVg>g2^(yZ^J;D= za-;8+8os%-vmx#~=v!qk{R6~YIp^4XFQyUoL@c#Xs$nm;Qh#zacVbShM{oY-g-YwN z(UpkzAb+uXk4o5(WMG1#8%*Ij!eUu1HIkj^mb(5R1jiqxOD6m&=-D^m{FjsS%mU25 zzJ(tuOhWZS3SG;Y`EibX*P{B;uE=`Q(`)Qy35glcAaeqjHszQAgM+@4HFgr^tbNHQ z?l57_pB3wy>2qrg19rVtxz;hS6=ukJ3ynJeE zpzzu|M%uMaAG=1@8TH<)%`lm$s|GkDI7UmEBghrCJlxL+0OhVUDM)vC%tZC&N;M%j zfsotIliYdQs$WA_UT9g*yIk;CNL+Z_cf5@^ME}75K&d}%7EEzj29&|xgyuomBDki> zwdUYNe+k|!;u-q5`g2=c?ynT7vM-R~@FL%B<=t&r``KSr?EJ(Vupg&yo6cpet<9HqAGNE6`9Mv`e`>T50* z;fg1`Cv>JHD!*^l04X;@G01B*5ftmG_R)T1<1tp<=yuW$t`RvQmZZiHx0>!hkqcCveDS@ZINL$5*4~@MdXK*2^=RWk05O(5G z>FcO2QiRy<7)5Nz2tF8V)lzQP7ksNB-<14dB|7h;9`&2X4&iV#-OBKRhFaQt@4m4$ zElW)-mkr59TP_!XL0A$>o@?a}VO9l=s0Y9&6ei%4X^RIrXnO%Q7*yQhhw^G~@c+G1B9Lguk%zE_O~8(SO)|%NOO_h7Weip7mUkXnDerXqB=1Oj z{g@qngc2i83y&8tj2MvySWvMYyfpxSEkL`92OV3wJltG%gY-^`X18#jkmj_!m(-o! zI|;lA09RkGp1uGyEdll5l9Q`9Y?)_uR+eVIdId4mX4v2eH0tb%y$>p<0?PalB3M9L zidzS&h@h==QPmvqyueSoGc3CA>;Mpc)90(^pRvr6@hp)Wznm_0+yjqc{;3njl3cAo82QNL?SpM1~T z$U`73p&}KziNuU;dcUN)G*skE99$HqGAj*r6hGRp`ynmRcBO2bYyc?(?}h2bPoWqb z^WO0`goU+d4WT~Jbkarv0E(^T4TpBb~)y9F@Q0oINanR%`1;x$FbFe zv9ujVa+JCqX0|K5q=3zzZ}M{QC5h-4+y)X?wJAa}6vLNY+ONdm44Pe0;aL>zTIxCZ z$5qfhdB_N=L*8NRJx?~KUdG~^@T#vaDPjAeS`1ote9?UF9wL*>%O(#HK1mAs`kCAG zB%}K|+Fzr)!2ywBpNTNbTL(f(Gv)&1Crl+uh(SW4wKm+i%87BPh4Yp4CW?#Vnn-p* zJp!X&2Qxa~(lo6HPQG0ppq<99_+;L+M543&nM+et=t2&#w`;;MEUKy+Yuwgw1iqKTKVpPH^~{e*>;vyP(MQgL3RG0I+qqmD%@4`Inja91+$ zYHtN6l1mw=18g;qf5jD-?T<%yfQ!3hamdhmxtA{G77v5D_)mp?|u37kv4kuHw`qn zaGS#;j`M-+-pebk+_%jiT5Y40F_ zSRb9HijS9%F*tWxT?;QZW&B<_J8&a>HfZ=n)q>84nsBZeQ>?{+?+oH`DbIH(e<{c| z>IUzX?8nO+K5R?u)4o&U@0lg$tGcc9otTLhm!+2V53KJzv!V3OoK|>CLmkTuUi~de z3uvb6yCamWg2E)C;!qfu=ZXE)jMtu(%p2%stWjA?QR&$Xa-ZOD_PBb1@$yU@)>-q= zR}pn){f&$O0XlmRaSgebpAgYG2fh-KT4*AFEOiF;0eG7Y!eD37hnF(5AMZTJww^x5 z(`l!;I6FI~()W$VK4@J6{Sl3e&$|8K<#yx|pZP4KjilTX=M!`E{vBlU_NkqMo>O zn9tdKnM$Lvw~*P%ye(ZNpsW*%5M2548It>QPmI=Rp%^l^qrT{@D|+DNBj_Eewd z%*ZLRmN$tgH-NZ7VvfD`o|~gA`c=85*sBC~Jk-_Z$uX(9*shhX^GYEqrP}2C$D^d@ zs0zuFUoQoDoR`j{p`_z3h*M)i=j-3RU#`Xrq#SA)?i4iZ?@ausUULxrJGo>?%f0pd z1;Z2ETvFc7%D7p_j*O>hM$u#hNOa{a!<@R!VN6MDIEBAY^LbXFi2CfQB!YZpAWqtv z{SCYE*jXJGF~Zp8_|f-V+aDa%Nocux_Tobpd6@-|^kweWd&l@f6RR2hV@Mu2nn4e- zA_?~HjQ{LPQ85`jV=WnXDqlP$xwvNa4Zhz>Z682=J#_QfUtrCsQRo$+dMelYvFsEG z0WjfkPsu&&9y2$T*{v^slUNJDeG$vA-kRP)#JxXqL*TL!{pl7t_w&D#)oC3fv~0#& zam|y$2k#n0j?soyokhgaG>iKd9^n6YK{W=j-ksOoT5(zjq8#4w&G9WY0qF>TNDzPN!C)X}q(}2x( zxaMuXND9c%&z5b5JB2oW9BGw+YAL`QyVEN#%z#JZ$Yum`W-?1t?lb+>J22V#!*;u0 z=V%g^04ys4xZ&JeE4)-EyOdG&wM5|*!FE^2wW?S&3J6RT`UH%2r%wo7k4qgn!?2w} z$y?a}=ml_KCd7Vd@ue8Rat!d%aOVY7mKiCt6rrFWSGamn2j7*!h1@iR2+T499_jd0 z9dAL*om~#{;ny}qHd_4f@jNe+@ADtuD$QEo%}2WseJ+vca|j;xyU)cZ)*)Z!@%e`{ zOUnntpDx`Yd)@za5cqD`(<&Nv`X9R(zRrl)kj7{;OE;@ z@7x%q+xA*DClBldYn!wTNfEzrc22cDZ9*XZ)r!yK48JnP^?d>MzbQ6-TYL|q#!`zi z&I+?@6$BuLu$cujwxGt~?P%NW%M*{OG*2|V zMi}FcvyN^mO#=Tydvv~}yGAN}=_lWNw7m8Gk$nw}z9YtixN zsQk5;)+mVE!8;%*WvP?>N3#<13b8hM5aT_J4|I}T3^{=zh_sxbbN_9^9VbE{*{7wZ zIc3#W+KsYOB{F};!tp1x1W%-Pq2Lc|B)$@xK40uQ>gJ6Q-id5Ba@(HHpkx8@>zR|4 zJr>IL{r|RvQN;2%ul$Hwqz$X*IPRGZIA4m&hNl|u-=aHR1Cg$#qHLLK=}K+L4`E(G zC}W+G$lK#Zs7H58I{bmv{SoQ?m>A7|;xln))9}XZ01hmf4xHQH3AM7dQMQ$`@&YipabO^BD4(y*Fdzt4n%2!KTljJOx!)h^EChina%v?bcx^ z6CWTN3@z(|d9!l^4u}y{>cC~s{ZXNOm59x>fy-0GY1cdU5=j~|UQ^T$N|UTb%9U<+ zukppARDkS7Isl|Ssmn;w{X}&aukaCj$3_>P!JSw02VzodN#J^;(rEfTM+;gb2~?32 z&n;AG%%4!cn4%32y+tNFnv^2)WE~yht)ngdsK3>0e)y-3{|WTe^@`N&*{EL136i^9 z3}tNSvFS==hPRD=e}d=8Tbe3!^}E|wDObOg&LHb`EVb~bE?y;3azQED0qe@J>kFWZ zn)xi(@QwMStgD99yK;K3D6wC(_JfI}Y`qe}k0jXLa zYQ})2(0F$G8DW+4iu_4=$ET@soGIrUU*&Epw~mii*bk%-8E1S#IR~CiO-*$ z!?QI|pDB@s+^O9{Ul~5o&qET^H?L=vri?hLBxLRL^T7>hcaXoXTA=5GhLy8lR9iqyc(GtT z?IbQ18~%IYpdf_{@e53SW;~_E-`HkfK{_Sv*cie2!gGg5Hfe+W&Bc{~Xh; zGNFgZ`P;Wcr(Nb7U4Ny_N$f^0!@G)$O)FLKKiU6)cnqeds4*RfK!kv5w<9y*y2AzN}ey48a`AA z@X#L6)^<@nm3_2zJBll_KUo79|BxQ($N%MwXQk_p2B}$R8-SnVy78tLU(BNvq7P)0 zr!`ZsJZYl(Bv#+MWsG2VwU7aaice&^M;b%^2pO+q{AkH|O6g=*?NcuL)tL}fHK^_5 zC?%@>u0)k7-Km*C-t4E_bH0s5!@UR~vfKaiu#iXoT$nbc?k;ksIAF?%z?CL`mD-k* zjye$I-vp1I2M$gZ`bpLPRrz5;eeEZ5h6CT6ME&gT7t0j9&MWiEDzfU?@klTzC31p0 ziZQOJIQRI$%Y6p*ti86Wr=Prm2?lQ8-(B_hDn%ZoCjkP-#+^FM&eVPL!#FKL00R>d ze!EBlY|g88Oa_Gn^h(X|=jYfKor1Mj+DYhF{$a%de4Gbi%2$x}6p`qWlvBCIY-u zfl7o$hM-q1fEgsqdenG3+P!LE2OS(XzqU31(k6$_14l%!V^ROq_wDe8Poged@J~_5 zjz@wLJ8y1jlF$kAcg(~w*cg`SA;(FRyKHkPB3jz(4tRuJ8&DsJLwSE(x)AvIf)8K& zg&cHQf!CqtIUQCWId{IbOu!|_WBbWOl)!ApKHni+`=>;O@&`*WU6%fa?*tH#RUm$3 zF3!Evle{gVpm!5FGx$0TYd@7AW=ImF*7aIm5n2Wpha-Y}or$bVV>u=+(7WlcIl4{l zupKHUdT=pso@r2rtxM?)h~rC zsD}D^a{#fZLOzG60Ngwu11QV7^}|6-QR5?*m)yi?b-e_cgl*8Ek!5fdPo=Hgb!9Q) zXCbjGCS$AwUf$j2_@TB@fd3bk9rvo_FlD**UPy;mZ)5d}MekTGr%CA=)X~obxNpz$ z;->r9av%}QnYv4ETMnn`*iCKZjE3>&d!2P-w)k*YR1?{|5V%E~+XwKda&sWxu3<~8 zL~_9&)#57*U>Bq<56&=G$~;2+d4Kt324RR96*hvkmA1h}Z?hF@ue;T~ljYkpT~e`bxv93-X`Q+>rV9+m@^Asa{ZQb~Zu^Ao#+ z!%bd#*!P?sC=wM%zBWL=+q8xfc2&c+wjveeSZ*O!I$UykwEEzG#Iv9~O#*<4r81g* zb;4wT-kj+@cFG9mKmiMNKqE@ZzAlwMB`)qde(_wiJgwduK)VS`Ts|L83{@nIdP&p9 zXOz^8l=O4<{Ld`o4VhxKYJoW;i68hi?jP7Z#9CB45)RahcHsXLPW45lR`6s3)vXv`E;rm**#G=R zyET#@sGeQs>{CSppwV&6{|0BACT#$j9;jO3!GKPHR>`xat5(9}j=K&J`hj{@I|MKv zBVhj6kv6MZyHly(4PkTp{17e@%diOQkJE8;JUz8Csm{Q9Hky1+L2=-@)L zSxM&qcY|@WGnz5W#hyH*kRRMGdlO&W`~^rIpp1OpDS2n z*toYJ;-4N3oxM&{fHZ&WO9*N_XIpn}gn3HC-pz*#H#FteP4ebbwQtNcL7*a24hs;q z#G8f9w}`~d#7yta4VC$W>s(=Bj8=6#KwRE17Z<1U?u;pwv9})*Cba|eI|=qqw#AF{ zoL(v^w;xKt{xZZ-cEgO-4BS)WT2yKtg_v-MsYOmzWS)n@$sm!fP4Jz~Y=B%O643EO+0H#lGD0}30$zkQe(_H0(u{}P!sHE{uE zhKzn``%_)V4Jl|<3xO}1W~pjg8UR%K(W*5m>ZQ{GS;-mz<;e+#k zpR~|E^3O&n`9D_zCsHm7*E|20M5B3P0(s zW$mJY^G8$v!RS_^FpO{SuY;BW7gp zB$e_+XzVNHjG81W%*QFT-NQ)EtHa1Ag?8TyE0NMs?X`)c`c}p)JNt?X0%GKxA@2!` zIjsosza;&Y&sRg4u`m|v5R{9KNbFOwSQ#$?dN{FTm^J)TN_MMZufhl8*2?wES>70L z1i)sdIE!N7V{cA|gPfccz~W|l?tQo%pGgL+=MyhF9@TQbzZ&A4*;ILPfB8PnY-m-f z2<5)9FZ~xZw0hlw75~BrSXqbv;jO z-3Bv|HCa;CL}>Zpqk}Nk!E1g|z~}Nohx>s`LU;hbKRQ13=j&yLRFAkOq4**=+w7dV zS<=TR&aL&J*)!NX>Lh(G>PpfTuUmF+f5Mrvy?!U7D#J@6c%F6tSWNGLA%-=zT@k~FR)(Y~ zI?4KkL8G)TI%$u6wvX!1Ndm+b*Tk!E#Hb7t;#~A+wi#Q3mcBRV@Q#ZcCSLT=4*yiA zAg`g*R?6)~jV3c+&c~?PAhYyJ$D6P(xT?2yaAD>UBMxrW*6jr@U>PJy83e!QEabs0CEJMFu%J4Q4<2CzOG()E>Zb$OrP zzA|rnXeA8(DbaiI2rOz&Y62uewO=HrIih(cD{ub2fd+&|tuR#T`_Y}3Jv&zW$?soa z4@obh1lEG&nUQ)Hu9D8GG^Y0^w~omn-b5{t!tfU$WQCsX3j#KV%VPoqWvDZeBRLXV zUIg{peJ#7N+zm-1ss9Czfa#!wBfvWyn~xFs4o0WOUH=&?y)$D(SF4L&{lPovc6;2P z*`O4%24;vACjzaJ&1{|jjB;Xgci){&>_(p!D|pHwme&UD!2J=({vybIf^BCV{)L|Kbv2u45W1vP^%MxZJe}?IDaB{Y+1S)R+{rC9#UVF}4 zyrcr?1?F9GqYo9q;xeTrQl#q7X-(i>{6+LlUBmBRDltBP0T}ylvEVn~V19k}ItDU#xuRAmQ_gP={!)>{v&n{lgyMC8{ z*yTSB>%sU^o}uWPX}i(ubo)#~|8xx;S|K|nYcarkWV5afZSRx9Hu z!B!OHpEKjTXRKn$bSr25RtPFD2NcE=n~Bh&9qH?R#j2DXscd)RJ|r?u6JsG!&TtUF zTIssYzv>}*JX35Z2xlUw1>_XSL5D7xuo2%KigNf}ocz9MB*gj;ZzX`x{@(2-!>jn^ z9FDJ5DS(Pk4b$p+`fdc@iX=Hwos}ksdtk>1sbW!!}miW z0#zp5i?G^+$|!;H7maJEwT2HD)>$bOhcnldBI2e30IBlQBBF-sSm-}et-BThJ>rEM zuxY)tke-VFMELI2A>!z8Y3{?R`+(ESjxvc`>`AAxaMt8I&t`)YXDc*a!= z2kOGN26W_!KiIv&KQ&6fLB8b4`?RLYsMLWSxXp4KC}co^=(0UxfY7_fR_f1%GsF2N zDz%({7~=OL+&cVnH&8BeXa4^8q^6PxbY6ib8WZUsD7_N+g;4fuy@r3)6QO_141SL% zCqV}us6kmHEf@fmBoIJY0jzYkzC;mqHv|XptPC)m2*e)1V=UMIUk~f(*Js|Zw14-5 ztG@dTs4jmggCEbFQewrw5`_0)TOBALGJ!76%7x1;*h@p0lP8q%D%iNh6wKpmM4aji zQZMOym;7|I<93!X2y<^2Yd3>VA68n!AmltA8 z1}!wl<7oOMKywHc{p@95C)v75w2JoE53T400zkUYxLk(uQtKjU`s?^O@9s0!A{o*b z6h`Pt9~Ewu9F`(tn*AuDcIZ{!|7T+9c0j)h{8V2pg1yYfO(9+>Xv`li&q1VD6yu|yA52CbSBqf7?`PxTJ&o7%A2KZ b95bGV*re8remR*cOn2SD