From 098e0816554d5c15e6f963aad380e2f1975ed530 Mon Sep 17 00:00:00 2001 From: jiangdw <598965300@qq.com> Date: Tue, 22 Sep 2020 15:46:35 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E5=A2=9E=E5=8A=A0parquet=E5=92=8Crabbitmq?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E6=BA=90=E7=9A=84=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 2 +- src/views/datax/jdbc-datasource/index.vue | 120 ++++++++- src/views/datax/json-build/index.vue | 10 +- .../json-build/reader/ParquetFileReader.vue | 216 ++++++++++++++++ .../json-build/writer/RabbitmqWriter.vue | 232 ++++++++++++++++++ vue.config.js | 2 +- 6 files changed, 572 insertions(+), 10 deletions(-) create mode 100644 src/views/datax/json-build/reader/ParquetFileReader.vue create mode 100644 src/views/datax/json-build/writer/RabbitmqWriter.vue diff --git a/package.json b/package.json index c05c6c15..889a0399 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,7 @@ "nprogress": "0.2.0", "path-to-regexp": "2.4.0", "screenfull": "4.2.0", - "showdown": "1.9.0", + "showdown": "^1.9.1", "sortablejs": "1.8.4", "tui-editor": "1.3.3", "vue": "2.6.10", diff --git a/src/views/datax/jdbc-datasource/index.vue b/src/views/datax/jdbc-datasource/index.vue index 57d2a63b..2dab1e67 100644 --- a/src/views/datax/jdbc-datasource/index.vue +++ b/src/views/datax/jdbc-datasource/index.vue @@ -95,10 +95,10 @@ - + - + @@ -117,6 +117,33 @@ style="width: 60%" /> + + + + + + + + + + + + 确认 - + 测试连接 @@ -228,7 +264,9 @@ export default { comments: '', datasource: '', zkAdress: '', - databaseName: '' + databaseName: '', + extra: '', + columnx: '' }, visible: true, dataSources: [ @@ -239,11 +277,16 @@ export default { { value: 'hive', label: 'hive' }, { value: 'hbase', label: 'hbase' }, { value: 'mongodb', label: 'mongodb' }, - { value: 'clickhouse', label: 'clickhouse' } + { value: 'clickhouse', label: 'clickhouse' }, + { value: 'parquetfile', label: 'parquetfile' }, + { value: 'rabbitmq', label: 'rabbitmq' } ], jdbc: true, hbase: false, - mongodb: false + mongodb: false, + enExtra: false, + rabbitmq: false, + parquetfile: false } }, created() { @@ -254,23 +297,68 @@ export default { if (datasource === 'mysql') { this.temp.jdbcUrl = 'jdbc:mysql://{host}:{port}/{database}' this.temp.jdbcDriverClass = 'com.mysql.jdbc.Driver' + this.temp.extra = '' + this.temp.columnx = '' + this.enExtra = false + this.rabbitmq = false + this.parquetfile = false } else if (datasource === 'oracle') { this.temp.jdbcUrl = 'jdbc:oracle:thin:@//{host}:{port}/{database}' this.temp.jdbcDriverClass = 'oracle.jdbc.OracleDriver' + this.temp.extra = '' + this.temp.columnx = '' + this.enExtra = false + this.rabbitmq = false + this.parquetfile = false } else if (datasource === 'postgresql') { this.temp.jdbcUrl = 'jdbc:postgresql://{host}:{port}/{database}' this.temp.jdbcDriverClass = 'org.postgresql.Driver' + this.temp.extra = '' + this.temp.columnx = '' + this.enExtra = false + this.rabbitmq = false + this.parquetfile = false } else if (datasource === 'sqlserver') { this.temp.jdbcUrl = 'jdbc:sqlserver://{host}:{port};DatabaseName={database}' this.temp.jdbcDriverClass = 'com.microsoft.sqlserver.jdbc.SQLServerDriver' + this.temp.extra = '' + this.temp.columnx = '' + this.enExtra = false + this.rabbitmq = false + this.parquetfile = false } else if (datasource === 'clickhouse') { this.temp.jdbcUrl = 'jdbc:clickhouse://{host}:{port}/{database}' this.temp.jdbcDriverClass = 'ru.yandex.clickhouse.ClickHouseDriver' + this.temp.extra = '' + this.temp.columnx = '' + this.enExtra = false + this.rabbitmq = false + this.parquetfile = false } else if (datasource === 'hive') { this.temp.jdbcUrl = 'jdbc:hive2://{host}:{port}/{database}' this.temp.jdbcDriverClass = 'org.apache.hive.jdbc.HiveDriver' + this.temp.extra = '' + this.temp.columnx = '' this.hbase = this.mongodb = false this.jdbc = true + this.enExtra = false + this.rabbitmq = false + this.parquetfile = false + } else if (datasource === 'parquetfile') { + this.temp.jdbcUrl = '/{local_dir_absolute_path}/*' + this.temp.extra = '' + this.temp.columnx = '[{"name": "mmsi","type": "String"},{"name": "rot","type": "Integer"}]' + // this.temp.columnx = '[{"index": 0,"type": "string"},{"index": 1,"type": "integer"}]' + this.rabbitmq = false + this.parquetfile = true + } else if (datasource === 'rabbitmq') { + this.temp.jdbcUrl = '{host}' + this.temp.extra = '{"port": "5672"}' + this.temp.columnx = '[{"name": "mmsi","type": "String"},{"name": "rot","type": "Integer"}]' + this.jdbc = false + this.enExtra = true + this.rabbitmq = true + this.parquetfile = false } this.getShowStrategy(datasource) }, @@ -293,7 +381,8 @@ export default { jdbcPassword: '', jdbcUrl: '', jdbcDriverClass: '', - comments: '' + comments: '', + extra: '' } }, handleCreate() { @@ -373,13 +462,30 @@ export default { if (datasource === 'hbase') { this.jdbc = this.mongodb = false this.hbase = true + this.parquetfile = false + this.rabbitmq = false } else if (datasource === 'mongodb') { this.jdbc = this.hbase = false this.mongodb = true + this.parquetfile = false + this.rabbitmq = false this.temp.jdbcUrl = 'mongodb://[username:password@]host1[:port1][,...hostN[:portN]]][/[database][?options]]' + } else if (datasource === 'rabbitmq') { + this.temp.extra = '{"port": "5672"}' + this.enExtra = true + this.jdbc = false + this.rabbitmq = true + this.parquetfile = false + } else if (datasource === 'parquetfile') { + this.enExtra = false + this.jdbc = false + this.parquetfile = true + this.rabbitmq = false } else { this.hbase = this.mongodb = false this.jdbc = true + this.parquetfile = false + this.rabbitmq = false } }, handleDelete(row) { diff --git a/src/views/datax/json-build/index.vue b/src/views/datax/json-build/index.vue index 3488be54..545a003a 100644 --- a/src/views/datax/json-build/index.vue +++ b/src/views/datax/json-build/index.vue @@ -229,6 +229,12 @@ export default { const rdbmsWriter = { preSql: writeData.preSql, postSql: writeData.postSql + } + const parquetFileReader = { + + } + const rabbitmqWriter = { + } const obj = { readerDatasourceId: readerData.datasourceId, @@ -244,7 +250,9 @@ export default { hbaseReader: hbaseReader, hbaseWriter: hbaseWriter, mongoDBReader: mongoDBReader, - mongoDBWriter: mongoDBWriter + mongoDBWriter: mongoDBWriter, + parquetFileReader: parquetFileReader, + rabbitmqWriter: rabbitmqWriter } // 调api dataxJsonApi.buildJobJson(obj).then(response => { diff --git a/src/views/datax/json-build/reader/ParquetFileReader.vue b/src/views/datax/json-build/reader/ParquetFileReader.vue new file mode 100644 index 00000000..9345bfcc --- /dev/null +++ b/src/views/datax/json-build/reader/ParquetFileReader.vue @@ -0,0 +1,216 @@ + + + diff --git a/src/views/datax/json-build/writer/RabbitmqWriter.vue b/src/views/datax/json-build/writer/RabbitmqWriter.vue new file mode 100644 index 00000000..c2e812cd --- /dev/null +++ b/src/views/datax/json-build/writer/RabbitmqWriter.vue @@ -0,0 +1,232 @@ + + + diff --git a/vue.config.js b/vue.config.js index b991ccb7..332d1828 100644 --- a/vue.config.js +++ b/vue.config.js @@ -11,7 +11,7 @@ const name = defaultSettings.title || 'DATAX-ADMIN' // page title // use administrator privileges to execute the command line. // For example, Mac: sudo npm run const port = 8082 // dev port -const apiPort = 8080 +const apiPort = 8088 // All configuration item explanations can be find in https://cli.vuejs.org/config/ module.exports = { From 8151a9f49c543f61f79c746ad990238756b49126 Mon Sep 17 00:00:00 2001 From: shengan Date: Wed, 23 Sep 2020 17:06:41 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E6=9E=84=E5=BB=BA?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0rabbitmq?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/views/datax/jdbc-datasource/index.vue | 118 ++++--- src/views/datax/json-build/index.vue | 268 ++++++++------- src/views/datax/json-build/reader.vue | 63 ++-- .../datax/json-build/reader/RDBMSReader.vue | 267 ++++++++++----- src/views/datax/json-build/writer.vue | 103 ++++-- .../datax/json-build/writer/RDBMSWriter.vue | 225 ++++++++----- .../json-build/writer/RabbitmqWriter.vue | 310 +++++++++++++----- 7 files changed, 876 insertions(+), 478 deletions(-) diff --git a/src/views/datax/jdbc-datasource/index.vue b/src/views/datax/jdbc-datasource/index.vue index 2dab1e67..898f36e3 100644 --- a/src/views/datax/jdbc-datasource/index.vue +++ b/src/views/datax/jdbc-datasource/index.vue @@ -9,15 +9,23 @@ class="filter-item" @keyup.enter.native="handleFilter" /> - - 搜索 - - - 添加 - + 搜索 + 添加 + --> + --> @@ -37,8 +45,7 @@ - + + + --> --> @@ -60,12 +68,13 @@ @@ -78,7 +87,13 @@ /> - + - + @@ -100,12 +120,24 @@ - + - + @@ -182,15 +214,9 @@
- - 取消 - - - 确认 - - - 测试连接 - + 取消 + 确认 + 测试连接
@@ -216,7 +242,7 @@ export default { components: { Pagination }, directives: { waves }, filters: { - statusFilter(status) { + statusFilter (status) { const statusMap = { published: 'success', draft: 'gray', @@ -225,7 +251,7 @@ export default { return statusMap[status] } }, - data() { + data () { return { list: null, listLoading: true, @@ -289,11 +315,11 @@ export default { parquetfile: false } }, - created() { + created () { this.fetchData() }, methods: { - selectDataSource(datasource) { + selectDataSource (datasource) { if (datasource === 'mysql') { this.temp.jdbcUrl = 'jdbc:mysql://{host}:{port}/{database}' this.temp.jdbcDriverClass = 'com.mysql.jdbc.Driver' @@ -362,7 +388,7 @@ export default { } this.getShowStrategy(datasource) }, - fetchData() { + fetchData () { this.listLoading = true datasourceApi.list(this.listQuery).then(response => { const { records } = response @@ -372,7 +398,7 @@ export default { this.listLoading = false }) }, - resetTemp() { + resetTemp () { this.temp = { id: undefined, datasourceName: '', @@ -385,7 +411,7 @@ export default { extra: '' } }, - handleCreate() { + handleCreate () { this.resetTemp() this.dialogStatus = 'create' this.dialogFormVisible = true @@ -393,7 +419,7 @@ export default { this.$refs['dataForm'].clearValidate() }) }, - createData() { + createData () { this.$refs['dataForm'].validate((valid) => { if (valid) { datasourceApi.created(this.temp).then(() => { @@ -409,7 +435,7 @@ export default { } }) }, - testDataSource() { + testDataSource () { this.$refs['dataForm'].validate((valid) => { if (valid) { datasourceApi.test(this.temp).then(response => { @@ -432,7 +458,7 @@ export default { } }) }, - handleUpdate(row) { + handleUpdate (row) { this.getShowStrategy(row.datasource) this.temp = Object.assign({}, row) // copy obj this.dialogStatus = 'update' @@ -441,7 +467,7 @@ export default { this.$refs['dataForm'].clearValidate() }) }, - updateData() { + updateData () { this.$refs['dataForm'].validate((valid) => { if (valid) { const tempData = Object.assign({}, this.temp) @@ -458,7 +484,7 @@ export default { } }) }, - getShowStrategy(datasource) { + getShowStrategy (datasource) { if (datasource === 'hbase') { this.jdbc = this.mongodb = false this.hbase = true @@ -488,7 +514,7 @@ export default { this.rabbitmq = false } }, - handleDelete(row) { + handleDelete (row) { console.log('删除') const idList = [] idList.push(row.id) @@ -505,13 +531,13 @@ export default { }) // const index = this.list.indexOf(row) }, - handleFetchPv(id) { + handleFetchPv (id) { datasourceApi.fetched(id).then(response => { this.pluginData = response this.dialogPvVisible = true }) }, - formatJson(filterVal, jsonData) { + formatJson (filterVal, jsonData) { return jsonData.map(v => filterVal.map(j => { if (j === 'timestamp') { return parseTime(v[j]) @@ -520,7 +546,7 @@ export default { } })) }, - changePass(value) { + changePass (value) { this.visible = !(value === 'show') } } diff --git a/src/views/datax/json-build/index.vue b/src/views/datax/json-build/index.vue index 545a003a..77b9969f 100644 --- a/src/views/datax/json-build/index.vue +++ b/src/views/datax/json-build/index.vue @@ -8,19 +8,23 @@ 4 -
+
-
+
-
+
-
+
1.构建 - {{ jobTemplate ? jobTemplate : "2.选择模板" }} - 复制json + {{ + jobTemplate ? jobTemplate : "2.选择模板" + }} + 复制json (步骤：构建->选择模板->下一步) {{ scope.row.jobDesc }} - + - + - + - +
- +
- 上一步 - 下一步 + 上一步 + 下一步
- + diff --git a/src/views/datax/json-build/reader.vue b/src/views/datax/json-build/reader.vue index 40fc5448..d42451e2 100644 --- a/src/views/datax/json-build/reader.vue +++ b/src/views/datax/json-build/reader.vue @@ -1,41 +1,62 @@ diff --git a/src/views/datax/json-build/reader/RDBMSReader.vue b/src/views/datax/json-build/reader/RDBMSReader.vue index 76e7bbe5..a016fde9 100644 --- a/src/views/datax/json-build/reader/RDBMSReader.vue +++ b/src/views/datax/json-build/reader/RDBMSReader.vue @@ -1,8 +1,18 @@ diff --git a/src/views/datax/json-build/writer.vue b/src/views/datax/json-build/writer.vue index 7c3bac6d..16ba03fc 100644 --- a/src/views/datax/json-build/writer.vue +++ b/src/views/datax/json-build/writer.vue @@ -1,60 +1,97 @@ diff --git a/src/views/datax/json-build/writer/RDBMSWriter.vue b/src/views/datax/json-build/writer/RDBMSWriter.vue index 48b990f8..527f5f15 100644 --- a/src/views/datax/json-build/writer/RDBMSWriter.vue +++ b/src/views/datax/json-build/writer/RDBMSWriter.vue @@ -1,6 +1,11 @@ diff --git a/src/views/datax/json-build/writer/RabbitmqWriter.vue b/src/views/datax/json-build/writer/RabbitmqWriter.vue index c2e812cd..f6a1e26f 100644 --- a/src/views/datax/json-build/writer/RabbitmqWriter.vue +++ b/src/views/datax/json-build/writer/RabbitmqWriter.vue @@ -1,6 +1,11 @@ From 32773a01642ecf6f9999f03bd2a7bf01c53b5a4b Mon Sep 17 00:00:00 2001 From: jiangdw <598965300@qq.com> Date: Thu, 24 Sep 2020 10:37:25 +0800 Subject: [PATCH 3/3] =?UTF-8?q?=E8=B0=83=E6=95=B4parquet=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E6=BA=90=E6=B7=BB=E5=8A=A0=E7=9A=84=E5=8F=82?= =?UTF-8?q?=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/views/datax/jdbc-datasource/index.vue | 9 ++++++--- src/views/datax/json-build/writer/RabbitmqWriter.vue | 1 - 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/views/datax/jdbc-datasource/index.vue b/src/views/datax/jdbc-datasource/index.vue index 898f36e3..6990d2f5 100644 --- a/src/views/datax/jdbc-datasource/index.vue +++ b/src/views/datax/jdbc-datasource/index.vue @@ -367,12 +367,12 @@ export default { this.temp.columnx = '' this.hbase = this.mongodb = false this.jdbc = true - this.enExtra = false + this.enExtra = true this.rabbitmq = false this.parquetfile = false } else if (datasource === 'parquetfile') { this.temp.jdbcUrl = '/{local_dir_absolute_path}/*' - this.temp.extra = '' + this.temp.extra = '{"encoding": "UTF-8"}' this.temp.columnx = '[{"name": "mmsi","type": "String"},{"name": "rot","type": "Integer"}]' // this.temp.columnx = '[{"index": 0,"type": "string"},{"index": 1,"type": "integer"}]' this.rabbitmq = false @@ -490,11 +490,13 @@ export default { this.hbase = true this.parquetfile = false this.rabbitmq = false + this.enExtra = false } else if (datasource === 'mongodb') { this.jdbc = this.hbase = false this.mongodb = true this.parquetfile = false this.rabbitmq = false + this.enExtra = false this.temp.jdbcUrl = 'mongodb://[username:password@]host1[:port1][,...hostN[:portN]]][/[database][?options]]' } else if (datasource === 'rabbitmq') { this.temp.extra = '{"port": "5672"}' @@ -503,7 +505,7 @@ export default { this.rabbitmq = true this.parquetfile = false } else if (datasource === 'parquetfile') { - this.enExtra = false + this.enExtra = true this.jdbc = false this.parquetfile = true this.rabbitmq = false @@ -512,6 +514,7 @@ export default { this.jdbc = true this.parquetfile = false this.rabbitmq = false + this.enExtra = false } }, handleDelete (row) { diff --git a/src/views/datax/json-build/writer/RabbitmqWriter.vue b/src/views/datax/json-build/writer/RabbitmqWriter.vue index f6a1e26f..d9bf75e1 100644 --- a/src/views/datax/json-build/writer/RabbitmqWriter.vue +++ b/src/views/datax/json-build/writer/RabbitmqWriter.vue @@ -88,7 +88,6 @@ }} - aaaaaaaaa