diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 000000000..bafd57195 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,103 @@ +name: Bug report +title: "[Bug] " +description: Problems and issues with code of Exchangis +labels: [bug, triage] +body: + - type: markdown + attributes: + value: | + Thank you for reporting the problem! + Please make sure what you are reporting is a bug with reproducible steps. To ask questions + or share ideas, pleae post on our [Discussion page](https://github.com/WeBankFinTech/Exchangis/discussions) instead. + + - type: checkboxes + attributes: + label: Search before asking + description: > + Please make sure to search in the [issues](https://github.com/WeBankFinTech/Exchangis/issues) first to see + whether the same issue was reported already. + options: + - label: > + I searched the [issues](https://github.com/WeBankFinTech/Exchangis/issues) and found no similar + issues. + required: true + + - type: dropdown + attributes: + label: Exchangis Component + description: | + What component are you using? Exchangis has many modules, please make sure to choose the module that + you found the bug. + multiple: true + options: + - "exchangis-datasource" + - "exchangis-job-launcher" + - "exchangis-job-server" + - "exchangis-job-builder" + - "exchangis-job-metrics" + - "exchangis-project" + - "exchangis-plugins" + - "exchangis-dao" + - "exchangis-web" + validations: + required: true + + - type: textarea + attributes: + label: What happened + What you expected to happen + description: Describe 1. the bug 2. expected behavior 3. useful information (e.g., logs) + placeholder: > + Please provide the context in which the problem occurred and explain what happened. Further, + To Reproduce Steps to reproduce the behavior: 1. Go to '...' 2. Click on '....' 3. Scroll down to '.... 4. See error + please also explain why you think the behaviour is erroneous. It is extremely helpful if you can + copy and paste the fragment of logs showing the exact error messages or wrong behaviour here. + + **NOTE**: Expected behavior A clear and concise description of what you expected to happen.Screenshots If applicable, add screenshots to help explain your problem. + validations: + required: true + + - type: textarea + attributes: + label: Relevent platform + description: The platform where you occurred this issue + placeholder: > + Please specify Desktop or Smartphone, Version / Dependencies / OS / Browser + validations: + required: true + + - type: textarea + attributes: + label: Reproduction script + description: > + Please provide a reproducible script. Providing a narrow reproduction (minimal / no external dependencies) will + help us triage and address issues in the timely manner! + placeholder: > + Please provide a short code snippet (less than 50 lines if possible) that can be copy-pasted to + reproduce the issue. The snippet should have **no external library dependencies** + (i.e., use fake or mock data / environments). + + **NOTE**: If the code snippet cannot be run by itself, the issue will be marked as "needs-repro-script" + until the repro instruction is updated. + validations: + required: true + + - type: textarea + attributes: + label: Anything else + description: Anything else we need to know? + placeholder: > + How often does this problem occur? (Once? Every time? Only when certain conditions are met?) + Any relevant logs to include? Are there other relevant issues? + + - type: checkboxes + attributes: + label: Are you willing to submit a PR? + description: > + This is absolutely not required, but we are happy to guide you in the contribution process + especially if you already have a good understanding of how to implement the fix. + options: + - label: Yes I am willing to submit a PR! + + - type: markdown + attributes: + value: "Thanks for completing our form!" diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..7c34114e9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: fasle +contact_links: + - name: Ask a question or get support + url: https://github.com/WeBankFinTech/Exchangis/discussions + about: Ask a question or request support for using Exchangis \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 000000000..357f173ff --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,63 @@ +name: Exchangis feature request +description: Suggest an idea for Exchangis project +title: "[Feature] " +labels: [enhancement] +body: + - type: markdown + attributes: + value: | + Thank you for finding the time to propose a new feature! + We really appreciate the community efforts to improve Exchangis. + - type: checkboxes + attributes: + label: Search before asking + description: > + Please make sure to search in the [issues](https://github.com/WeBankFinTech/Exchangis/issues) first to see + whether the same feature was requested already. + options: + - label: > + I had searched in the [issues](https://github.com/WeBankFinTech/Exchangis/issues) and found no similar + feature requirement. + required: true + - type: textarea + attributes: + label: Problem Description + description: Is your feature request related to a problem? Please describe. + + - type: textarea + attributes: + label: Description + description: A short description of your feature + + - type: textarea + attributes: + label: Use case + description: > + Describe the use case of your feature request. + placeholder: > + Describe the solution you'd like A clear and concise description of what you want to happen. + + - type: textarea + attributes: + label: solutions + description: Describe alternatives you've considered A clear and concise description of any alternative solutions or features you've considered. + + - type: textarea + attributes: + label: Anything else + description: Anything else we need to know? + placeholder: > + Additional context Add any other context or screenshots about the feature request here. + + - type: checkboxes + attributes: + label: Are you willing to submit a PR? + description: > + This is absolutely not required, but we are happy to guide you in the contribution process + especially if you already have a good understanding of how to implement the feature. + options: + - label: Yes I am willing to submit a PR! + + - type: markdown + attributes: + value: "Thanks for completing our form!" diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..57e883bcd --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,28 @@ +### What is the purpose of the change +(For example: Exchangis-Job defines the core ability of Exchangis, it provides the abilities of job management, job transform, and job launch. +Related issues: #50. ) + +### Brief change log +(for example:) +- defines the job server module of Exchangis; +- defines the job launcher module of Exchangis; +- defines the job metrics module of Exchangis. + +### Verifying this change +(Please pick either of the following options) +This change is a trivial rework / code cleanup without any test coverage. +(or) +This change is already covered by existing tests, such as (please describe tests). +(or) +This change added tests and can be verified as follows: +(example:) +- Added tests for creating and execute the Exchangis jobs and verify the availability of different Exchangis Job, such as sqoop job, datax job. + +### Does this pull request potentially affect one of the following parts: +- Dependencies (does it add or upgrade a dependency): (yes / no) +- Anything that affects deployment: (yes / no / don't know) +- The Core framework, i.e., JobManager, Server.: (yes / no) + +### Documentation +- Does this pull request introduce a new feature? (yes / no) +- If yes, how is the feature documented? (not applicable / docs / JavaDocs / not documented) \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..5f93411ce --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,55 @@ +# +# Copyright 2019 WeBank. +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Exchangis CI Actions + +on: + push: + pull_request: + +jobs: + build: + + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [14.17.3] + # See supported Node.js release schedule at https://nodejs.org/en/about/releases/ + + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up JDK 8 + uses: actions/setup-java@v2 + with: + distribution: 'adopt' + java-version: 8 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v2 + with: + node-version: ${{ matrix.node-version }} + - name: Build backend by maven + run: | + mvn -N install + mvn clean package + - name: Build frontend by node.js + run: | + cd web + npm install + npm run build diff --git a/.github/workflows/check_license.yml b/.github/workflows/check_license.yml new file mode 100644 index 000000000..10e3f9fde --- /dev/null +++ b/.github/workflows/check_license.yml @@ -0,0 +1,48 @@ +# +# Copyright 2019 WeBank. +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Exchangis License check + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v2 + - name: Set up JDK 8 + uses: actions/setup-java@v2 + with: + java-version: '8' + distribution: 'adopt' + - name: mvn -N install + run: + mvn -N install + - name: License check with Maven + run: | + rat_file=`mvn apache-rat:check | { grep -oe "\\S\\+/rat.txt" || true; }` + echo "rat_file=$rat_file" + if [[ -n "$rat_file" ]];then echo "check error!" && cat $rat_file && exit 123;else echo "check success!" ;fi + - name: Upload the report + uses: actions/upload-artifact@v2 + with: + name: license-check-report + path: "**/target/rat.txt" diff --git a/README-ZH.md b/README-ZH.md new file mode 100644 index 000000000..d92810f77 --- /dev/null +++ b/README-ZH.md @@ -0,0 +1,68 @@ +[![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) + +[English](README.md) | 中文 + +## 项目简介 + +Exchangis 是微众银行大数据平台 WeDataSphere 自研的数据交换工具，支持异构数据源之间的结构化和非结构化数据传输同步。 + +Exchangis 抽象了一套统一的数据源和同步作业定义插件，允许用户快速接入新的数据源，并只需在数据库中简单配置即可在页面中使用。 + +基于插件化的框架设计，及计算中间件 [Linkis](https://github.com/apache/incubator-linkis)，Exchangis 可快速集成对接 Linkis 已集成的数据同步引擎，将 Exchangis 的同步作业转换成 Linkis 数据同步引擎的数据同步作业。 + +借助于 [Linkis](https://github.com/apache/incubator-linkis) 计算中间件的连接、复用与简化能力，Exchangis 天生便具备了高并发、高可用、多租户隔离和资源管控的金融级数据同步能力。 + +## 核心特点 + +### 1. 数据源管理 + +基于 Linkis DataSource，抽象了底层数据源在 Exchangis 作为一个同步作业的 Source 和 Sink 所必须的所有能力。 + +- **多传输引擎支持** +传输引擎可横向扩展； +当前版本完整聚合了离线批量引擎DataX、部分聚合了大数据批量导数引擎SQOOP + +- **近实时任务管控** +快速抓取传输任务日志以及传输速率等信息，实时关闭任务； +可根据带宽状况对任务进行动态限流 + +- **支持无结构化传输** +DataX框架改造，单独构建二进制流快速通道，适用于无数据转换的纯数据同步场景。 + +- **任务状态自检** +监控长时间运行的任务和状态异常任务，及时释放占用的资源并发出告警。 + +## 与现有的系统的对比 +对现有的一些数据交换工具和平台的对比： + +| 功能模组 | 描述 | Exchangis | DataX | Sqoop | DataLink | DBus | +| :----: | :----: |-------|-------|-------|-------|-------| +| UI | 集成便捷的管理界面和监控窗口| 已集成 | 无 | 无 | 已集成 |已集成 | +| 安装部署 | 部署难易程度和第三方依赖 | 一键部署，无依赖 | 无依赖 | 依赖Hadoop环境 | 依赖Zookeeper | 依赖大量第三方组件 | +| 数据权限管理| 多租户权限配置和数据源权限管控 | 支持 | 不支持 | 不支持 | 不支持 | 支持 | +| |动态限流传输 | 支持 | 部分支持，无法动态调整 | 部分支持，无法动态调整| 支持 | 支持，借助Kafka | +| 数据传输| 无结构数据二进制传输 | 支持，快速通道 | 不支持 | 不支持 | 不支持，都是记录 | 不支持，需要转化为统一消息格式| +| | 嵌入处理代码 | 支持，动态编译 | 不支持 | 不支持 | 不支持 | 部分支持 | +| | 传输断点恢复 | 支持（未开源） | 不支持，只能重试 | 不支持，只能重试 | 支持 | 支持 | +| 服务高可用 | 服务多点，故障不影响使用| 应用高可用，传输单点（分布式架构规划中） | 单点服务（开源版本） | 传输多点 | 应用、传输高可用 | 应用、传输高可用 | +| 系统管理 | 节点、资源管理 | 支持 | 不支持 | 不支持 | 支持 | 支持 | + +## 整体设计 + +### 架构设计 + +![架构设计](../../../images/zh_CN/ch1/architecture.png) + +## 相关文档 +[安装部署文档](exchangis_deploy_cn.md) +[用户手册](exchangis_user_manual_cn.md) + +## 交流贡献 + +如果您想得到最快的响应，请给我们提 issue，或者扫码进群： + +![communication](../../../images/communication.png) + +## License + +Exchangis is under the Apache 2.0 License. See the [License](../../../LICENSE) file for details. \ No newline at end of file diff --git a/README.md b/README.md index d48f79fa6..d436f3f80 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) -English | [中文](docs/zh_CN/ch1/README.md) +English | [中文](README-ZH.md) ## Introduction Exchangis is a lightweight,highly extensible data exchange platform that supports data transmission between structured and unstructured heterogeneous data sources. On the application layer, it has business features such as data permission management and control, high availability of node services and multi-tenant resource isolation. On the data layer, it also has architectural characteristics such as diversified transmission architecture, module plug-in and low coupling of components. diff --git a/assembly-package/config/application-exchangis.yml b/assembly-package/config/application-exchangis.yml index 59caa497d..f7247d4aa 100644 --- a/assembly-package/config/application-exchangis.yml +++ b/assembly-package/config/application-exchangis.yml @@ -1,12 +1,12 @@ server: - port: 9500 + port: 9321 spring: application: name: exchangis-server eureka: client: serviceUrl: - defaultZone: http://127.0.0.1:20503/eureka/ + defaultZone: http://127.0.0.1:3306/eureka/ instance: metadata-map: test: wedatasphere diff --git a/assembly-package/config/exchangis-server.properties b/assembly-package/config/exchangis-server.properties index 07a162edc..81f4b441d 100644 --- a/assembly-package/config/exchangis-server.properties +++ b/assembly-package/config/exchangis-server.properties @@ -15,38 +15,55 @@ # # -wds.linkis.server.mybatis.datasource.url=jdbc:mysql://localhost:3306/database?useSSL=false&characterEncoding=UTF-8&allowMultiQueries=true +#wds.linkis.test.mode=true +wds.linkis.test.mode=false + +wds.linkis.server.mybatis.datasource.url=jdbc:mysql://127.0.0.1:3306/exchangis?useSSL=false&characterEncoding=UTF-8&allowMultiQueries=true wds.linkis.server.mybatis.datasource.username=username wds.linkis.server.mybatis.datasource.password=password +wds.linkis.gateway.ip=127.0.0.1 +wds.linkis.gateway.port=9001 +wds.linkis.gateway.url=http://127.0.0.1:9001/ + wds.linkis.log.clear=true wds.linkis.server.version=v1 -# datasource client -wds.exchangis.datasource.client.serverurl= +## datasource client +wds.exchangis.datasource.client.serverurl=http://127.0.0.1:9001 wds.exchangis.datasource.client.authtoken.key=DATASOURCE-AUTH wds.exchangis.datasource.client.authtoken.value=DATASOURCE-AUTH wds.exchangis.datasource.client.dws.version=v1 # launcher client -wds.exchangis.client.linkis.server-url= +wds.exchangis.client.linkis.server-url=http://127.0.0.1:9001 wds.exchangis.client.linkis.token.value=DATASOURCE-AUTH + wds.exchangis.datasource.extension.dir=exchangis-extds ##restful wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.exchangis.datasource.server.restful.api,\ com.webank.wedatasphere.exchangis.project.server.restful,\ com.webank.wedatasphere.exchangis.job.server.restful -wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/exchangis/job/server/mapper/impl/*.xml:\ - ,classpath*:com/webank/wedatasphere/exchangis/project/server/mapper/impl/*.xml +wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/framework/appconn/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/workflow/dao/impl/*.xml,\ +classpath*:com/webank/wedatasphere/exchangis/job/server/mapper/impl/*.xml,\ +classpath*:com/webank/wedatasphere/exchangis/project/server/mapper/impl/*.xml wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.exchangis.dao,\ com.webank.wedatasphere.exchangis.project.server.mapper,\ - com.webank.wedatasphere.exchangis.job.server.mapper + com.webank.wedatasphere.linkis.configuration.dao,\ + com.webank.wedatasphere.dss.framework.appconn.dao,\ + com.webank.wedatasphere.dss.workflow.dao,\ + com.webank.wedatasphere.linkis.metadata.dao,\ + com.webank.wedatasphere.exchangis.job.server.mapper,\ + com.webank.wedatasphere.exchangis.job.server.dao +wds.exchangis.job.task.scheduler.load-balancer.flexible.segments.min-occupy=0.25 +wds.exchangis.job.task.scheduler.load-balancer.flexible.segments.max-occupy=0.5 +#wds.exchangis.job.scheduler.group.max.running-jobs=4 diff --git a/assembly-package/sbin/install.sh b/assembly-package/sbin/install.sh new file mode 100644 index 000000000..8dc7b9f3f --- /dev/null +++ b/assembly-package/sbin/install.sh @@ -0,0 +1,275 @@ +#!/bin/bash +# +# Copyright 2020 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +SHELL_LOG="${DIR}/console.out" #console.out是什么文件？ +export SQL_SOURCE_PATH="${DIR}/../db/exchangis_ddl.sql" +PACKAGE_DIR="${DIR}/../packages" +# Home Path +EXCHNGIS_HOME_PATH="${DIR}/../" + +CONF_FILE_PATH="bin/configure.sh" +FORCE_INSTALL=false +SKIP_PACKAGE=false +USER=`whoami` +SUDO_USER=false + +CONF_PATH=${DIR}/../config + +usage(){ + printf "\033[1m Install project, run directly\n\033[0m" +} + +function LOG(){ + currentTime=`date "+%Y-%m-%d %H:%M:%S.%3N"` + echo -e "$currentTime [${1}] ($$) $2" | tee -a ${SHELL_LOG} # tee -a 输出是追加到文件里面 +} + +abs_path(){ + SOURCE="${BASH_SOURCE[0]}" + while [ -h "${SOURCE}" ]; do + DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "${SOURCE}")" + [[ ${SOURCE} != /* ]] && SOURCE="${DIR}/${SOURCE}" + done + echo "$( cd -P "$( dirname "${SOURCE}" )" && pwd )" +} + +BIN=`abs_path` + + +is_sudo_user(){ + sudo -v >/dev/null 2>&1 #因为 sudo 在第一次执行时或是在 N分钟内没有执行(N 预设为5)会问密码 + #这条命令的意思就是在后台执行这个程序,并将错误输出2重定向到标准输出1,然后将标准输出1全部放到/dev/null文件,也就是清空. + #所以可以看出" >/dev/null 2>&1 "常用来避免shell命令或者程序等运行中有内容输出。 +} + +uncompress_packages(){ + LOG INFO "\033[1m package dir is: [${PACKAGE_DIR}]\033[0m" + local list=`ls ${PACKAGE_DIR}` + LOG INFO "\033[1m package list is: [${list}]\033[0m" + for pack in ${list} + do + local uncompress=true + if [ ${#PACKAGE_NAMES[@]} -gt 0 ]; then + uncompress=false + for server in ${PACKAGE_NAMES[@]} + do + if [ ${server} == ${pack%%.tar.gz*} ] || [ ${server} == ${pack%%.zip*} ]; then + uncompress=true + break + fi + done + fi + if [ ${uncompress} == true ]; then + if [[ ${pack} =~ tar\.gz$ ]]; then + local do_uncompress=0 + if [ ${FORCE_INSTALL} == false ]; then + interact_echo "Do you want to decompress this package: [${pack}]?" + do_uncompress=$? + fi + if [ ${do_uncompress} == 0 ]; then + LOG INFO "\033[1m Uncompress package: [${pack}] to modules directory\033[0m" + tar --skip-old-files -zxf ${PACKAGE_DIR}/${pack} -C ../ + fi + elif [[ ${pack} =~ zip$ ]]; then + local do_uncompress=0 + if [ ${FORCE_INSTALL} == false ]; then + interact_echo "Do you want to decompress this package: [${pack}]?" + do_uncompress=$? + fi + if [ ${do_uncompress} == 0 ]; then + LOG INFO "\033[1m Uncompress package: [${pack}] to modules directory\033[0m" + unzip -nq ${PACKAGE_DIR}/${pack} -d # n 解压缩时不要覆盖原有的文件 + fi + fi + # skip other packages + fi + done +} + +interact_echo(){ + while [ 1 ]; do + read -p "$1 (Y/N)" yn + if [ "${yn}x" == "Yx" ] || [ "${yn}x" == "yx" ]; then + return 0 + elif [ "${yn}x" == "Nx" ] || [ "${yn}x" == "nx" ]; then + return 1 + else + echo "Unknown choise: [$yn], please choose again." + fi + done +} + +init_database(){ +BOOTSTRAP_PROP_FILE="${CONF_PATH}/exchangis-server.properties" +# Start to initalize database +if [ "x${SQL_SOURCE_PATH}" != "x" ] && [ -f "${SQL_SOURCE_PATH}" ]; then + `mysql --version >/dev/null 2>&1` + if [ $? == 0 ]; then + LOG INFO "\033[1m Scan out mysql command, so begin to initalize the database\033[0m" + interact_echo "Do you want to initalize database with sql: [${SQL_SOURCE_PATH}]?" + if [ $? == 0 ]; then + read -p "Please input the db host(default: 127.0.0.1): " HOST + if [ "x${HOST}" == "x" ]; then + HOST="127.0.0.1" + fi + while [ 1 ]; do + read -p "Please input the db port(default: 3306): " PORT + if [ "x${PORT}" == "x" ]; then + PORT=3306 + break + elif [ ${PORT} -gt 0 ] 2>/dev/null; then + break + else + echo "${PORT} is not a number, please input again" + fi + done + read -p "Please input the db username(default: root): " USERNAME + if [ "x${USERNAME}" == "x" ]; then + USERNAME="root" + fi + read -p "Please input the db password(default: ""): " PASSWORD + read -p "Please input the db name(default: exchangis)" DATABASE + if [ "x${DATABASE}" == "x" ]; then + DATABASE="exchangis" + fi + DATASOURCE_URL="jdbc:mysql:\/\/${HOST}:${PORT}\/${DATABASE}\?useSSL=false\&characterEncoding=UTF-8\&allowMultiQueries=true" + mysql -h ${HOST} -P ${PORT} -u ${USERNAME} -p${PASSWORD} --default-character-set=utf8 -e \ + "CREATE DATABASE IF NOT EXISTS ${DATABASE}; USE ${DATABASE}; source ${SQL_SOURCE_PATH};" + #sed -ri "s![#]?(DB_HOST=)\S*!\1${HOST}!g" ${BOOTSTRAP_PROP_FILE} + #sed -ri "s![#]?(DB_PORT=)\S*!\1${PORT}!g" ${BOOTSTRAP_PROP_FILE} + sed -ri "s![#]?(wds.linkis.server.mybatis.datasource.username=)\S*!\1${USERNAME}!g" ${BOOTSTRAP_PROP_FILE} + sed -ri "s![#]?(wds.linkis.server.mybatis.datasource.password=)\S*!\1${PASSWORD}!g" ${BOOTSTRAP_PROP_FILE} + sed -ri "s![#]?(wds.linkis.server.mybatis.datasource.url=)\S*!\1${DATASOURCE_URL}!g" ${BOOTSTRAP_PROP_FILE} + fi + fi +fi +} + +init_properties(){ +BOOTSTRAP_PROP_FILE="${CONF_PATH}/exchangis-server.properties" +# Start to initalize propertis + interact_echo "Do you want to initalize exchangis-server.properties?" + if [ $? == 0 ]; then + read -p "Please input the linkis gateway ip(default: 127.0.0.1): " HOST + if [ "x${HOST}" == "x" ]; then + HOST="127.0.0.1" + fi + while [ 1 ]; do + read -p "Please input the linkis gateway port(default: 3306): " PORT + if [ "x${PORT}" == "x" ]; then + PORT=3306 + break + elif [ ${PORT} -gt 0 ] 2>/dev/null; then + break + else + echo "${PORT} is not a number, please input again" + fi + done + + LINKIS_GATEWAY_URL="http:\/\/${HOST}:${PORT}\/" + + read -p "Please input the exchangis datasource client serverurl(default: http://127.0.0.1:3306): " EXCHANGIS_DATASOURCE_URL + if [ "x${EXCHANGIS_DATASOURCE_URL}" == "x" ]; then + EXCHANGIS_DATASOURCE_URL="http://127.0.0.1:3306" + fi + read -p "Please input the linkis server url(default: ""): " LINKIS_SERVER_URL + if [ "x${LINKIS_SERVER_URL}" == "x" ]; then + LINKIS_SERVER_URL="http://127.0.0.1:3306" + fi + + sed -ri "s![#]?(wds.linkis.gateway.ip=)\S*!\1${HOST}!g" ${BOOTSTRAP_PROP_FILE} + sed -ri "s![#]?(wds.linkis.gateway.port=)\S*!\1${PORT}!g" ${BOOTSTRAP_PROP_FILE} + sed -ri "s![#]?(wds.linkis.gateway.url=)\S*!\1${LINKIS_GATEWAY_URL}!g" ${BOOTSTRAP_PROP_FILE} + sed -ri "s![#]?(wds.exchangis.datasource.client.serverurl=)\S*!\1${EXCHANGIS_DATASOURCE_URL}!g" ${BOOTSTRAP_PROP_FILE} + sed -ri "s![#]?(wds.exchangis.client.linkis.server-url=)\S*!\1${LINKIS_SERVER_URL}!g" ${BOOTSTRAP_PROP_FILE} + fi +} + +install_modules(){ + LOG INFO "\033[1m ####### Start To Install project ######\033[0m" + echo "" + if [ ${FORCE_INSTALL} == false ]; then + interact_echo "Do you want to confiugre and install project?" + if [ $? == 0 ]; then #$? 执行上一个指令的返回值 (显示最后命令的退出状态。0表示没有错误，其他任何值表明有错误) + LOG INFO "\033[1m Install project ......\033[0m" + init_database + init_properties + fi + else + LOG INFO "\033[1m Install project ......\033[0m" + init_database + fi + LOG INFO "\033[1m ####### Finish To Install Project ######\033[0m" +} + + +while [ 1 ]; do + case ${!OPTIND} in + -h|--help) + usage + exit 0 + ;; + "") + break + ;; + *) + echo "Argument error! " 1>&2 + exit 1 + ;; + esac +done + +is_sudo_user +if [ $? == 0 ]; then + SUDO_USER=true +fi + +MODULE_LIST_RESOLVED=() +c=0 +RESOLVED_DIR=${PACKAGE_DIR} + +server="exchangis-server" +LOG INFO "\033[1m ####### server is [${server}] ######\033[0m" +server_list=`ls ${RESOLVED_DIR} | grep -E "^(${server}|${server}_[0-9]+\\.[0-9]+\\.[0-9]+)" | grep -E "(\\.tar\\.gz|\\.zip|)$"` +LOG INFO "\033[1m ####### server_list is [${server_list}] ######\033[0m" +for _server in ${server_list} + do + # More better method to cut string? + _server=${_server%%.tar.gz*} + _server=${_server%%zip*} + MODULE_LIST_RESOLVED[$c]=${_server} + c=$(($c + 1)) + done +if [ ${SKIP_PACKAGE} == true ]; then + MODULE_LIST=${MODULE_LIST_RESOLVED} +else + PACKAGE_NAMES=${MODULE_LIST_RESOLVED} +fi + + +LOG INFO "\033[1m ####### Start To Uncompress Packages ######\033[0m" +LOG INFO "Uncompressing...." +uncompress_packages +LOG INFO "\033[1m ####### Finish To Umcompress Packages ######\033[0m" + + install_modules + + +exit 0 + diff --git a/assembly-package/sbin/launcher.sh b/assembly-package/sbin/launcher.sh index f7aac9c7d..83bb37807 100644 --- a/assembly-package/sbin/launcher.sh +++ b/assembly-package/sbin/launcher.sh @@ -176,7 +176,7 @@ wait_for_startup(){ return 0 fi sleep ${SLEEP_TIMEREVAL_S} - now_s=`date '+%s'` + now_s=`date '+%s'` #计算当前时间时间戳 done return 1 } diff --git a/db/exchangis_ddl.sql b/db/exchangis_ddl.sql index 916523046..826c511c4 100644 --- a/db/exchangis_ddl.sql +++ b/db/exchangis_ddl.sql @@ -7,7 +7,7 @@ CREATE TABLE `exchangis_job_ds_bind` ( `source_ds_id` bigint(20) NOT NULL, `sink_ds_id` bigint(20) NOT NULL, PRIMARY KEY (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=5043 DEFAULT CHARSET=utf8 COLLATE=utf8_bin +) ENGINE=InnoDB AUTO_INCREMENT=59575 DEFAULT CHARSET=utf8 COLLATE=utf8_bin; -- exchangis_v4.exchangis_job_entity definition @@ -16,7 +16,7 @@ CREATE TABLE `exchangis_job_entity` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, `name` varchar(100) NOT NULL, `create_time` datetime DEFAULT NULL, - `last_update_time` datetime DEFAULT NULL, + `last_update_time` datetime(3) DEFAULT NULL, `engine_type` varchar(45) DEFAULT '', `job_labels` varchar(255) DEFAULT NULL, `create_user` varchar(100) DEFAULT NULL, @@ -29,7 +29,7 @@ CREATE TABLE `exchangis_job_entity` ( `source` text, `modify_user` varchar(50) DEFAULT NULL COMMENT '修改用户', PRIMARY KEY (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=129 DEFAULT CHARSET=utf8 +) ENGINE=InnoDB AUTO_INCREMENT=5793 DEFAULT CHARSET=utf8; -- exchangis_v4.exchangis_job_param_config definition @@ -60,7 +60,7 @@ CREATE TABLE `exchangis_job_param_config` ( `description` varchar(255) DEFAULT NULL, `status` tinyint(4) DEFAULT NULL, PRIMARY KEY (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=32 DEFAULT CHARSET=utf8 +) ENGINE=InnoDB AUTO_INCREMENT=32 DEFAULT CHARSET=utf8; -- exchangis_v4.exchangis_project_info definition @@ -69,7 +69,7 @@ CREATE TABLE `exchangis_project_info` ( `name` varchar(64) NOT NULL, `description` varchar(255) DEFAULT NULL, `create_time` datetime DEFAULT NULL, - `last_update_time` datetime DEFAULT NULL, + `last_update_time` datetime(3) DEFAULT NULL, `create_user` varchar(64) DEFAULT NULL, `last_update_user` varchar(64) DEFAULT NULL, `project_labels` varchar(255) DEFAULT NULL, @@ -79,28 +79,11 @@ CREATE TABLE `exchangis_project_info` ( `edit_users` varchar(255) DEFAULT NULL, `source` text, PRIMARY KEY (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=1497870871035973652 DEFAULT CHARSET=utf8 +) ENGINE=InnoDB AUTO_INCREMENT=1497870871035973934 DEFAULT CHARSET=utf8; -- exchangis_v4.exchangis_job_entity definition -CREATE TABLE `exchangis_job_entity` ( - `id` bigint(20) NOT NULL, - `name` varchar(100) NOT NULL, - `create_time` datetime DEFAULT NULL, - `last_update_time` datetime DEFAULT NULL, - `engine_type` varchar(45) DEFAULT '', - `job_labels` varchar(64) DEFAULT NULL, - `create_user` varchar(100) DEFAULT NULL, - `job_content` text NOT NULL, - `execute_user` varchar(100) DEFAULT '', - `job_params` text NOT NULL, - `project_id` bigint(13) DEFAULT NULL, - `source` varchar(255) NOT NULL, - `modify_user` varchar(50) DEFAULT NULL COMMENT '修改用户', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 - -- exchangis_v4.exchangis_launchable_task definition CREATE TABLE `exchangis_launchable_task` ( @@ -108,7 +91,7 @@ CREATE TABLE `exchangis_launchable_task` ( `name` varchar(100) NOT NULL, `job_execution_id` varchar(64) DEFAULT NULL, `create_time` datetime DEFAULT NULL, - `last_update_time` datetime DEFAULT NULL, + `last_update_time` datetime(3) DEFAULT NULL, `engine_type` varchar(45) DEFAULT '', `execute_user` varchar(50) DEFAULT '', `linkis_job_name` varchar(100) NOT NULL, @@ -117,7 +100,7 @@ CREATE TABLE `exchangis_launchable_task` ( `linkis_source` varchar(64) DEFAULT NULL, `labels` varchar(64) DEFAULT NULL, PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 +) ENGINE=InnoDB DEFAULT CHARSET=utf8; -- exchangis_v4.exchangis_launched_job_entity definition @@ -125,7 +108,7 @@ CREATE TABLE `exchangis_launched_job_entity` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, `name` varchar(100) NOT NULL, `create_time` datetime DEFAULT NULL, - `last_update_time` datetime DEFAULT NULL, + `last_update_time` datetime(3) DEFAULT NULL, `job_id` bigint(20) DEFAULT NULL, `launchable_task_num` int(20) DEFAULT '0', `engine_type` varchar(100) DEFAULT NULL, @@ -141,7 +124,7 @@ CREATE TABLE `exchangis_launched_job_entity` ( `create_user` varchar(100) DEFAULT NULL, PRIMARY KEY (`id`), UNIQUE KEY `job_execution_id_UNIQUE` (`job_execution_id`) -) ENGINE=InnoDB AUTO_INCREMENT=708 DEFAULT CHARSET=utf8 +) ENGINE=InnoDB AUTO_INCREMENT=8380 DEFAULT CHARSET=utf8; -- exchangis_v4.exchangis_launched_task_entity definition @@ -149,7 +132,7 @@ CREATE TABLE `exchangis_launched_task_entity` ( `id` bigint(20) NOT NULL, `name` varchar(100) NOT NULL, `create_time` datetime DEFAULT NULL, - `last_update_time` datetime DEFAULT NULL, + `last_update_time` datetime(3) DEFAULT NULL, `job_id` bigint(20) DEFAULT NULL, `engine_type` varchar(100) DEFAULT NULL, `execute_user` varchar(100) DEFAULT NULL, @@ -167,4 +150,21 @@ CREATE TABLE `exchangis_launched_task_entity` ( `metrics` text, `status` varchar(64) DEFAULT NULL, PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8 \ No newline at end of file +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +INSERT INTO exchangis_job_param_config (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status) VALUES +('setting.speed.bytes','作业速率限制','','DATAX','INPUT','setting.speed.bytes','作业速率限制','Mb/s',1,'NUMBER','','','REGEX','^[1-9]\\d*$','作业速率限制输入错误',0,0,'',1,'',1,'',1) +,('setting.speed.records','作业记录数限制','','DATAX','INPUT','setting.speed.records','作业记录数限制','条/s',1,'NUMBER','','','REGEX','^[1-9]\\d*$','作业记录数限制输入错误',0,0,'',1,'',2,'',1) +,('setting.max.parallelism','作业最大并行度','','DATAX','INPUT','setting.max.parallelism','作业最大并行度','个',1,'NUMBER','','1','REGEX','^[1-9]\\d*$','作业最大并行度输入错误',0,0,'',1,'',3,'',1) +,('setting.max.memory','作业最大使用内存','','DATAX','INPUT','setting.max.memory','作业最大使用内存','Mb',1,'NUMBER','','1024','REGEX','^[1-9]\\d*$','作业最大使用内存输入错误',0,0,'',1,'',4,'',1) +,('setting.errorlimit.record','最多错误记录数','','DATAX','INPUT','setting.errorlimit.record','最多错误记录数','条',1,'NUMBER','','','REGEX','^[1-9]\\d*$','最多错误记录数输入错误',0,0,'',1,'',5,'',1) +,('setting.max.parallelism','作业最大并行数','','SQOOP','INPUT','setting.max.parallelism','作业最大并行数','个',1,'NUMBER','','1','REGEX','^[1-9]\\d*$','作业最大并行数输入错误',0,0,'',1,'',1,'',1) +,('setting.max.memory','作业最大内存','','SQOOP','INPUT','setting.max.memory','作业最大内存','Mb',1,'NUMBER','','1024','REGEX','^[1-9]\\d*$','作业最大内存输入错误',0,0,'',1,'',2,'',1) +,('where','WHERE条件','SOURCE','MYSQL','INPUT','where','WHERE条件','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,500}$','WHERE条件输入过长',0,0,'',1,'',2,'',1) +,('writeMode','写入方式','SQOOP-SINK','HIVE','OPTION','writeMode','写入方式(OVERWRITE只对TEXT类型表生效)','',1,'OPTION','["OVERWRITE","APPEND"]','OVERWRITE','','','写入方式输入错误',0,0,'',1,'',1,'',1) +,('partition','分区信息','SINK','HIVE','MAP','partition','分区信息(文本)','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','分区信息过长',0,0,'/api/rest_j/v1/dss/exchangis/main/datasources/render/partition/element/map',1,'',2,'',1) +; +INSERT INTO exchangis_job_param_config (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status) VALUES +('partition','分区信息','SOURCE','HIVE','MAP','partition','分区信息(文本)',NULL,0,'VARCHAR',NULL,NULL,'REGEX','^[\\s\\S]{0,50}$','分区信息过长',0,0,'/api/rest_j/v1/dss/exchangis/main/datasources/render/partition/element/map',1,NULL,1,NULL,1) +,('writeMode','写入方式','SQOOP-SINK','MYSQL','OPTION','writeMode','写入方式',NULL,1,'OPTION','["INSERT","UPDATE"]','INSERT',NULL,NULL,'写入方式输入错误',0,0,NULL,1,NULL,1,NULL,1) +; \ No newline at end of file diff --git a/db/exchangis_dml.sql b/db/exchangis_dml.sql index a7172b2ee..6ea326fb9 100644 --- a/db/exchangis_dml.sql +++ b/db/exchangis_dml.sql @@ -1,5 +1,5 @@ -- 插入 job_param_config 记录 -INSERT INTO udes_gzpc_pub_sit_01.exchangis_job_param_config (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status) VALUES +INSERT INTO exchangis_job_param_config (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status) VALUES ('setting.speed.bytes','作业速率限制','','DATAX','INPUT','setting.speed.bytes','作业速率限制','Mb/s',1,'NUMBER','','','REGEX','^[1-9]\\d*$','作业速率限制输入错误',0,0,'',1,'',1,'',1) ,('setting.speed.records','作业记录数限制','','DATAX','INPUT','setting.speed.records','作业记录数限制','条/s',1,'NUMBER','','','REGEX','^[1-9]\\d*$','作业记录数限制输入错误',0,0,'',1,'',2,'',1) ,('setting.max.parallelism','作业最大并行度','','DATAX','INPUT','setting.max.parallelism','作业最大并行度','个',1,'NUMBER','','1','REGEX','^[1-9]\\d*$','作业最大并行度输入错误',0,0,'',1,'',3,'',1) @@ -9,7 +9,9 @@ INSERT INTO udes_gzpc_pub_sit_01.exchangis_job_param_config (config_key,config_n ,('setting.max.memory','作业最大内存','','SQOOP','INPUT','setting.max.memory','作业最大内存','Mb',1,'NUMBER','','1024','REGEX','^[1-9]\\d*$','作业最大内存输入错误',0,0,'',1,'',2,'',1) ,('where','WHERE条件','SOURCE','MYSQL','INPUT','where','WHERE条件','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,500}$','WHERE条件输入过长',0,0,'',1,'',2,'',1) ,('writeMode','写入方式','SQOOP-SINK','HIVE','OPTION','writeMode','写入方式(OVERWRITE只对TEXT类型表生效)','',1,'OPTION','["OVERWRITE","APPEND"]','OVERWRITE','','','写入方式输入错误',0,0,'',1,'',1,'',1) -,('partition','分区信息','SINK','HIVE','MAP','partition','分区信息(文本)','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','分区信息过长',0,0,'/api/rest_j/v1/exchangis/datasources/render/partition/element/map',1,'',2,'',1) -,('partition','分区信息','SOURCE','HIVE','MAP','partition','分区信息(文本)',NULL,0,'VARCHAR',NULL,NULL,'REGEX','^[\\s\\S]{0,50}$','分区信息过长',0,0,'/api/rest_j/v1/exchangis/datasources/render/partition/element/map',1,NULL,1,NULL,1) -,('writeMode','写入方式','SQOOP-SINK','MYSQL','OPTION','writeMode','写入方式',NULL,1,'OPTION','["INSERT","UPDATE"]','INSERT',NULL,NULL,'写入方式输入错误',0,0,NULL,1,NULL,1,NULL,1) +,('partition','分区信息','SINK','HIVE','MAP','partition','分区信息(文本)','',0,'VARCHAR','','','REGEX','^[\\s\\S]{0,50}$','分区信息过长',0,0,'/api/rest_j/v1/dss/exchangis/main/datasources/render/partition/element/map',1,'',2,'',1) ; +INSERT INTO exchangis_job_param_config (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status) VALUES +('partition','分区信息','SOURCE','HIVE','MAP','partition','分区信息(文本)',NULL,0,'VARCHAR',NULL,NULL,'REGEX','^[\\s\\S]{0,50}$','分区信息过长',0,0,'/api/rest_j/v1/dss/exchangis/main/datasources/render/partition/element/map',1,NULL,1,NULL,1) +,('writeMode','写入方式','SQOOP-SINK','MYSQL','OPTION','writeMode','写入方式',NULL,1,'OPTION','["INSERT","UPDATE"]','INSERT',NULL,NULL,'写入方式输入错误',0,0,NULL,1,NULL,1,NULL,1) +; \ No newline at end of file diff --git a/docs/zh_CN/ch1/exchangis_appconn_deploy_cn.md b/docs/zh_CN/ch1/exchangis_appconn_deploy_cn.md new file mode 100644 index 000000000..fa1842c47 --- /dev/null +++ b/docs/zh_CN/ch1/exchangis_appconn_deploy_cn.md @@ -0,0 +1,114 @@ +### 环境准备 +您在部署ExchangisAppConn之前，请按照Exchangis1.0.0安装部署文档安装完成Exchangis1.0.0及其他相关组件的安装，并确保工程基本功能可用。 + +### 安装包准备 +#### 1）下载二进制包 +[点击跳转 Release 界面](https://github.com/WeBankFinTech/Exchangis/releases) +#### 2） 编译打包 +如果您想自己开发和编译ExchangisAppConn，具体编译步骤如下： +1.clone Exchangis的代码 +2.在exchangis-plugins模块下，找到exchangis-appconn，单独编译exchangis-appconn +``` +cd {EXCHANGIS_CODE_HOME}/exchangis-plugins/exchangis-appconn +mvn clean install +``` +会在该路径下找到exchangis-appconn.zip安装包 +``` +{EXCHANGIS_CODE_HOME}\exchangis-plugins\exchangis-appconn\target\exchangis-appconn.zip +``` + +### 开始部署 +#### 1）ExchangisAppConn插件的部署和配置总体步骤 +1.拿到打包出来的exchangis-appconn.zip物料包 + +2.放置到如下目录并进行解压 +``` +cd {DSS_HOME}/dss/dss-appconns +unzip visualis-appconn.zip +``` +解压出来的目录结构为： +``` +conf +db +lib +appconn.properties +``` +3.配置ExchangisAppConn数据库表的相关信息 + +4.执行脚本进行自动化安装 +``` +cd {EXCHANGIS_INSTALL_HOME}/sbin +./install-appconn.sh +``` +该脚本为交互式的安装方案，您需要输入连接DSS数据库ip等相关信息，完成ExchangisApponn在DSS中的安装。我们将在接下来的章节中着重说明。 + +5.完成exchangis-appconn的安装后，需要重启dss服务，才能最终完成插件的更新 + +#### 2）Init.sql配置ExchangisAppConn数据库表的相关信息(重要) +想要使exchangis-appconn正确部署成功，最重要的是将init.sql中相关的字段值配置正确，错误的配置方式有可能导致某些appconn功能无法生效甚至dss无法启动。在这里，为了让您尽快部署一个可用的exchangis-appconn，我们已经提供给您一个配置较为完整的init.sql文件，但仍需您根据自身需要配置以下几个参数，除此之外，如果您需要更进一步的了解dss-appconn的架构和配置，可以参考dss文档进行学习： + +dss_application表：url字段修改为您的前端域名或IP加端口的形式，例如https://www.exchangis.com/或http://127.0.0.1:3306/。然后，project_url、homepage_url、redirect_url这三个字段值正常为exchangis的首页地址，三个字段值相同即可，例如https://www.exchangis.com/#/projectManage或http://127.0.0.1:3306/#/projectManage + +(注：只改域名部分或IP和端口部分即可！！) + +sql语句如下（在init.sql文件中）： +![image](https://user-images.githubusercontent.com/27387830/169785874-a9b87fd4-6846-4186-acd3-f3db91f64d79.png) + + +dss_appconn_instance表：url字段修改为您exchangis后端的IP加端口的形式，例如http://127.0.0.1:3306/。注意，这里不能用域名，否则APPCONN插件无法调用到exchangis后端接口。homepage_url为主页url，例如https://www.exchangis.com/#/projectManage或http://127.0.0.1:3306/#/projectManage。最后redirect_url直接填写域名或IP加端口，例如https://www.exchangis.com/或http://127.0.0.1:3306/ + +sql语句如下（在init.sql文件中）： +![image](https://user-images.githubusercontent.com/27387830/169786001-5151adac-7b2c-498a-bec4-f0b7f6e8a1e9.png) + +dss_workflow_node表：该表为dss工作流界面第三方节点的配置表，简单理解为配置了该表既会在dss工作流界面显示一个新的工作流节点。主要需要修改的只有jump_url字段，该字段用于配置双击sqoop节点时，跳转的前端界面url。字段值如下形式 +http://127.0.0.1:3306/#/childJobManage。只修改IP和端口部分即可 + +sql语句如下（在init.sql文件中）： +![image](https://user-images.githubusercontent.com/27387830/169786065-1391cf24-f88c-4f47-9a26-3d810fbafa22.png) + +#### 3）执行脚本进行自动化安装 +进入已经安装好的Exchangis目录，找到sbin目录下面的install-appconn.sh文件，目前安装采用默认安装的方式，将打包出来的exchangis-appconn.zip包和install-appconn.sh文件拷贝到{EXCHANGIS_INSTALL_HOME}/dss-appconns目录下。直接执行以下命令： + +``` +./sbin/install-appconn.sh +``` +该脚本为交互式安装，安装步骤依次分为以下几步： +1. 解压缩lib包 + +当出现该提醒时：Do you want to decompress this package: [exchangis-appconn.zip] + +输入y确认解压，就会将项目的实际zip包解压到dss项目的APPCONN目录{DSS_INSTALL_HOME}/dss-appconns下。 +2. exchangis-appconn数据库初始化。 + + +#### 4）exchangis-appconn数据库初始化 +如果你的服务上安装有mysql命令，在执行安装脚本的过程中则会出现以下提醒： +``` +Scan out mysql command, so begin to initalize the dss-appconn database +Do you want to initalize database with sql: [{INSTALL_PATH}/db/init.sql]? (Y/N)y +``` +依据init.sql中的数据库配置，大部分情况下即可快速完成初始化。 + +如果服务上并没有安装mysql命令，则可以取用目录下/db/init.sql脚本去手动执行。（需提前知道您的DSS数据库地址） + +注意，初始化数据库可能碰到的问题有：数据库访问权限不够，已存在同名数据库，防火墙未关闭等，视具体情况解决。 + + +#### 5）使部署好的APPCONN生效 +使用DSS启停脚本使APPCONN生效，脚本所在位置为{DSS_INSTALL_HOME}/sbin中，依次使用如下命令执行脚本 +``` +sh /sbin/dss-stop-all.sh +sh /sbin/dss-start-all.sh +``` +中途可能发生启动失败或者卡住，可以退出重复执行 + +#### 5）验证exchangis-appconn是否生效 +在安装部署完成exchangis-appconn之后，可通过以下步骤初步验证exchangis-appconn是否安装成功。 +1. 在DSS工作空间创建一个新的项目 +![image](https://user-images.githubusercontent.com/27387830/169782142-b2fc2633-e605-4553-9433-67756135a6f1.png) + +2. 在exchangis端查看是否同步创建项目，创建成功说明appconn安装成功 +![image](https://user-images.githubusercontent.com/27387830/169782337-678f2df0-080a-495a-b59f-a98c5a427cf8.png) + + + diff --git a/docs/zh_CN/ch1/exchangis_deploy_cn.md b/docs/zh_CN/ch1/exchangis_deploy_cn.md index 4fbc38c2a..2fe884a4f 100644 --- a/docs/zh_CN/ch1/exchangis_deploy_cn.md +++ b/docs/zh_CN/ch1/exchangis_deploy_cn.md @@ -3,8 +3,9 @@ - MySQL (5.5+) 必选，对应客户端可以选装, Linux服务上若安装mysql的客户端可以通过部署脚本快速初始化数据库 - JDK (1.8.0_141) 必选 - Maven (3.6.1+) 必选 -- SQOOP (1.4.6) 可选，如果想要SQOOP做传输引擎，可以安装SQOOP，SQOOP安装依赖Hive,Hadoop环境，这里就不展开来讲 -- Python (2.x) 可选，主要用于调度执行底层DataX的启动脚本，默认的方式是以Java子进程方式执行DataX，用户可以选择以Python方式来做自定义的改造 +- SQOOP (1.4.6) 必选，如果想要SQOOP做传输引擎，要安装SQOOP，SQOOP安装依赖Hive,Hadoop环境 +- DSS1.0.1必选，确保安装部署环境下有DSS服务，以便进行APPCONN接入 +- Linkis1.1.0必选，请求的路由规则，执行引擎等均需要linkis #### 2）选择用户 如果选择有sudo权限的用户来执行安装部署脚本，并启动服务，对于不同的数据交换作业，服务将会切换用户来执行，否则将以当前服务所属用户来执行。 @@ -17,9 +18,9 @@ ``` mvn clean install ``` -执行成功后将会在工程的build目录下生成安装包 +执行成功后将会在工程的${EXCHANGIS_HOME}/assembly-package/target目录下生成安装包 ``` -build/wedatasphere-exchangis-{VERSION}.tar.gz +target/wedatasphere-exchangis-{VERSION}.tar.gz ``` ### 开始部署 @@ -28,16 +29,31 @@ build/wedatasphere-exchangis-{VERSION}.tar.gz ``` tar -zxvf wedatasphere-exchangis-{VERSION}.tar.gz ``` -#### 2）执行一键安装脚本 -进入解压后的目录，找到bin目录下面的install.sh文件，如果选择交互式的安装，则直接执行 +在解压出来的目录结构为： ``` -./bin/install.sh +config + +db + +exchangis-extds + +packages + +sbin ``` -在交互模式下，对各个模块的package压缩包的解压以及configure配置脚本的调用，都会请求用户确认。 -如果不想使用交互模式，跳过确认过程，则执行以下命令安装 +其中，config为项目相关配置文件，db为数据库表sql文件夹，sbin为各种自动化脚本存放的文件夹。 +#### 2）执行一键安装脚本 +进入解压后的目录，找到sbin目录下面的install.sh文件，如果选择交互式的安装，则直接执行 ``` -./bin/install.sh --force +./sbin/install.sh ``` +该脚本为交互式安装，安装步骤依次分为以下几步： +1. 解压缩lib包 +当出现该提醒时：Do you want to decompress this package: [exchangis-server_1.0.0-RC1.tar.gz] +输入y确认解压，就会将项目的实际jar包解压到项目的根目录文件下lib下。 +2. 安装部署数据库 +3. 配置exchangis-server.properties中基本的配置参数 + #### 3）数据库初始化 如果你的服务上安装有mysql命令，在执行安装脚本的过程中则会出现以下提醒： @@ -51,38 +67,203 @@ Please input the db password(default: ): Please input the db name(default: exchangis) ``` 按照提示输入数据库地址，端口号，用户名，密码以及数据库名称，大部分情况下即可快速完成初始化。 -如果服务上并没有安装mysql命令，则可以取用目录下/bin/exchangis-init.sql脚本去手动执行，完成后修改相关配置文件 +同时，会自动配置exchangis-server.properties中的下列参数： ``` -vi ./modules/exchangis-service/conf/bootstrap.properties +#wds.linkis.server.mybatis.datasource.url= jdbc:mysql://localhost:3306/database?useSSL=false&characterEncoding=UTF-8&allowMultiQueries=true +#wds.linkis.server.mybatis.datasource.username= +#wds.linkis.server.mybatis.datasource.password= ``` +如果服务上并没有安装mysql命令，则可以取用目录下/db/exchangis-ddl.sql脚本去手动执行，完成后修改exchangis-server.properties相关数据库配置参数。 +注意，初始化数据库可能碰到的问题有：数据库访问权限不够，已存在同名数据库，防火墙未关闭等，视具体情况解决。 + +也可选择手动安装数据库，数据库表ddl和dml在db文件夹中，分别为exchangis_ddl.sql和exchangis_dml.sql。执行上述两个sql文件即刻完成库表创建。 + +#### 4）配置exchangis-server.properties中基本的配置参数 +在执行脚本过初中，出现以下提示，既说明需要配置除数据库参数外其他必须参数： ``` -#Database -#DB_HOST= -#DB_PORT= -#DB_USERNAME= -#DB_PASSWORD= -#DB_DATABASE= +Do you want to initalize exchangis-server.properties? (Y/N)y +Please input the linkis gateway ip(default: 127.0.0.1):（linkis gateway服务ip，必配） +Please input the linkis gateway port(default: 3306): （linkis gateway端口，必配） +Please input the exchangis datasource client serverurl(default: http://127.0.0.1:3306):（数据源服务url，用于执行数据同步任务，必配） +Please input the linkis server url(default: ""): （linkis服务url，必配） ``` -按照具体情况配置对应的值即可。 +以上参数均可自行在exchangis-server.properties文件中自行配置 + +配置datasource及launcher的token + +为了能够访问数据源服务及通过linkis服务认证，您需要在exchangis-server.properties配置以下几个token相关参数，该字段可在linkis表linkis_mg_gateway_auth_token的token_name字段获取，注意，需根据您实际安装linkis的数据库表内容做变动，此值不唯一 + +wds.exchangis.datasource.client.authtoken.key= +wds.exchangis.datasource.client.authtoken.value= +wds.exchangis.client.linkis.token.value= + +![image](https://user-images.githubusercontent.com/27387830/170611761-1ba315d8-04e3-4b6d-b85d-0b095ef17dce.png) + #### 4）启动服务 一键启动所有服务 ``` -./bin/start-all.sh +./sbin/start.sh或者./sbin/daemon.sh start +``` +中途可能发生启动失败或者卡住，可以退出重复执行 + +使用以下命令执行脚本，可一键完成服务的停止和重启 +``` +./sbin/daemon.sh restart server ``` -中途可能发生部分模块启动失败或者卡住，可以退出重复执行，如果需要改变某一模块服务端口号，则： +出现以下提示，说明exchangis服务启动成功 +![企业微信截图_16532930262583](https://user-images.githubusercontent.com/27387830/169773764-1c5ed6fb-35e9-48cb-bac8-6fa7f738368a.png) + +#### 5）查看服务 +Exchangis1.0通过EUREKA查看启动的服务，其端口号在配置文件application-exchangis.yml。通过服务端口在网页上查看。 +可根据需要修改服务IP及端口号，配置文件为application-exchangis.yml ``` -vi ./modules/{module_name}/bin/env.properties +port: XXXX +defaultZone: http://127.0.0.1:3306/eureka/ ``` -找到SERVER_PORT配置项，改变它的值即可。 -当然也可以单一地启动某一模块服务： + +#### 6）前端安装部署 +如果您想要通过前端界面访问Exchangis1.0，就要进行以下几步对前端进行安装配置操作： +1. 获取前端安装包 +这里提供Exchangis1.0前端包，您可以自行下载使用： +[点击跳转 Release 界面](https://github.com/WeBankFinTech/Exchangis/releases) + +如果您需要自行对前端代码进行开发编译，也可以在Exchangis1.0项目中找到前端web模块，路径为${EXCHANGIS_PROJECT_HOME}/web。这里以Fes编译为例，使用Fes命令对前端模块进行打包，编译步骤如下： ``` -./bin/start.sh -m {module_name} +cd ${EXCHANGIS_HOME}/web +npm i +npm run build ``` +通过上面的编译步骤，即可在${EXCHANGIS_PROJECT_HOME}/web/路径下生成编译好的dist.zip包，既为我们需要使用的前端包。 +获取到的前端包，您可以放在服务器上的任意位置，这里建议您与后端安装地址${EXCHANGIS_INSTALL_HOME}目录保持一致，在同一目录下放置并解压。 -#### 4）查看服务 -服务使用Eureka做注册中心，默认的Eureka端口是8500(可变), 可以在Eureka界面http://{EUREKA_IP}:{EUREKA_PORT}上观察服务是否正常启动。 -Exchangis的入口界面集成在Gateway中，Gateway的访问端口为9503（可变） --- +2. 配置nginx +为了正确找到前端资源，需要在服务器上配置nginx的conf文件，这里提供一个exchangis.conf示例配置，您可根据实际需要进行修改： +``` +#map $http_origin $cors_list { +# default https://www.oszone.space/wdsent/exchangis/; +# "~ https//sandbox.webank.com/*" https://sandbox.webank.com/wdsent/exchangis/; +#} + +map $upstream_http_Location $location{ + # 这种方案是把所有到 9098 端口的重定向都改成 80 端口 + # ~http://(?.*):9098/(?.*) http://$domains/$param; + + # 这种方案是针对特定域名 9098 端口的重定向，范围可控，写法冗长 + ~https://www.open.source:9098/(?.*) https://www.open.source/$param; + + # 默认情况，保持原状 + default $upstream_http_Location; +} + +#以前端域名是https://www.open.source/origin/exchangis/为例 +server { + listen 9098;# 访问端口 + server_name localhost; + gzip on; + location /{ + root /Install/exchangisPath/; # 静态文件目录，即为您的前端dist所在目录 + index index.html index.htm ; + autoindex on; + } + + location /api { + proxy_pass http://127.0.0.1:9001; #后端Linkis的地址 + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header x_real_ipP $remote_addr; + proxy_set_header remote_addr $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_http_version 1.1; + proxy_connect_timeout 4s; + proxy_read_timeout 600s; + proxy_send_timeout 12s; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection upgrade; + } + error_page 500 502 503 504 /50x.html; + + location = /50x.html { + + root /usr/share/nginx/html; + + } +} +location /api { + proxy_pass http://127.0.0.1:9001; #后端Linkis的地址 + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header x_real_ipP $remote_addr; + proxy_set_header remote_addr $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_http_version 1.1; + proxy_connect_timeout 4s; + proxy_read_timeout 600s; + proxy_send_timeout 12s; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection upgrade; + } + error_page 500 502 503 504 /50x.html; + + location = /50x.html { + + root /usr/share/nginx/html; + + } +} + +server { +listen 80; #前端实际监听端口 +server_name localhost; +location /origin/exchangis { + proxy_pass http://192.168.241.42:9098; + more_set_headers -s '301 302' 'Location $location'; +} +location /api { + proxy_pass http://192.168.241.42:9098/api; + more_set_headers -s '301 302' 'Location $location'; +} +location /origin/exchangis/api { + proxy_pass http://127.0.0.1:9098/api; #反向代理地址，即为前面的9098地址 + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header x_real_ipP $remote_addr; + proxy_set_header remote_addr $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_http_version 1.1; + proxy_connect_timeout 4s; + proxy_read_timeout 600s; + proxy_send_timeout 12s; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection upgrade; + more_clear_headers 'Access-Control-Allow-Credentials'; + more_clear_headers 'Access-Control-Allow-Headers'; + more_clear_headers 'Access-Control-Allow-Origin'; + more_clear_headers 'Access-Control-Allow-Methods'; + add_header 'Access-Control-Allow-Credentials' 'true'; + add_header 'Access-Control-Allow-Headers' 'authorization,Content-Type'; + add_header 'Access-Control-Allow-Origin' 'https://www.open.source/origin/exchangis/'; #您的前端域名，也可以是ip加端口的形式，例如https://127.0.0.1:9999 + add_header 'Access-Control-Allow-Methods' 'POST, GET, OPTIONS, PUT, HEAD, DELETE'; + + more_set_headers -s '301 302' 'Location $location'; +} +location / { + return 403; +} + error_page 500 502 503 504 /50x.html; + + location = /50x.html { + + root /usr/share/nginx/html; + + } +} +``` + +配置完成之后，使用以下命令重新启动nginx: +``` +nginx -s reload +``` -Tips: 脚本使用的都是bash指令集，如若使用sh调用脚本，可能会有未知的错误 \ No newline at end of file +访问域名 https://www.open.source/origin/exchangis/(此处为示例域名，请根据实际域名进行修改)，即可进入Exchangis主界面： +![image](https://user-images.githubusercontent.com/27387830/170417473-af0b4cbe-758e-4800-a58f-0972f83d87e6.png) diff --git a/docs/zh_CN/ch1/exchangis_sqoop_deploy_cn.md b/docs/zh_CN/ch1/exchangis_sqoop_deploy_cn.md new file mode 100644 index 000000000..7720e0193 --- /dev/null +++ b/docs/zh_CN/ch1/exchangis_sqoop_deploy_cn.md @@ -0,0 +1,48 @@ +### 环境准备 +Sqoop引擎是执行Exchangis数据同步任务不可或缺的组件，只有安装部署完成Sqoop引擎才能够成功执行数据同步任务。 + +您在安装部署Sqoop引擎之前，请按照Exchangis1.0.0安装部署文档安装完成Exchangis1.0.0及相关组件的安装，并确保工程基本功能可用。 + +### 安装包准备 +#### 1）下载二进制包 +[点击跳转 Release 界面](https://github.com/WeBankFinTech/Exchangis/releases) +#### 2） 编译打包 +如果您想自己开发和编译sqoop引擎，具体编译步骤如下： + +1.clone Exchangis的代码 + +2.在exchangis-plugins模块下，找到sqoop引擎，单独编译sqoop，操作如下 +``` +cd {EXCHANGIS_CODE_HOME}/exchangis-plugins/engine/sqoop +mvn clean install +``` +然后会在该路径下找到sqoop引擎安装包 +``` +{EXCHANGIS_CODE_HOME}\exchangis-plugins\sqoop\target\out\sqoop +``` + + +### 开始部署 +#### 1）sqoop引擎安装 +1.拿到打包出来的sqoop.zip物料包 + +2.放置到如下目录并进行解压 +``` +cd {LINKIS_HOME}/linkis/lib/linkis-engineconn-plugins +unzip.zip +``` +解压出来的目录结构为： +``` +dist +plugin +``` +(注意，看当前sqoop引擎对哪些用户有权限，不一定是root) + + +#### 2）重启linkis-engineplugin服务使sqoop引擎生效 +新加入linkis的引擎都要重启linkis的engineplugin服务才会生效，重启脚本为linkis安装目录下的./linkis-daemon.sh，具体步骤如下 +``` +{LINKIS _INSTALL_HOME}/links/sbin/ +./linkis-daemon.sh restart cg-engineplugin +``` +待服务启动成功，至此，sqoop安装部署就完成了。 diff --git a/docs/zh_CN/ch1/exchangis_sqoop_deply_cn.md b/docs/zh_CN/ch1/exchangis_sqoop_deply_cn.md new file mode 100644 index 000000000..7720e0193 --- /dev/null +++ b/docs/zh_CN/ch1/exchangis_sqoop_deply_cn.md @@ -0,0 +1,48 @@ +### 环境准备 +Sqoop引擎是执行Exchangis数据同步任务不可或缺的组件，只有安装部署完成Sqoop引擎才能够成功执行数据同步任务。 + +您在安装部署Sqoop引擎之前，请按照Exchangis1.0.0安装部署文档安装完成Exchangis1.0.0及相关组件的安装，并确保工程基本功能可用。 + +### 安装包准备 +#### 1）下载二进制包 +[点击跳转 Release 界面](https://github.com/WeBankFinTech/Exchangis/releases) +#### 2） 编译打包 +如果您想自己开发和编译sqoop引擎，具体编译步骤如下： + +1.clone Exchangis的代码 + +2.在exchangis-plugins模块下，找到sqoop引擎，单独编译sqoop，操作如下 +``` +cd {EXCHANGIS_CODE_HOME}/exchangis-plugins/engine/sqoop +mvn clean install +``` +然后会在该路径下找到sqoop引擎安装包 +``` +{EXCHANGIS_CODE_HOME}\exchangis-plugins\sqoop\target\out\sqoop +``` + + +### 开始部署 +#### 1）sqoop引擎安装 +1.拿到打包出来的sqoop.zip物料包 + +2.放置到如下目录并进行解压 +``` +cd {LINKIS_HOME}/linkis/lib/linkis-engineconn-plugins +unzip.zip +``` +解压出来的目录结构为： +``` +dist +plugin +``` +(注意，看当前sqoop引擎对哪些用户有权限，不一定是root) + + +#### 2）重启linkis-engineplugin服务使sqoop引擎生效 +新加入linkis的引擎都要重启linkis的engineplugin服务才会生效，重启脚本为linkis安装目录下的./linkis-daemon.sh，具体步骤如下 +``` +{LINKIS _INSTALL_HOME}/links/sbin/ +./linkis-daemon.sh restart cg-engineplugin +``` +待服务启动成功，至此，sqoop安装部署就完成了。 diff --git a/docs/zh_CN/ch1/exchangis_user_manual_new_cn.md b/docs/zh_CN/ch1/exchangis_user_manual_new_cn.md new file mode 100644 index 000000000..6428e482c --- /dev/null +++ b/docs/zh_CN/ch1/exchangis_user_manual_new_cn.md @@ -0,0 +1,177 @@ +Exchangis1.0用户手册 + +一.产品简介 + +本文是Exchangis1.0的快速入门文档，涵盖了Exchangis1.0的基本使用流程。Exchangis是一款轻量级的数据交换服务平台，支持不同类型数据源之间的数据同步。平台将数据交换流程进行拆分，抽象出数据源，数据交换任务，任务调度等概念，达到可视化管理数据同步流程的目的。而在实际数据传输过程中可集成多个传输组件特性，做到功能横向扩展。 + +二.登录Exchangis1.0 + +Exchangis1.0目前作为DSS《数据交换》组件的一部分，通过登录DSS的方式在组件列表中免密进入。所以，在使用Exchangis1.0之前，请对DSS，Exchangis1.0，Linkis等相关组件进行基本部署，保证组件功能可用，本文不进行赘述，详情见XXXXX。系统默认通过Linkis的Linux部署用户登录DSS，如使用hadoop部署Linkis和DSS，可以直接通过用户：hadoop，密码：hadoop(密码就是用户名)登录。 首先输入前端容器地址：192.168.xx.xx:8888 接着输入用户名密码：hadoop hadoop。 +进入步骤《首页》->《组件列表》->《数据质量》->《Exchangis》 + +图2.1 Exchangis1.0入口 + +三.数据源管理 + +进入《数据源管理》页面，可以对数据源进行配置和管理，为进行数据同步作业的起始步骤，目前Exchangis1.0支持对mysql和hive的数据相互导入导出。 +数据源主要功能如下： +1. 创建，编辑，删除数据源 +2. 根据类型和名称搜索数据源，支持对数据源快速定位 +3. 数据源连接测试操作 +4. 历史数据源版本发布及记录 + +创建数据源 + +点击《创建数据源》，选择自己想要创建的数据源，当前支持mysql和hive两种数据源的创建。 + +图3-1 数据源管理列表 + +图3-2 数据源展示 + +选择创建MYSQL数据源，填写配置参数，其中，带星号的为必填项，务必保证连接mysql数据库的Host，端口号，用户名和密码连接正确。《连接参数》为json格式，可在连接参数输入框中填入mysql的相关参数，填写完成点击确定即可保存。 + +图3-3 mysql数据源配置 + +对于HIVE数据源的配置，与mysql不太相同，暂时不提供用户自行在界面进行集群参数配置的功能，对于集群环境，由后端统一配置完成，用户只需要选择需要的集群环境即可，点击确定即可保存。 + +图3-4 hive数据源配置 + +数据源发布 + +数据源管理模块提供对配置数据源版本的《发布》功能，只有经过《发布》的数据源才能在配置导数任务的时候被使用，否则会提示不可用，只要再次编辑的数据源就会被视为一个新的版本，最新的版本在第一行。在版本列表中可以《查看》所有历史数据源版本的配置，您可在随时需要回滚时进行参考。 + +图3-5 版本列表界面 + +数据源管理《过期》按钮，用于提示此数据源已经逐渐要被替换，请及时更换使用该数据源的任务配置，避免直接删除数据源造成所配置的执行任务失效。 +数据源连接测试 + +为了能够确保当前配置数据源的有效性，可以通过《连接测试》按钮判断是否数据源能够正常连接。在版本列表及数据源管理首页，数据源编辑界面，均可进行该操作 + +四.项目管理 + +在进入《项目管理》之后，可以创建项目，在实际的导数任务中，一个项目下可以有多个导数任务，不同的项目之间互不影响，对于普通用户而言，可以操作的只有自己创建的项目。 +可以在项目管理首页，对项目进行管理，包括创建，修改和删除，同时能够通过名称进行项目的查询搜索。 + +图4-1 项目列表界面 + +任务列表 + +任务列表中可以对创建的job数据同步任务进行管理，与项目类似，包括创建，修改和删除，支持通过名称进行任务的搜索。 +除此之外，任务支持《复制》，能够增加需要的任务，复制的任务包含其原任务配置的所有信息。点击《创建任务》任务，能够选择任务类型和执行引擎，目前仅支持《离线任务》和《SQOOP》执行引擎，未来将会支持流式任务和DATAX引擎等。 + +图4-2 任务列表界面 + +数据同步任务配置和执行 + +数据同步任务的配置和执行是Exchangis1.0的核心功能，基本配置数据同步流程为：《添加子任务》->《选择源数据源库表和目的数据源库表》->《字段映射（可默认）》->《过程控制（可默认）》->《配置》（可默认）->《执行》。 + +任务执行主要功能包括： +1. 子任务卡片的添加，复制和删除; +2. 实现对两种不同类型数据源之间数据的导入导出； +3. 来源数据源和目的地数据源的库表选择； +4. 数据源字段映射； +5. 作业最大并发数和作业最大内存配置； +6. 数据同步任务执行情况查看； +7. 每个主任务和各个子任务的日志查看； +8. 任务执行历史状态查看； +9. 执行任务kill操作 + +数据源选择和配置 + +点击《添加子任务》开始创建一个数据同步任务，对于新创建的数据同步子任务，首先要进行数据源库表的选择，数据源要在《数据源管理》模块中提前配置好，才会在任务配置中出现。数据源选择支持搜索，搜索方式为先搜索库，再搜索表。 + +图4-3 库表搜索界面 + +Mysql为目的地数据源时，支持插入和更新两种写入方式；为源数据源时候，支持WHERE条件语句查询。 + +HIVE为目的地数据源时，支持分区信息配置，写入方式为追加数据和覆盖两种；为源数据源时，支持分区信息配置。 + +数据源字段映射 + +当配置完成数据源库表信息时，Exchangis1.0会自动在《字段映射》一行生成原数据源和目的数据源的字段自动映射，并且可以自行选择想要映射的字段；当HIVE为目的地数据源时，其映射字段不可修改，这点要注意。 + +图4-4 字段映射配置 + +过程控制 + +任务执行提供对任务的最大并行数配置，以及作业最大内存配置，可根据实际需要进行更改。 + +图4-5 过程控制配置 + +作业执行 + +Exchangis1.0支持多个子任务同时执行，任务配置完成后，点击执行，即开始数据同步任务，界面下方会弹出工作台，工作台主要包含三个部分功能：《运行情况》，《实时日志》和《执行历史》； +《运行情况》能够查看当前数据同步任务整体进度，包含task成功和失败的数量等，以及点击task的名字，能够展示每个task的各项运行指标信息。 + +图4-6 任务执行界面 + +《实时日志》主要展示的内容包含两大类，一是整个导数job的日志，能够输出每个task的状态日志，例如task是否被调度，是否运行中等；二是每个task的日志，输出的是各自相应的导数日志。《实时日志》能够根据关键字和忽略字进行日志筛选，并别提供获取最后n行日志功能。 +在《实时日志》中可以对Error、Warning和Info不同类型的日志进行筛选展示，只需点击相应的按钮即可。 + +图4-7 日志查看操作 + +《执行历史》能够展示该导数任务的历史执行信息，对历史执行过程提供初步的概览，如果想进一步查看详细历史信息，点击任务名称，即可跳转到同步历史界面进行查看。 + +图4-8 执行历史界面 + +数据同步任务执行需指定执行用户，默认为登录用户，具体情况需根据实际数据源的配置去调整。 + +图4-9 任务配置参数 + +五.同步历史 + +点击进入《同步历史》页面，可以查看到历史执行的所有数据同步任务，每个用户只能查看自己创建的任务，不同用户之间互相隔离。 + +主要功能如下： +1. 根据查询条件查找所需的历史任务信息； +2. 对于非终态的任务，提供终止任务的功能，能够kill掉非终态的任务； +3. 查看每个任务的运行情况和实时日志； +4. 查看每个同步任务的更细节配置信息和更新时间等。 + + +图5-1 同步历史界面 + +六.EXCHANGIS APPCONN使用 + +目前，Exchangis1.0支持以APPCONN形式与DSS对接，能够通过DSS的《工作空间》->《项目列表》，以工作流编排的模式创建数据交换《sqoop》工作流节点，在这里，能够进行数据同步任务的配置和执行。在DSS创建的Exchangis项目和数据交换任务，会同步在Exchangis中创建。 + +Exchangis Appconn主要支持以下功能： +1. 《项目操作》对DSS项目的创建，删除，修改操作会同步影响Exchangis端的项目； +2. 《工作流节点基本操作》DSS编排器中对sqoop工作流节点创建，删除，修改操作会同步到Exchangis端的任务； +3. 《工作流导数操作》支持sqoop工作流节点配置执行数据同步任务； +4. 《工作流发布操作》支持sqoop工作流节点发布至WTSS进行任务调度。 + +项目操作 + +这里以在DSS创建项目为例，给您展示exchangis appconn对项目的操作，流程为：《点击创建项目（DSS端）》->《填写项目信息》->《点击确认》->《进入Exchangis端》->《点击项目管理》，即可查看到同步创建的项目，如下图所示，同理对DSS端项目信息的修改和删除，也会同步对Exchangis端的项目进行操作： + +图6-1 DSS创建项目操作 + +图6-2 Exchangis端项目同步创建展示 + +工作流节点基本操作 + +这里以对sqoop工作流节点的创建操作为例，给您展示exchangis appconn对项目的操作，流程为：《创建一条工作流》->《从左侧插件栏拖动sqoop节点至右侧画布》->《点击确认创建sqoop节点任务》->《进入exchangis端查看同步创建的任务》，如下图所示，对sqoop节点任务的删除和修改同理。 + +图6-3 拖动创建sqoop工作流节点 + +图6-4 数据导入任务同步创建至Exchangis + +工作流导数操作 + +以工作流节点形式进行导数任务是Exchangis Appconn的核心功能，每一个sqoop节点代表一个数据同步任务，具体操作流程如下：《双击sqoop节点》->《弹出任务配置界面》->《配置任务信息》->《执行任务》。这里有两种执行方式，一种是在弹出的任务配置界面点击执行按钮进行执行。另一种是点击DSS编排器的《执行》按钮或者是《选中执行》按钮进行执行，《执行》按钮会对该工作流中的所有节点执行，《选中执行》仅会执行选中的工作流节点，不会执行全部节点，如下图所示： + +图6-5 双击sqoop工作流节点进入配置界面 + +图6-6配置工作流节点信息 + +图6-5执行任务 + +注：在DSS的sqoop节点中执行的数据同步任务，均可在Exchangis端查看相关信息。 + +工作流发布 + +工作流任务的《发布》功能，在开发中心创建和配置的数据交换任务信息，通过发布，发布到WTSS，能够在WTSS中进行任务调度。关于WTSS的使用说明，详见该地址XXXX + +图6-6 发布至WTSS进行任务调度 + diff --git a/exchangis-dao/pom.xml b/exchangis-dao/pom.xml index a8117fafc..9484df448 100644 --- a/exchangis-dao/pom.xml +++ b/exchangis-dao/pom.xml @@ -14,7 +14,7 @@ 8 8 - 6.0.15.Final + 5.1.1.Final diff --git a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/vo/DataSourceCreateVO.java b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/vo/DataSourceCreateVO.java index 69625f6d1..0b3e34b1a 100644 --- a/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/vo/DataSourceCreateVO.java +++ b/exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/vo/DataSourceCreateVO.java @@ -2,8 +2,8 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import javax.validation.constraints.NotBlank; -import javax.validation.constraints.NotEmpty; +//import javax.validation.constraints.NotBlank; +//import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; import javax.validation.constraints.Size; import java.util.Date; diff --git a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/vo/ExchangisJobVo.java b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/vo/ExchangisJobVo.java index d965b728e..b616880e2 100644 --- a/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/vo/ExchangisJobVo.java +++ b/exchangis-job/exchangis-job-common/src/main/java/com/webank/wedatasphere/exchangis/job/vo/ExchangisJobVo.java @@ -6,8 +6,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.webank.wedatasphere.exchangis.common.validator.groups.InsertGroup; import com.webank.wedatasphere.exchangis.job.domain.ExchangisJobInfo; +import org.hibernate.validator.constraints.NotBlank; -import javax.validation.constraints.NotBlank; import javax.validation.constraints.NotNull; import javax.validation.constraints.Size; import java.util.*; diff --git a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/external/ExchangisJobDssAppConnRestfulApi.java b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/external/ExchangisJobDssAppConnRestfulApi.java index f9c5c638f..269b73ba8 100644 --- a/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/external/ExchangisJobDssAppConnRestfulApi.java +++ b/exchangis-job/exchangis-job-server/src/main/java/com/webank/wedatasphere/exchangis/job/server/restful/external/ExchangisJobDssAppConnRestfulApi.java @@ -1,5 +1,6 @@ package com.webank.wedatasphere.exchangis.job.server.restful.external; +import com.fasterxml.jackson.core.JsonProcessingException; import com.webank.wedatasphere.exchangis.common.validator.groups.InsertGroup; import com.webank.wedatasphere.exchangis.job.domain.ExchangisJobInfo; import com.webank.wedatasphere.exchangis.job.launcher.domain.task.TaskStatus; @@ -13,6 +14,7 @@ import com.webank.wedatasphere.exchangis.project.server.entity.ExchangisProject; import com.webank.wedatasphere.exchangis.project.server.mapper.ProjectMapper; import org.apache.commons.lang.StringUtils; +import org.apache.linkis.server.BDPJettyServerHelper; import org.apache.linkis.server.Message; import org.apache.linkis.server.security.SecurityFilter; import org.slf4j.Logger; @@ -157,6 +159,13 @@ else if (!hasAuthority(userName, jobInfoService.getJob(id , true))){ */ @RequestMapping( value = "/execute/{id}", method = RequestMethod.POST) public Message executeJob(@PathVariable("id") Long id, HttpServletRequest request, @RequestBody Map params) { + try { + LOG.info("start to parse params8909"); + String paramString = BDPJettyServerHelper.jacksonJson().writeValueAsString(params); + LOG.error("paramString999879: {}", paramString); + } catch (JsonProcessingException e) { + LOG.error("parse execute content error: {}", e.getMessage()); + } String submitUser = params.get("submitUser").toString(); String loginUser = SecurityFilter.getLoginUsername(request); Message result = Message.ok(); diff --git a/exchangis-plugins/engine/sqoop/pom.xml b/exchangis-plugins/engine/sqoop/pom.xml new file mode 100644 index 000000000..d1724cd3c --- /dev/null +++ b/exchangis-plugins/engine/sqoop/pom.xml @@ -0,0 +1,285 @@ + + + + + + linkis + org.apache.linkis + 1.1.1 + ../../pom.xml + + 4.0.0 + + linkis-engineplugin-sqoop + + 1.4.6 + 3.1.2 + + + + + org.apache.commons + commons-exec + provided + 1.3 + + + org.apache.sqoop + sqoop + hadoop200 + ${sqoop.version} + + + org.apache.commons + commons-exec + provided + 1.3 + + + org.apache.avro + avro + provided + 1.10.2 + + + org.apache.hive + hive-common + provided + ${hive.version} + + + * + * + + + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + + + servlet-api + javax.servlet + + + + + org.apache.linkis + linkis-once-engineconn + ${linkis.version} + + + commons-logging + commons-logging + + + + + org.apache.linkis + linkis-computation-engineconn + ${linkis.version} + + + org.apache.hadoop + hadoop-client + ${hadoop.version} + + + log4j + log4j + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jetty-util + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-json + + + jsr311-api + javax.ws.rs + + + net.java.dev.jets3t + jets3t + + + com.jcraft + jsch + + + com.google.code.findbugs + jsr305 + + + xmlenc + xmlenc + + + net.java.dev.jets3t + jets3t + + + org.apache.avro + avro + + + org.apache.hadoop + hadoop-auth + + + com.jcraft + jsch + + + com.google.code.findbugs + jsr305 + + + servlet-api + javax.servlet + + + org.slf4j + slf4j-log4j12 + + + hadoop-hdfs + org.apache.hadoop + + + + + org.apache.linkis + linkis-engineconn-plugin-core + ${linkis.version} + + + + org.apache.linkis + linkis-rpc + ${linkis.version} + provided + + + + org.apache.linkis + linkis-storage + ${linkis.version} + provided + + + + org.apache.linkis + linkis-common + ${linkis.version} + provided + + + + org.apache.linkis + linkis-bml-engine-hook + ${linkis.version} + + + commons-logging + commons-logging + + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + + + + net.alchim31.maven + scala-maven-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + 2.3 + false + + + make-assembly + package + + single + + + + src/main/assembly/distribution.xml + + + + + + false + out + false + false + + src/main/assembly/distribution.xml + + + + + + + src/main/java + + **/*.xml + **/*.properties + + + + src/main/resources + + **/application.yml + **/bootstrap.yml + + + + + \ No newline at end of file diff --git a/exchangis-plugins/engine/sqoop/src/main/assembly/distribution.xml b/exchangis-plugins/engine/sqoop/src/main/assembly/distribution.xml new file mode 100644 index 000000000..78f54c4c0 --- /dev/null +++ b/exchangis-plugins/engine/sqoop/src/main/assembly/distribution.xml @@ -0,0 +1,324 @@ + + + + sqoop + + dir + + true + sqoop + + + + + + /dist/v${sqoop.version}/lib + true + true + false + false + true + + + antlr:antlr:jar + aopalliance:aopalliance:jar + asm:asm:jar + cglib:cglib:jar + com.amazonaws:aws-java-sdk-autoscaling:jar + com.amazonaws:aws-java-sdk-core:jar + com.amazonaws:aws-java-sdk-ec2:jar + com.amazonaws:aws-java-sdk-route53:jar + com.amazonaws:aws-java-sdk-sts:jar + com.amazonaws:jmespath-java:jar + com.fasterxml.jackson.core:jackson-annotations:jar + com.fasterxml.jackson.core:jackson-core:jar + com.fasterxml.jackson.core:jackson-databind:jar + com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:jar + com.fasterxml.jackson.datatype:jackson-datatype-jdk8:jar + com.fasterxml.jackson.datatype:jackson-datatype-jsr310:jar + com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:jar + com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:jar + com.fasterxml.jackson.module:jackson-module-jaxb-annotations:jar + com.fasterxml.jackson.module:jackson-module-parameter-names:jar + com.fasterxml.jackson.module:jackson-module-paranamer:jar + com.fasterxml.jackson.module:jackson-module-scala_2.11:jar + com.github.andrewoma.dexx:dexx-collections:jar + com.github.vlsi.compactmap:compactmap:jar + com.google.code.findbugs:annotations:jar + com.google.code.findbugs:jsr305:jar + com.google.code.gson:gson:jar + com.google.guava:guava:jar + com.google.inject:guice:jar + com.google.protobuf:protobuf-java:jar + com.netflix.archaius:archaius-core:jar + com.netflix.eureka:eureka-client:jar + com.netflix.eureka:eureka-core:jar + com.netflix.hystrix:hystrix-core:jar + com.netflix.netflix-commons:netflix-commons-util:jar + com.netflix.netflix-commons:netflix-eventbus:jar + com.netflix.netflix-commons:netflix-infix:jar + com.netflix.netflix-commons:netflix-statistics:jar + com.netflix.ribbon:ribbon:jar + com.netflix.ribbon:ribbon-core:jar + com.netflix.ribbon:ribbon-eureka:jar + com.netflix.ribbon:ribbon-httpclient:jar + com.netflix.ribbon:ribbon-loadbalancer:jar + com.netflix.ribbon:ribbon-transport:jar + com.netflix.servo:servo-core:jar + com.ning:async-http-client:jar + com.sun.jersey.contribs:jersey-apache-client4:jar + com.sun.jersey:jersey-client:jar + com.sun.jersey:jersey-core:jar + com.sun.jersey:jersey-json:jar + com.sun.jersey:jersey-server:jar + com.sun.jersey:jersey-servlet:jar + com.sun.xml.bind:jaxb-impl:jar + com.thoughtworks.paranamer:paranamer:jar + com.thoughtworks.xstream:xstream:jar + org.apache.linkis:linkis-common:jar + org.apache.linkis:linkis-module:jar + commons-beanutils:commons-beanutils:jar + commons-beanutils:commons-beanutils-core:jar + commons-cli:commons-cli:jar + commons-codec:commons-codec:jar + commons-collections:commons-collections:jar + commons-configuration:commons-configuration:jar + commons-daemon:commons-daemon:jar + commons-dbcp:commons-dbcp:jar + commons-digester:commons-digester:jar + commons-httpclient:commons-httpclient:jar + commons-io:commons-io:jar + commons-jxpath:commons-jxpath:jar + commons-lang:commons-lang:jar + commons-logging:commons-logging:jar + commons-net:commons-net:jar + commons-pool:commons-pool:jar + io.micrometer:micrometer-core:jar + io.netty:netty:jar + io.netty:netty-all:jar + io.netty:netty-buffer:jar + io.netty:netty-codec:jar + io.netty:netty-codec-http:jar + io.netty:netty-common:jar + io.netty:netty-handler:jar + io.netty:netty-transport:jar + io.netty:netty-transport-native-epoll:jar + io.reactivex:rxjava:jar + io.reactivex:rxnetty:jar + io.reactivex:rxnetty-contexts:jar + io.reactivex:rxnetty-servo:jar + javax.activation:activation:jar + javax.annotation:javax.annotation-api:jar + javax.inject:javax.inject:jar + javax.servlet:javax.servlet-api:jar + javax.servlet.jsp:jsp-api:jar + javax.validation:validation-api:jar + javax.websocket:javax.websocket-api:jar + javax.ws.rs:javax.ws.rs-api:jar + javax.xml.bind:jaxb-api:jar + javax.xml.stream:stax-api:jar + joda-time:joda-time:jar + log4j:log4j:jar + mysql:mysql-connector-java:jar + net.databinder.dispatch:dispatch-core_2.11:jar + net.databinder.dispatch:dispatch-json4s-jackson_2.11:jar + org.antlr:antlr-runtime:jar + org.antlr:stringtemplate:jar + org.apache.commons:commons-compress:jar + org.apache.commons:commons-math:jar + org.apache.commons:commons-math3:jar + org.apache.curator:curator-client:jar + org.apache.curator:curator-framework:jar + org.apache.curator:curator-recipes:jar + org.apache.directory.api:api-asn1-api:jar + org.apache.directory.api:api-util:jar + org.apache.directory.server:apacheds-i18n:jar + org.apache.directory.server:apacheds-kerberos-codec:jar + org.apache.hadoop:hadoop-annotations:jar + org.apache.hadoop:hadoop-auth:jar + org.apache.hadoop:hadoop-common:jar + org.apache.hadoop:hadoop-hdfs:jar + org.apache.htrace:htrace-core:jar + org.apache.httpcomponents:httpclient:jar + org.apache.httpcomponents:httpcore:jar + org.apache.logging.log4j:log4j-api:jar + org.apache.logging.log4j:log4j-core:jar + org.apache.logging.log4j:log4j-jul:jar + org.apache.logging.log4j:log4j-slf4j-impl:jar + org.apache.zookeeper:zookeeper:jar + org.aspectj:aspectjweaver:jar + org.bouncycastle:bcpkix-jdk15on:jar + org.bouncycastle:bcprov-jdk15on:jar + org.codehaus.jackson:jackson-jaxrs:jar + org.codehaus.jackson:jackson-xc:jar + org.codehaus.jettison:jettison:jar + org.codehaus.woodstox:stax2-api:jar + org.codehaus.woodstox:woodstox-core-asl:jar + org.eclipse.jetty:jetty-annotations:jar + org.eclipse.jetty:jetty-client:jar + org.eclipse.jetty:jetty-continuation:jar + org.eclipse.jetty:jetty-http:jar + org.eclipse.jetty:jetty-io:jar + org.eclipse.jetty:jetty-jndi:jar + org.eclipse.jetty:jetty-plus:jar + org.eclipse.jetty:jetty-security:jar + org.eclipse.jetty:jetty-server:jar + org.eclipse.jetty:jetty-servlet:jar + org.eclipse.jetty:jetty-servlets:jar + org.eclipse.jetty:jetty-util:jar + org.eclipse.jetty:jetty-webapp:jar + org.eclipse.jetty:jetty-xml:jar + org.eclipse.jetty.websocket:javax-websocket-client-impl:jar + org.eclipse.jetty.websocket:javax-websocket-server-impl:jar + org.eclipse.jetty.websocket:websocket-api:jar + org.eclipse.jetty.websocket:websocket-client:jar + org.eclipse.jetty.websocket:websocket-common:jar + org.eclipse.jetty.websocket:websocket-server:jar + org.eclipse.jetty.websocket:websocket-servlet:jar + org.fusesource.leveldbjni:leveldbjni-all:jar + org.glassfish.hk2:class-model:jar + org.glassfish.hk2:config-types:jar + org.glassfish.hk2.external:aopalliance-repackaged:jar + org.glassfish.hk2.external:asm-all-repackaged:jar + org.glassfish.hk2.external:bean-validator:jar + org.glassfish.hk2.external:javax.inject:jar + org.glassfish.hk2:hk2:jar + org.glassfish.hk2:hk2-api:jar + org.glassfish.hk2:hk2-config:jar + org.glassfish.hk2:hk2-core:jar + org.glassfish.hk2:hk2-locator:jar + org.glassfish.hk2:hk2-runlevel:jar + org.glassfish.hk2:hk2-utils:jar + org.glassfish.hk2:osgi-resource-locator:jar + org.glassfish.hk2:spring-bridge:jar + org.glassfish.jersey.bundles:jaxrs-ri:jar + org.glassfish.jersey.bundles.repackaged:jersey-guava:jar + org.glassfish.jersey.containers:jersey-container-servlet:jar + org.glassfish.jersey.containers:jersey-container-servlet-core:jar + org.glassfish.jersey.core:jersey-client:jar + org.glassfish.jersey.core:jersey-common:jar + org.glassfish.jersey.core:jersey-server:jar + org.glassfish.jersey.ext:jersey-entity-filtering:jar + org.glassfish.jersey.ext:jersey-spring3:jar + org.glassfish.jersey.media:jersey-media-jaxb:jar + org.glassfish.jersey.media:jersey-media-json-jackson:jar + org.glassfish.jersey.media:jersey-media-multipart:jar + org.hdrhistogram:HdrHistogram:jar + org.javassist:javassist:jar + org.json4s:json4s-ast_2.11:jar + org.json4s:json4s-core_2.11:jar + org.json4s:json4s-jackson_2.11:jar + org.jsoup:jsoup:jar + org.jvnet.mimepull:mimepull:jar + org.jvnet:tiger-types:jar + org.latencyutils:LatencyUtils:jar + org.mortbay.jasper:apache-el:jar + org.mortbay.jetty:jetty:jar + org.mortbay.jetty:jetty-util:jar + org.ow2.asm:asm-analysis:jar + org.ow2.asm:asm-commons:jar + org.ow2.asm:asm-tree:jar + org.reflections:reflections:jar + org.scala-lang.modules:scala-parser-combinators_2.11:jar + org.scala-lang.modules:scala-xml_2.11:jar + org.scala-lang:scala-compiler:jar + org.scala-lang:scala-library:jar + org.scala-lang:scala-reflect:jar + org.scala-lang:scalap:jar + org.slf4j:jul-to-slf4j:jar + org.slf4j:slf4j-api:jar + org.springframework.boot:spring-boot:jar + org.springframework.boot:spring-boot-actuator:jar + org.springframework.boot:spring-boot-actuator-autoconfigure:jar + org.springframework.boot:spring-boot-autoconfigure:jar + org.springframework.boot:spring-boot-starter:jar + org.springframework.boot:spring-boot-starter-actuator:jar + org.springframework.boot:spring-boot-starter-aop:jar + org.springframework.boot:spring-boot-starter-jetty:jar + org.springframework.boot:spring-boot-starter-json:jar + org.springframework.boot:spring-boot-starter-log4j2:jar + org.springframework.boot:spring-boot-starter-web:jar + org.springframework.cloud:spring-cloud-commons:jar + org.springframework.cloud:spring-cloud-config-client:jar + org.springframework.cloud:spring-cloud-context:jar + org.springframework.cloud:spring-cloud-netflix-archaius:jar + org.springframework.cloud:spring-cloud-netflix-core:jar + org.springframework.cloud:spring-cloud-netflix-eureka-client:jar + org.springframework.cloud:spring-cloud-netflix-ribbon:jar + org.springframework.cloud:spring-cloud-starter:jar + org.springframework.cloud:spring-cloud-starter-config:jar + org.springframework.cloud:spring-cloud-starter-eureka:jar + org.springframework.cloud:spring-cloud-starter-netflix-archaius:jar + org.springframework.cloud:spring-cloud-starter-netflix-eureka-client:jar + org.springframework.cloud:spring-cloud-starter-netflix-ribbon:jar + org.springframework.security:spring-security-crypto:jar + org.springframework.security:spring-security-rsa:jar + org.springframework:spring-aop:jar + org.springframework:spring-beans:jar + org.springframework:spring-context:jar + org.springframework:spring-core:jar + org.springframework:spring-expression:jar + org.springframework:spring-jcl:jar + org.springframework:spring-web:jar + org.springframework:spring-webmvc:jar + org.tukaani:xz:jar + org.yaml:snakeyaml:jar + software.amazon.ion:ion-java:jar + xerces:xercesImpl:jar + xmlenc:xmlenc:jar + xmlpull:xmlpull:jar + xpp3:xpp3_min:jar + + + + + + + + ${basedir}/src/main/resources + + * + + 0777 + 0755 + /dist/v${sqoop.version}/conf + unix + + + + ${basedir}/target + + *.jar + + + *doc.jar + + 0777 + /plugin/${sqoop.version} + + + + + + + diff --git a/exchangis-plugins/engine/sqoop/src/main/java/org/apache/linkis/engineconnplugin/sqoop/client/LinkisSqoopClient.java b/exchangis-plugins/engine/sqoop/src/main/java/org/apache/linkis/engineconnplugin/sqoop/client/LinkisSqoopClient.java new file mode 100644 index 000000000..9d364be57 --- /dev/null +++ b/exchangis-plugins/engine/sqoop/src/main/java/org/apache/linkis/engineconnplugin/sqoop/client/LinkisSqoopClient.java @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconnplugin.sqoop.client; + +import org.apache.linkis.common.exception.ErrorException; +import org.apache.linkis.engineconnplugin.sqoop.client.utils.JarLoader; +import org.apache.linkis.protocol.engine.JobProgressInfo; + +import org.apache.sqoop.SqoopOptions; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.io.Writer; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Map; +import java.util.function.Consumer; + +public class LinkisSqoopClient { + private static Class sqoopEngineClass; + private static Logger logger = LoggerFactory.getLogger(LinkisSqoopClient.class); + private static JarLoader jarLoader; + + public static int run(Map params) { + try { + jarLoader = + new JarLoader( + new String[] { + LinkisSqoopClient.class + .getProtectionDomain() + .getCodeSource() + .getLocation() + .getPath() + }); + // Load the sqoop class redefined by progress, notice that is not be resolved + jarLoader.loadClass("org.apache.sqoop.mapreduce.JobBase", false); + // Add the sqoop-{version}.jar to class path + jarLoader.addJarURL( + SqoopOptions.class + .getProtectionDomain() + .getCodeSource() + .getLocation() + .getPath()); + // Update the context loader + Thread.currentThread().setContextClassLoader(jarLoader); + sqoopEngineClass = + jarLoader.loadClass("org.apache.linkis.engineconnplugin.sqoop.client.Sqoop"); + Method method = sqoopEngineClass.getDeclaredMethod("main", Map.class); + return (Integer) method.invoke(null, params); + } catch (Throwable e) { + logger.error("Run Error Message:" + getLog(e), e); + return -1; + } + } + + /** Close */ + public static void close() { + operateInClassLoader( + jarLoader, + () -> { + Method method = sqoopEngineClass.getDeclaredMethod("close"); + method.invoke(null); + return null; + }, + e -> logger.error("Close Error Message: {}", getLog(e))); + } + + /** + * Fetch application id + * + * @return application id + */ + public static String getApplicationId() { + return operateInClassLoader( + jarLoader, + () -> { + Method method = sqoopEngineClass.getDeclaredMethod("getApplicationId"); + return (String) method.invoke(null); + }, + e -> logger.error("Linkis SqoopClient getApplicationId: {}", getLog(e))); + } + + /** + * Fetch application url + * + * @return url + */ + public static String getApplicationURL() { + return operateInClassLoader( + jarLoader, + () -> { + Method method = sqoopEngineClass.getDeclaredMethod("getApplicationURL"); + return (String) method.invoke(null); + }, + e -> logger.error("Linkis SqoopClient getApplicationURL: {}", getLog(e))); + } + + /** + * Progress value + * + * @return progress + */ + public static Float progress() { + return operateInClassLoader( + jarLoader, + () -> { + Method method = sqoopEngineClass.getDeclaredMethod("progress"); + return (Float) method.invoke(null); + }, + e -> logger.error("Linkis SqoopClient progress: {}", getLog(e))); + } + + /** + * ProgressInfo + * + * @return + */ + @SuppressWarnings("unchecked") + public static JobProgressInfo getProgressInfo() { + return operateInClassLoader( + jarLoader, + () -> { + Method method = sqoopEngineClass.getDeclaredMethod("getProgressInfo"); + return (JobProgressInfo) method.invoke(null); + }, + e -> logger.error("Linkis SqoopClient getProgressInfo: {}", getLog(e))); + } + + /** + * Get metrics + * + * @return map value + */ + @SuppressWarnings("unchecked") + public static Map getMetrics() { + return operateInClassLoader( + jarLoader, + () -> { + Method method = sqoopEngineClass.getDeclaredMethod("getMetrics"); + return (Map) method.invoke(null); + }, + e -> logger.error("Linkis SqoopClient getMetrics: {}", getLog(e))); + } + + /** + * Get diagnosis + * + * @return map value + */ + @SuppressWarnings("unchecked") + public static Map getDiagnosis() { + return operateInClassLoader( + jarLoader, + () -> { + Method method = sqoopEngineClass.getDeclaredMethod("getDiagnosis"); + return (Map) method.invoke(null); + }, + e -> logger.error("Linkis SqoopClient getDiagnosis: {}", getLog(e))); + } + + /** + * Console log + * + * @param e throwable + * @return log + */ + private static String getLog(Throwable e) { + Writer result = new StringWriter(); + PrintWriter printWriter = new PrintWriter(result); + e.printStackTrace(printWriter); + return e.toString(); + } + + /** + * Operate in special classloader + * + * @param classLoader classloader + * @param operation operation + * @param resolver resolver + * @param return type + * @return return + */ + private static R operateInClassLoader( + ClassLoader classLoader, ClientOperation operation, Consumer resolver) { + ClassLoader currentLoader = Thread.currentThread().getContextClassLoader(); + R result = null; + try { + Thread.currentThread().setContextClassLoader(classLoader); + result = operation.operate(); + } catch (Exception t) { + resolver.accept(t); + } finally { + Thread.currentThread().setContextClassLoader(currentLoader); + } + return result; + } + + @FunctionalInterface + interface ClientOperation { + + /** + * Operate + * + * @return T + * @throws ErrorException error exception + */ + T operate() + throws ErrorException, NoSuchMethodException, InvocationTargetException, + IllegalAccessException; + } +} diff --git a/exchangis-plugins/engine/sqoop/src/main/java/org/apache/linkis/engineconnplugin/sqoop/client/Sqoop.java b/exchangis-plugins/engine/sqoop/src/main/java/org/apache/linkis/engineconnplugin/sqoop/client/Sqoop.java new file mode 100644 index 000000000..bb1e0c093 --- /dev/null +++ b/exchangis-plugins/engine/sqoop/src/main/java/org/apache/linkis/engineconnplugin/sqoop/client/Sqoop.java @@ -0,0 +1,551 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconnplugin.sqoop.client; + +import org.apache.linkis.engineconnplugin.sqoop.client.config.ParamsMapping; +import org.apache.linkis.engineconnplugin.sqoop.client.exception.JobClosableException; +import org.apache.linkis.engineconnplugin.sqoop.context.SqoopEnvConfiguration; +import org.apache.linkis.engineconnplugin.sqoop.context.SqoopParamsConfiguration; +import org.apache.linkis.protocol.engine.JobProgressInfo; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.mapred.TIPStatus; +import org.apache.hadoop.mapreduce.*; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.apache.sqoop.manager.SqlManager; +import org.apache.sqoop.manager.oracle.OraOopManagerFactory; +import org.apache.sqoop.util.LoggingUtils; + +import com.cloudera.sqoop.SqoopOptions; +import com.cloudera.sqoop.manager.DefaultManagerFactory; +import com.cloudera.sqoop.tool.SqoopTool; +import com.cloudera.sqoop.util.OptionsFileUtil; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Field; +import java.net.MalformedURLException; +import java.nio.file.Paths; +import java.sql.SQLException; +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Main entry-point for Sqoop Usage: hadoop jar (this_jar_name) com.cloudera.sqoop.Sqoop (options) + * See the SqoopOptions class for options. + */ +public class Sqoop extends Configured implements Tool { + + public static final Log LOG = LogFactory.getLog(Sqoop.class.getName()); + public static volatile AtomicReference job = new AtomicReference<>(); + public static SqlManager sqlManager; + public static final String[] DEFAULT_FACTORY_CLASS_NAMES_ARR = { + OraOopManagerFactory.class.getName(), DefaultManagerFactory.class.getName(), + }; + public static final String FACTORY_CLASS_NAMES_KEY = "sqoop.connection.factories"; + public static final String METRICS_RUN_TIME = "MetricsRunTime"; + private static Float progress = 0.0f; + + /** + * If this System property is set, always throw an exception, do not just exit with status 1. + */ + public static final String SQOOP_RETHROW_PROPERTY = "sqoop.throwOnError"; + + /** The option to specify an options file from which other options to the tool are read. */ + public static final String SQOOP_OPTIONS_FILE_SPECIFIER = "--options-file"; + + static { + Configuration.addDefaultResource("sqoop-site.xml"); + } + + private SqoopTool tool; + private SqoopOptions options; + private String[] childPrgmArgs; + + /** + * Creates a new instance of Sqoop set to run the supplied SqoopTool with the default + * configuration. + * + * @param tool the SqoopTool to run in the main body of Sqoop. + */ + public Sqoop(SqoopTool tool) { + this(tool, (Configuration) null); + } + + /** + * Creates a new instance of Sqoop set to run the supplied SqoopTool with the provided + * configuration. + * + * @param tool the SqoopTool to run in the main body of Sqoop. + * @param conf the Configuration to use (e.g., from ToolRunner). + */ + public Sqoop(SqoopTool tool, Configuration conf) { + this(tool, conf, new SqoopOptions()); + } + + /** + * Creates a new instance of Sqoop set to run the supplied SqoopTool with the provided + * configuration and SqoopOptions. + * + * @param tool the SqoopTool to run in the main body of Sqoop. + * @param conf the Configuration to use (e.g., from ToolRunner). + * @param opts the SqoopOptions which control the tool's parameters. + */ + public Sqoop(SqoopTool tool, Configuration conf, SqoopOptions opts) { + /*LOG.info("Running Sqoop version: " + new SqoopVersion().VERSION);*/ + + if (null != conf) { + setConf(conf); + } + + this.options = opts; + this.options.setConf(getConf()); + + this.tool = tool; + } + + /** @return the SqoopOptions used in this Sqoop instance. */ + public SqoopOptions getOptions() { + return this.options; + } + + /** @return the SqoopTool used in this Sqoop instance. */ + public SqoopTool getTool() { + return this.tool; + } + + @Override + /** Actual main entry-point for the program */ + public int run(String[] args) { + if (options.getConf() == null) { + options.setConf(getConf()); + } + options.getConf().setStrings(FACTORY_CLASS_NAMES_KEY, DEFAULT_FACTORY_CLASS_NAMES_ARR); + try { + options = tool.parseArguments(args, null, options, false); + tool.appendArgs(this.childPrgmArgs); + tool.validateOptions(options); + if (options.getVerbose()) { + LoggingUtils.setDebugLevel(); + } + } catch (Exception e) { + LOG.error(e.getMessage(), e); + System.err.println(e.getMessage()); + return 1; + } + return tool.run(options); + } + + /** + * SqoopTools sometimes pass arguments to a child program (e.g., mysqldump). Users can specify + * additional args to these programs by preceeding the additional arguments with a standalone + * '--'; but ToolRunner/GenericOptionsParser will cull out this argument. We remove the + * child-program arguments in advance, and store them to be readded later. + * + * @param argv the argv in to the SqoopTool + * @return the argv with a "--" and any subsequent arguments removed. + */ + private String[] stashChildPrgmArgs(String[] argv) { + for (int i = 0; i < argv.length; i++) { + if ("--".equals(argv[i])) { + this.childPrgmArgs = Arrays.copyOfRange(argv, i, argv.length); + return Arrays.copyOfRange(argv, 0, i); + } + } + + // Didn't find child-program arguments. + return argv; + } + + /** + * Given a Sqoop object and a set of arguments to deliver to its embedded SqoopTool, run the + * tool, wrapping the call to ToolRunner. This entry-point is preferred to ToolRunner.run() + * because it has a chance to stash child program arguments before GenericOptionsParser would + * remove them. + */ + public static int runSqoop(Sqoop sqoop, String[] args) { + String[] toolArgs = sqoop.stashChildPrgmArgs(args); + try { + return ToolRunner.run(sqoop.getConf(), sqoop, toolArgs); + } catch (Exception e) { + LOG.error("Got exception running Sqoop: " + e.toString()); + e.printStackTrace(); + rethrowIfRequired(toolArgs, e); + return 1; + } + } + + public static void rethrowIfRequired(String[] toolArgs, Exception ex) { + final RuntimeException exceptionToThrow; + if (ex instanceof RuntimeException) { + exceptionToThrow = (RuntimeException) ex; + } else { + exceptionToThrow = new RuntimeException(ex); + } + + throw exceptionToThrow; + } + + /** + * Entry-point that parses the correct SqoopTool to use from the args, but does not call + * System.exit() as main() will. + */ + public static int runTool(Map argsMap, Configuration conf) { + + // Expand the options + String[] expandedArgs = null; + try { + String[] flatArgs = convertParamsMapToAarray(argsMap, conf); + expandedArgs = OptionsFileUtil.expandArguments(flatArgs); + } catch (Exception ex) { + LOG.error("Error while expanding arguments", ex); + System.err.println(ex.getMessage()); + System.err.println("Try 'sqoop help' for usage."); + return 1; + } + + String toolName = expandedArgs[0]; + Configuration pluginConf = SqoopTool.loadPlugins(conf); + SqoopTool tool = SqoopTool.getTool(toolName); + if (null == tool) { + System.err.println("No such sqoop tool: " + toolName + ". See 'sqoop help'."); + return 1; + } + + Sqoop sqoop = new Sqoop(tool, pluginConf); + return runSqoop(sqoop, Arrays.copyOfRange(expandedArgs, 1, expandedArgs.length)); + } + + private static String[] convertParamsMapToAarray( + Map paramsMap, Configuration conf) throws Exception { + List paramsList = new ArrayList<>(); + + for (Map.Entry entry : paramsMap.entrySet()) { + if (StringUtils.isNotBlank(entry.getKey())) { + String key = entry.getKey().toLowerCase(); + if (key.equals(SqoopParamsConfiguration.SQOOP_PARAM_MODE().getValue())) { + paramsList.add(0, entry.getValue()); + continue; + } + if (key.startsWith(SqoopParamsConfiguration.SQOOP_PARAM_ENV_PREFIX().getValue())) { + key = + key.substring( + SqoopParamsConfiguration.SQOOP_PARAM_ENV_PREFIX() + .getValue() + .length()); + conf.set(key, entry.getValue()); + continue; + } + String conKey = ParamsMapping.mapping.get(key); + if (conKey != null) { + if (entry.getValue() != null && entry.getValue().length() != 0) { + paramsList.add(conKey); + paramsList.add(entry.getValue()); + } else { + paramsList.add(conKey); + } + } else { + // Ignore the unrecognized params + LOG.warn("The Key " + entry.getKey() + " Is Not Supported"); + } + } + } + return paramsList.toArray(new String[0]); + } + + /** + * Entry-point that parses the correct SqoopTool to use from the args, but does not call + * System.exit() as main() will. + */ + public static int runTool(Map params) { + Configuration conf = new Configuration(); + try { + for (String fileName : + SqoopEnvConfiguration.SQOOP_HADOOP_SITE_FILE().getValue().split(";")) { + File resourceFile = Paths.get(fileName).toFile(); + if (resourceFile.exists()) { + LOG.info("Append resource: [" + resourceFile.getPath() + "] to configuration"); + conf.addResource(resourceFile.toURI().toURL()); + } + } + + } catch (MalformedURLException e) { + e.printStackTrace(); + System.exit(1); + } + return runTool(params, conf); + } + + public static int main(Map code) { + return runTool(code); + } + + /** + * Close method + * + * @throws JobClosableException + */ + public static void close() throws JobClosableException { + Job runnableJob = job.get(); + try { + if (Objects.nonNull(runnableJob)) { + runnableJob.killJob(); + } + if (sqlManager != null && sqlManager.getConnection() != null) { + sqlManager.getConnection().close(); + } + } catch (IllegalStateException se) { + if (isJobReady(runnableJob)) { + LOG.warn( + "Unable to close the mapReduce job, it seems that the job isn't connected to the cluster"); + } else if (Objects.nonNull(runnableJob)) { + String cluster = "UNKNOWN"; + try { + cluster = runnableJob.getCluster().getFileSystem().getCanonicalServiceName(); + } catch (Exception e) { + // Ignore + } + throw new JobClosableException( + "Unable to close the mapReduce job related to cluster [" + cluster + "]", + se); + } + } catch (IOException | SQLException e) { + throw new JobClosableException("Error in closing sqoop client", e); + } + } + + /** + * Get application id + * + * @return string value + */ + public static String getApplicationId() { + String applicationId = ""; + try { + Job runnableJob = job.get(); + if (Objects.nonNull(runnableJob)) { + JobID jobId = runnableJob.getJobID(); + if (Objects.nonNull(jobId)) { + applicationId = jobId.toString(); + } + } + } catch (Exception e) { + // Not throw exception + LOG.error("GetApplicationId in sqoop Error", e); + } + return applicationId; + } + + /** + * Get application url + * + * @return url + */ + public static String getApplicationURL() { + String applicationUrl = ""; + Job runnableJob = job.get(); + try { + if (Objects.nonNull(runnableJob)) { + return runnableJob.getTrackingURL(); + } + } catch (Exception e) { + if (e instanceof IllegalStateException && !isJobReady(runnableJob)) { + LOG.trace("The mapReduce job is not ready, wait for the job status to be Running"); + } else { + LOG.error("GetApplicationURL in sqoop Error", e); + } + } + return applicationUrl; + } + + /** + * Get progress value + * + * @return float value + */ + public static Float progress() { + Job runnableJob = job.get(); + try { + if (Objects.nonNull(runnableJob)) { + // Count by two paragraphs + progress = (runnableJob.mapProgress() + runnableJob.reduceProgress()) / 2.0f; + } + } catch (Exception e) { + if (e instanceof IllegalStateException && !isJobReady(runnableJob)) { + LOG.trace("The mapReduce job is not ready, the value of progress is 0.0 always"); + } else { + LOG.error("Get progress in sqoop Error", e); + } + } + return progress; + } + + /** + * Get progress info + * + * @return info + */ + public static JobProgressInfo getProgressInfo() { + Job runnableJob = job.get(); + try { + if (Objects.nonNull(runnableJob)) { + AtomicInteger totalTasks = new AtomicInteger(); + AtomicInteger failedTasks = new AtomicInteger(); + AtomicInteger runTasks = new AtomicInteger(); + AtomicInteger successTasks = new AtomicInteger(); + TaskType[] analyzeTypes = new TaskType[] {TaskType.MAP, TaskType.REDUCE}; + for (TaskType taskType : analyzeTypes) { + TaskReport[] taskReports = runnableJob.getTaskReports(taskType); + Optional.ofNullable(taskReports) + .ifPresent( + reports -> { + totalTasks.addAndGet(reports.length); + for (TaskReport report : reports) { + TIPStatus tipStatus = report.getCurrentStatus(); + switch (tipStatus) { + case FAILED: + case KILLED: + failedTasks.getAndIncrement(); + break; + case PENDING: + case RUNNING: + runTasks.getAndIncrement(); + break; + case COMPLETE: + successTasks.getAndIncrement(); + break; + default: + } + } + }); + } + return new JobProgressInfo( + getApplicationId(), + totalTasks.get(), + runTasks.get(), + failedTasks.get(), + successTasks.get()); + } + } catch (Exception e) { + if (e instanceof IllegalStateException && !isJobReady(runnableJob)) { + LOG.trace( + "The mapReduce job is not ready, the value of progressInfo is always empty"); + } else { + LOG.error("Get progress info in sqoop Error", e); + } + } + return new JobProgressInfo(getApplicationId(), 0, 0, 0, 0); + } + + /** + * Get metrics + * + * @return metrics map + */ + public static Map getMetrics() { + Job runnableJob = job.get(); + // Actual the counter map + Map metricsMap = new HashMap<>(); + try { + if (Objects.nonNull(runnableJob)) { + Counters counters = runnableJob.getCounters(); + counters.forEach( + group -> + metricsMap.computeIfAbsent( + group.getName(), + (groupName) -> { + Map map = new HashMap<>(); + group.forEach( + counter -> + map.put( + counter.getName(), + counter.getValue())); + return map; + })); + long startTime = runnableJob.getStartTime(); + long endTime = + runnableJob.getFinishTime() > 0 + ? runnableJob.getFinishTime() + : System.currentTimeMillis(); + // Analyze the run time + metricsMap.put(METRICS_RUN_TIME, startTime > 0 ? endTime - startTime : 0); + } + } catch (Exception e) { + if (e instanceof IllegalStateException && !isJobReady(runnableJob)) { + LOG.trace( + "The mapReduce job is not ready, the value of metricsMap is always empty"); + } else { + LOG.error("Get metrics info in sqoop Error", e); + } + } + return metricsMap; + } + + /** + * Get diagnosis + * + * @return + */ + public static Map getDiagnosis() { + Job runnableJob = job.get(); + Map diagnosis = new HashMap<>(); + try { + if (Objects.nonNull(runnableJob)) { + TaskType[] analyzeTypes = new TaskType[] {TaskType.MAP, TaskType.REDUCE}; + List listReports = new ArrayList<>(); + for (TaskType taskType : analyzeTypes) { + listReports.addAll(Arrays.asList(runnableJob.getTaskReports(taskType))); + } + listReports.forEach( + report -> diagnosis.put(report.getTaskId(), report.getDiagnostics())); + } + } catch (Exception e) { + if (e instanceof IllegalStateException && !isJobReady(runnableJob)) { + LOG.trace("The mapReduce job is not ready, the value of diagnosis is always empty"); + } else { + LOG.error("Get diagnosis info in sqoop Error", e); + } + } + return diagnosis; + } + + /** + * If the job is ready + * + * @param runnableJob job + * @return + */ + private static boolean isJobReady(Job runnableJob) { + boolean ready = false; + try { + Field stateField = Job.class.getDeclaredField("state"); + stateField.setAccessible(true); + Job.JobState state = (Job.JobState) stateField.get(runnableJob); + ready = state.equals(Job.JobState.RUNNING); + } catch (NoSuchFieldException | IllegalAccessException e) { + // Ignore + } + return ready; + } +} diff --git a/exchangis-plugins/engine/sqoop/src/main/java/org/apache/linkis/engineconnplugin/sqoop/client/config/ExecutionContext.java b/exchangis-plugins/engine/sqoop/src/main/java/org/apache/linkis/engineconnplugin/sqoop/client/config/ExecutionContext.java new file mode 100644 index 000000000..4dde08a76 --- /dev/null +++ b/exchangis-plugins/engine/sqoop/src/main/java/org/apache/linkis/engineconnplugin/sqoop/client/config/ExecutionContext.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconnplugin.sqoop.client.config; + +import org.apache.linkis.engineconn.common.creation.EngineCreationContext; + +public class ExecutionContext { + private final EngineCreationContext environmentContext; + + public ExecutionContext(EngineCreationContext environmentContext) { + this.environmentContext = environmentContext; + } +} diff --git a/exchangis-plugins/engine/sqoop/src/main/java/org/apache/linkis/engineconnplugin/sqoop/client/config/ParamsMapping.java b/exchangis-plugins/engine/sqoop/src/main/java/org/apache/linkis/engineconnplugin/sqoop/client/config/ParamsMapping.java new file mode 100644 index 000000000..d52d68176 --- /dev/null +++ b/exchangis-plugins/engine/sqoop/src/main/java/org/apache/linkis/engineconnplugin/sqoop/client/config/ParamsMapping.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconnplugin.sqoop.client.config; + +import org.apache.linkis.engineconnplugin.sqoop.context.SqoopParamsConfiguration; + +import java.util.HashMap; +import java.util.Map; + +/** Params mapping */ +public final class ParamsMapping { + public static Map mapping; + + static { + String paramPrefix = SqoopParamsConfiguration.SQOOP_PARAM_PREFIX().getValue(); + mapping = new HashMap<>(); + mapping.put(paramPrefix + "connect", "--connect"); + mapping.put(paramPrefix + "connection.manager", "--connection-manager"); + mapping.put(paramPrefix + "connection.param.file", "--connection-param-file"); + mapping.put(paramPrefix + "driver", "--driver"); + mapping.put(paramPrefix + "hadoop.home", "--hadoop-home"); + mapping.put(paramPrefix + "hadoop.mapred.home", "--hadoop-mapred-home"); + mapping.put(paramPrefix + "help", "help"); + mapping.put(paramPrefix + "password", "--password"); + mapping.put(paramPrefix + "password.alias", "--password-alias"); + mapping.put(paramPrefix + "password.file", "--password-file"); + mapping.put(paramPrefix + "relaxed.isolation", "--relaxed-isolation"); + mapping.put(paramPrefix + "skip.dist.cache", "--skip-dist-cache"); + mapping.put(paramPrefix + "username", "--username"); + mapping.put(paramPrefix + "verbose", "--verbose"); + mapping.put(paramPrefix + "append", "--append"); + mapping.put(paramPrefix + "as.avrodatafile", "--as-avrodatafile"); + mapping.put(paramPrefix + "as.parquetfile", "--as-parquetfile"); + mapping.put(paramPrefix + "as.sequencefile", "--as-sequencefile"); + mapping.put(paramPrefix + "as.textfile", "--as-textfile"); + mapping.put(paramPrefix + "autoreset.to.one.mapper", "--autoreset-to-one-mapper"); + mapping.put(paramPrefix + "boundary.query", "--boundary-query"); + mapping.put(paramPrefix + "case.insensitive", "--case-insensitive"); + mapping.put(paramPrefix + "columns", "--columns"); + mapping.put(paramPrefix + "compression.codec", "--compression-codec"); + mapping.put(paramPrefix + "delete.target.dir", "--delete-target-dir"); + mapping.put(paramPrefix + "direct", "--direct"); + mapping.put(paramPrefix + "direct.split.size", "--direct-split-size"); + mapping.put(paramPrefix + "query", "--query"); + mapping.put(paramPrefix + "fetch.size", "--fetch-size"); + mapping.put(paramPrefix + "inline.lob.limit", "--inline-lob-limit"); + mapping.put(paramPrefix + "num.mappers", "--num-mappers"); + mapping.put(paramPrefix + "mapreduce.job.name", "--mapreduce-job-name"); + mapping.put(paramPrefix + "merge.key", "--merge-key"); + mapping.put(paramPrefix + "split.by", "--split-by"); + mapping.put(paramPrefix + "table", "--table"); + mapping.put(paramPrefix + "target.dir", "--target-dir"); + mapping.put(paramPrefix + "validate", "--validate"); + mapping.put(paramPrefix + "validation.failurehandler", "--validation-failurehandler"); + mapping.put(paramPrefix + "validation.threshold", " --validation-threshold"); + mapping.put(paramPrefix + "validator", "--validator"); + mapping.put(paramPrefix + "warehouse.dir", "--warehouse-dir"); + mapping.put(paramPrefix + "where", "--where"); + mapping.put(paramPrefix + "compress", "--compress"); + mapping.put(paramPrefix + "check.column", "--check-column"); + mapping.put(paramPrefix + "incremental", "--incremental"); + mapping.put(paramPrefix + "last.value", "--last-value"); + mapping.put(paramPrefix + "enclosed.by", "--enclosed-by"); + mapping.put(paramPrefix + "escaped.by", "--escaped-by"); + mapping.put(paramPrefix + "fields.terminated.by", "--fields-terminated-by"); + mapping.put(paramPrefix + "lines.terminated.by", "--lines-terminated-by"); + mapping.put(paramPrefix + "mysql.delimiters", "--mysql-delimiters"); + mapping.put(paramPrefix + "optionally.enclosed.by", "--optionally-enclosed-by"); + mapping.put(paramPrefix + "input.enclosed.by", "--input-enclosed-by"); + mapping.put(paramPrefix + "input.escaped.by", "--input-escaped-by"); + mapping.put(paramPrefix + "input.fields.terminated.by", "--input-fields-terminated-by"); + mapping.put(paramPrefix + "input.lines.terminated.by", "--input-lines-terminated-by"); + mapping.put(paramPrefix + "input.optionally.enclosed.by", "--input-optionally-enclosed-by"); + mapping.put(paramPrefix + "create.hive.table", "--create-hive-table"); + mapping.put(paramPrefix + "hive.delims.replacement", "--hive-delims-replacement"); + mapping.put(paramPrefix + "hive.database", "--hive-database"); + mapping.put(paramPrefix + "hive.drop.import.delims", "--hive-drop-import-delims"); + mapping.put(paramPrefix + "hive.home", "--hive-home"); + mapping.put(paramPrefix + "hive.import", "--hive-import"); + mapping.put(paramPrefix + "hive.overwrite", "--hive-overwrite"); + mapping.put(paramPrefix + "hive.partition.value", "--hive-partition-value"); + mapping.put(paramPrefix + "hive.table", "--hive-table"); + mapping.put(paramPrefix + "column.family", "--column-family"); + mapping.put(paramPrefix + "hbase.bulkload", "--hbase-bulkload"); + mapping.put(paramPrefix + "hbase.create.table", "--hbase-create-table"); + mapping.put(paramPrefix + "hbase.row.key", "--hbase-row-key"); + mapping.put(paramPrefix + "hbase.table", "--hbase-table"); + mapping.put(paramPrefix + "hcatalog.database", "--hcatalog-database"); + mapping.put(paramPrefix + "hcatalog.home", "--hcatalog-home"); + mapping.put(paramPrefix + "hcatalog.partition.keys", "--hcatalog-partition-keys"); + mapping.put(paramPrefix + "hcatalog.partition.values", "--hcatalog-partition-values"); + mapping.put(paramPrefix + "hcatalog.table", "--hcatalog-table"); + mapping.put(paramPrefix + "hive.partition.key", "--hive-partition-key"); + mapping.put(paramPrefix + "map.column.hive", "--map-column-hive"); + mapping.put(paramPrefix + "create.hcatalog.table", "--create-hcatalog-table"); + mapping.put(paramPrefix + "hcatalog.storage.stanza", "--hcatalog-storage-stanza"); + mapping.put(paramPrefix + "accumulo.batch.size", "--accumulo-batch-size"); + mapping.put(paramPrefix + "accumulo.column.family", "--accumulo-column-family"); + mapping.put(paramPrefix + "accumulo.create.table", "--accumulo-create-table"); + mapping.put(paramPrefix + "accumulo.instance", "--accumulo-instance"); + mapping.put(paramPrefix + "accumulo.max.latency", "--accumulo-max-latency"); + mapping.put(paramPrefix + "accumulo.password", "--accumulo-password"); + mapping.put(paramPrefix + "accumulo.row.key", "--accumulo-row-key"); + mapping.put(paramPrefix + "accumulo.table", "--accumulo-table"); + mapping.put(paramPrefix + "accumulo.user", "--accumulo-user"); + mapping.put(paramPrefix + "accumulo.visibility", "--accumulo-visibility"); + mapping.put(paramPrefix + "accumulo.zookeepers", "--accumulo-zookeepers"); + mapping.put(paramPrefix + "bindir", "--bindir"); + mapping.put(paramPrefix + "class.name", "--class-name"); + mapping.put(paramPrefix + "input.null.non.string", "--input-null-non-string"); + mapping.put(paramPrefix + "input.null.string", "--input-null-string"); + mapping.put(paramPrefix + "jar.file", "--jar-file"); + mapping.put(paramPrefix + "map.column.java", "--map-column-java"); + mapping.put(paramPrefix + "null.non.string", "--null-non-string"); + mapping.put(paramPrefix + "null.string", "--null-string"); + mapping.put(paramPrefix + "outdir", "--outdir"); + mapping.put(paramPrefix + "package.name", "--package-name"); + mapping.put(paramPrefix + "conf", "-conf"); + mapping.put(paramPrefix + "D", "-D"); + mapping.put(paramPrefix + "fs", "-fs"); + mapping.put(paramPrefix + "jt", "-jt"); + mapping.put(paramPrefix + "files", "-files"); + mapping.put(paramPrefix + "libjars", "-libjars"); + mapping.put(paramPrefix + "archives", "-archives"); + mapping.put(paramPrefix + "update.key", "--update-key"); + mapping.put(paramPrefix + "update.mode", "--update-mode"); + mapping.put(paramPrefix + "export.dir", "--export-dir"); + } +} diff --git a/exchangis-plugins/engine/sqoop/src/main/java/org/apache/linkis/engineconnplugin/sqoop/client/exception/JobClosableException.java b/exchangis-plugins/engine/sqoop/src/main/java/org/apache/linkis/engineconnplugin/sqoop/client/exception/JobClosableException.java new file mode 100644 index 000000000..efbcd0e6b --- /dev/null +++ b/exchangis-plugins/engine/sqoop/src/main/java/org/apache/linkis/engineconnplugin/sqoop/client/exception/JobClosableException.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconnplugin.sqoop.client.exception; + +import org.apache.linkis.common.exception.ErrorException; + +/** Exception in closing/destroying the job */ +public class JobClosableException extends ErrorException { + private static final long serialVersionUID = 1L; + + public static final int ERROR_CODE = 16025; + + public JobClosableException(String message) { + super(ERROR_CODE, message); + } + + public JobClosableException(String message, Throwable e) { + super(ERROR_CODE, message); + this.initCause(e); + } +} diff --git a/exchangis-plugins/engine/sqoop/src/main/java/org/apache/linkis/engineconnplugin/sqoop/client/exception/JobExecutionException.java b/exchangis-plugins/engine/sqoop/src/main/java/org/apache/linkis/engineconnplugin/sqoop/client/exception/JobExecutionException.java new file mode 100644 index 000000000..168d54ff3 --- /dev/null +++ b/exchangis-plugins/engine/sqoop/src/main/java/org/apache/linkis/engineconnplugin/sqoop/client/exception/JobExecutionException.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconnplugin.sqoop.client.exception; + +import org.apache.linkis.common.exception.ErrorException; + +public class JobExecutionException extends ErrorException { + private static final long serialVersionUID = 1L; + + public static final int ERROR_CODE = 16023; + + public JobExecutionException(String message) { + super(ERROR_CODE, message); + } + + public JobExecutionException(String message, Throwable e) { + super(ERROR_CODE, message); + this.initCause(e); + } +} diff --git a/exchangis-plugins/engine/sqoop/src/main/java/org/apache/linkis/engineconnplugin/sqoop/client/utils/JarLoader.java b/exchangis-plugins/engine/sqoop/src/main/java/org/apache/linkis/engineconnplugin/sqoop/client/utils/JarLoader.java new file mode 100644 index 000000000..6140e68e8 --- /dev/null +++ b/exchangis-plugins/engine/sqoop/src/main/java/org/apache/linkis/engineconnplugin/sqoop/client/utils/JarLoader.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconnplugin.sqoop.client.utils; + +import org.apache.commons.lang3.Validate; + +import java.io.File; +import java.io.FileFilter; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.security.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +public class JarLoader extends URLClassLoader { + private AccessControlContext acc; + + public JarLoader(String[] paths) { + this(paths, false); + } + + public JarLoader(String[] paths, boolean recursive) { + this(paths, recursive, JarLoader.class.getClassLoader()); + } + + public JarLoader(String[] paths, boolean recursive, ClassLoader parent) { + super(getURLs(paths, recursive), parent); + } + + private static URL[] getURLs(String[] paths, boolean recursive) { + List urls = new ArrayList<>(); + if (recursive) { + List dirs = new ArrayList<>(); + for (String path : paths) { + dirs.add(path); + collectDirs(path, dirs); + } + for (String path : dirs) { + urls.addAll(doGetURLs(path)); + } + } else { + // For classpath, classloader will recursive automatically + urls.addAll( + Arrays.stream(paths) + .map(File::new) + .filter(File::exists) + .map( + f -> { + try { + return f.toURI().toURL(); + } catch (MalformedURLException e) { + // Ignore + return null; + } + }) + .collect(Collectors.toList())); + } + return urls.toArray(new URL[0]); + } + + public void addJarURL(String path) { + // Single jar + File singleJar = new File(path); + if (singleJar.exists() && singleJar.isFile()) { + try { + this.addURL(singleJar.toURI().toURL()); + } catch (MalformedURLException e) { + // Ignore + } + } + } + + private static void collectDirs(String path, List collector) { + + File current = new File(path); + if (!current.exists() || !current.isDirectory()) { + return; + } + + if (null != current.listFiles()) { + for (File child : Objects.requireNonNull(current.listFiles())) { + if (!child.isDirectory()) { + continue; + } + + collector.add(child.getAbsolutePath()); + collectDirs(child.getAbsolutePath(), collector); + } + } + } + + private static List doGetURLs(final String path) { + + File jarPath = new File(path); + + Validate.isTrue(jarPath.exists() && jarPath.isDirectory(), "jar包路径必须存在且为目录."); + + /* set filter */ + FileFilter jarFilter = pathname -> pathname.getName().endsWith(".jar"); + + /* iterate all jar */ + File[] allJars = new File(path).listFiles(jarFilter); + assert allJars != null; + List jarURLs = new ArrayList<>(allJars.length); + + for (File allJar : allJars) { + try { + jarURLs.add(allJar.toURI().toURL()); + } catch (Exception e) { + // Ignore + } + } + + return jarURLs; + } + + /** + * change the order to load class + * + * @param name name + * @param resolve isResolve + * @return + * @throws ClassNotFoundException + */ + @Override + public Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + synchronized (getClassLoadingLock(name)) { + // First, check if the class has already been loaded + Class c = findLoadedClass(name); + if (c == null) { + long t0 = System.nanoTime(); + try { + // invoke findClass in this class + c = findClass(name); + } catch (ClassNotFoundException e) { + // ClassNotFoundException thrown if class not found + } + if (c == null) { + return super.loadClass(name, resolve); + } + // For compatibility with higher versions > java 1.8.0_141 + // sun.misc.PerfCounter.getFindClasses().addElapsedTimeFrom(t0); + // sun.misc.PerfCounter.getFindClasses().increment(); + } + if (resolve) { + resolveClass(c); + } + return c; + } + } +} diff --git a/exchangis-plugins/engine/sqoop/src/main/java/org/apache/sqoop/mapreduce/JobBase.java b/exchangis-plugins/engine/sqoop/src/main/java/org/apache/sqoop/mapreduce/JobBase.java new file mode 100644 index 000000000..6703c303c --- /dev/null +++ b/exchangis-plugins/engine/sqoop/src/main/java/org/apache/sqoop/mapreduce/JobBase.java @@ -0,0 +1,410 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.sqoop.mapreduce; + +import org.apache.linkis.engineconnplugin.sqoop.client.Sqoop; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.mapreduce.InputFormat; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.OutputFormat; +import org.apache.hadoop.util.StringUtils; +import org.apache.sqoop.config.ConfigurationConstants; + +import com.cloudera.sqoop.SqoopOptions; +import com.cloudera.sqoop.config.ConfigurationHelper; +import com.cloudera.sqoop.manager.ConnManager; +import com.cloudera.sqoop.tool.SqoopTool; +import com.cloudera.sqoop.util.ClassLoaderStack; +import com.cloudera.sqoop.util.Jars; + +import java.io.File; +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Base class for configuring and running a MapReduce job. Allows dependency injection, etc, for + * easy customization of import job types. + */ +public class JobBase { + + public static final Log LOG = LogFactory.getLog(JobBase.class.getName()); + + public static final String SERIALIZE_SQOOPOPTIONS = "sqoop.jobbase.serialize.sqoopoptions"; + public static final boolean SERIALIZE_SQOOPOPTIONS_DEFAULT = false; + public static final String HADOOP_MAP_TASK_MAX_ATTEMTPS = "mapreduce.map.maxattempts"; + public static final String HADOOP_REDUCE_TASK_MAX_ATTEMTPS = "mapreduce.reduce.maxattempts"; + + protected SqoopOptions options; + protected Class mapperClass; + protected Class inputFormatClass; + protected Class outputFormatClass; + + private Job mrJob; + + private ClassLoader prevClassLoader = null; + protected final boolean isHCatJob; + + public static final String PROPERTY_VERBOSE = "sqoop.verbose"; + + public JobBase() { + this(null); + } + + public JobBase(final SqoopOptions opts) { + this(opts, null, null, null); + } + + public JobBase( + final SqoopOptions opts, + final Class mapperClass, + final Class inputFormatClass, + final Class outputFormatClass) { + System.out.println(SqoopOptions.class.getClassLoader()); + this.options = opts; + this.mapperClass = mapperClass; + this.inputFormatClass = inputFormatClass; + this.outputFormatClass = outputFormatClass; + isHCatJob = options.getHCatTableName() != null; + } + + /** @return the mapper class to use for the job. */ + protected Class getMapperClass() throws ClassNotFoundException { + return this.mapperClass; + } + + /** @return the inputformat class to use for the job. */ + protected Class getInputFormatClass() throws ClassNotFoundException { + return this.inputFormatClass; + } + + /** @return the outputformat class to use for the job. */ + protected Class getOutputFormatClass() throws ClassNotFoundException { + return this.outputFormatClass; + } + + /** Set the OutputFormat class to use for this job. */ + public void setOutputFormatClass(Class cls) { + this.outputFormatClass = cls; + } + + /** Set the InputFormat class to use for this job. */ + public void setInputFormatClass(Class cls) { + this.inputFormatClass = cls; + } + + /** Set the Mapper class to use for this job. */ + public void setMapperClass(Class cls) { + this.mapperClass = cls; + } + + /** Set the SqoopOptions configuring this job. */ + public void setOptions(SqoopOptions opts) { + this.options = opts; + } + + /** + * Put jar files required by Sqoop into the DistributedCache. + * + * @param job the Job being submitted. + * @param mgr the ConnManager to use. + */ + protected void cacheJars(Job job, ConnManager mgr) throws IOException { + if (options.isSkipDistCache()) { + LOG.info("Not adding sqoop jars to distributed cache as requested"); + return; + } + + Configuration conf = job.getConfiguration(); + FileSystem fs = FileSystem.getLocal(conf); + Set localUrls = new HashSet(); + + addToCache(Jars.getSqoopJarPath(), fs, localUrls); + if (null != mgr) { + addToCache(Jars.getDriverClassJar(mgr), fs, localUrls); + addToCache(Jars.getJarPathForClass(mgr.getClass()), fs, localUrls); + } + + SqoopTool tool = this.options.getActiveSqoopTool(); + if (null != tool) { + // Make sure the jar for the tool itself is on the classpath. (In case + // this is a third-party plugin tool.) + addToCache(Jars.getJarPathForClass(tool.getClass()), fs, localUrls); + List toolDeps = tool.getDependencyJars(); + if (null != toolDeps) { + for (String depFile : toolDeps) { + addToCache(depFile, fs, localUrls); + } + } + } + + // If the user specified a particular jar file name, + + // Add anything in $SQOOP_HOME/lib, if this is set. + String sqoopHome = System.getenv("SQOOP_HOME"); + if (null != sqoopHome) { + File sqoopHomeFile = new File(sqoopHome); + File sqoopLibFile = new File(sqoopHomeFile, "lib"); + if (sqoopLibFile.exists()) { + addDirToCache(sqoopLibFile, fs, localUrls); + } + } else { + LOG.warn("SQOOP_HOME is unset. May not be able to find " + "all job dependencies."); + } + + // If the user run import into hive as Parquet file, + // Add anything in $HIVE_HOME/lib. + if (options.doHiveImport() + && (options.getFileLayout() == SqoopOptions.FileLayout.ParquetFile)) { + String hiveHome = options.getHiveHome(); + if (null != hiveHome) { + File hiveHomeFile = new File(hiveHome); + File hiveLibFile = new File(hiveHomeFile, "lib"); + if (hiveLibFile.exists()) { + addDirToCache(hiveLibFile, fs, localUrls); + } + } else { + LOG.warn("HIVE_HOME is unset. Cannot add hive libs as dependencies."); + } + } + + String tmpjars = conf.get(ConfigurationConstants.MAPRED_DISTCACHE_CONF_PARAM); + StringBuilder sb = new StringBuilder(); + + // If we didn't put anything in our set, then there's nothing to cache. + if (localUrls.isEmpty() && (org.apache.commons.lang.StringUtils.isEmpty(tmpjars))) { + return; + } + + if (null != tmpjars) { + String[] tmpjarsElements = tmpjars.split(","); + for (String jarElement : tmpjarsElements) { + if (jarElement.isEmpty()) { + warn("Empty input is invalid and was removed from tmpjars."); + } else { + sb.append(jarElement); + sb.append(","); + } + } + } + + int lastComma = sb.lastIndexOf(","); + if (localUrls.isEmpty() && lastComma >= 0) { + sb.deleteCharAt(lastComma); + } + + // Add these to the 'tmpjars' array, which the MR JobSubmitter + // will upload to HDFS and put in the DistributedCache libjars. + sb.append(StringUtils.arrayToString(localUrls.toArray(new String[0]))); + conf.set(ConfigurationConstants.MAPRED_DISTCACHE_CONF_PARAM, sb.toString()); + } + + protected void warn(String message) { + LOG.warn(message); + } + + private void addToCache(String file, FileSystem fs, Set localUrls) { + if (null == file) { + return; + } + + Path p = new Path(file); + String qualified = p.makeQualified(fs).toString(); + LOG.debug("Adding to job classpath: " + qualified); + localUrls.add(qualified); + } + + /** Add the .jar elements of a directory to the DCache classpath, nonrecursively. */ + private void addDirToCache(File dir, FileSystem fs, Set localUrls) { + if (null == dir) { + return; + } + + for (File libfile : dir.listFiles()) { + if (libfile.exists() && !libfile.isDirectory() && libfile.getName().endsWith("jar")) { + addToCache(libfile.toString(), fs, localUrls); + } + } + } + + /** If jars must be loaded into the local environment, do so here. */ + protected void loadJars(Configuration conf, String ormJarFile, String tableClassName) + throws IOException { + + boolean isLocal = + "local".equals(conf.get("mapreduce.jobtracker.address")) + || "local".equals(conf.get("mapred.job.tracker")); + if (isLocal) { + // If we're using the LocalJobRunner, then instead of using the compiled + // jar file as the job source, we're running in the current thread. Push + // on another classloader that loads from that jar in addition to + // everything currently on the classpath. + this.prevClassLoader = ClassLoaderStack.addJarFile(ormJarFile, tableClassName); + } + } + + /** If any classloader was invoked by loadJars, free it here. */ + protected void unloadJars() { + if (null != this.prevClassLoader) { + // unload the special classloader for this jar. + ClassLoaderStack.setCurrentClassLoader(this.prevClassLoader); + } + } + + /** Configure the inputformat to use for the job. */ + protected void configureInputFormat( + Job job, String tableName, String tableClassName, String splitByCol) + throws ClassNotFoundException, IOException { + // TODO: 'splitByCol' is import-job specific; lift it out of this API. + Class ifClass = getInputFormatClass(); + LOG.debug("Using InputFormat: " + ifClass); + job.setInputFormatClass(ifClass); + } + + /** Configure the output format to use for the job. */ + protected void configureOutputFormat(Job job, String tableName, String tableClassName) + throws ClassNotFoundException, IOException { + Class ofClass = getOutputFormatClass(); + LOG.debug("Using OutputFormat: " + ofClass); + job.setOutputFormatClass(ofClass); + } + + /** + * Set the mapper class implementation to use in the job, as well as any related configuration + * (e.g., map output types). + */ + protected void configureMapper(Job job, String tableName, String tableClassName) + throws ClassNotFoundException, IOException { + job.setMapperClass(getMapperClass()); + } + + /** + * Configure the number of map/reduce tasks to use in the job, returning the number of map tasks + * for backward compatibility. + */ + protected int configureNumTasks(Job job) throws IOException { + int numMapTasks = configureNumMapTasks(job); + configureNumReduceTasks(job); + return numMapTasks; + } + + /** Configure the number of map tasks to use in the job. */ + protected int configureNumMapTasks(Job job) throws IOException { + int numMapTasks = options.getNumMappers(); + if (numMapTasks < 1) { + numMapTasks = SqoopOptions.DEFAULT_NUM_MAPPERS; + LOG.warn("Invalid mapper count; using " + numMapTasks + " mappers."); + } + ConfigurationHelper.setJobNumMaps(job, numMapTasks); + return numMapTasks; + } + + /** Configure the number of reduce tasks to use in the job. */ + protected int configureNumReduceTasks(Job job) throws IOException { + job.setNumReduceTasks(0); + return 0; + } + + /** Set the main job that will be run. */ + protected void setJob(Job job) { + LOG.info("Customize JobBase Set The Job"); + mrJob = job; + Sqoop.job.set(job); + } + + /** @return the main MapReduce job that is being run, or null if no job has started. */ + public Job getJob() { + return mrJob; + } + + /** + * Create new Job object in unified way for all types of jobs. + * + * @param configuration Hadoop configuration that should be used + * @return New job object, created object won't be persisted in the instance + */ + public Job createJob(Configuration configuration) throws IOException { + // Put the SqoopOptions into job if requested + if (configuration.getBoolean(SERIALIZE_SQOOPOPTIONS, SERIALIZE_SQOOPOPTIONS_DEFAULT)) { + putSqoopOptionsToConfiguration(options, configuration); + } + + return new Job(configuration); + } + + /** + * Iterates over serialized form of SqoopOptions and put them into Configuration object. + * + * @param opts SqoopOptions that should be serialized + * @param configuration Target configuration object + */ + public void putSqoopOptionsToConfiguration(SqoopOptions opts, Configuration configuration) { + for (Map.Entry e : opts.writeProperties().entrySet()) { + String key = (String) e.getKey(); + String value = (String) e.getValue(); + + // We don't need to do if(value is empty) because that is already done + // for us by the SqoopOptions.writeProperties() method. + configuration.set("sqoop.opt." + key, value); + } + } + + /** Actually run the MapReduce job. */ + protected boolean runJob(Job job) + throws ClassNotFoundException, IOException, InterruptedException { + return job.waitForCompletion(true); + } + + /** + * Display a notice on the log that the current MapReduce job has been retired, and thus + * Counters are unavailable. + * + * @param log the Log to display the info to. + */ + protected void displayRetiredJobNotice(Log log) { + log.info("The MapReduce job has already been retired. Performance"); + log.info("counters are unavailable. To get this information, "); + log.info("you will need to enable the completed job store on "); + log.info("the jobtracker with:"); + log.info("mapreduce.jobtracker.persist.jobstatus.active = true"); + log.info("mapreduce.jobtracker.persist.jobstatus.hours = 1"); + log.info("A jobtracker restart is required for these settings"); + log.info("to take effect."); + } + + /** + * Save interesting options to constructed job. Goal here is to propagate some of them to the + * job itself, so that they can be easily accessed. We're propagating only interesting global + * options (like verbose flag). + * + * @param job Destination job to save options + */ + protected void propagateOptionsToJob(Job job) { + Configuration configuration = job.getConfiguration(); + + // So far, propagate only verbose flag + configuration.setBoolean(PROPERTY_VERBOSE, options.getVerbose()); + } +} diff --git a/exchangis-plugins/engine/sqoop/src/main/resources/linkis-engineconn.properties b/exchangis-plugins/engine/sqoop/src/main/resources/linkis-engineconn.properties new file mode 100644 index 000000000..99b76eaea --- /dev/null +++ b/exchangis-plugins/engine/sqoop/src/main/resources/linkis-engineconn.properties @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +wds.linkis.server.version=v1 + +wds.linkis.engineconn.plugin.default.class=org.apache.linkis.engineconnplugin.sqoop.SqoopEngineConnPlugin + +wds.linkis.engine.connector.hooks=org.apache.linkis.engineconn.computation.executor.hook.ComputationEngineConnHook +# wds.linkis.hadoop.site.xml= \ No newline at end of file diff --git a/exchangis-plugins/engine/sqoop/src/main/resources/log4j2.xml b/exchangis-plugins/engine/sqoop/src/main/resources/log4j2.xml new file mode 100644 index 000000000..3b45ae2a1 --- /dev/null +++ b/exchangis-plugins/engine/sqoop/src/main/resources/log4j2.xml @@ -0,0 +1,82 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/SqoopEngineConnPlugin.scala b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/SqoopEngineConnPlugin.scala new file mode 100644 index 000000000..52266bb71 --- /dev/null +++ b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/SqoopEngineConnPlugin.scala @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconnplugin.sqoop + +import org.apache.linkis.manager.engineplugin.common.EngineConnPlugin +import org.apache.linkis.manager.engineplugin.common.creation.EngineConnFactory +import org.apache.linkis.manager.engineplugin.common.launch.EngineConnLaunchBuilder +import org.apache.linkis.manager.engineplugin.common.resource.{EngineResourceFactory, GenericEngineResourceFactory} +import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.engineconnplugin.sqoop.factory.SqoopEngineConnFactory +import org.apache.linkis.engineconnplugin.sqoop.launch.SqoopEngineConnLaunchBuilder + + +class SqoopEngineConnPlugin extends EngineConnPlugin{ + private val EP_CONTEXT_CONSTRUCTOR_LOCK = new Object() + private var engineResourceFactory: EngineResourceFactory = _ + private var engineConnLaunchBuilder: EngineConnLaunchBuilder = _ + private var engineConnFactory: EngineConnFactory = _ + override def init(params: java.util.Map[String, Any]): Unit = {} + + override def getEngineResourceFactory: EngineResourceFactory = { + + EP_CONTEXT_CONSTRUCTOR_LOCK.synchronized{ + if(null == engineResourceFactory){ + engineResourceFactory = new GenericEngineResourceFactory + } + engineResourceFactory + } + } + + override def getEngineConnLaunchBuilder: EngineConnLaunchBuilder = { + EP_CONTEXT_CONSTRUCTOR_LOCK.synchronized { + if (null == engineConnLaunchBuilder) { + engineConnLaunchBuilder = new SqoopEngineConnLaunchBuilder() + } + engineConnLaunchBuilder + } + } + + + override def getEngineConnFactory: EngineConnFactory = { + EP_CONTEXT_CONSTRUCTOR_LOCK.synchronized { + if (null == engineConnFactory) { + engineConnFactory = new SqoopEngineConnFactory + } + engineConnFactory + } + } + + override def getDefaultLabels: java.util.List[Label[_]] = new java.util.ArrayList[Label[_]] +} diff --git a/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/context/SqoopEngineConnContext.scala b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/context/SqoopEngineConnContext.scala new file mode 100644 index 000000000..be1d340db --- /dev/null +++ b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/context/SqoopEngineConnContext.scala @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconnplugin.sqoop.context + +import org.apache.linkis.engineconnplugin.sqoop.client.config.ExecutionContext + +class SqoopEngineConnContext{ + private var executionContext: ExecutionContext = _ + + def getExecutionContext: ExecutionContext = executionContext + + def setExecutionContext(executionContext: ExecutionContext): Unit = this.executionContext = executionContext + +} diff --git a/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/context/SqoopEnvConfiguration.scala b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/context/SqoopEnvConfiguration.scala new file mode 100644 index 000000000..63417dbca --- /dev/null +++ b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/context/SqoopEnvConfiguration.scala @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconnplugin.sqoop.context + +import org.apache.linkis.common.conf.{CommonVars, TimeType} + +object SqoopEnvConfiguration { + + val SQOOP_HADOOP_SITE_FILE: CommonVars[String] = CommonVars("wds.linkis.hadoop.site.xml", "core-site.xml;hdfs-site.xml;yarn-site.xml;mapred-site.xml") + + val SQOOP_STATUS_FETCH_INTERVAL: CommonVars[TimeType] = CommonVars("sqoop.fetch.status.interval", new TimeType("5s")) + + val LINKIS_DATASOURCE_SERVICE_NAME: CommonVars[String] = CommonVars("wds.linkis.datasource.service.name", "linkis-ps-data-source-manager") + + val SQOOP_HOME: CommonVars[String] = CommonVars("SQOOP_HOME", "") + + val SQOOP_CONF_DIR: CommonVars[String] = CommonVars("SQOOP_CONF_DIR", "") + + val SQOOP_HCAT_HOME: CommonVars[String] = CommonVars("HCAT_HOME", "") + + val SQOOP_HBASE_HOME: CommonVars[String] = CommonVars("HBASE_HOME", "") + + val SQOOP_ZOOCFGDIR: CommonVars[String] = CommonVars("ZOOCFGDIR", "") +} diff --git a/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/context/SqoopParamsConfiguration.scala b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/context/SqoopParamsConfiguration.scala new file mode 100644 index 000000000..0c449a9d4 --- /dev/null +++ b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/context/SqoopParamsConfiguration.scala @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconnplugin.sqoop.context + +import org.apache.linkis.common.conf.CommonVars + +/** + * Sqoop Params Configuration + */ +object SqoopParamsConfiguration { + + val SQOOP_PARAM_MODE: CommonVars[String] = CommonVars("sqoop.params.name.mode", "sqoop.mode") + + val SQOOP_PARAM_HOST: CommonVars[String] = CommonVars("sqoop.params.name.host", "sqoop.args.host") + + val SQOOP_PARAM_PORT: CommonVars[String] = CommonVars("sqoop.params.name.ip", "sqoop.args.port") + + val SQOOP_PARAM_CONNECT_PARAMS: CommonVars[String] = CommonVars("sqoop.params.name.ip", "sqoop.args.params") + + val SQOOP_PARAM_CONNECT: CommonVars[String] = CommonVars("sqoop.params.name.connect", "sqoop.args.connect") + + val SQOOP_PARAM_DATA_SOURCE: CommonVars[String] = CommonVars("sqoop.params.name.data-source", "sqoop.args.datasource.name") + + val SQOOP_PARAM_PREFIX: CommonVars[String] = CommonVars("sqoop.params.name.prefix", "sqoop.args.") + + val SQOOP_PARAM_ENV_PREFIX: CommonVars[String] = CommonVars("sqoop.params.name.env.prefix", "sqoop.env.") +} diff --git a/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/context/SqoopResourceConfiguration.scala b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/context/SqoopResourceConfiguration.scala new file mode 100644 index 000000000..710d28cc2 --- /dev/null +++ b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/context/SqoopResourceConfiguration.scala @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconnplugin.sqoop.context + +import org.apache.linkis.common.conf.CommonVars + + +object SqoopResourceConfiguration { + + val LINKIS_SQOOP_TASK_MAP_MEMORY: CommonVars[Int] = CommonVars[Int]("sqoop.task.map.memory", 2) + + val LINKIS_SQOOP_TASK_MAP_CPU_CORES: CommonVars[Int] = CommonVars[Int]("sqoop.task.map.cpu.cores", 1) + + val LINKIS_QUEUE_NAME: CommonVars[String] = CommonVars[String]("wds.linkis.rm.yarnqueue", "default") +} diff --git a/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/executor/SqoopExecutor.scala b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/executor/SqoopExecutor.scala new file mode 100644 index 000000000..5c43366ba --- /dev/null +++ b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/executor/SqoopExecutor.scala @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconnplugin.sqoop.executor + +import org.apache.linkis.engineconn.executor.entity.{LabelExecutor, ResourceExecutor, YarnExecutor} +import org.apache.linkis.engineconnplugin.sqoop.client.Sqoop +import org.apache.linkis.engineconnplugin.sqoop.context.SqoopResourceConfiguration.LINKIS_QUEUE_NAME +import org.apache.linkis.manager.common.entity.resource.NodeResource +import org.apache.linkis.manager.label.entity.Label +import java.util + +import org.apache.linkis.engineconnplugin.sqoop.client.Sqoop +import org.apache.linkis.engineconnplugin.sqoop.client.exception.JobExecutionException +import org.apache.linkis.engineconnplugin.sqoop.context.SqoopEngineConnContext + +trait SqoopExecutor extends YarnExecutor with LabelExecutor with ResourceExecutor{ + private var yarnMode: String = "Client" + private var executorLabels: util.List[Label[_]] = new util.ArrayList[Label[_]] + override def getApplicationId: String = Sqoop.getApplicationId + + override def getApplicationURL: String = Sqoop.getApplicationURL + + override def getYarnMode: String = yarnMode + def setYarnMode(yarnMode: String): Unit = this.yarnMode = yarnMode + + override def getQueue: String = LINKIS_QUEUE_NAME.getValue + + override def getExecutorLabels(): util.List[Label[_]] = executorLabels + + override def setExecutorLabels(labels: util.List[Label[_]]): Unit = this.executorLabels = labels + + override def requestExpectedResource(expectedResource: NodeResource): NodeResource = throw new JobExecutionException("Not support method for requestExpectedResource.") + + protected val sqoopEngineConnContext: SqoopEngineConnContext +} diff --git a/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/executor/SqoopOnceCodeExecutor.scala b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/executor/SqoopOnceCodeExecutor.scala new file mode 100644 index 000000000..1d7cb6af3 --- /dev/null +++ b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/executor/SqoopOnceCodeExecutor.scala @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconnplugin.sqoop.executor + +import org.apache.linkis.common.utils.{JsonUtils, OverloadUtils, Utils} +import org.apache.linkis.engineconn.once.executor.{OnceExecutorExecutionContext, OperableOnceExecutor} +import org.apache.linkis.engineconnplugin.sqoop.client.{LinkisSqoopClient, Sqoop} +import org.apache.linkis.engineconnplugin.sqoop.context.SqoopResourceConfiguration.{LINKIS_QUEUE_NAME, LINKIS_SQOOP_TASK_MAP_CPU_CORES, LINKIS_SQOOP_TASK_MAP_MEMORY} +import org.apache.linkis.engineconnplugin.sqoop.context.{SqoopEngineConnContext, SqoopParamsConfiguration} +import org.apache.linkis.manager.common.entity.resource.{CommonNodeResource, DriverAndYarnResource, LoadInstanceResource, NodeResource, YarnResource} +import org.apache.linkis.scheduler.executer.ErrorExecuteResponse +import java.util +import java.util.concurrent.{Future, TimeUnit} + +import org.apache.linkis.engineconn.common.creation.EngineCreationContext +import org.apache.linkis.engineconn.core.EngineConnObject +import org.apache.linkis.manager.engineplugin.common.conf.EngineConnPluginConf +import org.apache.linkis.protocol.engine.JobProgressInfo +import org.apache.linkis.engineconnplugin.sqoop.client.LinkisSqoopClient +import org.apache.linkis.engineconnplugin.sqoop.client.exception.JobExecutionException +import org.apache.linkis.engineconnplugin.sqoop.context.{SqoopEngineConnContext, SqoopEnvConfiguration} +import org.apache.linkis.engineconnplugin.sqoop.params.SqoopParamsResolver + + +class SqoopOnceCodeExecutor(override val id: Long, + override protected val sqoopEngineConnContext: SqoopEngineConnContext) extends SqoopOnceExecutor with OperableOnceExecutor{ + + + private var params: util.Map[String, String] = _ + private var future: Future[_] = _ + private var daemonThread: Future[_] = _ + private val paramsResolvers: Array[SqoopParamsResolver] = Array() + + override def doSubmit(onceExecutorExecutionContext: OnceExecutorExecutionContext, options: Map[String, String]): Unit = { + var isFailed = false + future = Utils.defaultScheduler.submit(new Runnable { + override def run(): Unit = { + // TODO filter job content + params = onceExecutorExecutionContext.getOnceExecutorContent.getJobContent.asInstanceOf[util.Map[String, String]] + info("Try to execute params." + params) + if(runSqoop(params, onceExecutorExecutionContext.getEngineCreationContext) != 0) { + isFailed = true + tryFailed() + setResponse(ErrorExecuteResponse("Run code failed!", new JobExecutionException("Exec Sqoop Code Error"))) + } + info("All codes completed, now to stop SqoopEngineConn.") + closeDaemon() + if (!isFailed) { + trySucceed() + } + this synchronized notify() + } + }) + } + protected def runSqoop(params: util.Map[String, String], context: EngineCreationContext): Int = { + Utils.tryCatch { + val finalParams = paramsResolvers.foldLeft(params) { + case (newParam, resolver) => resolver.resolve(newParam, context) + } + LinkisSqoopClient.run(finalParams) + }{ + case e: Exception => + error(s"Run Error Message: ${e.getMessage}", e) + -1 + } + + } + + override protected def waitToRunning(): Unit = { + if (!isCompleted) daemonThread = Utils.defaultScheduler.scheduleAtFixedRate(new Runnable { + override def run(): Unit = { + if (!(future.isDone || future.isCancelled)) { + info("The Sqoop Process In Running") + } + } + }, SqoopEnvConfiguration.SQOOP_STATUS_FETCH_INTERVAL.getValue.toLong, + SqoopEnvConfiguration.SQOOP_STATUS_FETCH_INTERVAL.getValue.toLong, TimeUnit.MILLISECONDS) + } + override def getCurrentNodeResource(): NodeResource = { + val memorySuffix = "g" + val properties = EngineConnObject.getEngineCreationContext.getOptions + Option(properties.get(EngineConnPluginConf.JAVA_ENGINE_REQUEST_MEMORY.key)).foreach(memory => { + if (! memory.toLowerCase.endsWith(memorySuffix)) { + properties.put(EngineConnPluginConf.JAVA_ENGINE_REQUEST_MEMORY.key, memory + memorySuffix) + } + }) + val resource = new DriverAndYarnResource( + new LoadInstanceResource(EngineConnPluginConf.JAVA_ENGINE_REQUEST_MEMORY.getValue(properties).toLong, + EngineConnPluginConf.JAVA_ENGINE_REQUEST_CORES.getValue(properties), + EngineConnPluginConf.JAVA_ENGINE_REQUEST_INSTANCE), + new YarnResource(LINKIS_SQOOP_TASK_MAP_MEMORY.getValue * getNumTasks, LINKIS_SQOOP_TASK_MAP_CPU_CORES.getValue * getNumTasks, 0, LINKIS_QUEUE_NAME.getValue) + ) + val engineResource = new CommonNodeResource + engineResource.setUsedResource(resource) + engineResource + } + + def getNumTasks: Int = { + if (params != null) { + params.getOrDefault("sqoop.args.num.mappers", "1").toInt + } else { + 0 + } + } + protected def closeDaemon(): Unit = { + if (daemonThread != null) daemonThread.cancel(true) + } + + override def getProgress: Float = LinkisSqoopClient.progress() + + override def getProgressInfo: Array[JobProgressInfo] = { + val progressInfo = LinkisSqoopClient.getProgressInfo + info(s"Progress Info, id: ${progressInfo.id}, total: ${progressInfo.totalTasks}, running: ${progressInfo.runningTasks}," + + s" succeed: ${progressInfo.succeedTasks}, fail: ${progressInfo.failedTasks}") + Array(progressInfo) + } + + + override def getMetrics: util.Map[String, Any] = { + val metrics = LinkisSqoopClient.getMetrics.asInstanceOf[util.Map[String, Any]] + // Report the resource + metrics.put("NodeResourceJson", getCurrentNodeResource().getUsedResource.toJson) + metrics + } + + override def getDiagnosis: util.Map[String, Any] = LinkisSqoopClient.getDiagnosis.asInstanceOf[util.Map[String, Any]] +} diff --git a/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/executor/SqoopOnceExecutor.scala b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/executor/SqoopOnceExecutor.scala new file mode 100644 index 000000000..727d1f640 --- /dev/null +++ b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/executor/SqoopOnceExecutor.scala @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconnplugin.sqoop.executor + +import org.apache.linkis.common.utils.Utils +import org.apache.linkis.engineconn.core.hook.ShutdownHook +import org.apache.linkis.engineconn.once.executor.{ManageableOnceExecutor, OnceExecutorExecutionContext} +import org.apache.linkis.engineconnplugin.sqoop.client.LinkisSqoopClient +import org.apache.linkis.manager.common.entity.enumeration.NodeStatus +import org.apache.linkis.engineconnplugin.sqoop.client.{LinkisSqoopClient, Sqoop} + +import scala.collection.convert.WrapAsScala._ + + +trait SqoopOnceExecutor extends ManageableOnceExecutor with SqoopExecutor{ + protected def submit(onceExecutorExecutionContext: OnceExecutorExecutionContext): Unit = { + val options = onceExecutorExecutionContext.getOnceExecutorContent.getJobContent.map { + case (k, v: String) => k -> v + case (k, v) if v != null => k -> v.toString + case (k, _) => k -> null + }.toMap + doSubmit(onceExecutorExecutionContext, options) + } + def doSubmit(onceExecutorExecutionContext: OnceExecutorExecutionContext, options: Map[String, String]): Unit + + val id: Long + + override def getId: String = "SqoopOnceApp_" + id + override def close(): Unit = { + Sqoop.close() + super.close() + } + override def trySucceed(): Boolean = { + super.trySucceed() + } + + + override def ensureAvailable[A](f: => A): A = { + // Not need to throws exception + Utils.tryQuietly{ super.ensureAvailable(f) } + } + + override def tryFailed(): Boolean = { + LinkisSqoopClient.close() + super.tryFailed() + } + + override def supportCallBackLogs(): Boolean = true + + + protected def isCompleted: Boolean = isClosed || NodeStatus.isCompleted(getStatus) +} diff --git a/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/factory/SqoopEngineConnFactory.scala b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/factory/SqoopEngineConnFactory.scala new file mode 100644 index 000000000..25666df2a --- /dev/null +++ b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/factory/SqoopEngineConnFactory.scala @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconnplugin.sqoop.factory + +import org.apache.linkis.common.utils.Logging +import org.apache.linkis.engineconn.common.creation.EngineCreationContext +import org.apache.linkis.engineconnplugin.sqoop.context.SqoopEngineConnContext +import org.apache.linkis.engineconnplugin.sqoop.util.ClassUtil +import org.apache.linkis.manager.engineplugin.common.creation.{ExecutorFactory, MultiExecutorEngineConnFactory} +import org.apache.linkis.manager.label.entity.engine.EngineType +import org.apache.linkis.manager.label.entity.engine.EngineType.EngineType +import org.apache.linkis.engineconnplugin.sqoop.context.SqoopEngineConnContext +import org.apache.linkis.engineconnplugin.sqoop.util.ClassUtil + +class SqoopEngineConnFactory extends MultiExecutorEngineConnFactory with Logging{ + override def getExecutorFactories: Array[ExecutorFactory] = executorFactoryArray + + override protected def getDefaultExecutorFactoryClass: Class[_ <: ExecutorFactory] = classOf[SqoopExecutorFactory] + + override protected def getEngineConnType: EngineType = EngineType.SQOOP + + override protected def createEngineConnSession(engineCreationContext: EngineCreationContext): Any = { + //val environmentContext = createEnvironmentContext(engineCreationContext) + val sqoopEngineConnContext = new SqoopEngineConnContext() + sqoopEngineConnContext + } + + + private val executorFactoryArray = Array[ExecutorFactory](ClassUtil.getInstance(classOf[SqoopExecutorFactory], new SqoopExecutorFactory)) +} diff --git a/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/factory/SqoopExecutorFactory.scala b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/factory/SqoopExecutorFactory.scala new file mode 100644 index 000000000..4b39fa0f0 --- /dev/null +++ b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/factory/SqoopExecutorFactory.scala @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconnplugin.sqoop.factory + +import org.apache.linkis.engineconn.common.creation.EngineCreationContext +import org.apache.linkis.engineconn.common.engineconn.EngineConn +import org.apache.linkis.engineconn.once.executor.OnceExecutor +import org.apache.linkis.engineconn.once.executor.creation.OnceExecutorFactory +import org.apache.linkis.engineconnplugin.sqoop.context.SqoopEngineConnContext +import org.apache.linkis.engineconnplugin.sqoop.executor.SqoopOnceCodeExecutor +import org.apache.linkis.manager.label.entity.Label +import org.apache.linkis.manager.label.entity.engine.RunType.{APPCONN, RunType} +import org.apache.linkis.engineconnplugin.sqoop.context.SqoopEngineConnContext +import org.apache.linkis.engineconnplugin.sqoop.executor.SqoopOnceCodeExecutor + +class SqoopExecutorFactory extends OnceExecutorFactory{ + + + override protected def getRunType: RunType = APPCONN + + override protected def newExecutor(id: Int, engineCreationContext: EngineCreationContext, engineConn: EngineConn, labels: Array[Label[_]]): OnceExecutor = { + engineConn.getEngineConnSession match { + case context: SqoopEngineConnContext => + new SqoopOnceCodeExecutor(id, context) + + } + } +} diff --git a/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/launch/SqoopEngineConnLaunchBuilder.scala b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/launch/SqoopEngineConnLaunchBuilder.scala new file mode 100644 index 000000000..a643b792c --- /dev/null +++ b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/launch/SqoopEngineConnLaunchBuilder.scala @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconnplugin.sqoop.launch + +import java.nio.file.Paths +import java.util +import java.util.concurrent.TimeUnit + +import org.apache.linkis.engineconnplugin.sqoop.context.SqoopEnvConfiguration._ +import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnBuildRequest +import org.apache.linkis.manager.engineplugin.common.launch.process.Environment.{variable, _} +import org.apache.linkis.manager.engineplugin.common.launch.process.JavaProcessEngineConnLaunchBuilder +import org.apache.linkis.manager.engineplugin.common.launch.process.LaunchConstants._ +import org.apache.commons.io.IOUtils +import org.apache.commons.lang3.StringUtils + +import scala.collection.JavaConverters._ + +class SqoopEngineConnLaunchBuilder extends JavaProcessEngineConnLaunchBuilder{ + + override protected def getEnvironment(implicit engineConnBuildRequest: EngineConnBuildRequest): util.Map[String, String] = { + val environment = super.getEnvironment + // Basic classpath + addPathToClassPath(environment, variable(HADOOP_CONF_DIR)) + addExistPathToClassPath(environment, Seq(SQOOP_CONF_DIR.getValue)) + if (StringUtils.isNotBlank(SQOOP_HOME.getValue)) { + addPathToClassPath(environment, Seq(SQOOP_HOME.getValue, "/*")) + addPathToClassPath(environment, Seq(SQOOP_HOME.getValue, "/lib/*")) + } + // HBase classpath + if (StringUtils.isNotBlank(SQOOP_HBASE_HOME.getValue) && Paths.get(SQOOP_HBASE_HOME.getValue).toFile.exists()) { + resolveCommandToClassPath(environment, SQOOP_HBASE_HOME.getValue + "/bin/hbase classpath") + } + // HCat classpath + if (StringUtils.isNotBlank(SQOOP_HCAT_HOME.getValue) && Paths.get(SQOOP_HCAT_HOME.getValue).toFile.exists()) { + resolveCommandToClassPath(environment, SQOOP_HCAT_HOME.getValue + "/bin/hcat -classpath") + } + addExistPathToClassPath(environment, Seq(SQOOP_ZOOCFGDIR.getValue)) + environment + } + + + override protected def getNecessaryEnvironment(implicit engineConnBuildRequest: EngineConnBuildRequest): Array[String] = { + // To submit a mapReduce job, we should load the configuration from hadoop config dir + Array(HADOOP_CONF_DIR.toString, SQOOP_HOME.key) + } + + private def addExistPathToClassPath(env: util.Map[String, String], path: String): Unit = { + if (StringUtils.isNotBlank(path) && Paths.get(path).toFile.exists()) { + addPathToClassPath(env, path) + } + } + private def resolveCommandToClassPath(env: util.Map[String, String], command: String): Unit = { + trace(s"Invoke command [${command}] to get class path sequence") + val builder = new ProcessBuilder(Array("/bin/bash", "-c", command): _*) + // Set the environment + builder.environment.putAll(sys.env.asJava) + builder.redirectErrorStream(false) + val process = builder.start() + if(process.waitFor(5, TimeUnit.SECONDS) && + process.waitFor() == 0) { + val jarPathSerial = IOUtils.toString(process.getInputStream).trim() + // TODO we should decide separator in different environment + val separatorChar = ":" + val jarPathList = StringUtils.split(jarPathSerial, separatorChar).filterNot(jarPath => { + val splitIndex = jarPath.lastIndexOf("/") + val jarName = if (splitIndex >= 0) jarPath.substring(splitIndex + 1) else jarPath + jarName.matches("^jasper-compiler-[\\s\\S]+?\\.jar$") || jarName.matches("^jsp-[\\s\\S]+?\\.jar$") || jarName.matches("^disruptor-[\\s\\S]+?\\.jar") + }).toList + addPathToClassPath(env, StringUtils.join(jarPathList.asJava, separatorChar)) + } + // Release the process + process.destroy(); + } + private implicit def buildPath(paths: Seq[String]): String = Paths.get(paths.head, paths.tail: _*).toFile.getPath + +} diff --git a/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/params/SqoopParamsResolver.scala b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/params/SqoopParamsResolver.scala new file mode 100644 index 000000000..464fc3922 --- /dev/null +++ b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/params/SqoopParamsResolver.scala @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconnplugin.sqoop.params + +import java.util + +import org.apache.linkis.engineconn.common.creation.EngineCreationContext +/** + * Resolve the engine job params + */ +trait SqoopParamsResolver { + + /** + * main method + * @param params input + * @return + */ + def resolve(params: util.Map[String, String], context: EngineCreationContext): util.Map[String, String] +} diff --git a/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/resource/SqoopEngineConnResourceFactory.scala b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/resource/SqoopEngineConnResourceFactory.scala new file mode 100644 index 000000000..1634e8b7e --- /dev/null +++ b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/resource/SqoopEngineConnResourceFactory.scala @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconnplugin.sqoop.resource + +import org.apache.linkis.manager.common.entity.resource.{LoadInstanceResource, Resource} +import org.apache.linkis.manager.engineplugin.common.resource.AbstractEngineResourceFactory + +import java.util + +class SqoopEngineConnResourceFactory extends AbstractEngineResourceFactory{ + override protected def getRequestResource(properties: util.Map[String, String]): Resource = { + new LoadInstanceResource(1, + 1, + 1) + } +} diff --git a/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/util/ClassUtil.scala b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/util/ClassUtil.scala new file mode 100644 index 000000000..c3476821f --- /dev/null +++ b/exchangis-plugins/engine/sqoop/src/main/scala/org/apache/linkis/engineconnplugin/sqoop/util/ClassUtil.scala @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.engineconnplugin.sqoop.util + +import org.apache.linkis.common.utils.{ClassUtils, Utils} +import org.apache.linkis.engineconnplugin.sqoop.client.exception.JobExecutionException + +import scala.collection.convert.wrapAsScala._ + +object ClassUtil { + + def getInstance[T](clazz: Class[T] , defaultValue: T): T = { + val classes = ClassUtils.reflections.getSubTypesOf(clazz).filterNot(ClassUtils.isInterfaceOrAbstract).toArray + if(classes.length <= 1) defaultValue + else if(classes.length == 2) { + val realClass = if(classes(0) == defaultValue.getClass) classes(1) else classes(0); + Utils.tryThrow(realClass.newInstance) { t => + new JobExecutionException(s"New a instance of ${clazz.getSimpleName} failed!", t); + } + } else { + throw new JobExecutionException(s"Too many subClasses of ${clazz.getSimpleName}, list: $classes."); + } + } + +} diff --git a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/impl/ExchangisOptStrategy.java b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/impl/ExchangisOptStrategy.java index 413bc8079..4e5c9b0c6 100644 --- a/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/impl/ExchangisOptStrategy.java +++ b/exchangis-plugins/exchangis-appconn/src/main/java/com/webank/wedatasphere/exchangis/dss/appconn/operation/impl/ExchangisOptStrategy.java @@ -147,6 +147,15 @@ public String submit(AsyncExecutionRequestRef ref, String baseUrl, SSORequestOpe ExchangisEntityPostAction exchangisEntityPostAction = new ExchangisEntityPostAction(); exchangisEntityPostAction.setUser(ref.getExecutionRequestRefContext().getRuntimeMap().get("wds.dss.workflow.submit.user").toString()); String originLabels = ref.getExecutionRequestRefContext().getRuntimeMap().get("labels").toString(); + String realLabels = null; + if (originLabels.contains("route")) { + Map changeData = BDPJettyServerHelper.gson().fromJson(originLabels, Map.class); + realLabels = changeData.get("route"); + } else { + realLabels = originLabels; + } + logger.info("realLables7777: {}", realLabels); + logger.info("originLables988888: {}", originLabels); setSSORequestService(developmentService); /*String realLabels = ""; try { @@ -158,7 +167,7 @@ public String submit(AsyncExecutionRequestRef ref, String baseUrl, SSORequestOpe }*/ String submitUser = ref.getExecutionRequestRefContext().getRuntimeMap().get("wds.dss.workflow.submit.user").toString(); HashMap labels = new HashMap<>(); - labels.put("route", originLabels); + labels.put("route", realLabels); exchangisEntityPostAction.addRequestPayload("labels", labels); exchangisEntityPostAction.addRequestPayload("submitUser",submitUser); diff --git a/web/src/common/constants.js b/web/src/common/constants.js index 4f25eb654..39e55f343 100644 --- a/web/src/common/constants.js +++ b/web/src/common/constants.js @@ -1,3 +1,3 @@ //export const BASE_URL = "/api/rest_j/v1/exchangis/"; //export const BASE_URL = "/api/rest_j/v1/dss/exchangis/" -export const BASE_URL = "/api/rest_j/v1/dss/exchangis/main/" +export const BASE_URL = "/wdsentl/exchangis/api/rest_j/v1/dss/exchangis/main/" diff --git a/web/src/pages/dataSourceManage/components/datasourceForm/index.vue b/web/src/pages/dataSourceManage/components/datasourceForm/index.vue index daa1485a0..96c5efb43 100644 --- a/web/src/pages/dataSourceManage/components/datasourceForm/index.vue +++ b/web/src/pages/dataSourceManage/components/datasourceForm/index.vue @@ -1,300 +1,317 @@ \ No newline at end of file + diff --git a/web/src/pages/dataSourceManage/components/editModal.vue b/web/src/pages/dataSourceManage/components/editModal.vue index f9da9edfe..b6949fc14 100644 --- a/web/src/pages/dataSourceManage/components/editModal.vue +++ b/web/src/pages/dataSourceManage/components/editModal.vue @@ -1,120 +1,122 @@ diff --git a/web/src/pages/dataSourceManage/index.vue b/web/src/pages/dataSourceManage/index.vue index 1349e9f5f..fa8389a2c 100644 --- a/web/src/pages/dataSourceManage/index.vue +++ b/web/src/pages/dataSourceManage/index.vue @@ -1,386 +1,369 @@